quda_internal.h

Go to the documentation of this file.

#ifndef _QUDA_INTERNAL_H
#define _QUDA_INTERNAL_H

#include <cuda.h>
#include <cuda_runtime.h>
#include <sys/time.h>
#include <string>
#include <complex>

#if ((defined(QMP_COMMS) || defined(MPI_COMMS)) && !defined(MULTI_GPU))
#error "MULTI_GPU must be enabled to use MPI or QMP"
#endif

#if (!defined(QMP_COMMS) && !defined(MPI_COMMS) && defined(MULTI_GPU))
#error "MPI or QMP must be enabled to use MULTI_GPU"
#endif

//#ifdef USE_QDPJIT
//#include "qdp_quda.h"
//#endif

#ifdef QMP_COMMS
#include <qmp.h>
#endif

#ifdef PTHREADS
#include <pthread.h>
#endif

#define MAX_SHORT 32767.0f

#define TEX_ALIGN_REQ (512*2) //Fermi, factor 2 comes from even/odd
#define ALIGNMENT_ADJUST(n) ( (n+TEX_ALIGN_REQ-1)/TEX_ALIGN_REQ*TEX_ALIGN_REQ)
#include <enum_quda.h>
#include <quda.h>
#include <util_quda.h>
#include <malloc_quda.h>

#include <vector>

// Use bindless texture on Kepler
#if (__COMPUTE_CAPABILITY__ >= 300) && (CUDA_VERSION >= 5000)
#define USE_TEXTURE_OBJECTS
#endif


#ifdef __cplusplus
extern "C" {
#endif
  
  typedef void *ParityGauge;

  // replace below with ColorSpinorField
  typedef struct {
    size_t bytes;
    QudaPrecision precision;
    int length; // total length
    int volume; // geometric volume (single parity)
    int X[QUDA_MAX_DIM]; // the geometric lengths (single parity)
    int Nc; // length of color dimension
    int Ns; // length of spin dimension
    void *data; // either (double2*), (float4 *) or (short4 *), depending on precision
    float *dataNorm; // used only when precision is QUDA_HALF_PRECISION
  } ParityHw;
  
  typedef struct {
    ParityHw odd;
    ParityHw even;
  } FullHw;

  struct QUDA_DiracField{
    void *field; 
  };

  extern cudaDeviceProp deviceProp;  
  extern cudaStream_t *streams;
 
#ifdef PTHREADS
  extern pthread_mutex_t pthread_mutex;
#endif
 
#ifdef __cplusplus
}
#endif

#define REAL(a) (*((double*)&a))
#define IMAG(a) (*((double*)&a+1))

namespace quda {

  typedef std::complex<double> Complex;

  struct Timer {
    double time;

    double last;

    timeval start;

    timeval stop;

    bool running;
    
    int count;

  Timer() : time(0.0), last(0.0), running(false), count(0) { ; } 

    void Start() {
      if (running) errorQuda("Cannot start an already running timer");
      gettimeofday(&start, NULL);
      running = true;
    }

    void Stop() {
      if (!running) errorQuda("Cannot stop an unstarted timer");
      gettimeofday(&stop, NULL);

      long ds = stop.tv_sec - start.tv_sec;
      long dus = stop.tv_usec - start.tv_usec;
      last = ds + 0.000001*dus;
      time += last;
      count++;

      running = false;
    }

    double Last() { return last; }

  };

  enum QudaProfileType {
    QUDA_PROFILE_H2D, 
    QUDA_PROFILE_D2H, 
    QUDA_PROFILE_INIT, 
    QUDA_PROFILE_PREAMBLE, 
    QUDA_PROFILE_COMPUTE, 
    QUDA_PROFILE_EPILOGUE, 
    QUDA_PROFILE_FREE, 
    // lower level counters used in the dslash
    QUDA_PROFILE_PACK_KERNEL, 
    QUDA_PROFILE_DSLASH_KERNEL, 
    QUDA_PROFILE_GATHER, 
    QUDA_PROFILE_SCATTER, 
    QUDA_PROFILE_EVENT_RECORD, 
    QUDA_PROFILE_EVENT_QUERY, 
    QUDA_PROFILE_STREAM_WAIT_EVENT, 
    QUDA_PROFILE_COMMS, 
    QUDA_PROFILE_COMMS_START, 
    QUDA_PROFILE_COMMS_QUERY, 
    QUDA_PROFILE_CONSTANT, 
    QUDA_PROFILE_TOTAL, 
    QUDA_PROFILE_COUNT 
  };

  struct TimeProfile {
    std::string fname;  
    Timer profile[QUDA_PROFILE_COUNT];
    static std::string pname[];

    bool switchOff;
    
    TimeProfile(std::string fname) : fname(fname), switchOff(false) { ; }

    void Print();

    void Start(QudaProfileType idx) { 
      // if total timer isn't running, then start it running
      if (!profile[QUDA_PROFILE_TOTAL].running && idx != QUDA_PROFILE_TOTAL) {
        profile[QUDA_PROFILE_TOTAL].Start(); 
        switchOff = true;
      }

      profile[idx].Start(); 
    }

    void Stop(QudaProfileType idx) { 
      profile[idx].Stop(); 

      // switch off total timer if we need to
      if (switchOff && idx != QUDA_PROFILE_TOTAL) {
        profile[QUDA_PROFILE_TOTAL].Stop(); 
        switchOff = false;
      }
    }

    double Last(QudaProfileType idx) { 
      return profile[idx].last;
    }

  };

#ifdef MULTI_GPU
#ifdef PTHREADS
  const int Nstream = 10;
#else
  const int Nstream = 9;
#endif
#else
  const int Nstream = 1;
#endif

} // namespace quda

#endif // _QUDA_INTERNAL_H