lattice_field.h

Go to the documentation of this file.

#pragma once
 
#include <map>
#include <quda.h>
#include <iostream>
#include <comm_quda.h>
#include <util_quda.h>
#include <object.h>
#include <quda_api.h>
 
namespace quda {
  
  // LatticeField is an abstract base clase for all Field objects.
 
  // Forward declaration of all children
  class LatticeField;
 
  class ColorSpinorField;
  class cudaColorSpinorField;
  class cpuColorSpinorField;
  
  class EigValueSet;
  class cudaEigValueSet;
  class cpuEigValueSet;
 
  class EigVecSet;
  class cpuEigVecSet;
  class cudaEigVecSet;
 
  class GaugeField;
  class cpuGaugeField;
  class cudaGaugeField;
 
  class CloverField;
  class cudaCloverField;
  class cpuCloverField;
 
  enum class QudaOffsetCopyMode { COLLECT, DISPERSE };
 
  struct LatticeFieldParam {
 
  protected:
    QudaPrecision precision;
 
    QudaPrecision ghost_precision;
 
  public:
    QudaPrecision Precision() const { return precision; }
 
    QudaPrecision GhostPrecision() const { return ghost_precision; }
 
    int nDim;
 
    int x[QUDA_MAX_DIM];
 
    int pad;
 
    QudaSiteSubset siteSubset;
 
    QudaMemoryType mem_type; 
 
    QudaGhostExchange ghostExchange;
 
    int r[QUDA_MAX_DIM];
 
    double scale;
 
    LatticeFieldParam()
    : precision(QUDA_INVALID_PRECISION), ghost_precision(QUDA_INVALID_PRECISION), nDim(4), pad(0),
      siteSubset(QUDA_INVALID_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(QUDA_GHOST_EXCHANGE_PAD), scale(1.0)
    {
      for (int i=0; i<nDim; i++) {
        x[i] = 0;
        r[i] = 0;
      }
    }
 
    LatticeFieldParam(int nDim, const int *x, int pad, QudaPrecision precision,
                      QudaGhostExchange ghostExchange=QUDA_GHOST_EXCHANGE_PAD)
    : precision(precision), ghost_precision(precision), nDim(nDim), pad(pad),
      siteSubset(QUDA_FULL_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(ghostExchange), scale(1.0)
    {
      if (nDim > QUDA_MAX_DIM) errorQuda("Number of dimensions too great");
      for (int i=0; i<nDim; i++) {
        this->x[i] = x[i];
        this->r[i] = 0;
      }
    }
    
    LatticeFieldParam(const QudaGaugeParam &param) 
    :  precision(param.cpu_prec), ghost_precision(param.cpu_prec), nDim(4), pad(0),
      siteSubset(QUDA_FULL_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(QUDA_GHOST_EXCHANGE_NO), scale(param.scale)
    {
      for (int i=0; i<nDim; i++) {
        this->x[i] = param.X[i];
        this->r[i] = 0;
      }
    }
 
    LatticeFieldParam(const LatticeField &field);
  };
 
  std::ostream& operator<<(std::ostream& output, const LatticeFieldParam& param);
 
  class LatticeField : public Object {
 
  protected:
    size_t volume;
 
    size_t volumeCB;
 
    size_t localVolume;
 
    size_t localVolumeCB;
 
    size_t stride;
    int pad;
 
    size_t total_bytes;
 
    int nDim;
 
    int x[QUDA_MAX_DIM];
 
    int surface[QUDA_MAX_DIM];
    int surfaceCB[QUDA_MAX_DIM];
 
    int r[QUDA_MAX_DIM];
 
    QudaPrecision precision;
 
    mutable QudaPrecision ghost_precision;
 
    mutable bool ghost_precision_reset;
 
    double scale;
 
    QudaSiteSubset siteSubset;
 
    QudaGhostExchange ghostExchange;
 
    // The below are additions for inter-GPU communication (merging FaceBuffer functionality)
 
    int nDimComms;
 
    /*
       The need for persistent message handlers (for GPUDirect support)
       means that we allocate different message handlers for each number of
       faces we can send.
    */
 
    static void *ghost_send_buffer_d[2];
 
    static void *ghost_recv_buffer_d[2];
 
    static void *ghost_pinned_send_buffer_h[2];
 
    static void *ghost_pinned_recv_buffer_h[2];
 
    static void *ghost_pinned_send_buffer_hd[2];
 
    static void *ghost_pinned_recv_buffer_hd[2];
 
    static void *ghost_remote_send_buffer_d[2][QUDA_MAX_DIM][2];
 
    static size_t ghostFaceBytes;
 
    static bool initGhostFaceBuffer;
 
    mutable size_t ghost_bytes;
 
    mutable size_t ghost_bytes_old;
 
    mutable size_t ghost_face_bytes[QUDA_MAX_DIM];
 
    mutable size_t ghost_face_bytes_aligned[QUDA_MAX_DIM];
 
    mutable size_t ghost_offset[QUDA_MAX_DIM][2];
 
    void *my_face_h[2];
 
    void *my_face_hd[2];
 
    void *my_face_d[2];
 
    void *my_face_dim_dir_h[2][QUDA_MAX_DIM][2];
 
    void *my_face_dim_dir_hd[2][QUDA_MAX_DIM][2];
 
    void *my_face_dim_dir_d[2][QUDA_MAX_DIM][2];
 
    void *from_face_h[2];
 
    void *from_face_hd[2];
 
    void *from_face_d[2];
 
    void *from_face_dim_dir_h[2][QUDA_MAX_DIM][2];
 
    void *from_face_dim_dir_hd[2][QUDA_MAX_DIM][2];
 
    void *from_face_dim_dir_d[2][QUDA_MAX_DIM][2];
 
    MsgHandle *mh_recv_fwd[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_recv_back[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_send_fwd[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_send_back[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_recv_rdma_fwd[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_recv_rdma_back[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_send_rdma_fwd[2][QUDA_MAX_DIM];
 
    MsgHandle *mh_send_rdma_back[2][QUDA_MAX_DIM];
 
    static MsgHandle *mh_send_p2p_fwd[2][QUDA_MAX_DIM];
 
    static MsgHandle *mh_send_p2p_back[2][QUDA_MAX_DIM];
 
    static MsgHandle *mh_recv_p2p_fwd[2][QUDA_MAX_DIM];
 
    static MsgHandle *mh_recv_p2p_back[2][QUDA_MAX_DIM];
 
    static int buffer_send_p2p_fwd[2][QUDA_MAX_DIM];
 
    static int buffer_recv_p2p_fwd[2][QUDA_MAX_DIM];
 
    static int buffer_send_p2p_back[2][QUDA_MAX_DIM];
 
    static int buffer_recv_p2p_back[2][QUDA_MAX_DIM];
 
    static cudaEvent_t ipcCopyEvent[2][2][QUDA_MAX_DIM];
 
    static cudaEvent_t ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];
 
    bool initComms;
 
    static bool initIPCComms;
 
    char vol_string[TuneKey::volume_n];
 
    char aux_string[TuneKey::aux_n];
 
    virtual void setTuningString();
 
    QudaMemoryType mem_type;
 
    void precisionCheck()
    {
      switch (precision) {
      case QUDA_QUARTER_PRECISION:
      case QUDA_HALF_PRECISION:
      case QUDA_SINGLE_PRECISION:
      case QUDA_DOUBLE_PRECISION: break;
      default: errorQuda("Unknown precision %d\n", precision);
      }
    }
 
    mutable char *backup_h;
    mutable char *backup_norm_h;
    mutable bool backed_up;
 
  public:
 
    LatticeField(const LatticeFieldParam &param);
 
    LatticeField(const LatticeField &field);
 
    virtual ~LatticeField();
    
    void allocateGhostBuffer(size_t ghost_bytes) const;
 
    static void freeGhostBuffer(void);
 
    void createComms(bool no_comms_fill=false, bool bidir=true);
 
    void destroyComms();
 
    void createIPCComms();
 
    static void destroyIPCComms();
 
    inline bool ipcCopyComplete(int dir, int dim);
 
    inline bool ipcRemoteCopyComplete(int dir, int dim);
 
    const cudaEvent_t& getIPCCopyEvent(int dir, int dim) const;
 
    const cudaEvent_t& getIPCRemoteCopyEvent(int dir, int dim) const;
 
    static int bufferIndex;
 
    static bool ghost_field_reset;
 
    int Ndim() const { return nDim; }
    
    const int* X() const { return x; }
 
    virtual int full_dim(int d) const = 0;
 
    size_t Volume() const { return volume; }
 
    size_t VolumeCB() const { return volumeCB; }
 
    size_t LocalVolume() const { return localVolume; }
 
    size_t LocalVolumeCB() const { return localVolumeCB; }
 
    const int* SurfaceCB() const { return surfaceCB; }
    
    int SurfaceCB(const int i) const { return surfaceCB[i]; }
 
    size_t Stride() const { return stride; }
 
    int Pad() const { return pad; }
    
    const int* R() const { return r; }
 
    QudaGhostExchange GhostExchange() const { return ghostExchange; }
 
    QudaPrecision Precision() const { return precision; }
 
    QudaPrecision GhostPrecision() const { return ghost_precision; }
 
    double Scale() const { return scale; }
 
    void Scale(double scale_) { scale = scale_; }
 
    virtual QudaSiteSubset SiteSubset() const { return siteSubset; }
 
    virtual QudaMemoryType MemType() const { return mem_type; }
 
    int Nvec() const;
 
    QudaFieldLocation Location() const;
 
    size_t GBytes() const { return total_bytes / (1<<30); }
 
    void checkField(const LatticeField &a) const;
 
    virtual void read(char *filename);
 
    virtual void write(char *filename);
 
    void *myFace_h(int dir, int dim) const { return my_face_dim_dir_h[bufferIndex][dim][dir]; }
 
    void *myFace_hd(int dir, int dim) const { return my_face_dim_dir_hd[bufferIndex][dim][dir]; }
 
    void *myFace_d(int dir, int dim) const { return my_face_dim_dir_d[bufferIndex][dim][dir]; }
 
    void *remoteFace_d(int dir, int dim) const { return ghost_remote_send_buffer_d[bufferIndex][dim][dir]; }
 
    void *remoteFace_r() const { return ghost_recv_buffer_d[bufferIndex]; }
 
    virtual void gather(int nFace, int dagger, int dir, qudaStream_t *stream_p = NULL) { errorQuda("Not implemented"); }
 
    virtual void commsStart(int nFace, int dir, int dagger = 0, qudaStream_t *stream_p = NULL, bool gdr_send = false,
                            bool gdr_recv = true)
    { errorQuda("Not implemented"); }
 
    virtual int commsQuery(int nFace, int dir, int dagger = 0, qudaStream_t *stream_p = NULL, bool gdr_send = false,
                           bool gdr_recv = true)
    { errorQuda("Not implemented"); return 0; }
 
    virtual void commsWait(int nFace, int dir, int dagger = 0, qudaStream_t *stream_p = NULL, bool gdr_send = false,
                           bool gdr_recv = true)
    { errorQuda("Not implemented"); }
 
    virtual void scatter(int nFace, int dagger, int dir)
    { errorQuda("Not implemented"); }
 
    inline const char *VolString() const { return vol_string; }
 
    inline const char *AuxString() const { return aux_string; }
 
    virtual void backup() const { errorQuda("Not implemented"); }
 
    virtual void restore() const { errorQuda("Not implemented"); }
 
    virtual void prefetch(QudaFieldLocation mem_space, qudaStream_t stream = 0) const { ; }
 
    virtual bool isNative() const = 0;
 
    virtual void copy_to_buffer(void *buffer) const = 0;
 
    virtual void copy_from_buffer(void *buffer) = 0;
  };
  
  inline QudaFieldLocation Location_(const char *func, const char *file, int line,
                                     const LatticeField &a, const LatticeField &b) {
    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION;
    if (a.Location() == b.Location()) location = a.Location();
    else errorQuda("Locations %d %d do not match  (%s:%d in %s())\n",
                   a.Location(), b.Location(), file, line, func);
    return location;
  }
 
  template <typename... Args>
  inline QudaFieldLocation Location_(const char *func, const char *file, int line,
                                     const LatticeField &a, const LatticeField &b, const Args &... args) {
    return static_cast<QudaFieldLocation>(Location_(func,file,line,a,b) & Location_(func,file,line,a,args...));
  }
 
#define checkLocation(...) Location_(__func__, __FILE__, __LINE__, __VA_ARGS__)
 
  inline QudaPrecision Precision_(const char *func, const char *file, int line,
                                  const LatticeField &a, const LatticeField &b) {
    QudaPrecision precision = QUDA_INVALID_PRECISION;
    if (a.Precision() == b.Precision()) precision = a.Precision();
    else errorQuda("Precisions %d %d do not match (%s:%d in %s())\n",
                   a.Precision(), b.Precision(), file, line, func);
    return precision;
  }
 
  template <typename... Args>
  inline QudaPrecision Precision_(const char *func, const char *file, int line,
                                  const LatticeField &a, const LatticeField &b,
                                  const Args &... args) {
    return static_cast<QudaPrecision>(Precision_(func,file,line,a,b) & Precision_(func,file,line,a,args...));
  }
 
#define checkPrecision(...) Precision_(__func__, __FILE__, __LINE__, __VA_ARGS__)
 
  inline bool Native_(const char *func, const char *file, int line, const LatticeField &a)
  {
    if (!a.isNative()) errorQuda("Non-native field detected (%s:%d in %s())\n", file, line, func);
    return true;
  }
 
  template <typename... Args>
  inline bool Native_(const char *func, const char *file, int line, const LatticeField &a, const Args &... args)
  {
    return (Native_(func, file, line, a) & Native_(func, file, line, args...));
  }
 
#define checkNative(...) Native_(__func__, __FILE__, __LINE__, __VA_ARGS__)
 
  QudaFieldLocation reorder_location();
 
  void reorder_location_set(QudaFieldLocation reorder_location_);
 
  inline const char *compile_type_str(const LatticeField &meta, QudaFieldLocation location_ = QUDA_INVALID_FIELD_LOCATION)
  {
    QudaFieldLocation location = (location_ == QUDA_INVALID_FIELD_LOCATION ? meta.Location() : location_);
#ifdef JITIFY
    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-jitify," : "CPU,";
#else
    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-offline," : "CPU,";
#endif
  }
 
} // namespace quda