lattice_field.h

Go to the documentation of this file.

#ifndef _LATTICE_FIELD_H
#define _LATTICE_FIELD_H

#include <map>
#include <quda.h>
#include <iostream>
#include <comm_quda.h>
#include <util_quda.h>
#include <object.h>
#include <cuda_runtime.h>

namespace quda {
  
  // LatticeField is an abstract base clase for all Field objects.

  // Forward declaration of all children
  class LatticeField;

  class ColorSpinorField;
  class cudaColorSpinorField;
  class cpuColorSpinorField;
  
  class EigValueSet;
  class cudaEigValueSet;
  class cpuEigValueSet;

  class EigVecSet;
  class cpuEigVecSet;
  class cudaEigVecSet;

  class GaugeField;
  class cpuGaugeField;
  class cudaGaugeField;

  class CloverField;
  class cudaCloverField;
  class cpuCloverField;

  struct LatticeFieldParam {

  protected:
    QudaPrecision precision;

    QudaPrecision ghost_precision;

  public:
    QudaPrecision Precision() const { return precision; }

    QudaPrecision GhostPrecision() const { return ghost_precision; }

    int nDim;

    int x[QUDA_MAX_DIM];

    int pad;

    QudaSiteSubset siteSubset;

    QudaMemoryType mem_type; 
 
    QudaGhostExchange ghostExchange;

    int r[QUDA_MAX_DIM];

    double scale;

    LatticeFieldParam()
    : precision(QUDA_INVALID_PRECISION), ghost_precision(QUDA_INVALID_PRECISION), nDim(4), pad(0),
      siteSubset(QUDA_INVALID_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(QUDA_GHOST_EXCHANGE_PAD), scale(1.0)
    {
      for (int i=0; i<nDim; i++) {
  x[i] = 0;
  r[i] = 0;
      }
    }

    LatticeFieldParam(int nDim, const int *x, int pad, QudaPrecision precision,
          QudaGhostExchange ghostExchange=QUDA_GHOST_EXCHANGE_PAD)
    : precision(precision), ghost_precision(precision), nDim(nDim), pad(pad),
      siteSubset(QUDA_FULL_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(ghostExchange), scale(1.0)
    {
      if (nDim > QUDA_MAX_DIM) errorQuda("Number of dimensions too great");
      for (int i=0; i<nDim; i++) {
  this->x[i] = x[i];
  this->r[i] = 0;
      }
    }
    
    LatticeFieldParam(const QudaGaugeParam &param) 
    :  precision(param.cpu_prec), ghost_precision(param.cpu_prec), nDim(4), pad(0),
      siteSubset(QUDA_FULL_SITE_SUBSET), mem_type(QUDA_MEMORY_DEVICE),
      ghostExchange(QUDA_GHOST_EXCHANGE_NO), scale(param.scale)
    {
      for (int i=0; i<nDim; i++) {
  this->x[i] = param.X[i];
  this->r[i] = 0;
      }
    }

    LatticeFieldParam(const LatticeField &field);
  };

  std::ostream& operator<<(std::ostream& output, const LatticeFieldParam& param);

  class LatticeField : public Object {

  protected:
    int volume;

    int volumeCB;

    int stride;
    int pad;

    size_t total_bytes;

    int nDim;
    
    int x[QUDA_MAX_DIM];

    int surface[QUDA_MAX_DIM];
    int surfaceCB[QUDA_MAX_DIM];

    int r[QUDA_MAX_DIM];

    QudaPrecision precision;
    
    mutable QudaPrecision ghost_precision;

    mutable bool ghost_precision_reset;

    double scale;

    QudaSiteSubset siteSubset;

    QudaGhostExchange ghostExchange;

    // The below are additions for inter-GPU communication (merging FaceBuffer functionality)

    int nDimComms;

    /* 
       The need for persistent message handlers (for GPUDirect support)
       means that we allocate different message handlers for each number of
       faces we can send.
    */

    static void *ghost_send_buffer_d[2];

    static void *ghost_recv_buffer_d[2];

    static void *ghost_pinned_send_buffer_h[2];

    static void *ghost_pinned_recv_buffer_h[2];

    static void *ghost_pinned_send_buffer_hd[2];

    static void *ghost_pinned_recv_buffer_hd[2];

    static void *ghost_remote_send_buffer_d[2][QUDA_MAX_DIM][2];

    static size_t ghostFaceBytes;

    static bool initGhostFaceBuffer;

    mutable size_t ghost_bytes;

    mutable size_t ghost_bytes_old;

    mutable size_t ghost_face_bytes[QUDA_MAX_DIM];

    mutable int ghostOffset[QUDA_MAX_DIM][2];

    mutable int ghostNormOffset[QUDA_MAX_DIM][2];

    void *my_face_h[2];

    void *my_face_hd[2];

    void *my_face_d[2];

    void *my_face_dim_dir_h[2][QUDA_MAX_DIM][2];

    void *my_face_dim_dir_hd[2][QUDA_MAX_DIM][2];

    void *my_face_dim_dir_d[2][QUDA_MAX_DIM][2];

    void *from_face_h[2];

    void *from_face_hd[2];

    void *from_face_d[2];

    void *from_face_dim_dir_h[2][QUDA_MAX_DIM][2];

    void *from_face_dim_dir_hd[2][QUDA_MAX_DIM][2];

    void *from_face_dim_dir_d[2][QUDA_MAX_DIM][2];
    
    MsgHandle *mh_recv_fwd[2][QUDA_MAX_DIM];

    MsgHandle *mh_recv_back[2][QUDA_MAX_DIM];

    MsgHandle *mh_send_fwd[2][QUDA_MAX_DIM];

    MsgHandle *mh_send_back[2][QUDA_MAX_DIM];

    MsgHandle *mh_recv_rdma_fwd[2][QUDA_MAX_DIM];

    MsgHandle *mh_recv_rdma_back[2][QUDA_MAX_DIM];

    MsgHandle *mh_send_rdma_fwd[2][QUDA_MAX_DIM];

    MsgHandle *mh_send_rdma_back[2][QUDA_MAX_DIM];

    static MsgHandle* mh_send_p2p_fwd[2][QUDA_MAX_DIM];

    static MsgHandle* mh_send_p2p_back[2][QUDA_MAX_DIM];

    static MsgHandle* mh_recv_p2p_fwd[2][QUDA_MAX_DIM];

    static MsgHandle* mh_recv_p2p_back[2][QUDA_MAX_DIM];

    static int buffer_send_p2p_fwd[2][QUDA_MAX_DIM];

    static int buffer_recv_p2p_fwd[2][QUDA_MAX_DIM];

    static int buffer_send_p2p_back[2][QUDA_MAX_DIM];

    static int buffer_recv_p2p_back[2][QUDA_MAX_DIM];

    static cudaEvent_t ipcCopyEvent[2][2][QUDA_MAX_DIM];

    static cudaEvent_t ipcRemoteCopyEvent[2][2][QUDA_MAX_DIM];

    bool initComms;

    static bool initIPCComms;

    char vol_string[TuneKey::volume_n];
    
    char aux_string[TuneKey::aux_n];

    virtual void setTuningString();

    QudaMemoryType mem_type;

    void precisionCheck() {
      switch(precision) {
      case QUDA_QUARTER_PRECISION:
      case QUDA_HALF_PRECISION:
      case QUDA_SINGLE_PRECISION:
      case QUDA_DOUBLE_PRECISION:
  break;
      default:
  errorQuda("Unknown precision %d\n", precision);
      }
    }

    mutable char *backup_h;
    mutable char *backup_norm_h;
    mutable bool backed_up;

  public:

    LatticeField(const LatticeFieldParam &param);

    LatticeField(const LatticeField &field);

    virtual ~LatticeField();
    
    void allocateGhostBuffer(size_t ghost_bytes) const;

    static void freeGhostBuffer(void);

    void createComms(bool no_comms_fill=false, bool bidir=true);

    void destroyComms();

    void createIPCComms();

    static void destroyIPCComms();

    inline bool ipcCopyComplete(int dir, int dim);

    inline bool ipcRemoteCopyComplete(int dir, int dim);

    const cudaEvent_t& getIPCCopyEvent(int dir, int dim) const;

    const cudaEvent_t& getIPCRemoteCopyEvent(int dir, int dim) const;

    static int bufferIndex;

    static bool ghost_field_reset;

    int Ndim() const { return nDim; }
    
    const int* X() const { return x; }
    
    int Volume() const { return volume; }
    
    int VolumeCB() const { return volumeCB; }
    
    const int* SurfaceCB() const { return surfaceCB; }
    
    int SurfaceCB(const int i) const { return surfaceCB[i]; }
    
    int Stride() const { return stride; }
    
    int Pad() const { return pad; }
    
    const int* R() const { return r; }

    QudaGhostExchange GhostExchange() const { return ghostExchange; }

    QudaPrecision Precision() const { return precision; }

    QudaPrecision GhostPrecision() const { return ghost_precision; }

    double Scale() const { return scale; }

    void Scale(double scale_) { scale = scale_; }

    virtual QudaSiteSubset SiteSubset() const { return siteSubset; }

    virtual QudaMemoryType MemType() const { return mem_type; }

    int Nvec() const;

    QudaFieldLocation Location() const;

    size_t GBytes() const { return total_bytes / (1<<30); }

    void checkField(const LatticeField &a) const;

    virtual void read(char *filename);

    virtual void write(char *filename);
    
    virtual void gather(int nFace, int dagger, int dir, cudaStream_t *stream_p=NULL)
    { errorQuda("Not implemented"); }

    virtual void commsStart(int nFace, int dir, int dagger=0, cudaStream_t *stream_p=NULL, bool gdr_send=false, bool gdr_recv=true)
    { errorQuda("Not implemented"); }

    virtual int commsQuery(int nFace, int dir, int dagger=0, cudaStream_t *stream_p=NULL, bool gdr_send=false, bool gdr_recv=true)
    { errorQuda("Not implemented"); return 0; }

    virtual void commsWait(int nFace, int dir, int dagger=0, cudaStream_t *stream_p=NULL, bool gdr_send=false, bool gdr_recv=true)
    { errorQuda("Not implemented"); }

    virtual void scatter(int nFace, int dagger, int dir)
    { errorQuda("Not implemented"); }

    inline const char *VolString() const { return vol_string; }

    inline const char *AuxString() const { return aux_string; }

    virtual void backup() const { errorQuda("Not implemented"); }

    virtual void restore() const { errorQuda("Not implemented"); }
  };
  
  inline QudaFieldLocation Location_(const char *func, const char *file, int line,
             const LatticeField &a, const LatticeField &b) {
    QudaFieldLocation location = QUDA_INVALID_FIELD_LOCATION;
    if (a.Location() == b.Location()) location = a.Location();
    else errorQuda("Locations %d %d do not match  (%s:%d in %s())\n",
       a.Location(), b.Location(), file, line, func);
    return location;
  }

  template <typename... Args>
  inline QudaFieldLocation Location_(const char *func, const char *file, int line,
             const LatticeField &a, const LatticeField &b, const Args &... args) {
    return static_cast<QudaFieldLocation>(Location_(func,file,line,a,b) & Location_(func,file,line,a,args...));
  }

#define checkLocation(...)Location_(__func__, __FILE__, __LINE__, __VA_ARGS__)

  inline QudaPrecision Precision_(const char *func, const char *file, int line,
          const LatticeField &a, const LatticeField &b) {
    QudaPrecision precision = QUDA_INVALID_PRECISION;
    if (a.Precision() == b.Precision()) precision = a.Precision();
    else errorQuda("Precisions %d %d do not match (%s:%d in %s())\n",
       a.Precision(), b.Precision(), file, line, func);
    return precision;
  }

  template <typename... Args>
  inline QudaPrecision Precision_(const char *func, const char *file, int line,
          const LatticeField &a, const LatticeField &b,
          const Args &... args) {
    return static_cast<QudaPrecision>(Precision_(func,file,line,a,b) & Precision_(func,file,line,a,args...));
  }

#define checkPrecision(...) Precision_(__func__, __FILE__, __LINE__, __VA_ARGS__)

  QudaFieldLocation reorder_location();

  void reorder_location_set(QudaFieldLocation reorder_location_);

  inline const char *compile_type_str(const LatticeField &meta, QudaFieldLocation location_ = QUDA_INVALID_FIELD_LOCATION)
  {
    QudaFieldLocation location = (location_ == QUDA_INVALID_FIELD_LOCATION ? meta.Location() : location_);
#ifdef JITIFY
    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-jitify," : "CPU,";
#else
    return location == QUDA_CUDA_FIELD_LOCATION ? "GPU-offline," : "CPU,";
#endif
  }

} // namespace quda

#endif // _LATTICE_FIELD_H