QUDA  v0.7.0
A library for QCD on GPUs
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
dslash_ndeg_twisted_mass.cu
Go to the documentation of this file.
1 #include <cstdlib>
2 #include <cstdio>
3 #include <string>
4 #include <iostream>
5 
6 #include <color_spinor_field.h>
7 #include <clover_field.h>
8 
9 // these control the Wilson-type actions
10 #ifdef GPU_WILSON_DIRAC
11 //#define DIRECT_ACCESS_LINK
12 //#define DIRECT_ACCESS_WILSON_SPINOR
13 //#define DIRECT_ACCESS_WILSON_ACCUM
14 //#define DIRECT_ACCESS_WILSON_INTER
15 //#define DIRECT_ACCESS_WILSON_PACK_SPINOR
16 //#define DIRECT_ACCESS_CLOVER
17 #endif // GPU_WILSON_DIRAC
18 
19 
20 #include <quda_internal.h>
21 #include <dslash_quda.h>
22 #include <sys/time.h>
23 #include <blas_quda.h>
24 #include <face_quda.h>
25 
26 #include <inline_ptx.h>
27 
28 namespace quda {
29 
30  namespace ndegtwisted {
31 
32 #include <dslash_constants.h>
33 #include <dslash_textures.h>
34 #include <dslash_index.cuh>
35 
36  // Enable shared memory dslash for Fermi architecture
37  //#define SHARED_WILSON_DSLASH
38  //#define SHARED_8_BYTE_WORD_SIZE // 8-byte shared memory access
39 
40  //#if (__COMPUTE_CAPABILITY__) >= 200 && defined(GPU_NDEG_TWISTED_MASS_DIRAC)
41 #if (__COMPUTE_CAPABILITY__ >= 200) && defined(GPU_NDEG_TWISTED_MASS_DIRAC)
42 #include <tm_ndeg_dslash_def.h> // Non-degenerate twisted Mass
43 #endif
44 
45 #ifndef NDEGTM_SHARED_FLOATS_PER_THREAD
46 #define NDEGTM_SHARED_FLOATS_PER_THREAD 0
47 #endif
48 
49 #include <dslash_quda.cuh>
50 
51  } // end namespace twisted
52 
53  // declare the dslash events
54 #include <dslash_events.cuh>
55 
56  using namespace ndegtwisted;
57 
58 #if (__COMPUTE_CAPABILITY__ >= 200) && defined(GPU_NDEG_TWISTED_MASS_DIRAC)
59  template <typename sFloat, typename gFloat>
60  class NdegTwistedDslashCuda : public SharedDslashCuda {
61 
62  private:
63  const gFloat *gauge0, *gauge1;
64  const QudaTwistDslashType dslashType;
65  double a, b, c, d;
66 
67  protected:
68  unsigned int sharedBytesPerThread() const
69  {
70 #if (__COMPUTE_CAPABILITY__ >= 200)
71  if (dslashParam.kernel_type == INTERIOR_KERNEL) {
72  int reg_size = (typeid(sFloat)==typeid(double2) ? sizeof(double) : sizeof(float));
73  return NDEGTM_SHARED_FLOATS_PER_THREAD * reg_size;
74  } else {
75  return 0;
76  }
77 #else
78  int reg_size = (typeid(sFloat)==typeid(double2) ? sizeof(double) : sizeof(float));
79  return NDEGTM_SHARED_FLOATS_PER_THREAD * reg_size;
80 #endif
81  }
82 
83  public:
84  NdegTwistedDslashCuda(cudaColorSpinorField *out, const gFloat *gauge0, const gFloat *gauge1,
85  const QudaReconstructType reconstruct, const cudaColorSpinorField *in, const cudaColorSpinorField *x,
86  const QudaTwistDslashType dslashType, const double kappa, const double mu,
87  const double epsilon, const double k, const int dagger)
88  : SharedDslashCuda(out, in, x, reconstruct, dagger), gauge0(gauge0), gauge1(gauge1), dslashType(dslashType)
89  {
90  bindSpinorTex<sFloat>(in, out, x);
91  a = kappa;
92  b = mu;
93  c = epsilon;
94  d = k;
95  if (dslashType != QUDA_NONDEG_DSLASH) errorQuda("Invalid dslashType for non-degenerate twisted-mass Dslash");
96  dslashParam.fl_stride = in->VolumeCB()/2;
97  }
98  virtual ~NdegTwistedDslashCuda() { unbindSpinorTex<sFloat>(in, out, x); }
99 
100  TuneKey tuneKey() const
101  {
102  TuneKey key = DslashCuda::tuneKey();
103  strcat(key.aux,",NdegDslash");
104  return key;
105  }
106 
107  void apply(const cudaStream_t &stream)
108  {
109 
110 #ifdef SHARED_WILSON_DSLASH
111  if (dslashParam.kernel_type == EXTERIOR_KERNEL_X)
112  errorQuda("Shared dslash does not yet support X-dimension partitioning");
113 #endif
114  TuneParam tp = tuneLaunch(*this, getTuning(), getVerbosity());
115  NDEG_TM_DSLASH(twistedNdegMassDslash, tp.grid, tp.block, tp.shared_bytes, stream, dslashParam,
116  (sFloat*)out->V(), (float*)out->Norm(), gauge0, gauge1,
117  (sFloat*)in->V(), (float*)in->Norm(), a, b, c, d, (sFloat*)(x ? x->V() : 0), (float*)(x ? x->Norm() : 0));
118  }
119 
120  long long flops() const { return (x ? 1416ll : 1392ll) * in->VolumeCB(); } // FIXME for multi-GPU
121  };
122 #endif // GPU_NDEG_TWISTED_MASS_DIRAC
123 
124 
125 #include <dslash_policy.cuh>
126 
128  const cudaColorSpinorField *in, const int parity, const int dagger,
129  const cudaColorSpinorField *x, const QudaTwistDslashType type,
130  const double &kappa, const double &mu, const double &epsilon,
131  const double &k, const int *commOverride, TimeProfile &profile,
132  const QudaDslashPolicy &dslashPolicy)
133  {
134  inSpinor = (cudaColorSpinorField*)in; // EVIL
135 #if (__COMPUTE_CAPABILITY__ >= 200) && defined(GPU_NDEG_TWISTED_MASS_DIRAC)
136  int Npad = (in->Ncolor()*in->Nspin()*2)/in->FieldOrder(); // SPINOR_HOP in old code
137 
138  int ghost_threads[4] = {0};
139  int bulk_threads = in->Volume() / 2;
140 
141  for(int i=0;i<4;i++){
142  dslashParam.ghostDim[i] = commDimPartitioned(i); // determines whether to use regular or ghost indexing at boundary
143  dslashParam.ghostOffset[i] = Npad*(in->GhostOffset(i) + in->Stride());
144  dslashParam.ghostNormOffset[i] = in->GhostNormOffset(i) + in->Stride();
145  dslashParam.commDim[i] = (!commOverride[i]) ? 0 : commDimPartitioned(i); // switch off comms if override = 0
146  ghost_threads[i] = in->GhostFace()[i] / 2;
147  }
148 
149  void *gauge0, *gauge1;
150  bindGaugeTex(gauge, parity, &gauge0, &gauge1);
151 
152  if (in->Precision() != gauge.Precision())
153  errorQuda("Mixing gauge and spinor precision not supported");
154 
155  DslashCuda *dslash = 0;
156  size_t regSize = sizeof(float);
157 
158  if (in->Precision() == QUDA_DOUBLE_PRECISION) {
159 #if (__COMPUTE_CAPABILITY__ >= 130)
160  dslash = new NdegTwistedDslashCuda<double2,double2>(out, (double2*)gauge0,(double2*)gauge1, gauge.Reconstruct(), in, x, type, kappa, mu, epsilon, k, dagger);
161  regSize = sizeof(double);
162 #else
163  errorQuda("Double precision not supported on this GPU");
164 #endif
165  } else if (in->Precision() == QUDA_SINGLE_PRECISION) {
166  dslash = new NdegTwistedDslashCuda<float4,float4>(out, (float4*)gauge0,(float4*)gauge1, gauge.Reconstruct(), in, x, type, kappa, mu, epsilon, k, dagger);
167 
168  } else if (in->Precision() == QUDA_HALF_PRECISION) {
169  dslash = new NdegTwistedDslashCuda<short4,short4>(out, (short4*)gauge0,(short4*)gauge1, gauge.Reconstruct(), in, x, type, kappa, mu, epsilon, k, dagger);
170  }
171 
172 #ifndef GPU_COMMS
173  DslashPolicyImp* dslashImp = DslashFactory::create(dslashPolicy);
174 #else
175  DslashPolicyImp* dslashImp = DslashFactory::create(QUDA_GPU_COMMS_DSLASH);
176 #endif
177  (*dslashImp)(*dslash, const_cast<cudaColorSpinorField*>(in), regSize, parity, dagger, bulk_threads, ghost_threads, profile);
178  delete dslashImp;
179 
180  delete dslash;
181 
182  unbindGaugeTex(gauge);
183 
184  checkCudaError();
185 #else
186 
187 #if (__COMPUTE_CAPABILITY__ < 200)
188  errorQuda("Non-degenerate twisted-mass fermions not supported on pre-Fermi architecture");
189 #else
190  errorQuda("Non-degenerate twisted mass dslash has not been built");
191 #endif
192 
193 #endif
194  }
195 
196 }
void unbindGaugeTex(const cudaGaugeField &gauge)
#define NDEGTM_SHARED_FLOATS_PER_THREAD
int commDimPartitioned(int dir)
QudaVerbosity getVerbosity()
Definition: util_quda.cpp:20
#define errorQuda(...)
Definition: util_quda.h:73
__global__ void const RealA *const const RealA *const const RealA *const const RealB *const const RealB *const int int mu
cudaStream_t * stream
int GhostNormOffset(const int i) const
QudaDagType dagger
Definition: test_util.cpp:1558
QudaPrecision Precision() const
void ndegTwistedMassDslashCuda(cudaColorSpinorField *out, const cudaGaugeField &gauge, const cudaColorSpinorField *in, const int parity, const int dagger, const cudaColorSpinorField *x, const QudaTwistDslashType type, const double &kappa, const double &mu, const double &epsilon, const double &k, const int *commDim, TimeProfile &profile, const QudaDslashPolicy &dslashPolicy=QUDA_DSLASH)
VOLATILE spinorFloat kappa
cpuColorSpinorField * in
enum QudaDslashPolicy_s QudaDslashPolicy
TuneParam & tuneLaunch(Tunable &tunable, QudaTune enabled, QudaVerbosity verbosity)
Definition: tune.cpp:271
QudaReconstructType Reconstruct() const
Definition: gauge_field.h:168
int x[4]
QudaFieldOrder FieldOrder() const
enum QudaTwistDslashType_s QudaTwistDslashType
cpuColorSpinorField * out
enum QudaReconstructType_s QudaReconstructType
QudaPrecision Precision() const
void bindGaugeTex(const cudaGaugeField &gauge, const int oddBit, void **gauge0, void **gauge1)
#define checkCudaError()
Definition: util_quda.h:110
QudaTune getTuning()
Definition: util_quda.cpp:32
int GhostOffset(const int i) const
const QudaParity parity
Definition: dslash_test.cpp:29
void * gauge[4]
Definition: su3_test.cpp:15
const int * GhostFace() const