dslash_quda.h

Go to the documentation of this file.

#ifndef _DSLASH_QUDA_H
#define _DSLASH_QUDA_H
 
#include <quda_internal.h>
#include <tune_quda.h>
#include <color_spinor_field.h>
#include <gauge_field.h>
#include <clover_field.h>
#include <worker.h>
#ifdef NVSHMEM_COMMS
#include <cuda/atomic>
#endif
 
namespace quda {
 
  void setKernelPackT(bool pack);
 
  bool getKernelPackT();
 
  void pushKernelPackT(bool pack);
  void popKernelPackT();
 
  void setPackComms(const int *dim_pack);
 
  bool getDslashLaunch();
 
  void createDslashEvents();
  void destroyDslashEvents();
 
  namespace dslash
  {
    using shmem_sync_t = uint64_t;
 
    shmem_sync_t get_shmem_sync_counter();
 
    shmem_sync_t set_shmem_sync_counter(shmem_sync_t count);
 
    shmem_sync_t inc_shmem_sync_counter();
#ifdef NVSHMEM_COMMS
    using shmem_retcount_intra_t = cuda::atomic<int, cuda::thread_scope_system>;
    using shmem_retcount_inter_t = cuda::atomic<int, cuda::thread_scope_device>;
    using shmem_interior_done_t = cuda::atomic<shmem_sync_t, cuda::thread_scope_device>;
    using shmem_interior_count_t = cuda::atomic<int, cuda::thread_scope_block>;
 
    shmem_sync_t *get_shmem_sync_arr();
 
    shmem_retcount_intra_t *get_shmem_retcount_intra();
 
    shmem_retcount_inter_t *get_shmem_retcount_inter();
 
    shmem_interior_done_t *get_shmem_interior_done();
 
    shmem_interior_count_t *get_shmem_interior_count();
#endif
  } // namespace dslash
 
  void ApplyWilson(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double kappa,
                   const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyWilsonClover(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &A,
      double kappa, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyWilsonCloverHasenbuschTwist(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                        const CloverField &A, double kappa, double mu, const ColorSpinorField &x,
                                        int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyWilsonCloverPreconditioned(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
      const CloverField &A, double kappa, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
      TimeProfile &profile);
 
  void ApplyWilsonCloverHasenbuschTwistPCClovInv(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                                                 const CloverField &A, double kappa, double mu,
                                                 const ColorSpinorField &x, int parity, bool dagger,
                                                 const int *comm_override, TimeProfile &profile);
 
  void ApplyWilsonCloverHasenbuschTwistPCNoClovInv(ColorSpinorField &out, const ColorSpinorField &in,
                                                   const GaugeField &U, const CloverField &A, double kappa, double mu,
                                                   const ColorSpinorField &x, int parity, bool dagger,
                                                   const int *comm_override, TimeProfile &profile);
 
  // old
  void ApplyTwistedMass(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
                        const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
                        TimeProfile &profile);
 
  void ApplyTwistedMassPreconditioned(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
      double b, bool xpay, const ColorSpinorField &x, int parity, bool dagger, bool asymmetric,
      const int *comm_override, TimeProfile &profile);
 
  void ApplyNdegTwistedMass(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double b,
      double c, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyNdegTwistedMassPreconditioned(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
      double a, double b, double c, bool xpay, const ColorSpinorField &x, int parity, bool dagger, bool asymmetric,
      const int *comm_override, TimeProfile &profile);
 
  void ApplyTwistedClover(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, const CloverField &C,
      double a, double b, const ColorSpinorField &x, int parity, bool dagger, const int *comm_override,
      TimeProfile &profile);
 
  void ApplyTwistedCloverPreconditioned(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
      const CloverField &C, double a, double b, bool xpay, const ColorSpinorField &x, int parity, bool dagger,
      const int *comm_override, TimeProfile &profile);
 
  void ApplyDomainWall5D(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double m_f,
      const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyDomainWall4D(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a, double m_5,
                         const Complex *b_5, const Complex *c_5, const ColorSpinorField &x, int parity, bool dagger,
                         const int *comm_override, TimeProfile &profile);
 
  enum Dslash5Type {
    DSLASH5_DWF,
    DSLASH5_MOBIUS_PRE,
    DSLASH5_MOBIUS,
    M5_INV_DWF,
    M5_INV_MOBIUS,
    M5_INV_ZMOBIUS,
    M5_EOFA,
    M5INV_EOFA
  };
 
  enum class MdwfFusedDslashType {
    D4_D5INV_D5PRE,
    D4_D5INV_D5INVDAG,
    D4DAG_D5PREDAG_D5INVDAG,
    D4DAG_D5PREDAG,
    D5PRE,
  };
 
  void ApplyDslash5(ColorSpinorField &out, const ColorSpinorField &in, const ColorSpinorField &x, double m_f,
                    double m_5, const Complex *b_5, const Complex *c_5, double a, bool dagger, Dslash5Type type);
 
  // Tensor core functions for Mobius DWF
  namespace mobius_tensor_core
  {
    void apply_fused_dslash(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, ColorSpinorField &y,
                            const ColorSpinorField &x, double m_f, double m_5, const Complex *b_5, const Complex *c_5,
                            bool dagger, int parity, int shift[4], int halo_shift[4], MdwfFusedDslashType type);
  }
 
  // The EOFA stuff
  namespace mobius_eofa
  {
    void apply_dslash5(ColorSpinorField &out, const ColorSpinorField &in, const ColorSpinorField &x, double m_f,
                       double m_5, const Complex *b_5, const Complex *c_5, double a, int eofa_pm, double inv,
                       double kappa, const double *eofa_u, const double *eofa_x, const double *eofa_y,
                       double sherman_morrison, bool dagger, Dslash5Type type);
  }
 
  void ApplyLaplace(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int dir, double a, double b,
                    const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyCovDev(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, int mu, int parity,
                   bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyClover(
      ColorSpinorField &out, const ColorSpinorField &in, const CloverField &clover, bool inverse, int parity);
 
  void ApplyStaggered(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U, double a,
                      const ColorSpinorField &x, int parity, bool dagger, const int *comm_override, TimeProfile &profile);
 
  void ApplyImprovedStaggered(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &U,
                              const GaugeField &L, double a, const ColorSpinorField &x, int parity, bool dagger,
                              const int *comm_override, TimeProfile &profile);
 
  void ApplyStaggeredKahlerDiracInverse(ColorSpinorField &out, const ColorSpinorField &in, const GaugeField &Xinv,
                                        bool dagger);
 
  void ApplyTwistGamma(ColorSpinorField &out, const ColorSpinorField &in, int d, double kappa, double mu,
                       double epsilon, int dagger, QudaTwistGamma5Type type);
 
  void ApplyTwistClover(ColorSpinorField &out, const ColorSpinorField &in, const CloverField &clover,
                        double kappa, double mu, double epsilon, int parity, int dagger, QudaTwistGamma5Type twist);
 
  void PackGhost(void *ghost[2 * QUDA_MAX_DIM], const ColorSpinorField &field, MemoryLocation location, int nFace,
                 bool dagger, int parity, bool spin_project, double a, double b, double c, int shmem,
                 const qudaStream_t &stream);
 
  void gamma5(ColorSpinorField &out, const ColorSpinorField &in);
 
}
 
#endif // _DSLASH_QUDA_H