quda.h

Go to the documentation of this file.

#pragma once
 
#include <enum_quda.h>
#include <stdio.h> /* for FILE */
#include <quda_define.h>
#include <quda_constants.h>
 
#ifndef __CUDACC_RTC__
#define double_complex double _Complex
#else // keep NVRTC happy since it can't handle C types
#define double_complex double2
#endif
 
#ifdef __cplusplus
extern "C" {
#endif
 
  typedef struct QudaGaugeParam_s {
    size_t struct_size; 
    QudaFieldLocation location; 
    int X[4];             
    double anisotropy;    
    double tadpole_coeff; 
    double scale; 
    QudaLinkType type; 
    QudaGaugeFieldOrder gauge_order; 
    QudaTboundary t_boundary;  
    QudaPrecision cpu_prec; 
    QudaPrecision cuda_prec; 
    QudaReconstructType reconstruct; 
    QudaPrecision cuda_prec_sloppy; 
    QudaReconstructType reconstruct_sloppy; 
    QudaPrecision cuda_prec_refinement_sloppy; 
    QudaReconstructType reconstruct_refinement_sloppy; 
    QudaPrecision cuda_prec_precondition; 
    QudaReconstructType reconstruct_precondition; 
    QudaPrecision cuda_prec_eigensolver;         
    QudaReconstructType reconstruct_eigensolver; 
    QudaGaugeFixed gauge_fix; 
    int ga_pad;       
    int site_ga_pad;  
    int staple_pad;   
    int llfat_ga_pad; 
    int mom_ga_pad;   
    QudaStaggeredPhase staggered_phase_type; 
    int staggered_phase_applied; 
    double i_mu; 
    int overlap; 
    int overwrite_mom; 
    int use_resident_gauge;  
    int use_resident_mom;    
    int make_resident_gauge; 
    int make_resident_mom;   
    int return_result_gauge; 
    int return_result_mom;   
    size_t gauge_offset; 
    size_t mom_offset; 
    size_t site_size; 
  } QudaGaugeParam;
 
 
  typedef struct QudaInvertParam_s {
 
    size_t struct_size;
 
    QudaFieldLocation input_location; 
    QudaFieldLocation output_location; 
    QudaDslashType dslash_type; 
    QudaInverterType inv_type; 
    double mass;  
    double kappa; 
    double m5;    
    int Ls;       
    double_complex b_5[QUDA_MAX_DWF_LS]; 
    double_complex c_5[QUDA_MAX_DWF_LS]; 
    double eofa_shift;
    int eofa_pm;
    double mq1;
    double mq2;
    double mq3;
 
    double mu;    
    double epsilon; 
    QudaTwistFlavorType twist_flavor;  
    int laplace3D; 
    double tol;    
    double tol_restart;   
    double tol_hq; 
    int compute_true_res; 
    double true_res; 
    double true_res_hq; 
    int maxiter; 
    double reliable_delta; 
    double reliable_delta_refinement; 
    int use_alternative_reliable; 
    int use_sloppy_partial_accumulator; 
    int solution_accumulator_pipeline;
 
    int max_res_increase;
 
    int max_res_increase_total;
 
    int max_hq_res_increase;
 
    int max_hq_res_restart_total;
 
    int heavy_quark_check;
 
    int pipeline; 
    int num_offset; 
    int num_src; 
    int num_src_per_sub_partition; 
    int split_grid[QUDA_MAX_DIM];
 
    int overlap; 
    double offset[QUDA_MAX_MULTI_SHIFT];
 
    double tol_offset[QUDA_MAX_MULTI_SHIFT];
 
    double tol_hq_offset[QUDA_MAX_MULTI_SHIFT];
 
    double true_res_offset[QUDA_MAX_MULTI_SHIFT];
 
    double iter_res_offset[QUDA_MAX_MULTI_SHIFT];
 
    double true_res_hq_offset[QUDA_MAX_MULTI_SHIFT];
 
    double residue[QUDA_MAX_MULTI_SHIFT];
 
    int compute_action;
 
    double action[2];
 
    QudaSolutionType solution_type;  
    QudaSolveType solve_type;        
    QudaMatPCType matpc_type;        
    QudaDagType dagger;              
    QudaMassNormalization mass_normalization; 
    QudaSolverNormalization solver_normalization; 
    QudaPreserveSource preserve_source;       
    QudaPrecision cpu_prec;                
    QudaPrecision cuda_prec;               
    QudaPrecision cuda_prec_sloppy;        
    QudaPrecision cuda_prec_refinement_sloppy; 
    QudaPrecision cuda_prec_precondition;  
    QudaPrecision cuda_prec_eigensolver;   
    QudaDiracFieldOrder dirac_order;       
    QudaGammaBasis gamma_basis;            
    QudaFieldLocation clover_location;     
    QudaPrecision clover_cpu_prec;         
    QudaPrecision clover_cuda_prec;        
    QudaPrecision clover_cuda_prec_sloppy; 
    QudaPrecision clover_cuda_prec_refinement_sloppy; 
    QudaPrecision clover_cuda_prec_precondition; 
    QudaPrecision clover_cuda_prec_eigensolver;  
    QudaCloverFieldOrder clover_order;     
    QudaUseInitGuess use_init_guess;       
    double clover_csw;                     
    double clover_coeff;                   
    double clover_rho;                     
    int compute_clover_trlog;              
    double trlogA[2];                      
    int compute_clover;                    
    int compute_clover_inverse;            
    int return_clover;                     
    int return_clover_inverse;             
    QudaVerbosity verbosity;               
    int sp_pad;                            
    int cl_pad;                            
    int iter;                              
    double gflops;                         
    double secs;                           
    QudaTune tune; 
    int Nsteps;
 
    int gcrNkrylov;
 
    /*
     * The following parameters are related to the solver
     * preconditioner, if enabled.
     */
 
    QudaInverterType inv_type_precondition;
 
    void *preconditioner;
 
    void *deflation_op;
 
    void *eig_param;
 
    QudaBoolean deflate;
 
    QudaDslashType dslash_type_precondition;
    QudaVerbosity verbosity_precondition;
 
    double tol_precondition;
 
    int maxiter_precondition;
 
    double omega;
 
    QudaCABasis ca_basis;
 
    double ca_lambda_min;
 
    double ca_lambda_max;
 
    int precondition_cycle;
 
    QudaSchwarzType schwarz_type;
 
    QudaResidualType residual_type;
 
    QudaPrecision cuda_prec_ritz;
    int n_ev;
    int max_search_dim;
    int rhs_idx;
    int deflation_grid;
    double eigenval_tol;
    int eigcg_max_restarts;
    int max_restart_num;
    double inc_tol;
 
    int make_resident_solution;
 
    int use_resident_solution;
 
    int chrono_make_resident;
 
    int chrono_replace_last;
 
    int chrono_use_resident;
 
    int chrono_max_dim;
 
    int chrono_index;
 
    QudaPrecision chrono_precision;
 
    QudaExtLibType extlib_type;
 
    QudaBoolean native_blas_lapack;
  } QudaInvertParam;
 
  // Parameter set for solving eigenvalue problems.
  typedef struct QudaEigParam_s {
    size_t struct_size;
 
    // EIGENSOLVER PARAMS
    //-------------------------------------------------
    QudaInvertParam *invert_param;
 
    QudaEigType eig_type;
 
    QudaBoolean use_poly_acc;
 
    int poly_deg;
 
    double a_min;
    double a_max;
 
    QudaBoolean preserve_deflation;
 
    void *preserve_deflation_space;
 
    QudaBoolean preserve_evals;
 
    QudaBoolean use_dagger;
    QudaBoolean use_norm_op;
 
    QudaBoolean use_eigen_qr;
 
    QudaBoolean compute_svd;
 
    QudaBoolean compute_gamma5;
 
    QudaBoolean require_convergence;
 
    QudaEigSpectrumType spectrum;
 
    int n_ev;
    int n_kr;
    int nLockedMax;
    int n_conv;
    int n_ev_deflate;
    double tol;
    double qr_tol;
    int check_interval;
    int max_restarts;
    int batched_rotate;
    int block_size;
 
    QudaBoolean arpack_check;
    char arpack_logfile[512];
 
    char QUDA_logfile[512];
 
    //-------------------------------------------------
 
    // EIG-CG PARAMS
    //-------------------------------------------------
    int nk;
    int np;
 
    QudaBoolean import_vectors;
 
    QudaPrecision cuda_prec_ritz;
 
    QudaMemoryType mem_type_ritz;
 
    QudaFieldLocation location;
 
    QudaBoolean run_verify;
 
    char vec_infile[256];
 
    char vec_outfile[256];
 
    QudaPrecision save_prec;
 
    QudaBoolean io_parity_inflate;
 
    double gflops;
 
    double secs;
 
    QudaExtLibType extlib_type;
    //-------------------------------------------------
  } QudaEigParam;
 
  typedef struct QudaMultigridParam_s {
 
    size_t struct_size;
 
    QudaInvertParam *invert_param;
 
    QudaEigParam *eig_param[QUDA_MAX_MG_LEVEL];
 
    int n_level;
 
    int geo_block_size[QUDA_MAX_MG_LEVEL][QUDA_MAX_DIM];
 
    int spin_block_size[QUDA_MAX_MG_LEVEL];
 
    int n_vec[QUDA_MAX_MG_LEVEL];
 
    QudaPrecision precision_null[QUDA_MAX_MG_LEVEL];
 
    int n_block_ortho[QUDA_MAX_MG_LEVEL];
 
    QudaVerbosity verbosity[QUDA_MAX_MG_LEVEL];
 
    QudaInverterType setup_inv_type[QUDA_MAX_MG_LEVEL];
 
    int num_setup_iter[QUDA_MAX_MG_LEVEL];
 
    double setup_tol[QUDA_MAX_MG_LEVEL];
 
    int setup_maxiter[QUDA_MAX_MG_LEVEL];
 
    int setup_maxiter_refresh[QUDA_MAX_MG_LEVEL];
 
    QudaCABasis setup_ca_basis[QUDA_MAX_MG_LEVEL];
 
    int setup_ca_basis_size[QUDA_MAX_MG_LEVEL];
 
    double setup_ca_lambda_min[QUDA_MAX_MG_LEVEL];
 
    double setup_ca_lambda_max[QUDA_MAX_MG_LEVEL];
 
    QudaSetupType setup_type;
 
    QudaBoolean pre_orthonormalize;
 
    QudaBoolean post_orthonormalize;
 
    QudaInverterType coarse_solver[QUDA_MAX_MG_LEVEL];
 
    double coarse_solver_tol[QUDA_MAX_MG_LEVEL];
 
    int coarse_solver_maxiter[QUDA_MAX_MG_LEVEL];
 
    QudaCABasis coarse_solver_ca_basis[QUDA_MAX_MG_LEVEL];
 
    int coarse_solver_ca_basis_size[QUDA_MAX_MG_LEVEL];
 
    double coarse_solver_ca_lambda_min[QUDA_MAX_MG_LEVEL];
 
    double coarse_solver_ca_lambda_max[QUDA_MAX_MG_LEVEL];
 
    QudaInverterType smoother[QUDA_MAX_MG_LEVEL];
 
    double smoother_tol[QUDA_MAX_MG_LEVEL];
 
    int nu_pre[QUDA_MAX_MG_LEVEL];
 
    int nu_post[QUDA_MAX_MG_LEVEL];
 
    double omega[QUDA_MAX_MG_LEVEL];
 
    QudaPrecision smoother_halo_precision[QUDA_MAX_MG_LEVEL];
 
    QudaSchwarzType smoother_schwarz_type[QUDA_MAX_MG_LEVEL];
 
    int smoother_schwarz_cycle[QUDA_MAX_MG_LEVEL];
 
    QudaSolutionType coarse_grid_solution_type[QUDA_MAX_MG_LEVEL];
 
    QudaSolveType smoother_solve_type[QUDA_MAX_MG_LEVEL];
 
    QudaMultigridCycleType cycle_type[QUDA_MAX_MG_LEVEL];
 
    QudaBoolean global_reduction[QUDA_MAX_MG_LEVEL];
 
    QudaFieldLocation location[QUDA_MAX_MG_LEVEL];
 
    QudaFieldLocation setup_location[QUDA_MAX_MG_LEVEL];
 
    QudaBoolean use_eig_solver[QUDA_MAX_MG_LEVEL];
 
    QudaBoolean setup_minimize_memory;
 
    QudaComputeNullVector compute_null_vector;
 
    QudaBoolean generate_all_levels;
 
    QudaBoolean run_verify;
 
    QudaBoolean run_low_mode_check;
 
    QudaBoolean run_oblique_proj_check;
 
    QudaBoolean vec_load[QUDA_MAX_MG_LEVEL];
 
    char vec_infile[QUDA_MAX_MG_LEVEL][256];
 
    QudaBoolean vec_store[QUDA_MAX_MG_LEVEL];
 
    char vec_outfile[QUDA_MAX_MG_LEVEL][256];
 
    QudaBoolean coarse_guess;
 
    QudaBoolean preserve_deflation;
 
    double gflops;
 
    double secs;
 
    double mu_factor[QUDA_MAX_MG_LEVEL];
 
    QudaTransferType transfer_type[QUDA_MAX_MG_LEVEL];
 
    QudaBoolean use_mma;
 
    QudaBoolean thin_update_only;
  } QudaMultigridParam;
 
  typedef struct QudaGaugeObservableParam_s {
    size_t struct_size; 
    QudaBoolean su_project;              
    QudaBoolean compute_plaquette;       
    double plaquette[3];                 
    QudaBoolean compute_qcharge;         
    double qcharge;                      
    double energy[3];                    
    QudaBoolean compute_qcharge_density; 
    void *qcharge_density; 
  } QudaGaugeObservableParam;
 
  typedef struct QudaBLASParam_s {
    size_t struct_size; 
    QudaBLASOperation trans_a; 
    QudaBLASOperation trans_b; 
    int m;                     
    int n;                     
    int k;                     
    int lda;                   
    int ldb;                   
    int ldc;                   
    int a_offset;              
    int b_offset;              
    int c_offset;              
    int a_stride;              
    int b_stride;              
    int c_stride;              
    double_complex alpha; 
    double_complex beta;  
    int batch_count; 
    QudaBLASDataType data_type;   
    QudaBLASDataOrder data_order; 
  } QudaBLASParam;
 
  /*
   * Interface functions, found in interface_quda.cpp
   */
 
  void setVerbosityQuda(QudaVerbosity verbosity, const char prefix[],
                        FILE *outfile);
 
  typedef int (*QudaCommsMap)(const int *coords, void *fdata);
 
  void qudaSetCommHandle(void *mycomm);
 
  void initCommsGridQuda(int nDim, const int *dims, QudaCommsMap func, void *fdata);
 
  void initQudaDevice(int device);
 
  void initQudaMemory();
 
  void initQuda(int device);
 
  void endQuda(void);
 
  void updateR();
 
  QudaGaugeParam newQudaGaugeParam(void);
 
  QudaInvertParam newQudaInvertParam(void);
 
  QudaMultigridParam newQudaMultigridParam(void);
 
  QudaEigParam newQudaEigParam(void);
 
  QudaGaugeObservableParam newQudaGaugeObservableParam(void);
 
  QudaBLASParam newQudaBLASParam(void);
 
  void printQudaGaugeParam(QudaGaugeParam *param);
 
  void printQudaInvertParam(QudaInvertParam *param);
 
  void printQudaMultigridParam(QudaMultigridParam *param);
 
  void printQudaEigParam(QudaEigParam *param);
 
  void printQudaGaugeObservableParam(QudaGaugeObservableParam *param);
 
  void printQudaBLASParam(QudaBLASParam *param);
 
  void loadGaugeQuda(void *h_gauge, QudaGaugeParam *param);
 
  void freeGaugeQuda(void);
 
  void saveGaugeQuda(void *h_gauge, QudaGaugeParam *param);
 
  void loadCloverQuda(void *h_clover, void *h_clovinv,
      QudaInvertParam *inv_param);
 
  void freeCloverQuda(void);
 
  void lanczosQuda(int k0, int m, void *hp_Apsi, void *hp_r, void *hp_V, void *hp_alpha, void *hp_beta,
                   QudaEigParam *eig_param);
 
  void eigensolveQuda(void **h_evecs, double_complex *h_evals, QudaEigParam *param);
 
  void invertQuda(void *h_x, void *h_b, QudaInvertParam *param);
 
  void invertMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, void *h_gauge, QudaGaugeParam *gauge_param);
 
  void invertMultiSrcStaggeredQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, void *milc_fatlinks,
                                   void *milc_longlinks, QudaGaugeParam *gauge_param);
 
  void invertMultiSrcCloverQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, void *h_gauge,
                                QudaGaugeParam *gauge_param, void *h_clover, void *h_clovinv);
 
  void invertMultiShiftQuda(void **_hp_x, void *_hp_b, QudaInvertParam *param);
 
  void* newMultigridQuda(QudaMultigridParam *param);
 
  void destroyMultigridQuda(void *mg_instance);
 
  void updateMultigridQuda(void *mg_instance, QudaMultigridParam *param);
 
  void dumpMultigridQuda(void *mg_instance, QudaMultigridParam *param);
 
  void dslashQuda(void *h_out, void *h_in, QudaInvertParam *inv_param, QudaParity parity);
 
  void dslashMultiSrcQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, QudaParity parity, void *h_gauge,
                          QudaGaugeParam *gauge_param);
  void dslashMultiSrcStaggeredQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, QudaParity parity,
                                   void *milc_fatlinks, void *milc_longlinks, QudaGaugeParam *gauge_param);
 
  void dslashMultiSrcCloverQuda(void **_hp_x, void **_hp_b, QudaInvertParam *param, QudaParity parity, void *h_gauge,
                                QudaGaugeParam *gauge_param, void *h_clover, void *h_clovinv);
 
  void cloverQuda(void *h_out, void *h_in, QudaInvertParam *inv_param, QudaParity parity, int inverse);
 
  void MatQuda(void *h_out, void *h_in, QudaInvertParam *inv_param);
 
  void MatDagMatQuda(void *h_out, void *h_in, QudaInvertParam *inv_param);
 
 
  /*
   * The following routines are temporary additions used by the HISQ
   * link-fattening code.
   */
 
  void set_dim(int *);
  void pack_ghost(void **cpuLink, void **cpuGhost, int nFace,
      QudaPrecision precision);
 
  void computeKSLinkQuda(void* fatlink, void* longlink, void* ulink, void* inlink,
                         double *path_coeff, QudaGaugeParam *param);
 
  void momResidentQuda(void *mom, QudaGaugeParam *param);
 
  int computeGaugeForceQuda(void* mom, void* sitelink,  int*** input_path_buf, int* path_length,
                            double* loop_coeff, int num_paths, int max_length, double dt,
                            QudaGaugeParam* qudaGaugeParam);
 
  void updateGaugeFieldQuda(void* gauge, void* momentum, double dt,
      int conj_mom, int exact, QudaGaugeParam* param);
 
  void staggeredPhaseQuda(void *gauge_h, QudaGaugeParam *param);
 
  void projectSU3Quda(void *gauge_h, double tol, QudaGaugeParam *param);
 
  double momActionQuda(void* momentum, QudaGaugeParam* param);
 
  void* createGaugeFieldQuda(void* gauge, int geometry, QudaGaugeParam* param);
 
  void  saveGaugeFieldQuda(void* outGauge, void* inGauge, QudaGaugeParam* param);
 
  void destroyGaugeFieldQuda(void* gauge);
 
  void createCloverQuda(QudaInvertParam* param);
 
  void computeCloverForceQuda(void *mom, double dt, void **x, void **p, double *coeff, double kappa2, double ck,
                              int nvector, double multiplicity, void *gauge,
                              QudaGaugeParam *gauge_param, QudaInvertParam *inv_param);
 
  void computeStaggeredForceQuda(void *mom, double dt, double delta, void *gauge, void **x, QudaGaugeParam *gauge_param,
                                 QudaInvertParam *invert_param);
 
  void computeHISQForceQuda(void* momentum,
                            double dt,
                            const double level2_coeff[6],
                            const double fat7_coeff[6],
                            const void* const w_link,
                            const void* const v_link,
                            const void* const u_link,
                            void** quark,
                            int num,
                            int num_naik,
                            double** coeff,
                            QudaGaugeParam* param);
 
  void gaussGaugeQuda(unsigned long long seed, double sigma);
 
  void plaqQuda(double plaq[3]);
 
  void copyExtendedResidentGaugeQuda(void* resident_gauge, QudaFieldLocation loc);
 
  void performWuppertalnStep(void *h_out, void *h_in, QudaInvertParam *param, unsigned int n_steps, double alpha);
 
  void performAPEnStep(unsigned int n_steps, double alpha, int meas_interval);
 
  void performSTOUTnStep(unsigned int n_steps, double rho, int meas_interval);
 
  void performOvrImpSTOUTnStep(unsigned int n_steps, double rho, double epsilon, int meas_interval);
 
  void performWFlownStep(unsigned int n_steps, double step_size, int meas_interval, QudaWFlowType wflow_type);
 
  void gaugeObservablesQuda(QudaGaugeObservableParam *param);
 
  void contractQuda(const void *x, const void *y, void *result, const QudaContractType cType, QudaInvertParam *param,
                    const int *X);
 
  int computeGaugeFixingOVRQuda(void *gauge, const unsigned int gauge_dir, const unsigned int Nsteps,
                                const unsigned int verbose_interval, const double relax_boost, const double tolerance,
                                const unsigned int reunit_interval, const unsigned int stopWtheta,
                                QudaGaugeParam *param, double *timeinfo);
  int computeGaugeFixingFFTQuda(void *gauge, const unsigned int gauge_dir, const unsigned int Nsteps,
                                const unsigned int verbose_interval, const double alpha, const unsigned int autotune,
                                const double tolerance, const unsigned int stopWtheta, QudaGaugeParam *param,
                                double *timeinfo);
 
  void blasGEMMQuda(void *arrayA, void *arrayB, void *arrayC, QudaBoolean native, QudaBLASParam *param);
 
  void flushChronoQuda(int index);
 
  void openMagma();
 
  void closeMagma();
 
  void* newDeflationQuda(QudaEigParam *param);
 
  void destroyDeflationQuda(void *df_instance);
 
  void setMPICommHandleQuda(void *mycomm);
 
#ifdef __cplusplus
}
#endif
 
// remove NVRTC WAR
#undef double_complex
 
/* #include <quda_new_interface.h> */