QUDA
1.0.0
|
#include <color_spinor_field.h>
Public Member Functions | |
cudaColorSpinorField (const cudaColorSpinorField &) | |
cudaColorSpinorField (const ColorSpinorField &, const ColorSpinorParam &) | |
cudaColorSpinorField (const ColorSpinorField &) | |
cudaColorSpinorField (const ColorSpinorParam &) | |
virtual | ~cudaColorSpinorField () |
ColorSpinorField & | operator= (const ColorSpinorField &) |
cudaColorSpinorField & | operator= (const cudaColorSpinorField &) |
cudaColorSpinorField & | operator= (const cpuColorSpinorField &) |
void | switchBufferPinned () |
void | createComms (int nFace, bool spin_project=true) |
Create the communication handlers and buffers. More... | |
void | allocateGhostBuffer (int nFace, bool spin_project=true) const |
Allocate the ghost buffers. More... | |
void | packGhost (const int nFace, const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream, MemoryLocation location[2 *QUDA_MAX_DIM], MemoryLocation location_label, bool spin_project, double a=0, double b=0, double c=0) |
Packs the cudaColorSpinorField's ghost zone. More... | |
void | packGhostExtended (const int nFace, const int R[], const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream, bool zero_copy=false) |
void | sendGhost (void *ghost_spinor, const int nFace, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream) |
void | unpackGhost (const void *ghost_spinor, const int nFace, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream) |
void | unpackGhostExtended (const void *ghost_spinor, const int nFace, const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream, bool zero_copy) |
void | streamInit (cudaStream_t *stream_p) |
void | pack (int nFace, int parity, int dagger, int stream_idx, MemoryLocation location[], MemoryLocation location_label, bool spin_project=true, double a=0, double b=0, double c=0) |
void | packExtended (const int nFace, const int R[], const int parity, const int dagger, const int dim, cudaStream_t *stream_p, const bool zeroCopyPack=false) |
void | gather (int nFace, int dagger, int dir, cudaStream_t *stream_p=NULL) |
void | recvStart (int nFace, int dir, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr=false) |
Initiate halo communication receive. More... | |
void | sendStart (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr=false, bool remote_write=false) |
Initiate halo communication sending. More... | |
void | commsStart (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false) |
Initiate halo communication. More... | |
int | commsQuery (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false) |
Non-blocking query if the halo communication has completed. More... | |
void | commsWait (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false) |
Wait on halo communication to complete. More... | |
void | scatter (int nFace, int dagger, int dir, cudaStream_t *stream_p) |
void | scatter (int nFace, int dagger, int dir) |
void | scatterExtended (int nFace, int parity, int dagger, int dir) |
const void * | Ghost2 () const |
void | exchangeGhost (QudaParity parity, int nFace, int dagger, const MemoryLocation *pack_destination=nullptr, const MemoryLocation *halo_location=nullptr, bool gdr_send=false, bool gdr_recv=false, QudaPrecision ghost_precision=QUDA_INVALID_PRECISION) const |
This is a unified ghost exchange function for doing a complete halo exchange regardless of the type of field. All dimensions are exchanged and no spin projection is done in the case of Wilson fermions. More... | |
cudaColorSpinorField & | Component (const int idx) const |
for composite fields: More... | |
CompositeColorSpinorField & | Components () const |
void | CopySubset (cudaColorSpinorField &dst, const int range, const int first_element=0) const |
void | zero () |
void | getTexObjectInfo () const |
void | Source (const QudaSourceType sourceType, const int st=0, const int s=0, const int c=0) |
void | PrintVector (unsigned int x) const |
void | backup () const |
Backs up the cudaColorSpinorField. More... | |
void | restore () const |
Restores the cudaColorSpinorField. More... | |
![]() | |
ColorSpinorField (const ColorSpinorField &) | |
ColorSpinorField (const ColorSpinorParam &) | |
virtual | ~ColorSpinorField () |
int | Ncolor () const |
int | Nspin () const |
int | Nvec () const |
QudaTwistFlavorType | TwistFlavor () const |
int | Ndim () const |
const int * | X () const |
int | X (int d) const |
size_t | RealLength () const |
size_t | Length () const |
int | Stride () const |
int | Volume () const |
int | VolumeCB () const |
int | Pad () const |
size_t | Bytes () const |
size_t | NormBytes () const |
size_t | GhostBytes () const |
size_t | GhostNormBytes () const |
void | PrintDims () const |
void * | V () |
const void * | V () const |
void * | Norm () |
const void * | Norm () const |
void | exchange (void **ghost, void **sendbuf, int nFace=1) const |
bool | isNative () const |
bool | IsComposite () const |
bool | IsComponent () const |
int | CompositeDim () const |
int | ComponentId () const |
int | ComponentVolume () const |
int | ComponentVolumeCB () const |
int | ComponentStride () const |
size_t | ComponentLength () const |
size_t | ComponentRealLength () const |
size_t | ComponentBytes () const |
size_t | ComponentNormBytes () const |
QudaPCType | PCType () const |
QudaSiteSubset | SiteSubset () const |
QudaSiteOrder | SiteOrder () const |
QudaFieldOrder | FieldOrder () const |
QudaGammaBasis | GammaBasis () const |
const int * | GhostFace () const |
const int * | GhostFaceCB () const |
int | GhostOffset (const int i) const |
int | GhostOffset (const int i, const int j) const |
int | GhostNormOffset (const int i) const |
int | GhostNormOffset (const int i, const int j) const |
void * | Ghost (const int i) |
const void * | Ghost (const int i) const |
void * | GhostNorm (const int i) |
const void * | GhostNorm (const int i) const |
void *const * | Ghost () const |
const DslashConstant & | getDslashConstant () const |
Get the dslash_constant structure from this field. More... | |
const ColorSpinorField & | Even () const |
const ColorSpinorField & | Odd () const |
ColorSpinorField & | Even () |
ColorSpinorField & | Odd () |
ColorSpinorField & | Component (const int idx) const |
ColorSpinorField & | Component (const int idx) |
CompositeColorSpinorField & | Components () |
void | LatticeIndex (int *y, int i) const |
void | OffsetIndex (int &i, int *y) const |
ColorSpinorField * | CreateCoarse (const int *geoBlockSize, int spinBlockSize, int Nvec, QudaPrecision precision=QUDA_INVALID_PRECISION, QudaFieldLocation location=QUDA_INVALID_FIELD_LOCATION, QudaMemoryType mem_Type=QUDA_MEMORY_INVALID) |
Create a coarse color-spinor field, using this field to set the meta data. More... | |
ColorSpinorField * | CreateFine (const int *geoblockSize, int spinBlockSize, int Nvec, QudaPrecision precision=QUDA_INVALID_PRECISION, QudaFieldLocation location=QUDA_INVALID_FIELD_LOCATION, QudaMemoryType mem_type=QUDA_MEMORY_INVALID) |
Create a fine color-spinor field, using this field to set the meta data. More... | |
![]() | |
LatticeField (const LatticeFieldParam ¶m) | |
LatticeField (const LatticeField &field) | |
virtual | ~LatticeField () |
void | allocateGhostBuffer (size_t ghost_bytes) const |
Allocate the static ghost buffers. More... | |
void | createComms (bool no_comms_fill=false, bool bidir=true) |
void | destroyComms () |
void | createIPCComms () |
bool | ipcCopyComplete (int dir, int dim) |
bool | ipcRemoteCopyComplete (int dir, int dim) |
const cudaEvent_t & | getIPCCopyEvent (int dir, int dim) const |
const cudaEvent_t & | getIPCRemoteCopyEvent (int dir, int dim) const |
int | Ndim () const |
const int * | X () const |
int | Volume () const |
int | VolumeCB () const |
const int * | SurfaceCB () const |
int | SurfaceCB (const int i) const |
int | Stride () const |
int | Pad () const |
const int * | R () const |
QudaGhostExchange | GhostExchange () const |
QudaPrecision | Precision () const |
QudaPrecision | GhostPrecision () const |
double | Scale () const |
void | Scale (double scale_) |
Set the scale factor for a fixed-point field. More... | |
virtual QudaMemoryType | MemType () const |
int | Nvec () const |
QudaFieldLocation | Location () const |
size_t | GBytes () const |
void | checkField (const LatticeField &a) const |
virtual void | read (char *filename) |
virtual void | write (char *filename) |
const char * | VolString () const |
const char * | AuxString () const |
![]() | |
Object () | |
virtual | ~Object () |
Private Member Functions | |
void | create (const QudaFieldCreate) |
void | destroy () |
void | copy (const cudaColorSpinorField &) |
void | zeroPad () |
Zero the padded regions added on to the field. Ensures correct reductions and silences false positive warnings regarding uninitialized memory. More... | |
void | copySpinorField (const ColorSpinorField &src) |
void | loadSpinorField (const ColorSpinorField &src) |
void | saveSpinorField (ColorSpinorField &src) const |
Private Attributes | |
bool | alloc |
bool | init |
bool | texInit |
bool | ghostTexInit |
QudaPrecision | ghost_precision_tex |
bool | reference |
void * | ghost_field_tex [4] |
Friends | |
class | cpuColorSpinorField |
std::ostream & | operator<< (std::ostream &out, const cudaColorSpinorField &) |
Definition at line 575 of file color_spinor_field.h.
quda::cudaColorSpinorField::cudaColorSpinorField | ( | const cudaColorSpinorField & | src | ) |
Definition at line 43 of file cuda_color_spinor_field.cpp.
References copySpinorField(), create(), and QUDA_COPY_FIELD_CREATE.
Referenced by CopySubset(), and create().
quda::cudaColorSpinorField::cudaColorSpinorField | ( | const ColorSpinorField & | src, |
const ColorSpinorParam & | param | ||
) |
Definition at line 57 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::bytes, quda::ColorSpinorField::composite_descr, copySpinorField(), create(), errorQuda, quda::CompositeColorSpinorFieldDescriptor::id, quda::CompositeColorSpinorFieldDescriptor::is_component, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, param, QUDA_COPY_FIELD_CREATE, QUDA_FULL_SITE_SUBSET, QUDA_NULL_FIELD_CREATE, QUDA_PARITY_SITE_SUBSET, QUDA_REFERENCE_FIELD_CREATE, QUDA_ZERO_FIELD_CREATE, quda::ColorSpinorField::reset(), quda::ColorSpinorField::SiteSubset(), quda::ColorSpinorField::v, and zero().
quda::cudaColorSpinorField::cudaColorSpinorField | ( | const ColorSpinorField & | src | ) |
Definition at line 110 of file cuda_color_spinor_field.cpp.
References copySpinorField(), create(), and QUDA_COPY_FIELD_CREATE.
quda::cudaColorSpinorField::cudaColorSpinorField | ( | const ColorSpinorParam & | param | ) |
Definition at line 15 of file cuda_color_spinor_field.cpp.
References create(), errorQuda, quda::ColorSpinorField::norm, param, QUDA_COPY_FIELD_CREATE, QUDA_NULL_FIELD_CREATE, QUDA_REFERENCE_FIELD_CREATE, QUDA_ZERO_FIELD_CREATE, quda::ColorSpinorField::v, and zero().
|
virtual |
Definition at line 159 of file cuda_color_spinor_field.cpp.
References destroy(), and quda::LatticeField::destroyComms().
void quda::cudaColorSpinorField::allocateGhostBuffer | ( | int | nFace, |
bool | spin_project = true |
||
) | const |
Allocate the ghost buffers.
[in] | nFace | Depth of each halo |
[in] | spin_project | Whether the halos are spin projected (Wilson-type fermions only) |
Definition at line 747 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::allocateGhostBuffer(), quda::ColorSpinorField::createGhostZone(), quda::LatticeField::ghost_bytes, ghost_field_tex, quda::LatticeField::ghost_pinned_recv_buffer_hd, quda::LatticeField::ghost_precision_reset, quda::LatticeField::ghost_recv_buffer_d, and ghostTexInit.
Referenced by quda::computeStaggeredOprod(), and createComms().
|
virtual |
Backs up the cudaColorSpinorField.
Reimplemented from quda::LatticeField.
Definition at line 540 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::backed_up, quda::LatticeField::backup_h, quda::LatticeField::backup_norm_h, quda::ColorSpinorField::bytes, checkCudaError, errorQuda, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, and quda::ColorSpinorField::v.
|
virtual |
Non-blocking query if the halo communication has completed.
[in] | Depth | of face exchange |
[in] | d | d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards) |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream to be used (unused) |
[in] | gdr_send | Whether we are using GDR on the send side |
[in] | gdr_recv | Whether we are using GDR on the receive side |
Reimplemented from quda::LatticeField.
Definition at line 1127 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_gdr_enabled(), comm_peer2peer_enabled(), comm_query(), commDimPartitioned(), errorQuda, quda::LatticeField::mh_recv_back, quda::LatticeField::mh_recv_fwd, quda::LatticeField::mh_recv_p2p_back, quda::LatticeField::mh_recv_p2p_fwd, quda::LatticeField::mh_recv_rdma_back, quda::LatticeField::mh_recv_rdma_fwd, quda::LatticeField::mh_send_back, quda::LatticeField::mh_send_fwd, quda::LatticeField::mh_send_p2p_back, quda::LatticeField::mh_send_p2p_fwd, quda::LatticeField::mh_send_rdma_back, and quda::LatticeField::mh_send_rdma_fwd.
Referenced by quda::dslash::commsComplete(), quda::exchangeExtendedGhost(), and exchangeGhost().
|
virtual |
Initiate halo communication.
[in] | Depth | of face exchange |
[in] | d | d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards) |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream to be used (unused) |
[in] | gdr_send | Whether we are using GDR on the send side |
[in] | gdr_recv | Whether we are using GDR on the receive side |
Reimplemented from quda::LatticeField.
Definition at line 1116 of file cuda_color_spinor_field.cpp.
References recvStart(), and sendStart().
Referenced by quda::exchangeExtendedGhost().
|
virtual |
Wait on halo communication to complete.
[in] | Depth | of face exchange |
[in] | d | d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards) |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream to be used (unused) |
[in] | gdr_send | Whether we are using GDR on the send side |
[in] | gdr_recv | Whether we are using GDR on the receive side |
Reimplemented from quda::LatticeField.
Definition at line 1195 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_gdr_enabled(), comm_peer2peer_enabled(), comm_wait(), commDimPartitioned(), errorQuda, quda::LatticeField::ipcCopyEvent, quda::LatticeField::ipcRemoteCopyEvent, quda::LatticeField::mh_recv_back, quda::LatticeField::mh_recv_fwd, quda::LatticeField::mh_recv_p2p_back, quda::LatticeField::mh_recv_p2p_fwd, quda::LatticeField::mh_recv_rdma_back, quda::LatticeField::mh_recv_rdma_fwd, quda::LatticeField::mh_send_back, quda::LatticeField::mh_send_fwd, quda::LatticeField::mh_send_p2p_back, quda::LatticeField::mh_send_p2p_fwd, quda::LatticeField::mh_send_rdma_back, and quda::LatticeField::mh_send_rdma_fwd.
cudaColorSpinorField & quda::cudaColorSpinorField::Component | ( | const int | idx | ) | const |
for composite fields:
Definition at line 1445 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::components, quda::ColorSpinorField::CompositeDim(), errorQuda, and quda::ColorSpinorField::IsComposite().
CompositeColorSpinorField& quda::cudaColorSpinorField::Components | ( | ) | const |
|
private |
Definition at line 621 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::checkField(), quda::blas::copy(), errorQuda, and quda::ColorSpinorField::GammaBasis().
Referenced by copySpinorField().
|
private |
This function is responsible for calling the correct copy kernel given the nature of the source field and the desired destination.
Definition at line 627 of file cuda_color_spinor_field.cpp.
References copy(), quda::copyGenericColorSpinor(), errorQuda, quda::ColorSpinorField::GammaBasis(), quda::ColorSpinorField::isNative(), loadSpinorField(), and QUDA_CUDA_FIELD_LOCATION.
Referenced by cudaColorSpinorField(), and operator=().
void quda::cudaColorSpinorField::CopySubset | ( | cudaColorSpinorField & | dst, |
const int | range, | ||
const int | first_element = 0 |
||
) | const |
Definition at line 1460 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::checkField(), quda::ColorSpinorParam::create, cudaColorSpinorField(), errorQuda, quda::ColorSpinorParam::fieldOrder, quda::ColorSpinorField::fieldOrder, quda::ColorSpinorParam::gammaBasis, quda::ColorSpinorField::gammaBasis, quda::ColorSpinorParam::nColor, quda::ColorSpinorField::nColor, quda::LatticeFieldParam::nDim, quda::ColorSpinorField::nDim, quda::ColorSpinorParam::norm, quda::ColorSpinorField::norm, quda::ColorSpinorParam::nSpin, quda::ColorSpinorField::nSpin, quda::LatticeFieldParam::pad, quda::ColorSpinorField::pad, param, quda::LatticeFieldParam::precision, quda::LatticeField::precision, QUDA_PARITY_SITE_SUBSET, QUDA_REFERENCE_FIELD_CREATE, quda::ColorSpinorParam::siteOrder, quda::ColorSpinorField::siteOrder, quda::LatticeFieldParam::siteSubset, quda::ColorSpinorField::siteSubset, quda::ColorSpinorParam::twistFlavor, quda::ColorSpinorField::twistFlavor, quda::ColorSpinorParam::v, quda::ColorSpinorField::v, quda::LatticeFieldParam::x, and quda::ColorSpinorField::x.
|
private |
setup an object for selected eigenvector (the 1st one as a default):
Definition at line 164 of file cuda_color_spinor_field.cpp.
References alloc, quda::ColorSpinorField::bytes, checkCudaError, quda::ColorSpinorParam::component_id, quda::ColorSpinorField::components, quda::ColorSpinorField::composite_descr, quda::ColorSpinorParam::composite_dim, quda::ColorSpinorParam::create, cudaColorSpinorField(), deviceProp, quda::CompositeColorSpinorFieldDescriptor::dim, errorQuda, quda::ColorSpinorField::even, quda::ColorSpinorField::fieldOrder, getVerbosity(), quda::LatticeField::ghost_bytes, ghost_field_tex, quda::LatticeField::ghost_pinned_recv_buffer_hd, quda::LatticeField::ghost_precision, ghost_precision_tex, quda::LatticeField::ghost_recv_buffer_d, ghostTexInit, quda::CompositeColorSpinorFieldDescriptor::id, quda::is_aligned(), quda::CompositeColorSpinorFieldDescriptor::is_component, quda::ColorSpinorParam::is_component, quda::CompositeColorSpinorFieldDescriptor::is_composite, quda::ColorSpinorParam::is_composite, quda::ColorSpinorField::isNative(), mapped_malloc, quda::LatticeFieldParam::mem_type, quda::LatticeField::mem_type, memset(), quda::LatticeFieldParam::nDim, quda::ColorSpinorField::nDim, quda::ColorSpinorParam::norm, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::norm_h, quda::ColorSpinorField::nVec, quda::ColorSpinorField::odd, param, pool_device_malloc, quda::LatticeField::precision, printfQuda, QUDA_DEBUG_VERBOSE, QUDA_DOUBLE_PRECISION, QUDA_EVEN_ODD_SITE_ORDER, QUDA_FLOAT2_FIELD_ORDER, QUDA_FULL_SITE_SUBSET, QUDA_HALF_PRECISION, QUDA_INVALID_PRECISION, QUDA_MEMORY_DEVICE, QUDA_MEMORY_MAPPED, QUDA_PARITY_SITE_SUBSET, QUDA_QUARTER_PRECISION, QUDA_REFERENCE_FIELD_CREATE, QUDA_SINGLE_PRECISION, quda::ColorSpinorField::siteOrder, quda::LatticeFieldParam::siteSubset, quda::ColorSpinorField::siteSubset, texInit, quda::ColorSpinorParam::v, quda::ColorSpinorField::v, quda::ColorSpinorField::v_h, quda::LatticeFieldParam::x, quda::ColorSpinorField::x, and zeroPad().
Referenced by cudaColorSpinorField(), and operator=().
void quda::cudaColorSpinorField::createComms | ( | int | nFace, |
bool | spin_project = true |
||
) |
Create the communication handlers and buffers.
[in] | nFace | Depth of each halo |
[in] | spin_project | Whether the halos are spin projected (Wilson-type fermions only) |
Definition at line 899 of file cuda_color_spinor_field.cpp.
References allocateGhostBuffer(), commDimPartitioned(), quda::LatticeField::createComms(), quda::LatticeField::createIPCComms(), quda::LatticeField::destroyIPCComms(), quda::LatticeField::from_face_d, quda::LatticeField::from_face_h, quda::ColorSpinorField::ghost, quda::LatticeField::ghost_field_reset, quda::LatticeField::ghost_pinned_recv_buffer_h, quda::LatticeField::ghost_pinned_send_buffer_h, quda::LatticeField::ghost_precision, quda::LatticeField::ghost_precision_reset, quda::LatticeField::ghost_recv_buffer_d, quda::LatticeField::ghost_send_buffer_d, quda::ColorSpinorField::ghostNorm, quda::LatticeField::ghostNormOffset, quda::LatticeField::ghostOffset, quda::LatticeField::initComms, quda::LatticeField::my_face_d, quda::LatticeField::my_face_h, quda::LatticeField::nDimComms, QUDA_HALF_PRECISION, QUDA_QUARTER_PRECISION, and QUDA_SINGLE_PRECISION.
Referenced by exchangeGhost(), pack(), and packExtended().
|
private |
Definition at line 502 of file cuda_color_spinor_field.cpp.
References alloc, quda::ColorSpinorField::components, quda::ColorSpinorField::composite_descr, errorQuda, quda::ColorSpinorField::even, host_free, quda::CompositeColorSpinorFieldDescriptor::is_component, quda::CompositeColorSpinorFieldDescriptor::is_composite, quda::LatticeField::mem_type, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_h, quda::ColorSpinorField::odd, pool_device_free, quda::LatticeField::precision, QUDA_FULL_SITE_SUBSET, QUDA_HALF_PRECISION, QUDA_MEMORY_DEVICE, QUDA_MEMORY_MAPPED, QUDA_QUARTER_PRECISION, quda::ColorSpinorField::siteSubset, quda::ColorSpinorField::v, and quda::ColorSpinorField::v_h.
Referenced by operator=(), and ~cudaColorSpinorField().
|
virtual |
This is a unified ghost exchange function for doing a complete halo exchange regardless of the type of field. All dimensions are exchanged and no spin projection is done in the case of Wilson fermions.
[in] | parity | Field parity |
[in] | nFace | Depth of halo exchange |
[in] | dagger | Is this for a dagger operator (only relevant for spin projected Wilson) |
[in] | pack_destination | Destination of the packing buffer |
[in] | halo_location | Destination of the halo reading buffer |
[in] | gdr_send | Are we using GDR for sending |
[in] | gdr_recv | Are we using GDR for receiving |
[in] | ghost_precision | The precision used for the ghost exchange |
Implements quda::ColorSpinorField.
Definition at line 1290 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_dim_partitioned(), comm_gdr_enabled(), comm_peer2peer_enabled(), comm_peer2peer_enabled_global(), commsQuery(), createComms(), quda::Device, errorQuda, quda::LatticeField::from_face_dim_dir_d, quda::LatticeField::from_face_dim_dir_h, quda::LatticeField::from_face_dim_dir_hd, quda::LatticeField::from_face_h, quda::genericPackGhost(), quda::ColorSpinorField::ghost_buf, quda::LatticeField::ghost_face_bytes, quda::LatticeField::ghost_precision, quda::LatticeField::ghost_precision_reset, quda::LatticeField::ghost_recv_buffer_d, quda::LatticeField::ghost_send_buffer_d, quda::Host, quda::LatticeField::ipcRemoteCopyEvent, quda::LatticeField::my_face_dim_dir_d, quda::LatticeField::my_face_dim_dir_h, quda::LatticeField::my_face_dim_dir_hd, quda::LatticeField::my_face_h, quda::LatticeField::nDimComms, quda::popKernelPackT(), quda::LatticeField::precision, quda::pushKernelPackT(), QUDA_INVALID_PRECISION, QUDA_MAX_DIM, qudaDeviceSynchronize, quda::qudaStreamWaitEvent(), recvStart(), sendStart(), streamInit(), streams, and quda::LatticeField::total_bytes.
|
virtual |
Reimplemented from quda::LatticeField.
Definition at line 960 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), quda::LatticeField::my_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and sendGhost().
Referenced by quda::exchangeExtendedGhost(), and quda::dslash::issueGather().
void quda::cudaColorSpinorField::getTexObjectInfo | ( | ) | const |
Definition at line 1518 of file cuda_color_spinor_field.cpp.
References printfQuda.
|
inlinevirtual |
Reimplemented from quda::ColorSpinorField.
Definition at line 806 of file color_spinor_field.h.
References dagger, quda::operator<<(), out, parity, QUDA_INVALID_PRECISION, quda::s, and quda::zero().
|
private |
Definition at line 643 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::bytes, quda::ColorSpinorField::Bytes(), checkCudaError, quda::copyGenericColorSpinor(), errorQuda, quda::ColorSpinorField::FieldOrder(), memset(), quda::ColorSpinorField::norm, quda::ColorSpinorField::Norm(), quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::NormBytes(), pool_device_free, pool_device_malloc, pool_pinned_free, pool_pinned_malloc, QUDA_CPU_FIELD_LOCATION, QUDA_CUDA_FIELD_LOCATION, QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER, qudaDeviceSynchronize, qudaMemcpy, quda::reorder_location(), quda::ColorSpinorField::v, quda::ColorSpinorField::V(), and zeroCopy.
Referenced by copySpinorField(), and operator=().
|
virtual |
Reimplemented from quda::ColorSpinorField.
Definition at line 123 of file cuda_color_spinor_field.cpp.
References cpuColorSpinorField, and errorQuda.
cudaColorSpinorField & quda::cudaColorSpinorField::operator= | ( | const cudaColorSpinorField & | src | ) |
Definition at line 134 of file cuda_color_spinor_field.cpp.
References copySpinorField(), create(), destroy(), quda::LatticeField::destroyComms(), quda::ColorSpinorField::init, quda::ColorSpinorField::operator=(), and QUDA_COPY_FIELD_CREATE.
cudaColorSpinorField & quda::cudaColorSpinorField::operator= | ( | const cpuColorSpinorField & | src | ) |
Definition at line 148 of file cuda_color_spinor_field.cpp.
References create(), destroy(), quda::ColorSpinorField::init, loadSpinorField(), quda::ColorSpinorField::operator=(), and QUDA_COPY_FIELD_CREATE.
void quda::cudaColorSpinorField::pack | ( | int | nFace, |
int | parity, | ||
int | dagger, | ||
int | stream_idx, | ||
MemoryLocation | location[], | ||
MemoryLocation | location_label, | ||
bool | spin_project = true , |
||
double | a = 0 , |
||
double | b = 0 , |
||
double | c = 0 |
||
) |
Pack the field halos in preparation for halo exchange, e.g., for Dslash
[in] | nFace | Depth of faces |
[in] | parity | Field parity |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream index to be used for packing kernel |
[in] | location | Array of field locations where each halo will be sent (Host, Device or Remote) |
[in] | location_label | Consistent label used for labeling the packing tunekey since location can be difference for each process |
[in] | spin_project | Whether we are spin projecting when face packing |
[in] | a | Used for twisted mass (scale factor) |
[in] | b | Used for twisted mass (chiral twist factor) |
[in] | c | Used for twisted mass (flavor twist factor) |
Definition at line 937 of file cuda_color_spinor_field.cpp.
References createComms(), packGhost(), and QUDA_BOTH_DIRS.
Referenced by quda::dslash::issuePack().
void quda::cudaColorSpinorField::packExtended | ( | const int | nFace, |
const int | R[], | ||
const int | parity, | ||
const int | dagger, | ||
const int | dim, | ||
cudaStream_t * | stream_p, | ||
const bool | zeroCopyPack = false |
||
) |
Definition at line 949 of file cuda_color_spinor_field.cpp.
References createComms(), quda::Nstream, packGhostExtended(), and QUDA_BOTH_DIRS.
Referenced by quda::exchangeExtendedGhost().
void quda::cudaColorSpinorField::packGhost | ( | const int | nFace, |
const QudaParity | parity, | ||
const int | dim, | ||
const QudaDirection | dir, | ||
const int | dagger, | ||
cudaStream_t * | stream, | ||
MemoryLocation | location[2 *QUDA_MAX_DIM], | ||
MemoryLocation | location_label, | ||
bool | spin_project, | ||
double | a = 0 , |
||
double | b = 0 , |
||
double | c = 0 |
||
) |
Packs the cudaColorSpinorField's ghost zone.
[in] | nFace | How many faces to pack (depth) |
[in] | parity | Parity of the field |
[in] | dim | Labels space-time dimensions |
[in] | dir | Pack data to send in forward of backward directions, or both |
[in] | dagger | Whether the operator is the Hermitian conjugate or not |
[in] | stream | Which stream to use for the kernel |
[out] | buffer | Optional parameter where the ghost should be stored (default is to use cudaColorSpinorField::ghostFaceBuffer) |
[in] | location | Are we packing directly into local device memory, zero-copy memory or remote memory |
[in] | location_label | Consistent label used for labeling the packing tunekey since location can be difference for each process |
[in] | spin_project | Whether we are spin projecting when face packing |
[in] | a | Twisted mass parameter (scale factor, default=0) |
[in] | b | Twisted mass parameter (flavor twist factor, default=0) |
[in] | c | Twisted mass parameter (chiral twist factor, default=0) |
Definition at line 763 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, quda::Device, errorQuda, quda::LatticeField::ghost_remote_send_buffer_d, quda::LatticeField::ghostOffset, quda::Host, quda::LatticeField::my_face_dim_dir_d, quda::LatticeField::my_face_dim_dir_hd, quda::PackGhost(), quda::LatticeField::precision, QUDA_MAX_DIM, and quda::Remote.
Referenced by pack().
void quda::cudaColorSpinorField::packGhostExtended | ( | const int | nFace, |
const int | R[], | ||
const QudaParity | parity, | ||
const int | dim, | ||
const QudaDirection | dir, | ||
const int | dagger, | ||
cudaStream_t * | stream, | ||
bool | zero_copy = false |
||
) |
Definition at line 880 of file cuda_color_spinor_field.cpp.
References errorQuda.
Referenced by packExtended().
|
virtual |
Implements quda::ColorSpinorField.
Definition at line 1545 of file cuda_color_spinor_field.cpp.
References quda::genericCudaPrintVector().
void quda::cudaColorSpinorField::recvStart | ( | int | nFace, |
int | dir, | ||
int | dagger = 0 , |
||
cudaStream_t * | stream_p = nullptr , |
||
bool | gdr = false |
||
) |
Initiate halo communication receive.
[in] | Depth | of face exchange |
[in] | d | d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards) |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream to be used (unused) |
[in] | gdr | Whether we are using GDR on the receive side |
Definition at line 981 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_gdr_enabled(), comm_peer2peer_enabled(), comm_start(), commDimPartitioned(), errorQuda, quda::LatticeField::mh_recv_back, quda::LatticeField::mh_recv_fwd, quda::LatticeField::mh_recv_p2p_back, quda::LatticeField::mh_recv_p2p_fwd, quda::LatticeField::mh_recv_rdma_back, and quda::LatticeField::mh_recv_rdma_fwd.
Referenced by commsStart(), exchangeGhost(), and quda::dslash::issueRecv().
|
virtual |
Restores the cudaColorSpinorField.
Reimplemented from quda::LatticeField.
Definition at line 552 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::backed_up, quda::LatticeField::backup_h, quda::LatticeField::backup_norm_h, quda::ColorSpinorField::bytes, checkCudaError, errorQuda, quda::ColorSpinorField::norm_bytes, and quda::ColorSpinorField::v.
|
private |
Definition at line 695 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::bytes, quda::ColorSpinorField::Bytes(), checkCudaError, quda::copyGenericColorSpinor(), errorQuda, quda::ColorSpinorField::FieldOrder(), quda::ColorSpinorField::norm, quda::ColorSpinorField::Norm(), quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::NormBytes(), pool_device_free, pool_device_malloc, pool_pinned_free, pool_pinned_malloc, QUDA_CPU_FIELD_LOCATION, QUDA_CUDA_FIELD_LOCATION, QUDA_PADDED_SPACE_SPIN_COLOR_FIELD_ORDER, qudaDeviceSynchronize, qudaMemcpy, quda::reorder_location(), quda::ColorSpinorField::v, quda::ColorSpinorField::V(), and zeroCopy.
Referenced by quda::cpuColorSpinorField::operator=().
void quda::cudaColorSpinorField::scatter | ( | int | nFace, |
int | dagger, | ||
int | dir, | ||
cudaStream_t * | stream_p | ||
) |
Definition at line 1255 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhost().
Referenced by quda::dslash::commsComplete().
|
virtual |
Reimplemented from quda::LatticeField.
Definition at line 1268 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhost().
void quda::cudaColorSpinorField::scatterExtended | ( | int | nFace, |
int | parity, | ||
int | dagger, | ||
int | dir | ||
) |
Definition at line 1281 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhostExtended().
Referenced by quda::exchangeExtendedGhost().
void quda::cudaColorSpinorField::sendGhost | ( | void * | ghost_spinor, |
const int | nFace, | ||
const int | dim, | ||
const QudaDirection | dir, | ||
const int | dagger, | ||
cudaStream_t * | stream | ||
) |
Initiate the gpu to cpu send of the ghost zone (halo)
ghost_spinor | Where to send the ghost zone |
nFace | Number of face to send |
dim | The lattice dimension we are sending |
dir | The direction (QUDA_BACKWARDS or QUDA_FORWARDS) |
dagger | Whether the operator is daggerer or not |
stream | The array of streams to use |
Definition at line 795 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, quda::ColorSpinorField::bytes, errorQuda, quda::getKernelPackT(), quda::LatticeField::ghost_face_bytes, quda::LatticeField::ghost_precision, quda::ColorSpinorField::ghostFaceCB, quda::LatticeField::my_face_dim_dir_d, quda::ColorSpinorField::nColor, quda::ColorSpinorField::nDim, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::nSpin, quda::ColorSpinorField::Nvec(), parity, quda::popKernelPackT(), quda::LatticeField::precision, quda::pushKernelPackT(), QUDA_BACKWARDS, QUDA_DOUBLE_PRECISION, QUDA_FORWARDS, QUDA_HALF_PRECISION, QUDA_QUARTER_PRECISION, qudaMemcpy2DAsync, qudaMemcpyAsync, quda::s, quda::ColorSpinorField::siteSubset, quda::ColorSpinorField::stride, quda::ColorSpinorField::v, quda::ColorSpinorField::volumeCB, and quda::ColorSpinorField::x.
Referenced by gather().
void quda::cudaColorSpinorField::sendStart | ( | int | nFace, |
int | d, | ||
int | dagger = 0 , |
||
cudaStream_t * | stream_p = nullptr , |
||
bool | gdr = false , |
||
bool | remote_write = false |
||
) |
Initiate halo communication sending.
[in] | nFace | Depth of face exchange |
[in] | d | d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards) |
[in] | dagger | Whether this exchange is for the conjugate operator |
[in] | stream | CUDA stream that we will post the p2p event synchronization to (if nullptr then stream+d will be used |
[in] | gdr | Whether we are using GDR on the send side |
[in] | remote_write | Whether we are writing direct to remote memory (or using copy engines) |
Definition at line 1013 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, quda::ColorSpinorField::bytes, comm_gdr_enabled(), comm_peer2peer_enabled(), comm_start(), commDimPartitioned(), errorQuda, quda::getKernelPackT(), quda::LatticeField::ghost_face_bytes, quda::LatticeField::ghost_precision, quda::LatticeField::ghost_remote_send_buffer_d, quda::ColorSpinorField::ghostFaceCB, quda::LatticeField::ghostOffset, quda::LatticeField::ipcCopyEvent, quda::LatticeField::mh_send_back, quda::LatticeField::mh_send_fwd, quda::LatticeField::mh_send_p2p_back, quda::LatticeField::mh_send_p2p_fwd, quda::LatticeField::mh_send_rdma_back, quda::LatticeField::mh_send_rdma_fwd, quda::LatticeField::my_face_dim_dir_d, quda::ColorSpinorField::nColor, quda::ColorSpinorField::nDim, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::nSpin, quda::ColorSpinorField::Nvec(), parity, quda::popKernelPackT(), quda::LatticeField::precision, quda::pushKernelPackT(), QUDA_DOUBLE_PRECISION, QUDA_HALF_PRECISION, QUDA_QUARTER_PRECISION, quda::qudaEventRecord(), quda::s, quda::ColorSpinorField::siteSubset, quda::ColorSpinorField::stride, quda::ColorSpinorField::v, quda::ColorSpinorField::volumeCB, and quda::ColorSpinorField::x.
Referenced by commsStart(), exchangeGhost(), quda::dslash::DslashBasic< Dslash >::operator()(), quda::dslash::DslashFusedExterior< Dslash >::operator()(), quda::dslash::DslashGDR< Dslash >::operator()(), quda::dslash::DslashFusedGDR< Dslash >::operator()(), quda::dslash::DslashGDRRecv< Dslash >::operator()(), quda::dslash::DslashFusedGDRRecv< Dslash >::operator()(), quda::dslash::DslashZeroCopyPack< Dslash >::operator()(), quda::dslash::DslashFusedZeroCopyPack< Dslash >::operator()(), quda::dslash::DslashZeroCopyPackGDRRecv< Dslash >::operator()(), quda::dslash::DslashFusedZeroCopyPackGDRRecv< Dslash >::operator()(), quda::dslash::DslashZeroCopy< Dslash >::operator()(), and quda::dslash::DslashFusedZeroCopy< Dslash >::operator()().
|
virtual |
Implements quda::ColorSpinorField.
Definition at line 1532 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorParam::create, quda::ColorSpinorParam::fieldOrder, quda::ColorSpinorParam::location, param, quda::LatticeField::precision, QUDA_CPU_FIELD_LOCATION, QUDA_INVALID_PRECISION, QUDA_NULL_FIELD_CREATE, QUDA_POINT_SOURCE, QUDA_SINGLE_PRECISION, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER, QUDA_ZERO_FIELD_CREATE, quda::ColorSpinorParam::setPrecision(), quda::cpuColorSpinorField::Source(), and tmp.
void quda::cudaColorSpinorField::streamInit | ( | cudaStream_t * | stream_p | ) |
Definition at line 933 of file cuda_color_spinor_field.cpp.
Referenced by quda::dslash::DslashPolicyTune< Dslash >::DslashPolicyTune(), and exchangeGhost().
void quda::cudaColorSpinorField::switchBufferPinned | ( | ) |
void quda::cudaColorSpinorField::unpackGhost | ( | const void * | ghost_spinor, |
const int | nFace, | ||
const int | dim, | ||
const QudaDirection | dir, | ||
const int | dagger, | ||
cudaStream_t * | stream | ||
) |
Initiate the cpu to gpu send of the ghost zone (halo)
ghost_spinor | Source of the ghost zone |
nFace | Number of face to send |
dim | The lattice dimension we are sending |
dir | The direction (QUDA_BACKWARDS or QUDA_FORWARDS) |
dagger | Whether the operator is daggerer or not |
stream | The array of streams to use |
Definition at line 866 of file cuda_color_spinor_field.cpp.
References quda::LatticeField::bufferIndex, quda::LatticeField::ghost_face_bytes, quda::LatticeField::ghost_precision, quda::LatticeField::ghost_recv_buffer_d, quda::LatticeField::ghostOffset, QUDA_BACKWARDS, and qudaMemcpyAsync.
Referenced by scatter().
void quda::cudaColorSpinorField::unpackGhostExtended | ( | const void * | ghost_spinor, |
const int | nFace, | ||
const QudaParity | parity, | ||
const int | dim, | ||
const QudaDirection | dir, | ||
const int | dagger, | ||
cudaStream_t * | stream, | ||
bool | zero_copy | ||
) |
Initiate the cpu to gpu copy of the extended border region
ghost_spinor | Source of the ghost zone |
parity | Parity of the field |
nFace | Number of face to send |
dim | The lattice dimension we are sending |
dir | The direction (QUDA_BACKWARDS or QUDA_FORWARDS) |
dagger | Whether the operator is daggered or not |
stream | The array of streams to use |
zero_copy | Whether we are unpacking from zero_copy memory |
Definition at line 889 of file cuda_color_spinor_field.cpp.
References errorQuda.
Referenced by scatterExtended().
void quda::cudaColorSpinorField::zero | ( | ) |
Definition at line 567 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::bytes, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::LatticeField::precision, QUDA_HALF_PRECISION, QUDA_QUARTER_PRECISION, and quda::ColorSpinorField::v.
Referenced by cudaColorSpinorField().
|
private |
Zero the padded regions added on to the field. Ensures correct reductions and silences false positive warnings regarding uninitialized memory.
Definition at line 572 of file cuda_color_spinor_field.cpp.
References quda::ColorSpinorField::bytes, checkCudaError, quda::ColorSpinorField::composite_descr, quda::CompositeColorSpinorFieldDescriptor::dim, quda::ColorSpinorField::fieldOrder, quda::CompositeColorSpinorFieldDescriptor::is_component, quda::CompositeColorSpinorFieldDescriptor::is_composite, quda::ColorSpinorField::length, quda::ColorSpinorField::nColor, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::ColorSpinorField::nSpin, quda::LatticeField::precision, quda::ColorSpinorField::siteSubset, quda::CompositeColorSpinorFieldDescriptor::stride, quda::ColorSpinorField::stride, quda::ColorSpinorField::v, quda::CompositeColorSpinorFieldDescriptor::volumeCB, and quda::ColorSpinorField::volumeCB.
Referenced by create().
|
friend |
Definition at line 577 of file color_spinor_field.h.
Referenced by operator=().
|
friend |
Definition at line 1435 of file cuda_color_spinor_field.cpp.
|
private |
Definition at line 580 of file color_spinor_field.h.
Referenced by create(), destroy(), and quda::operator<<().
|
mutableprivate |
Definition at line 600 of file color_spinor_field.h.
Referenced by allocateGhostBuffer(), and create().
|
mutableprivate |
Definition at line 585 of file color_spinor_field.h.
Referenced by create().
|
mutableprivate |
Definition at line 584 of file color_spinor_field.h.
Referenced by allocateGhostBuffer(), and create().
|
private |
Definition at line 581 of file color_spinor_field.h.
Referenced by quda::operator<<().
|
private |
the precision allocated for the ghost texture
Definition at line 598 of file color_spinor_field.h.
|
private |
Definition at line 583 of file color_spinor_field.h.
Referenced by create().