#include <color_spinor_field.h>

Inheritance diagram for quda::cudaColorSpinorField:

Collaboration diagram for quda::cudaColorSpinorField:

[legend]

Public Member Functions
	cudaColorSpinorField (const cudaColorSpinorField &)

	cudaColorSpinorField (const ColorSpinorField &, const ColorSpinorParam &)

	cudaColorSpinorField (const ColorSpinorField &)

	cudaColorSpinorField (const ColorSpinorParam &)

virtual	~cudaColorSpinorField ()

ColorSpinorField &	operator= (const ColorSpinorField &)

cudaColorSpinorField &	operator= (const cudaColorSpinorField &)

cudaColorSpinorField &	operator= (const cpuColorSpinorField &)

void	switchBufferPinned ()

void	createComms (int nFace, bool spin_project=true)
	Create the communication handlers and buffers. More...

void	allocateGhostBuffer (int nFace, bool spin_project=true) const
	Allocate the ghost buffers. More...

void	packGhost (const int nFace, const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t stream, MemoryLocation location[2 QUDA_MAX_DIM], MemoryLocation location_label, bool spin_project, double a=0, double b=0, double c=0)
	Packs the cudaColorSpinorField's ghost zone. More...

void	packGhostExtended (const int nFace, const int R[], const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t *stream, bool zero_copy=false)

void	sendGhost (void ghost_spinor, const int nFace, const int dim, const QudaDirection dir, const int dagger, cudaStream_t stream)

void	unpackGhost (const void ghost_spinor, const int nFace, const int dim, const QudaDirection dir, const int dagger, cudaStream_t stream)

void	unpackGhostExtended (const void ghost_spinor, const int nFace, const QudaParity parity, const int dim, const QudaDirection dir, const int dagger, cudaStream_t stream, bool zero_copy)

void	streamInit (cudaStream_t *stream_p)

void	pack (int nFace, int parity, int dagger, int stream_idx, MemoryLocation location[], MemoryLocation location_label, bool spin_project=true, double a=0, double b=0, double c=0)

void	packExtended (const int nFace, const int R[], const int parity, const int dagger, const int dim, cudaStream_t *stream_p, const bool zeroCopyPack=false)

void	gather (int nFace, int dagger, int dir, cudaStream_t *stream_p=NULL)

void	recvStart (int nFace, int dir, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr=false)
	Initiate halo communication receive. More...

void	sendStart (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr=false, bool remote_write=false)
	Initiate halo communication sending. More...

void	commsStart (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false)
	Initiate halo communication. More...

int	commsQuery (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false)
	Non-blocking query if the halo communication has completed. More...

void	commsWait (int nFace, int d, int dagger=0, cudaStream_t *stream_p=nullptr, bool gdr_send=false, bool gdr_recv=false)
	Wait on halo communication to complete. More...

void	scatter (int nFace, int dagger, int dir, cudaStream_t *stream_p)

void	scatter (int nFace, int dagger, int dir)

void	scatterExtended (int nFace, int parity, int dagger, int dir)

const void *	Ghost2 () const

void	exchangeGhost (QudaParity parity, int nFace, int dagger, const MemoryLocation pack_destination=nullptr, const MemoryLocation halo_location=nullptr, bool gdr_send=false, bool gdr_recv=false, QudaPrecision ghost_precision=QUDA_INVALID_PRECISION) const
	This is a unified ghost exchange function for doing a complete halo exchange regardless of the type of field. All dimensions are exchanged and no spin projection is done in the case of Wilson fermions. More...

cudaColorSpinorField &	Component (const int idx) const
	for composite fields: More...

CompositeColorSpinorField &	Components () const

void	CopySubset (cudaColorSpinorField &dst, const int range, const int first_element=0) const

void	zero ()

void	getTexObjectInfo () const

void	Source (const QudaSourceType sourceType, const int st=0, const int s=0, const int c=0)

void	PrintVector (unsigned int x) const

void	backup () const
	Backs up the cudaColorSpinorField. More...

void	restore () const
	Restores the cudaColorSpinorField. More...

Public Member Functions inherited from quda::ColorSpinorField
	ColorSpinorField (const ColorSpinorField &)

	ColorSpinorField (const ColorSpinorParam &)

virtual	~ColorSpinorField ()

int	Ncolor () const

int	Nspin () const

int	Nvec () const

QudaTwistFlavorType	TwistFlavor () const

int	Ndim () const

const int *	X () const

int	X (int d) const

size_t	RealLength () const

size_t	Length () const

int	Stride () const

int	Volume () const

int	VolumeCB () const

int	Pad () const

size_t	Bytes () const

size_t	NormBytes () const

size_t	GhostBytes () const

size_t	GhostNormBytes () const

void	PrintDims () const

void *	V ()

const void *	V () const

void *	Norm ()

const void *	Norm () const

void	exchange (void ghost, void sendbuf, int nFace=1) const

bool	isNative () const

bool	IsComposite () const

bool	IsComponent () const

int	CompositeDim () const

int	ComponentId () const

int	ComponentVolume () const

int	ComponentVolumeCB () const

int	ComponentStride () const

size_t	ComponentLength () const

size_t	ComponentRealLength () const

size_t	ComponentBytes () const

size_t	ComponentNormBytes () const

QudaPCType	PCType () const

QudaSiteSubset	SiteSubset () const

QudaSiteOrder	SiteOrder () const

QudaFieldOrder	FieldOrder () const

QudaGammaBasis	GammaBasis () const

const int *	GhostFace () const

const int *	GhostFaceCB () const

int	GhostOffset (const int i) const

int	GhostOffset (const int i, const int j) const

int	GhostNormOffset (const int i) const

int	GhostNormOffset (const int i, const int j) const

void *	Ghost (const int i)

const void *	Ghost (const int i) const

void *	GhostNorm (const int i)

const void *	GhostNorm (const int i) const

void const	Ghost () const

const DslashConstant &	getDslashConstant () const
	Get the dslash_constant structure from this field. More...

const ColorSpinorField &	Even () const

const ColorSpinorField &	Odd () const

ColorSpinorField &	Even ()

ColorSpinorField &	Odd ()

ColorSpinorField &	Component (const int idx) const

ColorSpinorField &	Component (const int idx)

CompositeColorSpinorField &	Components ()

void	LatticeIndex (int *y, int i) const

void	OffsetIndex (int &i, int *y) const

ColorSpinorField *	CreateCoarse (const int *geoBlockSize, int spinBlockSize, int Nvec, QudaPrecision precision=QUDA_INVALID_PRECISION, QudaFieldLocation location=QUDA_INVALID_FIELD_LOCATION, QudaMemoryType mem_Type=QUDA_MEMORY_INVALID)
	Create a coarse color-spinor field, using this field to set the meta data. More...

ColorSpinorField *	CreateFine (const int *geoblockSize, int spinBlockSize, int Nvec, QudaPrecision precision=QUDA_INVALID_PRECISION, QudaFieldLocation location=QUDA_INVALID_FIELD_LOCATION, QudaMemoryType mem_type=QUDA_MEMORY_INVALID)
	Create a fine color-spinor field, using this field to set the meta data. More...

Public Member Functions inherited from quda::LatticeField
	LatticeField (const LatticeFieldParam &param)

	LatticeField (const LatticeField &field)

virtual	~LatticeField ()

void	allocateGhostBuffer (size_t ghost_bytes) const
	Allocate the static ghost buffers. More...

void	createComms (bool no_comms_fill=false, bool bidir=true)

void	destroyComms ()

void	createIPCComms ()

bool	ipcCopyComplete (int dir, int dim)

bool	ipcRemoteCopyComplete (int dir, int dim)

const cudaEvent_t &	getIPCCopyEvent (int dir, int dim) const

const cudaEvent_t &	getIPCRemoteCopyEvent (int dir, int dim) const

int	Ndim () const

const int *	X () const

int	Volume () const

int	VolumeCB () const

const int *	SurfaceCB () const

int	SurfaceCB (const int i) const

int	Stride () const

int	Pad () const

const int *	R () const

QudaGhostExchange	GhostExchange () const

QudaPrecision	Precision () const

QudaPrecision	GhostPrecision () const

double	Scale () const

void	Scale (double scale_)
	Set the scale factor for a fixed-point field. More...

virtual QudaMemoryType	MemType () const

int	Nvec () const

QudaFieldLocation	Location () const

size_t	GBytes () const

void	checkField (const LatticeField &a) const

virtual void	read (char *filename)

virtual void	write (char *filename)

const char *	VolString () const

const char *	AuxString () const

Public Member Functions inherited from quda::Object
	Object ()

virtual	~Object ()

Private Member Functions
void	create (const QudaFieldCreate)

void	destroy ()

void	copy (const cudaColorSpinorField &)

void	zeroPad ()
	Zero the padded regions added on to the field. Ensures correct reductions and silences false positive warnings regarding uninitialized memory. More...

void	copySpinorField (const ColorSpinorField &src)

void	loadSpinorField (const ColorSpinorField &src)

void	saveSpinorField (ColorSpinorField &src) const

Private Attributes
bool	alloc

bool	init

bool	texInit

bool	ghostTexInit

QudaPrecision	ghost_precision_tex

bool	reference

void *	ghost_field_tex [4]

Friends
class	cpuColorSpinorField

std::ostream &	operator<< (std::ostream &out, const cudaColorSpinorField &)

Additional Inherited Members
Static Public Member Functions inherited from quda::ColorSpinorField
static ColorSpinorField *	Create (const ColorSpinorParam &param)

static ColorSpinorField *	Create (const ColorSpinorField &src, const ColorSpinorParam &param)

Static Public Member Functions inherited from quda::LatticeField
static void	freeGhostBuffer (void)
	Free statically allocated ghost buffers. More...

static void	destroyIPCComms ()

Static Public Member Functions inherited from quda::Object
static void *	operator new (std::size_t size)

static void	operator delete (void *p)

static void *	operator new[] (std::size_t size)

static void	operator delete[] (void *p)

Static Public Attributes inherited from quda::LatticeField
static int	bufferIndex = 0

static bool	ghost_field_reset = false

Protected Member Functions inherited from quda::ColorSpinorField
void	createGhostZone (int nFace, bool spin_project=true) const

void	reset (const ColorSpinorParam &)

void	fill (ColorSpinorParam &) const

void	setTuningString ()
	Set the vol_string and aux_string for use in tuning. More...

Protected Member Functions inherited from quda::LatticeField
void	precisionCheck ()

Static Protected Member Functions inherited from quda::ColorSpinorField
static void	checkField (const ColorSpinorField &, const ColorSpinorField &)

Protected Attributes inherited from quda::ColorSpinorField
bool	init

QudaPrecision	ghost_precision_allocated

int	nColor

int	nSpin

int	nVec

int	nDim

int	x [QUDA_MAX_DIM]

int	volume

int	volumeCB

int	pad

int	stride

QudaTwistFlavorType	twistFlavor

QudaPCType	pc_type

size_t	real_length

size_t	length

void *	v

void *	norm

void *	v_h

void *	norm_h

void *	ghost [2][QUDA_MAX_DIM]

void *	ghostNorm [2][QUDA_MAX_DIM]

int	ghostFace [QUDA_MAX_DIM]

int	ghostFaceCB [QUDA_MAX_DIM]

void *	ghost_buf [2 *QUDA_MAX_DIM]

DslashConstant	dslash_constant

size_t	bytes

size_t	norm_bytes

QudaSiteSubset	siteSubset

QudaSiteOrder	siteOrder

QudaFieldOrder	fieldOrder

QudaGammaBasis	gammaBasis

ColorSpinorField *	even

ColorSpinorField *	odd

CompositeColorSpinorFieldDescriptor	composite_descr
	used for deflation eigenvector sets etc.: More...

CompositeColorSpinorField	components

Protected Attributes inherited from quda::LatticeField
int	volume

int	volumeCB

int	stride

int	pad

size_t	total_bytes

int	nDim

int	x [QUDA_MAX_DIM]

int	surface [QUDA_MAX_DIM]

int	surfaceCB [QUDA_MAX_DIM]

int	r [QUDA_MAX_DIM]

QudaPrecision	precision

QudaPrecision	ghost_precision

bool	ghost_precision_reset

double	scale

QudaSiteSubset	siteSubset

QudaGhostExchange	ghostExchange

int	nDimComms

size_t	ghost_bytes

size_t	ghost_bytes_old

size_t	ghost_face_bytes [QUDA_MAX_DIM]

int	ghostOffset [QUDA_MAX_DIM][2]

int	ghostNormOffset [QUDA_MAX_DIM][2]

void *	my_face_h [2]

void *	my_face_hd [2]

void *	my_face_d [2]

void *	my_face_dim_dir_h [2][QUDA_MAX_DIM][2]

void *	my_face_dim_dir_hd [2][QUDA_MAX_DIM][2]

void *	my_face_dim_dir_d [2][QUDA_MAX_DIM][2]

void *	from_face_h [2]

void *	from_face_hd [2]

void *	from_face_d [2]

void *	from_face_dim_dir_h [2][QUDA_MAX_DIM][2]

void *	from_face_dim_dir_hd [2][QUDA_MAX_DIM][2]

void *	from_face_dim_dir_d [2][QUDA_MAX_DIM][2]

MsgHandle *	mh_recv_fwd [2][QUDA_MAX_DIM]

MsgHandle *	mh_recv_back [2][QUDA_MAX_DIM]

MsgHandle *	mh_send_fwd [2][QUDA_MAX_DIM]

MsgHandle *	mh_send_back [2][QUDA_MAX_DIM]

MsgHandle *	mh_recv_rdma_fwd [2][QUDA_MAX_DIM]

MsgHandle *	mh_recv_rdma_back [2][QUDA_MAX_DIM]

MsgHandle *	mh_send_rdma_fwd [2][QUDA_MAX_DIM]

MsgHandle *	mh_send_rdma_back [2][QUDA_MAX_DIM]

bool	initComms

char	vol_string [TuneKey::volume_n]

char	aux_string [TuneKey::aux_n]

QudaMemoryType	mem_type

char *	backup_h

char *	backup_norm_h

bool	backed_up

Static Protected Attributes inherited from quda::LatticeField
static void *	ghost_send_buffer_d [2] = {nullptr, nullptr}

static void *	ghost_recv_buffer_d [2] = {nullptr, nullptr}

static void *	ghost_pinned_send_buffer_h [2] = {nullptr, nullptr}

static void *	ghost_pinned_recv_buffer_h [2] = {nullptr, nullptr}

static void *	ghost_pinned_send_buffer_hd [2] = {nullptr, nullptr}

static void *	ghost_pinned_recv_buffer_hd [2] = {nullptr, nullptr}

static void *	ghost_remote_send_buffer_d [2][QUDA_MAX_DIM][2]

static size_t	ghostFaceBytes = 0

static bool	initGhostFaceBuffer = false

static MsgHandle *	mh_send_p2p_fwd [2][QUDA_MAX_DIM] { }

static MsgHandle *	mh_send_p2p_back [2][QUDA_MAX_DIM] { }

static MsgHandle *	mh_recv_p2p_fwd [2][QUDA_MAX_DIM] { }

static MsgHandle *	mh_recv_p2p_back [2][QUDA_MAX_DIM] { }

static int	buffer_send_p2p_fwd [2][QUDA_MAX_DIM] { }

static int	buffer_recv_p2p_fwd [2][QUDA_MAX_DIM] { }

static int	buffer_send_p2p_back [2][QUDA_MAX_DIM] { }

static int	buffer_recv_p2p_back [2][QUDA_MAX_DIM] { }

static cudaEvent_t	ipcCopyEvent [2][2][QUDA_MAX_DIM]

static cudaEvent_t	ipcRemoteCopyEvent [2][2][QUDA_MAX_DIM]

static bool	initIPCComms = false

Detailed Description

Definition at line 575 of file color_spinor_field.h.

Constructor & Destructor Documentation

◆ cudaColorSpinorField() [1/4]

quda::cudaColorSpinorField::cudaColorSpinorField ( const cudaColorSpinorField & src )

Definition at line 43 of file cuda_color_spinor_field.cpp.

References copySpinorField(), create(), and QUDA_COPY_FIELD_CREATE.

Referenced by CopySubset(), and create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ cudaColorSpinorField() [2/4]

quda::cudaColorSpinorField::cudaColorSpinorField	(	const ColorSpinorField &	src,
		const ColorSpinorParam &	param
	)

◆ cudaColorSpinorField() [3/4]

quda::cudaColorSpinorField::cudaColorSpinorField ( const ColorSpinorField & src )

Definition at line 110 of file cuda_color_spinor_field.cpp.

References copySpinorField(), create(), and QUDA_COPY_FIELD_CREATE.

Here is the call graph for this function:

◆ cudaColorSpinorField() [4/4]

quda::cudaColorSpinorField::cudaColorSpinorField ( const ColorSpinorParam & param )

Definition at line 15 of file cuda_color_spinor_field.cpp.

References create(), errorQuda, quda::ColorSpinorField::norm, param, QUDA_COPY_FIELD_CREATE, QUDA_NULL_FIELD_CREATE, QUDA_REFERENCE_FIELD_CREATE, QUDA_ZERO_FIELD_CREATE, quda::ColorSpinorField::v, and zero().

Here is the call graph for this function:

◆ ~cudaColorSpinorField()

quda::cudaColorSpinorField::~cudaColorSpinorField ( )

virtual

Definition at line 159 of file cuda_color_spinor_field.cpp.

References destroy(), and quda::LatticeField::destroyComms().

Here is the call graph for this function:

Member Function Documentation

◆ allocateGhostBuffer()

void quda::cudaColorSpinorField::allocateGhostBuffer	(	int	nFace,
		bool	spin_project = `true`
	)		const

Allocate the ghost buffers.

Parameters

[in]	nFace	Depth of each halo
[in]	spin_project	Whether the halos are spin projected (Wilson-type fermions only)

Definition at line 747 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::allocateGhostBuffer(), quda::ColorSpinorField::createGhostZone(), quda::LatticeField::ghost_bytes, ghost_field_tex, quda::LatticeField::ghost_pinned_recv_buffer_hd, quda::LatticeField::ghost_precision_reset, quda::LatticeField::ghost_recv_buffer_d, and ghostTexInit.

Referenced by quda::computeStaggeredOprod(), and createComms().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ backup()

void quda::cudaColorSpinorField::backup ( ) const

virtual

Backs up the cudaColorSpinorField.

Reimplemented from quda::LatticeField.

Definition at line 540 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::backed_up, quda::LatticeField::backup_h, quda::LatticeField::backup_norm_h, quda::ColorSpinorField::bytes, checkCudaError, errorQuda, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, and quda::ColorSpinorField::v.

◆ commsQuery()

int quda::cudaColorSpinorField::commsQuery	(	int	nFace,
		int	d,
		int	dagger = `0`,
		cudaStream_t *	stream_p = `nullptr`,
		bool	gdr_send = `false`,
		bool	gdr_recv = `false`
	)

virtual

Non-blocking query if the halo communication has completed.

Parameters

[in]	Depth	of face exchange
[in]	d	d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards)
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream to be used (unused)
[in]	gdr_send	Whether we are using GDR on the send side
[in]	gdr_recv	Whether we are using GDR on the receive side

Reimplemented from quda::LatticeField.

Definition at line 1127 of file cuda_color_spinor_field.cpp.

Referenced by quda::dslash::commsComplete(), quda::exchangeExtendedGhost(), and exchangeGhost().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ commsStart()

void quda::cudaColorSpinorField::commsStart	(	int	nFace,
		int	d,
		int	dagger = `0`,
		cudaStream_t *	stream_p = `nullptr`,
		bool	gdr_send = `false`,
		bool	gdr_recv = `false`
	)

virtual

Initiate halo communication.

Parameters

[in]	Depth	of face exchange
[in]	d	d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards)
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream to be used (unused)
[in]	gdr_send	Whether we are using GDR on the send side
[in]	gdr_recv	Whether we are using GDR on the receive side

Reimplemented from quda::LatticeField.

Definition at line 1116 of file cuda_color_spinor_field.cpp.

References recvStart(), and sendStart().

Referenced by quda::exchangeExtendedGhost().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ commsWait()

void quda::cudaColorSpinorField::commsWait	(	int	nFace,
		int	d,
		int	dagger = `0`,
		cudaStream_t *	stream_p = `nullptr`,
		bool	gdr_send = `false`,
		bool	gdr_recv = `false`
	)

virtual

Wait on halo communication to complete.

Parameters

[in]	Depth	of face exchange
[in]	d	d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards)
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream to be used (unused)
[in]	gdr_send	Whether we are using GDR on the send side
[in]	gdr_recv	Whether we are using GDR on the receive side

Reimplemented from quda::LatticeField.

Definition at line 1195 of file cuda_color_spinor_field.cpp.

Here is the call graph for this function:

◆ Component()

cudaColorSpinorField & quda::cudaColorSpinorField::Component ( const int idx ) const

for composite fields:

Definition at line 1445 of file cuda_color_spinor_field.cpp.

References quda::ColorSpinorField::components, quda::ColorSpinorField::CompositeDim(), errorQuda, and quda::ColorSpinorField::IsComposite().

Here is the call graph for this function:

◆ Components()

CompositeColorSpinorField& quda::cudaColorSpinorField::Components ( ) const

◆ copy()

void quda::cudaColorSpinorField::copy ( const cudaColorSpinorField & src )

private

Definition at line 621 of file cuda_color_spinor_field.cpp.

References quda::ColorSpinorField::checkField(), quda::blas::copy(), errorQuda, and quda::ColorSpinorField::GammaBasis().

Referenced by copySpinorField().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ copySpinorField()

void quda::cudaColorSpinorField::copySpinorField ( const ColorSpinorField & src )

private

This function is responsible for calling the correct copy kernel given the nature of the source field and the desired destination.

Definition at line 627 of file cuda_color_spinor_field.cpp.

References copy(), quda::copyGenericColorSpinor(), errorQuda, quda::ColorSpinorField::GammaBasis(), quda::ColorSpinorField::isNative(), loadSpinorField(), and QUDA_CUDA_FIELD_LOCATION.

Referenced by cudaColorSpinorField(), and operator=().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ CopySubset()

void quda::cudaColorSpinorField::CopySubset	(	cudaColorSpinorField &	dst,
		const int	range,
		const int	first_element = `0`
	)		const

Definition at line 1460 of file cuda_color_spinor_field.cpp.

Here is the call graph for this function:

◆ create()

void quda::cudaColorSpinorField::create ( const QudaFieldCreate create )

private

setup an object for selected eigenvector (the 1st one as a default):

Definition at line 164 of file cuda_color_spinor_field.cpp.

Referenced by cudaColorSpinorField(), and operator=().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ createComms()

void quda::cudaColorSpinorField::createComms	(	int	nFace,
		bool	spin_project = `true`
	)

Create the communication handlers and buffers.

Parameters

[in]	nFace	Depth of each halo
[in]	spin_project	Whether the halos are spin projected (Wilson-type fermions only)

Definition at line 899 of file cuda_color_spinor_field.cpp.

Referenced by exchangeGhost(), pack(), and packExtended().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ destroy()

void quda::cudaColorSpinorField::destroy ( )

private

◆ exchangeGhost()

void quda::cudaColorSpinorField::exchangeGhost	(	QudaParity	parity,
		int	nFace,
		int	dagger,
		const MemoryLocation *	pack_destination = `nullptr`,
		const MemoryLocation *	halo_location = `nullptr`,
		bool	gdr_send = `false`,
		bool	gdr_recv = `false`,
		QudaPrecision	ghost_precision = `QUDA_INVALID_PRECISION`
	)		const

virtual

This is a unified ghost exchange function for doing a complete halo exchange regardless of the type of field. All dimensions are exchanged and no spin projection is done in the case of Wilson fermions.

Parameters

[in]	parity	Field parity
[in]	nFace	Depth of halo exchange
[in]	dagger	Is this for a dagger operator (only relevant for spin projected Wilson)
[in]	pack_destination	Destination of the packing buffer
[in]	halo_location	Destination of the halo reading buffer
[in]	gdr_send	Are we using GDR for sending
[in]	gdr_recv	Are we using GDR for receiving
[in]	ghost_precision	The precision used for the ghost exchange

Implements quda::ColorSpinorField.

Definition at line 1290 of file cuda_color_spinor_field.cpp.

Here is the call graph for this function:

◆ gather()

void quda::cudaColorSpinorField::gather	(	int	nFace,
		int	dagger,
		int	dir,
		cudaStream_t *	stream_p = `NULL`
	)

virtual

Reimplemented from quda::LatticeField.

Definition at line 960 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), quda::LatticeField::my_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and sendGhost().

Referenced by quda::exchangeExtendedGhost(), and quda::dslash::issueGather().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getTexObjectInfo()

void quda::cudaColorSpinorField::getTexObjectInfo ( ) const

Definition at line 1518 of file cuda_color_spinor_field.cpp.

References printfQuda.

◆ Ghost2()

const void* quda::cudaColorSpinorField::Ghost2 ( ) const

inlinevirtual

Reimplemented from quda::ColorSpinorField.

Definition at line 806 of file color_spinor_field.h.

References dagger, quda::operator<<(), out, parity, QUDA_INVALID_PRECISION, quda::s, and quda::zero().

Here is the call graph for this function:

◆ loadSpinorField()

void quda::cudaColorSpinorField::loadSpinorField ( const ColorSpinorField & src )

private

Definition at line 643 of file cuda_color_spinor_field.cpp.

Referenced by copySpinorField(), and operator=().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ operator=() [1/3]

ColorSpinorField & quda::cudaColorSpinorField::operator= ( const ColorSpinorField & src )

virtual

Reimplemented from quda::ColorSpinorField.

Definition at line 123 of file cuda_color_spinor_field.cpp.

References cpuColorSpinorField, and errorQuda.

◆ operator=() [2/3]

cudaColorSpinorField & quda::cudaColorSpinorField::operator= ( const cudaColorSpinorField & src )

Definition at line 134 of file cuda_color_spinor_field.cpp.

References copySpinorField(), create(), destroy(), quda::LatticeField::destroyComms(), quda::ColorSpinorField::init, quda::ColorSpinorField::operator=(), and QUDA_COPY_FIELD_CREATE.

Here is the call graph for this function:

◆ operator=() [3/3]

cudaColorSpinorField & quda::cudaColorSpinorField::operator= ( const cpuColorSpinorField & src )

Definition at line 148 of file cuda_color_spinor_field.cpp.

References create(), destroy(), quda::ColorSpinorField::init, loadSpinorField(), quda::ColorSpinorField::operator=(), and QUDA_COPY_FIELD_CREATE.

Here is the call graph for this function:

◆ pack()

void quda::cudaColorSpinorField::pack	(	int	nFace,
		int	parity,
		int	dagger,
		int	stream_idx,
		MemoryLocation	location[],
		MemoryLocation	location_label,
		bool	spin_project = `true`,
		double	a = `0`,
		double	b = `0`,
		double	c = `0`
	)

Pack the field halos in preparation for halo exchange, e.g., for Dslash

Parameters

[in]	nFace	Depth of faces
[in]	parity	Field parity
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream index to be used for packing kernel
[in]	location	Array of field locations where each halo will be sent (Host, Device or Remote)
[in]	location_label	Consistent label used for labeling the packing tunekey since location can be difference for each process
[in]	spin_project	Whether we are spin projecting when face packing
[in]	a	Used for twisted mass (scale factor)
[in]	b	Used for twisted mass (chiral twist factor)
[in]	c	Used for twisted mass (flavor twist factor)

Definition at line 937 of file cuda_color_spinor_field.cpp.

References createComms(), packGhost(), and QUDA_BOTH_DIRS.

Referenced by quda::dslash::issuePack().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ packExtended()

void quda::cudaColorSpinorField::packExtended	(	const int	nFace,
		const int	R[],
		const int	parity,
		const int	dagger,
		const int	dim,
		cudaStream_t *	stream_p,
		const bool	zeroCopyPack = `false`
	)

Definition at line 949 of file cuda_color_spinor_field.cpp.

References createComms(), quda::Nstream, packGhostExtended(), and QUDA_BOTH_DIRS.

Referenced by quda::exchangeExtendedGhost().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ packGhost()

void quda::cudaColorSpinorField::packGhost	(	const int	nFace,
		const QudaParity	parity,
		const int	dim,
		const QudaDirection	dir,
		const int	dagger,
		cudaStream_t *	stream,
		MemoryLocation	location[2 *QUDA_MAX_DIM],
		MemoryLocation	location_label,
		bool	spin_project,
		double	a = `0`,
		double	b = `0`,
		double	c = `0`
	)

Packs the cudaColorSpinorField's ghost zone.

Parameters

[in]	nFace	How many faces to pack (depth)
[in]	parity	Parity of the field
[in]	dim	Labels space-time dimensions
[in]	dir	Pack data to send in forward of backward directions, or both
[in]	dagger	Whether the operator is the Hermitian conjugate or not
[in]	stream	Which stream to use for the kernel
[out]	buffer	Optional parameter where the ghost should be stored (default is to use cudaColorSpinorField::ghostFaceBuffer)
[in]	location	Are we packing directly into local device memory, zero-copy memory or remote memory
[in]	location_label	Consistent label used for labeling the packing tunekey since location can be difference for each process
[in]	spin_project	Whether we are spin projecting when face packing
[in]	a	Twisted mass parameter (scale factor, default=0)
[in]	b	Twisted mass parameter (flavor twist factor, default=0)
[in]	c	Twisted mass parameter (chiral twist factor, default=0)

Definition at line 763 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, quda::Device, errorQuda, quda::LatticeField::ghost_remote_send_buffer_d, quda::LatticeField::ghostOffset, quda::Host, quda::LatticeField::my_face_dim_dir_d, quda::LatticeField::my_face_dim_dir_hd, quda::PackGhost(), quda::LatticeField::precision, QUDA_MAX_DIM, and quda::Remote.

Referenced by pack().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ packGhostExtended()

void quda::cudaColorSpinorField::packGhostExtended	(	const int	nFace,
		const int	R[],
		const QudaParity	parity,
		const int	dim,
		const QudaDirection	dir,
		const int	dagger,
		cudaStream_t *	stream,
		bool	zero_copy = `false`
	)

Definition at line 880 of file cuda_color_spinor_field.cpp.

References errorQuda.

Referenced by packExtended().

Here is the caller graph for this function:

◆ PrintVector()

void quda::cudaColorSpinorField::PrintVector ( unsigned int x ) const

virtual

Implements quda::ColorSpinorField.

Definition at line 1545 of file cuda_color_spinor_field.cpp.

References quda::genericCudaPrintVector().

Here is the call graph for this function:

◆ recvStart()

void quda::cudaColorSpinorField::recvStart	(	int	nFace,
		int	dir,
		int	dagger = `0`,
		cudaStream_t *	stream_p = `nullptr`,
		bool	gdr = `false`
	)

Initiate halo communication receive.

Parameters

[in]	Depth	of face exchange
[in]	d	d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards)
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream to be used (unused)
[in]	gdr	Whether we are using GDR on the receive side

Definition at line 981 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, comm_gdr_enabled(), comm_peer2peer_enabled(), comm_start(), commDimPartitioned(), errorQuda, quda::LatticeField::mh_recv_back, quda::LatticeField::mh_recv_fwd, quda::LatticeField::mh_recv_p2p_back, quda::LatticeField::mh_recv_p2p_fwd, quda::LatticeField::mh_recv_rdma_back, and quda::LatticeField::mh_recv_rdma_fwd.

Referenced by commsStart(), exchangeGhost(), and quda::dslash::issueRecv().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ restore()

void quda::cudaColorSpinorField::restore ( ) const

virtual

Restores the cudaColorSpinorField.

Reimplemented from quda::LatticeField.

Definition at line 552 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::backed_up, quda::LatticeField::backup_h, quda::LatticeField::backup_norm_h, quda::ColorSpinorField::bytes, checkCudaError, errorQuda, quda::ColorSpinorField::norm_bytes, and quda::ColorSpinorField::v.

◆ saveSpinorField()

void quda::cudaColorSpinorField::saveSpinorField ( ColorSpinorField & src ) const

private

Definition at line 695 of file cuda_color_spinor_field.cpp.

Referenced by quda::cpuColorSpinorField::operator=().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ scatter() [1/2]

void quda::cudaColorSpinorField::scatter	(	int	nFace,
		int	dagger,
		int	dir,
		cudaStream_t *	stream_p
	)

Definition at line 1255 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhost().

Referenced by quda::dslash::commsComplete().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ scatter() [2/2]

void quda::cudaColorSpinorField::scatter	(	int	nFace,
		int	dagger,
		int	dir
	)

virtual

Reimplemented from quda::LatticeField.

Definition at line 1268 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, comm_peer2peer_enabled(), commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhost().

Here is the call graph for this function:

◆ scatterExtended()

void quda::cudaColorSpinorField::scatterExtended	(	int	nFace,
		int	parity,
		int	dagger,
		int	dir
	)

Definition at line 1281 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, commDimPartitioned(), quda::LatticeField::from_face_dim_dir_h, QUDA_BACKWARDS, QUDA_FORWARDS, and unpackGhostExtended().

Referenced by quda::exchangeExtendedGhost().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendGhost()

void quda::cudaColorSpinorField::sendGhost	(	void *	ghost_spinor,
		const int	nFace,
		const int	dim,
		const QudaDirection	dir,
		const int	dagger,
		cudaStream_t *	stream
	)

Initiate the gpu to cpu send of the ghost zone (halo)

Parameters

ghost_spinor	Where to send the ghost zone
nFace	Number of face to send
dim	The lattice dimension we are sending
dir	The direction (QUDA_BACKWARDS or QUDA_FORWARDS)
dagger	Whether the operator is daggerer or not
stream	The array of streams to use

Definition at line 795 of file cuda_color_spinor_field.cpp.

Referenced by gather().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendStart()

void quda::cudaColorSpinorField::sendStart	(	int	nFace,
		int	d,
		int	dagger = `0`,
		cudaStream_t *	stream_p = `nullptr`,
		bool	gdr = `false`,
		bool	remote_write = `false`
	)

Initiate halo communication sending.

Parameters

[in]	nFace	Depth of face exchange
[in]	d	d=[2*dim+dir], where dim is dimension and dir is the scatter-centric direction (0=backwards,1=forwards)
[in]	dagger	Whether this exchange is for the conjugate operator
[in]	stream	CUDA stream that we will post the p2p event synchronization to (if nullptr then stream+d will be used
[in]	gdr	Whether we are using GDR on the send side
[in]	remote_write	Whether we are writing direct to remote memory (or using copy engines)

Definition at line 1013 of file cuda_color_spinor_field.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ Source()

void quda::cudaColorSpinorField::Source	(	const QudaSourceType	sourceType,
		const int	st = `0`,
		const int	s = `0`,
		const int	c = `0`
	)

virtual

Implements quda::ColorSpinorField.

Definition at line 1532 of file cuda_color_spinor_field.cpp.

References quda::ColorSpinorParam::create, quda::ColorSpinorParam::fieldOrder, quda::ColorSpinorParam::location, param, quda::LatticeField::precision, QUDA_CPU_FIELD_LOCATION, QUDA_INVALID_PRECISION, QUDA_NULL_FIELD_CREATE, QUDA_POINT_SOURCE, QUDA_SINGLE_PRECISION, QUDA_SPACE_SPIN_COLOR_FIELD_ORDER, QUDA_ZERO_FIELD_CREATE, quda::ColorSpinorParam::setPrecision(), quda::cpuColorSpinorField::Source(), and tmp.

Here is the call graph for this function:

◆ streamInit()

void quda::cudaColorSpinorField::streamInit ( cudaStream_t * stream_p )

Definition at line 933 of file cuda_color_spinor_field.cpp.

Referenced by quda::dslash::DslashPolicyTune< Dslash >::DslashPolicyTune(), and exchangeGhost().

Here is the caller graph for this function:

◆ switchBufferPinned()

void quda::cudaColorSpinorField::switchBufferPinned ( )

◆ unpackGhost()

void quda::cudaColorSpinorField::unpackGhost	(	const void *	ghost_spinor,
		const int	nFace,
		const int	dim,
		const QudaDirection	dir,
		const int	dagger,
		cudaStream_t *	stream
	)

Initiate the cpu to gpu send of the ghost zone (halo)

Parameters

ghost_spinor	Source of the ghost zone
nFace	Number of face to send
dim	The lattice dimension we are sending
dir	The direction (QUDA_BACKWARDS or QUDA_FORWARDS)
dagger	Whether the operator is daggerer or not
stream	The array of streams to use

Definition at line 866 of file cuda_color_spinor_field.cpp.

References quda::LatticeField::bufferIndex, quda::LatticeField::ghost_face_bytes, quda::LatticeField::ghost_precision, quda::LatticeField::ghost_recv_buffer_d, quda::LatticeField::ghostOffset, QUDA_BACKWARDS, and qudaMemcpyAsync.

Referenced by scatter().

Here is the caller graph for this function:

◆ unpackGhostExtended()

void quda::cudaColorSpinorField::unpackGhostExtended	(	const void *	ghost_spinor,
		const int	nFace,
		const QudaParity	parity,
		const int	dim,
		const QudaDirection	dir,
		const int	dagger,
		cudaStream_t *	stream,
		bool	zero_copy
	)

Initiate the cpu to gpu copy of the extended border region

Parameters

ghost_spinor	Source of the ghost zone
parity	Parity of the field
nFace	Number of face to send
dim	The lattice dimension we are sending
dir	The direction (QUDA_BACKWARDS or QUDA_FORWARDS)
dagger	Whether the operator is daggered or not
stream	The array of streams to use
zero_copy	Whether we are unpacking from zero_copy memory

Definition at line 889 of file cuda_color_spinor_field.cpp.

References errorQuda.

Referenced by scatterExtended().

Here is the caller graph for this function:

◆ zero()

void quda::cudaColorSpinorField::zero ( )

Definition at line 567 of file cuda_color_spinor_field.cpp.

References quda::ColorSpinorField::bytes, quda::ColorSpinorField::norm, quda::ColorSpinorField::norm_bytes, quda::LatticeField::precision, QUDA_HALF_PRECISION, QUDA_QUARTER_PRECISION, and quda::ColorSpinorField::v.

Referenced by cudaColorSpinorField().

Here is the caller graph for this function:

◆ zeroPad()

void quda::cudaColorSpinorField::zeroPad ( )

private

Zero the padded regions added on to the field. Ensures correct reductions and silences false positive warnings regarding uninitialized memory.

Definition at line 572 of file cuda_color_spinor_field.cpp.

Referenced by create().

Here is the caller graph for this function:

Friends And Related Function Documentation

◆ cpuColorSpinorField

friend class cpuColorSpinorField

friend

Definition at line 577 of file color_spinor_field.h.

Referenced by operator=().

◆ operator<<

std::ostream& operator<<	(	std::ostream &	out,
		const cudaColorSpinorField &	a
	)

friend

Definition at line 1435 of file cuda_color_spinor_field.cpp.

Member Data Documentation

◆ alloc

bool quda::cudaColorSpinorField::alloc

private

Definition at line 580 of file color_spinor_field.h.

Referenced by create(), destroy(), and quda::operator<<().

◆ ghost_field_tex

void* quda::cudaColorSpinorField::ghost_field_tex[4]

mutableprivate

Definition at line 600 of file color_spinor_field.h.

Referenced by allocateGhostBuffer(), and create().

◆ ghost_precision_tex

QudaPrecision quda::cudaColorSpinorField::ghost_precision_tex

mutableprivate

Definition at line 585 of file color_spinor_field.h.

Referenced by create().

◆ ghostTexInit

bool quda::cudaColorSpinorField::ghostTexInit

mutableprivate

Definition at line 584 of file color_spinor_field.h.

Referenced by allocateGhostBuffer(), and create().

◆ init

bool quda::cudaColorSpinorField::init

private

Definition at line 581 of file color_spinor_field.h.

Referenced by quda::operator<<().

◆ reference

bool quda::cudaColorSpinorField::reference

private

the precision allocated for the ghost texture

Definition at line 598 of file color_spinor_field.h.

◆ texInit

bool quda::cudaColorSpinorField::texInit

private

Definition at line 583 of file color_spinor_field.h.

Referenced by create().

The documentation for this class was generated from the following files:

include/color_spinor_field.h
lib/cuda_color_spinor_field.cpp

Public Member Functions

Private Member Functions

Private Attributes

Friends

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ cudaColorSpinorField() [1/4]

◆ cudaColorSpinorField() [2/4]

◆ cudaColorSpinorField() [3/4]

◆ cudaColorSpinorField() [4/4]

◆ ~cudaColorSpinorField()

Member Function Documentation

◆ allocateGhostBuffer()

◆ backup()

◆ commsQuery()

◆ commsStart()

◆ commsWait()

◆ Component()

◆ Components()

◆ copy()

◆ copySpinorField()

◆ CopySubset()

◆ create()

◆ createComms()

◆ destroy()

◆ exchangeGhost()

◆ gather()

◆ getTexObjectInfo()

◆ Ghost2()

◆ loadSpinorField()

◆ operator=() [1/3]

◆ operator=() [2/3]

◆ operator=() [3/3]

◆ pack()

◆ packExtended()

◆ packGhost()

◆ packGhostExtended()

◆ PrintVector()

◆ recvStart()

◆ restore()

◆ saveSpinorField()

◆ scatter() [1/2]

◆ scatter() [2/2]

◆ scatterExtended()

◆ sendGhost()

◆ sendStart()

◆ Source()

◆ streamInit()

◆ switchBufferPinned()

◆ unpackGhost()

◆ unpackGhostExtended()

◆ zero()

◆ zeroPad()

Friends And Related Function Documentation

◆ cpuColorSpinorField

◆ operator<<

Member Data Documentation

◆ alloc

◆ ghost_field_tex

◆ ghost_precision_tex

◆ ghostTexInit

◆ init

◆ reference

◆ texInit