#include <communicator_quda.h>

Public Member Functions
void	comm_peer2peer_init (const char *hostname_recv_buf)

bool	comm_peer2peer_present ()

bool	comm_peer2peer_enabled (int dir, int dim)

int	comm_peer2peer_enabled_global ()

void	comm_enable_peer2peer (bool enable)

bool	comm_intranode_enabled (int dir, int dim)

void	comm_enable_intranode (bool enable)

void	comm_set_default_topology (Topology *topo)

Topology *	comm_default_topology (void)

void	comm_set_neighbor_ranks (Topology *topo=nullptr)

int	comm_neighbor_rank (int dir, int dim)

int	comm_dim (int dim)

int	comm_coord (int dim)

void	comm_finalize (void)

void	comm_dim_partitioned_set (int dim)

void	comm_dim_partitioned_reset ()

int	comm_dim_partitioned (int)

int	comm_partitioned ()

bool	comm_gdr_enabled ()

bool	comm_gdr_blacklist ()

bool	comm_nvshmem_enabled ()

void	comm_init_common (int ndim, const int dims, QudaCommsMap rank_from_coords, void map_data)

const char *	comm_config_string ()

const char *	comm_dim_partitioned_string (const int *comm_dim_override)

const char *	comm_dim_topology_string ()

bool	comm_deterministic_reduce ()

void	reduceMaxDouble (double &max)

void	reduceDouble (double &sum)

void	reduceDoubleArray (double *sum, const int len)

int	commDim (int dir)

int	commCoords (int dir)

int	commDimPartitioned (int dir)

void	commDimPartitionedSet (int dir)

void	commDimPartitionedReset ()

bool	commGlobalReduction ()

void	commGlobalReductionSet (bool global_reduction)

bool	commAsyncReduction ()

void	commAsyncReductionSet (bool async_reduction)

	Communicator ()

	Communicator (Communicator &other, const int *comm_split)

	Communicator (int nDim, const int commDims, QudaCommsMap rank_from_coords, void map_data, bool user_set_comm_handle=false, void *user_comm=nullptr)

	~Communicator ()

void	comm_gather_hostname (char *hostname_recv_buf)

void	comm_gather_gpuid (int *gpuid_recv_buf)

void	comm_init (int ndim, const int dims, QudaCommsMap rank_from_coords, void map_data)

int	comm_rank (void)

int	comm_size (void)

int	comm_rank_from_coords (const int *coords)

MsgHandle *	comm_declare_send_rank (void *buffer, int rank, int tag, size_t nbytes)

MsgHandle *	comm_declare_recv_rank (void *buffer, int rank, int tag, size_t nbytes)

MsgHandle *	comm_declare_send_displaced (void *buffer, const int displacement[], size_t nbytes)

MsgHandle *	comm_declare_receive_displaced (void *buffer, const int displacement[], size_t nbytes)

MsgHandle *	comm_declare_strided_send_displaced (void *buffer, const int displacement[], size_t blksize, int nblocks, size_t stride)

MsgHandle *	comm_declare_strided_receive_displaced (void *buffer, const int displacement[], size_t blksize, int nblocks, size_t stride)

void	comm_free (MsgHandle *&mh)

void	comm_start (MsgHandle *mh)

void	comm_wait (MsgHandle *mh)

int	comm_query (MsgHandle *mh)

template<typename T >
T	deterministic_reduce (T *array, int n)

void	comm_allreduce (double *data)

void	comm_allreduce_max (double *data)

void	comm_allreduce_min (double *data)

void	comm_allreduce_array (double *data, size_t size)

void	comm_allreduce_max_array (double *data, size_t size)

void	comm_allreduce_int (int *data)

void	comm_allreduce_xor (uint64_t *data)

void	comm_broadcast (void *data, size_t nbytes)

void	comm_barrier (void)

Static Public Member Functions
static int	comm_gpuid ()

static void	comm_abort_ (int status)

static int	comm_rank_global ()

Public Attributes
bool	user_set_comm_handle

bool	peer2peer_enabled [2][4] = {{false, false, false, false}, {false, false, false, false}}

bool	peer2peer_init = false

bool	intranode_enabled [2][4] = {{false, false, false, false}, {false, false, false, false}}

bool	peer2peer_present = false

int	enable_peer_to_peer = 3

int	enable_p2p_max_access_rank = std::numeric_limits<int>::max()

bool	enable_p2p = true

bool	init = false

bool	p2p_global = false

bool	enable_intranode = true

Topology *	default_topo = nullptr

int	neighbor_rank [2][4] = {{-1, -1, -1, -1}, {-1, -1, -1, -1}}

bool	neighbors_cached = false

char	partition_string [16]

char	topology_string [128]

char	partition_override_string [16]

int	manual_set_partition [QUDA_MAX_DIM] = {0}

bool	gdr_enabled = false

bool	blacklist = false

bool	blacklist_init = false

bool	use_deterministic_reduce = false

char	config_string [64]

bool	config_init = false

bool	globalReduce = true

bool	asyncReduce = false

int	rank = -1

int	size = -1

Static Public Attributes
static int	gpuid = -1

Detailed Description

Definition at line 163 of file communicator_quda.h.

Constructor & Destructor Documentation

◆ Communicator() [1/3]

Communicator::Communicator ( )

inline

Definition at line 709 of file communicator_quda.h.

◆ Communicator() [2/3]

Communicator::Communicator	(	Communicator &	other,
		const int *	comm_split
	)

Definition at line 54 of file communicator_mpi.cpp.

◆ Communicator() [3/3]

Communicator::Communicator	(	int	nDim,
		const int *	commDims,
		QudaCommsMap	rank_from_coords,
		void *	map_data,
		bool	user_set_comm_handle = `false`,
		void *	user_comm = `nullptr`
	)

Dummy communications layer for single-GPU backend.

Definition at line 35 of file communicator_mpi.cpp.

◆ ~Communicator()

Communicator::~Communicator ( )

Definition at line 83 of file communicator_mpi.cpp.

Member Function Documentation

◆ comm_abort_()

void Communicator::comm_abort_ ( int status )

static

Definition at line 355 of file communicator_mpi.cpp.

◆ comm_allreduce()

void Communicator::comm_allreduce ( double * data )

Definition at line 271 of file communicator_mpi.cpp.

◆ comm_allreduce_array()

void Communicator::comm_allreduce_array	(	double *	data,
		size_t	size
	)

Definition at line 300 of file communicator_mpi.cpp.

◆ comm_allreduce_int()

void Communicator::comm_allreduce_int ( int * data )

Definition at line 332 of file communicator_mpi.cpp.

◆ comm_allreduce_max()

void Communicator::comm_allreduce_max ( double * data )

Definition at line 286 of file communicator_mpi.cpp.

◆ comm_allreduce_max_array()

void Communicator::comm_allreduce_max_array	(	double *	data,
		size_t	size
	)

Definition at line 324 of file communicator_mpi.cpp.

◆ comm_allreduce_min()

void Communicator::comm_allreduce_min ( double * data )

Definition at line 293 of file communicator_mpi.cpp.

◆ comm_allreduce_xor()

void Communicator::comm_allreduce_xor ( uint64_t * data )

Definition at line 339 of file communicator_mpi.cpp.

◆ comm_barrier()

void Communicator::comm_barrier ( void )

Definition at line 353 of file communicator_mpi.cpp.

◆ comm_broadcast()

void Communicator::comm_broadcast	(	void *	data,
		size_t	nbytes
	)

broadcast from rank 0

Definition at line 348 of file communicator_mpi.cpp.

◆ comm_config_string()

const char* Communicator::comm_config_string ( )

inline

Definition at line 621 of file communicator_quda.h.

◆ comm_coord()

int Communicator::comm_coord ( int dim )

inline

Definition at line 424 of file communicator_quda.h.

◆ comm_declare_receive_displaced()

MsgHandle * Communicator::comm_declare_receive_displaced	(	void *	buffer,
		const int	displacement[],
		size_t	nbytes
	)

Declare a message handle for receiving from a node displaced in (x,y,z,t) according to "displacement"

Definition at line 176 of file communicator_mpi.cpp.

◆ comm_declare_recv_rank()

MsgHandle * Communicator::comm_declare_recv_rank	(	void *	buffer,
		int	rank,
		int	tag,
		size_t	nbytes
	)

Declare a message handle for receiving nbytes from the rank with tag.

Definition at line 142 of file communicator_mpi.cpp.

◆ comm_declare_send_displaced()

MsgHandle * Communicator::comm_declare_send_displaced	(	void *	buffer,
		const int	displacement[],
		size_t	nbytes
	)

Declare a message handle for sending to a node displaced in (x,y,z,t) according to "displacement"

Definition at line 154 of file communicator_mpi.cpp.

◆ comm_declare_send_rank()

MsgHandle * Communicator::comm_declare_send_rank	(	void *	buffer,
		int	rank,
		int	tag,
		size_t	nbytes
	)

Declare a message handle for sending nbytes to the rank with tag.

Definition at line 130 of file communicator_mpi.cpp.

◆ comm_declare_strided_receive_displaced()

MsgHandle * Communicator::comm_declare_strided_receive_displaced	(	void *	buffer,
		const int	displacement[],
		size_t	blksize,
		int	nblocks,
		size_t	stride
	)

Declare a message handle for receiving from a node displaced in (x,y,z,t) according to "displacement"

Declare a message handle for strided receiving from a node displaced in (x,y,z,t) according to "displacement"

Definition at line 226 of file communicator_mpi.cpp.

◆ comm_declare_strided_send_displaced()

MsgHandle * Communicator::comm_declare_strided_send_displaced	(	void *	buffer,
		const int	displacement[],
		size_t	blksize,
		int	nblocks,
		size_t	stride
	)

Declare a message handle for sending to a node displaced in (x,y,z,t) according to "displacement"

Declare a message handle for strided sending to a node displaced in (x,y,z,t) according to "displacement"

Definition at line 198 of file communicator_mpi.cpp.

◆ comm_default_topology()

Topology* Communicator::comm_default_topology ( void )

inline

Definition at line 383 of file communicator_quda.h.

◆ comm_deterministic_reduce()

bool Communicator::comm_deterministic_reduce ( )

inline

Definition at line 657 of file communicator_quda.h.

◆ comm_dim()

int Communicator::comm_dim ( int dim )

inline

Definition at line 418 of file communicator_quda.h.

◆ comm_dim_partitioned()

int Communicator::comm_dim_partitioned ( int )

inline

Definition at line 464 of file communicator_quda.h.

◆ comm_dim_partitioned_reset()

void Communicator::comm_dim_partitioned_reset ( )

inline

Definition at line 453 of file communicator_quda.h.

◆ comm_dim_partitioned_set()

void Communicator::comm_dim_partitioned_set ( int dim )

inline

Definition at line 443 of file communicator_quda.h.

◆ comm_dim_partitioned_string()

const char* Communicator::comm_dim_partitioned_string ( const int * comm_dim_override )

inline

Definition at line 640 of file communicator_quda.h.

◆ comm_dim_topology_string()

const char* Communicator::comm_dim_topology_string ( )

inline

Definition at line 655 of file communicator_quda.h.

◆ comm_enable_intranode()

void Communicator::comm_enable_intranode ( bool enable )

inline

Definition at line 377 of file communicator_quda.h.

◆ comm_enable_peer2peer()

void Communicator::comm_enable_peer2peer ( bool enable )

inline

Definition at line 371 of file communicator_quda.h.

◆ comm_finalize()

void Communicator::comm_finalize ( void )

inline

Definition at line 430 of file communicator_quda.h.

◆ comm_free()

void Communicator::comm_free ( MsgHandle *& mh )

Definition at line 251 of file communicator_mpi.cpp.

◆ comm_gather_gpuid()

void Communicator::comm_gather_gpuid ( int * gpuid_recv_buf )

Definition at line 96 of file communicator_mpi.cpp.

◆ comm_gather_hostname()

void Communicator::comm_gather_hostname ( char * hostname_recv_buf )

Definition at line 89 of file communicator_mpi.cpp.

◆ comm_gdr_blacklist()

bool Communicator::comm_gdr_blacklist ( )

inline

Definition at line 496 of file communicator_quda.h.

◆ comm_gdr_enabled()

bool Communicator::comm_gdr_enabled ( )

inline

Definition at line 480 of file communicator_quda.h.

◆ comm_gpuid()

static int Communicator::comm_gpuid ( )

inlinestatic

Definition at line 169 of file communicator_quda.h.

◆ comm_init()

void Communicator::comm_init	(	int	ndim,
		const int *	dims,
		QudaCommsMap	rank_from_coords,
		void *	map_data
	)

Definition at line 102 of file communicator_mpi.cpp.

◆ comm_init_common()

void Communicator::comm_init_common	(	int	ndim,
		const int *	dims,
		QudaCommsMap	rank_from_coords,
		void *	map_data
	)

inline

Definition at line 546 of file communicator_quda.h.

◆ comm_intranode_enabled()

bool Communicator::comm_intranode_enabled	(	int	dir,
		int	dim
	)

inline

Definition at line 375 of file communicator_quda.h.

◆ comm_neighbor_rank()

int Communicator::comm_neighbor_rank	(	int	dir,
		int	dim
	)

inline

Definition at line 412 of file communicator_quda.h.

◆ comm_nvshmem_enabled()

bool Communicator::comm_nvshmem_enabled ( )

inline

Definition at line 527 of file communicator_quda.h.

◆ comm_partitioned()

int Communicator::comm_partitioned ( )

inline

Definition at line 467 of file communicator_quda.h.

◆ comm_peer2peer_enabled()

bool Communicator::comm_peer2peer_enabled	(	int	dir,
		int	dim
	)

inline

Definition at line 349 of file communicator_quda.h.

◆ comm_peer2peer_enabled_global()

int Communicator::comm_peer2peer_enabled_global ( )

inline

Definition at line 354 of file communicator_quda.h.

◆ comm_peer2peer_init()

void Communicator::comm_peer2peer_init ( const char * hostname_recv_buf )

inline

Definition at line 191 of file communicator_quda.h.

◆ comm_peer2peer_present()

bool Communicator::comm_peer2peer_present ( )

inline

Definition at line 345 of file communicator_quda.h.

◆ comm_query()

int Communicator::comm_query ( MsgHandle * mh )

Definition at line 263 of file communicator_mpi.cpp.

◆ comm_rank()

int Communicator::comm_rank ( void )

Definition at line 123 of file communicator_mpi.cpp.

◆ comm_rank_from_coords()

int Communicator::comm_rank_from_coords ( const int * coords )

inline

Definition at line 728 of file communicator_quda.h.

◆ comm_rank_global()

int Communicator::comm_rank_global ( )

static

Definition at line 357 of file communicator_mpi.cpp.

◆ comm_set_default_topology()

void Communicator::comm_set_default_topology ( Topology * topo )

inline

Definition at line 381 of file communicator_quda.h.

◆ comm_set_neighbor_ranks()

void Communicator::comm_set_neighbor_ranks ( Topology * topo = nullptr )

inline

Definition at line 393 of file communicator_quda.h.

◆ comm_size()

int Communicator::comm_size ( void )

Definition at line 125 of file communicator_mpi.cpp.

◆ comm_start()

void Communicator::comm_start ( MsgHandle * mh )

Definition at line 259 of file communicator_mpi.cpp.

◆ comm_wait()

void Communicator::comm_wait ( MsgHandle * mh )

Definition at line 261 of file communicator_mpi.cpp.

◆ commAsyncReduction()

bool Communicator::commAsyncReduction ( )

inline

Definition at line 688 of file communicator_quda.h.

◆ commAsyncReductionSet()

void Communicator::commAsyncReductionSet ( bool async_reduction )

inline

Definition at line 690 of file communicator_quda.h.

◆ commCoords()

int Communicator::commCoords ( int dir )

inline

Definition at line 676 of file communicator_quda.h.

◆ commDim()

int Communicator::commDim ( int dir )

inline

Definition at line 674 of file communicator_quda.h.

◆ commDimPartitioned()

int Communicator::commDimPartitioned ( int dir )

inline

Definition at line 678 of file communicator_quda.h.

◆ commDimPartitionedReset()

void Communicator::commDimPartitionedReset ( )

inline

Definition at line 682 of file communicator_quda.h.

◆ commDimPartitionedSet()

void Communicator::commDimPartitionedSet ( int dir )

inline

Definition at line 680 of file communicator_quda.h.

◆ commGlobalReduction()

bool Communicator::commGlobalReduction ( )

inline

Definition at line 684 of file communicator_quda.h.

◆ commGlobalReductionSet()

void Communicator::commGlobalReductionSet ( bool global_reduction )

inline

Definition at line 686 of file communicator_quda.h.

◆ deterministic_reduce()

template<typename T >

T Communicator::deterministic_reduce	(	T *	array,
		int	n
	)

inline

Definition at line 774 of file communicator_quda.h.

◆ reduceDouble()

void Communicator::reduceDouble ( double & sum )

inline

Definition at line 664 of file communicator_quda.h.

◆ reduceDoubleArray()

void Communicator::reduceDoubleArray	(	double *	sum,
		const int	len
	)

inline

Definition at line 669 of file communicator_quda.h.

◆ reduceMaxDouble()

void Communicator::reduceMaxDouble ( double & max )

inline

Definition at line 662 of file communicator_quda.h.

Member Data Documentation

◆ asyncReduce

bool Communicator::asyncReduce = false

Definition at line 660 of file communicator_quda.h.

◆ blacklist

bool Communicator::blacklist = false

Definition at line 493 of file communicator_quda.h.

◆ blacklist_init

bool Communicator::blacklist_init = false

Definition at line 494 of file communicator_quda.h.

◆ config_init

bool Communicator::config_init = false

Definition at line 619 of file communicator_quda.h.

◆ config_string

char Communicator::config_string[64]

Definition at line 618 of file communicator_quda.h.

◆ default_topo

Topology* Communicator::default_topo = nullptr

Definition at line 379 of file communicator_quda.h.

◆ enable_intranode

bool Communicator::enable_intranode = true

Definition at line 373 of file communicator_quda.h.

◆ enable_p2p

bool Communicator::enable_p2p = true

Definition at line 347 of file communicator_quda.h.

◆ enable_p2p_max_access_rank

int Communicator::enable_p2p_max_access_rank = std::numeric_limits<int>::max()

sets whether we cap which peers can use peer-to-peer

Definition at line 189 of file communicator_quda.h.

◆ enable_peer_to_peer

int Communicator::enable_peer_to_peer = 3

by default enable both copy engines and load/store access

Definition at line 186 of file communicator_quda.h.

◆ gdr_enabled

bool Communicator::gdr_enabled = false

Definition at line 474 of file communicator_quda.h.

◆ globalReduce

bool Communicator::globalReduce = true

Definition at line 659 of file communicator_quda.h.

◆ gpuid

int Communicator::gpuid = -1

static

The gpuid is static, and it's set when the default communicator is initialized.

Definition at line 168 of file communicator_quda.h.

◆ init

bool Communicator::init = false

Definition at line 351 of file communicator_quda.h.

◆ intranode_enabled

bool Communicator::intranode_enabled[2][4] = {{false, false, false, false}, {false, false, false, false}}

Definition at line 179 of file communicator_quda.h.

◆ manual_set_partition

int Communicator::manual_set_partition[QUDA_MAX_DIM] = {0}

string that contains any overridden partitioning

Definition at line 441 of file communicator_quda.h.

◆ neighbor_rank

int Communicator::neighbor_rank[2][4] = {{-1, -1, -1, -1}, {-1, -1, -1, -1}}

Definition at line 389 of file communicator_quda.h.

◆ neighbors_cached

bool Communicator::neighbors_cached = false

Definition at line 391 of file communicator_quda.h.

◆ p2p_global

bool Communicator::p2p_global = false

Definition at line 352 of file communicator_quda.h.

◆ partition_override_string

char Communicator::partition_override_string[16]

string that contains the job topology

Definition at line 439 of file communicator_quda.h.

◆ partition_string

char Communicator::partition_string[16]

Definition at line 437 of file communicator_quda.h.

◆ peer2peer_enabled

bool Communicator::peer2peer_enabled[2][4] = {{false, false, false, false}, {false, false, false, false}}

Definition at line 176 of file communicator_quda.h.

◆ peer2peer_init

bool Communicator::peer2peer_init = false

Definition at line 177 of file communicator_quda.h.

◆ peer2peer_present

bool Communicator::peer2peer_present = false

this records whether there is any peer-2-peer capability (regardless whether it is enabled or not)

Definition at line 183 of file communicator_quda.h.

◆ rank

int Communicator::rank = -1

Definition at line 706 of file communicator_quda.h.

◆ size

int Communicator::size = -1

Definition at line 707 of file communicator_quda.h.

◆ topology_string

char Communicator::topology_string[128]

string that contains the job partitioning

Definition at line 438 of file communicator_quda.h.

◆ use_deterministic_reduce

bool Communicator::use_deterministic_reduce = false

Definition at line 544 of file communicator_quda.h.

◆ user_set_comm_handle

bool Communicator::user_set_comm_handle

Whether or not the MPI_COMM_HANDLE is created by user, in which case we should not free it.

Definition at line 174 of file communicator_quda.h.

The documentation for this struct was generated from the following files:

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ Communicator() [1/3]

◆ Communicator() [2/3]

◆ Communicator() [3/3]

◆ ~Communicator()

Member Function Documentation

◆ comm_abort_()

◆ comm_allreduce()

◆ comm_allreduce_array()

◆ comm_allreduce_int()

◆ comm_allreduce_max()

◆ comm_allreduce_max_array()

◆ comm_allreduce_min()

◆ comm_allreduce_xor()

◆ comm_barrier()

◆ comm_broadcast()

◆ comm_config_string()

◆ comm_coord()

◆ comm_declare_receive_displaced()

◆ comm_declare_recv_rank()

◆ comm_declare_send_displaced()

◆ comm_declare_send_rank()

◆ comm_declare_strided_receive_displaced()

◆ comm_declare_strided_send_displaced()

◆ comm_default_topology()

◆ comm_deterministic_reduce()

◆ comm_dim()

◆ comm_dim_partitioned()

◆ comm_dim_partitioned_reset()

◆ comm_dim_partitioned_set()

◆ comm_dim_partitioned_string()

◆ comm_dim_topology_string()

◆ comm_enable_intranode()

◆ comm_enable_peer2peer()

◆ comm_finalize()

◆ comm_free()

◆ comm_gather_gpuid()

◆ comm_gather_hostname()

◆ comm_gdr_blacklist()

◆ comm_gdr_enabled()

◆ comm_gpuid()

◆ comm_init()

◆ comm_init_common()

◆ comm_intranode_enabled()

◆ comm_neighbor_rank()

◆ comm_nvshmem_enabled()

◆ comm_partitioned()

◆ comm_peer2peer_enabled()

◆ comm_peer2peer_enabled_global()

◆ comm_peer2peer_init()

◆ comm_peer2peer_present()

◆ comm_query()

◆ comm_rank()

◆ comm_rank_from_coords()

◆ comm_rank_global()

◆ comm_set_default_topology()

◆ comm_set_neighbor_ranks()

◆ comm_size()

◆ comm_start()

◆ comm_wait()

◆ commAsyncReduction()

◆ commAsyncReductionSet()

◆ commCoords()

◆ commDim()

◆ commDimPartitioned()

◆ commDimPartitionedReset()

◆ commDimPartitionedSet()

◆ commGlobalReduction()

◆ commGlobalReductionSet()

◆ deterministic_reduce()

◆ reduceDouble()

◆ reduceDoubleArray()

◆ reduceMaxDouble()

Member Data Documentation

◆ asyncReduce