comm_quda.h

Go to the documentation of this file.

#pragma once
#include <cstdint>
 
#ifdef __cplusplus
extern "C" {
#endif
 
  typedef struct MsgHandle_s MsgHandle;
  typedef struct Topology_s Topology;
 
  /* defined in quda.h; redefining here to avoid circular references */
  typedef int (*QudaCommsMap)(const int *coords, void *fdata);
 
  char *comm_hostname(void);
  double comm_drand(void);
  Topology *comm_create_topology(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data);
  void comm_destroy_topology(Topology *topo);
  int comm_ndim(const Topology *topo);
  const int *comm_dims(const Topology *topo);
  const int *comm_coords(const Topology *topo);
  const int *comm_coords_from_rank(const Topology *topo, int rank);
  int comm_rank_from_coords(const Topology *topo, const int *coords);
  int comm_rank_displaced(const Topology *topo, const int displacement[]);
  void comm_set_default_topology(Topology *topo);
  Topology *comm_default_topology(void);
 
  // routines related to direct peer-2-peer access
  void comm_set_neighbor_ranks(Topology *topo=NULL);
  int comm_neighbor_rank(int dir, int dim);
 
  int comm_dim(int dim);
 
  int comm_coord(int dim);
 
  MsgHandle *comm_declare_send_rank(void *buffer, int rank, int tag, size_t nbytes);
 
  MsgHandle *comm_declare_recv_rank(void *buffer, int rank, int tag, size_t nbytes);
 
  MsgHandle *comm_declare_send_relative_(const char *func, const char *file, int line,
                                         void *buffer, int dim, int dir, size_t nbytes);
 
#define comm_declare_send_relative(buffer, dim, dir, nbytes)            \
  comm_declare_send_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, nbytes)
 
  MsgHandle *comm_declare_receive_relative_(const char *func, const char *file, int line,
                                            void *buffer, int dim, int dir, size_t nbytes);
 
#define comm_declare_receive_relative(buffer, dim, dir, nbytes)         \
  comm_declare_receive_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, nbytes)
 
  MsgHandle *comm_declare_strided_send_relative_(const char *func, const char *file, int line,
                                                 void *buffer, int dim, int dir,
                                                 size_t blksize, int nblocks, size_t stride);
 
#define comm_declare_strided_send_relative(buffer, dim, dir, blksize, nblocks, stride) \
  comm_declare_strided_send_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, blksize, nblocks, stride)
 
  MsgHandle *comm_declare_strided_receive_relative_(const char *func, const char *file, int line,
                                                    void *buffer, int dim, int dir,
                                                    size_t blksize, int nblocks, size_t stride);
 
#define comm_declare_strided_receive_relative(buffer, dim, dir, blksize, nblocks, stride) \
  comm_declare_strided_receive_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, blksize, nblocks, stride)
 
  void comm_finalize(void);
  void comm_dim_partitioned_set(int dim);
  int comm_dim_partitioned(int dim);
 
  int comm_partitioned();
 
  void comm_set_tunekey_string();
 
  const char *comm_dim_partitioned_string(const int *comm_dim_override = 0);
 
  const char* comm_dim_topology_string();
 
  const char *comm_config_string();
 
  void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data,
                 bool user_set_comm_handle = false, void *user_comm = nullptr);
 
  void comm_init_common(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data);
 
  int comm_rank(void);
 
  int comm_rank_global(void);
 
  int comm_size(void);
 
  int comm_gpuid(void);
 
  bool comm_deterministic_reduce();
 
  void comm_gather_hostname(char *hostname_recv_buf);
 
  void comm_gather_gpuid(int *gpuid_recv_buf);
 
  void comm_peer2peer_init(const char *hostname_recv_buf);
 
  bool comm_peer2peer_present();
 
  int comm_peer2peer_enabled_global();
 
  bool comm_peer2peer_enabled(int dir, int dim);
 
  void comm_enable_peer2peer(bool enable);
 
  bool comm_intranode_enabled(int dir, int dim);
 
  void comm_enable_intranode(bool enable);
 
  bool comm_gdr_enabled();
 
  bool comm_nvshmem_enabled();
 
  bool comm_gdr_blacklist();
 
  MsgHandle *comm_declare_send_displaced(void *buffer, const int displacement[], size_t nbytes);
 
  MsgHandle *comm_declare_receive_displaced(void *buffer, const int displacement[], size_t nbytes);
 
  MsgHandle *comm_declare_strided_send_displaced(
      void *buffer, const int displacement[], size_t blksize, int nblocks, size_t stride);
 
  MsgHandle *comm_declare_strided_receive_displaced(void *buffer, const int displacement[],
                                                    size_t blksize, int nblocks, size_t stride);
 
  void comm_free(MsgHandle *&mh);
  void comm_start(MsgHandle *mh);
  void comm_wait(MsgHandle *mh);
  int comm_query(MsgHandle *mh);
  void comm_allreduce(double* data);
  void comm_allreduce_max(double* data);
  void comm_allreduce_min(double* data);
  void comm_allreduce_array(double* data, size_t size);
  void comm_allreduce_max_array(double* data, size_t size);
  void comm_allreduce_int(int* data);
  void comm_allreduce_xor(uint64_t *data);
  void comm_broadcast(void *data, size_t nbytes);
  void comm_barrier(void);
  void comm_abort(int status);
  void comm_abort_(int status);
 
  void reduceMaxDouble(double &);
  void reduceDouble(double &);
  void reduceDoubleArray(double *, const int len);
  int commDim(int);
  int commCoords(int);
  int commDimPartitioned(int dir);
  void commDimPartitionedSet(int dir);
 
  void commDimPartitionedReset();
  bool commGlobalReduction();
  void commGlobalReductionSet(bool global_reduce);
 
  bool commAsyncReduction();
  void commAsyncReductionSet(bool global_reduce);
 
#ifdef __cplusplus
}
#endif