comm_quda.h

Go to the documentation of this file.

#pragma once
#include <cstdint>

#ifdef __cplusplus
extern "C" {
#endif

  typedef struct MsgHandle_s MsgHandle;
  typedef struct Topology_s Topology;

  /* defined in quda.h; redefining here to avoid circular references */ 
  typedef int (*QudaCommsMap)(const int *coords, void *fdata);

  /* implemented in comm_common.cpp */

  char *comm_hostname(void);
  double comm_drand(void);
  Topology *comm_create_topology(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data);
  void comm_destroy_topology(Topology *topo);
  int comm_ndim(const Topology *topo);
  const int *comm_dims(const Topology *topo);
  const int *comm_coords(const Topology *topo);
  const int *comm_coords_from_rank(const Topology *topo, int rank);
  int comm_rank_from_coords(const Topology *topo, const int *coords);
  int comm_rank_displaced(const Topology *topo, const int displacement[]);
  void comm_set_default_topology(Topology *topo);
  Topology *comm_default_topology(void);

  // routines related to direct peer-2-peer access
  void comm_set_neighbor_ranks(Topology *topo=NULL);
  int comm_neighbor_rank(int dir, int dim);

  int comm_dim(int dim);

  int comm_coord(int dim);

  MsgHandle *comm_declare_send_relative_(const char *func, const char *file, int line,
           void *buffer, int dim, int dir, size_t nbytes);

#define comm_declare_send_relative(buffer, dim, dir, nbytes)    \
  comm_declare_send_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, nbytes)

  MsgHandle *comm_declare_receive_relative_(const char *func, const char *file, int line,
              void *buffer, int dim, int dir, size_t nbytes);

#define comm_declare_receive_relative(buffer, dim, dir, nbytes)   \
  comm_declare_receive_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, nbytes)

  MsgHandle *comm_declare_strided_send_relative_(const char *func, const char *file, int line,
             void *buffer, int dim, int dir,
             size_t blksize, int nblocks, size_t stride);

#define comm_declare_strided_send_relative(buffer, dim, dir, blksize, nblocks, stride) \
  comm_declare_strided_send_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, blksize, nblocks, stride)

  MsgHandle *comm_declare_strided_receive_relative_(const char *func, const char *file, int line,
                void *buffer, int dim, int dir,
                size_t blksize, int nblocks, size_t stride);

#define comm_declare_strided_receive_relative(buffer, dim, dir, blksize, nblocks, stride) \
  comm_declare_strided_receive_relative_(__func__, __FILE__, __LINE__, buffer, dim, dir, blksize, nblocks, stride)

  void comm_finalize(void);
  void comm_dim_partitioned_set(int dim);
  int comm_dim_partitioned(int dim);

  int comm_partitioned();

  const char* comm_dim_partitioned_string();

  const char* comm_dim_topology_string();

  /* implemented in comm_single.cpp, comm_qmp.cpp, and comm_mpi.cpp */

  void comm_init(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data);
  int comm_rank(void);
  int comm_size(void);
  int comm_gpuid(void);

  void comm_gather_hostname(char *hostname_recv_buf);

  void comm_gather_gpuid(int *gpuid_recv_buf);

  void comm_peer2peer_init(const char *hostname_recv_buf);

  int comm_peer2peer_enabled_global();

  bool comm_peer2peer_enabled(int dir, int dim);

  void comm_enable_peer2peer(bool enable);

  bool comm_intranode_enabled(int dir, int dim);

  void comm_enable_intranode(bool enable);

  bool comm_gdr_enabled();

  bool comm_gdr_blacklist();

  MsgHandle *comm_declare_send_displaced(void *buffer, const int displacement[], size_t nbytes);

  MsgHandle *comm_declare_receive_displaced(void *buffer, const int displacement[], size_t nbytes);

  MsgHandle *comm_declare_strided_send_displaced(void *buffer, const int displacement[], 
             size_t blksize, int nblocks, size_t stride);

  MsgHandle *comm_declare_strided_receive_displaced(void *buffer, const int displacement[],
                size_t blksize, int nblocks, size_t stride);

  void comm_free(MsgHandle *mh);
  void comm_start(MsgHandle *mh);
  void comm_wait(MsgHandle *mh);
  int comm_query(MsgHandle *mh);
  void comm_allreduce(double* data);
  void comm_allreduce_max(double* data);
  void comm_allreduce_array(double* data, size_t size);
  void comm_allreduce_int(int* data);
  void comm_allreduce_xor(uint64_t *data);
  void comm_broadcast(void *data, size_t nbytes);
  void comm_barrier(void);
  void comm_abort(int status);

  void reduceMaxDouble(double &);
  void reduceDouble(double &);
  void reduceDoubleArray(double *, const int len);
  int commDim(int);
  int commCoords(int);
  int commDimPartitioned(int dir);
  void commDimPartitionedSet(int dir);
  
  void commDimPartitionedReset();
  bool commGlobalReduction();
  void commGlobalReductionSet(bool global_reduce);

  bool commAsyncReduction();
  void commAsyncReductionSet(bool global_reduce);

#ifdef __cplusplus
}
#endif