comm_common.cpp

Go to the documentation of this file.

#include <unistd.h> // for gethostname()
#include <assert.h>
#include <limits>
 
#include <quda_internal.h>
#include <communicator_quda.h>
#include <comm_quda.h>
 
 
char *comm_hostname(void)
{
  static bool cached = false;
  static char hostname[128];
 
  if (!cached) {
    gethostname(hostname, 128);
    hostname[127] = '\0';
    cached = true;
  }
 
  return hostname;
}
 
static unsigned long int rand_seed = 137;
 
double comm_drand(void)
{
  const double twoneg48 = 0.35527136788005009e-14;
  const unsigned long int m = 25214903917, a = 11, mask = 281474976710655;
  rand_seed = (m * rand_seed + a) & mask;
  return (twoneg48 * rand_seed);
}
 
MsgHandle *comm_declare_send_relative_(const char *func, const char *file, int line, void *buffer, int dim, int dir,
                                       size_t nbytes)
{
#ifdef HOST_DEBUG
  checkCudaError(); // check and clear error state first
 
  if (isHost(buffer)) {
    // test this memory allocation is ok by doing a memcpy from it
    void *tmp = safe_malloc(nbytes);
    try {
      std::copy(static_cast<char *>(buffer), static_cast<char *>(buffer) + nbytes, static_cast<char *>(tmp));
    } catch (std::exception &e) {
      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, nbytes=%zu)\n", file, line, func, dim, dir,
                 nbytes);
      errorQuda("aborting");
    }
    host_free(tmp);
  } else {
    // test this memory allocation is ok by doing a memcpy from it
    void *tmp = device_malloc(nbytes);
    qudaMemcpy(tmp, buffer, nbytes, cudaMemcpyDeviceToDevice);
    device_free(tmp);
  }
#endif
 
  int disp[QUDA_MAX_DIM] = {0};
  disp[dim] = dir;
 
  return comm_declare_send_displaced(buffer, disp, nbytes);
}
 
MsgHandle *comm_declare_receive_relative_(const char *func, const char *file, int line, void *buffer, int dim, int dir,
                                          size_t nbytes)
{
#ifdef HOST_DEBUG
  checkCudaError(); // check and clear error state first
 
  if (isHost(buffer)) {
    // test this memory allocation is ok by filling it
    try {
      std::fill(static_cast<char *>(buffer), static_cast<char *>(buffer) + nbytes, 0);
    } catch (std::exception &e) {
      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, nbytes=%zu)\n", file, line, func, dim, dir,
                 nbytes);
      errorQuda("aborting");
    }
  } else {
    // test this memory allocation is ok by doing a memset
    qudaMemset(buffer, 0, nbytes);
  }
#endif
 
  int disp[QUDA_MAX_DIM] = {0};
  disp[dim] = dir;
 
  return comm_declare_receive_displaced(buffer, disp, nbytes);
}
 
MsgHandle *comm_declare_strided_send_relative_(const char *func, const char *file, int line, void *buffer, int dim,
                                               int dir, size_t blksize, int nblocks, size_t stride)
{
#ifdef HOST_DEBUG
  checkCudaError(); // check and clear error state first
 
  if (isHost(buffer)) {
    // test this memory allocation is ok by doing a memcpy from it
    void *tmp = safe_malloc(blksize * nblocks);
    try {
      for (int i = 0; i < nblocks; i++)
        std::copy(static_cast<char *>(buffer) + i * stride, static_cast<char *>(buffer) + i * stride + blksize,
                  static_cast<char *>(tmp));
    } catch (std::exception &e) {
      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, blksize=%zu nblocks=%d stride=%zu)\n", file,
                 line, func, dim, dir, blksize, nblocks, stride);
      errorQuda("aborting");
    }
    host_free(tmp);
  } else {
    // test this memory allocation is ok by doing a memcpy from it
 
    void *tmp = device_malloc(blksize*nblocks);
    qudaMemcpy2D(tmp, blksize, buffer, stride, blksize, nblocks, cudaMemcpyDeviceToDevice);
    device_free(tmp);
  }
#endif
 
  int disp[QUDA_MAX_DIM] = {0};
  disp[dim] = dir;
 
  return comm_declare_strided_send_displaced(buffer, disp, blksize, nblocks, stride);
}
 
MsgHandle *comm_declare_strided_receive_relative_(const char *func, const char *file, int line, void *buffer, int dim,
                                                  int dir, size_t blksize, int nblocks, size_t stride)
{
#ifdef HOST_DEBUG
  checkCudaError(); // check and clear error state first
 
  if (isHost(buffer)) {
    // test this memory allocation is ok by filling it
    try {
      for (int i = 0; i < nblocks; i++)
        std::fill(static_cast<char *>(buffer) + i * stride, static_cast<char *>(buffer) + i * stride + blksize, 0);
    } catch (std::exception &e) {
      printfQuda("ERROR: buffer failed (%s:%d in %s(), dim=%d, dir=%d, blksize=%zu nblocks=%d stride=%zu)\n", file,
                 line, func, dim, dir, blksize, nblocks, stride);
      errorQuda("aborting");
    }
  } else {
    // test this memory allocation is ok by doing a memset
    qudaMemset2D(buffer, stride, 0, blksize, nblocks);
  }
#endif
 
  int disp[QUDA_MAX_DIM] = {0};
  disp[dim] = dir;
 
  return comm_declare_strided_receive_displaced(buffer, disp, blksize, nblocks, stride);
}
 
Topology *comm_create_topology(int ndim, const int *dims, QudaCommsMap rank_from_coords, void *map_data, int my_rank)
{
  if (ndim > QUDA_MAX_DIM) { errorQuda("ndim exceeds QUDA_MAX_DIM"); }
 
  Topology *topo = new Topology;
 
  topo->ndim = ndim;
 
  int nodes = 1;
  for (int i = 0; i < ndim; i++) {
    topo->dims[i] = dims[i];
    nodes *= dims[i];
  }
 
  topo->ranks = new int[nodes];
  topo->coords = (int(*)[QUDA_MAX_DIM])new int[QUDA_MAX_DIM * nodes];
 
  int x[QUDA_MAX_DIM];
  for (int i = 0; i < QUDA_MAX_DIM; i++) x[i] = 0;
 
  do {
    int rank = rank_from_coords(x, map_data);
    topo->ranks[index(ndim, dims, x)] = rank;
    for (int i = 0; i < ndim; i++) { topo->coords[rank][i] = x[i]; }
  } while (advance_coords(ndim, dims, x));
 
  topo->my_rank = my_rank;
  for (int i = 0; i < ndim; i++) { topo->my_coords[i] = topo->coords[my_rank][i]; }
 
  // initialize the random number generator with a rank-dependent seed and initialized it only once.
  if (comm_gpuid() < 0) { rand_seed = 17 * my_rank + 137; }
 
  return topo;
}
 
void comm_abort(int status)
{
#ifdef HOST_DEBUG
  raise(SIGABRT);
#endif
#ifdef QUDA_BACKWARDSCPP
  backward::StackTrace st;
  st.load_here(32);
  backward::Printer p;
  p.print(st, getOutputFile());
#endif
  comm_abort_(status);
}