mpi_stencil_gt.cc

@@ -8,7 +8,6 @@
 #include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <time.h>
 
 #include "gtensor/gtensor.h"
 #include "gtensor/reductions.h"
@@ -92,6 +91,7 @@ void boundary_exchange(MPI_Comm comm, int world_size, int rank,
   int rank_r = rank + 1;
 
   if (rank_l >= 0) {
+    printf("%d left\n", rank);
     // send/recv left boundary
     MPI_Irecv(d_y_data, n_bnd, MPI_DOUBLE, rank_l, 123, comm, &req_l[0]);
     MPI_Isend(d_y_data + n_bnd, n_bnd, MPI_DOUBLE, rank_l, 456, comm,
@@ -99,6 +99,7 @@ void boundary_exchange(MPI_Comm comm, int world_size, int rank,
   }
 
   if (rank_r < world_size) {
+    printf("%d right\n", rank);
     // send/recv right boundary
     MPI_Irecv(d_y_data_end - n_bnd, n_bnd, MPI_DOUBLE, rank_r, 456, comm,
               &req_r[0]);
@@ -108,12 +109,14 @@ void boundary_exchange(MPI_Comm comm, int world_size, int rank,
 
   int mpi_rval;
   if (rank_l >= 0) {
+    printf("%d wait left\n", rank);
     mpi_rval = MPI_Waitall(2, req_l, MPI_STATUSES_IGNORE);
     if (mpi_rval != MPI_SUCCESS) {
       printf("send_l error: %d\n", mpi_rval);
     }
   }
   if (rank_r < world_size) {
+    printf("%d wait right\n", rank);
     mpi_rval = MPI_Waitall(2, req_r, MPI_STATUSES_IGNORE);
     if (mpi_rval != MPI_SUCCESS) {
       printf("send_r error: %d\n", mpi_rval);
@@ -165,9 +168,7 @@ int main(int argc, char **argv) {
   auto fn_x_cubed = [](double x) { return x * x * x; };
   auto fn_x_cubed_deriv = [](double x) { return 3 * x * x; };
 
-  struct timespec start, end;
+  printf("%d Init\n", world_rank);
-  double seconds = 0.0;
-
   double x_start = world_rank * lx_local;
   for (int i = 0; i < n_local; i++) {
     double xtmp = x_start + i * dx;
@@ -190,20 +191,20 @@ int main(int argc, char **argv) {
   }
 
   gt::copy(h_y, d_y);
-  // gt::synchronize();
+  gt::synchronize();
+
+  printf("%d Ex\n", world_rank);
 
-  clock_gettime(CLOCK_MONOTONIC, &start);
   boundary_exchange(MPI_COMM_WORLD, world_size, world_rank, d_y, n_bnd);
-  // gt::synchronize();
+  gt::synchronize();
-  clock_gettime(CLOCK_MONOTONIC, &end);
-  seconds = ((end.tv_sec - start.tv_sec) + (end.tv_nsec - start.tv_nsec) * 1.0e-9);
-  printf("%d/%d exchange time %0.4f\n", world_rank, world_size, seconds);
 
+  printf("%d Sten\n", world_rank);
   d_dydx_numeric = stencil1d_5(d_y, stencil5) * scale;
-  // gt::synchronize();
+  gt::synchronize();
 
+  printf("%d Copy\n", world_rank);
   gt::copy(d_dydx_numeric, h_dydx_numeric);
-  // gt::synchronize();
+  gt::synchronize();
 
   /*
   for (int i = 0; i < 5; i++) {
@@ -217,6 +218,7 @@ int main(int argc, char **argv) {
   }
   */
 
+  printf("%d Err calc\n", world_rank);
   double err_norm = std::sqrt(gt::sum_squares(h_dydx_numeric - h_dydx_actual));
 
   printf("%d/%d [%d:0x%08x] err_norm = %.8f\n", world_rank, world_size,