add env var debugging

Makefile

@@ -1,11 +1,14 @@
 .PHONY: all
-all: daxpy mpi_daxpy
+all: daxpy mpi_daxpy mpienv
 
 daxpy: daxpy.cu cuda_error.h
 	nvcc -lcublas -o daxpy daxpy.cu
 
 mpi_daxpy: mpi_daxpy.cc cuda_error.h
-	mpic++ -lcudart -lcublas -I$(CUDA_HOME)/include -o mpi_daxpy mpi_daxpy.cc
+	mpic++ -lcudart -lcublas -I$(CUDA_HOME)/include -L$(CUDA_HOME)/lib64 -o mpi_daxpy mpi_daxpy.cc
+
+mpienv: mpienv.f90
+	mpif90 -o mpienv mpienv.f90
 
 .PHONY: clean
 clean:

mpi_daxpy.cc

@@ -64,7 +64,6 @@ void set_rank_device(int n_ranks, int rank) {
 
 int main(int argc, char **argv) {
     int n = 1024;
-
     int world_size, world_rank;
 
     double a = 2.0;
@@ -73,6 +72,8 @@ int main(int argc, char **argv) {
     double *x, *y, *d_x, *d_y;
     double *m_x, *m_y;
 
+    char *mb_per_core;
+
     MPI_Init(NULL, NULL);
 
     MPI_Comm_size(MPI_COMM_WORLD, &world_size);
@@ -91,10 +92,21 @@ int main(int argc, char **argv) {
     }
 
     for (int i=0; i<n; i++) {
-        x[i] = i+1;
+        x[i] =  i+1;
         y[i] = -i-1;
     }
 
+    // DEBUG weirdness on summit where GENE can't see MEMORY_PER_CORE,
+    // possibly because the system spectrum mpi uses it in some way.
+    if (world_rank == 0) {
+        mb_per_core = getenv("MEMORY_PER_CORE");
+        if (mb_per_core == NULL) {
+            printf("MEMORY_PER_CORE is not set\n");
+        } else {
+            printf("MEMORY_PER_CORE=%s\n", mb_per_core);
+        }
+    }
+
     set_rank_device(world_size, world_rank);
     //CHECK("setDevice", cudaSetDevice(0));
 

mpienv.f90

@@ -0,0 +1,35 @@
+program mpienv
+use mpi
+use iso_c_binding
+implicit none
+
+integer :: rank, ierr, nmpi
+
+integer :: strlen, err, memory_per_core
+character(len=5) :: read_env
+
+call MPI_Init(ierr)
+if (ierr /= 0) then
+    print *, 'Failed MPI_Init: ', ierr
+    stop
+end if
+
+call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr)
+if (ierr /= 0) then
+    print *, 'Failed MPI_COMM_RANK: ', ierr
+    stop
+end if
+
+call MPI_COMM_SIZE(MPI_COMM_WORLD, nmpi, ierr)
+if (ierr /= 0) then
+    print *, 'Failed MPI_COMM_SIZE: ', ierr
+    stop
+end if
+
+call get_environment_variable('MEMORY_PER_CORE',read_env,strlen,err)
+read(read_env, '(i6)') memory_per_core
+
+print *, 'rank ', rank, ' MEMORY_PER_CORE=', memory_per_core
+
+
+end program mpienv