bda/gpu-mpi-tests
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gt: don't use circular exchange for y deriv

Browse Source
This commit is contained in:
Bryce Allen
2023-03-25 12:58:11 -07:00
parent 7967991bd9
commit 84deab6ced
1 changed files with 29 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
mpi_stencil2d_gt.cc
View File

@@ -273,21 +273,20 @@ void boundary_exchange_y(MPI_Comm comm, int world_size, int rank,
MPI_Request req_l[2]; MPI_Request req_l[2];
MPI_Request req_r[2]; MPI_Request req_r[2];
int rank_l = (rank - 1) % world_size; int rank_l = rank - 1;
int rank_r = (rank + 1) % world_size; int rank_r = rank + 1;
auto sv_l = gt::view_strided(d_z, _all, _s(n_bnd, 2 * n_bnd)); auto sv_l = gt::view_strided(d_z, _all, _s(n_bnd, 2 * n_bnd));
auto sv_r = gt::view_strided(d_z, _all, _s(-2 * n_bnd, -n_bnd)); auto sv_r = gt::view_strided(d_z, _all, _s(-2 * n_bnd, -n_bnd));
auto rv_l = gt::view_strided(d_z, _all, _s(0, n_bnd)); auto rv_l = gt::view_strided(d_z, _all, _s(0, n_bnd));
auto rv_r = gt::view_strided(d_z, _all, _s(-n_bnd, _)); auto rv_r = gt::view_strided(d_z, _all, _s(-n_bnd, _));
// start async copy of ghost points into send buffers // start async copy of ghost points into send buffers
if (rank_l >= 0) {
if (stage_device) { if (stage_device) {
if (rank_l >= 0) {
sbuf_l = sv_l; sbuf_l = sv_l;
} }
}
if (rank_r <= world_size) { if (rank_r <= world_size) {
if (stage_device) {
sbuf_r = sv_r; sbuf_r = sv_r;
} }
} }
@@ -316,7 +315,9 @@ void boundary_exchange_y(MPI_Comm comm, int world_size, int rank,
} }
// wait for send buffer fill // wait for send buffer fill
if (stage_device) {
gt::synchronize(); gt::synchronize();
}
// initiate async sends // initiate async sends
if (rank_l >= 0) { if (rank_l >= 0) {
@@ -381,7 +382,7 @@ void print_test_name(bool use_buffers)
template <typename S, int Dim> template <typename S, int Dim>
void test(int device_id, uint32_t vendor_id, int world_size, int world_rank, void test(int device_id, uint32_t vendor_id, int world_size, int world_rank,
int n_global, int n_iter, bool use_buffers, int n_warmup=5) int n_global, int n_iter, bool use_buffers, int n_warmup = 5)
{ {
// Note: domain will be n_global x n_global plus ghost points in one dimension // Note: domain will be n_global x n_global plus ghost points in one dimension
@@ -436,7 +437,7 @@ void test(int device_id, uint32_t vendor_id, int world_size, int world_rank,
double iter_time = 0.0; double iter_time = 0.0;
double total_time = 0.0; double total_time = 0.0;
double x_start=0, y_start=0; double x_start = 0, y_start = 0;
if constexpr (Dim == 0) { if constexpr (Dim == 0) {
x_start = world_rank * ln_local; x_start = world_rank * ln_local;
} else { } else {
@@ -554,7 +555,6 @@ void test(int device_id, uint32_t vendor_id, int world_size, int world_rank,
printf("%d/%d [%d:0x%08x] err_norm = %.8f\n", world_rank, world_size, printf("%d/%d [%d:0x%08x] err_norm = %.8f\n", world_rank, world_size,
device_id, vendor_id, err_norm); device_id, vendor_id, err_norm);
} }
int main(int argc, char** argv) int main(int argc, char** argv)
@@ -603,21 +603,23 @@ int main(int argc, char** argv)
fflush(stdout); fflush(stdout);
/* // test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank,
test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); // n_global, n_iter, true, 5);
test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5); // test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank,
test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); // n_global, n_iter, false, 5);
test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5); // test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank,
// n_global, n_iter, true, 5);
// test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank,
// n_global, n_iter, false, 5);
test<gt::space::device, 1>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); test<gt::space::device, 1>(device_id, vendor_id, world_size, world_rank,
test<gt::space::device, 1>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5); n_global, n_iter, true, 5);
test<gt::space::managed, 1>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); test<gt::space::device, 1>(device_id, vendor_id, world_size, world_rank,
test<gt::space::managed, 1>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5); n_global, n_iter, false, 5);
*/ // test<gt::space::managed, 1>(device_id, vendor_id, world_size, world_rank,
test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); // n_global, n_iter, true, 5);
test<gt::space::managed, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5); // test<gt::space::managed, 1>(device_id, vendor_id, world_size, world_rank,
test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, true, 5); // n_global, n_iter, false, 5);
test<gt::space::device, 0>(device_id, vendor_id, world_size, world_rank, n_global, n_iter, false, 5);
MPI_Finalize(); MPI_Finalize();