Skip to content

gpu-direct: make it optional and have fallback solution with device-host explicit transfers #389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
svalat wants to merge 17 commits into
base: develop
Choose a base branch
from
Draft

gpu-direct: make it optional and have fallback solution with device-host explicit transfers #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
fec3e60
gpu-direct: handle communications when GPU DIRECT is not there.
svalat Jul 8, 2026
7967a1b
gpu-direct: Implement extensions of MPI API using views and provde th…
svalat Jul 9, 2026
a15c690
gpu-direct: use the MPI view semantic in EMF, axis and exganger
svalat Jul 9, 2026
4742f74
gpu-direct: rename the API to use a namespace and fallback on the ori…
svalat Jul 9, 2026
2af6af6
gpu-direct: fix some extra parts of idefix using MPI
svalat Jul 9, 2026
f38f106
gpu-direct: wrap all the internal MPI_.+ using .data() internally wit…
svalat Jul 10, 2026
7b89da6
gpu-direct: Add a macro to activate to validate by forcing the transf…
svalat Jul 10, 2026
9b58b1c
gpu-direct: fix an issue in MPI_Gather
svalat Jul 10, 2026
75a4c33
gpu-direct: Fix some compilation warnings with CUDA
svalat Jul 10, 2026
433c9c1
gpu-direct: fix mistake in lazy-init, should be done on all entry points
svalat Jul 10, 2026
17e0c76
gpu-direct: Ooops, no cannot lazy init due to the copy semantic
svalat Jul 10, 2026
411866f
gpu-direct: try to avoid the warning in normal way to run, need to fi…
svalat Jul 10, 2026
4a5b620
gpu-direct: Cleanup the way to switch the class ON/OFF by using a sin…
svalat Jul 10, 2026
afa7a4e
gpu-direct: Opps wrong name in option declaration
svalat Jul 10, 2026
03e0f40
gpu-direct: remove a useless scafolding
svalat Jul 10, 2026
76d46ff
gpu-direct: try to implement the copy/move constructor & operator= to…
svalat Jul 10, 2026
aab2e8a
gpu-direct: solve most of the compiler warnings when compiling with CUDA
svalat Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OF
option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF)
option(Idefix_WERROR "Treat compiler warnings as errors" OFF)
option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF)
option(Idefix_MPI_GPU_DIRECT "Enable usage of GPU direct to avoid CPU-GPU copies for MPI messages." ON)
set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.")
set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag")
set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file")
Expand Down Expand Up @@ -104,6 +105,10 @@ if(Idefix_MPI)
add_subdirectory(src/mpi)
endif()

if (Idefix_MPI_GPU_DIRECT)
add_compile_definitions("WITH_MPI_GPU_DIRECT")
endif()

if(Idefix_HDF5)
add_compile_definitions("WITH_HDF5")
if(Idefix_MPI)
Expand Down Expand Up @@ -264,6 +269,7 @@ endif()
message(STATUS " MPI: ${Idefix_MPI}")
message(STATUS " HDF5: ${Idefix_HDF5}")
message(STATUS " Python: ${Idefix_PYTHON}")
message(STATUS " GPU direct: ${Idefix_MPI_GPU_DIRECT}")
message(STATUS " Reconstruction: ${Idefix_RECONSTRUCTION}")
message(STATUS " Precision: ${Idefix_PRECISION}")
message(STATUS " Version: ${Idefix_VERSION}")
Expand Down
32 changes: 16 additions & 16 deletions src/fluid/boundary/axis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ void Axis::ShowConfig() {
void Axis::SymmetrizeEx1Side(int jref, IdefixArray3D<real> Ex1) {
#if DIMENSIONS == 3

IdefixArray1D<real> Ex1Avg = this->Ex1Avg;
auto Ex1Avg = this->Ex1Avg.deviceView();

idefix_for("Ex1_ini",0,data->np_tot[IDIR],
KOKKOS_LAMBDA(int i) {
Expand All @@ -40,7 +40,7 @@ void Axis::SymmetrizeEx1Side(int jref, IdefixArray3D<real> Ex1) {
#ifdef WITH_MPI
Kokkos::fence();
// sum along all of the processes on the same r
MPI_Allreduce(MPI_IN_PLACE, Ex1Avg.data(), data->np_tot[IDIR], realMPI,
idefix::MPI_Allreduce(MPI_IN_PLACE, this->Ex1Avg, data->np_tot[IDIR], realMPI,
MPI_SUM, data->mygrid->AxisComm);
#endif
}
Expand Down Expand Up @@ -88,7 +88,8 @@ void Axis::RegularizeCurrentSide(int side) {
jc = data->end[JDIR]-1;
sign = -1;
}
IdefixArray1D<real> BAvg = this->Ex1Avg;
auto BAvg = this->Ex1Avg.deviceView();
auto BAvgComm = this->Ex1Avg;
IdefixArray1D<real> x1 = data->x[IDIR];
IdefixArray1D<real> dx3 = data->dx[KDIR];
IdefixArray1D<real> dx2 = data->dx[JDIR];
Expand All @@ -107,7 +108,7 @@ void Axis::RegularizeCurrentSide(int side) {
#ifdef WITH_MPI
Kokkos::fence();
// sum along all of the processes on the same r
MPI_Allreduce(MPI_IN_PLACE, BAvg.data(), data->np_tot[IDIR], realMPI,
MPI_Allreduce(MPI_IN_PLACE, BAvgComm, data->np_tot[IDIR], realMPI,
MPI_SUM, data->mygrid->AxisComm);
#endif
}
Expand Down Expand Up @@ -169,7 +170,7 @@ void Axis::FixBx2sAxis(int side) {
// Compute the values of Bx and By that are consistent with BX2 along the axis
#if DIMENSIONS == 3
IdefixArray4D<real> Vs = this->Vs;
IdefixArray2D<real> BAvg = this->BAvg;
auto BAvg = this->BAvg.deviceView();
IdefixArray1D<real> phi = data->x[KDIR];

int jin = 0;
Expand Down Expand Up @@ -208,7 +209,7 @@ void Axis::FixBx2sAxis(int side) {
Kokkos::fence();
#ifdef WITH_MPI
// sum along all of the processes on the same r
MPI_Allreduce(MPI_IN_PLACE, BAvg.data(), 2*data->np_tot[IDIR], realMPI,
idefix::MPI_Allreduce(MPI_IN_PLACE, this->BAvg, 2*data->np_tot[IDIR], realMPI,
MPI_SUM, data->mygrid->AxisComm);
#endif
}
Expand Down Expand Up @@ -392,7 +393,7 @@ void Axis::ExchangeMPI(int side) {
idfx::pushRegion("Axis::ExchangeMPI");
#ifdef WITH_MPI
// Load the buffers with data
int ibeg,iend,jbeg,jend,kbeg,kend,offset;
int offset;
int ny;
Buffer bufferSend = this->bufferSend;
IdefixArray1D<int> map = this->mapVars;
Expand All @@ -408,7 +409,7 @@ void Axis::ExchangeMPI(int side) {
MPI_Status recvStatus;

double tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Start(&recvRequest));
MPI_SAFE_CALL(idefix::MPI_Start(&recvRequest));
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Coordinates of the ghost region which needs to be transfered
Expand Down Expand Up @@ -470,8 +471,8 @@ void Axis::ExchangeMPI(int side) {
Kokkos::fence();

tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Start(&sendRequest));
MPI_Wait(&recvRequest,&recvStatus);
MPI_SAFE_CALL(idefix::MPI_Start(&sendRequest));
idefix::MPI_Wait(&recvRequest,&recvStatus);
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Unpack
Expand Down Expand Up @@ -525,8 +526,7 @@ void Axis::ExchangeMPI(int side) {
} // MHD
}

MPI_Wait(&sendRequest, &sendStatus);

idefix::MPI_Wait(&sendRequest, &sendStatus);
idfx::mpiCallsTimer += MPI_Wtime() - tStart;


Expand Down Expand Up @@ -593,11 +593,11 @@ void Axis::InitMPI() {
MPI_SAFE_CALL(MPI_Cart_shift(data->mygrid->AxisComm,0,data->mygrid->nproc[KDIR]/2,
&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(bufferSend.data(), bufferSend.Size(), realMPI, procSend,
650, data->mygrid->AxisComm, &sendRequest));
MPI_SAFE_CALL(idefix::MPI_Send_init(bufferSend.commView(), bufferSend.Size(),
realMPI, procSend, 650, data->mygrid->AxisComm, &sendRequest));

MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferRecv.Size(), realMPI, procRecv,
650, data->mygrid->AxisComm, &recvRequest));
MPI_SAFE_CALL(idefix::MPI_Recv_init(bufferRecv.commView(), bufferRecv.Size(),
realMPI, procRecv, 650, data->mygrid->AxisComm, &recvRequest));

#endif
idfx::popRegion();
Expand Down
12 changes: 6 additions & 6 deletions src/fluid/boundary/axis.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ class Axis {

enum {faceTop, faceBot};
#ifdef WITH_MPI
MPI_Request sendRequest;
MPI_Request recvRequest;
idefix::MPI_Request_1D<real> sendRequest;
idefix::MPI_Request_1D<real> recvRequest;

Buffer bufferSend;
Buffer bufferRecv;
Expand All @@ -71,8 +71,8 @@ class Axis {
#endif
void InitMPI();

IdefixArray1D<real> Ex1Avg;
IdefixArray2D<real> BAvg;
idefix::IdefixCommArray1D<real> Ex1Avg;
idefix::IdefixCommArray2D<real> BAvg;
bool haveCurrent;
IdefixArray2D<real> JAvg;
IdefixArray1D<int> symmetryVc;
Expand Down Expand Up @@ -155,8 +155,8 @@ Axis::Axis(Boundary<Phys> *boundary) {
}
Kokkos::deep_copy(symmetryVs, symmetryVsHost);

this->Ex1Avg = IdefixArray1D<real>("Axis:Ex1Avg",data->np_tot[IDIR]);
this->BAvg = IdefixArray2D<real>("Axis:BxAvg",data->np_tot[IDIR],2);
this->Ex1Avg = idefix::IdefixCommArray1D<real>("Axis:Ex1Avg",data->np_tot[IDIR]);
this->BAvg = idefix::IdefixCommArray2D<real>("Axis:BxAvg",data->np_tot[IDIR],2);
if(haveCurrent) {
this->JAvg = IdefixArray2D<real>("Axis:JAvg",data->np_tot[IDIR],3);
}
Expand Down
25 changes: 12 additions & 13 deletions src/fluid/constrainedTransport/EMFexchange.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ void ConstrainedTransport<Phys>::ExchangeX1(IdefixArray3D<real> ey, IdefixArray3
MPI_Status recvStatus[2];

double tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1));
MPI_SAFE_CALL(idefix::MPI_Startall(2, recvRequestX1));
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

BoundaryType lbound = data->lbound[IDIR];
Expand Down Expand Up @@ -82,11 +82,10 @@ void ConstrainedTransport<Phys>::ExchangeX1(IdefixArray3D<real> ey, IdefixArray3
Kokkos::fence();

tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1));
MPI_SAFE_CALL(idefix::MPI_Startall(2, sendRequestX1));
// Wait for buffers to be received

MPI_Waitall(2,recvRequestX1,recvStatus);
MPI_Waitall(2, sendRequestX1, sendStatus);
idefix::MPI_Waitall(2,recvRequestX1,recvStatus);
idefix::MPI_Waitall(2, sendRequestX1, sendStatus);
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Unpack
Expand Down Expand Up @@ -130,7 +129,7 @@ void ConstrainedTransport<Phys>::ExchangeX2(IdefixArray3D<real> ex, IdefixArray3
double tStart = MPI_Wtime();
MPI_Status sendStatus[2];
MPI_Status recvStatus[2];
MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2));
MPI_SAFE_CALL(idefix::MPI_Startall(2, recvRequestX2));
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

BoundaryType lbound = data->lbound[JDIR];
Expand Down Expand Up @@ -169,10 +168,10 @@ void ConstrainedTransport<Phys>::ExchangeX2(IdefixArray3D<real> ex, IdefixArray3
Kokkos::fence();

tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2));
MPI_SAFE_CALL(idefix::MPI_Startall(2, sendRequestX2));
// Wait for buffers to be received
MPI_Waitall(2, recvRequestX2, recvStatus);
MPI_Waitall(2, sendRequestX2, sendStatus);
idefix::MPI_Waitall(2, recvRequestX2, recvStatus);
idefix::MPI_Waitall(2, sendRequestX2, sendStatus);
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Unpack
Expand Down Expand Up @@ -216,7 +215,7 @@ void ConstrainedTransport<Phys>::ExchangeX3(IdefixArray3D<real> ex, IdefixArray3
double tStart = MPI_Wtime();
MPI_Status sendStatus[2];
MPI_Status recvStatus[2];
MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3));
MPI_SAFE_CALL(idefix::MPI_Startall(2, recvRequestX3));
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

BoundaryType lbound = data->lbound[KDIR];
Expand Down Expand Up @@ -256,10 +255,10 @@ void ConstrainedTransport<Phys>::ExchangeX3(IdefixArray3D<real> ex, IdefixArray3
Kokkos::fence();

tStart = MPI_Wtime();
MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3));
MPI_SAFE_CALL(idefix::MPI_Startall(2, sendRequestX3));
// Wait for buffers to be received
MPI_Waitall(2, recvRequestX3, recvStatus);
MPI_Waitall(2, sendRequestX3, sendStatus);
idefix::MPI_Waitall(2, recvRequestX3, recvStatus);
idefix::MPI_Waitall(2, sendRequestX3, sendStatus);
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Unpack
Expand Down
84 changes: 42 additions & 42 deletions src/fluid/constrainedTransport/constrainedTransport.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ class ConstrainedTransport {
IdefixArray3D<real> Ex3;

// Helper arrays for shearing box
IdefixArray2D<real> sbEyL;
IdefixArray2D<real> sbEyR;
IdefixArray2D<real> sbEyRL;
idefix::IdefixCommArray2D<real> sbEyL;
idefix::IdefixCommArray2D<real> sbEyR;
idefix::IdefixCommArray2D<real> sbEyRL;

// Range of existence

Expand Down Expand Up @@ -137,12 +137,12 @@ class ConstrainedTransport {
int bufferSizeX3;

// Requests for MPI persistent communications
MPI_Request sendRequestX1[2];
MPI_Request sendRequestX2[2];
MPI_Request sendRequestX3[2];
MPI_Request recvRequestX1[2];
MPI_Request recvRequestX2[2];
MPI_Request recvRequestX3[2];
idefix::MPI_Request_1D<real> sendRequestX1[2];
idefix::MPI_Request_1D<real> sendRequestX2[2];
idefix::MPI_Request_1D<real> sendRequestX3[2];
idefix::MPI_Request_1D<real> recvRequestX1[2];
idefix::MPI_Request_1D<real> recvRequestX2[2];
idefix::MPI_Request_1D<real> recvRequestX3[2];

Kokkos::Timer timer; // Internal MPI timer
#endif
Expand Down Expand Up @@ -188,9 +188,9 @@ ConstrainedTransport<Phys>::ConstrainedTransport(Input &input, Fluid<Phys> *hydr

// Allocate shearing box arrays
if(hydro->haveShearingBox == true) {
sbEyL = IdefixArray2D<real>("EMF_sbEyL", data->np_tot[KDIR], data->np_tot[JDIR]);
sbEyR = IdefixArray2D<real>("EMF_sbEyR", data->np_tot[KDIR], data->np_tot[JDIR]);
sbEyRL = IdefixArray2D<real>("EMF_sbEyRL", data->np_tot[KDIR], data->np_tot[JDIR]);
sbEyL = idefix::IdefixCommArray2D<real>("EMF_sbEyL", data->np_tot[KDIR], data->np_tot[JDIR]);
sbEyR = idefix::IdefixCommArray2D<real>("EMF_sbEyR", data->np_tot[KDIR], data->np_tot[JDIR]);
sbEyRL = idefix::IdefixCommArray2D<real>("EMF_sbEyRL", data->np_tot[KDIR], data->np_tot[JDIR]);
}

D_EXPAND( ez = IdefixArray3D<real>("EMF_ez",
Expand Down Expand Up @@ -331,11 +331,11 @@ ConstrainedTransport<Phys>::ConstrainedTransport(Input &input, Fluid<Phys> *hydr
if(data->rbound[IDIR] == shearingbox ) procSend = MPI_PROC_NULL;
if(data->lbound[IDIR] == shearingbox ) procRecv = MPI_PROC_NULL;

MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, 100,
mygrid->CartComm, &sendRequestX1[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX1[faceRight].commView(), bufferSizeX1,
realMPI, procSend, 100, mygrid->CartComm, &sendRequestX1[faceRight]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, 100,
mygrid->CartComm, &recvRequestX1[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX1[faceLeft].commView(), bufferSizeX1,
realMPI, procRecv, 100, mygrid->CartComm, &recvRequestX1[faceLeft]));

// Send to the left
// We receive from procRecv, and we send to procSend
Expand All @@ -344,52 +344,52 @@ ConstrainedTransport<Phys>::ConstrainedTransport(Input &input, Fluid<Phys> *hydr
if(data->lbound[IDIR] == shearingbox ) procSend = MPI_PROC_NULL;
if(data->rbound[IDIR] == shearingbox ) procRecv = MPI_PROC_NULL;

MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, 101,
mygrid->CartComm, &sendRequestX1[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX1[faceLeft].commView(), bufferSizeX1,
realMPI, procSend, 101, mygrid->CartComm, &sendRequestX1[faceLeft]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, 101,
mygrid->CartComm, &recvRequestX1[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX1[faceRight].commView(), bufferSizeX1,
realMPI, procRecv, 101, mygrid->CartComm, &recvRequestX1[faceRight]));

#if DIMENSIONS >= 2
// We receive from procRecv, and we send to procSend
MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,1,&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, 200,
mygrid->CartComm, &sendRequestX2[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX2[faceRight].commView(), bufferSizeX2,
realMPI, procSend, 200, mygrid->CartComm, &sendRequestX2[faceRight]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, 200,
mygrid->CartComm, &recvRequestX2[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX2[faceLeft].commView(), bufferSizeX2,
realMPI, procRecv, 200, mygrid->CartComm, &recvRequestX2[faceLeft]));

// Send to the left
// We receive from procRecv, and we send to procSend
MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,-1,&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, 201,
mygrid->CartComm, &sendRequestX2[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX2[faceLeft].commView(), bufferSizeX2,
realMPI, procSend, 201, mygrid->CartComm, &sendRequestX2[faceLeft]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, 201,
mygrid->CartComm, &recvRequestX2[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX2[faceRight].commView(), bufferSizeX2,
realMPI, procRecv, 201, mygrid->CartComm, &recvRequestX2[faceRight]));
#endif

#if DIMENSIONS == 3
// We receive from procRecv, and we send to procSend
MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,1,&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, 300,
mygrid->CartComm, &sendRequestX3[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX3[faceRight].commView(), bufferSizeX3,
realMPI, procSend, 300, mygrid->CartComm, &sendRequestX3[faceRight]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, 300,
mygrid->CartComm, &recvRequestX3[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX3[faceLeft].commView(), bufferSizeX3,
realMPI, procRecv, 300, mygrid->CartComm, &recvRequestX3[faceLeft]));

// Send to the left
// We receive from procRecv, and we send to procSend
MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,-1,&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, 301,
mygrid->CartComm, &sendRequestX3[faceLeft]));
MPI_SAFE_CALL(idefix::MPI_Send_init(BufferSendX3[faceLeft].commView(), bufferSizeX3,
realMPI, procSend, 301, mygrid->CartComm, &sendRequestX3[faceLeft]));

MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, 301,
mygrid->CartComm, &recvRequestX3[faceRight]));
MPI_SAFE_CALL(idefix::MPI_Recv_init(BufferRecvX3[faceRight].commView(), bufferSizeX3,
realMPI, procRecv, 301, mygrid->CartComm, &recvRequestX3[faceRight]));
#endif

#endif // WITH_MPI
Expand All @@ -409,17 +409,17 @@ ConstrainedTransport<Phys>::~ConstrainedTransport() {
// Properly clean up the mess
idfx::cout << "Emf: Cleaning up MPI persistent communication channels" << std::endl;
for(int i=0 ; i< 2; i++) {
MPI_Request_free( &sendRequestX1[i]);
MPI_Request_free( &recvRequestX1[i]);
idefix::MPI_Request_free( &sendRequestX1[i]);
idefix::MPI_Request_free( &recvRequestX1[i]);

#if DIMENSIONS >= 2
MPI_Request_free( &sendRequestX2[i]);
MPI_Request_free( &recvRequestX2[i]);
idefix::MPI_Request_free( &sendRequestX2[i]);
idefix::MPI_Request_free( &recvRequestX2[i]);
#endif

#if DIMENSIONS == 3
MPI_Request_free( &sendRequestX3[i]);
MPI_Request_free( &recvRequestX3[i]);
idefix::MPI_Request_free( &sendRequestX3[i]);
idefix::MPI_Request_free( &recvRequestX3[i]);
#endif
}
#endif
Expand Down
Loading
Loading