From 5b97e546719973be9717a87d7c038eda7465d925 Mon Sep 17 00:00:00 2001 From: Alankar Dutta Date: Fri, 27 Sep 2024 13:23:54 +0200 Subject: [PATCH 01/10] [ENH] customize log_dir location from runtime ini --- src/global.cpp | 36 +++++++++++++++++++++++++++++++++--- src/global.hpp | 1 + src/input.cpp | 19 +++++++++++++------ src/input.hpp | 3 +++ 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/global.cpp b/src/global.cpp index 1e9c11eef..25098f3a5 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -7,6 +7,16 @@ #include #include +#if __has_include() + #include + namespace fs = std::filesystem; +#elif __has_include() + #include + namespace fs = std::experimental::filesystem; +#else + #error "Missing the header." +#endif + #include "idefix.hpp" #include "global.hpp" #include "profiler.hpp" @@ -26,6 +36,7 @@ bool warningsAreErrors{false}; IdefixOutStream cout; IdefixErrStream cerr; +std::string logFileDir; Profiler prof; LoopPattern defaultLoopPattern; @@ -103,11 +114,30 @@ void IdefixOutStream::init(int rank) { // disable the log file void IdefixOutStream::enableLogFile() { std::stringstream sslogFileName; - sslogFileName << "idefix." << idfx::prank << ".log"; - + + sslogFileName << idfx::logFileDir << "/./" << "idefix." << idfx::prank << ".log"; std::string logFileName(sslogFileName.str()); + + if(idfx::prank==0) { + if(!fs::is_directory(logFileDir)) { + try { + if(!fs::create_directories(logFileDir)) { + std::stringstream msg; + msg << "Cannot create directory " << logFileDir << std::endl; + IDEFIX_ERROR(msg); + } + } catch(std::exception &e) { + std::stringstream msg; + msg << "Cannot create directory " << logFileDir << std::endl; + msg << e.what(); + IDEFIX_ERROR(msg); + } + } + } +#ifdef WITH_MPI + MPI_Barrier(MPI_COMM_WORLD); +#endif this->my_fstream.open(logFileName.c_str()); - this->logFileEnabled = true; } diff --git a/src/global.hpp b/src/global.hpp index 47fca42ee..78747f57e 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -21,6 +21,7 @@ class Profiler; extern int prank; //< parallel rank extern int psize; +extern std::string logFileDir; //< logfileDir extern IdefixOutStream cout; //< custom cout for idefix extern IdefixErrStream cerr; //< custom cerr for idefix extern Profiler prof; //< profiler (for memory & performance usage) diff --git a/src/input.cpp b/src/input.cpp index 336414953..d5c6d9bff 100644 --- a/src/input.cpp +++ b/src/input.cpp @@ -43,6 +43,8 @@ Input::Input(int argc, char* argv[] ) { bool haveBlock = false; std::stringstream msg; int nParameters = 0; // # of parameters in current block + // Log files are enabled by default + this->enableLogs = true; // Tell the system we want to catch the SIGUSR2 signals signal(SIGUSR2, signalHandler); @@ -108,12 +110,20 @@ Input::Input(int argc, char* argv[] ) { } } file.close(); + + if(this->enableLogs) { + if(CheckEntry("Output","log_dir")>=0) { + idfx::logFileDir = Get("Output", "log_dir", 0); + } else { + idfx::logFileDir = "./"; + } + idfx::cout.enableLogFile(); + } } // This routine parse command line options void Input::ParseCommandLine(int argc, char **argv) { std::stringstream msg; - bool enableLogs = true; for(int i = 1 ; i < argc ; i++) { // MPI decomposition argument if(std::string(argv[i]) == "-dec") { @@ -171,9 +181,9 @@ void Input::ParseCommandLine(int argc, char **argv) { this->forceInitRequested = true; } else if(std::string(argv[i]) == "-nowrite") { this->forceNoWrite = true; - enableLogs = false; + this->enableLogs = false; } else if(std::string(argv[i]) == "-nolog") { - enableLogs = false; + this->enableLogs = false; } else if(std::string(argv[i]) == "-profile") { idfx::prof.EnablePerformanceProfiling(); } else if(std::string(argv[i]) == "-Werror") { @@ -190,9 +200,6 @@ void Input::ParseCommandLine(int argc, char **argv) { IDEFIX_ERROR(msg); } } - if(enableLogs) { - idfx::cout.enableLogFile(); - } } diff --git a/src/input.hpp b/src/input.hpp index 18fcf2601..947b65625 100644 --- a/src/input.hpp +++ b/src/input.hpp @@ -24,6 +24,9 @@ class Input { // Constructor from a file Input (int, char ** ); void ShowConfig(); + + // flag if logging is to be done + bool enableLogs; // Accessor to input parameters // the parameters are always: BlockName, EntryName, ParameterNumber (starting from 0) From 73fc68b553d8aecfcfe9ebe775f32cf3af56eba1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:28:40 +0000 Subject: [PATCH 02/10] [pre-commit.ci lite] apply automatic fixes --- src/global.cpp | 4 ++-- src/input.cpp | 2 +- src/input.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/global.cpp b/src/global.cpp index 25098f3a5..26dfcc096 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -114,10 +114,10 @@ void IdefixOutStream::init(int rank) { // disable the log file void IdefixOutStream::enableLogFile() { std::stringstream sslogFileName; - + sslogFileName << idfx::logFileDir << "/./" << "idefix." << idfx::prank << ".log"; std::string logFileName(sslogFileName.str()); - + if(idfx::prank==0) { if(!fs::is_directory(logFileDir)) { try { diff --git a/src/input.cpp b/src/input.cpp index d5c6d9bff..2e628f855 100644 --- a/src/input.cpp +++ b/src/input.cpp @@ -110,7 +110,7 @@ Input::Input(int argc, char* argv[] ) { } } file.close(); - + if(this->enableLogs) { if(CheckEntry("Output","log_dir")>=0) { idfx::logFileDir = Get("Output", "log_dir", 0); diff --git a/src/input.hpp b/src/input.hpp index 947b65625..727920597 100644 --- a/src/input.hpp +++ b/src/input.hpp @@ -24,7 +24,7 @@ class Input { // Constructor from a file Input (int, char ** ); void ShowConfig(); - + // flag if logging is to be done bool enableLogs; From ee6dd0bc67ae06d5446df016f26f69c207c6606c Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 13:49:14 +0100 Subject: [PATCH 03/10] update version number --- CHANGELOG.md | 12 ++++++++++++ CMakeLists.txt | 6 +++--- doc/source/conf.py | 2 +- src/main.cpp | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d791f8f0..a675e1768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.00] 2025-01-17 +### Changed + +- + +### Added + +- + +### Removed + + ## [2.1.02] 2024-10-24 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index d1a2e3d36..a8c7ff94e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,10 @@ endif() set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) -set(Idefix_VERSION_MINOR 1) -set(Idefix_VERSION_PATCH 02) +set(Idefix_VERSION_MINOR 2) +set(Idefix_VERSION_PATCH 00) -project (idefix VERSION 2.1.02) +project (idefix VERSION 2.2.00) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) diff --git a/doc/source/conf.py b/doc/source/conf.py index b712d0264..24f9709bb 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,7 @@ author = 'Geoffroy Lesur' # The full version, including alpha/beta/rc tags -release = '2.1.02' +release = '2.2.00' diff --git a/src/main.cpp b/src/main.cpp index 06f408ffe..6b65c4f1c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 2.1.00 +// IDEFIX v 2.2.00 // // ************************************************************************ //@HEADER From 5d1a15da396f4a0f8b7cd1a6d6579a2f868e8a83 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 15 Jan 2025 14:07:03 +0100 Subject: [PATCH 04/10] update changelog --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a675e1768..66f01f7be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.2.00] 2025-01-17 ### Changed -- +- Fix a bug that could lead to a segmentation fault when MHD+Fargo+MPI (with X3 domain decomposition) are all enabled (#295) +- Fix a bug that could result in an incorrect magnetic field when initialising B from the vector potential in non-axisymmetric spherical geometry (#293) +- Fix a bug that could result in Idefix believing the MPI library is not Cuda aware for some versions of OpenMPI (#310) +- Ensure that the behaviour in 1D Spherical geometry is identical to Pluto (#291) ### Added -- - -### Removed +- Add the python interface "pydefix", allowing users to initialise and analyse Idefix simulations live from Python without writing any file (#277) +- Add the native Idefix coordinates in VTK file to simplify postprocessing (#292) +- Add code testing on CPU targets using gcc and Intel Oneapi (#300) ## [2.1.02] 2024-10-24 From ede2e1f5e30e4486bd1dfcf141b4a67126634093 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 16 Apr 2025 16:38:05 +0200 Subject: [PATCH 05/10] V2.2.01 (#336) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [2.2.01] 2025-04-16 ### Changed - Fix a bug that led to instabilities in the RKL scheme with very small grid spacings (#323) - Fix a bug that prevented Idefix from running with Sycl backend (required on Intel GPUs) (#331) - Fix an error that led to incorrect electrical current regularisation around the polar axis in non-Ideal MHD (#333) - Improve div(B) checks with a dimensionless implementation, avoiding too large divB errors in grids with large stretch factors (#334) ### Added - Time-Implicit drag for multiple dust species, preventing small time steps for strongly coupled dust grains (#321) - Collisionless heat flux added to the Braginskii module (#317) - New global `idfx::DumpArray` debugging function to dump any Idefix array into a numpy array that can read from python (#318) - Automatic benchmark plots in the documentation (#319) - More CI tests of grid coarsening (#329) - Dump outputs based on wallclock time (#335) --------- Co-authored-by: Victor Réville Co-authored-by: Hal Bal Co-authored-by: Jean Kempf Co-authored-by: Victor Réville <47865059+vreville@users.noreply.github.com> Co-authored-by: Marc Coiffier Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .github/workflows/idefix-ci-jobs.yml | 6 + .gitignore | 1 + CHANGELOG.md | 18 ++ CMakeLists.txt | 4 +- doc/python_requirements.txt | 1 + doc/source/bench.json | 162 ++++++++++++ doc/source/conf.py | 1 + doc/source/faq.rst | 3 + doc/source/index.rst | 7 +- doc/source/modules/braginskii.rst | 64 ++++- doc/source/modules/dust.rst | 41 ++- doc/source/performances.rst | 23 +- doc/source/plot_idefix_bench.py | 26 ++ doc/source/programmingguide.rst | 21 ++ doc/source/reference/idefix.ini.rst | 6 +- doc/source/reference/makefile.rst | 3 +- reference | 2 +- src/dataBlock/coarsen.cpp | 30 ++- src/dataBlock/evolveStage.cpp | 10 + src/fluid/boundary/axis.cpp | 4 +- src/fluid/braginskii/bragThermalDiffusion.cpp | 22 +- src/fluid/braginskii/bragThermalDiffusion.hpp | 224 ++++++++++++++-- src/fluid/braginskii/bragViscosity.cpp | 9 +- src/fluid/braginskii/bragViscosity.hpp | 4 + src/fluid/calcRightHandSide.hpp | 70 ++--- src/fluid/checkDivB.hpp | 18 +- src/fluid/drag.cpp | 243 ++++++++++++++---- src/fluid/drag.hpp | 109 +++++--- src/fluid/evolveStage.hpp | 6 +- src/fluid/fluid.hpp | 91 +++---- src/fluid/fluid_defs.hpp | 8 +- src/global.hpp | 15 ++ src/loop.hpp | 15 ++ src/main.cpp | 2 +- src/output/output.cpp | 9 +- src/rkl/rkl.hpp | 89 +++++-- src/utils/CMakeLists.txt | 2 + src/utils/column.cpp | 240 +++++++++++++++++ src/utils/column.hpp | 74 ++++++ test/Dust/DustEnergy/idefix-implicit.ini | 36 +++ test/Dust/DustEnergy/testme.py | 2 +- test/Dust/DustyShock/definitions.hpp | 5 + test/Dust/DustyShock/idefix-implicit.ini | 36 +++ test/Dust/DustyShock/idefix.ini | 35 +++ test/Dust/DustyShock/python/testidefix.py | 97 +++++++ test/Dust/DustyShock/setup.cpp | 99 +++++++ test/Dust/DustyShock/testme.py | 39 +++ test/Dust/DustyWave/idefix-implicit.ini | 36 +++ test/Dust/DustyWave/python/testidefix.py | 7 +- test/Dust/DustyWave/testme.py | 2 +- test/Dust/FargoPlanet/setup.cpp | 2 +- test/MHD/AxisFluxTube/setup.cpp | 2 +- test/MHD/MTI/idefix-rkl.ini | 2 +- test/MHD/MTI/idefix-sl.ini | 2 +- test/MHD/MTI/idefix.ini | 2 +- test/MHD/MTI/setup.cpp | 6 +- test/MHD/clessTDiffusion/CMakeLists.txt | 1 + test/MHD/clessTDiffusion/definitions.hpp | 5 + test/MHD/clessTDiffusion/idefix.ini | 41 +++ test/MHD/clessTDiffusion/python/testidefix.py | 40 +++ test/MHD/clessTDiffusion/setup.cpp | 158 ++++++++++++ test/MHD/clessTDiffusion/testme.py | 42 +++ test/MHD/sphBragTDiffusion/idefix.ini | 2 +- test/utils/columnDensity/definitions.hpp | 4 + test/utils/columnDensity/idefix.ini | 24 ++ test/utils/columnDensity/setup.cpp | 169 ++++++++++++ test/utils/columnDensity/testme.py | 23 ++ 67 files changed, 2313 insertions(+), 289 deletions(-) create mode 100644 doc/source/bench.json create mode 100644 doc/source/plot_idefix_bench.py create mode 100644 src/utils/column.cpp create mode 100644 src/utils/column.hpp create mode 100644 test/Dust/DustEnergy/idefix-implicit.ini create mode 100644 test/Dust/DustyShock/definitions.hpp create mode 100644 test/Dust/DustyShock/idefix-implicit.ini create mode 100644 test/Dust/DustyShock/idefix.ini create mode 100755 test/Dust/DustyShock/python/testidefix.py create mode 100644 test/Dust/DustyShock/setup.cpp create mode 100755 test/Dust/DustyShock/testme.py create mode 100644 test/Dust/DustyWave/idefix-implicit.ini create mode 100644 test/MHD/clessTDiffusion/CMakeLists.txt create mode 100644 test/MHD/clessTDiffusion/definitions.hpp create mode 100644 test/MHD/clessTDiffusion/idefix.ini create mode 100644 test/MHD/clessTDiffusion/python/testidefix.py create mode 100644 test/MHD/clessTDiffusion/setup.cpp create mode 100755 test/MHD/clessTDiffusion/testme.py create mode 100644 test/utils/columnDensity/definitions.hpp create mode 100644 test/utils/columnDensity/idefix.ini create mode 100644 test/utils/columnDensity/setup.cpp create mode 100755 test/utils/columnDensity/testme.py diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index c0e622966..275ea1ce2 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -65,6 +65,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang -all $TESTME_OPTIONS - name: Orszag Tang 3D+restart tests run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS + - name: Axis Flux tube + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AxisFluxTube -all $TESTME_OPTIONS ParabolicMHD: runs-on: self-hosted @@ -177,6 +179,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragTDiffusion -all $TESTME_OPTIONS - name: Spherical anisotropic viscosity run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sphBragViscosity -all $TESTME_OPTIONS + - name: Collisionless thermal conduction + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/clessTDiffusion -all $TESTME_OPTIONS Examples: needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] @@ -201,3 +205,5 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/lookupTable -all $TESTME_OPTIONS - name: Dump Image run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/dumpImage -all $TESTME_OPTIONS + - name: Column density + run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/columnDensity -all $TESTME_OPTIONS diff --git a/.gitignore b/.gitignore index 422647c3f..4de7fb76e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ doc/source/_static/* doc/source/_public/* doc/source/api/* doc/source/xml/* +doc/env/* # compiled files **/__pycache__ diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f01f7be..b0732bb58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.01] 2025-04-16 +### Changed + +- Fix a bug that led to instabilities in the RKL scheme with very small grid spacings (#323) +- Fix a bug that prevented Idefix from running with Sycl backend (required on Intel GPUs) (#331) +- Fix an error that led to incorrect electrical current regularisation around the polar axis in non-Ideal MHD (#333) +- Improve div(B) checks with a dimensionless implementation, avoiding too large divB errors in grids with large stretch factors (#334) + +### Added + +- Time-Implicit drag for multiple dust species, preventing small time steps for strongly coupled dust grains (#321) +- Collisionless heat flux added to the Braginskii module (#317) +- New global `idfx::DumpArray` debugging function to dump any Idefix array into a numpy array that can read from python (#318) +- Automatic benchmark plots in the documentation (#319) +- More CI tests of grid coarsening (#329) +- Dump outputs based on wallclock time (#335) + + ## [2.2.00] 2025-01-17 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index a8c7ff94e..6b62b7542 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) set(Idefix_VERSION_MINOR 2) -set(Idefix_VERSION_PATCH 00) +set(Idefix_VERSION_PATCH 01) -project (idefix VERSION 2.2.00) +project (idefix VERSION 2.2.01) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) diff --git a/doc/python_requirements.txt b/doc/python_requirements.txt index 35f72dcf7..c75fa02b7 100644 --- a/doc/python_requirements.txt +++ b/doc/python_requirements.txt @@ -14,3 +14,4 @@ exhale==0.3.7 m2r2==0.3.2 sphinx-copybutton==0.5.2 #sphinxcontrib-applehelp==1.0.7 +matplotlib==3.10.0 diff --git a/doc/source/bench.json b/doc/source/bench.json new file mode 100644 index 000000000..08a3c8040 --- /dev/null +++ b/doc/source/bench.json @@ -0,0 +1,162 @@ +[ + { + "date": "2025-03-04_12:57:12", + "gpumodel": "v100", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "37161676db15115c38fed3f35c94fa447cbac7bd", + "results": [ + { + "nbgpu": 1, + "cell_updates": 1.193720E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.178864E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.155336E+8 + }, + { + "nbgpu": 8, + "cell_updates": 1.014338E+8 + }, + { + "nbgpu": 16, + "cell_updates": 9.855007E+7 + }, + { + "nbgpu": 32, + "cell_updates": 9.012061E+7 + }, + { + "nbgpu": 64, + "cell_updates": 8.538461E+7 + }, + { + "nbgpu": 128, + "cell_updates": 8.531021E+7 + } + ] + }, + { + "date": "2025-03-04_13:07:10", + "gpumodel": "a100", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "b536949200e50fac68d8a46d5db38fc8e3f02da5", + "results": [ + { + "nbgpu": 1, + "cell_updates": 2.044728E+8 + }, + { + "nbgpu": 2, + "cell_updates": 2.003563E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.963512E+8 + }, + { + "nbgpu": 8, + "cell_updates": 1.933039E+8 + }, + { + "nbgpu": 16, + "cell_updates": 9.759154E+7 + }, + { + "nbgpu": 32, + "cell_updates": 6.369645E+7 + }, + { + "nbgpu": 64, + "cell_updates": 4.629474E+7 + }, + { + "nbgpu": 128, + "cell_updates": 4.580281E+7 + } + ] + }, + { + "date": "2025-03-04_13:16:01", + "gpumodel": "h100", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "b536949200e50fac68d8a46d5db38fc8e3f02da5", + "results": [ + { + "nbgpu": 1, + "cell_updates": 3.079643E+8 + }, + { + "nbgpu": 2, + "cell_updates": 3.012300E+8 + }, + { + "nbgpu": 4, + "cell_updates": 2.944091E+8 + }, + { + "nbgpu": 8, + "cell_updates": 2.837224E+8 + }, + { + "nbgpu": 16, + "cell_updates": 2.827778E+8 + }, + { + "nbgpu": 32, + "cell_updates": 2.822657E+8 + }, + { + "nbgpu": 64, + "cell_updates": 2.767820E+8 + }, + { + "nbgpu": 128, + "cell_updates": 2.767322E+8 + } + ] + }, + { + "date": "2025-03-06_11:21:56", + "gpumodel": "mi250x", + "idefix_commit": "2bc09a0d218459f278e2b28506a09e4591b103ae", + "bench_commit": "868be0a87c6fcda665cbb62db7020aeff70dc62d", + "results": [ + { + "nbgpu": 1, + "cell_updates": 1.436580E+8 + }, + { + "nbgpu": 2, + "cell_updates": 1.372499E+8 + }, + { + "nbgpu": 4, + "cell_updates": 1.344528E+8 + }, + { + "nbgpu": 8, + "cell_updates": 1.293602E+8 + }, + { + "nbgpu": 16, + "cell_updates": 1.260359E+8 + }, + { + "nbgpu": 32, + "cell_updates": 1.204980E+8 + }, + { + "nbgpu": 64, + "cell_updates": 1.163099E+8 + }, + { + "nbgpu": 128, + "cell_updates": 1.192343E+8 + } + ] + } +] diff --git a/doc/source/conf.py b/doc/source/conf.py index 24f9709bb..6f0cffa44 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -35,6 +35,7 @@ extensions = [ "sphinx_rtd_theme", 'sphinx_git', + 'matplotlib.sphinxext.plot_directive', "breathe", "exhale", "m2r2", diff --git a/doc/source/faq.rst b/doc/source/faq.rst index 75a07a68b..4c7bc056b 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -73,3 +73,6 @@ I want to test a modification to *Idefix* source code specific to my problem wit I want to use a lookup table from a CSV file in my idefix_loop. How could I proceed? Use the ``LookupTable`` class (see :ref:`LookupTableClass`) + +I want to compute a cumulative sum (e.g a column density) on the fly. How could I proceed? + Use the ``Column`` class (see :class:`::Column`) diff --git a/doc/source/index.rst b/doc/source/index.rst index 128d591ae..411d73d6d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -84,8 +84,11 @@ Jonah Mauxion Clément Robert gitlab integration, linter -Jean Kempf & François Rincon - anisotropic diffusion +Jean Kempf, Victor Reville, François Rincon + anisotropic diffusion and collisionless thermal conduction + +Marc Coiffier + Continuous integration, automatic benchmarking ======================== About this documentation diff --git a/doc/source/modules/braginskii.rst b/doc/source/modules/braginskii.rst index e657cedd0..fd5f2b8fc 100644 --- a/doc/source/modules/braginskii.rst +++ b/doc/source/modules/braginskii.rst @@ -4,7 +4,7 @@ Braginskii module =================== Equations solved and methods ---------------------------- +---------------------------- The ``Braginskii`` module implements the anisotropic heat and momentum fluxes specific to weakly collisional, magnetised plasma like the intracluster medium @@ -37,7 +37,7 @@ though adapted to vector quantities. cell interface by a simple arithmetic average (Eq. (6)-(7) from Sharma & Hammett 2007). However in the same paper, the authors showed that this implementation can lead to - unphysical heat flux from high to low temperature regions. + unphysical heat flux from low to high temperature regions. So we implemented slope limiters for the computation of these transverse heat fluxes, as described in Eq. (17) from Sharma & Hammett (2007). Only the van Leer and the Monotonized Central (MC) limiters are available @@ -72,18 +72,68 @@ of the Braginskii heat flux and viscosity. .. _braginskiiParameterSection: + +Saturation with collisionless heat flux +--------------------------------------- + +The ``Braginskii`` module can include a collisionless saturation of the Braginskii heat flux, typically due to supra-thermal electrons. +The heat flux is then computed as follows: + +:math:`q = \alpha (q_B + q_\perp) + (1-\alpha)\beta*p*v`, + +where :math:`\alpha \in [0,1]` controls the transition between the Braginskii heat flux and the collisionless heat flux +and :math:`\beta` controls the amplitude of the collisionless heat flux (typically :math:`\beta \in [1,4]`, see Hollweg 1976). + +.. note:: + As a result, even with :math:`\kappa_\perp = 0`, the heat flux is no longer necessarilly strictly aligned with the magnetic field. +.. note:: + The collisionless heat flux is a hyperbolic term and the diffusion coefficient is set proportional to :math:`\alpha`. +.. note:: + If selected, slope limiters are also used in the collisionless flux, where an upwind scheme has been implemented for stability. +.. note:: + This saturation has been thought to be used mostly using the userdef function that takes four userdef arrays as input. + Main parameters of the module ----------------------------- The ``Braginskii`` module can be enabled adding one or two lines in the ``[Hydro]`` section -starting with the keyword -`bragTDiffusion` or/and *bragViscosity*. The following table summarises the different options +starting with the keyword `bragTDiffusion` or/and *bragViscosity*. The following tables summarise the different options associated to the activation of the Braginskii module: +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ | Column | Entry name | Parameter type | Comment | +========+=======================+=========================+=======================================================================================+ -| 0 | bragModule | string | | Activates Braginskii diffusion. Can be ``bragTDiffusion`` or ``bragViscosity``. | +| 0 | bragTDiffusion | string | | Activates Braginskii thermal diffusion. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 1 | integration | string | | Specifies the type of scheme to be used to integrate the parabolic term. | +| | | | | Can be ``rkl`` or ``explicit``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 2 | slope limiter | string | | Choose the type of limiter to be used to compute anisotropic transverse flux terms. | +| | | | | Can be ``mc``, ``vanleer`` or ``nolimiter``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 3 | saturation mode | string | | Include or not collisionless saturation. Can be ``nosat`` or ``wcless``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 4 | diffusivity type | string | | Specifies the type of diffusivity wanted. Can be ``constant`` or ``userdef``. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 5 | parallel diffusivity | real | | Mandatory if the diffusivity type is ``constant``. Not needed otherwise. | +| | | | | Value of the parallel diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 6 | normal diffusivity | real | | When bragModule ``bragTDiffusion`` and diffusivity type ``constant``, | +| | | | | value of the normal diffusivity. Should be a real number. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 7 | alpha collisionless | real | | If the diffusivity type is ``constant`` and saturation is ``wcless``. | +| | | | | Set to 1 if not provided. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| 8 | beta collisionless | real | | If the diffusivity type is ``constant`` and saturation is ``wcless``. | +| | | | | Set to 0 if not provided. | ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ + +for the *bragViscosity*: + ++--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ +| Column | Entry name | Parameter type | Comment | ++========+=======================+=========================+=======================================================================================+ +| 0 | bragViscosity | string | | Activates Braginskii viscosity. | +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ | 1 | integration | string | | Specifies the type of scheme to be used to integrate the parabolic term. | | | | | | Can be ``rkl`` or ``explicit``. | @@ -101,7 +151,7 @@ associated to the activation of the Braginskii module: +--------+-----------------------+-------------------------+---------------------------------------------------------------------------------------+ Numerical checks ---------------- +---------------- In Cartesian geometry, the ``Braginskii`` module has been tested with many setups and in all configurations of magnetic polarisation: @@ -119,3 +169,5 @@ The same goes for the anisotropic heat flux in Cylindrical/Polar geometry while the anisotropic viscosity has *never* been tested in this geometry. In spherical geometry, both ``Braginskii`` operators have been partially validated (diffusion along the polar axis has not been directly tested). + +The collisionless saturation has been tested in 1D and 2D spherical geometry. diff --git a/doc/source/modules/dust.rst b/doc/source/modules/dust.rst index ca371dade..0cdac49ba 100644 --- a/doc/source/modules/dust.rst +++ b/doc/source/modules/dust.rst @@ -39,15 +39,45 @@ which guarantees total momentum conservation. -Drag CFL condition -------------------- -*Idefix* computes the drag terms with a time-explicit scheme. Hence, an addition CFL constraint arrises because of the drag: +Time Integration +---------------- + +*Idefix* can compute the drag with a 2nd order time-explicit (default) or 1st order time-implicit scheme. Each scheme has its pros and cons. The choice +is made by the user in the input file. + +Explicit scheme: drag CFL condition ++++++++++++++++++++++++++++++++++++ + +When using the time explicit scheme, an addition CFL constraint arrises because of the drag: .. math:: dt < \min(\frac{1}{\sum_i\gamma_i(\rho_i+\rho)}) -*Idefix* automatically adjusts the CFL to satisfy this inequality, in addition to the usual CFL condition. +*Idefix* automatically adjusts the CFL to satisfy this inequality, in addition to the usual CFL condition. This can severly limit the time step, +especially when the gas density is high. + +Implicit scheme ++++++++++++++++ + +When the drag is applied with a 1st order implicit scheme, the drag force is applied at the end of each step. In order to avoid +the complete inversion of the system, we follow a simplified inversion procedure, where we first update the gas momentum as: + +.. math:: + + v_g^{(n+1)}=\left(v_g^{(n)}+\sum_i\frac{\rho_i\gamma_i dt}{1+\rho \gamma_i dt}v_i^{(n)}\right)/\left(1+\sum_i\frac{\rho_i\gamma_i dt}{1+\rho\gamma_idt}\right) + +And then update the dust momentum as: + +.. math:: + + v_i^{(n+1)}=\left(v_i^{(n)}+\rho\gamma_i v_g^{(n+1)}dt\right)/(1+\rho\gamma_i dt) + +Note that the latter equation relies on the *updated* gas velocity. + +.. warning:: + While the implicit scheme is more stable than the explicit one, and it does not require any additional CFL condition, it is less accurate and + possibly lead to inacurrate dust velocities when :math:`dt\gg (\gamma_i\rho)^{-1}`. Use it at your own risk. Dust parameters --------------- @@ -66,6 +96,9 @@ The dust module can be enabled adding a block `[Dust]` in your input .ini file. +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | drag_feedback | bool | | (optionnal) whether the gas feedback is enabled (default true). | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +| drag_implicit | bool | | (optionnal) whether the drag uses a 1st order implicit method. Otherwise use the | +| | | | 2nd order time-explicit scheme (default is false=time explicit) | ++----------------+-------------------------+---------------------------------------------------------------------------------------------+ The drag parameter :math:`\beta_i` above sets the functional form of :math:`\gamma_i(\rho, \rho_i, c_s)` depending on the drag type: diff --git a/doc/source/performances.rst b/doc/source/performances.rst index 84042685e..deae1194e 100644 --- a/doc/source/performances.rst +++ b/doc/source/performances.rst @@ -6,9 +6,8 @@ We report below the performances obtained on various architectures using Idefix. is the 3D MHD Orszag-Tang test problem with 2nd order reconstruction and uct_contact EMFS bundled in Idefix test suite, disabling passive tracers. The test is computed with a 128\ :sup:`3` resolution per MPI sub-domain on GPUs or 32\ :sup:`3` per MPI sub-domain on CPUs. All of the performances measures -have been obtained enabling MPI on *one full node*, but we report here the performance *per GPU* -(i.e. with 2 GCDs on AMD Mi250) or *per core* (on CPU), i.e. dividing the node performance by the number of GPU/core -to simplify the comparison with other clusters. +have been obtained enabling MPI and we reporte here the performance *per GPU*, *per GCD* (on Mi250) +or *per core* (on CPU). The complete scalability tests are available in Idefix `method paper `_. The performances mentionned below are updated for each major revision of Idefix, so they might slightly differ from the method paper. @@ -33,16 +32,14 @@ CPU performances | IDRIS/Jean Zay | Intel Cascade Lake | 0.62 | +---------------------+--------------------+----------------------------------------------------+ - GPU performances ================ -+----------------------+--------------------+----------------------------------------------------+ -| Cluster name | GPU | Performances (in 10\ :sup:`6` cell/s/GPU) | -+======================+====================+====================================================+ -| IDRIS/Jean Zay | NVIDIA V100 | 110 | -+----------------------+--------------------+----------------------------------------------------+ -| IDRIS/Jean Zay | NVIDIA A100 | 194 | -+----------------------+--------------------+----------------------------------------------------+ -| CINES/Adastra | AMD Mi250 | 250 | -+----------------------+--------------------+----------------------------------------------------+ +.. plot:: + + import plot_idefix_bench + plot_idefix_bench.do_plot('Performance on NVidia and AMD GPUs', 'bench.json', ['v100','a100','h100','mi250x']) + +.. note:: + + The inter-node communication on Jean Zay is not optimal on A100 nodes. A ticket is opened with IDRIS support to fix this issue. diff --git a/doc/source/plot_idefix_bench.py b/doc/source/plot_idefix_bench.py new file mode 100644 index 000000000..43482eb04 --- /dev/null +++ b/doc/source/plot_idefix_bench.py @@ -0,0 +1,26 @@ +import matplotlib.pyplot as plt +import json + +def do_plot(title, bench_file, gpumodels): + with open(bench_file, 'r') as f: + benches = json.load(f) + + plt.figure() + xmax=0 + ymax=0 + for gpumodel in gpumodels: + select = [bench for bench in benches if bench['gpumodel'] == gpumodel][-1] + + xs = [r['nbgpu'] for r in select['results']] + ys = [r['cell_updates'] for r in select['results']] + plt.plot(xs, ys,'o-',label=gpumodel) + xmax=max(xmax,max(xs)) + ymax=max(ymax,max(ys)) + + plt.xscale("log", base=2) + plt.ylim(0,ymax*1.1) + plt.xlim(1,xmax*1.1) + plt.legend() + plt.xlabel("Number of GPUs/GCDs") + plt.ylabel("Performance (cells / second / GPU)") + plt.title(title) diff --git a/doc/source/programmingguide.rst b/doc/source/programmingguide.rst index 18795b50a..4bd7c69f7 100644 --- a/doc/source/programmingguide.rst +++ b/doc/source/programmingguide.rst @@ -645,6 +645,27 @@ It may also be useful to implement debug-only safeguards with custom logic that fit `RUNTIME_CHECK_*` macros. This can be achieved by using the compiler directive `#ifdef RUNTIME_CHECKS` directly. +Dump an array to a file +----------------------- + +It is usually difficult to know what Idefix arrays effectively contains, especially when running on GPU. +To help with this difficulty, Idefix provides a global function ``DumpArray`` that can be used +to dump a ``IdefixArray`` to a numpy file (that can be read from python). This feature can be used +for debugging purpose as: + + +.. code-block:: c++ + + #include "idefix.hpp" + + IdefixArray3D myArray("debugMe",10,10,10); + + idfx::DumpArray("myFilename.npy",myArray); // Dump the array content to a numpy file named "myFilename.npy" + + +Note that the array is automatically transfered from the GPU, if needed. + + Minimal skeleton ================ diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index c9f9c5785..1fa05cc67 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -388,8 +388,10 @@ This section describes the outputs *Idefix* produces. For more details about eac +================+=========================+==================================================================================================+ | log | integer | | Time interval between log outputs, in code steps (default 100). | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ -| dmp | float | | Time interval between dump outputs, in code units. | -| | | | If negative, periodic dump outputs are disabled. | +| dmp | float, float+char | | 1st parameter: Code time interval between dump outputs, in code units. | +| | | | If negative, the first parameter is ignored. | +| | | | 2nd parameter (optional): Wallclock time interval between two dumps. The ending character | +| | | | can be "s" (seconds) "m" (minutes) "h" (hours) or "d" (days) | +----------------+-------------------------+--------------------------------------------------------------------------------------------------+ | dmp_dir | string | | directory for dump file outputs. Default to "./" | | | | | The directory is automatically created if it does not exist. | diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index aadc205c3..23052d92c 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -111,7 +111,8 @@ We recommend the following modules and environement variables on AdAstra: module load cpe/24.07 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray - module load amd-mixed + module load amd-mixed/6.1.2 + module load rocm/6.1.2 module load cray-python/3.11.7 Finally, *Idefix* can be configured to run on Mi250 by enabling HIP and the desired architecture with the following options to ccmake: diff --git a/reference b/reference index b675bceaa..c32894208 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit b675bceaa6aabc01dded346e2d631857f698dc76 +Subproject commit c328942083b618a85b84eb16ce9fd35e67c00597 diff --git a/src/dataBlock/coarsen.cpp b/src/dataBlock/coarsen.cpp index 2da3049cb..4c6b60ccd 100644 --- a/src/dataBlock/coarsen.cpp +++ b/src/dataBlock/coarsen.cpp @@ -66,27 +66,33 @@ void DataBlock::CheckCoarseningLevels() { idfx::pushRegion("DataBlock::CheckCoarseningLevels()"); // Check that the coarsening levels we have are valid // NB: this is a costly procedure, we can't repeat it at each loop! - DataBlockHost d(*this); - d.SyncFromDevice(); for(int dir = 0 ; dir < DIMENSIONS ; dir++) { if(mygrid->coarseningDirection[dir]) { - IdefixHostArray2D arr = d.coarseningLevel[dir]; - const int Xt = (dir == IDIR ? JDIR : IDIR); - const int Xb = (dir == KDIR ? JDIR : KDIR); - for(int i = beg[Xt] ; i < end[Xt] ; i++) { - for(int j = beg[Xb] ; j < end[Xb] ; j++) { + IdefixHostArray2D arr = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), + coarseningLevel[dir]); + for(int j = 0 ; j < arr.extent(0) ; j++) { + for(int i = 0 ; i < arr.extent(1) ; i++) { + if(std::isnan(arr(j,i))) { + std::stringstream str; + str << "Nan in grid coarsening levels" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): Coarsening level is NaN!" << std::endl; + IDEFIX_ERROR(str); + } if(arr(j,i) < 1) { std::stringstream str; - str << "Incorrect grid coarsening levels" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level < 1!" << std::endl; + str << "Coarsening level < 1!" << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; IDEFIX_ERROR(str); } const int factor = 1 << (arr(j,i) - 1); if(np_int[dir] % factor != 0) { std::stringstream str; - str << "local grid size not divisible by coarsening level" << std::endl; - str << "at (i,j)=("<< i << "," << j << "): Coarsening level: "; - str << arr(j,i) << std::endl; + str << "Local grid size not divisible by coarsening level." << std::endl; + str << "at (i,j)=("<< i << "," << j << "): "; + str << "coarsening level= " << arr(j,i) << std::endl; + str << np_int[dir] << " cannot be divided by 2^" << arr(j,i)-1; + str << " = " << factor << std::endl; IDEFIX_ERROR(str); } } diff --git a/src/dataBlock/evolveStage.cpp b/src/dataBlock/evolveStage.cpp index 981c060ae..c97c9f713 100644 --- a/src/dataBlock/evolveStage.cpp +++ b/src/dataBlock/evolveStage.cpp @@ -19,6 +19,16 @@ void DataBlock::EvolveStage() { for(int i = 0 ; i < dust.size() ; i++) { dust[i]->EvolveStage(this->t,this->dt); } + // Add implicit term for dust drag + if(dust[0]->drag->IsImplicit()) { + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->drag->AddImplicitBackReaction(this->dt,dust[0]->drag->implicitFactor); + } + dust[0]->drag->NormalizeImplicitBackReaction(this->dt); + for(int i = 0 ; i < dust.size() ; i++) { + dust[i]->drag->AddImplicitFluidMomentum(this->dt); + } + } } idfx::popRegion(); diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index b77fe3b5f..3ad67f0a0 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -128,12 +128,12 @@ void Axis::RegularizeCurrentSide(int side) { real deltaPhi = data->mygrid->xend[KDIR] - data->mygrid->xbeg[KDIR]; // Use the circulation around the pole of Bphi to determine Jr on the pole: - // Delta phi r^2(1-cos theta) Jr = int r sin(theta) Bphi dphi + // 1/2*Delta phi r^2 sin theta^2 Jr = int r sin(theta) Bphi dphi idefix_for("fixJ",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { real th = x2(jc); - real fact = sign*sin(th)/(deltaPhi*x1(i)*(1-cos(th))); + real fact = 2*sign/(deltaPhi*x1(i)*sin(th)); J(IDIR, k,js,i) = BAvg(i)*fact; }); diff --git a/src/fluid/braginskii/bragThermalDiffusion.cpp b/src/fluid/braginskii/bragThermalDiffusion.cpp index 82d5dac94..f251d0b0c 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.cpp +++ b/src/fluid/braginskii/bragThermalDiffusion.cpp @@ -19,7 +19,6 @@ #include "eos.hpp" - void BragThermalDiffusion::ShowConfig() { if(status.status==Constant) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with constant diffusivity kpar=" @@ -27,11 +26,11 @@ void BragThermalDiffusion::ShowConfig() { } else if (status.status==UserDefFunction) { idfx::cout << "Braginskii Thermal Diffusion: ENABLED with user-defined diffusivity function." << std::endl; - if(!diffusivityFunc) { + if(!bragDiffusivityFunc) { IDEFIX_ERROR("No braginskii thermal diffusion function has been enrolled"); } } else { - IDEFIX_ERROR("Unknown braginskii thermal diffusion mode"); + IDEFIX_ERROR("Unknown Braginskii thermal diffusion mode"); } if(status.isExplicit) { idfx::cout << "Braginskii Thermal Diffusion: uses an explicit time integration." << std::endl; @@ -42,16 +41,27 @@ void BragThermalDiffusion::ShowConfig() { IDEFIX_ERROR("Unknown time integrator for braginskii thermal diffusion."); } if(haveSlopeLimiter) { - idfx::cout << "Braginskii Thermal Diffusion: uses a slope limiter." << std::endl; + if(haveMonotonizedCentral) { + idfx::cout << "Braginskii Thermal Diffusion: " + "uses the monotonized central slope limiter." << std::endl; + } else if(haveVanLeer) { + idfx::cout << "Braginskii Thermal Diffusion: uses the van Leer slope limiter." << std::endl; + } else { + IDEFIX_ERROR("Unknown slope limiter for braginskii thermal diffusion."); + } + } + if(includeCollisionlessTD) { + idfx::cout << "Braginskii Thermal Diffusion: saturation" + " with collisionless flux is enabled." << std::endl; } } void BragThermalDiffusion::EnrollBragThermalDiffusivity(BragDiffusivityFunc myFunc) { if(this->status.status != UserDefFunction) { IDEFIX_WARNING("Braginskii thermal diffusivity enrollment requires Hydro/BragThermalDiffusion " - "to be set to userdef in .ini file"); + "to be set to userdef in .ini file"); } - this->diffusivityFunc = myFunc; + this->bragDiffusivityFunc = myFunc; } void BragThermalDiffusion::AddBragDiffusiveFlux(int dir, const real t, diff --git a/src/fluid/braginskii/bragThermalDiffusion.hpp b/src/fluid/braginskii/bragThermalDiffusion.hpp index 1fe774a41..116ff9a03 100644 --- a/src/fluid/braginskii/bragThermalDiffusion.hpp +++ b/src/fluid/braginskii/bragThermalDiffusion.hpp @@ -9,6 +9,7 @@ #define FLUID_BRAGINSKII_BRAGTHERMALDIFFUSION_HPP_ #include +#include #include "idefix.hpp" #include "input.hpp" @@ -40,6 +41,8 @@ class BragThermalDiffusion { IdefixArray3D heatSrc; // Source terms of the thermal operator IdefixArray3D knorArr; IdefixArray3D kparArr; + IdefixArray3D clessAlphaArr; // Transition from Brag to collisionless tc + IdefixArray3D clessBetaArr; // Collisionless tc flux coefficient (before pv) // pre-computed geometrical factors in non-cartesian geometry IdefixArray1D one_dmu; @@ -50,9 +53,14 @@ class BragThermalDiffusion { // status of the module ParabolicModuleStatus &status; - BragDiffusivityFunc diffusivityFunc; + BragDiffusivityFunc bragDiffusivityFunc; + + bool includeCollisionlessTD{false}; bool haveSlopeLimiter{false}; + bool haveMonotonizedCentral{false}; + bool haveVanLeer{false}; + bool haveMinmod{false}; // helper array IdefixArray4D &Vc; @@ -61,6 +69,7 @@ class BragThermalDiffusion { // constant diffusion coefficient (when needed) real knor, kpar; + real clessAlpha, clessBeta; // equation of state (required to get the heat capacity) EquationOfState *eos; @@ -84,9 +93,11 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid if(input.CheckEntry("Hydro","bragTDiffusion")>=0) { if(input.Get("Hydro","bragTDiffusion",1).compare("mc") == 0) { this->haveSlopeLimiter = true; + this->haveMonotonizedCentral = true; limiter = PLMLimiter::McLim; } else if(input.Get("Hydro","bragTDiffusion",1).compare("vanleer") == 0) { this->haveSlopeLimiter = true; + this->haveVanLeer = true; limiter = PLMLimiter::VanLeer; } else if(input.Get("Hydro","bragTDiffusion",1).compare("minmod") == 0) { IDEFIX_ERROR("The minmod slope limiter is not available because it has been " @@ -98,21 +109,55 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid IDEFIX_ERROR("Unknown braginskii thermal diffusion limiter in idefix.ini. " "Can only be vanleer, mc or nolimiter."); } - if(input.Get("Hydro","bragTDiffusion",2).compare("constant") == 0) { - this->kpar = input.Get("Hydro","bragTDiffusion",3); - this->knor = input.GetOrSet("Hydro","bragTDiffusion",4,0.); - this->status.status = Constant; - } else if(input.Get("Hydro","bragTDiffusion",2).compare("userdef") == 0) { - this->status.status = UserDefFunction; - this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); - this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], - data->np_tot[JDIR], - data->np_tot[IDIR]); + if(input.Get("Hydro","bragTDiffusion",2).compare("nosat") == 0) { + if(input.Get("Hydro","bragTDiffusion",3).compare("constant") == 0) { + this->kpar = input.Get("Hydro","bragTDiffusion",4); + this->knor = input.GetOrSet("Hydro","bragTDiffusion",5,0.); + this->status.status = Constant; + } else if(input.Get("Hydro","bragTDiffusion",3).compare("userdef") == 0) { + this->status.status = UserDefFunction; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } + } else if(input.Get("Hydro","bragTDiffusion",2).compare("wcless") == 0.0) { + if(input.Get("Hydro","bragTDiffusion",3).compare("constant") == 0) { + this->includeCollisionlessTD = true; + this->kpar = input.Get("Hydro","bragTDiffusion",4); + this->knor = input.GetOrSet("Hydro","bragTDiffusion",5,0.); + this->clessAlpha = input.GetOrSet("Hydro","bragTDiffusion",6,1.); + this->clessBeta = input.GetOrSet("Hydro","bragTDiffusion",7,0.); + this->status.status = Constant; + } else if(input.Get("Hydro","bragTDiffusion",3).compare("userdef") == 0) { + this->includeCollisionlessTD = true; + this->status.status = UserDefFunction; + this->kparArr = IdefixArray3D("BragThermalDiffusionKparArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->knorArr = IdefixArray3D("BragThermalDiffusionKnorArray",data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessAlphaArr = IdefixArray3D("ClessThermalDiffusionAlphaArray", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + this->clessBetaArr = IdefixArray3D("ClessThermalDiffusionBetaArray", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + IDEFIX_ERROR("Unknown braginskii/collisionless thermal diffusion definition in idefix.ini. " + "Can only be constant or userdef."); + } } else { - IDEFIX_ERROR("Unknown braginskii thermal diffusion definition in idefix.ini. " - "Can only be constant or userdef."); + IDEFIX_ERROR("Unknown braginskii thermal diffusion saturation in idefix.ini. " + "Can only be nosat or wcless."); } } else { IDEFIX_ERROR("I cannot create a BragThermalDiffusion object without bragTDiffusion defined" @@ -135,7 +180,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid // and therefore not available as such in the DataBlock. // It is rather defined as PRS/RHO. // Special spatial derivative macros are therefore needed and defined here -// directly at the right cell interface according to the direciton of the flux. +// directly at the right cell interface according to the direction of the flux. #define D_DX_I_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j,i - 1)/q(RHO,k,j,i - 1)) #define D_DY_J_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k,j - 1,i)/q(RHO,k,j - 1,i)) #define D_DZ_K_T(q) (q(PRS,k,j,i)/q(RHO,k,j,i) - q(PRS,k - 1,j,i)/q(RHO,k - 1,j,i)) @@ -179,7 +224,7 @@ BragThermalDiffusion::BragThermalDiffusion(Input &input, Grid &grid, Fluid //We now define spatial average macros for the magnetic field. // The magnetic field appears in the expression of the Braginskii heat flux. -// It is therefore needed at the right cell interface according to the direction of the flux. +// It is therefore needed at the left cell interface according to the direction of the flux. #define BX_I Vs(BX1s,k,j,i) #define BY_J Vs(BX2s,k,j,i) #define BZ_K Vs(BX3s,k,j,i) @@ -210,6 +255,7 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, HydroModuleStatus haveThermalDiffusion = this->status.status; bool haveSlopeLimiter = this->haveSlopeLimiter; + bool includeCollisionlessTD = this->includeCollisionlessTD; using SL = SlopeLimiter; int ibeg, iend, jbeg, jend, kbeg, kend; @@ -250,16 +296,33 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, #endif real knorConstant = this->knor; real kparConstant = this->kpar; + real clessAlphaConst = this->clessAlpha; + real clessBetaConst = this->clessBeta; IdefixArray3D knorArr = this->knorArr; IdefixArray3D kparArr = this->kparArr; + IdefixArray3D clessAlphaArr = this->clessAlphaArr; + IdefixArray3D clessBetaArr = this->clessBetaArr; - if(haveThermalDiffusion == UserDefFunction && dir == IDIR) { - if(diffusivityFunc) { + if(includeCollisionlessTD == false && haveThermalDiffusion == UserDefFunction && dir == IDIR) { + if(bragDiffusivityFunc) { idfx::pushRegion("UserDef::BragThermalDiffusivityFunction"); - diffusivityFunc(*this->data, t, kparArr, knorArr); + std::vector> userdefArr = {kparArr, knorArr}; + bragDiffusivityFunc(*this->data, t, userdefArr); + idfx::popRegion(); + } + else { + IDEFIX_ERROR("No user-defined Braginskii thermal diffusion function has been enrolled"); + } + } else if(includeCollisionlessTD == true && haveThermalDiffusion == UserDefFunction + && dir == IDIR) { + if (bragDiffusivityFunc) { + idfx::pushRegion("UserDef::ClessThermalDiffusivityFunction"); + std::vector> userdefArr = {kparArr, knorArr, clessAlphaArr, clessBetaArr}; + bragDiffusivityFunc(*this->data, t, userdefArr); idfx::popRegion(); } else { - IDEFIX_ERROR("No user-defined thermal diffusion function has been enrolled"); + IDEFIX_ERROR("No user-defined Braginskii/collisionless " + "thermal diffusion function has been enrolled"); } } @@ -273,6 +336,8 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, [[maybe_unused]] real dTi, dTj, dTk, dTn; dTi = dTj = dTk = dTn = ZERO_F; + /* For the collisionless / saturated flux */ + [[maybe_unused]] real clessAlpha, clessBeta, dvp, dvm, dpp, dpm, Pn, Vn; real locdmax = 0; /////////////////////////////////////////// @@ -287,14 +352,22 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } else { kpar = HALF_F*(kparArr(k,j,i-1)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k,j,i-1)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j,i-1)+clessBetaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } - EXPAND( Bi = BX_I; , - Bj = BY_I; , - Bk = BZ_I; ) + D_EXPAND( Bi = BX_I; , + Bj = BY_I; , + Bk = BZ_I; ) Bn = BX_I; #if GEOMETRY == CARTESIAN @@ -370,7 +443,9 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } dTi = D_DX_I_T(Vc)/dx1(i); + #if DIMENSIONS >= 2 + if (haveSlopeLimiter) { dTj = 1./x1(i-1)*SL::PLMLim(SL_DY_T(Vc,k,j,i-1)/dx2(j), SL_DY_T(Vc,k,j+1,i-1)/dx2(j+1)); @@ -399,6 +474,30 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j,i - 1)))*gamma_m1; dTn = dTi; + + /* For collisionless / saturated heat flux */ + if (includeCollisionlessTD) { + if(haveSlopeLimiter) { + dvp = Vc(VX1,k,j,i+1) - Vc(VX1,k,j,i); + dvm = Vc(VX1,k,j,i) - Vc(VX1,k,j,i-1); + + dpp = Vc(PRS,k,j,i+1) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j,i-1); + + /* Upwind scheme */ + if (Vc(VX1,k,j,i) > 0.0) { + Vn = Vc(VX1,k,j,i-1)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i-1)+HALF_F*SL::PLMLim(dpm, dpp); + } + else { + Vn = Vc(VX1,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } + } else { + Vn = HALF_F*(Vc(VX1,k,j,i) + Vc(VX1,k,j,i-1)); + Pn = HALF_F*(Vc(PRS,k,j,i) + Vc(PRS,k,j,i-1)); + } + } } else if(dir == JDIR) { ////////////// // JDIR @@ -411,9 +510,17 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } else { kpar = HALF_F*(kparArr(k,j-1,i)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k,j-1,i)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k,j-1,i)+clessBetaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } EXPAND( Bi = BX_J; , @@ -533,6 +640,29 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k,j - 1,i)))*gamma_m1; dTn = dTj; + + /* For the collisionless / saturated flux */ + if(includeCollisionlessTD) { + if(haveSlopeLimiter) { + dvp = Vc(VX2,k,j+1,i) - Vc(VX2,k,j,i); + dvm = Vc(VX2,k,j,i) - Vc(VX2,k,j-1,i); + + dpp = Vc(PRS,k,j+1,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k,j-1,i); + + /* Upwind scheme */ + if (Vc(VX2,k,j,i) > 0.0) { + Vn = Vc(VX2,k,j-1,i)+HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j-1,i)+HALF_F*SL::PLMLim(dpm, dpp); + } else { + Vn = Vc(VX2,k,j,i)-HALF_F*SL::PLMLim(dvm, dvp); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpm, dpp); + } + } else { + Vn = HALF_F*(Vc(VX2,k,j-1,i) + Vc(VX2,k,j,i)); + Pn = HALF_F*(Vc(PRS,k,j-1,i) + Vc(PRS,k,j,i)); + } + } } else if(dir == KDIR) { ////////////// // KDIR @@ -544,9 +674,17 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, } else { kpar = HALF_F*(kparArr(k-1,j,i)+kparArr(k,j,i)); } + if(includeCollisionlessTD) { + clessAlpha=HALF_F*(clessAlphaArr(k-1,j,i)+clessAlphaArr(k,j,i)); + clessBeta=HALF_F*(clessBetaArr(k-1,j,i)+clessBetaArr(k,j,i)); + } } else { knor = knorConstant; kpar = kparConstant; + if(includeCollisionlessTD) { + clessAlpha=clessAlphaConst; + clessBeta=clessBetaConst; + } } Bi = BX_K; @@ -625,11 +763,36 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, locdmax = FMAX(kpar,knor)/(0.5*(Vc(RHO,k,j,i) + Vc(RHO,k-1,j,i)))*gamma_m1; dTn = dTk; + + /* For the collisionless / saturated flux */ + if(includeCollisionlessTD) { + if(haveSlopeLimiter) { + dvp = Vc(VX3,k+1,j,i) - Vc(VX3,k,j,i); + dvm = Vc(VX3,k,j,i) - Vc(VX3,k-1,j,i); + + dpp = Vc(PRS,k+1,j,i) - Vc(PRS,k,j,i); + dpm = Vc(PRS,k,j,i) - Vc(PRS,k-1,j,i); + + /* Upwind scheme */ + if (Vc(VX3,k,j,i) > 0.0) { + Vn = Vc(VX3,k-1,j,i)+HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k-1,j,i)+HALF_F*SL::PLMLim(dpp, dpm); + } else { + Vn = Vc(VX3,k,j,i)-HALF_F*SL::PLMLim(dvp, dvm); + Pn = Vc(PRS,k,j,i)-HALF_F*SL::PLMLim(dpp, dpm); + } + } else { + Vn = HALF_F*(Vc(VX3,k-1,j,i) + Vc(VX3,k,j,i)); + Pn = HALF_F*(Vc(PRS,k-1,j,i) + Vc(PRS,k,j,i)); + } + } } // From here, gradients and normal have been computed, so we just need to get the fluxes - bgradT = EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); - Bmag = EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + bgradT = D_EXPAND( Bi*dTi , + Bj*dTj, +Bk*dTk); + Bmag = D_EXPAND( Bi*Bi , + Bj*Bj, + Bk*Bk); + // EXPAND can yield unexpected behaviour when DIMENSIONS < COMPONENTS + //printf("%f , %f\n", Bmag, EXPAND(Bi*Bi, + Bj*Bj, + Bk*Bk)); Bmag = sqrt(Bmag); Bmag = FMAX(1e-6*SMALL_NUMBER,Bmag); @@ -637,10 +800,15 @@ void BragThermalDiffusion::AddBragDiffusiveFluxLim(int dir, const real t, bn = Bn/Bmag; /* -- unit vector component -- */ q = kpar*bgradT*bn + knor*(dTn - bn*bgradT); - + if(includeCollisionlessTD) { + q = clessAlpha*q + (1-clessAlpha)*clessBeta*Pn*Vn; + } Flux(ENG, k, j, i) -= q; - dMax(k,j,i) = FMAX(dMax(k,j,i),locdmax); + + if(includeCollisionlessTD) { + dMax(k,j,i) *= clessAlpha; + } }); idfx::popRegion(); } diff --git a/src/fluid/braginskii/bragViscosity.cpp b/src/fluid/braginskii/bragViscosity.cpp index 6d5007b50..e53ab2c65 100644 --- a/src/fluid/braginskii/bragViscosity.cpp +++ b/src/fluid/braginskii/bragViscosity.cpp @@ -59,7 +59,14 @@ void BragViscosity::ShowConfig() { IDEFIX_ERROR("Unknown time integrator for braginskii viscosity."); } if(haveSlopeLimiter) { - idfx::cout << "Braginskii Viscosity: uses a slope limiter." << std::endl; + if(haveMonotizedCentral) { + idfx::cout << "Braginskii Viscosity: uses the " + "monotonized central slope limiter." << std::endl; + } else if(haveVanLeer) { + idfx::cout << "Braginskii Viscosity: uses the van Leer slope limiter." << std::endl; + } else { + IDEFIX_ERROR("Unknown slope limiter for braginskii viscosity."); + } } #if GEOMETRY == CYLINDRICAL || GEOMETRY == POLAR diff --git a/src/fluid/braginskii/bragViscosity.hpp b/src/fluid/braginskii/bragViscosity.hpp index 7a8d2b756..5340110a8 100644 --- a/src/fluid/braginskii/bragViscosity.hpp +++ b/src/fluid/braginskii/bragViscosity.hpp @@ -52,6 +52,8 @@ class BragViscosity { DiffusivityFunc bragViscousDiffusivityFunc; bool haveSlopeLimiter{false}; + bool haveMonotizedCentral{false}; + bool haveVanLeer{false}; IdefixArray4D &Vc; IdefixArray4D &Vs; @@ -78,9 +80,11 @@ BragViscosity::BragViscosity(Input &input, Grid &grid, Fluid *hydroin): if(input.CheckEntry("Hydro","bragViscosity")>=0) { if(input.Get("Hydro","bragViscosity",1).compare("vanleer") == 0) { this->haveSlopeLimiter = true; + this->haveVanLeer = true; limiter = PLMLimiter::VanLeer; } else if(input.Get("Hydro","bragViscosity",1).compare("mc") == 0) { this->haveSlopeLimiter = true; + this->haveMonotizedCentral = true; limiter = PLMLimiter::McLim; } else if (input.Get("Hydro","bragViscosity",1).compare("nolimiter") == 0) { this->haveSlopeLimiter = false; diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp index c8738b4b2..38ef76d5d 100644 --- a/src/fluid/calcRightHandSide.hpp +++ b/src/fluid/calcRightHandSide.hpp @@ -149,47 +149,49 @@ struct Fluid_CorrectFluxFunctor { Flux(MX1+meanDir,k,j,i) += meanV * Flux(RHO,k,j,i); } // Fargo & Rotation corrections - real Ax = A(k,j,i); + ////////////////////////////////////////////// + // Define correction factor for the fluxes + ////////////////////////////////////////////// - for(int nv = 0 ; nv < Phys::nvar ; nv++) { - Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; - } + real Ax[Phys::nvar]; // corrected Area + for(int nv = 0 ; nv < Phys::nvar ; nv++) Ax[nv] = A(k,j,i); // Curvature terms -#if (GEOMETRY == POLAR && COMPONENTS >= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if constexpr (dir==IDIR) { - // Conserve angular momentum, hence flux is R*Bphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - if constexpr(Phys::mhd) { - if(Ax= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if constexpr (dir==IDIR) { + // Conserve angular momentum, hence flux is R*Bphi + Ax[iMPHI] *= FABS(x1m(i)); + if constexpr(Phys::mhd) { + Ax[iBPHI] = 1.0; // corrected Flux is simply Bphi + } } - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL + #endif // GEOMETRY==POLAR OR CYLINDRICAL -#if GEOMETRY == SPHERICAL - if constexpr(dir==IDIR) { - #if COMPONENTS == 3 - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - #endif // COMPONENTS == 3 - if constexpr(Phys::mhd) { - if(Ax::CheckDivB() { data->beg[IDIR], data->end[IDIR], KOKKOS_LAMBDA (int k, int j, int i, real &divBmax) { [[maybe_unused]] real dB1,dB2,dB3; + [[maybe_unused]] real d1, d2, d3; + [[maybe_unused]] real B1,B2,B3; dB1=dB2=dB3=ZERO_F; + d1=d2=d3=ZERO_F; + B1=B2=B3=ZERO_F; + D_EXPAND( dB1=(Ax1(k,j,i+1)*Vs(BX1s,k,j,i+1)-Ax1(k,j,i)*Vs(BX1s,k,j,i)); , dB2=(Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i)-Ax2(k,j,i)*Vs(BX2s,k,j,i)); , dB3=(Ax3(k+1,j,i)*Vs(BX3s,k+1,j,i)-Ax3(k,j,i)*Vs(BX3s,k,j,i)); ) - divBmax=FMAX(FABS(D_EXPAND(dB1, +dB2, +dB3))/dV(k,j,i),divBmax); + D_EXPAND( d1=0.5*(Ax1(k,j,i+1) + Ax1(k,j,i)); , + d2=0.5*(Ax2(k,j+1,i) + Ax2(k,j,i)); , + d3=0.5*(Ax3(k+1,j,i) + Ax3(k,j,i)); ) + + D_EXPAND( B1=0.5*(Vs(BX1s,k,j,i+1) + Vs(BX1s,k,j,i)); , + B2=0.5*(Vs(BX2s,k,j+1,i) + Vs(BX2s,k,j,i)); , + B3=0.5*(Vs(BX3s,k+1,j,i) + Vs(BX3s,k,j,i)); ) + + real amplitude = 1e-40; + amplitude += D_EXPAND( std::fabs(B1)*d1, + std::fabs(B2)*d2, + std::fabs(B3)*d3 ); + + divBmax=FMAX(FABS(D_EXPAND(dB1, +dB2, +dB3))/amplitude,divBmax); }, Kokkos::Max(divB) // reduction ); diff --git a/src/fluid/drag.cpp b/src/fluid/drag.cpp index d24e0bea7..4cf879cdd 100644 --- a/src/fluid/drag.cpp +++ b/src/fluid/drag.cpp @@ -5,7 +5,9 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** #include "drag.hpp" +#include #include "physics.hpp" + void Drag::AddDragForce(const real dt) { idfx::pushRegion("Drag::AddDragForce"); @@ -15,83 +17,182 @@ void Drag::AddDragForce(const real dt) { auto VcDust = this->VcDust; auto InvDt = this->InvDt; - const Type type = this->type; - real dragCoeff = this->dragCoeff; bool feedback = this->feedback; - EquationOfState eos = *(this->eos); - - auto userGammai = this->gammai; - if(type == Type::Userdef) { - if(userDrag != NULL) { - idfx::pushRegion("Drag::UserDrag"); - userDrag(data, dragCoeff, userGammai); - idfx::popRegion(); - } else { - IDEFIX_ERROR("No User-defined drag function has been enrolled"); - } + if(implicit) { + IDEFIX_ERROR("Add DragForce should not be called when drag is implicit"); } + + auto gammaDrag = this->gammaDrag; + gammaDrag.RefreshUserDrag(data); + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) // Where gamma is computed according to the choice of drag type idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { - real gamma; // The drag coefficient - if(type == Type::Gamma) { - gamma = dragCoeff; - - } else if(type == Type::Tau) { - // In this case, the coefficient is the stopping time (assumed constant) - gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); - } else if(type == Type::Size) { - real cs; - // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs - // Get the sound speed - #if HAVE_ENERGY == 1 - cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) - *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); - #else - cs = eos.GetWaveSpeed(k,j,i); - #endif - gamma = cs/dragCoeff; - } else if(type == Type::Userdef) { - gamma = userGammai(k,j,i); - } + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient real dp = dt * gamma * VcDust(RHO,k,j,i) * VcGas(RHO,k,j,i); for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { real dv = VcDust(n,k,j,i) - VcGas(n,k,j,i); UcDust(n,k,j,i) -= dp*dv; - if(feedback) UcGas(n,k,j,i) += dp*dv; + if(feedback) { + UcGas(n,k,j,i) += dp*dv; + #if HAVE_ENERGY == 1 + // We add back the energy dissipated for the dust which is not accounted for + // (since there is no energy equation for dust grains) + + // TODO(GL): this should be disabled in the case of a true multifluid system where + // both fluids have a proper energy equation + UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); + #endif + } // feedback + } + // Cfl constraint + real idt = gamma*VcGas(RHO,k,j,i); + if(feedback) idt += gamma*VcDust(RHO,k,j,i); + InvDt(k,j,i) += idt; + }); + idfx::popRegion(); +} +// +// $ p_g^{(n+1)}=p_g^{(n)}+\sum_i\frac{\rho_g\gamma_i dt}{1+\rho_g \gamma_i dt}p_i^{(n)} $ +// We accumulate in the array "prefactor" +// $ \sum_i\frac{\rho_i\gamma_i dt}{1+\rho_g\gamma_i dt} +// + +void Drag::AddImplicitBackReaction(const real dt, IdefixArray3D preFactor) { + if(!feedback) { + // no feedback, no need for anything + return; + } + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto VcGas = this->VcGas; + auto VcDust = this->VcDust; + auto UcDust = this->UcDust; + + bool isFirst = this->instanceNumber == 0; + + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + auto gammaDrag = this->gammaDrag; + gammaDrag.RefreshUserDrag(data); + + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient + + + const real factor = UcDust(RHO,k,j,i)*gamma*dt/(1+UcGas(RHO,k,j,i)*gamma*dt); + if(isFirst) { + preFactor(k,j,i) = factor; + } else { + preFactor(k,j,i) += factor; + } + + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + UcGas(n,k,j,i) += dt * gamma * UcGas(RHO,k,j,i) * UcDust(n,k,j,i) / + (1 + UcGas(RHO,k,j,i)*dt*gamma); + } + }); + idfx::popRegion(); +} + +void Drag::NormalizeImplicitBackReaction(const real dt) { + if(!feedback) { + // no feedback, no need for anything + return; + } + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto preFactor = this->implicitFactor; + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + const real factor = 1+preFactor(k,j,i); + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + UcGas(n,k,j,i) /= factor; + } + }); + idfx::popRegion(); +} + +void Drag::AddImplicitFluidMomentum(const real dt) { + idfx::pushRegion("AddImplicitFluidMomentum"); + + auto UcGas = this->UcGas; + auto VcGas = this->VcGas; + auto UcDust = this->UcDust; + auto VcDust = this->VcDust; + auto InvDt = this->InvDt; + + bool feedback = this->feedback; + + if(!implicit) { + IDEFIX_ERROR("AddImplicitGasMomentum should not be called when drag is explicit"); + } + + auto gammaDrag = this->gammaDrag; + if(!feedback) gammaDrag.RefreshUserDrag(data); + + // Compute a drag force fd = - gamma*rhod*rhog*(vd-vg) + // Where gamma is computed according to the choice of drag type + idefix_for("DragForce",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + real gamma = gammaDrag.GetGamma(k,j,i); // The drag coefficient + + for(int n = MX1 ; n < MX1+COMPONENTS ; n++) { + real oldUc = UcDust(n,k,j,i); + UcDust(n,k,j,i) = (oldUc + dt * gamma * UcDust(RHO,k,j,i) * UcGas(n,k,j,i)) / + (1 + UcGas(RHO,k,j,i)*dt*gamma); + #if HAVE_ENERGY == 1 + real dp = UcDust(n,k,j,i) - oldUc; + // We add back the energy dissipated for the dust which is not accounted for // (since there is no energy equation for dust grains) // TODO(GL): this should be disabled in the case of a true multifluid system where // both fluids have a proper energy equation - UcGas(ENG,k,j,i) += dp*dv*VcDust(n,k,j,i); + if(feedback) UcGas(ENG,k,j,i) -= dp*VcDust(n,k,j,i); #endif } - // Cfl constraint - real idt = gamma*VcGas(RHO,k,j,i); - if(feedback) idt += gamma*VcDust(RHO,k,j,i); - InvDt(k,j,i) += idt; }); idfx::popRegion(); } void Drag::ShowConfig() { idfx::cout << "Drag: Using "; - switch(type) { - case Type::Gamma: + if(implicit) { + idfx::cout << " IMPLICIT "; + } else { + idfx::cout << " EXPLICIT "; + } + switch(gammaDrag.type) { + case GammaDrag::Type::Gamma: idfx::cout << "constant gamma"; break; - case Type::Tau: + case GammaDrag::Type::Tau: idfx::cout << "constant stopping time"; break; - case Type::Size: + case GammaDrag::Type::Size: idfx::cout << "constant dust size"; break; - case Type::Userdef: + case GammaDrag::Type::Userdef: idfx::cout << "user-defined"; break; } @@ -105,8 +206,60 @@ void Drag::ShowConfig() { } void Drag::EnrollUserDrag(UserDefDragFunc func) { + gammaDrag.EnrollUserDrag(func); +} + +//////////////////////////////////////////// +// GammaDrag function definitions +//////////////////////////////////////////// + +void GammaDrag::RefreshUserDrag(DataBlock *data) { + if(type == Type::Userdef) { + if(userDrag != NULL) { + idfx::pushRegion("GammaDrag::UserDrag"); + userDrag(data, instanceNumber, dragCoeff, gammai); + idfx::popRegion(); + } else { + IDEFIX_ERROR("No User-defined drag function has been enrolled"); + } + } +} + +void GammaDrag::EnrollUserDrag(UserDefDragFunc func) { if(type != Type::Userdef) { IDEFIX_ERROR("User-defined drag function requires drag entry to be set to \"userdef\""); } this->userDrag = func; } + +GammaDrag::GammaDrag(Input &input, std::string BlockName, int instanceNumber, DataBlock *data) { + if(input.CheckEntry(BlockName,"drag")>=0) { + std::string dragType = input.Get(BlockName,"drag",0); + if(dragType.compare("gamma") == 0) { + this->type = Type::Gamma; + } else if(dragType.compare("tau") == 0) { + this->type = Type::Tau; + this->VcGas = data->hydro->Vc; + } else if(dragType.compare("size") == 0) { + this->type = Type::Size; + this->eos = *(data->hydro->eos.get()); + this->VcGas = data->hydro->Vc; + } else if(dragType.compare("userdef") == 0) { + this->type = Type::Userdef; + this->gammai = IdefixArray3D("UserDrag", + data->np_tot[KDIR], + data->np_tot[JDIR], + data->np_tot[IDIR]); + } else { + std::stringstream msg; + msg << "Unknown drag type \"" << dragType + << "\" in your input file." << std::endl + << "Allowed values are: gamma, tau, epstein, stokes, userdef." << std::endl; + + IDEFIX_ERROR(msg); + } + } + // Fetch the drag coefficient for the current specie. + this->dragCoeff = input.Get(BlockName,"drag",instanceNumber+1); + this->instanceNumber = instanceNumber; +} diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp index d5f7a5a73..29eca3097 100644 --- a/src/fluid/drag.hpp +++ b/src/fluid/drag.hpp @@ -13,38 +13,88 @@ #include "fluid_defs.hpp" #include "eos.hpp" -using UserDefDragFunc = void (*) (DataBlock *, real beta, IdefixArray3D &gammai); +using UserDefDragFunc = void (*) (DataBlock *, int n, real beta, IdefixArray3D &gammai); + +/* A class that holds the kind of drag law we intend to use */ +class GammaDrag { + public: + enum class Type{Gamma, Tau, Size, Userdef}; + GammaDrag() = default; + GammaDrag(Input &, std::string BlockName, int instanceNumber, DataBlock *data); + void EnrollUserDrag(UserDefDragFunc); + void RefreshUserDrag(DataBlock *); + + KOKKOS_INLINE_FUNCTION real GetGamma(const int k, const int j, const int i) const { + real gamma; // The drag coefficient + if(type == Type::Gamma) { + gamma = dragCoeff; + + } else if(type == Type::Tau) { + // In this case, the coefficient is the stopping time (assumed constant) + gamma = 1/(dragCoeff*VcGas(RHO,k,j,i)); + } else if(type == Type::Size) { + real cs; + // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs + // Get the sound speed + #if HAVE_ENERGY == 1 + cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + #else + cs = eos.GetWaveSpeed(k,j,i); + #endif + gamma = cs/dragCoeff; + } else if(type == Type::Userdef) { + gamma = gammai(k,j,i); + } + return gamma; + } + + Type type; + real dragCoeff; + EquationOfState eos; + IdefixArray3D gammai; + IdefixArray4D VcGas; + + int instanceNumber; + UserDefDragFunc userDrag{NULL}; +}; class Drag { public: - enum class Type{Gamma, Tau, Size, Userdef}; // Different types of implementation for the drag force. template Drag(Input &, Fluid *); void ShowConfig(); // print configuration void AddDragForce(const real); + + ////////////////////////// + // Implicit functions + void AddImplicitBackReaction(const real, IdefixArray3D); // Add the back reaction + void NormalizeImplicitBackReaction(const real); // Normalize the implicit back reaction + void AddImplicitFluidMomentum(const real); // Add the implicit drag force on dust grains + ///////////////////////// + void EnrollUserDrag(UserDefDragFunc); // User defined drag function enrollment + bool IsImplicit() const { return implicit; } // Check if the drag is implicit IdefixArray4D UcDust; // Dust conservative quantities IdefixArray4D UcGas; // Gas conservative quantities IdefixArray4D VcDust; // Gas primitive quantities IdefixArray4D VcGas; // Gas primitive quantities IdefixArray3D InvDt; // The InvDt of current dust specie - IdefixArray3D gammai; // the drag coefficient (only used for user-defined dust grains) - Type type; + IdefixArray3D implicitFactor; // The prefactor used by the implicit timestepping + + GammaDrag gammaDrag; // The drag law private: DataBlock* data; - real dragCoeff; bool feedback{false}; - - UserDefDragFunc userDrag{NULL}; - - // Sound speed computation - EquationOfState *eos; + bool implicit{false}; + int instanceNumber; }; + #include "fluid.hpp" template @@ -58,45 +108,30 @@ Drag::Drag(Input &input, Fluid *hydroin): // Save the parent hydro object this->data = hydroin->data; - // We use the EOS of the basic fluid instance, as there is no EOS for dust! - this->eos = hydroin->data->hydro->eos.get(); // Check in which block we should fetch our information - std::string BlockName; + std::string blockName; if(Phys::dust) { - BlockName = "Dust"; + blockName = "Dust"; } else { IDEFIX_ERROR("Drag is currently implemented only for dusty fuids"); } - if(input.CheckEntry(BlockName,"drag")>=0) { - std::string dragType = input.Get(BlockName,"drag",0); - if(dragType.compare("gamma") == 0) { - this->type = Type::Gamma; - } else if(dragType.compare("tau") == 0) { - this->type = Type::Tau; - } else if(dragType.compare("size") == 0) { - this->type = Type::Size; - } else if(dragType.compare("userdef") == 0) { - this->type = Type::Userdef; - this->gammai = IdefixArray3D("UserDrag", + if(input.CheckEntry(blockName,"drag")>=0) { + this->instanceNumber = hydroin->instanceNumber; + this->gammaDrag = GammaDrag(input,blockName,instanceNumber,data); + + // Feedback is true by default, but can be switched off. + this->feedback = input.GetOrSet(blockName,"drag_feedback",0,true); + this->implicit = input.GetOrSet(blockName,"drag_implicit",0,false); + + if(implicit && instanceNumber == 0) { + this->implicitFactor = IdefixArray3D("ImplicitFactor", data->np_tot[KDIR], data->np_tot[JDIR], data->np_tot[IDIR]); - } else { - std::stringstream msg; - msg << "Unknown drag type \"" << dragType - << "\" in your input file." << std::endl - << "Allowed values are: gamma, tau, epstein, stokes, userdef." << std::endl; - - IDEFIX_ERROR(msg); } - // Fetch the drag coefficient for the current specie. - const int n = hydroin->instanceNumber; - this->dragCoeff = input.Get(BlockName,"drag",n+1); - // Feedback is true by default, but can be switched off. - this->feedback = input.GetOrSet(BlockName,"drag_feedback",0,true); } else { IDEFIX_ERROR("A [Drag] block is required in your input file to define the drag force."); } diff --git a/src/fluid/evolveStage.hpp b/src/fluid/evolveStage.hpp index 06754d0b7..31dae5b7d 100644 --- a/src/fluid/evolveStage.hpp +++ b/src/fluid/evolveStage.hpp @@ -61,7 +61,11 @@ void Fluid::EvolveStage(const real t, const real dt) { if(haveSourceTerms) AddSourceTerms(t, dt); // Step 5: add drag when needed - if(haveDrag) drag->AddDragForce(dt); + if(haveDrag) { + if(!drag->IsImplicit()) { + drag->AddDragForce(dt); + } + } if constexpr(Phys::mhd) { #if DIMENSIONS >= 2 diff --git a/src/fluid/fluid.hpp b/src/fluid/fluid.hpp index ae809b489..44cace6bf 100644 --- a/src/fluid/fluid.hpp +++ b/src/fluid/fluid.hpp @@ -585,52 +585,55 @@ Fluid::Fluid(Grid &grid, Input &input, DataBlock *datain, int n) { } for(int i = 0 ; i < Phys::nvar+nTracer ; i++) { - switch(i) { - case RHO: - VcName.push_back("RHO"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-RHO", RHO); - break; - case VX1: - VcName.push_back("VX1"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX1", VX1, IDIR); - break; - case VX2: - VcName.push_back("VX2"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX2", VX2, JDIR); - break; - case VX3: - VcName.push_back("VX3"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX3", VX3, KDIR); - break; - case BX1: - VcName.push_back("BX1"); - // never save cell-centered BX1 in dumps - break; - case BX2: - VcName.push_back("BX2"); - #if DIMENSIONS < 2 - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX2", BX2, JDIR); - #endif - break; - case BX3: - VcName.push_back("BX3"); - #if DIMENSIONS < 3 - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX3", BX3, KDIR); - #endif - break; - case PRS: - VcName.push_back("PRS"); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-PRS", PRS); - break; - default: - if(i>=Phys::nvar) { - std::string tracerLabel = std::string("TR")+std::to_string(i-Phys::nvar); // ="TRn" - VcName.push_back(tracerLabel); - data->dump->RegisterVariable(Vc, outputPrefix+"Vc-"+tracerLabel, i); - } else { + if(i < Phys::nvar) { + // These are standard variable names + switch(i) { + case RHO: + VcName.push_back("RHO"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-RHO", RHO); + break; + case VX1: + VcName.push_back("VX1"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX1", VX1, IDIR); + break; + case VX2: + VcName.push_back("VX2"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX2", VX2, JDIR); + break; + case VX3: + VcName.push_back("VX3"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-VX3", VX3, KDIR); + break; + case BX1: + VcName.push_back("BX1"); + // never save cell-centered BX1 in dumps + break; + case BX2: + VcName.push_back("BX2"); + #if DIMENSIONS < 2 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX2", BX2, JDIR); + #endif + break; + case BX3: + VcName.push_back("BX3"); + #if DIMENSIONS < 3 + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-BX3", BX3, KDIR); + #endif + break; + case PRS: + VcName.push_back("PRS"); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-PRS", PRS); + break; + default: + // unknown variable name (this should never happen, but who knows...) VcName.push_back("Vc-"+std::to_string(i)); data->vtk->RegisterVariable(Vc, outputPrefix+"Vc-"+std::to_string(i), i); - } + break; + } + } else { //if(i>=Phys::nvar) + std::string tracerLabel = std::string("TR")+std::to_string(i-Phys::nvar); // ="TRn" + VcName.push_back(tracerLabel); + data->dump->RegisterVariable(Vc, outputPrefix+"Vc-"+tracerLabel, i); } data->vtk->RegisterVariable(Vc, outputPrefix+VcName[i], i); #ifdef WITH_HDF5 diff --git a/src/fluid/fluid_defs.hpp b/src/fluid/fluid_defs.hpp index 2269fc803..3f68c9b49 100644 --- a/src/fluid/fluid_defs.hpp +++ b/src/fluid/fluid_defs.hpp @@ -8,6 +8,7 @@ #ifndef FLUID_FLUID_DEFS_HPP_ #define FLUID_FLUID_DEFS_HPP_ +#include #include "../idefix.hpp" @@ -25,7 +26,7 @@ class Fluid; // Parabolic terms can have different status -enum HydroModuleStatus {Disabled, Constant, UserDefFunction }; +enum HydroModuleStatus {Disabled, Constant, UserDefFunction}; // Structure to describe the status of parabolic modules struct ParabolicModuleStatus { @@ -49,8 +50,9 @@ using SrcTermFunc = void (*) (Fluid *, const real t, const real dt); using EmfBoundaryFunc = void (*) (DataBlock &, const real t); using DiffusivityFunc = void (*) (DataBlock &, const real t, IdefixArray3D &); -using BragDiffusivityFunc = void (*) (DataBlock &, const real t, - IdefixArray3D &, IdefixArray3D &); + +using BragDiffusivityFunc = void (*) (DataBlock &, const real, + std::vector> &); // Deprecated signatures using SrcTermFuncOld = void (*) (DataBlock &, const real t, const real dt); diff --git a/src/global.hpp b/src/global.hpp index 8651f9748..265699b29 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -11,6 +11,7 @@ #include #include #include "arrays.hpp" +#include "npy.hpp" namespace idfx { int initialize(); // Initialisation routine for idefix @@ -44,6 +45,20 @@ IdefixArray1D ConvertVectorToIdefixArray(std::vector &inputVector) { return(outArr); } +///< dump Idefix array to a numpy array on disk +template +void DumpArray(std::string filename, ArrayType array) { + auto hArray = Kokkos::create_mirror(array); + Kokkos::deep_copy(hArray, array); + + std::array shape; + bool fortran_order{false}; + for (size_t i = 0; i < ArrayType::rank; ++i) { + shape[i] = array.extent(i); + } + npy::SaveArrayAsNumpy(filename, fortran_order, ArrayType::rank, shape.data(), hArray.data()); +} + } // namespace idfx class idfx::IdefixOutStream { diff --git a/src/loop.hpp b/src/loop.hpp index 8ad345dea..42192fa84 100644 --- a/src/loop.hpp +++ b/src/loop.hpp @@ -34,6 +34,12 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; // Check if the user requested a specific loop unrolling strategy #if defined(LOOP_PATTERN_SIMD) + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from SIMD loop. You should try another loop pattern." + ); constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; #elif defined(LOOP_PATTERN_1DRANGE) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; @@ -51,7 +57,16 @@ typedef Kokkos::TeamPolicy<>::member_type member_type; constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_HIP) constexpr LoopPattern defaultLoop = LoopPattern::RANGE; + #elif defined(KOKKOS_ENABLE_SYCL) + constexpr LoopPattern defaultLoop = LoopPattern::RANGE; #elif defined(KOKKOS_ENABLE_SERIAL) + // Check that Idefix Arrays can be assigned from SIMD loop + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Idefix Arrays cannot be accessed from Host, but Device is unknown/untested. " + "Ask the developers to add support." + ); constexpr LoopPattern defaultLoop = LoopPattern::SIMDFOR; #else #warning "Unknown target architeture: default to MDrange" diff --git a/src/main.cpp b/src/main.cpp index 6b65c4f1c..d21793d1d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 2.2.00 +// IDEFIX v 2.2.01 // // ************************************************************************ //@HEADER diff --git a/src/output/output.cpp b/src/output/output.cpp index a67cb1666..9df188926 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -55,8 +55,13 @@ Output::Output(Input &input, DataBlock &data) dumpPeriod = input.Get("Output","dmp",0); dumpLast = data.t - dumpPeriod; // dump something in the next CheckForWrite() if(input.CheckEntry("Output","dmp")>1) { - dumpTimePeriod = input.Get("Output","dmp",1); - std::string dumpTimeExtension = input.Get("Output","dmp",2); + std::string dumpString = input.Get("Output","dmp",1); + try { + dumpTimePeriod = std::stod(dumpString.substr(0, dumpString.size()-1), NULL); + } catch(const std::exception& e) { + IDEFIX_ERROR("The dump time period should be a number followed by a unit (s, m, h or d)"); + } + std::string dumpTimeExtension = dumpString.substr(dumpString.size()-1,1); if(dumpTimeExtension.compare("s")==0) { dumpTimePeriod *= 1.0; // Dump time period is in seconds by default } else if (dumpTimeExtension.compare("m")==0) { diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index a2f6734f9..c8dd8cb96 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -762,32 +762,42 @@ void RKLegendre::CalcParabolicRHS(real t) { data->beg[IDIR],data->end[IDIR]+ioffset, KOKKOS_LAMBDA (int n, int k, int j, int i) { real Ax = A(k,j,i); - -#if GEOMETRY != CARTESIAN - if(Ax= 2) \ - || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) - if(dir==IDIR && nv==iMPHI) { - // Conserve angular momentum, hence flux is R*Vphi - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } -#endif // GEOMETRY==POLAR OR CYLINDRICAL + #if (GEOMETRY == POLAR && COMPONENTS >= 2) \ + || (GEOMETRY == CYLINDRICAL && COMPONENTS == 3) + if(dir==IDIR) { + if(nv==iMPHI) { + Ax *= FABS(x1m(i)); + } + if constexpr(Phys::mhd) { + if(nv==iBPHI) { + // No area for this one + Ax = 1; + } + } + } + #endif // GEOMETRY==POLAR OR CYLINDRICAL + + #if GEOMETRY == SPHERICAL + if(dir==IDIR && nv==VX3) { + Ax *= FABS(x1m(i)); + } else if(dir==JDIR && nv==VX3) { + Ax *= FABS(sm(j)); + } + + if constexpr(Phys::mhd) { + if(dir == IDIR && (nv==BX3 || nv == BX2)) { + Ax = x1m(i); + } + if(dir==JDIR && nv==BX3) { + Ax = 1.0; + } + } + #endif // GEOMETRY == SPHERICAL -#if GEOMETRY == SPHERICAL && COMPONENTS == 3 - if(dir==IDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(x1m(i)); - } else if(dir==JDIR && nv==iMPHI) { - Flux(iMPHI,k,j,i) = Flux(iMPHI,k,j,i) * FABS(sm(j)); - } -#endif // GEOMETRY == SPHERICAL && COMPONENTS == 3 + Flux(nv,k,j,i) = Flux(nv,k,j,i) * Ax; } ); @@ -818,17 +828,42 @@ void RKLegendre::CalcParabolicRHS(real t) { } #if GEOMETRY != CARTESIAN - #ifdef iMPHI - if((dir==IDIR) && (nv == iMPHI)) { + if(dir==IDIR) { + #ifdef iMPHI + if((nv == iMPHI)) { rhs /= x1(i); } + #endif // iMPHI + real dx_ = dx(i); + real x1_ = x1(i); + + if constexpr(Phys::mhd) { + #if (GEOMETRY == POLAR || GEOMETRY == CYLINDRICAL) && (defined iBPHI) + if(nv==iBPHI) rhs = - 1 / dx_ * (Flux(iBPHI, k, j, i+1) - Flux(iBPHI, k, j, i) ); + + #elif (GEOMETRY == SPHERICAL) + real q = 1 / (x1_*dx_); + if(nv == BX2 || nv == BX3) { + rhs = -q * ((Flux(nv, k, j, i+1) - Flux(nv, k, j, i) )); + } + #endif // GEOMETRY + } // MHD + } // dir==IDIR + if(dir==JDIR) { #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) - if((dir==JDIR) && (nv == iMPHI)) { + if(nv == iMPHI) { rhs /= FABS(s(j)); } + real dx_ = dx(j); + real rt_ = rt(i); + if constexpr(Phys::mhd) { + if(nv == iBPHI) { + rhs = - 1 / (rt_*dx_) * (Flux(nv, k, j+1, i) - Flux(nv, k, j, i)); + } + } #endif // GEOMETRY - // Nothing for KDIR - #endif // iMPHI + } // dir==JDIR + // Nothing for KDIR #endif // GEOMETRY != CARTESIAN // store the field components diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index b36f8a08e..afb4f57d4 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -5,4 +5,6 @@ target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/dumpImage.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lookupTable.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/column.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/column.hpp ) diff --git a/src/utils/column.cpp b/src/utils/column.cpp new file mode 100644 index 000000000..34d55f56b --- /dev/null +++ b/src/utils/column.cpp @@ -0,0 +1,240 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + +#include +#include + +#include "column.hpp" +#include "idefix.hpp" +#include "input.hpp" +#include "output.hpp" +#include "grid.hpp" +#include "dataBlock.hpp" +#include "dataBlockHost.hpp" + +Column::Column(int dir, int sign, DataBlock *data) + : direction(dir), sign(sign) { + idfx::pushRegion("Column::Column"); + this->np_tot = data->np_tot; + this->np_int = data->np_int; + this->beg = data->beg; + + if(dir>= DIMENSIONS || dir < IDIR) IDEFIX_ERROR("Unknown direction for Column constructor"); + + // Allocate the array on which we do the average + this->ColumnArray = IdefixArray3D("ColumnArray",np_tot[KDIR], np_tot[JDIR], np_tot[IDIR]); + + // Elementary volumes and area + this->Volume = data->dV; + this->Area = data->A[dir]; + + // allocate helper array + if(dir == IDIR) { + localSum = IdefixArray2D("localSum",np_tot[KDIR], np_tot[JDIR]); + } + if(dir == JDIR) { + localSum = IdefixArray2D("localSum",np_tot[KDIR], np_tot[IDIR]); + } + if(dir == KDIR) { + localSum = IdefixArray2D("localSum",np_tot[JDIR], np_tot[IDIR]); + } + #ifdef WITH_MPI + // Create sub-MPI communicator dedicated to scan + int remainDims[3] = {false, false, false}; + remainDims[dir] = true; + MPI_Cart_sub(data->mygrid->CartComm, remainDims, &this->ColumnComm); + MPI_Comm_rank(this->ColumnComm, &this->MPIrank); + MPI_Comm_size(this->ColumnComm, &this->MPIsize); + + // create MPI class for boundary Xchanges + int ntarget = 0; + std::vector mapVars; + mapVars.push_back(ntarget); + + this->mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data()); + this->nproc = data->mygrid->nproc; + #endif + idfx::popRegion(); +} + +void Column::ComputeColumn(IdefixArray4D in, const int var) { + idfx::pushRegion("Column::ComputeColumn"); + const int nk = np_int[KDIR]; + const int nj = np_int[JDIR]; + const int ni = np_int[IDIR]; + const int kb = beg[KDIR]; + const int jb = beg[JDIR]; + const int ib = beg[IDIR]; + const int ke = kb+nk; + const int je = jb+nj; + const int ie = ib+ni; + + const int direction = this->direction; + auto column = this->ColumnArray; + auto dV = this->Volume; + auto A = this->Area; + auto localSum = this->localSum; + + if(direction==IDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX1", team_policy (nk*nj, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int k = team_member.league_rank() / nj; + int j = team_member.league_rank() - k*nj + jb; + k += kb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,ib,ie), + [=] (int i, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j,i+1))); + if(is_final) column(k,j,i) = partial_sum; + //partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j,i+1))); + }); + }); + } + if(direction==JDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX2", team_policy (nk*ni, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int k = team_member.league_rank() / ni; + int i = team_member.league_rank() - k*ni + ib; + k += kb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,jb,je), + [=] (int j, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k,j+1,i))); + if(is_final) column(k,j,i) = partial_sum; + }); + }); + } + if(direction==KDIR) { + // Inspired from loop.hpp + Kokkos::parallel_for("ColumnX2", team_policy (nj*ni, Kokkos::AUTO), + KOKKOS_LAMBDA (member_type team_member) { + int j = team_member.league_rank() / ni; + int i = team_member.league_rank() - j*ni + ib; + j += jb; + Kokkos::parallel_scan(Kokkos::TeamThreadRange<>(team_member,kb,ke), + [=] (int k, real &partial_sum, bool is_final) { + partial_sum += in(var,k,j,i)*dV(k,j,i) / (0.5*(A(k,j,i)+A(k+1,j,i))); + if(is_final) column(k,j,i) = partial_sum; + }); + }); + } + + #ifdef WITH_MPI + // Load the current sum + int dst,src; + MPI_Cart_shift(this->ColumnComm,0, 1, &src, &dst); + int size = localSum.extent(0)*localSum.extent(1); + if(MPIrank>0) { + MPI_Status status; + // Get the cumulative sum from previous processes + Kokkos::fence(); + MPI_Recv(localSum.data(), size, realMPI, src, 20, ColumnComm, &status); + // Add this to our cumulative sum + idefix_for("Addsum",kb,ke,jb,je,ib,ie, + KOKKOS_LAMBDA(int k, int j, int i) { + if(direction == IDIR) column(k,j,i) += localSum(k,j); + if(direction == JDIR) column(k,j,i) += localSum(k,i); + if(direction == KDIR) column(k,j,i) += localSum(j,i); + }); + } // rank =0 + // Get the final sum + if(MPIrank < MPIsize-1) { + if(direction==IDIR) { + idefix_for("Loadsum",kb,ke,jb,je, + KOKKOS_LAMBDA(int k, int j) { + localSum(k,j) = column(k,j,ie-1); + }); + } + if(direction==JDIR) { + idefix_for("Loadsum",kb,ke,ib,ie, + KOKKOS_LAMBDA(int k, int i) { + localSum(k,i) = column(k,je-1,i); + }); + } + if(direction==KDIR) { + idefix_for("Loadsum",jb,je,ib,ie, + KOKKOS_LAMBDA(int j, int i) { + localSum(j,i) = column(ke-1,j,i); + }); + } + // And send it + Kokkos::fence(); + MPI_Send(localSum.data(),size, realMPI, dst, 20, ColumnComm); + } // MPIrank small enough + #endif + // If we need it backwards + if(sign<0) { + // Compute total cumulative sum in the last subdomain + if(direction == IDIR) { + idefix_for("Loadsum",kb,ke,jb,je, + KOKKOS_LAMBDA(int k, int j) { + localSum(k,j) = column(k,j,ie-1) + in(var,k,j,ie-1) * dV(k,j,ie-1) + / (0.5*(A(k,j,ie-1)+A(k,j,ie))); + }); + } + if(direction == JDIR) { + idefix_for("Loadsum",kb,ke,ib,ie, + KOKKOS_LAMBDA(int k, int i) { + localSum(k,i) = column(k,je-1,i) + in(var,k,je-1,i) * dV(k,je-1,i) + / (0.5*(A(k,je-1,i)+A(k,je,i))); + }); + } + if(direction == KDIR) { + idefix_for("Loadsum",jb,je,ib,ie, + KOKKOS_LAMBDA(int j, int i) { + localSum(j,i) = column(ke-1,j,i) + in(var,ke-1,j,i) * dV(ke-1,j,i) + / (0.5*(A(ke-1,j,i)+A(ke,j,i))); + }); + } + + #ifdef WITH_MPI + Kokkos::fence(); + MPI_Bcast(localSum.data(),size, realMPI, MPIsize-1, ColumnComm); + #endif + // All substract the local column from the full column + + idefix_for("Subsum",kb,ke,jb,je,ib,ie, + KOKKOS_LAMBDA(int k, int j, int i) { + if(direction==IDIR) column(k,j,i) = localSum(k,j)-column(k,j,i); + if(direction==JDIR) column(k,j,i) = localSum(k,i)-column(k,j,i); + if(direction==KDIR) column(k,j,i) = localSum(j,i)-column(k,j,i); + }); + } // sign <0 + // Xchange boundary elements when using MPI to ensure that column + // density in the ghost zones are coherent + #ifdef WITH_MPI + // Create a 4D array that contains our column data + IdefixArray4D arr4D(column.data(), 1, this->np_tot[KDIR], + this->np_tot[JDIR], + this->np_tot[IDIR]); + + for(int dir = 0 ; dir < DIMENSIONS ; dir++) { + // MPI Exchange data when needed + if(nproc[dir]>1) { + switch(dir) { + case 0: + this->mpi.ExchangeX1(arr4D); + break; + case 1: + this->mpi.ExchangeX2(arr4D); + break; + case 2: + this->mpi.ExchangeX3(arr4D); + break; + } + } + } + #endif + idfx::popRegion(); +} + +void Column::ComputeColumn(IdefixArray3D in) { + // 4D alias + IdefixArray4D arr4D(in.data(), 1, in.extent(0), in.extent(1), in.extent(2)); + return this->ComputeColumn(arr4D,0); +} diff --git a/src/utils/column.hpp b/src/utils/column.hpp new file mode 100644 index 000000000..43383cbe6 --- /dev/null +++ b/src/utils/column.hpp @@ -0,0 +1,74 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UTILS_COLUMN_HPP_ +#define UTILS_COLUMN_HPP_ + +#include +#include + +#include "idefix.hpp" +#include "dataBlock.hpp" + + + +// A class to implement a parralel cumulative sum +class Column { + public: + //////////////////////////////////////////////////////////////////////////////////// + /// @brief Constructor of a cumulative sum(=column density) from setup's arguments + /// @param dir direction along which the integration is performed + /// @param sign: +1 for an integration from left to right, -1 for an integration from right to + /// left i.e (backwards) + /////////////////////////////////////////////////////////////////////////////////// + Column(int dir, int sign, DataBlock *); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Effectively compute integral from the input array in argument + /// @param in: 4D input array + /// @param variable: index of the variable along which we do the integral (since the + /// intput array is 4D) + /////////////////////////////////////////////////////////////////////////////////// + void ComputeColumn(IdefixArray4D in, int variable); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Effectively compute integral from the input array in argument + /// @param in: 3D input array + /////////////////////////////////////////////////////////////////////////////////// + void ComputeColumn(IdefixArray3D in); + + /////////////////////////////////////////////////////////////////////////////////// + /// @brief Get a reference to the computed column density array + /////////////////////////////////////////////////////////////////////////////////// + IdefixArray3D GetColumn() { + return (this->ColumnArray); + } + + private: + IdefixArray3D ColumnArray; + int direction; // The direction along which the column is computed + int sign; // whether we integrate from the left or from the right + std::array np_tot; + std::array np_int; + std::array beg; + + IdefixArray3D Area; + IdefixArray3D Volume; + + IdefixArray2D localSum; + #ifdef WITH_MPI + Mpi mpi; // Mpi object when WITH_MPI is set + MPI_Comm ColumnComm; + int MPIrank; + int MPIsize; + + std::array nproc; // 3D size of the MPI cartesian geometry + + #endif +}; + +#endif // UTILS_COLUMN_HPP_ diff --git a/test/Dust/DustEnergy/idefix-implicit.ini b/test/Dust/DustEnergy/idefix-implicit.ini new file mode 100644 index 000000000..9ba3c781a --- /dev/null +++ b/test/Dust/DustEnergy/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks that the total energy (thermal+dust kinetic+gas kinetic) +# is effectively conserved when drag is present + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 1.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +gamma 1.4 + +[Dust] +nSpecies 1 +drag tau 1.0 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 1.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustEnergy/testme.py b/test/Dust/DustEnergy/testme.py index f3b413e83..7187d299b 100755 --- a/test/Dust/DustEnergy/testme.py +++ b/test/Dust/DustEnergy/testme.py @@ -15,7 +15,7 @@ def testMe(test): test.configure() test.compile() - inifiles=["idefix.ini"] + inifiles=["idefix.ini","idefix-implicit.ini"] # loop on all the ini files for this test for ini in inifiles: diff --git a/test/Dust/DustyShock/definitions.hpp b/test/Dust/DustyShock/definitions.hpp new file mode 100644 index 000000000..cdbe23d6e --- /dev/null +++ b/test/Dust/DustyShock/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 1 +#define DIMENSIONS 1 +#define ISOTHERMAL + +#define GEOMETRY CARTESIAN diff --git a/test/Dust/DustyShock/idefix-implicit.ini b/test/Dust/DustyShock/idefix-implicit.ini new file mode 100644 index 000000000..2d92a0a52 --- /dev/null +++ b/test/Dust/DustyShock/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks the behaviour of a dust sound shock +# following the 4 fluids test of Benitez-Llambay+ 2019 + +[Grid] +X1-grid 1 0.0 400 u 40.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 500.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 3 +drag userdef 1.0 3.0 5.0 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 500.0 +vtk 500.0 +log 1000 diff --git a/test/Dust/DustyShock/idefix.ini b/test/Dust/DustyShock/idefix.ini new file mode 100644 index 000000000..a98ebd097 --- /dev/null +++ b/test/Dust/DustyShock/idefix.ini @@ -0,0 +1,35 @@ +# This test checks the behaviour of a dust sound shock +# following the 4 fluids test of Benitez-Llambay+ 2019 + +[Grid] +X1-grid 1 0.0 400 u 40.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 500.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 3 +drag userdef 1.0 3.0 5.0 +drag_feedback yes + +[Boundary] +X1-beg userdef +X1-end userdef +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 500.0 +vtk 500.0 +log 1000 diff --git a/test/Dust/DustyShock/python/testidefix.py b/test/Dust/DustyShock/python/testidefix.py new file mode 100755 index 000000000..8fca15a3f --- /dev/null +++ b/test/Dust/DustyShock/python/testidefix.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 5 11:29:41 2020 + +@author: glesur +""" + +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import argparse +import numpy as np +import matplotlib.pyplot as plt +from scipy.integrate import solve_ivp + +parser = argparse.ArgumentParser() +parser.add_argument("-noplot", + default=False, + help="disable plotting", + action="store_true") + + +args, unknown=parser.parse_known_args() + +V=readVTK('../data.0001.vtk', geometry='cartesian') + +# Find where the shock starts +istart = np.argwhere(V.data['RHO'][:,0,0]>2.0)[0][0]-1 + +# Compute analytical solution +M=2 # Mach number +K=np.asarray([1,3,5])/M # Drag coefficients + +# (56) from Benitez-Llambay 2018 +def get_wg(Mach, wi): + coeff=[1,np.sum(wi-1)-1/Mach**2-1,1/Mach**2] + solution=np.roots(coeff) + return(np.min(solution)) + +# (55) from Benitez-Llambay 2018 +def rhs(t,wi,Mach,K): + wg = get_wg(Mach,wi) + return K*(wg-wi) + +wi0=np.asarray([1.0,1.0,1.0]) +x=V.x[istart:] +arguments=M,K + +sol = solve_ivp(rhs, t_span=[x[0],x[-1]], y0=wi0, t_eval=x,args=arguments) + +# compute gas velocity +wg=np.zeros(len(x)) +for i in range(0,len(wg)): + wg[i]=get_wg(M,sol.y[:,i]) + + +if(not args.noplot): + plt.figure(1) + plt.plot(V.x,V.data['RHO'][:,0,0],'o',markersize=4,color='tab:purple') + plt.plot(x,1/wg,color='tab:purple') + plt.plot(V.x,V.data['Dust0_RHO'][:,0,0],'o',markersize=4,color='tab:orange') + plt.plot(x,1/sol.y[0,:],color='tab:orange') + plt.plot(V.x,V.data['Dust1_RHO'][:,0,0],'o',markersize=4,color='tab:blue') + plt.plot(x,1/sol.y[1,:],color='tab:blue') + plt.plot(V.x,V.data['Dust2_RHO'][:,0,0],'o',markersize=4,color='tab:green') + plt.plot(x,1/sol.y[2,:],color='tab:green') + plt.xlim([0,10]) + plt.title('Density') + + plt.figure(2) + plt.plot(V.x,V.data['VX1'][:,0,0],'o',markersize=4,color='tab:purple') + plt.plot(x,M*wg,color='tab:purple') + plt.plot(V.x[::4],V.data['Dust0_VX1'][::4,0,0],'o',markersize=4,color='tab:orange') + plt.plot(x,M*sol.y[0,:],color='tab:orange') + plt.plot(V.x[::4],V.data['Dust1_VX1'][::4,0,0],'o',markersize=4,color='tab:blue') + plt.plot(x,M*sol.y[1,:],color='tab:blue') + plt.plot(V.x[::4],V.data['Dust2_VX1'][::4,0,0],'o',markersize=4,color='tab:green') + plt.plot(x,M*sol.y[2,:],color='tab:green') + + plt.xlim([0,10]) + plt.title('Velocity') + + plt.ioff() + plt.show() + +# Compute the error on the dust densities + +error=np.mean(np.fabs(V.data['Dust0_RHO'][istart:,0,0]-1/sol.y[0,:]) + np.fabs(V.data['Dust1_RHO'][istart:,0,0]-1/sol.y[1,:]) + np.fabs(V.data['Dust2_RHO'][istart:,0,0]-1/sol.y[2,:])) / 3 +print("Error=%e"%error) +if error<3.6e-2: + print("SUCCESS!") + sys.exit(0) +else: + print("FAILURE!") + sys.exit(1) diff --git a/test/Dust/DustyShock/setup.cpp b/test/Dust/DustyShock/setup.cpp new file mode 100644 index 000000000..f8632248a --- /dev/null +++ b/test/Dust/DustyShock/setup.cpp @@ -0,0 +1,99 @@ +#include "idefix.hpp" +#include "setup.hpp" + +#define FILENAME "timevol.dat" + +void MyDrag(DataBlock *data, int n, real beta, IdefixArray3D &gamma) { + // Compute the drag coefficient gamma from the input beta + auto VcGas = data->hydro->Vc; + auto VcDust = data->dust[n]->Vc; + + idefix_for("MyDrag",0,data->np_tot[KDIR],0,data->np_tot[JDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + gamma(k,j,i) = beta/VcGas(RHO,k,j,i)/VcDust(RHO,k,j,i); + }); +} + + +void ApplyBoundary(DataBlock *data, IdefixArray4D Vc, int dir, BoundarySide side) { + if(dir==IDIR) { + int iref,ibeg,iend; + if(side == left) { + iref = data->beg[IDIR]; + ibeg = 0; + iend = data->beg[IDIR]; + } else { + iref =data->end[IDIR] - 1; + ibeg=data->end[IDIR]; + iend=data->np_tot[IDIR]; + } + int nvar = Vc.extent(0); + idefix_for("UserDefBoundary",0,data->np_tot[KDIR],0,data->np_tot[JDIR],ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + for(int n = 0 ; n < nvar ; n++) { + Vc(n,k,j,i) = Vc(n,k,j,iref ); + } + }); + } +} + +void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + ApplyBoundary(hydro->data, hydro->Vc, dir, side); +} + +void UserdefBoundaryDust(Fluid *dust, int dir, BoundarySide side, real t) { + ApplyBoundary(dust->data, dust->Vc, dir, side); +} + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + + data.hydro->EnrollUserDefBoundary(&UserdefBoundary); + if(data.haveDust) { + int nSpecies = data.dust.size(); + for(int n = 0 ; n < nSpecies ; n++) { + data.dust[n]->EnrollUserDefBoundary(&UserdefBoundaryDust); + data.dust[n]->drag->EnrollUserDrag(&MyDrag); + } + } + +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + const real x0 = 4.0; + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + real x = d.x[IDIR](i); + + d.Vc(RHO,k,j,i) = (x < x0) ? 1.0 : 16.0; + d.Vc(VX1,k,j,i) = (x < x0) ? 2.0 : 0.125; + + for(int n = 0 ; n < data.dust.size(); n++) { + d.dustVc[n](RHO,k,j,i) = (x < x0) ? 1.0 : 16.0; + d.dustVc[n](VX1,k,j,i) = (x < x0) ? 2.0 : 0.125; + } + + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} diff --git a/test/Dust/DustyShock/testme.py b/test/Dust/DustyShock/testme.py new file mode 100755 index 000000000..2b03d1636 --- /dev/null +++ b/test/Dust/DustyShock/testme.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini","idefix-implicit.ini"] + + # loop on all the ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name,tolerance=1e-14) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename=name) + else: + testMe(test) +else: + test.noplot = True + test.reconstruction=2 + testMe(test) diff --git a/test/Dust/DustyWave/idefix-implicit.ini b/test/Dust/DustyWave/idefix-implicit.ini new file mode 100644 index 000000000..726585e0b --- /dev/null +++ b/test/Dust/DustyWave/idefix-implicit.ini @@ -0,0 +1,36 @@ +# This test checks the dissipation of a sound wave by a dust grains +# partially coupled to the gas (Riols & Lesur 2018, appendix A) + +[Grid] +X1-grid 1 0.0 500 u 1.0 +X2-grid 1 0.0 1 u 1.0 +X3-grid 1 0.0 1 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 10.0 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hllc +csiso constant 1.0 + +[Dust] +nSpecies 1 +drag tau 1.0 +drag_feedback yes +drag_implicit yes + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg outflow +X2-end outflow +X3-beg outflow +X3-end outflow + +[Output] +dmp 10.0 +analysis 0.01 +log 1000 diff --git a/test/Dust/DustyWave/python/testidefix.py b/test/Dust/DustyWave/python/testidefix.py index c98bc8213..ad52d3306 100755 --- a/test/Dust/DustyWave/python/testidefix.py +++ b/test/Dust/DustyWave/python/testidefix.py @@ -35,9 +35,6 @@ # Get the minimal decay rate (this should be the one that pops up) tau=np.amax(np.imag(sol)) - - - # load the dat file produced by the setup raw=np.loadtxt('../timevol.dat',skiprows=1) t=raw[:,0] @@ -58,10 +55,10 @@ # Compute decay rate: tau_measured=t[-1]/(2*np.log(etot[-1]/etot[0])) # error on the decay rate: -error=(tau_measured-tau)/tau +error=np.abs((tau_measured-tau)/tau) print("error=%f"%error) -if(error<0.065): +if(error<0.07): print("Success!") else: print("Failure!") diff --git a/test/Dust/DustyWave/testme.py b/test/Dust/DustyWave/testme.py index b6a75ce3c..2b03d1636 100755 --- a/test/Dust/DustyWave/testme.py +++ b/test/Dust/DustyWave/testme.py @@ -15,7 +15,7 @@ def testMe(test): test.configure() test.compile() - inifiles=["idefix.ini"] + inifiles=["idefix.ini","idefix-implicit.ini"] # loop on all the ini files for this test for ini in inifiles: diff --git a/test/Dust/FargoPlanet/setup.cpp b/test/Dust/FargoPlanet/setup.cpp index af611911d..8447efe0b 100644 --- a/test/Dust/FargoPlanet/setup.cpp +++ b/test/Dust/FargoPlanet/setup.cpp @@ -42,7 +42,7 @@ void MySoundSpeed(DataBlock &data, const real t, IdefixArray3D &cs) { // a = 20 cm * beta * (Sigma_phys/100 g.cm^-2) / (rho_s / 2 g.cm^-3) // NB: checked against A. Johansen+ (2007) -void MyDrag(DataBlock *data, real beta, IdefixArray3D &gamma) { +void MyDrag(DataBlock *data, int nSpecie, real beta, IdefixArray3D &gamma) { // Compute the drag coefficient gamma from the input beta auto x1 = data->x[IDIR]; real sigma0 = sigma0Glob; diff --git a/test/MHD/AxisFluxTube/setup.cpp b/test/MHD/AxisFluxTube/setup.cpp index 34009da85..4c30506d3 100644 --- a/test/MHD/AxisFluxTube/setup.cpp +++ b/test/MHD/AxisFluxTube/setup.cpp @@ -112,7 +112,7 @@ void CoarsenFunction(DataBlock &data) { Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { output.EnrollUserDefVariables(&ComputeUserVars); data.hydro->EnrollUserDefBoundary(&UserdefBoundary); - output.EnrollAnalysis(&Analysis); + //output.EnrollAnalysis(&Analysis); Rtorus = input.Get("Setup","Rtorus",0); Ztorus = input.Get("Setup","Ztorus",0); Rin = input.Get("Setup","Rin",0); diff --git a/test/MHD/MTI/idefix-rkl.ini b/test/MHD/MTI/idefix-rkl.ini index 1b9b11eb2..09e9eb72d 100644 --- a/test/MHD/MTI/idefix-rkl.ini +++ b/test/MHD/MTI/idefix-rkl.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion rkl nolimiter userdef +bragTDiffusion rkl nolimiter nosat userdef bragViscosity rkl nolimiter userdef [Gravity] diff --git a/test/MHD/MTI/idefix-sl.ini b/test/MHD/MTI/idefix-sl.ini index f2c9deb2b..aa707914f 100644 --- a/test/MHD/MTI/idefix-sl.ini +++ b/test/MHD/MTI/idefix-sl.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion rkl mc userdef +bragTDiffusion rkl mc nosat userdef bragViscosity rkl vanleer userdef [Gravity] diff --git a/test/MHD/MTI/idefix.ini b/test/MHD/MTI/idefix.ini index 7a09c406c..a159c4f70 100644 --- a/test/MHD/MTI/idefix.ini +++ b/test/MHD/MTI/idefix.ini @@ -12,7 +12,7 @@ nstages 3 [Hydro] solver hlld gamma 1.6666666666666666 -bragTDiffusion explicit nolimiter userdef +bragTDiffusion explicit nolimiter nosat userdef bragViscosity explicit nolimiter userdef [Gravity] diff --git a/test/MHD/MTI/setup.cpp b/test/MHD/MTI/setup.cpp index e631f7b2b..552f76465 100644 --- a/test/MHD/MTI/setup.cpp +++ b/test/MHD/MTI/setup.cpp @@ -39,9 +39,13 @@ void Potential(DataBlock& data, const real t, IdefixArray1D& x1, IdefixArr } -void MyBragThermalConductivity(DataBlock &data, const real t, IdefixArray3D &kparArr, IdefixArray3D &knorArr) { +void MyBragThermalConductivity(DataBlock &data, const real t, std::vector> &userdefArr) { IdefixArray4D Vc = data.hydro->Vc; IdefixArray1D x2 = data.x[JDIR]; + + IdefixArray3D kparArr = userdefArr.at(0); + IdefixArray3D knorArr = userdefArr.at(1); + real ksi = ksiGlob; idefix_for("MyThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], KOKKOS_LAMBDA (int k, int j, int i) { diff --git a/test/MHD/clessTDiffusion/CMakeLists.txt b/test/MHD/clessTDiffusion/CMakeLists.txt new file mode 100644 index 000000000..629aed2d1 --- /dev/null +++ b/test/MHD/clessTDiffusion/CMakeLists.txt @@ -0,0 +1 @@ +enable_idefix_property(Idefix_MHD) diff --git a/test/MHD/clessTDiffusion/definitions.hpp b/test/MHD/clessTDiffusion/definitions.hpp new file mode 100644 index 000000000..21b1f8733 --- /dev/null +++ b/test/MHD/clessTDiffusion/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 3 +#define DIMENSIONS 1 +#define GEOMETRY SPHERICAL + +#define SMALL_PRESSURE_TEMPERATURE (0.1) diff --git a/test/MHD/clessTDiffusion/idefix.ini b/test/MHD/clessTDiffusion/idefix.ini new file mode 100644 index 000000000..9c444b928 --- /dev/null +++ b/test/MHD/clessTDiffusion/idefix.ini @@ -0,0 +1,41 @@ +[Grid] +X1-grid 1 1.0 256 l 40.0 +X2-grid 1 1.4708 1 u 1.6708 +X3-grid 1 0.9892 1 u 1.0108 + +[TimeIntegrator] +CFL 0.4 +CFL_max_var 1.1 # not used +tstop 300.0 +first_dt 1.e-6 +nstages 2 + +[Hydro] +solver hlld +gamma 1.6666666666666 +bragTDiffusion rkl mc wcless userdef + +[Gravity] +potential central +Mcentral 1.0 + +[Boundary] +X1-beg userdef +X1-end outflow +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Setup] +UNIT_DENSITY 1.6726e-16 +UNIT_LENGTH 6.9570e10 +UNIT_VELOCITY 4.3670e7 +cs_vesc 0.26 +va_vesc 0.3 +k0 9e-7 + +[Output] +vtk 100.0 +dmp 300.0 +log 10 diff --git a/test/MHD/clessTDiffusion/python/testidefix.py b/test/MHD/clessTDiffusion/python/testidefix.py new file mode 100644 index 000000000..9427aca82 --- /dev/null +++ b/test/MHD/clessTDiffusion/python/testidefix.py @@ -0,0 +1,40 @@ +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +from pytools.vtk_io import readVTK +import numpy as np +import inifix + +conf = inifix.load("../idefix.ini") +gamma = conf["Hydro"]["gamma"] +kappa = conf["Hydro"]["bragTDiffusion"][-1] + +current_VTK = readVTK('../data.0003.vtk', geometry='spherical') + +rho = current_VTK.data['RHO'].squeeze() +prs = current_VTK.data['PRS'].squeeze() +r = current_VTK.r + +idx1=np.where(r > 20)[0][0] +idx2=np.where(r > 30)[0][0] + +def gamma_prime(gamma, beta): + return (gamma+beta*(gamma-1))/(1+beta*(gamma-1)) + +success = True +eps = 3e-4 + +gamma_eff = np.gradient(prs, r)/np.gradient(rho, r)*rho/prs + +Error=np.abs(gamma_eff[idx1:idx2]-gamma_prime(gamma, 1.5)).mean() +if Error > eps: + success=False + +if success: + print("SUCCESS") + print("Error: {0}".format(Error)) + sys.exit(0) +else: + print("Failed") + print("Error: {0}".format(Error)) + sys.exit(1) diff --git a/test/MHD/clessTDiffusion/setup.cpp b/test/MHD/clessTDiffusion/setup.cpp new file mode 100644 index 000000000..c540c0def --- /dev/null +++ b/test/MHD/clessTDiffusion/setup.cpp @@ -0,0 +1,158 @@ +#include "idefix.hpp" +#include "setup.hpp" + +real udenGlob; +real ulenGlob; +real uvelGlob; +real gammaGlob; +real cs_vescGlob; +real va_vescGlob; +real k0_Glob; +real ParkerWind(real); + +const real kB{1.3807e-16}; +const real mp{1.6726e-24}; + +void MyClessThermalConductivity(DataBlock &data, const real t, std::vector> &userdefArr) { + IdefixArray4D Vc = data.hydro->Vc; + IdefixArray1D x1 = data.x[IDIR]; + IdefixArray1D x2 = data.x[JDIR]; + + IdefixArray3D kparArr = userdefArr.at(0); + IdefixArray3D knorArr = userdefArr.at(1); + IdefixArray3D clessAlpha = userdefArr.at(2); + IdefixArray3D clessBeta = userdefArr.at(3); + + real norm = mp*0.5/(udenGlob*uvelGlob*ulenGlob*kB); + real uTemp=0.5*uvelGlob*uvelGlob*mp/kB; + real k0 = k0_Glob*norm; + idefix_for("MyClessThConductivity",0,data.np_tot[KDIR],0,data.np_tot[JDIR],0,data.np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + kparArr(k,j,i) = k0*pow(Vc(PRS,k,j,i)/Vc(RHO,k,j,i)*uTemp,2.5); + knorArr(k,j,i) = 0.; + clessAlpha(k,j,i) = (1.0-tanh(x1(i)-10))/2; + clessBeta(k,j,i) = -1.5; + }); +} + +// User-defined boundaries +void UserDefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { + + DataBlock *data = hydro->data; + + if( (dir==IDIR) && (side == left)) { + IdefixArray4D Vc = hydro->Vc; + IdefixArray4D Vs = hydro->Vs; + IdefixArray1D x1 = data->x[IDIR]; + + real rc,vwind0; + real cs=cs_vescGlob*sqrt(2.); + real va_vesc = va_vescGlob; + real mu = va_vesc * sqrt(2.); + real PonRho; + + PonRho = cs*cs; + rc = 0.25 / (cs_vescGlob*cs_vescGlob); + vwind0 = ParkerWind(1./rc) * cs; + + hydro->boundary->BoundaryFor("UserDefBoundary", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + real r = x1(i); + + Vc(RHO,k,j,i) = vwind0/(vwind0 * r * r); + Vc(PRS,k,j,i) = PonRho * Vc(RHO, k, j, i); + Vc(VX1,k,j,i) = vwind0; + Vc(VX2,k,j,i) = 0.0; + Vc(VX3,k,j,i) = 0.0; + Vc(BX1,k,j,i) = mu / (r*r); + Vc(BX2,k,j,i) = 0.0; + Vc(BX3,k,j,i) = 0.0; + + }); + } +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + // Set the function for userdefboundary + data.hydro->EnrollUserDefBoundary(&UserDefBoundary); + data.hydro->bragThermalDiffusion->EnrollBragThermalDiffusivity(&MyClessThermalConductivity); + gammaGlob=input.Get("Hydro", "gamma", 0); + udenGlob=input.Get("Setup", "UNIT_DENSITY",0); + ulenGlob=input.Get("Setup", "UNIT_LENGTH",0); + uvelGlob=input.Get("Setup", "UNIT_VELOCITY",0); + cs_vescGlob=input.Get("Setup", "cs_vesc", 0); + va_vescGlob=input.Get("Setup", "va_vesc", 0); + k0_Glob = input.Get("Setup", "k0", 0); +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + real r,rl; + real PonRho, vwind0, rc; + real cs=cs_vescGlob*sqrt(2.); + + + rc = 0.25 / (cs_vescGlob*cs_vescGlob); + vwind0 = ParkerWind(1./rc) * cs; + PonRho = cs*cs; + real mu = va_vescGlob * sqrt(2.); + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + r=d.x[IDIR](i); + + real vwind; + + vwind = ParkerWind(r/rc) * cs; + + d.Vc(RHO,k,j,i) = 1.0*vwind0/(vwind * r * r); + d.Vc(PRS,k,j,i) = PonRho * d.Vc(RHO, k, j, i); + d.Vc(VX1,k,j,i) = vwind; + d.Vc(VX2,k,j,i) = 0.0; + d.Vc(VX3,k,j,i) = 0.0; + + rl=d.xl[IDIR](i); // Radius on the left side of the cell + d.Vs(BX1s, k, j, i) = mu / (rl*rl); + + } + } + } + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { +} + +/**************************************************/ +real ParkerWind(real x) +/* Parker wind velocity in unit of iso sound speed + x = radius / critical radius. +**************************************************/ +{ + real v, f; + real vref; + + v = 1e-7; + f = v*v-2*log(v)-4/x-4*log(x)+3; + if (x>1) {v=10;} + while (fabs(f) > 1e-10){ + vref = v; + v = v - 0.5*f/(v-1/v); + while (v < 0){ + v = (v + vref)/2; + } + f = v*v-2*log(v)-4/x-4*log(x)+3; + } + return v; +} diff --git a/test/MHD/clessTDiffusion/testme.py b/test/MHD/clessTDiffusion/testme.py new file mode 100755 index 000000000..6ed5b1648 --- /dev/null +++ b/test/MHD/clessTDiffusion/testme.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + + +def testMe(test): + test.configure() + test.compile() + inifiles=["idefix.ini"] + + # loop on all the ini files for this test + name = "dump.0001.dmp" + for ini in inifiles: + test.run(inputFile=ini) + test.standardTest() + if test.init: + test.makeReference(filename=name) + test.nonRegressionTest(filename=name, tolerance=2e-15) + + +test=tst.idfxTest() + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp") + else: + testMe(test) +else: + test.noplot = True + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) diff --git a/test/MHD/sphBragTDiffusion/idefix.ini b/test/MHD/sphBragTDiffusion/idefix.ini index 35ac54e8b..795ae7bb2 100644 --- a/test/MHD/sphBragTDiffusion/idefix.ini +++ b/test/MHD/sphBragTDiffusion/idefix.ini @@ -13,7 +13,7 @@ nstages 3 [Hydro] solver hlld gamma 1.4 -bragTDiffusion rkl nolimiter constant 50.0 +bragTDiffusion rkl nolimiter nosat constant 50.0 [Setup] amplitude 1e-4 diff --git a/test/utils/columnDensity/definitions.hpp b/test/utils/columnDensity/definitions.hpp new file mode 100644 index 000000000..3b581bd62 --- /dev/null +++ b/test/utils/columnDensity/definitions.hpp @@ -0,0 +1,4 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 + +#define GEOMETRY CARTESIAN diff --git a/test/utils/columnDensity/idefix.ini b/test/utils/columnDensity/idefix.ini new file mode 100644 index 000000000..5e80c8bce --- /dev/null +++ b/test/utils/columnDensity/idefix.ini @@ -0,0 +1,24 @@ +[Grid] +X1-grid 1 0.0 64 u 1.0 +X2-grid 1 0.0 32 u 1.0 +X3-grid 1 0.0 16 u 1.0 + +[TimeIntegrator] +CFL 0.8 +tstop 0.0 +nstages 2 + +[Hydro] +solver hllc +gamma 1.4 + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +analysis 0.01 diff --git a/test/utils/columnDensity/setup.cpp b/test/utils/columnDensity/setup.cpp new file mode 100644 index 000000000..fe7c57e69 --- /dev/null +++ b/test/utils/columnDensity/setup.cpp @@ -0,0 +1,169 @@ +#include "idefix.hpp" +#include "setup.hpp" +#include "column.hpp" + + +Column *columnX1Left; +Column *columnX1Right; +Column *columnX2Left; +Column *columnX2Right; +Column *columnX3Left; +Column *columnX3Right; + +// Analyse data to check that column density works as expected +void Analysis(DataBlock & data) { + + DataBlockHost d(data); + + // Try the 4D array interface + columnX1Left->ComputeColumn(data.hydro->Vc,RHO); + columnX1Right->ComputeColumn(data.hydro->Vc,RHO); + + // Try the 3D array interface + IdefixArray3D rho("rho",data.np_tot[KDIR],data.np_tot[JDIR],data.np_tot[IDIR]); + auto Vc = data.hydro->Vc; + + idefix_for("init rho",data.beg[KDIR],data.end[KDIR],data.beg[JDIR],data.end[JDIR],data.beg[IDIR],data.end[IDIR], + KOKKOS_LAMBDA(int k, int j, int i) { + rho(k,j,i) = Vc(RHO,k,j,i); + }); + columnX2Left->ComputeColumn(rho); + columnX2Right->ComputeColumn(rho); + columnX3Left->ComputeColumn(rho); + columnX3Right->ComputeColumn(rho); + + IdefixArray3D columnDensityLeft, columnDensityRight; + IdefixArray3D::HostMirror columnDensityLeftHost, columnDensityRightHost; + // IDIR + columnDensityLeft = columnX1Left->GetColumn(); + columnDensityRight = columnX1Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + real errMax = 0.0; + + // Because this particular setup has a constant density =1, we expect + // the column density from the left to match the cell right coordinates, + // and the column density from the right to match L-the cell left coordinate (where L is the box size) + // here we have L=1 + // This routine checks that all of the available column densities do match the expected values. + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[IDIR](i)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[IDIR](i))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in IDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + // JDIR + columnDensityLeft = columnX2Left->GetColumn(); + columnDensityRight = columnX2Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + errMax = 0.0; + + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[JDIR](j)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[JDIR](j))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in JDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + // KDIR + columnDensityLeft = columnX3Left->GetColumn(); + columnDensityRight = columnX3Right->GetColumn(); + columnDensityLeftHost = Kokkos::create_mirror_view(columnDensityLeft); + columnDensityRightHost = Kokkos::create_mirror_view(columnDensityRight); + Kokkos::deep_copy(columnDensityLeftHost,columnDensityLeft); + Kokkos::deep_copy(columnDensityRightHost,columnDensityRight); + + errMax = 0.0; + + for(int k = data.beg[KDIR]; k < data.end[KDIR] ; k++) { + for(int j = data.beg[JDIR]; j < data.end[JDIR] ; j++) { + for(int i = data.beg[IDIR]; i < data.end[IDIR] ; i++) { + real err = std::fabs(columnDensityLeftHost(k,j,i)-d.xr[KDIR](k)); + if(err>errMax) errMax=err; + err = std::fabs(columnDensityRightHost(k,j,i)-(1-d.xl[KDIR](k))); + if(err>errMax) errMax=err; + } + } + } + idfx::cout << "Error on column density in KDIR=" << std::scientific << errMax << std::endl; + if(errMax>1e-14) { + IDEFIX_ERROR("Error above tolerance"); + } + +} + +void InternalBoundary(Fluid * hydro, const real t) { + IdefixArray4D Vc = hydro->Vc; + idefix_for("InternalBoundary",0,hydro->data->np_tot[KDIR], + 0,hydro->data->np_tot[JDIR], + 0,hydro->data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int j, int i) { + // Cancel any motion that could be happening + Vc(VX1,k,j,i) = 0.0; + Vc(VX2,k,j,i) = 0.0; + Vc(VX3,k,j,i) = 0.0; + }); +} + + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + output.EnrollAnalysis(&Analysis); + data.hydro->EnrollInternalBoundary(&InternalBoundary); + + columnX1Left = new Column(IDIR, 1, &data); + columnX1Right = new Column(IDIR, -1, &data); + columnX2Left = new Column(JDIR, 1, &data); + columnX2Right = new Column(JDIR, -1, &data); + columnX3Left = new Column(KDIR, 1, &data); + columnX3Right = new Column(KDIR, -1, &data); + // Initialise the output file +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + + d.Vc(RHO,k,j,i) = 1.0; + d.Vc(VX1,k,j,i) = ZERO_F; + d.Vc(PRS,k,j,i) = 1.0; + + } + } + } + + // Send it all, if needed + d.SyncToDevice(); +} diff --git a/test/utils/columnDensity/testme.py b/test/utils/columnDensity/testme.py new file mode 100755 index 000000000..6f017d7d8 --- /dev/null +++ b/test/utils/columnDensity/testme.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) +import pytools.idfx_test as tst + +test=tst.idfxTest() + +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run() + +test.mpi = True +test.configure() +test.compile() +# this test succeeds if it runs successfully +test.run() From 4a8b8a2abdbc3237f3270e08cf933ad7333abeaa Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Sun, 19 Oct 2025 08:48:57 +0200 Subject: [PATCH 06/10] V2.2.02 (#355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [2.2.02] 2025-10-18 ### Changed - Added a module to add explicit units (#338) - fixed a bug that could lead to incorrect profiling information on non-blocking cuda loops (#341) - fixed a bug that could lead to incorrect energy budget when shearing box and fargo were both enabled (#346) - fixed a bug that led to incorrect BX2 reconstruction when axis is not used on both sides of the domain (#345) - fixed a bug that led to incorrect reflective boundary conditions on B when DIMENSIONS < 3 (#345) - fixed a bug that led to incorrect dust stopping time when the adiabatic equation of state is used with "size" drag law (#353) ### Added - documentation for the continuous integration (#354) --------- Co-authored-by: Victor Réville Co-authored-by: Hal Bal Co-authored-by: Jean Kempf Co-authored-by: Victor Réville <47865059+vreville@users.noreply.github.com> --- CHANGELOG.md | 14 +++ CMakeLists.txt | 4 +- doc/source/conf.py | 2 +- doc/source/faq.rst | 13 ++- doc/source/index.rst | 1 + doc/source/reference/idefix.ini.rst | 25 +++- doc/source/testing.rst | 117 +++++++++++++++++++ doc/source/testing/idfxTest.rst | 172 ++++++++++++++++++++++++++++ reference | 2 +- src/CMakeLists.txt | 2 + src/fluid/addSourceTerms.hpp | 3 - src/fluid/boundary/axis.cpp | 53 ++------- src/fluid/boundary/axis.hpp | 5 +- src/fluid/boundary/boundary.hpp | 50 ++++---- src/fluid/calcRightHandSide.hpp | 12 +- src/fluid/drag.hpp | 4 +- src/global.cpp | 3 + src/global.hpp | 2 + src/gravity/gravity.cpp | 16 ++- src/main.cpp | 5 + src/units.cpp | 55 +++++++++ src/units.hpp | 87 ++++++++++++++ 22 files changed, 563 insertions(+), 84 deletions(-) create mode 100644 doc/source/testing.rst create mode 100644 doc/source/testing/idfxTest.rst create mode 100644 src/units.cpp create mode 100644 src/units.hpp diff --git a/CHANGELOG.md b/CHANGELOG.md index b0732bb58..3778d514e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.2.02] 2025-10-18 +### Changed + +- Added a module to add explicit units (#338) +- fixed a bug that could lead to incorrect profiling information on non-blocking cuda loops (#341) +- fixed a bug that could lead to incorrect energy budget when shearing box and fargo were both enabled (#346) +- fixed a bug that led to incorrect BX2 reconstruction when axis is not used on both sides of the domain (#345) +- fixed a bug that led to incorrect reflective boundary conditions on B when DIMENSIONS < 3 (#345) +- fixed a bug that led to incorrect dust stopping time when the adiabatic equation of state is used with "size" drag law (#353) + +### Added + +- documentation for the continuous integration (#354) + ## [2.2.01] 2025-04-16 ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b62b7542..364ee4ef3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) set(Idefix_VERSION_MINOR 2) -set(Idefix_VERSION_PATCH 01) +set(Idefix_VERSION_PATCH 02) -project (idefix VERSION 2.2.01) +project (idefix VERSION 2.2.02) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) diff --git a/doc/source/conf.py b/doc/source/conf.py index 6f0cffa44..595672c25 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,7 @@ author = 'Geoffroy Lesur' # The full version, including alpha/beta/rc tags -release = '2.2.00' +release = '2.2.02' diff --git a/doc/source/faq.rst b/doc/source/faq.rst index 4c7bc056b..4dd198318 100644 --- a/doc/source/faq.rst +++ b/doc/source/faq.rst @@ -61,9 +61,18 @@ How can I stop the code without loosing the current calculation? I'm doing performance measures. How do I disable all outputs in *Idefix*? Add ``-nowrite`` when you call *Idefix* executable. +VTK output appears corrupted when running with MPI (OpenMPI) + Some OpenMPI configurations (notably OpenMPI 4 with the ompio component) can produce corrupted VTK/VTU output when running with MPI enabled. This appears to be caused by bugs in OpenMPI's ompio I/O component. + Disable ompio so OpenMPI falls back to ROMIO (MPICH's MPI-IO), which is typically more stable: -Developement ------------- + .. code-block:: console + + mpirun --mca io ^ompio ... + + This has resolved intermittent corruption for several users. See issue #348 for discussion and reports. + +Development +----------- I have a serious bug (e.g. segmentation fault), in my setup, how do I proceed? Add ``-DIdefix_DEBUG=ON`` to ``cmake`` and recompile to find out exactly where the code crashes (see :ref:`debugging`). diff --git a/doc/source/index.rst b/doc/source/index.rst index 411d73d6d..d36ab39be 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -122,6 +122,7 @@ The Idefix collaboration benefited from funding from the “Programme National d reference modules programmingguide + testing performances kokkos contributing diff --git a/doc/source/reference/idefix.ini.rst b/doc/source/reference/idefix.ini.rst index 1fa05cc67..8b558722f 100644 --- a/doc/source/reference/idefix.ini.rst +++ b/doc/source/reference/idefix.ini.rst @@ -265,8 +265,10 @@ section as followed: +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | Mcentral | real | | Mass of the central object when a central potential is enabled (see above). Default is 1. | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ -| gravCst | real | | Set the value of the gravitational constant :math:`G_c` used by the central | +| gravCst | real or string | | Set the value of the gravitational constant :math:`G_c` used by the central | | | | | mass potential and self-gravitational potential (when enabled) ). Default is 1. | +| | | | If set to "units", Idefix will compute the gravitational constant from the user units | +| | | | defined in the units section (see :ref:`unitsSection`). | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ | bodyForce | string | | Adds an acceleration vector to each cell of the domain. The only value allowed | | | | | is ``userdef``. The ``Gravity`` class then expects a user-defined bodyforce function to | @@ -310,7 +312,28 @@ of gravitational potential in the ``Gravity`` section (see :ref:`gravitySection` | | | | Default is 1 (i.e. self-gravity is computed at every cycle). | +----------------+-------------------------+---------------------------------------------------------------------------------------------+ +.. _unitsSection: +``Units`` section +------------------ + +This optional section controls how the code handles units. By default, Idefix work "unit free" and this section is not useful. +However, some physical processes like self-gravity or radiative transfer require the explicit use of units to define natural constants. +These entries define how one code units should be converted to CGS units. When used, the following values should be set: + ++----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ +| Entry name | Parameter type | Comment | ++================+====================+===========================================================================================================+ +| length | float | The code unit length: 1 code unit = `length` cm. 1 by default. | ++----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ +| velocity | float | The code unit velocity: 1 code unit = `velocity` cm/s. 1 by default. | ++----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ +| density | float | The code unit density: 1 code unit = `density` g/cm^3. 1 by default. | ++----------------+--------------------+-----------------------------------------------------------------------------------------------------------+ + +.. note:: + The units module automatically reconstruct all of the units (time, temperature, magnetic field, etc.) from the above 3 fundamental constants. These can + all be obtained from the global object ``idfx::units``. Note however that the code outputs remain in code units, even if `[Units]` are defined. ``RKL`` section ------------------ diff --git a/doc/source/testing.rst b/doc/source/testing.rst new file mode 100644 index 000000000..888ac2057 --- /dev/null +++ b/doc/source/testing.rst @@ -0,0 +1,117 @@ +Continuous Integration (CI) tests +================================ + +This document describes the GitHub Actions continuous-integration setup used to run the Idefix +test-suite. The CI is implemented by two workflows checked in .github/workflows: + +- .github/workflows/idefix-ci.yml +- .github/workflows/idefix-ci-jobs.yml + +Overview +-------- + +The CI is split in two layers: + +- A top-level workflow (.github/workflows/idefix-ci.yml) that: + + - runs a Linter job (pre-commit) on push / PR / manual dispatch, + - then calls a reusable workflow for different compiler/backends (intel, gcc, cuda) + providing two inputs: TESTME_OPTIONS and IDEFIX_COMPILER. + +- A reusable workflow (.github/workflows/idefix-ci-jobs.yml) that: + + - defines the actual test jobs grouped by physics domain (ShocksHydro, ParabolicHydro, + ShocksMHD, ParabolicMHD, Fargo, ShearingBox, SelfGravity, Planet, Dust, Braginskii, + Examples, Utils), + - runs test scripts on self-hosted runners, + - expects the repository to be checked out with submodules, + - invokes the repository-provided CI helper scripts to configure / build / run tests. + +Key configuration points +------------------------ + +- Inputs passed from the top-level workflow: + + - TESTME_OPTIONS (string): flags forwarded to the per-test runner (examples: -cuda, -Werror, + -intel, -all). + - IDEFIX_COMPILER (string): which compiler the tests should use (e.g. icc, gcc, nvcc). + +- Environment variables set by the reusable workflow: + + - IDEFIX_COMPILER, TESTME_OPTIONS, PYTHONPATH, IDEFIX_DIR + +- Linter job: + + - Runs only when repository is the main project (not arbitrary forks). + - Uses actions/setup-python and runs pre-commit (pre-commit/action@v3 and pre-commit-ci/lite). + - Prevents regressions in style and common mistakes before running heavy test jobs. + +- Test execution: + + - All test jobs call the repository script scripts/ci/run-tests with a test directory + and the TESTME_OPTIONS flags. Example invocation (from the workflows): + scripts/ci/run-tests $IDEFIX_DIR/test/HD/sod -all $TESTME_OPTIONS + + - The reusable workflow is written to execute many test directories in separate job steps, + so each physics group is kept logically separated in CI logs. + +Runners and prerequisites +------------------------- + +- The heavy numerical tests run on self-hosted runners (see runs-on: self-hosted). + The CI assumes appropriate hardware and dependencies are available on those runners + (compilers, MPI, GPUs when CUDA/HIP flags are used, required system libraries). + +- The workflows check out the repository and its submodules. Submodules must be available + on the CI machines. + +How tests are driven (testme scripts) +------------------------------------- + +Each test directory contains a small Python "testMe" driver that uses the helper Python +class documented in the repository: + +- See the test helper documentation: :doc:`idfxTest ` + +That helper (idfxTest) is responsible for: + +- parsing TESTME_OPTIONS-like flags (precision, MPI, CUDA, reconstruction, vector potential, etc.), +- calling configure / compile / run, +- performing standard python checks and non-regression (RMSE) comparisons against + reference dumps, +- optionally creating / updating reference dumps (init mode). + +Practical examples +------------------ + +- Example of a CI invocation (triggered by workflows): + + - Top-level workflow calls the reusable jobs workflow for each compiler/back-end, e.g. + TESTME_OPTIONS="-cuda -Werror" IDEFIX_COMPILER=nvcc + +- Running tests locally (developer machine) + - You can mimic what CI does by calling the repository helper script directly. Example: + scripts/ci/run-tests /path/to/idefix/test/HD/sod -all -mpi -dec 2 2 -reconstruction 3 -single + +Notes for maintainers +--------------------- + +- The reusable jobs workflow contains a commented concurrency block for optional cancellation + of in-flight runs — consider enabling it if you want to auto-cancel redundant CI runs. +- Because tests are run on self-hosted runners, ensure the pools have the required compilers, + MPI stacks and GPU drivers for the requested TESTME_OPTIONS. +- Keep TESTME_OPTIONS in sync with the options understood by the test helper documented in + :doc:`idfxTest `. + +Relevant files +-------------- + +- Workflow entry point: .github/workflows/idefix-ci.yml +- Reusable jobs: .github/workflows/idefix-ci-jobs.yml +- Test helper documentation: :doc:`idfxTest ` + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + testing/idfxTest.rst diff --git a/doc/source/testing/idfxTest.rst b/doc/source/testing/idfxTest.rst new file mode 100644 index 000000000..b425d3fe2 --- /dev/null +++ b/doc/source/testing/idfxTest.rst @@ -0,0 +1,172 @@ +========= +idfxTest +========= + +.. autoclass:: idfxTest + :members: + :undoc-members: + :show-inheritance: + +Overview +-------- + +The ``idfxTest`` class provides a high-level interface for automating the configuration, compilation, execution, and regression testing of Idefix simulations. It is designed to be used in test scripts (such as ``testme.py``) to streamline the testing workflow, including handling reference files and plotting differences. + +Constructor and Command-Line Options +------------------------------------ + +The constructor parses command-line arguments using ``argparse``. These options can be passed directly to the test script or via the command line. The following options are available: + +.. list-table:: + :header-rows: 1 + + * - Option + - Attribute + - Description + * - ``-noplot`` + - ``noplot`` + - Disable plotting in standard tests (default: True). + * - ``-ploterr`` + - ``ploterr`` + - Enable plotting of differences when regression tests fail. + * - ``-cmake OPT [OPT ...]`` + - ``cmake`` + - Extra CMake options (list of strings). + * - ``-definitions FILE`` + - ``definitions`` + - Specify a custom ``definitions.hpp`` file. + * - ``-dec NX NY NZ`` + - ``dec`` + - MPI domain decomposition (list of integers). + * - ``-check`` + - ``check`` + - Only perform regression tests without compilation. + * - ``-cuda`` + - ``cuda`` + - Enable CUDA backend for Nvidia GPUs. + * - ``-intel`` + - ``intel`` + - Use Intel OneAPI compilers. + * - ``-hip`` + - ``hip`` + - Enable HIP backend for AMD GPUs. + * - ``-single`` + - ``single`` + - Enable single precision. + * - ``-vectPot`` + - ``vectPot`` + - Enable vector potential formulation. + * - ``-reconstruction N`` + - ``reconstruction`` + - Set reconstruction scheme (2=PLM, 3=LimO3, 4=PPM). + * - ``-idefixDir PATH`` + - ``idefixDir`` + - Set the directory for Idefix source files (default: ``$IDEFIX_DIR``). + * - ``-mpi`` + - ``mpi`` + - Enable MPI parallelism. + * - ``-all`` + - ``all`` + - Run the full test suite with multiple configurations. + * - ``-init`` + - ``init`` + - Reinitialize reference files for non-regression tests. + * - ``-Werror`` + - ``Werror`` + - Treat compiler warnings as errors. + +Main Methods +------------ + +.. list-table:: + :header-rows: 1 + + * - Method + - Description + * - ``configure`` + - Runs CMake to configure the build system for Idefix, using options set by command-line flags (e.g., precision, MPI, CUDA, etc.). + * - ``compile`` + - Compiles the Idefix code using ``make`` with the specified number of parallel jobs. + * - ``run`` + - Executes the Idefix binary, optionally with MPI, using the provided input file and runtime options. + * - ``checkOnly`` + - Performs regression testing only, without compiling or running the code (useful for checking outputs after a manual run). + * - ``standardTest`` + - Runs any Python-based standard tests (e.g., ``testidefix.py``) present in the test directory for additional validation. + * - ``nonRegressionTest`` + - Compares the output dump file to a reference file using RMSE; fails if the error exceeds the tolerance. + * - ``compareDump`` + - Compares two arbitrary dump files using the same logic as ``nonRegressionTest``. + * - ``makeReference`` + - Copies the specified output file to the reference directory, updating the reference for future regression tests. + +Usage Example +------------- + +Below is an example inspired by ``testme.py`` from ``test/HD/sod/testme.py``. This demonstrates a typical workflow for running tests and performing regression checks. + +.. code-block:: python + + import pytools.idfx_test as tst + + name = "dump.0001.dmp" + + def testMe(test): + test.configure() + test.compile() + inifiles = ["idefix.ini", "idefix-hll.ini", "idefix-hllc.ini", "idefix-tvdlf.ini"] + if test.reconstruction == 4: + inifiles = ["idefix-rk3.ini", "idefix-hllc-rk3.ini"] + + # Loop over all ini files for this test + for ini in inifiles: + test.run(inputFile=ini) + if test.init: + test.makeReference(filename=name) + test.standardTest() + test.nonRegressionTest(filename=name) + + test = tst.idfxTest() + + if not test.all: + if test.check: + test.checkOnly(filename=name) + else: + testMe(test) + else: + test.noplot = True + for rec in range(2, 5): + test.vectPot = False + test.single = False + test.reconstruction = rec + test.mpi = False + testMe(test) + + # Test in single precision + test.reconstruction = 2 + test.single = True + testMe(test) + +How to Run +---------- + +You can run the test script from the command line, passing any of the supported options. For example: + +.. code-block:: bash + + python testme.py -mpi -dec 2 2 -reconstruction 3 -single -ploterr -idefixDir /path/to/idefix + +This will configure, compile, and run the test in MPI mode with a 2x2 domain decomposition, third-order reconstruction, single precision, and plotting enabled for regression errors. + +Reference File Management +------------------------ + +- Reference files are stored in ``$IDEFIX_DIR/reference/``. +- The filename is generated based on precision, reconstruction, input file, and vector potential settings. +- Use ``test.init`` to regenerate reference files (dangerous: overwrites existing references). + +Regression Testing +------------------ + +- The ``nonRegressionTest`` method compares output dumps to reference files using RMSE. +- If the error exceeds the tolerance, the test fails and (optionally) plots the difference. diff --git a/reference b/reference index c32894208..c4082b99a 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit c328942083b618a85b84eb16ce9fd35e67c00597 +Subproject commit c4082b99a4c542def3177c96cb35b1c9d9002f18 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 727f21504..08dd93c61 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,6 +28,8 @@ target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/reduce.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/setup.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/setup.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/units.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/units.cpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/timeIntegrator.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/timeIntegrator.cpp ) diff --git a/src/fluid/addSourceTerms.hpp b/src/fluid/addSourceTerms.hpp index a0cb5a719..e3f745868 100644 --- a/src/fluid/addSourceTerms.hpp +++ b/src/fluid/addSourceTerms.hpp @@ -83,9 +83,6 @@ struct Fluid_AddSourceTermsFunctor { Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX2,k,j,i); Uc(MX2,k,j,i) += - TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * Vc(VX1,k,j,i); } - if(haveFargo) { - Uc(MX1,k,j,i) += TWO_F * dt * Vc(RHO,k,j,i) * OmegaZ * sbS * x1(i); - } #endif // fetch fargo velocity when required [[maybe_unused]] real fargoV = ZERO_F; diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index 3ad67f0a0..e447aefc8 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -358,65 +358,28 @@ void Axis::EnforceAxisBoundary(int side) { // Reconstruct Bx2s taking care of the sides where an axis is lying -void Axis::ReconstructBx2s() { - idfx::pushRegion("Axis::ReconstructBx2s"); +void Axis::RegularizeBX2s() { + idfx::pushRegion("Axis::RegularizeBX2s"); #if DIMENSIONS >= 2 && MHD == YES IdefixArray4D Vs = this->Vs; - IdefixArray3D Ax1=data->A[IDIR]; - IdefixArray3D Ax2=data->A[JDIR]; - IdefixArray3D Ax3=data->A[KDIR]; - int nstart = data->beg[JDIR]-1; - int nend = data->end[JDIR]; - int ntot = data->np_tot[JDIR]; - - bool haveleft = axisLeft; - bool haveright = axisRight; - - if(haveleft) { - // This loop is a copy of ReconstructNormalField, with the proper sign when we cross the axis - idefix_for("Axis::ReconstructBX2sLeft",0,data->np_tot[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int i) { - for(int j = nstart ; j>=0 ; j-- ) { - Vs(BX2s,k,j,i) = 1.0 / Ax2(k,j,i) * ( Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i) - +(D_EXPAND( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) , - , - - ( Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - - Ax3(k,j,i) * Vs(BX3s,k,j,i) )))); - } - } - ); - } - if(haveright) { - // This loop is a copy of ReconstructNormalField, with the proper sign when we cross the axis - idefix_for("Axis::ReconstructBX2sRight",0,data->np_tot[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int i) { - for(int j = nend ; jend[JDIR]; int jleft = data->beg[JDIR]; if(isTwoPi) { #ifdef AXIS_BX2S_USE_ATHENA_REGULARISATION - if(haveleft) FixBx2sAxisGhostAverage(left); - if(haveright) FixBx2sAxisGhostAverage(right); + if(axisLeft) FixBx2sAxisGhostAverage(left); + if(axisRight) FixBx2sAxisGhostAverage(right); #else - if(haveleft) FixBx2sAxis(left); - if(haveright) FixBx2sAxis(right); + if(axisLeft) FixBx2sAxis(left); + if(axisRight) FixBx2sAxis(right); #endif } else { + bool haveleft = axisLeft; + bool haveright = axisRight; idefix_for("Axis:BoundaryAvg",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA (int k, int i) { if(haveleft) { diff --git a/src/fluid/boundary/axis.hpp b/src/fluid/boundary/axis.hpp index a1c3054b3..b74697862 100644 --- a/src/fluid/boundary/axis.hpp +++ b/src/fluid/boundary/axis.hpp @@ -29,7 +29,7 @@ class Axis { void RegularizeEMFs(); // Regularize the EMF sitting on the axis void RegularizeCurrent(); // Regularize the currents along the axis void EnforceAxisBoundary(int side); // Enforce the boundary conditions (along X2) - void ReconstructBx2s(); // Reconstruct BX2s in the ghost zone using divB=0 + void RegularizeBX2s(); // Regularize BX2s on the axis void ShowConfig(); @@ -42,6 +42,8 @@ class Axis { void ExchangeMPI(int side); // Function has to be public for GPU, but its technically // a private function + bool haveLeftAxis() const { return axisLeft; } ///< Check if the axis is on the left + bool haveRightAxis() const { return axisRight; } ///< Check if the axis is on the right private: bool isTwoPi = false; @@ -147,7 +149,6 @@ Axis::Axis(Boundary *boundary) { Kokkos::deep_copy(symmetryVc, symmetryVcHost); if constexpr(Phys::mhd) { - idfx::cout << "Phys MHD" << std::endl; symmetryVs = IdefixArray1D("Axis:SymmetryVs",DIMENSIONS); IdefixArray1D::HostMirror symmetryVsHost = Kokkos::create_mirror_view(symmetryVs); // Init the array diff --git a/src/fluid/boundary/boundary.hpp b/src/fluid/boundary/boundary.hpp index 86909936f..897c1ce10 100644 --- a/src/fluid/boundary/boundary.hpp +++ b/src/fluid/boundary/boundary.hpp @@ -112,6 +112,8 @@ class Boundary { IdefixArray4D Vs; ///< reference to face-centered array that we should sync std::unique_ptr axis; ///< Axis object, initialised if needed. bool haveAxis{false}; + bool haveLeftAxis{false}; ///< True if the left boundary is an axis + bool haveRightAxis{false}; ///< True if the right boundary is an axis private: friend class Axis; @@ -137,6 +139,8 @@ Boundary::Boundary(Fluid* fluid) { if(data->haveAxis) { this->axis = std::make_unique(this); this->haveAxis = true; + this->haveLeftAxis = axis->haveLeftAxis(); + this->haveRightAxis = axis->haveRightAxis(); } @@ -464,30 +468,34 @@ void Boundary::ReconstructNormalField(int dir) { if(dir==JDIR) { nstart = data->beg[JDIR]-1; nend = data->end[JDIR]; - if(!this->haveAxis) { - idefix_for("ReconstructBX2s",0,data->np_tot[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA (int k, int i) { - if(reconstructLeft) { - for(int j = nstart ; j>=0 ; j-- ) { - Vs(BX2s,k,j,i) = 1.0 / Ax2(k,j,i) * ( Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i) - +(D_EXPAND( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) , - , - + Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) ))); - } + #if DIMENSIONS == 3 + const int signLeft = haveLeftAxis ? -1 : 1; // left axis is in the negative direction + const int signRight = haveRightAxis ? -1 : 1; // right axis is in the negative direction + #endif + + idefix_for("ReconstructBX2s",0,data->np_tot[KDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA (int k, int i) { + if(reconstructLeft) { + for(int j = nstart ; j>=0 ; j-- ) { + Vs(BX2s,k,j,i) = 1.0 / Ax2(k,j,i) * ( Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i) + +(D_EXPAND( Ax1(k,j,i+1) * Vs(BX1s,k,j,i+1) - Ax1(k,j,i) * Vs(BX1s,k,j,i) , + , + + signLeft*(Ax3(k+1,j,i) * Vs(BX3s,k+1,j,i) - Ax3(k,j,i) * Vs(BX3s,k,j,i) )))); } - if(reconstructRight) { - for(int j = nend ; jReconstructBx2s(); + axis->RegularizeBX2s(); } } #endif @@ -686,7 +694,7 @@ void Boundary::EnforceReflective(int dir, BoundarySide side ) { const int jref = (dir==JDIR) ? 2*(jghost + side*nxj) - j - 1 : j; const int kref = (dir==KDIR) ? 2*(kghost + side*nxk) - k - 1 : k; - const int sign = (n == VX1+dir) ? -1.0 : 1.0; + const int sign = (n == VX1+dir || (n >= BX1 && n != BX1+dir)) ? -1.0 : 1.0; Vc(n,k,j,i) = sign * Vc(n,kref,jref,iref); }); diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp index 38ef76d5d..7777d1c22 100644 --- a/src/fluid/calcRightHandSide.hpp +++ b/src/fluid/calcRightHandSide.hpp @@ -472,12 +472,18 @@ struct Fluid_CalcRHSFunctor { // Body force if(needBodyForce) { - rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * bodyForce(dir,k,j,i); + real bf = bodyForce(dir,k,j,i); + #if GEOMETRY == CARTESIAN + if(haveFargo && dir==IDIR) { + // Remove Body force that is already compensated by Fargo shear*Rotation + bf -= -2*Omega*sbS * x1(i); + } + #endif + rhs[MX1+dir] += dt * Vc(RHO,k,j,i) * bf; if constexpr(Phys::pressure) { // rho * v . f, where rhov is taken as a volume average of Flux(RHO) rhs[ENG] += HALF_F * dtdV * dl * - (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * - bodyForce(dir,k,j,i); + (Flux(RHO,k,j,i) + Flux(RHO, k+koffset, j+joffset, i+ioffset)) * bf; } // Pressure // Particular cases if we do not sweep all of the components diff --git a/src/fluid/drag.hpp b/src/fluid/drag.hpp index 29eca3097..63b5e1bdb 100644 --- a/src/fluid/drag.hpp +++ b/src/fluid/drag.hpp @@ -37,8 +37,8 @@ class GammaDrag { // Assume a fixed size, hence for both Epstein or Stokes, gamma~1/rho_g/cs // Get the sound speed #if HAVE_ENERGY == 1 - cs = std::sqrt(eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i) - *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i))); + cs = std::sqrt( eos.GetGamma(VcGas(PRS,k,j,i),VcGas(RHO,k,j,i)) + *VcGas(PRS,k,j,i)/VcGas(RHO,k,j,i)); #else cs = eos.GetWaveSpeed(k,j,i); #endif diff --git a/src/global.cpp b/src/global.cpp index 4d9499d96..1ed33ba11 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -11,6 +11,7 @@ #include "idefix.hpp" #include "global.hpp" #include "profiler.hpp" +#include "units.hpp" #ifdef WITH_MPI #include "mpi.hpp" @@ -29,6 +30,7 @@ IdefixOutStream cout; IdefixErrStream cerr; Profiler prof; LoopPattern defaultLoopPattern; +Units units; #ifdef DEBUG static int regionIndent = 0; @@ -64,6 +66,7 @@ int initialize() { void pushRegion(const std::string& kName) { Kokkos::Profiling::pushRegion(kName); if(prof.perfEnabled) { + Kokkos::fence(); prof.currentRegion = prof.currentRegion->GetChild(kName); prof.currentRegion->Start(); } diff --git a/src/global.hpp b/src/global.hpp index 265699b29..5a9ea70fb 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -20,6 +20,7 @@ void safeExit(int ); // Exit the code class IdefixOutStream; class IdefixErrStream; class Profiler; +class Units; extern int prank; //< parallel rank extern int psize; @@ -29,6 +30,7 @@ extern Profiler prof; //< profiler (for memory & performance u extern double mpiCallsTimer; //< time significant MPI calls extern LoopPattern defaultLoopPattern; //< default loop patterns (for idefix_for loops) extern bool warningsAreErrors; //< whether warnings should be considered as errors +extern Units units; //< Units for the run void pushRegion(const std::string&); void popRegion(); diff --git a/src/gravity/gravity.cpp b/src/gravity/gravity.cpp index c0096c51e..a39b00063 100644 --- a/src/gravity/gravity.cpp +++ b/src/gravity/gravity.cpp @@ -12,13 +12,27 @@ #include "dataBlock.hpp" #include "output.hpp" #include "input.hpp" +#include "units.hpp" Gravity::Gravity(Input &input, DataBlock *datain) { idfx::pushRegion("Gravity::Gravity"); this->data = datain; // Gravitational constant G - this->gravCst = input.GetOrSet("Gravity","gravCst",0, 1.0); + // should we compute it from the units? + if(input.CheckEntry("Gravity","gravCst")>=0) { + if(input.Get("Gravity","gravCst",0).compare("units") == 0) { + // User ask us to compute the gravitational constants from the units + + this->gravCst = idfx::units.GetDensity() * idfx::units.GetTime() * idfx::units.GetTime() + * idfx::units.G; + } else { + this->gravCst = input.Get("Gravity","gravCst",0); + } + } else { // default value to 1.0 + this->gravCst = 1.0; + } + // Gravitational potential int nPotential = input.CheckEntry("Gravity","potential"); if(nPotential >=0) { diff --git a/src/main.cpp b/src/main.cpp index d21793d1d..164c134ee 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -30,6 +30,7 @@ #include "idefix.hpp" #include "profiler.hpp" #include "input.hpp" +#include "units.hpp" #include "grid.hpp" #include "gridHost.hpp" #include "fluid.hpp" @@ -79,6 +80,9 @@ int main( int argc, char* argv[] ) { input.PrintLogo(); idfx::cout << "Main: initialization stage." << std::endl; + // Init the units when needed + idfx::units.Init(input); + // Allocate the grid on device Grid grid(input); // Allocate the grid image on host @@ -112,6 +116,7 @@ int main( int argc, char* argv[] ) { << std::endl; } input.ShowConfig(); + idfx::units.ShowConfig(); grid.ShowConfig(); data.ShowConfig(); Tint.ShowConfig(); diff --git a/src/units.cpp b/src/units.cpp new file mode 100644 index 000000000..2baf97b1e --- /dev/null +++ b/src/units.cpp @@ -0,0 +1,55 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include "units.hpp" +#include "idefix.hpp" +#include "input.hpp" + + +void idfx::Units::Init(Input &input) { + if(input.CheckBlock("Units")) { + this->_length = input.GetOrSet("Units","length",0,1.0); + this->_velocity = input.GetOrSet("Units","velocity",0,1.0); + this->_density = input.GetOrSet("Units","density",0,1.0); + this->ComputeUnits(); + } +} + +void idfx::Units::SetDensity(const real in) { + this->_density = in; + this->ComputeUnits(); +} + +void idfx::Units::SetLength(const real in) { + this->_length = in; + this->ComputeUnits(); +} + +void idfx::Units::SetVelocity(const real in) { + this->_velocity = in; + this->ComputeUnits(); +} + +void idfx::Units::ShowConfig() { + if(_isInitialized) { + idfx::cout << "Units: [Length] = " << this->_length << " cm" << std::endl; + idfx::cout << "Units: [Velocity] = " << this->_velocity << " cm/s" << std::endl; + idfx::cout << "Units: [Density] = " << this->_density << " g/cm3" << std::endl; + idfx::cout << "Units: [Energy] = " << this->_energy << " erg/cm3" << std::endl; + idfx::cout << "Units: [Time] = " << this->_time << " s" << std::endl; + idfx::cout << "Units: [Temperature] = " << this->_Kelvin << " K" << std::endl; + idfx::cout << "Units: [Mag. Field] = " << this->_magField << " G" << std::endl; + } +} + +void idfx::Units::ComputeUnits() { + this->_isInitialized = true; + this->_magField = std::sqrt(4*M_PI*_density*_velocity*_velocity); + this->_Kelvin = u * _velocity*_velocity/k_B; + this->_energy = _density*_velocity*_velocity; + this->_time = _length/_velocity; +} diff --git a/src/units.hpp b/src/units.hpp new file mode 100644 index 000000000..525e44bc0 --- /dev/null +++ b/src/units.hpp @@ -0,0 +1,87 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef UNITS_HPP_ +#define UNITS_HPP_ + +#include "idefix.hpp" + +// Units class implemented in CGS +// Constants taken from Astropy +class Input; +namespace idfx { +class Units { + public: + void Init(Input &input); + + + const real u{1.6605390666e-24}; // Atomic mass unit (g) + const real m_p{1.67262192369e-24}; // Proton mass unit (g) + const real m_n{1.67492749804e-24}; // neutron mass unit (g) + const real m_e{9.1093837015e-28}; // electron mass unit (g) + const real k_B{1.380649e-16}; // Boltzmann constant (erg/K) + const real sigma_sb{5.6703744191844314e-05}; // Stephan Boltzmann constant (g/(K^4 s^3)) + const real ar{7.5646e-15}; // Radiation constant + // = 4*sigma_sb/c (g/(K^4 s^2 cm)) + const real c{29979245800.0}; // Speed of light (cm/s) + const real M_sun{1.988409870698051e+33}; // Solar mass (g) + const real R_sun{69570000000.0}; // Solar radius (cm) + const real M_earth{5.972167867791379e+27}; // Earth mass (g) + const real R_earth{6.3781e8}; // Earth radius (cm) + const real G{6.674299999999999e-8}; // Gravitational constant (cm3 / (g s2)) + const real h{6.62607015e-27}; // Planck constant (erg.s) + const real pc{3.08568e+18}; // Parsec (cm) + const real au{1.49597892e13}; // Astronomical unit (cm) + const real e{4.80320425e-10}; // Elementary charge (cm^3/2 g^1/2 s^-1) + + + // User-defined units, non user-modifiable + // L (cm) = L (code) * Units::length + KOKKOS_INLINE_FUNCTION real GetLength() const {return _length;} + // V(cm/s) = V(code) * Units::velocity + KOKKOS_INLINE_FUNCTION real GetVelocity() const {return _velocity;} + // density (g/cm^3) = density(code) * Units::density + KOKKOS_INLINE_FUNCTION real GetDensity() const {return _density;} + + // Deduced units from user-defined units + // T(K) = P(code)/rho(code) * mu * Units::Kelvin + KOKKOS_INLINE_FUNCTION real GetKelvin() const {return _Kelvin;} + // B(G) = B(code) * Units::MagField + KOKKOS_INLINE_FUNCTION real GetMagField() const {return _magField;} + // energy(erg/cm3) = energy(code) * Units::energy + KOKKOS_INLINE_FUNCTION real GetEnergy() const {return _energy;} + // time(s) = time(code) * Units::time + KOKKOS_INLINE_FUNCTION real GetTime() const {return _time;} + + KOKKOS_INLINE_FUNCTION bool GetIsInitialized() const {return _isInitialized;} + + // code-style change of the units + void SetLength(const real); + void SetVelocity(const real); + void SetDensity(const real); + + // Show the configuration + void ShowConfig(); + + private: + bool _isInitialized{false}; + // read-write variables + real _length{1.0}; + real _velocity{1.0}; + real _density{1.0}; + + // Deduced units from user-defined units + real _Kelvin{1.0}; + real _magField{1.0}; + real _time{1.0}; + real _energy{1.0}; + + // Recompute deduced user-units + void ComputeUnits(); +}; +} // namespace idfx +#endif // UNITS_HPP_ From 9e255516faf5fd4c3903eed5933e0fc91c222a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 14 Apr 2026 13:35:29 +0200 Subject: [PATCH 07/10] SEC: fix insecure GitHub Actions settings and enable automated audits with zizmor + pre-commit (#373) * SEC: switch GHA refs to immutable hashes with pinact * SEC: disable default gha permissions * SEC: avoid leaking credentials * SEC: enable security audits with zizmor + pre-commit --- .github/workflows/idefix-ci-doc.yml | 5 +++- .github/workflows/idefix-ci-jobs.yml | 38 +++++++++++++++++++--------- .github/workflows/idefix-ci.yml | 12 ++++++--- .pre-commit-config.yaml | 5 ++++ 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.github/workflows/idefix-ci-doc.yml b/.github/workflows/idefix-ci-doc.yml index 036404654..7422be2f3 100644 --- a/.github/workflows/idefix-ci-doc.yml +++ b/.github/workflows/idefix-ci-doc.yml @@ -9,13 +9,16 @@ on: paths-ignore: - '.github/ISSUE_TEMPLATE/*' +permissions: {} jobs: ReadTheDocs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + persist-credentials: false - name: install doxygen run: sudo apt-get install -y doxygen - name: install python dependencies diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index 275ea1ce2..32eef5911 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -21,14 +21,17 @@ env: PYTHONPATH: ${{ github.workspace }} IDEFIX_DIR: ${{ github.workspace }} +permissions: {} + jobs: ShocksHydro: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Sod test run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/sod -all $TESTME_OPTIONS - name: Isothermal Sod test @@ -40,9 +43,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Viscous flow past cylinder run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/ViscousFlowPastCylinder -all $TESTME_OPTIONS - name: Viscous disk @@ -54,9 +58,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: MHD Sod test run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sod -all $TESTME_OPTIONS - name: MHD Isothermal Sod test @@ -72,9 +77,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Ambipolar C Shock run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AmbipolarCshock -all $TESTME_OPTIONS - name: Ambipolar C Shock 3D @@ -91,9 +97,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Fargo + planet run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/FargoPlanet -all $TESTME_OPTIONS - name: Fargo MHD spherical @@ -104,9 +111,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Hydro shearing box run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/ShearingBox -all $TESTME_OPTIONS - name: MHD shearing box @@ -117,9 +125,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Jeans Instability run: scripts/ci/run-tests $IDEFIX_DIR/test/SelfGravity/JeansInstability -all $TESTME_OPTIONS - name: Random sphere spherical @@ -136,9 +145,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: 3 body run: scripts/ci/run-tests $IDEFIX_DIR/test/Planet/Planet3Body -all $TESTME_OPTIONS - name: migration @@ -157,9 +167,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Energy conservation run: scripts/ci/run-tests $IDEFIX_DIR/test/Dust/DustEnergy -all $TESTME_OPTIONS - name: Dusty wave @@ -170,9 +181,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: MTI run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/MTI -all $TESTME_OPTIONS - name: Spherical anisotropic diffusion @@ -187,9 +199,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Run examples test run: cd test && ./checks_examples.sh $TEST_OPTIONS @@ -198,9 +211,10 @@ jobs: runs-on: self-hosted steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: submodules: recursive + persist-credentials: false - name: Lookup table run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/lookupTable -all $TESTME_OPTIONS - name: Dump Image diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index cb9461431..b6ed719e9 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -9,18 +9,22 @@ on: paths-ignore: - '.github/ISSUE_TEMPLATE/*' +permissions: {} + jobs: Linter: # Don't do this in forks if: ${{ github.repository == 'idefix-code/idefix' || github.repository == 'glesur/idefix' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + persist-credentials: false + - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 with: python-version: 3.x - - uses: pre-commit/action@v3.0.0 - - uses: pre-commit-ci/lite-action@v1.0.0 + - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0 + - uses: pre-commit-ci/lite-action@50143aaf27e2c42e75a5e06185a471d9582e89df # v1.0.0 if: always() icc-jobs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72360011b..f0b285315 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,11 @@ repos: - id: check-added-large-files args: ['--maxkb=100'] ## prevent files larger than 100kB from being commited (exclude git lfs files) + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.23.1 + hooks: + - id: zizmor + - repo: https://github.com/Lucas-C/pre-commit-hooks-nodejs rev: v1.1.2 hooks: From e9e68b28f23dcfaafe5b88592f1cfe26bc0b4801 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 22 Apr 2026 09:58:35 +0200 Subject: [PATCH 08/10] v2.3.0 (#376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Changed - fixed a bug that could lead to diverging results around the spherical axis in non-ideal MHD in 2.5D and 3D (#356) - refactor of the MPI exchange routines and boundary routines to avoid buildup of roundoff errors at domain faces/edges that could lead to the sudden burst of div(B) or incoherences between MPI sub-domains (#357) - fixed a bug that could lead to compilations error when targetting AMD APUs (#359) - fixed a bug that led to the generation of incorrect subviews in 2.5D with vector_potential enabled (#362) - fixed a bug that could lead to memory corruption when using the UCT_HLLD emf reconstruction scheme and DIMENSIONS < COMPONENTS (#363) - reorganise the test to separate specific IO tests from physics tests (#367) ### Added - magnetic vector potential is now accessible from pydefix when enabled (#361) - use ccache in the test suite to reduce the runtime of continuous integration (#364) - automatically detects pybind11 path with cmake when using pydefix (#367) - skeleton to run the test suite with pytest (#366) --------------------------------------------------------------------------- Co-authored-by: Clément Robert Co-authored-by: Sébastien Valat --- .github/workflows/idefix-ci-jobs.yml | 27 +- .github/workflows/idefix-ci.yml | 4 +- .gitignore | 6 + .pre-commit-config.yaml | 3 +- CHANGELOG.md | 19 + CMakeLists.txt | 51 +- doc/python_requirements.txt | 8 +- doc/source/conf.py | 2 +- doc/source/modules/pydefix.rst | 3 +- doc/source/reference/makefile.rst | 6 + doc/source/testing.rst | 12 +- doc/source/testing/idfxTest.rst | 3 + doc/source/testing/testLauncher.rst | 383 +++++++ pytest.ini | 6 + pytools/idfx_test.py | 225 +++- pytools/idfx_test_gen.py | 249 +++++ pytools/idfx_test_run.py | 257 +++++ pytools/pytest.ini | 2 + pytools/tests/__init__.py | 0 pytools/tests/test/pb1/testme.json | 11 + pytools/tests/test/pb2/testme.json | 18 + pytools/tests/test_idfx_test_gen.py | 208 ++++ pytools/tests/test_idfx_test_run.py | 88 ++ reference | 2 +- src/dataBlock/dataBlock.cpp | 1 + src/dataBlock/dataBlockHost.cpp | 35 +- src/dataBlock/dumpToFile.cpp | 14 +- src/dataBlock/fargo.cpp | 9 +- src/dataBlock/fargo.hpp | 10 +- .../RiemannSolver/MHDsolvers/storeFlux.hpp | 8 +- src/fluid/boundary/axis.cpp | 72 +- src/fluid/boundary/axis.hpp | 17 +- src/fluid/boundary/boundary.hpp | 181 ++-- src/fluid/calcRightHandSide.hpp | 2 +- src/fluid/checkDivB.hpp | 9 +- src/fluid/checkNan.hpp | 4 +- src/fluid/constrainedTransport/CMakeLists.txt | 1 + .../constrainedTransport/EMFexchange.hpp | 76 +- .../constrainedTransport.hpp | 13 +- .../enforceEMFBoundary.hpp | 119 +-- .../enforceVectorPotentialBoundary.hpp | 44 + src/global.hpp | 2 +- src/gravity/laplacian.cpp | 2 +- src/grid.hpp | 2 +- src/gridHost.cpp | 8 +- src/macros.hpp | 1 - src/main.cpp | 6 +- src/mpi.cpp | 992 ------------------ src/mpi.hpp | 269 ----- src/mpi/CMakeLists.txt | 7 + src/mpi/buffer.hpp | 195 ++++ src/mpi/exchanger.cpp | 308 ++++++ src/mpi/exchanger.hpp | 82 ++ src/mpi/mpi.cpp | 265 +++++ src/mpi/mpi.hpp | 74 ++ src/output/dump.cpp | 2 +- src/output/dump.hpp | 4 +- src/output/scalarField.hpp | 2 +- src/output/xdmf.cpp | 14 +- src/pydefix.cpp | 12 +- src/rkl/rkl.hpp | 3 +- src/setup.cpp | 4 +- src/utils/column.cpp | 2 +- src/utils/lookupTable.hpp | 24 +- test.py | 26 + test/Dust/DustEnergy/testme.json | 12 + test/Dust/DustEnergy/testme.py | 2 +- test/Dust/DustyShock/testme.json | 11 + test/Dust/DustyShock/testme.py | 2 +- test/Dust/DustyWave/testme.json | 12 + test/Dust/DustyWave/testme.py | 2 +- test/HD/FargoPlanet/testme.json | 25 + test/HD/FargoPlanet/testme.py | 2 +- test/HD/MachReflection/testme.json | 15 + test/HD/MachReflection/testme.py | 2 +- test/HD/SedovBlastWave/idefix.ini | 5 +- test/HD/SedovBlastWave/testme.json | 28 + test/HD/SedovBlastWave/testme.py | 2 +- test/HD/ShearingBox/testme.json | 15 + test/HD/ShearingBox/testme.py | 2 +- test/HD/ViscousDisk/testme.json | 15 + test/HD/ViscousDisk/testme.py | 2 +- test/HD/ViscousFlowPastCylinder/testme.json | 23 + test/HD/ViscousFlowPastCylinder/testme.py | 2 +- test/HD/sod-iso/testme.json | 33 + test/HD/sod-iso/testme.py | 2 +- test/HD/sod/testme.json | 33 + test/HD/sod/testme.py | 2 +- test/HD/thermalDiffusion/testme.json | 15 + test/HD/thermalDiffusion/testme.py | 2 +- test/IO/dump/CMakeLists.txt | 1 + test/IO/dump/definitions.hpp | 5 + .../dump/idefix.ini} | 0 test/IO/dump/setup.cpp | 277 +++++ test/IO/dump/testme.json | 28 + test/IO/dump/testme.py | 59 ++ test/IO/pydefix/README.md | 7 + test/IO/pydefix/idefix.ini | 3 +- test/IO/pydefix/python_requirements.txt | 3 +- test/IO/pydefix/testme.json | 17 + test/IO/pydefix/testme.py | 41 + .../SedovBlastWave => IO/xdmf}/CMakeLists.txt | 1 + test/IO/xdmf/definitions.hpp | 5 + test/IO/xdmf/idefix.ini | 26 + test/IO/xdmf/setup.cpp | 75 ++ test/IO/xdmf/testme.json | 17 + test/IO/xdmf/testme.py | 45 + test/MHD/AmbipolarCshock/testme.json | 14 + test/MHD/AmbipolarCshock/testme.py | 2 +- test/MHD/AmbipolarCshock3D/idefix-rkl.ini | 4 +- test/MHD/AmbipolarCshock3D/idefix.ini | 4 +- test/MHD/AmbipolarCshock3D/setup.cpp | 47 +- test/MHD/AmbipolarCshock3D/testme.json | 35 + test/MHD/AmbipolarCshock3D/testme.py | 4 +- test/MHD/AxisFluxTube/testme.json | 16 + test/MHD/AxisFluxTube/testme.py | 2 +- test/MHD/Coarsening/testme.json | 12 + test/MHD/Coarsening/testme.py | 2 +- test/MHD/FargoMHDSpherical/testme.json | 17 + test/MHD/FargoMHDSpherical/testme.py | 2 +- test/MHD/HallWhistler/testme.json | 15 + test/MHD/HallWhistler/testme.py | 2 +- test/MHD/LinearWaveTest/idefix-entropy.ini | 2 +- test/MHD/LinearWaveTest/testme.json | 24 + test/MHD/LinearWaveTest/testme.py | 4 +- test/MHD/MTI/testme.json | 15 + test/MHD/MTI/testme.py | 2 +- test/MHD/OrszagTang/testme.json | 74 ++ test/MHD/OrszagTang/testme.py | 2 +- test/MHD/OrszagTang3D/setup.cpp | 188 ---- test/MHD/OrszagTang3D/testme.json | 34 + test/MHD/OrszagTang3D/testme.py | 8 +- test/MHD/ResistiveAlfvenWave/testme.json | 22 + test/MHD/ResistiveAlfvenWave/testme.py | 2 +- test/MHD/ShearingBox/testme.json | 15 + test/MHD/ShearingBox/testme.py | 5 +- test/MHD/clessTDiffusion/testme.json | 15 + test/MHD/clessTDiffusion/testme.py | 2 +- test/MHD/sod-iso/testme.json | 43 + test/MHD/sod-iso/testme.py | 2 +- test/MHD/sod/testme.json | 33 + test/MHD/sod/testme.py | 2 +- test/MHD/sphBragTDiffusion/testme.json | 15 + test/MHD/sphBragTDiffusion/testme.py | 2 +- test/MHD/sphBragViscosity/testme.json | 16 + test/MHD/sphBragViscosity/testme.py | 2 +- test/Planet/Planet3Body/testme.json | 15 + test/Planet/Planet3Body/testme.py | 2 +- test/Planet/PlanetMigration2D/testme.json | 16 + test/Planet/PlanetMigration2D/testme.py | 2 +- test/Planet/PlanetPlanetRK42D/testme.json | 25 + test/Planet/PlanetPlanetRK42D/testme.py | 2 +- test/Planet/PlanetSpiral2D/testme.json | 16 + test/Planet/PlanetSpiral2D/testme.py | 2 +- test/Planet/PlanetTorque3D/testme.json | 16 + test/Planet/PlanetTorque3D/testme.py | 2 +- test/Planet/PlanetsIsActiveRK52D/testme.json | 17 + test/Planet/PlanetsIsActiveRK52D/testme.py | 2 +- test/SelfGravity/DustyCollapse/testme.json | 16 + test/SelfGravity/DustyCollapse/testme.py | 2 +- test/SelfGravity/JeansInstability/testme.json | 17 + test/SelfGravity/JeansInstability/testme.py | 2 +- test/SelfGravity/RandomSphere/testme.json | 14 + test/SelfGravity/RandomSphere/testme.py | 2 +- .../RandomSphereCartesian/testme.json | 12 + .../RandomSphereCartesian/testme.py | 2 +- test/SelfGravity/UniformCollapse/testme.json | 16 + test/SelfGravity/UniformCollapse/testme.py | 2 +- test/python_requirements.txt | 4 + test/utils/columnDensity/testme.json | 13 + test/utils/columnDensity/testme.py | 2 +- test/utils/dumpImage/testme.json | 13 + test/utils/dumpImage/testme.py | 2 +- test/utils/lookupTable/testme.json | 12 + test/utils/lookupTable/testme.py | 2 +- 175 files changed, 4441 insertions(+), 1906 deletions(-) create mode 100644 doc/source/testing/testLauncher.rst create mode 100644 pytest.ini create mode 100644 pytools/idfx_test_gen.py create mode 100644 pytools/idfx_test_run.py create mode 100644 pytools/pytest.ini create mode 100644 pytools/tests/__init__.py create mode 100644 pytools/tests/test/pb1/testme.json create mode 100644 pytools/tests/test/pb2/testme.json create mode 100644 pytools/tests/test_idfx_test_gen.py create mode 100644 pytools/tests/test_idfx_test_run.py create mode 100644 src/fluid/constrainedTransport/enforceVectorPotentialBoundary.hpp delete mode 100644 src/mpi.cpp delete mode 100644 src/mpi.hpp create mode 100644 src/mpi/CMakeLists.txt create mode 100644 src/mpi/buffer.hpp create mode 100644 src/mpi/exchanger.cpp create mode 100644 src/mpi/exchanger.hpp create mode 100644 src/mpi/mpi.cpp create mode 100644 src/mpi/mpi.hpp create mode 100755 test.py create mode 100644 test/Dust/DustEnergy/testme.json create mode 100644 test/Dust/DustyShock/testme.json create mode 100644 test/Dust/DustyWave/testme.json create mode 100644 test/HD/FargoPlanet/testme.json create mode 100644 test/HD/MachReflection/testme.json create mode 100644 test/HD/SedovBlastWave/testme.json create mode 100644 test/HD/ShearingBox/testme.json create mode 100644 test/HD/ViscousDisk/testme.json create mode 100644 test/HD/ViscousFlowPastCylinder/testme.json create mode 100644 test/HD/sod-iso/testme.json create mode 100644 test/HD/sod/testme.json create mode 100644 test/HD/thermalDiffusion/testme.json create mode 100644 test/IO/dump/CMakeLists.txt create mode 100644 test/IO/dump/definitions.hpp rename test/{MHD/OrszagTang3D/idefix-checkrestart.ini => IO/dump/idefix.ini} (100%) create mode 100644 test/IO/dump/setup.cpp create mode 100644 test/IO/dump/testme.json create mode 100755 test/IO/dump/testme.py create mode 100644 test/IO/pydefix/testme.json create mode 100755 test/IO/pydefix/testme.py rename test/{HD/SedovBlastWave => IO/xdmf}/CMakeLists.txt (50%) create mode 100644 test/IO/xdmf/definitions.hpp create mode 100644 test/IO/xdmf/idefix.ini create mode 100644 test/IO/xdmf/setup.cpp create mode 100644 test/IO/xdmf/testme.json create mode 100755 test/IO/xdmf/testme.py create mode 100644 test/MHD/AmbipolarCshock/testme.json create mode 100644 test/MHD/AmbipolarCshock3D/testme.json create mode 100644 test/MHD/AxisFluxTube/testme.json create mode 100644 test/MHD/Coarsening/testme.json create mode 100644 test/MHD/FargoMHDSpherical/testme.json create mode 100644 test/MHD/HallWhistler/testme.json create mode 100644 test/MHD/LinearWaveTest/testme.json create mode 100644 test/MHD/MTI/testme.json create mode 100644 test/MHD/OrszagTang/testme.json create mode 100644 test/MHD/OrszagTang3D/testme.json create mode 100644 test/MHD/ResistiveAlfvenWave/testme.json create mode 100644 test/MHD/ShearingBox/testme.json create mode 100644 test/MHD/clessTDiffusion/testme.json create mode 100644 test/MHD/sod-iso/testme.json create mode 100644 test/MHD/sod/testme.json create mode 100644 test/MHD/sphBragTDiffusion/testme.json create mode 100644 test/MHD/sphBragViscosity/testme.json create mode 100644 test/Planet/Planet3Body/testme.json create mode 100644 test/Planet/PlanetMigration2D/testme.json create mode 100644 test/Planet/PlanetPlanetRK42D/testme.json create mode 100644 test/Planet/PlanetSpiral2D/testme.json create mode 100644 test/Planet/PlanetTorque3D/testme.json create mode 100644 test/Planet/PlanetsIsActiveRK52D/testme.json create mode 100644 test/SelfGravity/DustyCollapse/testme.json create mode 100644 test/SelfGravity/JeansInstability/testme.json create mode 100644 test/SelfGravity/RandomSphere/testme.json create mode 100644 test/SelfGravity/RandomSphereCartesian/testme.json create mode 100644 test/SelfGravity/UniformCollapse/testme.json create mode 100644 test/utils/columnDensity/testme.json create mode 100644 test/utils/dumpImage/testme.json create mode 100644 test/utils/lookupTable/testme.json diff --git a/.github/workflows/idefix-ci-jobs.yml b/.github/workflows/idefix-ci-jobs.yml index 32eef5911..847ba3cbe 100644 --- a/.github/workflows/idefix-ci-jobs.yml +++ b/.github/workflows/idefix-ci-jobs.yml @@ -38,6 +38,8 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/sod-iso -all $TESTME_OPTIONS - name: Mach reflection test run: scripts/ci/run-tests $IDEFIX_DIR/test/HD//MachReflection -all $TESTME_OPTIONS + - name: Sedov blast wave + run: scripts/ci/run-tests $IDEFIX_DIR/test/HD/SedovBlastWave -all $TESTME_OPTIONS ParabolicHydro: runs-on: self-hosted @@ -68,8 +70,10 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/sod-iso -all $TESTME_OPTIONS - name: Orszag Tang run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang -all $TESTME_OPTIONS - - name: Orszag Tang 3D+restart tests + - name: Orszag Tang 3D run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/OrszagTang3D -all $TESTME_OPTIONS + - name: Linear wave test + run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/LinearWaveTest -all $TESTME_OPTIONS - name: Axis Flux tube run: scripts/ci/run-tests $IDEFIX_DIR/test/MHD/AxisFluxTube -all $TESTME_OPTIONS @@ -221,3 +225,24 @@ jobs: run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/dumpImage -all $TESTME_OPTIONS - name: Column density run: scripts/ci/run-tests $IDEFIX_DIR/test/utils/columnDensity -all $TESTME_OPTIONS + + IOs: + needs: [Fargo, Dust, Planet, ShearingBox, SelfGravity] + runs-on: self-hosted + steps: + - name: Check out repo + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + submodules: recursive + persist-credentials: false + - name: Restart dumps + run: scripts/ci/run-tests $IDEFIX_DIR/test/IO/dump -all $TESTME_OPTIONS + - name: Pydefix + run: | + python3 -m venv $IDEFIX_DIR/test/IO/pydefix/env + source $IDEFIX_DIR/test/IO/pydefix/env/bin/activate + python3 -m pip install -r $IDEFIX_DIR/test/IO/pydefix/python_requirements.txt + scripts/ci/run-tests $IDEFIX_DIR/test/IO/pydefix -all $TESTME_OPTIONS + + - name: xdmf + run: scripts/ci/run-tests $IDEFIX_DIR/test/IO/xdmf -all $TESTME_OPTIONS diff --git a/.github/workflows/idefix-ci.yml b/.github/workflows/idefix-ci.yml index b6ed719e9..50fd26eee 100644 --- a/.github/workflows/idefix-ci.yml +++ b/.github/workflows/idefix-ci.yml @@ -32,7 +32,7 @@ jobs: name: CPU Jobs (intel OneApi) uses: ./.github/workflows/idefix-ci-jobs.yml with: - TESTME_OPTIONS: -intel -Werror + TESTME_OPTIONS: -intel -Werror -ccache IDEFIX_COMPILER: icc gcc-jobs: @@ -40,7 +40,7 @@ jobs: name: CPU Jobs (gcc) uses: ./.github/workflows/idefix-ci-jobs.yml with: - TESTME_OPTIONS: -Werror + TESTME_OPTIONS: -Werror -ccache IDEFIX_COMPILER: gcc cuda-jobs: diff --git a/.gitignore b/.gitignore index 4de7fb76e..e3bf696a2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ CMakeFiles CMakeCache.txt cmake_install.cmake Makefile.local +git-state.txt +Kokkos_Version_info.* + # test artifacts test/**/*.o @@ -36,6 +39,9 @@ test/**/KokkosCore* test/**/*.csv test/**/*.pyc test/**/*.dat +test/**/cmake_packages* +test/**/ +idefix-tests.junit.xml # machine specific cache and hidden files .* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f0b285315..8249b7b03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,9 +45,10 @@ repos: - F403 # ignore import * - repo: https://github.com/neutrinoceros/inifix - rev: v5.0.2 + rev: v6.1.2 hooks: - id: inifix-format + files: ^(test/).*\.(ini)$ # want to skip pytest.ini - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3778d514e..6bb7f979e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.3.0] 2026-04-21 +### Changed + +- fixed a bug that could lead to diverging results around the spherical axis in non-ideal MHD in 2.5D and 3D (#356) +- refactor of the MPI exchange routines and boundary routines to avoid buildup of roundoff errors at domain faces/edges that could lead to the sudden burst of div(B) or incoherences between MPI sub-domains (#357) +- fixed a bug that could lead to compilations error when targetting AMD APUs (#359) +- fixed a bug that led to the generation of incorrect subviews in 2.5D with vector_potential enabled (#362) +- fixed a bug that could lead to memory corruption when using the UCT_HLLD emf reconstruction scheme and DIMENSIONS < COMPONENTS (#363) +- reorganise the test to separate specific IO tests from physics tests (#367) + +### Added +- magnetic vector potential is now accessible from pydefix when enabled (#361) +- use ccache in the test suite to reduce the runtime of continuous integration (#364) +- automatically detects pybind11 path with cmake when using pydefix (#367) +- skeleton to run the test suite with pytest (#366) + + ## [2.2.02] 2025-10-18 ### Changed @@ -13,6 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - fixed a bug that led to incorrect BX2 reconstruction when axis is not used on both sides of the domain (#345) - fixed a bug that led to incorrect reflective boundary conditions on B when DIMENSIONS < 3 (#345) - fixed a bug that led to incorrect dust stopping time when the adiabatic equation of state is used with "size" drag law (#353) +- fixed div(B) normalisation to avoid "too large div(B)" errors when this is actually due to nulls in |B| +- fixed insecure github actions settings (#373) ### Added diff --git a/CMakeLists.txt b/CMakeLists.txt index 364ee4ef3..6698aaec6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,10 @@ endif() set (CMAKE_CXX_STANDARD 17) set(Idefix_VERSION_MAJOR 2) -set(Idefix_VERSION_MINOR 2) -set(Idefix_VERSION_PATCH 02) +set(Idefix_VERSION_MINOR 3) +set(Idefix_VERSION_PATCH 0) -project (idefix VERSION 2.2.02) +project (idefix VERSION 2.3.0) option(Idefix_MHD "enable MHD" OFF) option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF) option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF) @@ -16,6 +16,7 @@ option(Idefix_DEBUG "Enable Idefix debug features (makes the code very slow)" OF option(Idefix_RUNTIME_CHECKS "Enable runtime sanity checks" OFF) option(Idefix_WERROR "Treat compiler warnings as errors" OFF) option(Idefix_PYTHON "Enable python bindings (requires pybind11)" OFF) +set(Idefix_PROBLEM_DIR "${CMAKE_BINARY_DIR}" CACHE STRING "Problem directory to build for.") set(Idefix_CXX_FLAGS "" CACHE STRING "Additional compiler/linker flag") set(Idefix_DEFS "definitions.hpp" CACHE FILEPATH "Problem definition header file") option(Idefix_CUSTOM_EOS "Use custom equation of state" OFF) @@ -34,7 +35,6 @@ set_property(CACHE Idefix_PRECISION PROPERTY STRINGS Double Single) set(Idefix_LOOP_PATTERN "Default" CACHE STRING "Loop pattern for idefix_for") set_property(CACHE Idefix_LOOP_PATTERN PROPERTY STRINGS Default SIMD Range MDRange TeamPolicy TeamPolicyInnerVector) - # load git revision tools list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") include(GetGitRevisionDescription) @@ -51,7 +51,7 @@ if(Kokkos_ENABLE_CUDA) endif() # Add kokkos CMAKE files (required early since these set compiler options) -add_subdirectory(src/kokkos build/kokkos) +add_subdirectory(src/kokkos ${CMAKE_BINARY_DIR}/build/kokkos) include_directories(${Kokkos_INCLUDE_DIRS_RET}) # Add Idefix CXX Flags @@ -75,20 +75,22 @@ endif() add_executable(idefix) add_subdirectory(src build) -if(EXISTS ${PROJECT_BINARY_DIR}/setup.cpp) - target_sources(idefix PUBLIC ${PROJECT_BINARY_DIR}/setup.cpp) +# make absolute +get_filename_component(Idefix_PROBLEM_DIR_ABS ${Idefix_PROBLEM_DIR} ABSOLUTE BASE_DIR ${PROJECT_BINARY_DIR}) + +if(EXISTS ${Idefix_PROBLEM_DIR_ABS}/setup.cpp) + target_sources(idefix PUBLIC ${Idefix_PROBLEM_DIR_ABS}/setup.cpp) else() message(WARNING "No specific setup.cpp found in the problem directory (this message can be ignored if using python to define your problem)") endif() # If a CMakeLists.txt is in the problem dir (for problem-specific source files) # then read it -if(EXISTS ${PROJECT_BINARY_DIR}/CMakeLists.txt) - message(STATUS "Including problem-specific CMakeLists in '${PROJECT_BINARY_DIR}'") - add_subdirectory(${PROJECT_BINARY_DIR} build/setup) +if(EXISTS ${Idefix_PROBLEM_DIR_ABS}/CMakeLists.txt) + message(STATUS "Including problem-specific CMakeLists in '${Idefix_PROBLEM_DIR_ABS}'") + add_subdirectory(${Idefix_PROBLEM_DIR_ABS} build/setup) endif() - if(Idefix_MHD) add_compile_definitions("MHD=YES") else() @@ -99,10 +101,7 @@ if(Idefix_MPI) add_compile_definitions("WITH_MPI") find_package(MPI REQUIRED) target_link_libraries(idefix MPI::MPI_CXX) - target_sources(idefix - PUBLIC src/mpi.cpp - PUBLIC src/mpi.hpp - ) + add_subdirectory(src/mpi) endif() if(Idefix_HDF5) @@ -116,7 +115,13 @@ if(Idefix_HDF5) ) find_package(HDF5 REQUIRED) target_link_libraries(idefix "${HDF5_LIBRARIES}") - target_include_directories(idefix "${HDF5_INCLUDE_DIRS}") + message(STATUS "Found HDF5 include directories: ${HDF5_INCLUDE_DIRS}") + target_include_directories(idefix PUBLIC "${HDF5_INCLUDE_DIRS}") + if(Idefix_MPI) + if(NOT HDF5_IS_PARALLEL) + message(FATAL_ERROR "Parallel HDF5 required for Idefix_MPI but the found HDF5 library does not support it") + endif() + endif() message(STATUS "XDMF (hdf5+xmf) dumps enabled") else() set(Idefix_HDF5 OFF) @@ -124,6 +129,16 @@ endif() if(Idefix_PYTHON) add_compile_definitions("WITH_PYTHON") + find_package(Python3 REQUIRED COMPONENTS Interpreter Development) + + execute_process( + COMMAND "${Python3_EXECUTABLE}" -m pybind11 --cmakedir + OUTPUT_VARIABLE PYBIND11_CMAKE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY + ) + + list(APPEND CMAKE_PREFIX_PATH "${PYBIND11_CMAKE_DIR}") if (NOT DEFINED Python_FIND_FRAMEWORK) set(Python_FIND_FRAMEWORK "LAST") # Use Apple's python only at last resort on Macos endif () @@ -213,7 +228,7 @@ if(${Idefix_PRECISION} STREQUAL "Single") endif() target_include_directories(idefix PUBLIC - "${PROJECT_BINARY_DIR}" + "${Idefix_PROBLEM_DIR_ABS}" ) target_include_directories(idefix PUBLIC src/kokkos/core/src @@ -234,6 +249,7 @@ target_include_directories(idefix PUBLIC src/gravity src/utils src/utils/iterativesolver + src/mpi src ) @@ -251,6 +267,7 @@ message(STATUS " Python: ${Idefix_PYTHON}") message(STATUS " Reconstruction: ${Idefix_RECONSTRUCTION}") message(STATUS " Precision: ${Idefix_PRECISION}") message(STATUS " Version: ${Idefix_VERSION}") +message(STATUS " Problem directory: '${Idefix_PROBLEM_DIR}'") message(STATUS " Problem definitions: '${Idefix_DEFS}'") if(Idefix_CUSTOM_EOS) message(STATUS " EOS: Custom file '${Idefix_CUSTOM_EOS_FILE}'") diff --git a/doc/python_requirements.txt b/doc/python_requirements.txt index c75fa02b7..28715fb71 100644 --- a/doc/python_requirements.txt +++ b/doc/python_requirements.txt @@ -6,12 +6,12 @@ # python -m pip install -r python_requirements.txt wheel>=0.38.4 # help forward compatibility for pip with old sphinx plugins -sphinx==5.3.0 -sphinx_rtd_theme==1.3.0 +sphinx==9.1.0 +sphinx_rtd_theme==3.1.0 sphinx_git==11.0.0 -breathe==4.34.0 +breathe==4.36.0 exhale==0.3.7 -m2r2==0.3.2 +m2r2==0.3.4 sphinx-copybutton==0.5.2 #sphinxcontrib-applehelp==1.0.7 matplotlib==3.10.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 595672c25..f47f6659c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,7 @@ author = 'Geoffroy Lesur' # The full version, including alpha/beta/rc tags -release = '2.2.02' +release = '2.3.0' diff --git a/doc/source/modules/pydefix.rst b/doc/source/modules/pydefix.rst index f68284203..0a8daf753 100644 --- a/doc/source/modules/pydefix.rst +++ b/doc/source/modules/pydefix.rst @@ -23,7 +23,8 @@ Pydefix usage Idefix Configuration ++++++++++++++++++++ -In order to use Pydefix, you need to switch on ``Idefix_PYTHON`` in cmake. This will auto-detect Python and check that pybind11 can be used effectively. +In order to use Pydefix, you need to switch on ``Idefix_PYTHON`` in cmake. This will auto-detect Python and check that pybind11 can be used effectively. If you are using a python environement manager such as venv or conda, make sure to activate the right environement *before* running cmake, as idefix +now includes the full path to python and pybind11 in its executable file. Run Idefix with Pydefix diff --git a/doc/source/reference/makefile.rst b/doc/source/reference/makefile.rst index 23052d92c..8f58ceedb 100644 --- a/doc/source/reference/makefile.rst +++ b/doc/source/reference/makefile.rst @@ -59,6 +59,12 @@ Several options can be enabled from the command line (or are accessible with ``c The number of ghost cells is automatically adjusted as a function of the order of the reconstruction scheme. *Idefix* uses 2 ghost cells when ``ORDER < 4`` and 3 ghost cells when ``ORDER = 4`` +``-D Idefix_PROBLEM_DIR=.`` + Specify where to find the problem directory to build *Idefix* out of source. + Place yourself in the ``build`` directory you want to build in and call the ``cmake`` by : + + pointing the *Idefix* source directory via ``IDEFIX_DIR`` as usual. + + using ``-D Idefix_PROBLEM_DIR`` to point the problem to build. + ``-D Kokkos_ENABLE_OPENMP=ON`` Enable OpenMP parallelisation on supported compilers. Note that this can be enabled simultaneously with MPI, resulting in a hybrid MPI+OpenMP compilation. diff --git a/doc/source/testing.rst b/doc/source/testing.rst index 888ac2057..59bd2177a 100644 --- a/doc/source/testing.rst +++ b/doc/source/testing.rst @@ -71,9 +71,17 @@ How tests are driven (testme scripts) Each test directory contains a small Python "testMe" driver that uses the helper Python class documented in the repository: +- See the test launcher documentation: :doc:`test.py ` - See the test helper documentation: :doc:`idfxTest ` -That helper (idfxTest) is responsible for: +The test launcher (test.py) is responsible for: + +- Loading all the test definitions by search all the ``testme.json`` files in the ``test`` + directory. +- Calling the :doc:`idfxTest ` helper to run the particular test. +- Generate reports about success and failures. + +The helper (idfxTest) is responsible for: - parsing TESTME_OPTIONS-like flags (precision, MPI, CUDA, reconstruction, vector potential, etc.), - calling configure / compile / run, @@ -108,10 +116,12 @@ Relevant files - Workflow entry point: .github/workflows/idefix-ci.yml - Reusable jobs: .github/workflows/idefix-ci-jobs.yml +- Test launcher documentation: :doc:`test launcher ` - Test helper documentation: :doc:`idfxTest ` .. toctree:: :maxdepth: 2 :caption: Contents: + testing/testLauncher.rst testing/idfxTest.rst diff --git a/doc/source/testing/idfxTest.rst b/doc/source/testing/idfxTest.rst index b425d3fe2..96e364465 100644 --- a/doc/source/testing/idfxTest.rst +++ b/doc/source/testing/idfxTest.rst @@ -74,6 +74,9 @@ The constructor parses command-line arguments using ``argparse``. These options * - ``-Werror`` - ``Werror`` - Treat compiler warnings as errors. + * - ``-ccache`` + - ``ccache`` + - Enable usage of ccache to build the tests and reduce the build time. Main Methods ------------ diff --git a/doc/source/testing/testLauncher.rst b/doc/source/testing/testLauncher.rst new file mode 100644 index 000000000..aea7f5c3d --- /dev/null +++ b/doc/source/testing/testLauncher.rst @@ -0,0 +1,383 @@ +========================== +Test launcher and reporter +========================== + +Overview +-------- + +The class :doc:`idfxTest ` provides the toolbox to implement an *Idefix* integration test for validation. +In order to ease launching all the tests, the user might prefer to use directly the ``./test.py`` command at the +root of the *Idefix* sources. + +This script will run all the listed variants of *Idefix* and build a report in the terminal. At the +end of the run a standard ``junit.xml`` file is produced. This one can be translated into a browsable +HTML file. + +Depencencies +------------ + +Before using :doc:`idfxTest ` you need to install some Python depencencies (possibly in a ``virtual env``): + +.. code-block:: shell + + pip install -r test/python_requirements.txt + +Running +------- + +To run the test you can basically : + +.. code-block:: shell + + # Run all tests + ./test.py + + # Run all tests in ./tests/HD + ./test.py -subdir=./tests/HD + + # Select in more details the tests containing the "single" keyword + # See pytest documentation for the exact advanced semantic + ./test.py -subdir=./tests/HD -k single + + # Run in verbose + ./test.py -v + +The result of the execution will be an output like : + +.. code-block:: text + + ============================================ test session starts ============================================= + collected 52 items / 44 deselected / 8 selected + + test.py::test_idefix_build_run_check[HD/sod-iso-idefix.ini-single-reconstruction-2] PASSED [ 12%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-hll.ini-single-reconstruction-2] PASSED [ 25%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-hllc.ini-single-reconstruction-2] PASSED [ 37%] + test.py::test_idefix_build_run_check[HD/sod-iso-idefix-tvdlf.ini-single-reconstruction-2] PASSED [ 50%] + test.py::test_idefix_build_run_check[HD/sod-idefix.ini-single-reconstruction-2] PASSED [ 62%] + test.py::test_idefix_build_run_check[HD/sod-idefix-hll.ini-single-reconstruction-2] PASSED [ 75%] + test.py::test_idefix_build_run_check[HD/sod-idefix-hllc.ini-single-reconstruction-2] PASSED [ 87%] + test.py::test_idefix_build_run_check[HD/sod-idefix-tvdlf.ini-single-reconstruction-2] PASSED [100%] + + ---------- generated xml file: idefix-tests.junit.xml --------------------------------------------------------- + =============================== 8 passed, 44 deselected in 73.03s (0:01:13) =================================== + +When an error is detected, the output of the command will be printed at the end. + +HTML report +----------- + +If you want to to generate an HTML page from the report you can proceed by using the Python package ``junit2html`` : + +.. code-block:: shell + + # install junit2html + pip install junit2html + + # convert the report + junit2html ./idefix-tests.junit.xml ./idefix-tests.junit.html + +Advanced usage of the command +----------------------------- + +Here the options supported by the test script : + +.. code-block:: text + + usage: test.py [-h] [-noplot] [-ploterr] [-cmake CMAKE [CMAKE ...]] [-definitions DEFINITIONS] + [-dec DEC [DEC ...]] [-check] [-cuda] [-intel] [-hip] [-single] [-vectPot] + [-reconstruction RECONSTRUCTION] [-idefixDir IDEFIXDIR] [-mpi] [-all] [-init] + [-Werror] [-ccache] [-restart] [-v] [--help-pytest] [-fake] [-subdir SUBDIR] + + options: + -h, --help show this help message and exit + -noplot disable plotting in standard tests + -ploterr Enable plotting on error in regression tests + -cmake CMAKE [CMAKE ...] + CMake options + -definitions DEFINITIONS + definitions.hpp file + -dec DEC [DEC ...] MPI domain decomposition + -check Only perform regression tests without compilation + -cuda Test on Nvidia GPU using CUDA + -intel Test compiling with Intel OneAPI + -hip Test on AMD GPU using HIP + -single Enable single precision + -vectPot Enable vector potential formulation + -reconstruction RECONSTRUCTION + set reconstruction scheme (2=PLM, 3=LimO3, 4=PPM) + -idefixDir IDEFIXDIR Set directory for idefix source files (default $IDEFIX_DIR) + -mpi Enable MPI + -all Do all test suite (otherwise, just do the test with the current configuration) + -init Reinit reference files for non-regression tests (dangerous!) + -Werror Consider warnings as errors + -ccache Use ccache to reduce the build time over multiple run of the test suite. + -restart Enable creating a restart from a checkpoint. + -v, --verbose Enable verbose mode, by not capturing the output. + --help-pytest Display the options you can transmit directly to pytest in addition to the specific to idefix tests. + -fake Make a fake run by just logging the actions to validate that we generate same command over refactoring. + -subdir SUBDIR Select the test in the given subdir not to run all. + +The script is built on top of the ``pytest`` command so you automatically get access +to all the advanced option this command provide. Here a few examples : + +.. code-block:: shell + + # get the pytest help + ./test.py --help-pytest + + # let pytest filtering the tests + ./test.py -k "single and mpi" + + # stop on first failure + ./test.py -x + + # re-run only the last failed tests + ./test.py --lf + +Definition of the tests +----------------------- + +The script looks for all the files named ``testme.json`` in the ``test`` directory and all its sub-directories. +This file describes the combination of parameters used to produce the list of *Idefix* runs to check. +For a single basic configuration one can use : + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": "idefix.ini", + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0 + } + } + +Available parameters +-------------------- + +The parameters in the ``variants`` dictionnary correspond to the options supported by the +:doc:`idfxTest ` script to configure the build and run of *Idefix*. + +In addition there is some extra keys which are dedicated to the json interpretation layer : + +.. list-table:: + :header-rows: 1 + + * - Option + - Default + - Description + * - ``dumname`` + - ``dump.0001.dmp`` + - Dump file to use to check the results after the run. + * - ``ini`` + - ``idefix.ini`` + - The configuration file to use. + * - ``tolerance`` + - ``0`` + - The margins to allow when checking the results. + * - ``standardTest`` + - ``true`` + - Runs any Python-based standard tests (e.g., ``testidefix.py``) present in the test directory for additional validation. + * - ``nonRegressionTest`` + - ``true`` + - Compares the output dump file to a reference file using RMSE; fails if the error exceeds the tolerance. + * - ``nonRegressionTestIni`` + - Same than ``ini`` + - When making restart you might want to make the check using the inirial configuration file. + * - ``multirun`` + - ``{}`` + - See the multi-run section below. + +Looping over parameters +----------------------- + +You might want to explore running Idefix within parameter ranges (configuration files, modes). +For this simply list the values you want as a list. The test script will automatically +generate all combinations. + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": [false, true], + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + } + +It will automatically produce the tests : + +* HD/sod-iso-idefix.ini +* HD/sod-iso-idefix.ini-vectPot +* HD/sod-iso-idefix.ini-mpi +* HD/sod-iso-idefix.ini-vectPot-mpi +* HD/sod-iso-idefix-hll.ini +* HD/sod-iso-idefix-hll.ini-vectPot +* HD/sod-iso-idefix-hll.ini-mpi +* HD/sod-iso-idefix-hll.ini-vectPot-mpi + +Specific keys +------------- + +There is some keys which are by default some arrays, they will not be considered +as combination rules : + +* ``dec`` +* ``multirun`` +* ``restart_no_overwrite`` +* ``tolerance`` + +Reduce the combinations +----------------------- + +You might not want to see all the combinations but just a few, for this, you +can list several sets as a list. Here using single only on half of the modes. + +.. code-block:: json + + { + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-hll.ini"], + "vectPot": true, + "single": true, + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + ] + } + +* HD/sod-iso-idefix.ini +* HD/sod-iso-idefix.ini-mpi +* HD/sod-iso-idefix-hll.ini-single-vectPot +* HD/sod-iso-idefix-hll.ini-single-vectPot-mpi + +Naming the test +--------------- + +If you prefer to see the options appearing in a specific order in the generated test name, +you can provide the key ``namings`` listing as a comma separated list the variables in +the order you want to see them composing the test name. + +.. code-block:: json + + { + "namings": "ini,single,mpi", + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": false, + "single": [false, true], + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + } + } + + +By default, the alphabetical order is used. + +When clauses +------------ + +You can also dynamically override a specific value when a parameter value is selected. +It is just like if you used and IF statement. + +.. code-block:: json + + { + "namings": "ini,single,mpi", + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini"], + "vectPot": false, + "single": [false, true], + "reconstruction": 2, + "mpi": [false, true], + "standardTest": false, + "tolerance": 0 + }, + "when": { + "conditions": { + "single": true + }, + "apply": { + "reconstruction": 1 + } + } + } + +In this case, ``reconstruction`` will be set to 1 when ``single`` is equal to ``true``. + +Note that the ``conditions`` field can contains several values which will be treaded +as and AND logical operator. + +You can provide several ``when`` clauses by using a list of them instead of directly +the dictionnary. + +Making multi-run steps +---------------------- + +In order to validate checkpoint restart, or continuing a simulation with dirfferent +tunnings the script supports decribing multi-run configurations. + +They are described like : + +.. code-block:: json + + { + "variants": { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0, + "dec": [2,2,2], + "multirun": [ + { + },{ + "mpi": true, + "restart": true, + "restart_no_overwrite": ["dump.0001.dmp", "data.0005.vtk"] + } + ] + }, + } + +Using the idfxTest options +-------------------------- + +As the ``test.py`` uses the class described in :doc:`idfxTest ` it also supports all the +command line options it offers. + +Most usefull might be the enabling of ``ccache`` to reduce the compilation time from one +run to another. + +.. code-block:: shell + + ./test.py -ccache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..b3482664f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +markers= + default: Test to run by default. +python_files="test_*.py" +junit_logging="system-out" +junit_log_passing_tests=false diff --git a/pytools/idfx_test.py b/pytools/idfx_test.py index 25bac76b4..fd28ccb61 100644 --- a/pytools/idfx_test.py +++ b/pytools/idfx_test.py @@ -4,6 +4,7 @@ import subprocess import sys import re +import json import numpy as np import matplotlib.pyplot as plt @@ -22,7 +23,7 @@ class bcolors: UNDERLINE = '\033[4m' class idfxTest: - def __init__ (self): + def __init__ (self, current_test_file, name=""): parser = argparse.ArgumentParser() idefix_dir_env = os.getenv("IDEFIX_DIR") @@ -99,19 +100,94 @@ def __init__ (self): help="Consider warnings as errors", action="store_true") + parser.add_argument("-ccache", + help="Use ccache to reduce the build time over multiple run of the test suite.", + action="store_true") + + parser.add_argument("-restart", + help="Enable creating a restart from a checkpoint.", + action='store_true') + + parser.add_argument("-v", "--verbose", + help="Enable verbose mode, by not capturing the output.", + action="store_true") + + parser.add_argument("--help-pytest", + help="Display the options you can transmit directly to pytest in addition to the specific to idefix tests.", + action="store_true") + + parser.add_argument("-fake", + help="Make a fake run by just logging the actions to validate that we generate same command over refactoring.", + action="store_true") + + # this option is not used directly by direct users of idfxTest but by idx_test_gen + parser.add_argument("-subdir", + default="./test", + help="Select the test in the given subdir not to run all.", + type=str) args, unknown=parser.parse_known_args() # transform all arguments from args into attributes of this instance self.__dict__.update(vars(args)) + # store the full path of problem directory + self.currentTestFile = current_test_file + self.currentTestName = name + self.problemDir=os.path.dirname(current_test_file) self.referenceDirectory = os.path.join(idefix_dir_env,"reference") # current directory relative to $IDEFIX_DIR/test (used to retrieve the path ot reference files) - self.testDir=os.path.relpath(os.curdir,os.path.join(idefix_dir_env,"test")) - - def configure(self,definitionFile=""): + self.testDir=os.path.relpath(self.problemDir,os.path.join(idefix_dir_env,"test")) + # build directory, currently inside the test named build-test + self.buildDir=os.path.join(self.problemDir,"build-test") + # remind what build we dit last + self.lastCmakeCmd="" + # subdir + self.filterSubdir=args.subdir + # save + self.cmdArgs = vars(args) + self.cmdArgs.update({ + "restart_no_overwrite": [], + }) + self.log=[] + # when making a restart we should not overrite those files (will be checked) + self.restart_no_overwrite=[] + + # forward args for pytest + if args.verbose: + unknown.append("--capture=no") + if args.help_pytest: + unknown.append("--help") + # remaining args + self.remainingArgs=unknown + + def addLog(self, entry): + if self.fake: + self.log.append(entry) + with open(os.path.join(self.problemDir,"testsuite.log.json"), "w+") as fp: + json.dump(self.log, fp, indent='\t') + + def applyConfig(self, config: dict={}): + # check args + for key, value in config.items(): + if key not in ['ini', 'testfile', 'testname', 'dumpname']: + assert key in self.cmdArgs, f"The given configuration overriding try to set an invalid paramater : {key}={value}" + + # override options + newArgs = {} + newArgs.update(self.cmdArgs) + newArgs.update(config) + + # replace in dict + self.__dict__.update(newArgs) + + def _genCmakeCommand(self,definitionFile=""): comm=["cmake"] # add source directory comm.append(self.idefixDir) + + # we will build in ./build-test so problem dir is parent + comm.append("-DIdefix_PROBLEM_DIR="+self.problemDir) + # add specific options for opt in self.cmake: comm.append("-D"+opt) @@ -155,7 +231,7 @@ def configure(self,definitionFile=""): else: self.definitions="definitions.hpp" - comm.append("-DIdefix_DEFS="+self.definitions) + comm.append("-DIdefix_DEFS="+os.path.join(self.problemDir, self.definitions)) if(self.mpi): comm.append("-DIdefix_MPI=ON") @@ -169,20 +245,88 @@ def configure(self,definitionFile=""): elif(self.reconstruction==4): comm.append("-DIdefix_RECONSTRUCTION=Parabolic") + # export ccache env + if self.ccache: + comm.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache") + + # ok + return comm + + + def configure(self,definitionFile="", reuse_last_same_build=True, override: dict={}): + # log + self.addLog({"call": "configure", "args":{ + 'definitionFile': definitionFile, + 'reuse_last_same_build': reuse_last_same_build, + 'override': override + }}) + + # gen command + comm = self._genCmakeCommand(definitionFile) + # log + print("***************** CALLING CMAKE *******************") + print(f"mkdir -p {self.buildDir}") + print(f"cd {self.buildDir}") + print(' '.join(comm)) + print("***************************************************") + + # not action needed if same command or force no rebuild + if reuse_last_same_build == True and self.lastCmakeCmd == ' '.join(comm): + print("SKIP ALREADY DONE") + return + + # run try: - cmake=subprocess.run(comm) - cmake.check_returncode() + # do cleanup + self.clean() + + # log and in fake mode we do not execute + self.addLog({"command": comm}) + + # call cmake + if not self.fake: + cmake=subprocess.run(comm, cwd=os.path.abspath(self.buildDir)) + cmake.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Cmake failed") print("***************************************************"+bcolors.ENDC) raise e + finally: + # remind for next time + self.lastCmakeCmd = ' '.join(comm) + + def clean(self): + # log and in fake mode we do not execute + self.addLog({"call": "clean", "args":{}}) + if self.fake: + return + + # remove the build directory before re-creating it + if os.path.exists(self.buildDir): + shutil.rmtree(self.buildDir) + + # recreate + os.makedirs(self.buildDir, exist_ok=False) def compile(self,jobs=8): + self.addLog({"call": "compile", "args":{ + 'jobs': jobs, + }}) + try: - make=subprocess.run(["make","-j"+str(jobs)]) - make.check_returncode() + comm = ["make","-j"+str(jobs)] + self.addLog({"command": comm}) + + print("***************************************************") + print(f"cd {os.getcwd()}") + print(' '.join(comm)) + print("***************************************************") + + if not self.fake: + make=subprocess.run(comm, cwd=os.path.abspath(self.buildDir)) + make.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Compilation failed") @@ -190,7 +334,14 @@ def compile(self,jobs=8): raise e def run(self, inputFile="", np=2, nowrite=False, restart=-1): - comm=["./idefix"] + # log + self.addLog({"call": "run", "args":{ + 'np': np, + 'nowrite': nowrite, + 'restart': restart, + }}) + + comm=[os.path.join(self.buildDir,"idefix")] if inputFile: comm.append("-i") comm.append(inputFile) @@ -218,9 +369,16 @@ def run(self, inputFile="", np=2, nowrite=False, restart=-1): comm.append("-restart") comm.append(str(restart)) + print("***************************************************") + print(f"cd {os.getcwd()}") + print(' '.join(comm)) + print("***************************************************") + try: - make=subprocess.run(comm) - make.check_returncode() + self.addLog({"command": comm}) + if not self.fake: + make=subprocess.run(comm, cwd=self.problemDir) + make.check_returncode() except subprocess.CalledProcessError as e: print(bcolors.FAIL+"***************************************************") print("Execution failed") @@ -230,6 +388,9 @@ def run(self, inputFile="", np=2, nowrite=False, restart=-1): self._readLog() def _readLog(self): + if self.fake: + return + if not os.path.exists('./idefix.0.log'): # When no idefix file is produced, we leave return @@ -274,6 +435,12 @@ def _readLog(self): self.perf=float(line.group(1)) def checkOnly(self, filename, tolerance=0): + # log + self.addLog({"call": "checkOnly", "args":{ + 'filename': filename, + 'tolerance': tolerance, + }}) + # Assumes the code has been run manually using some configuration, so we simply # do the test suite witout configure/compile/run self._readLog() @@ -288,8 +455,14 @@ def checkOnly(self, filename, tolerance=0): self.nonRegressionTest(filename, tolerance) def standardTest(self): - if os.path.exists(os.path.join('python', 'testidefix.py')): - os.chdir("python") + # log and in fake mode do not execute. + self.addLog({"call": "standardTest", "args":{}}) + if self.fake: + return + + if os.path.exists(os.path.join(self.problemDir, 'python', 'testidefix.py')): + oldPwd = os.getcwd() + os.chdir(os.path.join(self.problemDir, "python")) comm = [sys.executable, "testidefix.py"] if self.noplot: comm.append("-noplot") @@ -304,12 +477,19 @@ def standardTest(self): print("***************************************************"+bcolors.ENDC) raise e print(bcolors.OKCYAN+"Standard test succeeded"+bcolors.ENDC) - os.chdir("..") + os.chdir(oldPwd) else: print(bcolors.WARNING+"No standard testidefix.py for this test"+bcolors.ENDC) sys.stdout.flush() def nonRegressionTest(self, filename,tolerance=0): + # log and in fake mode do not execute. + self.addLog({"call": "nonRegressionTest", "args":{ + "filename": filename, + "tolerance": tolerance + }}) + if self.fake: + return fileref=os.path.join(self.referenceDirectory, self.testDir, self._getReferenceFilename()) if not(os.path.exists(fileref)): @@ -333,6 +513,14 @@ def nonRegressionTest(self, filename,tolerance=0): sys.stdout.flush() def compareDump(self, file1, file2,tolerance=0): + self.addLog({"call": "compareDump", "args":{ + "file1": file1, + "file2": file2, + "tolerance": tolerance, + }}) + if self.fake: + return + Vref=readDump(file1) Vtest=readDump(file2) error=self._computeError(Vref,Vtest) @@ -347,6 +535,13 @@ def compareDump(self, file1, file2,tolerance=0): def makeReference(self,filename): + # log and in fake mode do not execute. + self.addLog({"call": "compareDump", "args":{ + "filename": filename, + }}) + if self.fake: + return + self._readLog() targetDir = os.path.join(self.referenceDirectory,self.testDir) if not os.path.exists(targetDir): diff --git a/pytools/idfx_test_gen.py b/pytools/idfx_test_gen.py new file mode 100644 index 000000000..e552c321d --- /dev/null +++ b/pytools/idfx_test_gen.py @@ -0,0 +1,249 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +import copy +import pytest + +DO_NOT_LOOP_ON = ['restart_no_overwrite', "dec", "multirun", "check_file_produced"] + +class IdefixDirTestGenerator: + ''' + Class used to generate the various configuration to run by parsing the + files `testme.py` found in the hierarchy of the /test directory of Idefix. + ''' + + def __init__(self, currentTestFile: str, name: str = ""): + ''' + Constructor or the class. + + Args: + currentTestFile (str): + Path the the current python file. Normally you + simply pass __file__ to this parameter. + name (str): + Define a name for the test, can be empty. + ''' + + self.currentTestFile = currentTestFile + self.currentTestName = name + + # generate the list of configs to run + def genTestConfigs(self, names:str, params, whenClauses = {}) -> list: + ''' + Generate the the list of configurations as pytest parameters. + It will unpack the configuration set by looping on all combinations defined + by the given sets. + + Args: + names (str): + Comma separated list of variables do consider to build the + name of the file. + params (dict|list): + A configuration set as a dictionnary or a list of + configuration set. + whenCaluses (dict|list): + Provide a set of clauses to apply after unpacking + the configuration so we can patch some values depending on some others. + + Returns: + A list of pytest.param() ready to be fiven to parametrized pytest functions. + ''' + # get name ordering list + nameList = names.split(',') + + # gen list of complete configs + all_configs = [] + if isinstance(params, dict): + all_configs += self._genOneConfigSeries(names, params) + elif isinstance(params, list): + for p in params: + all_configs += self._genOneConfigSeries(names, p) + else: + raise Exception("Should never be called !") + + # convert as parametrize with nice name + result = [] + for config in all_configs: + # append the file + config['testfile'] = self.currentTestFile + config['testname'] = self.currentTestName + # gen name + nameParts = [self.currentTestName] + for name in nameList: + if isinstance(config[name], bool): + if config[name]: + nameParts.append(name) + elif isinstance(config[name], str): + nameParts.append(str(config[name])) + else: + nameParts.append(f"{name}-{config[name]}") + confName = "-".join(nameParts) + + # apply when clause + config = self._applyWhen(config, whenClauses) + + result.append(pytest.param(config, id=confName)) + + # ok + return result + + def extractNamingParameters(self, params) -> list: + ''' + Loop on the parameters and check automatically what are the list of variable parameters. + + Args: + params (list|dict): + The list of configuration sets to scan or a single set. + + Returns: + The list of names of the variable parameters. + ''' + + # if not a list make a list + if not isinstance(params, list): + params = [params] + + # see params + seen = {} + variables = [] + + # loop + for param_set in params: + for key, value in param_set.items(): + if key in variables: + pass + elif key in DO_NOT_LOOP_ON: + pass + elif isinstance(value, list): + variables.append(key) + elif key in seen and seen[key] != value: + variables.append(key) + elif key not in seen: + seen[key] = value + + # by default sort by alphabetic order about var names + # TODO make something better by assigning a priority to vars + variables.sort() + + # ok + return ','.join(variables) + + def _genNextLevelCombinations(self, input: list, paramName: str, paramValues: list) -> list: + ''' + Take an input case list and unpack the given parameter to build the new combinations. + + Args: + input (list): + The incoming list of combinations already unpacked before this call. + paramName (str): + Name of the parameter to unpack. + paramValues (list): + The list of values to unpack and to build combinations for. + + Returns: + The updated list of run sets. + ''' + result = [] + for entry in input: + for value in paramValues: + v = copy.deepcopy(entry) + v[paramName] = value + result.append(v) + return result + + def _genOneConfigSeries(self, names: str, config: dict) -> list: + ''' + Generate the the list of configurations as pytest parameters. + It will unpack the configuration set by looping on all combinations defined + by the given sets. + + Args: + names (str): + Comma separated list of variables do consider to build the + name of the file. + config (dict): + A configuration set as a dictionnary. + + Returns: + A list of pytest.param() ready to be fiven to parametrized pytest functions. + ''' + # get name ordering list + nameList = names.split(',') + + # if there is ini in the list we put it at the end + loopOrder = nameList.copy() + if 'ini' in loopOrder: + loopOrder.remove('ini') + loopOrder.append('ini') + + # init core with everything not a list + core = {} + for key, value in config.items(): + if isinstance(value, list) and key not in DO_NOT_LOOP_ON: + assert key in nameList, f"All variable parameteres should be ordered in the names list, '{key}' is not." + else: + core[key] = copy.deepcopy(value) + + # at start we have only default core + result = [core] + + # loop + for key in loopOrder: + value = config[key] + assert isinstance(value, list), f"This parameter is marked as a list but is not a list : {key}={value} !" + result = self._genNextLevelCombinations(result, key, value) + + # ok + return result + + def _matchWhenClause(self, config: dict, when_clause: dict) -> bool: + ''' + Check the matching of a given when clause on the given configuration. + + Args: + config (dict): The configuration. + when_clause (dict): The when clause to check. + + Returns: + True if the clause, match, False otherwise. + ''' + for key, value in when_clause.items(): + if config[key] != value: + return False + return True + + def _applyWhen(self, config: dict, when: dict) -> dict: + ''' + Check if the when clause applies and apply if if any. + + Args: + config (dict): The configuration. + when_clause (dict): The when clause to check and apply. + + Returns: + The fixed config. + ''' + # nothing to do + if when == {}: + return config + + # clone + result = copy.deepcopy(config) + + # loop on when + if isinstance(when, list): + for clause in when: + if self._matchWhenClause(config, clause['conditions']): + result.update(clause['apply']) + elif isinstance(when, dict): + if self._matchWhenClause(config, when['conditions']): + result.update(when['apply']) + else: + raise Exception(f"Invalid type for 'when' : {when}") + + # ok + return result diff --git a/pytools/idfx_test_run.py b/pytools/idfx_test_run.py new file mode 100644 index 000000000..d14de0b41 --- /dev/null +++ b/pytools/idfx_test_run.py @@ -0,0 +1,257 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +import os +import sys +import json +import glob +import copy +import pytest +# idefix test class +import pytools.idfx_test as tst +from pytools.idfx_test_gen import IdefixDirTestGenerator +from contextlib import contextmanager + +@contextmanager +def moveInDir(path): + ''' + Change current working dir for the given directoy for what is inside the + with statement. + ''' + oldpwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(oldpwd) + +class IdexPytestRunner: + ''' + Implement the Idefix pytest runner to scan and run all the tests described by the files + testme.json into the /test directory of Idefix sources. + ''' + + def __init__(self, parentScriptFile: str): + self.currentTestFile="" + self.currentTestRunner: tst.idfxTest=None + self.parentScritFile=parentScriptFile + self.filterSubdir = os.environ.get("IDEFIX_TEST_FILTER_SUBDIR", "./test/") + + def _validateNaming(self, namings: str, autoExtracted: str, file: str): + names = namings.split(',') + for n in autoExtracted.split(','): + if not n in names: + raise Exception(f"Naming parameter list not match the auto-detectection, some are missinge. You gave : '{names}', detected '{autoExtracted}' into {file}") + + def _makeVariableArgAsList(self, namings: str, variants) -> list: + # make as a list + if isinstance(variants, list) == False: + variants = [variants] + + # split namings + namings = namings.split(",") + + # loop on each + for variant in variants: + for name in namings: + if name in variant and isinstance(variant[name], list) == False: + variant[name] = [variant[name]] + + def genTests(self) -> list: + sourceDir = os.path.dirname(self.parentScritFile) + + # loop over all tests + result = [] + with moveInDir(sourceDir): + # if missing / + if self.filterSubdir != "" and self.filterSubdir[-1] != "/": + self.filterSubdir += "/" + + # walk in test dir to find the tests & sort by name + testfiles = glob.glob(self.filterSubdir + "**/testme.json", recursive=True) + testfiles.sort() + + # loop over each + for testfile in testfiles: + try: + # calc some paths + testfileRelPath = os.path.relpath(testfile, os.path.join(sourceDir, 'test')) + testfilePath = os.path.abspath(os.path.join('test',testfileRelPath)) + testfileDir = os.path.dirname(testfileRelPath) + + # load json & build the inner test combinations + with open(testfilePath, 'r') as fp: + test = json.load(fp) + idefixTestGenerator=IdefixDirTestGenerator(testfilePath, testfileDir) + if 'namings' in test: + namings = test['namings'] + autoExtracted = idefixTestGenerator.extractNamingParameters(test['variants']) + self._validateNaming(namings, autoExtracted, testfilePath) + else: + namings = idefixTestGenerator.extractNamingParameters(test['variants']) + + # required to simplify the algos later, if var is listed as variable, we need to loop over it. + self._makeVariableArgAsList(namings, test['variants']) + + # gen + result += idefixTestGenerator.genTestConfigs(namings, test['variants'], test.get('when', {})) + except Exception as e: + raise Exception(f"Fail to generate tests from {testfileRelPath} : {e}") + + # ok + return result + + def run(self, config: dict) -> None: + # clone before modify to not modity for caller + config = copy.deepcopy(config) + + # print config + print("***************************************************") + print(json.dumps(config, indent='\t')) + print("***************************************************") + + # extract some infos for local usage + testfile = config["testfile"] + dumpname = config['dumpname'] + testname = config["testname"] + tolerance = config.get("tolerance", 0) + definitionFile = config.get("definitionFile", "") + standardTest = config.get("standardTest", True) + nonRegressionTest = config.get("nonRegressionTest", True) + nonRegressionTestIni = config.get("nonRegressionTestIni", None) + check_file_produced = config.get("check_file_produced", []) + problemDir = os.path.dirname(testfile) + + # cleanup some keyword not handled at the + # level of idx_test so we don't perturbate it + del config['dumpname'] + if 'definitionFile' in config: + del config['definitionFile'] + if 'tolerance' in config: + del config['tolerance'] + if 'standardTest' in config: + del config['standardTest'] + if 'nonRegressionTest' in config: + del config['nonRegressionTest'] + if 'nonRegressionTestIni' in config: + del config['nonRegressionTestIni'] + + # if switch from test, rebuild the runner (a runner make for one dir) + if self.currentTestFile != testfile: + self.currentTestRunner = tst.idfxTest(testfile, name=testname) + self.currentTestFile = testfile + + # run + with moveInDir(problemDir): + self._runNonRegression(dumpname, config['ini'], config, tolerance=tolerance, definitionFile=definitionFile, standardTest=standardTest, nonReg=nonRegressionTest, nonRegIni=nonRegressionTestIni) + + # check produced + for file in check_file_produced: + if not os.path.exists(file): + raise Exception(f"Don't find expected file to be produced by the run : {file} !") + + def _runNonRegression(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + if 'multirun' in config_override: + self._runNonRegMultirun(dumpname, ini, config_override, tolerance=tolerance, nonReg=nonReg, nonRegIni=nonRegIni, standardTest=standardTest, configure_and_compile=configure_and_compile, definitionFile=definitionFile, first_run_ini=first_run_ini, first_run_dumpname=first_run_dumpname) + else: + # single basic run + self._runNonRegSingleRun(dumpname, ini, config_override, tolerance=tolerance, nonReg=nonReg, standardTest=standardTest, configure_and_compile=configure_and_compile, definitionFile=definitionFile, first_run_ini=first_run_ini, first_run_dumpname=first_run_dumpname) + + def _runNonRegMultirun(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + # check + assert 'multirun' in config_override + + # loop over runs + for run in config_override['multirun']: + # copy config + run_config = copy.deepcopy(config_override) + + # patch a bit + del run_config['multirun'] + run_config.update(run) + nonReg=run_config.get('nonRegressionTest', nonReg) + dumpname=run_config.get('dumpname', dumpname) + if 'nonRegressionTest' in run_config: + del run_config['nonRegressionTest'] + standardTest=run_config.get('standardTest', standardTest) + if 'standardTest' in run_config: + del run_config['standardTest'] + + # make single run + self._runNonRegSingleRun(dumpname, run_config['ini'], run_config, definitionFile=definitionFile, tolerance=tolerance, nonReg=nonReg, nonRegIni=nonRegIni, standardTest=standardTest) + + def _runNonRegSingleRun(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True): + # build the runner + idefixTest = self.currentTestRunner + + # handle special override which should not go into + # idefixTest because it is not supported by the idfx_test layer. + config_override = copy.deepcopy(config_override) + nonRegIni = config_override.get("nonRegressionTestIni", nonRegIni) + if 'nonRegressionTestIni' in config_override: + del config_override['nonRegressionTestIni'] + + # apply config + idefixTest.applyConfig(config_override) + + # recompile if needed + if configure_and_compile: + idefixTest.configure(override=config_override, definitionFile=definitionFile) + idefixTest.compile() + + if first_run_ini: + idefixTest.run(inputFile=first_run_ini) + if nonReg: + idefixTest.nonRegressionTest(filename=first_run_dumpname, tolerance=tolerance) + + # Test the restart option + file_mtime={} + if not idefixTest.fake: + for file in idefixTest.restart_no_overwrite: + file_mtime[file] = os.path.getmtime(file) + + # restart + if idefixTest.restart: + restart = 1 + else: + restart = -1 + + # run + idefixTest.run(inputFile=ini, restart=restart) + + # regen ref if needed + if idefixTest.init: + idefixTest.makeReference(filename=dumpname) + + # check outputs + if standardTest: + idefixTest.standardTest() + if nonReg: + if nonRegIni: + idefixTest.inifile = nonRegIni + idefixTest.nonRegressionTest(filename=dumpname, tolerance=tolerance) + + # check that we didn't overrite the file during the restart + if not idefixTest.fake: + for file in idefixTest.restart_no_overwrite: + assert file_mtime[file] == os.path.getmtime(file), f"Dump file {file} was overwritten on restart" + + def main(self, all: bool = False): + if all: + sys.argv.append("-all") + idefixTest = tst.idfxTest(self.parentScritFile, name="main") + os.environ["IDEFIX_TEST_FILTER_SUBDIR"] = idefixTest.filterSubdir + + if idefixTest.all: + pytest.main(['-v', '--no-header', '--junit-xml=idefix-tests.junit.xml', '--tb=short'] + idefixTest.remainingArgs + [self.parentScritFile]) + else: + assert False, "Not yet supported !" + #elif self.check: + # idefixTest.checkOnly(filename=dumpname, tolerance=tolerance) + #else: + # for ini in ini_list: + # self.runNonRegression(dumpname, ini, {}, tolerance=tolerance) diff --git a/pytools/pytest.ini b/pytools/pytest.ini new file mode 100644 index 000000000..d1f826516 --- /dev/null +++ b/pytools/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files="test_*.py" diff --git a/pytools/tests/__init__.py b/pytools/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pytools/tests/test/pb1/testme.json b/pytools/tests/test/pb1/testme.json new file mode 100644 index 000000000..d8ddad44f --- /dev/null +++ b/pytools/tests/test/pb1/testme.json @@ -0,0 +1,11 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/pytools/tests/test/pb2/testme.json b/pytools/tests/test/pb2/testme.json new file mode 100644 index 000000000..96c1753ab --- /dev/null +++ b/pytools/tests/test/pb2/testme.json @@ -0,0 +1,18 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + }, + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": false, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/pytools/tests/test_idfx_test_gen.py b/pytools/tests/test_idfx_test_gen.py new file mode 100644 index 000000000..fc4f6261e --- /dev/null +++ b/pytools/tests/test_idfx_test_gen.py @@ -0,0 +1,208 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +from ..idfx_test_gen import IdefixDirTestGenerator +import pytest + +def test_extractNamingParameters(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + lst = gen.extractNamingParameters([ + { + "dec": [1,2,3], + "a": 10, + "b": 11, + "c": [True, False] + }, + { + "dec": [1,2,4], + "a": 10, + "b": 12, + "d": [True, False] + }, + ]) + + # valid + assert lst == 'b,c,d' + +def test_genNextLevelCombinations(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # init with single element + core = {} + init = [core] + + # gen first set of combinations + lst1 = gen._genNextLevelCombinations(init, "mpi", [True, False]) + assert lst1 == [ + {"mpi": True}, + {"mpi": False}, + ] + + # gen first second set of combinations + lst3 = gen._genNextLevelCombinations(lst1, "name", ["a", "b"]) + assert lst3 == [ + {"mpi": True, "name": "a"}, + {"mpi": True, "name": "b"}, + {"mpi": False, "name": "a"}, + {"mpi": False, "name": "b"}, + ] + +def test_genOneConfigSeries(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # gen + result = gen._genOneConfigSeries("mpi,name", { + "mpi": [True, False], + "name": ["a", "b"] + }) + + # check + assert result == [ + { + 'mpi': True, + 'name': 'a', + },{ + 'mpi': True, + 'name': 'b', + },{ + 'mpi': False, + 'name': 'a', + },{ + 'mpi': False, + 'name': 'b', + }, + ] + +def test_matchWhenClause_single(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # calls + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":10}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":11}) == False + assert gen._matchWhenClause({"a":10, "b": 11}, {"b":11}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"b":10}) == False + +def test_matchWhenClause_and(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # calls + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":10, "b":11}) == True + assert gen._matchWhenClause({"a":10, "b": 11}, {"a":11, "b":11}) == False + +def test_applyWhen_none(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={}) + + # check result + assert res == {"a": 10, "b": 11} + +def test_applyWhen_single_apply_yes(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={ + "conditions": { + "a": 10, + }, + "apply": { + "b": 12 + } + }) + + # check result + assert res == {"a": 10, "b": 12} + +def test_applyWhen_single_apply_no(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when={ + "conditions": { + "a": 9, + }, + "apply": { + "b": 12 + } + }) + + # check result + assert res == {"a": 10, "b": 11} + +def test_applyWhen_list_apply_yes(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # call when clause + res = gen._applyWhen({ + "a": 10, + "b": 11 + }, when=[ + { + "conditions": { + "a": 10, + }, + "apply": { + "b": 12 + } + }, + { + "conditions": { + "a": 13, + }, + "apply": { + "b": 13 + } + } + ]) + + # check result + assert res == {"a": 10, "b": 12} + +def test_gen_full(): + # build generator + gen = IdefixDirTestGenerator(__file__, "unit-test") + + # gen + result = gen.genTestConfigs("mpi,name", { + "mpi": [True, False], + "name": ["a", "b"] + }, { + "conditions": { + "mpi": True + }, + "apply": { + "extra": 10, + } + }) + + # check + assert result == [ + pytest.param({'mpi': True, 'name': 'a', 'testfile': __file__, 'testname': 'unit-test', 'extra': 10}, id='unit-test-mpi-a'), + pytest.param({'mpi': True, 'name': 'b', 'testfile': __file__, 'testname': 'unit-test', 'extra': 10}, id='unit-test-mpi-b'), + pytest.param({'mpi': False, 'name': 'a', 'testfile': __file__, 'testname': 'unit-test'}, id='unit-test-a'), + pytest.param({'mpi': False, 'name': 'b', 'testfile': __file__, 'testname': 'unit-test'}, id='unit-test-b'), + ] diff --git a/pytools/tests/test_idfx_test_run.py b/pytools/tests/test_idfx_test_run.py new file mode 100644 index 000000000..8843705ee --- /dev/null +++ b/pytools/tests/test_idfx_test_run.py @@ -0,0 +1,88 @@ +##################################################################################### +# Idefix MHD astrophysical code +# Copyright(C) Sébastien Valat +# and other code contributors +# Licensed under CeCILL 2.1 License, see COPYING for more information +##################################################################################### + +from ..idfx_test_run import IdexPytestRunner +import pytest +import os + +def test_genTests(): + # build runner + runner = IdexPytestRunner(__file__) + + # dir + dir = os.path.dirname(__file__) + + # generate + result = runner.genTests() + assert result == [ + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb1/testme.json', + 'testname': 'pb1' + }, marks=(), id='pb1-idefix.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb1/testme.json', + 'testname': 'pb1' + }, marks=(), id='pb1-idefix-implicit.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix.ini-noplot' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': True, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix-implicit.ini-noplot' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': False, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix.ini' + ), + pytest.param( + { + 'dumpname': 'dump.0001.dmp', + 'noplot': False, + 'reconstruction': 2, + 'tolerance': 1e-14, + 'ini': 'idefix-implicit.ini', + 'testfile': dir + '/test/pb2/testme.json', + 'testname': 'pb2' + }, marks=(), id='pb2-idefix-implicit.ini' + ), + ] diff --git a/reference b/reference index c4082b99a..32499c14c 160000 --- a/reference +++ b/reference @@ -1 +1 @@ -Subproject commit c4082b99a4c542def3177c96cb35b1c9d9002f18 +Subproject commit 32499c14cfacd05e1c9499bd23198a53c416c3c2 diff --git a/src/dataBlock/dataBlock.cpp b/src/dataBlock/dataBlock.cpp index 81874cef6..2877a8346 100644 --- a/src/dataBlock/dataBlock.cpp +++ b/src/dataBlock/dataBlock.cpp @@ -367,6 +367,7 @@ real DataBlock::ComputeTimestep() { void DataBlock::DeriveVectorPotential() { if constexpr(DefaultPhysics::mhd) { #ifdef EVOLVE_VECTOR_POTENTIAL + hydro->emf->EnforceVectorPotentialBoundary(hydro->Ve); hydro->emf->ComputeMagFieldFromA(hydro->Ve, hydro->Vs); #endif } diff --git a/src/dataBlock/dataBlockHost.cpp b/src/dataBlock/dataBlockHost.cpp index d570648fa..c7811b391 100644 --- a/src/dataBlock/dataBlockHost.cpp +++ b/src/dataBlock/dataBlockHost.cpp @@ -21,11 +21,11 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { // Create mirrors (should be mirror_view) for(int dir = 0 ; dir < 3 ; dir++) { - x[dir] = Kokkos::create_mirror_view(data->x[dir]); - xr[dir] = Kokkos::create_mirror_view(data->xr[dir]); - xl[dir] = Kokkos::create_mirror_view(data->xl[dir]); - dx[dir] = Kokkos::create_mirror_view(data->dx[dir]); - A[dir] = Kokkos::create_mirror_view(data->A[dir]); + x[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->x[dir]); + xr[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->xr[dir]); + xl[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->xl[dir]); + dx[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->dx[dir]); + A[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->A[dir]); } np_tot = data->np_tot; @@ -47,30 +47,30 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { // TO BE COMPLETED... - dV = Kokkos::create_mirror_view(data->dV); - Vc = Kokkos::create_mirror_view(data->hydro->Vc); - Uc = Kokkos::create_mirror_view(data->hydro->Uc); - InvDt = Kokkos::create_mirror_view(data->hydro->InvDt); + dV = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->dV); + Vc = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->Vc); + Uc = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->Uc); + InvDt = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->InvDt); #if MHD == YES - Vs = Kokkos::create_mirror_view(data->hydro->Vs); + Vs = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->Vs); this->haveCurrent = data->hydro->haveCurrent; if(data->hydro->haveCurrent) { - J = Kokkos::create_mirror_view(data->hydro->J); + J = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->J); } #ifdef EVOLVE_VECTOR_POTENTIAL - Ve = Kokkos::create_mirror_view(data->hydro->Ve); + Ve = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->Ve); #endif - D_EXPAND( Ex3 = Kokkos::create_mirror_view(data->hydro->emf->ez); , + D_EXPAND( Ex3 = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->emf->ez); , , - Ex1 = Kokkos::create_mirror_view(data->hydro->emf->ex); - Ex2 = Kokkos::create_mirror_view(data->hydro->emf->ey); ) + Ex1 = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->emf->ex); + Ex2 = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->hydro->emf->ey); ) #endif if(haveDust) { dustVc = std::vector>(data->dust.size()); for(int i = 0 ; i < data->dust.size() ; i++) { - dustVc[i] = Kokkos::create_mirror_view(data->dust[i]->Vc); + dustVc[i] = Kokkos::create_mirror_view(Kokkos::HostSpace(), data->dust[i]->Vc); } } @@ -80,7 +80,8 @@ DataBlockHost::DataBlockHost(DataBlock& datain) { this->coarseningDirection = data->coarseningDirection; for(int dir = 0 ; dir < 3 ; dir++) { if(coarseningDirection[dir]) { - coarseningLevel[dir] = Kokkos::create_mirror_view(data->coarseningLevel[dir]); + coarseningLevel[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), + data->coarseningLevel[dir]); } } } diff --git a/src/dataBlock/dumpToFile.cpp b/src/dataBlock/dumpToFile.cpp index 35b60b81d..51347e6ed 100644 --- a/src/dataBlock/dumpToFile.cpp +++ b/src/dataBlock/dumpToFile.cpp @@ -45,14 +45,15 @@ void DataBlock::DumpToFile(std::string filebase) { // TODO(lesurg) Make datablock a friend of hydro to get the Riemann flux? - //IdefixArray4D::HostMirror locFlux = Kokkos::create_mirror_view(this->hydro->FluxRiemann); + //IdefixArray4D::HostMirror locFlux = Kokkos::create_mirror_view(Kokkos::HostSpace(), + // this->hydro->FluxRiemann); //Kokkos::deep_copy(locFlux, this->FluxRiemann); #if MHD == YES IdefixArray4D::HostMirror locJ; if(hydro->haveCurrent) { - locJ = Kokkos::create_mirror_view(this->hydro->J); + locJ = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->hydro->J); Kokkos::deep_copy(locJ, this->hydro->J); } #endif @@ -120,7 +121,8 @@ void DataBlock::DumpToFile(std::string filebase) { // Write Vs #if MHD == YES // Write Vs - IdefixArray4D::HostMirror locVs = Kokkos::create_mirror_view(this->hydro->Vs); + IdefixArray4D::HostMirror locVs = Kokkos::create_mirror_view(Kokkos::HostSpace(), + this->hydro->Vs); Kokkos::deep_copy(locVs,this->hydro->Vs); dims[0] = this->np_tot[IDIR]+IOFFSET; dims[1] = this->np_tot[JDIR]+JOFFSET; @@ -137,7 +139,8 @@ void DataBlock::DumpToFile(std::string filebase) { dims[2] = this->np_tot[KDIR]; std::snprintf(fieldName,NAMESIZE,"Ex3"); - IdefixArray3D::HostMirror locE = Kokkos::create_mirror_view(this->hydro->emf->ez); + IdefixArray3D::HostMirror locE = Kokkos::create_mirror_view(Kokkos::HostSpace(), + this->hydro->emf->ez); Kokkos::deep_copy(locE,this->hydro->emf->ez); WriteVariable(fileHdl, 3, dims, fieldName, locE.data()); @@ -152,7 +155,8 @@ void DataBlock::DumpToFile(std::string filebase) { if(hydro->haveCurrent) { - IdefixArray4D::HostMirror locJ = Kokkos::create_mirror_view(this->hydro->J); + IdefixArray4D::HostMirror locJ = Kokkos::create_mirror_view(Kokkos::HostSpace(), + this->hydro->J); Kokkos::deep_copy(locJ,this->hydro->J); dims[0] = this->np_tot[IDIR]; dims[1] = this->np_tot[JDIR]; diff --git a/src/dataBlock/fargo.cpp b/src/dataBlock/fargo.cpp index a509ef886..7aa2d26a5 100644 --- a/src/dataBlock/fargo.cpp +++ b/src/dataBlock/fargo.cpp @@ -138,10 +138,15 @@ Fargo::Fargo(Input &input, int nmax, DataBlock *data) { for(int i=0 ; i < nvar ; i++) { vars.push_back(i); } + #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR + const int dirShift = JDIR; + #elif GEOMETRY == SPHERICAL + const int dirShift = KDIR; + #endif #if MHD == YES - this->mpi.Init(data->mygrid, vars, this->nghost.data(), data->np_int.data(), true); + this->mpiExchanger.Init(data->mygrid, dirShift, vars, this->nghost, data->np_int, true); #else - this->mpi.Init(data->mygrid, vars, this->nghost.data(), data->np_int.data()); + this->mpiExchanger.Init(data->mygrid, dirShift, vars, this->nghost, data->np_int, false); #endif } #endif diff --git a/src/dataBlock/fargo.hpp b/src/dataBlock/fargo.hpp index 099c99350..a1efcb2b6 100644 --- a/src/dataBlock/fargo.hpp +++ b/src/dataBlock/fargo.hpp @@ -11,7 +11,7 @@ #include #include "idefix.hpp" #ifdef WITH_MPI - #include "mpi.hpp" + #include "exchanger.hpp" #endif #include "physics.hpp" @@ -63,7 +63,7 @@ class Fargo { IdefixArray4D scrhVs; #ifdef WITH_MPI - Mpi mpi; // Fargo-specific MPI layer + Exchanger mpiExchanger; // Fargo-specific MPI layer #endif std::array beg; @@ -290,11 +290,7 @@ void Fargo::StoreToScratch(Fluid* hydro) { } #if WITH_MPI if(haveDomainDecomposition) { - #if GEOMETRY == CARTESIAN || GEOMETRY == POLAR - this->mpi.ExchangeX2(scrhUc, scrhVs); - #elif GEOMETRY == SPHERICAL - this->mpi.ExchangeX3(scrhUc, scrhVs); - #endif + this->mpiExchanger.Exchange(scrhUc, scrhVs); } #endif } diff --git a/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp b/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp index c50cadb83..921a312ac 100644 --- a/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp +++ b/src/fluid/RiemannSolver/MHDsolvers/storeFlux.hpp @@ -87,9 +87,9 @@ KOKKOS_FORCEINLINE_FUNCTION void K_StoreHLLD( const int i, const int j, const in const IdefixArray3D &aR, const IdefixArray3D &dL, const IdefixArray3D &dR) { - EXPAND( const int Xn = DIR+MX1; , - const int Xt = (DIR == IDIR ? MX2 : MX1); , - const int Xb = (DIR == KDIR ? MX2 : MX3); ) + EXPAND( const int Xn = DIR+MX1; , + const int Xt = (DIR == IDIR ? MX2 : MX1); , + [[maybe_unused]] const int Xb = (DIR == KDIR ? MX2 : MX3); ) // Compute magnetic pressure [[maybe_unused]] real ptR, ptL; @@ -201,7 +201,7 @@ KOKKOS_FORCEINLINE_FUNCTION void K_StoreHLLD( const int i, const int j, const in } #if COMPONENTS > 1 - EXPAND( Et(k,j,i) = -st*(ar*vL[Xt] - al*vR[Xt])*scrh; , + D_EXPAND( Et(k,j,i) = -st*(ar*vL[Xt] - al*vR[Xt])*scrh; , , Eb(k,j,i) = -sb*(ar*vL[Xb] - al*vR[Xb])*scrh; ); #endif diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index e447aefc8..47f0c2954 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -22,45 +22,36 @@ void Axis::ShowConfig() { } -void Axis::SymmetrizeEx1Side(int jref) { +void Axis::SymmetrizeEx1Side(int jref, IdefixArray3D Ex1) { #if DIMENSIONS == 3 - IdefixArray3D Ex1 = this->ex; IdefixArray1D Ex1Avg = this->Ex1Avg; - if(isTwoPi) { - idefix_for("Ex1_ini",0,data->np_tot[IDIR], - KOKKOS_LAMBDA(int i) { - Ex1Avg(i) = ZERO_F; - }); - - idefix_for("Ex1_Symmetrize",data->beg[KDIR],data->end[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA(int k,int i) { - Kokkos::atomic_add(&Ex1Avg(i), Ex1(k,jref,i)); + idefix_for("Ex1_ini",0,data->np_tot[IDIR], + KOKKOS_LAMBDA(int i) { + Ex1Avg(i) = ZERO_F; }); - if(needMPIExchange) { - #ifdef WITH_MPI - Kokkos::fence(); - // sum along all of the processes on the same r - MPI_Allreduce(MPI_IN_PLACE, Ex1Avg.data(), data->np_tot[IDIR], realMPI, - MPI_SUM, data->mygrid->AxisComm); - #endif - } - int ncells=data->mygrid->np_int[KDIR]; - - idefix_for("Ex1_Store",0,data->np_tot[KDIR],0,data->np_tot[IDIR], - KOKKOS_LAMBDA(int k,int i) { - Ex1(k,jref,i) = Ex1Avg(i)/((real) ncells); - }); - } else { - // if we're not doing full two pi, the flow is symmetric with respect to the axis, and the axis - // EMF is simply zero - idefix_for("Ex1_Store",0,data->np_tot[KDIR],0,data->np_tot[IDIR], + idefix_for("Ex1_Symmetrize",data->beg[KDIR],data->end[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { - Ex1(k,jref,i) = ZERO_F; + Kokkos::atomic_add(&Ex1Avg(i), Ex1(k,jref,i)); }); + if(needMPIExchange) { + #ifdef WITH_MPI + Kokkos::fence(); + // sum along all of the processes on the same r + MPI_Allreduce(MPI_IN_PLACE, Ex1Avg.data(), data->np_tot[IDIR], realMPI, + MPI_SUM, data->mygrid->AxisComm); + #endif } + + int ncells=data->mygrid->np_int[KDIR]; + + idefix_for("Ex1_Store",0,data->np_tot[KDIR],0,data->np_tot[IDIR], + KOKKOS_LAMBDA(int k,int i) { + Ex1(k,jref,i) = Ex1Avg(i)/((real) ncells); + }); + #endif } @@ -71,9 +62,7 @@ void Axis::SymmetrizeEx1Side(int jref) { // Hence, we enforce a regularisation of Ex3 for consistancy. -void Axis::RegularizeEx3side(int jref) { - IdefixArray3D Ex3 = this->ez; - +void Axis::RegularizeEx3side(int jref, IdefixArray3D Ex3) { idefix_for("Ex3_Regularise",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { Ex3(k,jref,i) = 0.0; @@ -100,9 +89,9 @@ void Axis::RegularizeCurrentSide(int side) { sign = -1; } IdefixArray1D BAvg = this->Ex1Avg; - IdefixArray1D x2 = data->x[JDIR]; IdefixArray1D x1 = data->x[IDIR]; IdefixArray1D dx3 = data->dx[KDIR]; + IdefixArray1D dx2 = data->dx[JDIR]; idefix_for("B_ini",0,data->np_tot[IDIR], KOKKOS_LAMBDA(int i) { @@ -132,8 +121,7 @@ void Axis::RegularizeCurrentSide(int side) { idefix_for("fixJ",0,data->np_tot[KDIR],0,data->np_tot[IDIR], KOKKOS_LAMBDA(int k,int i) { - real th = x2(jc); - real fact = 2*sign/(deltaPhi*x1(i)*sin(th)); + real fact = 2*sign/(deltaPhi*x1(i)*dx2(jc)); J(IDIR, k,js,i) = BAvg(i)*fact; }); @@ -142,18 +130,20 @@ void Axis::RegularizeCurrentSide(int side) { // Average the Emf component along the axis -void Axis::RegularizeEMFs() { +void Axis::RegularizeEMFs(IdefixArray3D ex, + IdefixArray3D ey, + IdefixArray3D ez) { idfx::pushRegion("Axis::RegularizeEMFs"); if(this->axisLeft) { int jref = data->beg[JDIR]; - SymmetrizeEx1Side(jref); - RegularizeEx3side(jref); + SymmetrizeEx1Side(jref, ex); + RegularizeEx3side(jref, ez); } if(this->axisRight) { int jref = data->end[JDIR]; - SymmetrizeEx1Side(jref); - RegularizeEx3side(jref); + SymmetrizeEx1Side(jref, ex); + RegularizeEx3side(jref, ez); } idfx::popRegion(); diff --git a/src/fluid/boundary/axis.hpp b/src/fluid/boundary/axis.hpp index b74697862..1a1227b61 100644 --- a/src/fluid/boundary/axis.hpp +++ b/src/fluid/boundary/axis.hpp @@ -26,15 +26,16 @@ class Axis { public: template explicit Axis(Boundary *); // Initialisation - void RegularizeEMFs(); // Regularize the EMF sitting on the axis + void RegularizeEMFs(IdefixArray3D, IdefixArray3D, IdefixArray3D); + // Regularize the EMF sitting on the axis void RegularizeCurrent(); // Regularize the currents along the axis void EnforceAxisBoundary(int side); // Enforce the boundary conditions (along X2) void RegularizeBX2s(); // Regularize BX2s on the axis void ShowConfig(); - - void SymmetrizeEx1Side(int); // Symmetrize on a specific side (internal method) - void RegularizeEx3side(int); // Regularize Ex3 along the axis (internal method) + // Internal methods + void SymmetrizeEx1Side(int, IdefixArray3D); // Symmetrize on a specific side + void RegularizeEx3side(int side, IdefixArray3D ex3); // Regularize Ex3 along the axis void RegularizeCurrentSide(int); // Regularize J along the axis (internal method) void FixBx2sAxis(int side); // Fix BX2s on the axis using the field around it (internal) void FixBx2sAxisGhostAverage(int side); //Fix BX2s on the axis using the average of neighbouring @@ -76,9 +77,6 @@ class Axis { IdefixArray1D symmetryVc; IdefixArray1D symmetryVs; - IdefixArray3D ex; - IdefixArray3D ey; - IdefixArray3D ez; IdefixArray4D J; IdefixArray4D Vc; @@ -94,11 +92,6 @@ Axis::Axis(Boundary *boundary) { Vc = boundary->Vc; Vs = boundary->Vs; J = boundary->fluid->J; - if constexpr(Phys::mhd) { - ex = boundary->fluid->emf->ex; - ey = boundary->fluid->emf->ey; - ez = boundary->fluid->emf->ez; - } data = boundary->data; haveMHD = Phys::mhd; diff --git a/src/fluid/boundary/boundary.hpp b/src/fluid/boundary/boundary.hpp index 897c1ce10..4065cc60d 100644 --- a/src/fluid/boundary/boundary.hpp +++ b/src/fluid/boundary/boundary.hpp @@ -34,6 +34,8 @@ template using InternalBoundaryFunc = void (*) (Fluid *, const real t); using InternalBoundaryFuncOld = void (*) (DataBlock &, const real t); // DEPRECATED +using BoundingBox = std::array,3>; + template class Boundary { public: @@ -83,12 +85,22 @@ class Boundary { const BoundarySide &, Function ); + template + void BoundaryFor(const std::string &, + BoundingBox box, + Function ); + template void BoundaryForAll(const std::string &, const int &, const BoundarySide &, Function ); + template + void BoundaryForAll(const std::string &, + BoundingBox box, + Function ); + template void BoundaryForX1s(const std::string &, const int &, @@ -115,6 +127,10 @@ class Boundary { bool haveLeftAxis{false}; ///< True if the left boundary is an axis bool haveRightAxis{false}; ///< True if the right boundary is an axis + std::array,3> GhostBoxVc; ///< A bounding box for each ghost regions + std::array,3>,3> + GhostBoxVs; ///< A bounding box each Vs component + private: friend class Axis; Fluid *fluid; // pointer to parent hydro object @@ -154,6 +170,49 @@ Boundary::Boundary(Fluid* fluid) { data->nghost[IDIR]); } + // Initialise the Bounding Boxes for cell-centered variables + for(int dir = 0 ; dir < 3 ; dir++) { + // dir=direction along which we plan to apply the boundary conditions + for(int side = 0; side < 2 ; side++) { + // Side on which we apply the boundaries + for(int dim = 0 ; dim < 3 ; dim++) { + // Dimension of the datacube + if(dim != dir) { + GhostBoxVc[dir][side][dim][0] = 0; + GhostBoxVc[dir][side][dim][1] = data->np_tot[dim]; + } else { + GhostBoxVc[dir][side][dim][0] = side*(data->end[dim]); + GhostBoxVc[dir][side][dim][1] = side*(data->end[dim])+data->nghost[dim]; + } + } + } + } + + // Initialise the Bounding Boxes for face-centered variables (NB: we need one for each component) + for(int component = 0 ; component < DIMENSIONS ; component++) { + // Initialise the boxes for face-centered variables with the same bounding box + GhostBoxVs[component] = GhostBoxVc; + for(int dir = 0 ; dir < 3 ; dir++) { + for(int side = 0 ; side < 2 ; side++) { + // Add one element in the normal direction since we're staggered + GhostBoxVs[component][dir][side][component][1] += 1; + // Do not overwrite last active BXs normal if not serial+periodic + if(dir == component) { + if(side==left) { + if(data->mygrid->nproc[dir] > 1 || data->lbound[dir] != BoundaryType::periodic) { + GhostBoxVs[component][dir][side][component][1] -= 1; + } + } + if(side==right) { + if(data->mygrid->nproc[dir] > 1 || data->rbound[dir] != BoundaryType::periodic) { + GhostBoxVs[component][dir][side][component][0] += 1; + } + } + } + } + } + } + // Init MPI stack when needed #ifdef WITH_MPI //////////////////////////////////////////////////////////////////////////// @@ -185,7 +244,8 @@ Boundary::Boundary(Fluid* fluid) { } } - mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data(), Phys::mhd); + mpi.Init(data->mygrid, mapVars, data->nghost, data->np_int, + data->lbound, data->rbound, Phys::mhd); #endif // MPI idfx::popRegion(); @@ -970,54 +1030,47 @@ template template inline void Boundary::BoundaryForAll( const std::string & name, - const int &dir, - const BoundarySide &side, + BoundingBox box, Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, 0, this->nVar, kbeg, kend, jbeg, jend, ibeg, iend, function); + idefix_for(name, 0, this->nVar, + box[KDIR][0], box[KDIR][1], + box[JDIR][0], box[JDIR][1], + box[IDIR][0], box[IDIR][1], + function); } template template -inline void Boundary::BoundaryFor( +inline void Boundary::BoundaryForAll( const std::string & name, const int &dir, const BoundarySide &side, Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; + BoundaryForAll(name,GhostBoxVc[dir][side],function); +} +template +template +inline void Boundary::BoundaryFor( + const std::string & name, + BoundingBox box, + Function function) { + idefix_for(name, + box[KDIR][0], box[KDIR][1], + box[JDIR][0], box[JDIR][1], + box[IDIR][0], box[IDIR][1], + function); +} - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); +template +template +inline void Boundary::BoundaryFor( + const std::string & name, + const int &dir, + const BoundarySide &side, + Function function) { + BoundaryFor(name,GhostBoxVc[dir][side],function); } template @@ -1027,23 +1080,7 @@ inline void Boundary::BoundaryForX1s( const int &dir, const BoundarySide &side, Function function) { - const int nxi = data->np_int[IDIR]+1; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]+1; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); + BoundaryFor(name,GhostBoxVs[BX1s][dir][side],function); } template @@ -1053,23 +1090,7 @@ inline void Boundary::BoundaryForX2s( const int &dir, const BoundarySide &side, Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]+1; - const int nxk = data->np_int[KDIR]; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]+1; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); + BoundaryFor(name,GhostBoxVs[BX2s][dir][side],function); } template @@ -1079,23 +1100,7 @@ inline void Boundary::BoundaryForX3s( const int &dir, const BoundarySide &side, Function function) { - const int nxi = data->np_int[IDIR]; - const int nxj = data->np_int[JDIR]; - const int nxk = data->np_int[KDIR]+1; - - const int ighost = data->nghost[IDIR]; - const int jghost = data->nghost[JDIR]; - const int kghost = data->nghost[KDIR]; - - // Boundaries of the loop - const int ibeg = (dir == IDIR) ? side*(ighost+nxi) : 0; - const int iend = (dir == IDIR) ? ighost + side*(ighost+nxi) : data->np_tot[IDIR]; - const int jbeg = (dir == JDIR) ? side*(jghost+nxj) : 0; - const int jend = (dir == JDIR) ? jghost + side*(jghost+nxj) : data->np_tot[JDIR]; - const int kbeg = (dir == KDIR) ? side*(kghost+nxk) : 0; - const int kend = (dir == KDIR) ? kghost + side*(kghost+nxk) : data->np_tot[KDIR]+1; - - idefix_for(name, kbeg, kend, jbeg, jend, ibeg, iend, function); + BoundaryFor(name,GhostBoxVs[BX3s][dir][side],function); } diff --git a/src/fluid/calcRightHandSide.hpp b/src/fluid/calcRightHandSide.hpp index 7777d1c22..0a0d3e194 100644 --- a/src/fluid/calcRightHandSide.hpp +++ b/src/fluid/calcRightHandSide.hpp @@ -363,7 +363,7 @@ struct Fluid_CalcRHSFunctor { #if (GEOMETRY == SPHERICAL) && (COMPONENTS == 3) rhs[iMPHI] /= FABS(sinx2(j)); if constexpr(Phys::mhd) { - rhs[iBPHI] = -dt / (rt(i)*dx(j)) * (Flux(iBPHI, k, j+1, i) - Flux(iBPHI, k, j, i)); + rhs[iBPHI] = -dt / (x1(i)*dx(j)) * (Flux(iBPHI, k, j+1, i) - Flux(iBPHI, k, j, i)); } // MHD #endif // GEOMETRY } diff --git a/src/fluid/checkDivB.hpp b/src/fluid/checkDivB.hpp index c78f6b3e0..6e47d4c8c 100644 --- a/src/fluid/checkDivB.hpp +++ b/src/fluid/checkDivB.hpp @@ -31,12 +31,9 @@ real Fluid::CheckDivB() { KOKKOS_LAMBDA (int k, int j, int i, real &divBmax) { [[maybe_unused]] real dB1,dB2,dB3; [[maybe_unused]] real d1, d2, d3; - [[maybe_unused]] real B1,B2,B3; dB1=dB2=dB3=ZERO_F; d1=d2=d3=ZERO_F; - B1=B2=B3=ZERO_F; - D_EXPAND( dB1=(Ax1(k,j,i+1)*Vs(BX1s,k,j,i+1)-Ax1(k,j,i)*Vs(BX1s,k,j,i)); , dB2=(Ax2(k,j+1,i)*Vs(BX2s,k,j+1,i)-Ax2(k,j,i)*Vs(BX2s,k,j,i)); , @@ -46,12 +43,8 @@ real Fluid::CheckDivB() { d2=0.5*(Ax2(k,j+1,i) + Ax2(k,j,i)); , d3=0.5*(Ax3(k+1,j,i) + Ax3(k,j,i)); ) - D_EXPAND( B1=0.5*(Vs(BX1s,k,j,i+1) + Vs(BX1s,k,j,i)); , - B2=0.5*(Vs(BX2s,k,j+1,i) + Vs(BX2s,k,j,i)); , - B3=0.5*(Vs(BX3s,k+1,j,i) + Vs(BX3s,k,j,i)); ) - real amplitude = 1e-40; - amplitude += D_EXPAND( std::fabs(B1)*d1, + std::fabs(B2)*d2, + std::fabs(B3)*d3 ); + const real amplitude = D_EXPAND( d1, + d2, + d3 ); divBmax=FMAX(FABS(D_EXPAND(dB1, +dB2, +dB3))/amplitude,divBmax); }, diff --git a/src/fluid/checkNan.hpp b/src/fluid/checkNan.hpp index 88e206e5e..0e8890725 100644 --- a/src/fluid/checkNan.hpp +++ b/src/fluid/checkNan.hpp @@ -67,7 +67,7 @@ int Fluid::CheckNan() { DataBlockHost dataHost(*data); - IdefixHostArray4D VcHost = Kokkos::create_mirror_view(this->Vc); + IdefixHostArray4D VcHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->Vc); Kokkos::deep_copy(VcHost,Vc); int nerrormax=10; @@ -95,7 +95,7 @@ int Fluid::CheckNan() { } if constexpr(Phys::mhd) { - IdefixHostArray4D VsHost = Kokkos::create_mirror_view(this->Vs); + IdefixHostArray4D VsHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->Vs); Kokkos::deep_copy(VsHost,Vs); for(int k = data->beg[KDIR] ; k < data->end[KDIR]+KOFFSET ; k++) { for(int j = data->beg[JDIR] ; j < data->end[JDIR]+JOFFSET ; j++) { diff --git a/src/fluid/constrainedTransport/CMakeLists.txt b/src/fluid/constrainedTransport/CMakeLists.txt index 06efc782b..7d99b805c 100644 --- a/src/fluid/constrainedTransport/CMakeLists.txt +++ b/src/fluid/constrainedTransport/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources(idefix PUBLIC ${CMAKE_CURRENT_LIST_DIR}/constrainedTransport.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/EMFexchange.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/enforceEMFBoundary.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/enforceVectorPotentialBoundary.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveMagField.hpp PUBLIC ${CMAKE_CURRENT_LIST_DIR}/evolveVectorPotential.hpp ) diff --git a/src/fluid/constrainedTransport/EMFexchange.hpp b/src/fluid/constrainedTransport/EMFexchange.hpp index 187186271..f6e90f75a 100644 --- a/src/fluid/constrainedTransport/EMFexchange.hpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -14,16 +14,18 @@ #ifdef WITH_MPI template -void ConstrainedTransport::ExchangeAll() { - if(data->mygrid->nproc[IDIR]>1) this->ExchangeX1(); - if(data->mygrid->nproc[JDIR]>1) this->ExchangeX2(); - if(data->mygrid->nproc[KDIR]>1) this->ExchangeX3(); +void ConstrainedTransport::ExchangeAll(IdefixArray3D ex, + IdefixArray3D ey, + IdefixArray3D ez) { + if(data->mygrid->nproc[IDIR]>1) this->ExchangeX1(ey,ez); + if(data->mygrid->nproc[JDIR]>1) this->ExchangeX2(ex,ez); + if(data->mygrid->nproc[KDIR]>1) this->ExchangeX3(ex,ey); } // Exchange EMFs in X1 template -void ConstrainedTransport::ExchangeX1() { +void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3D ez) { idfx::pushRegion("Emf::ExchangeX1"); @@ -34,8 +36,6 @@ void ConstrainedTransport::ExchangeX1() { IdefixArray1D BufferLeft=BufferSendX1[faceLeft]; IdefixArray1D BufferRight=BufferSendX1[faceRight]; - IdefixArray3D ey=this->ey; - IdefixArray3D ez=this->ez; // If MPI Persistent, start receiving even before the buffers are filled @@ -61,7 +61,6 @@ void ConstrainedTransport::ExchangeX1() { idefix_for("LoadBufferX1Emfz",kbeg,kend,jbeg,jend+1, KOKKOS_LAMBDA (int k, int j) { - BufferLeft( (j-jbeg) + (k-kbeg)*(ny+1) ) = ez(k,j,ileft); BufferRight( (j-jbeg) + (k-kbeg)*(ny+1) ) = ez(k,j,iright); } ); @@ -70,7 +69,6 @@ void ConstrainedTransport::ExchangeX1() { idefix_for("LoadBufferX1Emfy",kbeg,kend+1,jbeg,jend, KOKKOS_LAMBDA (int k, int j) { - BufferLeft( (j-jbeg) + (k-kbeg)*ny + Vsindex ) = ey(k,j,ileft); BufferRight( (j-jbeg) + (k-kbeg)*ny + Vsindex ) = ey(k,j,iright); } ); @@ -91,29 +89,21 @@ void ConstrainedTransport::ExchangeX1() { BufferLeft=BufferRecvX1[faceLeft]; BufferRight=BufferRecvX1[faceRight]; - // We average the edge emfs zones + // Erase the emf with the one coming from the left process + idefix_for("StoreBufferX1Emfz",kbeg,kend,jbeg,jend+1, KOKKOS_LAMBDA (int k, int j) { if(lbound == internal || lbound == periodic) { - ez(k,j,ileft) = HALF_F*( - BufferLeft( (j-jbeg) + (k-kbeg)*(ny+1) ) + ez(k,j,ileft) ); - } - if(rbound == internal || rbound == periodic) { - ez(k,j,iright) = HALF_F*( - BufferRight( (j-jbeg) + (k-kbeg)*(ny+1) ) + ez(k,j,iright) ); + ez(k,j,ileft) = BufferLeft( (j-jbeg) + (k-kbeg)*(ny+1)); } }); + #if DIMENSIONS == 3 Vsindex = (ny+1)*nz; idefix_for("StoreBufferX1Emfy",kbeg,kend+1,jbeg,jend, KOKKOS_LAMBDA (int k, int j) { if(lbound == internal || lbound == periodic) { - ey(k,j,ileft) = HALF_F*( - BufferLeft( (j-jbeg) + (k-kbeg)*ny +Vsindex) + ey(k,j,ileft) ); - } - if(rbound == internal || rbound == periodic) { - ey(k,j,iright) = HALF_F*( - BufferRight( (j-jbeg) + (k-kbeg)*ny +Vsindex) + ey(k,j,iright) ); + ey(k,j,ileft) = BufferLeft( (j-jbeg) + (k-kbeg)*ny +Vsindex); } }); #endif @@ -124,7 +114,7 @@ void ConstrainedTransport::ExchangeX1() { // Exchange EMFs in X2 template -void ConstrainedTransport::ExchangeX2() { +void ConstrainedTransport::ExchangeX2(IdefixArray3D ex, IdefixArray3D ez) { idfx::pushRegion("Emf::ExchangeX2"); // Load the buffers with data @@ -133,8 +123,6 @@ void ConstrainedTransport::ExchangeX2() { [[maybe_unused]] int nz; IdefixArray1D BufferLeft=BufferSendX2[faceLeft]; IdefixArray1D BufferRight=BufferSendX2[faceRight]; - IdefixArray3D ex=this->ex; - IdefixArray3D ez=this->ez; // If MPI Persistent, start receiving even before the buffers are filled double tStart = MPI_Wtime(); @@ -158,7 +146,6 @@ void ConstrainedTransport::ExchangeX2() { idefix_for("LoadBufferX2Emfz",kbeg,kend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int i) { - BufferLeft( (i-ibeg) + (k-kbeg)*(nx+1) ) = ez(k,jleft,i); BufferRight( (i-ibeg) + (k-kbeg)*(nx+1) ) = ez(k,jright,i); } ); @@ -167,7 +154,6 @@ void ConstrainedTransport::ExchangeX2() { idefix_for("LoadBufferX1Emfx",kbeg,kend+1,ibeg,iend, KOKKOS_LAMBDA (int k, int i) { - BufferLeft( (i-ibeg) + (k-kbeg)*nx + Vsindex ) = ex(k,jleft,i); BufferRight( (i-ibeg) + (k-kbeg)*nx + Vsindex ) = ex(k,jright,i); } ); @@ -191,12 +177,7 @@ void ConstrainedTransport::ExchangeX2() { idefix_for("StoreBufferX2Emfz",kbeg,kend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int i) { if(lbound == internal || lbound == periodic) { - ez(k,jleft,i) = HALF_F*( - BufferLeft( (i-ibeg) + (k-kbeg)*(nx+1) ) + ez(k,jleft,i) ); - } - if(rbound == internal || rbound == periodic) { - ez(k,jright,i) = HALF_F*( - BufferRight( (i-ibeg) + (k-kbeg)*(nx+1) ) + ez(k,jright,i) ); + ez(k,jleft,i) = BufferLeft( (i-ibeg) + (k-kbeg)*(nx+1) ); } }); #if DIMENSIONS == 3 @@ -204,12 +185,7 @@ void ConstrainedTransport::ExchangeX2() { idefix_for("StoreBufferX1Emfy",kbeg,kend+1,ibeg,iend, KOKKOS_LAMBDA (int k, int i) { if(lbound == internal || lbound == periodic) { - ex(k,jleft,i) = HALF_F*( - BufferLeft( (i-ibeg) + (k-kbeg)*nx +Vsindex) + ex(k,jleft,i) ); - } - if(rbound == internal || rbound == periodic) { - ex(k,jright,i) = HALF_F*( - BufferRight( (i-ibeg) + (k-kbeg)*nx +Vsindex) + ex(k,jright,i) ); + ex(k,jleft,i) = BufferLeft( (i-ibeg) + (k-kbeg)*nx +Vsindex); } }); #endif @@ -220,7 +196,7 @@ void ConstrainedTransport::ExchangeX2() { // Exchange EMFs in X3 template -void ConstrainedTransport::ExchangeX3() { +void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3D ey) { idfx::pushRegion("Emf::ExchangeX3"); @@ -229,8 +205,6 @@ void ConstrainedTransport::ExchangeX3() { int nx,ny; IdefixArray1D BufferLeft=BufferSendX3[faceLeft]; IdefixArray1D BufferRight=BufferSendX3[faceRight]; - IdefixArray3D ex=this->ex; - IdefixArray3D ey=this->ey; int Vsindex = 0; @@ -259,7 +233,6 @@ void ConstrainedTransport::ExchangeX3() { idefix_for("LoadBufferX3Emfx",jbeg,jend+1,ibeg,iend, KOKKOS_LAMBDA (int j, int i) { - BufferLeft( (i-ibeg) + (j-jbeg)*nx ) = ex(kleft,j,i); BufferRight( (i-ibeg) + (j-jbeg)*nx ) = ex(kright,j,i); } ); @@ -267,7 +240,6 @@ void ConstrainedTransport::ExchangeX3() { idefix_for("LoadBufferX3Emfy",jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int j, int i) { - BufferLeft( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) = ey(kleft,j,i); BufferRight( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) = ey(kright,j,i); } ); @@ -290,12 +262,7 @@ void ConstrainedTransport::ExchangeX3() { idefix_for("StoreBufferX3Emfx",jbeg,jend+1,ibeg,iend, KOKKOS_LAMBDA (int j, int i) { if(lbound == internal || lbound == periodic) { - ex(kleft,j,i) = HALF_F*( - BufferLeft( (i-ibeg) + (j-jbeg)*nx ) + ex(kleft,j,i) ); - } - if(rbound == internal || rbound == periodic) { - ex(kright,j,i) = HALF_F*( - BufferRight( (i-ibeg) + (j-jbeg)*nx ) + ex(kright,j,i) ); + ex(kleft,j,i) = BufferLeft( (i-ibeg) + (j-jbeg)*nx ); } }); @@ -303,16 +270,9 @@ void ConstrainedTransport::ExchangeX3() { idefix_for("StoreBufferX3Emfy",jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int j, int i) { if(lbound == internal || lbound == periodic) { - ey(kleft,j,i) = HALF_F*( - BufferLeft( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) + ey(kleft,j,i) ); - } - if(rbound == internal || rbound == periodic) { - ey(kright,j,i) = HALF_F*( - BufferRight( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) + ey(kright,j,i) ); + ey(kleft,j,i) = BufferLeft( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ); } }); - - idfx::popRegion(); } diff --git a/src/fluid/constrainedTransport/constrainedTransport.hpp b/src/fluid/constrainedTransport/constrainedTransport.hpp index c32e4ec96..24c317c09 100644 --- a/src/fluid/constrainedTransport/constrainedTransport.hpp +++ b/src/fluid/constrainedTransport/constrainedTransport.hpp @@ -101,13 +101,17 @@ class ConstrainedTransport { // Routines for evolving the magnetic potential (only available when EVOLVE_VECTOR_POTENTIAL) void EvolveVectorPotential(real, IdefixArray4D &); void ComputeMagFieldFromA(IdefixArray4D &Vein, IdefixArray4D &Vsout); + void EnforceVectorPotentialBoundary(IdefixArray4D &Vein); // Enforce BCs on A + void EnforceEMFBoundaryPeriodic(IdefixArray3D ex, + IdefixArray3D ey, + IdefixArray3D ez); #ifdef WITH_MPI // Exchange surface EMFs to remove interprocess round off errors - void ExchangeAll(); - void ExchangeX1(); - void ExchangeX2(); - void ExchangeX3(); + void ExchangeAll(IdefixArray3D ex, IdefixArray3D ey, IdefixArray3D ez); + void ExchangeX1(IdefixArray3D ey, IdefixArray3D ez); + void ExchangeX2(IdefixArray3D ex, IdefixArray3D ez); + void ExchangeX3(IdefixArray3D ex, IdefixArray3D ey); #endif private: @@ -449,6 +453,7 @@ void ConstrainedTransport::ShowConfig() { #include "calcRiemannEmf.hpp" #include "EMFexchange.hpp" #include "enforceEMFBoundary.hpp" +#include "enforceVectorPotentialBoundary.hpp" #include "evolveMagField.hpp" #include "evolveVectorPotential.hpp" diff --git a/src/fluid/constrainedTransport/enforceEMFBoundary.hpp b/src/fluid/constrainedTransport/enforceEMFBoundary.hpp index 192ec12e4..9d3d6913f 100644 --- a/src/fluid/constrainedTransport/enforceEMFBoundary.hpp +++ b/src/fluid/constrainedTransport/enforceEMFBoundary.hpp @@ -19,8 +19,9 @@ // This is because in some specific cases involving curvilinear coordinates, roundoff errors // can accumulate, leading to a small drift of face-values that should be strictly equal. // This behaviour is enabled using the flag below. - -//#define ENFORCE_EMF_CONSISTENCY +#ifdef EVOLVE_VECTOR_POTENTIAL + #define ENFORCE_EMF_CONSISTENCY +#endif template void ConstrainedTransport::EnforceEMFBoundary() { @@ -30,81 +31,73 @@ void ConstrainedTransport::EnforceEMFBoundary() { this->data->hydro->emfBoundaryFunc(*data, data->t); if(this->data->hydro->haveAxis) { - this->data->hydro->boundary->axis->RegularizeEMFs(); + this->data->hydro->boundary->axis->RegularizeEMFs(this->ex, this->ey, this->ez); } #ifdef ENFORCE_EMF_CONSISTENCY #ifdef WITH_MPI // This average the EMFs at the domain surface with immediate neighbours // to ensure the EMFs exactly match - this->ExchangeAll(); + this->ExchangeAll(this->ex, this->ey, this->ez); #endif #endif - IdefixArray3D ex = this->ex; - IdefixArray3D ey = this->ey; - IdefixArray3D ez = this->ez; - // Enforce specific EMF regularisation for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { if(data->lbound[dir] == shearingbox || data->rbound[dir] == shearingbox) { SymmetrizeEMFShearingBox(); } - #ifdef ENFORCE_EMF_CONSISTENCY - if(data->lbound[dir] == periodic && data->rbound[dir] == periodic) { - // If domain decomposed, periodicity is already enforced by ExchangeAll - if(data->mygrid->nproc[dir] == 1) { - int ioffset = (dir == IDIR) ? data->np_int[IDIR] : 0; - int joffset = (dir == JDIR) ? data->np_int[JDIR] : 0; - int koffset = (dir == KDIR) ? data->np_int[KDIR] : 0; - - int ibeg = (dir == IDIR) ? data->beg[IDIR] : 0; - int iend = (dir == IDIR) ? data->beg[IDIR]+1 : data->np_tot[IDIR]; - int jbeg = (dir == JDIR) ? data->beg[JDIR] : 0; - int jend = (dir == JDIR) ? data->beg[JDIR]+1 : data->np_tot[JDIR]; - int kbeg = (dir == KDIR) ? data->beg[KDIR] : 0; - int kend = (dir == KDIR) ? data->beg[KDIR]+1 : data->np_tot[KDIR]; - idefix_for("BoundaryEMFPeriodic",kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - real em; - - if(dir==IDIR) { - em = HALF_F*(ez(k,j,i)+ez(k,j,i+ioffset)); - ez(k,j,i) = em; - ez(k,j,i+ioffset) = em; - - #if DIMENSIONS == 3 - em = HALF_F*(ey(k,j,i)+ey(k,j,i+ioffset)); - ey(k,j,i) = em; - ey(k,j,i+ioffset) = em; - #endif - } - - if(dir==JDIR) { - em = HALF_F*(ez(k,j,i)+ez(k,j+joffset,i)); - ez(k,j,i) = em; - ez(k,j+joffset,i) = em; - - #if DIMENSIONS == 3 - em = HALF_F*(ex(k,j,i)+ex(k,j+joffset,i)); - ex(k,j,i) = em; - ex(k,j+joffset,i) = em; - #endif - } - - if(dir==KDIR) { - em = HALF_F*(ex(k,j,i)+ex(k+koffset,j,i)); - ex(k,j,i) = em; - ex(k+koffset,j,i) = em; - - em = HALF_F*(ey(k,j,i)+ey(k+koffset,j,i)); - ey(k,j,i) = em; - ey(k+koffset,j,i) = em; - } - }); - } + } + #ifdef ENFORCE_EMF_CONSISTENCY + EnforceEMFBoundaryPeriodic(this->ex, this->ey, this->ez); + #endif //ENFORCE_EMF_CONSISTENCY +#endif // MHD==YES + idfx::popRegion(); +} + +template +void ConstrainedTransport::EnforceEMFBoundaryPeriodic(IdefixArray3D ex, + IdefixArray3D ey, + IdefixArray3D ez) { + idfx::pushRegion("Emf::EnforceEMFBoundaryPeriodic"); + #if MHD == YES + for(int dir=0 ; dir < DIMENSIONS ; dir++ ) { + if(data->lbound[dir] == periodic && data->rbound[dir] == periodic) { + // If domain decomposed, periodicity is already enforced by ExchangeAll + if(data->mygrid->nproc[dir] == 1) { + int ioffset = (dir == IDIR) ? data->np_int[IDIR] : 0; + int joffset = (dir == JDIR) ? data->np_int[JDIR] : 0; + int koffset = (dir == KDIR) ? data->np_int[KDIR] : 0; + + int ibeg = (dir == IDIR) ? data->beg[IDIR] : 0; + int iend = (dir == IDIR) ? data->beg[IDIR]+1 : data->np_tot[IDIR]; + int jbeg = (dir == JDIR) ? data->beg[JDIR] : 0; + int jend = (dir == JDIR) ? data->beg[JDIR]+1 : data->np_tot[JDIR]; + int kbeg = (dir == KDIR) ? data->beg[KDIR] : 0; + int kend = (dir == KDIR) ? data->beg[KDIR]+1 : data->np_tot[KDIR]; + idefix_for("BoundaryEMFPeriodic",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + if(dir==IDIR) { + ez(k,j,i+ioffset) = ez(k,j,i); + #if DIMENSIONS == 3 + ey(k,j,i+ioffset) = ey(k,j,i); + #endif + } + + if(dir==JDIR) { + ez(k,j+joffset,i) = ez(k,j,i); + #if DIMENSIONS == 3 + ex(k,j+joffset,i) = ex(k,j,i); + #endif + } + + if(dir==KDIR) { + ex(k+koffset,j,i) = ex(k,j,i); + ey(k+koffset,j,i) = ey(k,j,i); + } + }); } - #endif //ENFORCE_EMF_CONSISTENCY + } } #endif // MHD==YES idfx::popRegion(); @@ -148,11 +141,13 @@ void ConstrainedTransport::SymmetrizeEMFShearingBox() { if(data->lbound[IDIR]==shearingbox) { // We send to our left (which, by periodicity, is the right end of the domain) // our value of sbEyL and get + Kokkos::fence(); MPI_Sendrecv(sbEyL.data(), size, realMPI, procLeft, 2001, sbEyR.data(), size, realMPI, procLeft, 2002, data->mygrid->CartComm, &status ); } if(data->rbound[IDIR]==shearingbox) { + Kokkos::fence(); // We send to our right (which, by periodicity, is the left end (=beginning) // of the domain) our value of sbEyR and get sbEyL MPI_Sendrecv(sbEyR.data(), size, realMPI, procRight, 2002, diff --git a/src/fluid/constrainedTransport/enforceVectorPotentialBoundary.hpp b/src/fluid/constrainedTransport/enforceVectorPotentialBoundary.hpp new file mode 100644 index 000000000..3019edfd8 --- /dev/null +++ b/src/fluid/constrainedTransport/enforceVectorPotentialBoundary.hpp @@ -0,0 +1,44 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef FLUID_CONSTRAINEDTRANSPORT_ENFORCEVECTORPOTENTIALBOUNDARY_HPP_ +#define FLUID_CONSTRAINEDTRANSPORT_ENFORCEVECTORPOTENTIALBOUNDARY_HPP_ +#include "constrainedTransport.hpp" + +template +void ConstrainedTransport::EnforceVectorPotentialBoundary(IdefixArray4D &Vein) { + idfx::pushRegion("Emf::EnforceVectorPotentialBoundary"); + + + IdefixArray3D Ax1, Ax2, Ax3; + + #ifdef EVOLVE_VECTOR_POTENTIAL + #if DIMENSIONS == 3 + Ax1 = Kokkos::subview(Vein, AX1e, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); + Ax2 = Kokkos::subview(Vein, AX2e, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); + #endif + Ax3 = Kokkos::subview(Vein, AX3e, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); + + + if(this->hydro->haveAxis) { + this->hydro->boundary->axis->RegularizeEMFs(Ax1, Ax2, Ax3); + } + + #ifdef ENFORCE_EMF_CONSISTENCY + #ifdef WITH_MPI + // This average the vector potential at the domain surface with immediate neighbours + // to ensure the vector potentials exactly match + + this->ExchangeAll(Ax1, Ax2, Ax3); + #endif + EnforceEMFBoundaryPeriodic(Ax1, Ax2, Ax3); + #endif + #endif // EVOLVE_VECTOR_POTENTIAL + + idfx::popRegion(); +} +#endif // FLUID_CONSTRAINEDTRANSPORT_ENFORCEVECTORPOTENTIALBOUNDARY_HPP_ diff --git a/src/global.hpp b/src/global.hpp index 5a9ea70fb..c4de3926e 100644 --- a/src/global.hpp +++ b/src/global.hpp @@ -39,7 +39,7 @@ template IdefixArray1D ConvertVectorToIdefixArray(std::vector &inputVector) { IdefixArray1D outArr = IdefixArray1D("Vector",inputVector.size()); IdefixHostArray1D outArrHost; - outArrHost = Kokkos::create_mirror_view(outArr); + outArrHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), outArr); for(int i = 0; i < inputVector.size() ; i++) { outArrHost(i) = inputVector[i]; } diff --git a/src/gravity/laplacian.cpp b/src/gravity/laplacian.cpp index 2300db82d..7ac329ea3 100644 --- a/src/gravity/laplacian.cpp +++ b/src/gravity/laplacian.cpp @@ -103,7 +103,7 @@ Laplacian::Laplacian(DataBlock *datain, std::array left std::vector mapVars; mapVars.push_back(ntarget); - this->mpi.Init(data->mygrid, mapVars, this->nghost.data(), this->np_int.data()); + this->mpi.Init(data->mygrid, mapVars, nghost, np_int, datain->lbound, datain->rbound, false); #endif idfx::popRegion(); diff --git a/src/grid.hpp b/src/grid.hpp index 0ffdb3a58..947d05d12 100644 --- a/src/grid.hpp +++ b/src/grid.hpp @@ -81,7 +81,7 @@ class SubGrid { parentGrid(grid), type(type), direction(d) { idfx::pushRegion("SubGrid::SubGrid()"); // Find the index of the current subgrid. - auto x = Kokkos::create_mirror_view(parentGrid->x[direction]); + auto x = Kokkos::create_mirror_view(Kokkos::HostSpace(), parentGrid->x[direction]); Kokkos::deep_copy(x,parentGrid->x[direction]); int iref = -1; for(int i = 0 ; i < x.extent(0) - 1 ; i++) { diff --git a/src/gridHost.cpp b/src/gridHost.cpp index ad61f0436..0c85d8e41 100644 --- a/src/gridHost.cpp +++ b/src/gridHost.cpp @@ -32,10 +32,10 @@ GridHost::GridHost(Grid &grid) { // Create mirrors on host for(int dir = 0 ; dir < 3 ; dir++) { - x[dir] = Kokkos::create_mirror_view(grid.x[dir]); - xr[dir] = Kokkos::create_mirror_view(grid.xr[dir]); - xl[dir] = Kokkos::create_mirror_view(grid.xl[dir]); - dx[dir] = Kokkos::create_mirror_view(grid.dx[dir]); + x[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), grid.x[dir]); + xr[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), grid.xr[dir]); + xl[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), grid.xl[dir]); + dx[dir] = Kokkos::create_mirror_view(Kokkos::HostSpace(), grid.dx[dir]); } idfx::popRegion(); diff --git a/src/macros.hpp b/src/macros.hpp index 2c7bda127..0df5f9430 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -25,7 +25,6 @@ #if COMPONENTS == 3 #define EXPAND(a,b,c) a b c #define SELECT(a,b,c) c - #endif #if DIMENSIONS == 1 diff --git a/src/main.cpp b/src/main.cpp index 164c134ee..a2dcd630a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ //@HEADER // ************************************************************************ // -// IDEFIX v 2.2.01 +// IDEFIX v 2.3.0 // // ************************************************************************ //@HEADER @@ -192,7 +192,9 @@ int main( int argc, char* argv[] ) { try { Tint.Cycle(data); } catch(std::exception &e) { - idfx::cout << "Main: WARNING! Caught an exception in TimeIntegrator." << std::endl; + idfx::cout << std::endl + << "Main: WARNING! Caught an exception in TimeIntegrator." + << std::endl; #ifdef WITH_MPI if(!Mpi::CheckSync(5)) { std::stringstream message; diff --git a/src/mpi.cpp b/src/mpi.cpp deleted file mode 100644 index 4de7739cb..000000000 --- a/src/mpi.cpp +++ /dev/null @@ -1,992 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - - -#include "mpi.hpp" -#include -#include -#include // NOLINT [build/c++11] -#include // NOLINT [build/c++11] -#include -#include -#include "idefix.hpp" -#include "dataBlock.hpp" - - -#if defined(OPEN_MPI) && OPEN_MPI -#include "mpi-ext.h" // Needed for CUDA-aware check */ -#endif - - -//#define MPI_NON_BLOCKING -#define MPI_PERSISTENT - -// init the number of instances -int Mpi::nInstances = 0; - -// MPI Routines exchange -void Mpi::ExchangeAll() { - IDEFIX_ERROR("Not Implemented"); -} - -/// -/// Initialise an instance of the MPI class. -/// @param grid: pointer to the grid object (needed to get the MPI neighbours) -/// @param inputMap: 1st indices of inputVc which are to be exchanged (i.e, the list of variables) -/// @param nghost: size of the ghost region in each direction -/// @param nint: size of the internal region in each direction -/// @param inputHaveVs: whether the instance should also treat face-centered variable -/// (optional, default false) -/// - -void Mpi::Init(Grid *grid, std::vector inputMap, - int nghost[3], int nint[3], - bool inputHaveVs) { - idfx::pushRegion("Mpi::Init"); - this->mygrid = grid; - - // increase the number of instances - nInstances++; - thisInstance=nInstances; - - // Transfer the vector of indices as an IdefixArray on the target - - // Allocate mapVars on target and copy it from the input argument list - this->mapVars = idfx::ConvertVectorToIdefixArray(inputMap); - this->mapNVars = inputMap.size(); - this->haveVs = inputHaveVs; - - // Compute indices of arrays we will be working with - for(int dir = 0 ; dir < 3 ; dir++) { - this->nghost[dir] = nghost[dir]; - this->nint[dir] = nint[dir]; - this->ntot[dir] = nint[dir]+2*nghost[dir]; - this->beg[dir] = nghost[dir]; - this->end[dir] = nghost[dir]+nint[dir]; - } - - ///////////////////////////////////////////////////////////////////////////// - // Init exchange datasets - bufferSizeX1 = 0; - bufferSizeX2 = 0; - bufferSizeX3 = 0; - - // Number of cells in X1 boundary condition: - bufferSizeX1 = nghost[IDIR] * nint[JDIR] * nint[KDIR] * mapNVars; - - if(haveVs) { - bufferSizeX1 += nghost[IDIR] * nint[JDIR] * nint[KDIR]; - #if DIMENSIONS>=2 - bufferSizeX1 += nghost[IDIR] * (nint[JDIR]+1) * nint[KDIR]; - #endif - - #if DIMENSIONS==3 - bufferSizeX1 += nghost[IDIR] * nint[JDIR] * (nint[KDIR]+1); - #endif // DIMENSIONS - } - - - BufferRecvX1[faceLeft ] = Buffer(bufferSizeX1); - BufferRecvX1[faceRight] = Buffer(bufferSizeX1); - BufferSendX1[faceLeft ] = Buffer(bufferSizeX1); - BufferSendX1[faceRight] = Buffer(bufferSizeX1); - - // Number of cells in X2 boundary condition (only required when problem >2D): -#if DIMENSIONS >= 2 - bufferSizeX2 = ntot[IDIR] * nghost[JDIR] * nint[KDIR] * mapNVars; - if(haveVs) { - // IDIR - bufferSizeX2 += (ntot[IDIR]+1) * nghost[JDIR] * nint[KDIR]; - #if DIMENSIONS>=2 - bufferSizeX2 += ntot[IDIR] * nghost[JDIR] * nint[KDIR]; - #endif - #if DIMENSIONS==3 - bufferSizeX2 += ntot[IDIR] * nghost[JDIR] * (nint[KDIR]+1); - #endif // DIMENSIONS - } - - BufferRecvX2[faceLeft ] = Buffer(bufferSizeX2); - BufferRecvX2[faceRight] = Buffer(bufferSizeX2); - BufferSendX2[faceLeft ] = Buffer(bufferSizeX2); - BufferSendX2[faceRight] = Buffer(bufferSizeX2); - -#endif -// Number of cells in X3 boundary condition (only required when problem is 3D): -#if DIMENSIONS ==3 - bufferSizeX3 = ntot[IDIR] * ntot[JDIR] * nghost[KDIR] * mapNVars; - - if(haveVs) { - // IDIR - bufferSizeX3 += (ntot[IDIR]+1) * ntot[JDIR] * nghost[KDIR]; - // JDIR - bufferSizeX3 += ntot[IDIR] * (ntot[JDIR]+1) * nghost[KDIR]; - // KDIR - bufferSizeX3 += ntot[IDIR] * ntot[JDIR] * nghost[KDIR]; - } - - BufferRecvX3[faceLeft ] = Buffer(bufferSizeX3); - BufferRecvX3[faceRight] = Buffer(bufferSizeX3); - BufferSendX3[faceLeft ] = Buffer(bufferSizeX3); - BufferSendX3[faceRight] = Buffer(bufferSizeX3); -#endif // DIMENSIONS - -#ifdef MPI_PERSISTENT - // Init persistent MPI communications - int procSend, procRecv; - - // X1-dir exchanges - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, - thisInstance*1000, mygrid->CartComm, &sendRequestX1[faceRight])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, - thisInstance*1000, mygrid->CartComm, &recvRequestX1[faceLeft])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, - thisInstance*1000+1,mygrid->CartComm, &sendRequestX1[faceLeft])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, - thisInstance*1000+1,mygrid->CartComm, &recvRequestX1[faceRight])); - - #if DIMENSIONS >= 2 - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, - thisInstance*1000+10, mygrid->CartComm, &sendRequestX2[faceRight])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, - thisInstance*1000+10, mygrid->CartComm, &recvRequestX2[faceLeft])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, - thisInstance*1000+11, mygrid->CartComm, &sendRequestX2[faceLeft])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, - thisInstance*1000+11, mygrid->CartComm, &recvRequestX2[faceRight])); - #endif - - #if DIMENSIONS == 3 - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, - thisInstance*1000+20, mygrid->CartComm, &sendRequestX3[faceRight])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, - thisInstance*1000+20, mygrid->CartComm, &recvRequestX3[faceLeft])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, - thisInstance*1000+21, mygrid->CartComm, &sendRequestX3[faceLeft])); - - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, - thisInstance*1000+21, mygrid->CartComm, &recvRequestX3[faceRight])); - #endif - -#endif // MPI_Persistent - - // say this instance is initialized. - isInitialized = true; - - idfx::popRegion(); -} - -// Destructor (clean up persistent communication channels) -Mpi::~Mpi() { - idfx::pushRegion("Mpi::~Mpi"); - if(isInitialized) { - // Properly clean up the mess - #ifdef MPI_PERSISTENT - idfx::cout << "Mpi(" << thisInstance - << "): Cleaning up MPI persistent communication channels" << std::endl; - for(int i=0 ; i< 2; i++) { - MPI_Request_free( &sendRequestX1[i]); - MPI_Request_free( &recvRequestX1[i]); - - #if DIMENSIONS >= 2 - MPI_Request_free( &sendRequestX2[i]); - MPI_Request_free( &recvRequestX2[i]); - #endif - - #if DIMENSIONS == 3 - MPI_Request_free( &sendRequestX3[i]); - MPI_Request_free( &recvRequestX3[i]); - #endif - } - #endif - if(thisInstance==1) { - idfx::cout << "Mpi(" << thisInstance << "): measured throughput is " - << bytesSentOrReceived/myTimer/1024.0/1024.0 << " MB/s" << std::endl; - idfx::cout << "Mpi(" << thisInstance << "): message sizes were " << std::endl; - idfx::cout << " X1: " << bufferSizeX1*sizeof(real)/1024.0/1024.0 << " MB" << std::endl; - idfx::cout << " X2: " << bufferSizeX2*sizeof(real)/1024.0/1024.0 << " MB" << std::endl; - idfx::cout << " X3: " << bufferSizeX3*sizeof(real)/1024.0/1024.0 << " MB" << std::endl; - } - isInitialized = false; - } - idfx::popRegion(); -} - -void Mpi::ExchangeX1(IdefixArray4D Vc, IdefixArray4D Vs) { - idfx::pushRegion("Mpi::ExchangeX1"); - - // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset,nx; - Buffer BufferLeft = BufferSendX1[faceLeft]; - Buffer BufferRight = BufferSendX1[faceRight]; - IdefixArray1D map = this->mapVars; - - // If MPI Persistent, start receiving even before the buffers are filled - myTimer -= MPI_Wtime(); - double tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1)); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; -#endif - myTimer += MPI_Wtime(); - - // Coordinates of the ghost region which needs to be transfered - ibeg = 0; - iend = nghost[IDIR]; - nx = nghost[IDIR]; // Number of points in x - offset = end[IDIR]; // Distance between beginning of left and right ghosts - jbeg = beg[JDIR]; - jend = end[JDIR]; - - kbeg = beg[KDIR]; - kend = end[KDIR]; - - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - BufferLeft.Pack(Vc, map, std::make_pair(ibeg+nx, iend+nx), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vc, map, std::make_pair(ibeg+offset-nx, iend+offset-nx), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - // Load face-centered field in the buffer - if(haveVs) { - BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg+nx+1, iend+nx+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg+offset-nx, iend+offset-nx), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - #if DIMENSIONS >= 2 - - BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg+nx, iend+nx), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg+offset-nx, iend+offset-nx), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg , kend)); - - #endif - - #if DIMENSIONS == 3 - - BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg+nx, iend+nx), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend+1)); - - BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg+offset-nx, iend+offset-nx), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend+1)); - - #endif - } - - // Wait for completion before sending out everything - Kokkos::fence(); - myTimer -= MPI_Wtime(); - tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1)); - // Wait for buffers to be received - MPI_Waitall(2,recvRequestX1,recvStatus); - -#else - int procSend, procRecv; - - #ifdef MPI_NON_BLOCKING - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - MPI_Request sendRequest[2]; - MPI_Request recvRequest[2]; - - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, 100, - mygrid->CartComm, &sendRequest[0])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, 100, - mygrid->CartComm, &recvRequest[0])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, 101, - mygrid->CartComm, &sendRequest[1])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, 101, - mygrid->CartComm, &recvRequest[1])); - - // Wait for recv to complete (we don't care about the sends) - MPI_Waitall(2, recvRequest, recvStatus); - - #else - MPI_Status status; - // Send to the right - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, 100, - BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, 100, - mygrid->CartComm, &status)); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,0,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, 101, - BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, 101, - mygrid->CartComm, &status)); - #endif -#endif - myTimer += MPI_Wtime(); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; - // Unpack - BufferLeft=BufferRecvX1[faceLeft]; - BufferRight=BufferRecvX1[faceRight]; - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vc, map,std::make_pair(ibeg+offset, iend+offset), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - // We fill the ghost zones - - if(haveVs) { - BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg+offset+1, iend+offset+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - #if DIMENSIONS >= 2 - BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vs, BX2s, std::make_pair(ibeg+offset, iend+offset), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg , kend)); - #endif - - #if DIMENSIONS == 3 - BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend+1)); - - BufferRight.Unpack(Vs, BX3s, std::make_pair(ibeg+offset, iend+offset), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend+1)); - #endif - } - -myTimer -= MPI_Wtime(); -#ifdef MPI_NON_BLOCKING - // Wait for the sends if they have not yet completed - MPI_Waitall(2, sendRequest, sendStatus); -#endif - -#ifdef MPI_PERSISTENT - MPI_Waitall(2, sendRequestX1, sendStatus); -#endif - myTimer += MPI_Wtime(); - bytesSentOrReceived += 4*bufferSizeX1*sizeof(real); - - idfx::popRegion(); -} - - -void Mpi::ExchangeX2(IdefixArray4D Vc, IdefixArray4D Vs) { - idfx::pushRegion("Mpi::ExchangeX2"); - - // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset,ny; - Buffer BufferLeft=BufferSendX2[faceLeft]; - Buffer BufferRight=BufferSendX2[faceRight]; - IdefixArray1D map = this->mapVars; - -// If MPI Persistent, start receiving even before the buffers are filled - myTimer -= MPI_Wtime(); - double tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2)); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; -#endif - myTimer += MPI_Wtime(); - - // Coordinates of the ghost region which needs to be transfered - ibeg = 0; - iend = ntot[IDIR]; - - jbeg = 0; - jend = nghost[JDIR]; - offset = end[JDIR]; // Distance between beginning of left and right ghosts - ny = nghost[JDIR]; - - kbeg = beg[KDIR]; - kend = end[KDIR]; - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - BufferLeft.Pack(Vc, map, std::make_pair(ibeg , iend), - std::make_pair(jbeg+ny , jend+ny), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vc, map, std::make_pair(ibeg , iend), - std::make_pair(jbeg+offset-ny , jend+offset-ny), - std::make_pair(kbeg , kend)); - - // Load face-centered field in the buffer - if(haveVs) { - BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg , iend+1), - std::make_pair(jbeg+ny , jend+ny), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg , iend+1), - std::make_pair(jbeg+offset-ny , jend+offset-ny), - std::make_pair(kbeg , kend)); - #if DIMENSIONS >= 2 - BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg , iend), - std::make_pair(jbeg+ny+1 , jend+ny+1), - std::make_pair(kbeg , kend)); - - BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg , iend), - std::make_pair(jbeg+offset-ny , jend+offset-ny), - std::make_pair(kbeg , kend)); - #endif - #if DIMENSIONS == 3 - - BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg , iend), - std::make_pair(jbeg+ny , jend+ny), - std::make_pair(kbeg , kend+1)); - - BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg , iend), - std::make_pair(jbeg+offset-ny , jend+offset-ny), - std::make_pair(kbeg , kend+1)); - - #endif - } - - // Send to the right - Kokkos::fence(); - - myTimer -= MPI_Wtime(); - tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2)); - MPI_Waitall(2,recvRequestX2,recvStatus); - -#else - int procSend, procRecv; - - #ifdef MPI_NON_BLOCKING - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - MPI_Request sendRequest[2]; - MPI_Request recvRequest[2]; - - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, 100, - mygrid->CartComm, &sendRequest[0])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, 100, - mygrid->CartComm, &recvRequest[0])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, 101, - mygrid->CartComm, &sendRequest[1])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, 101, - mygrid->CartComm, &recvRequest[1])); - - // Wait for recv to complete (we don't care about the sends) - MPI_Waitall(2, recvRequest, recvStatus); - - #else - MPI_Status status; - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, 200, - BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, 200, - mygrid->CartComm, &status)); - - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,1,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, 201, - BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, 201, - mygrid->CartComm, &status)); - #endif -#endif - myTimer += MPI_Wtime(); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; - // Unpack - BufferLeft=BufferRecvX2[faceLeft]; - BufferRight=BufferRecvX2[faceRight]; - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - // We fill the ghost zones - BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vc, map,std::make_pair(ibeg , iend), - std::make_pair(jbeg+offset , jend+offset), - std::make_pair(kbeg , kend)); - // We fill the ghost zones - - if(haveVs) { - BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), - std::make_pair(jbeg+offset , jend+offset), - std::make_pair(kbeg , kend)); - #if DIMENSIONS >= 2 - BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), - std::make_pair(jbeg+offset+1, jend+offset+1), - std::make_pair(kbeg , kend)); - #endif - #if DIMENSIONS == 3 - BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), - std::make_pair(jbeg, jend), - std::make_pair(kbeg, kend+1)); - - BufferRight.Unpack(Vs, BX3s,std::make_pair(ibeg , iend), - std::make_pair(jbeg+offset, jend+offset), - std::make_pair(kbeg , kend+1)); - #endif - } - - myTimer -= MPI_Wtime(); -#ifdef MPI_NON_BLOCKING - // Wait for the sends if they have not yet completed - MPI_Waitall(2, sendRequest, sendStatus); -#endif - -#ifdef MPI_PERSISTENT - MPI_Waitall(2, sendRequestX2, sendStatus); -#endif - myTimer += MPI_Wtime(); - bytesSentOrReceived += 4*bufferSizeX2*sizeof(real); - - idfx::popRegion(); -} - - -void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { - idfx::pushRegion("Mpi::ExchangeX3"); - - - // Load the buffers with data - int ibeg,iend,jbeg,jend,kbeg,kend,offset,nz; - Buffer BufferLeft=BufferSendX3[faceLeft]; - Buffer BufferRight=BufferSendX3[faceRight]; - IdefixArray1D map = this->mapVars; - - // If MPI Persistent, start receiving even before the buffers are filled - myTimer -= MPI_Wtime(); - - double tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3)); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; -#endif - myTimer += MPI_Wtime(); - // Coordinates of the ghost region which needs to be transfered - ibeg = 0; - iend = ntot[IDIR]; - - jbeg = 0; - jend = ntot[JDIR]; - - kbeg = 0; - kend = nghost[KDIR]; - offset = end[KDIR]; // Distance between beginning of left and right ghosts - nz = nghost[KDIR]; - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - BufferLeft.Pack(Vc, map, std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+nz, kend+nz)); - - BufferRight.Pack(Vc, map, std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg + offset-nz, kend+ offset-nz)); - - // Load face-centered field in the buffer - if(haveVs) { - BufferLeft.Pack(Vs, BX1s,std::make_pair(ibeg , iend+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+nz , kend+nz)); - - BufferRight.Pack(Vs, BX1s, std::make_pair(ibeg , iend+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg + offset-nz, kend+ offset-nz)); - - #if DIMENSIONS >= 2 - - BufferLeft.Pack(Vs, BX2s,std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg+nz , kend+nz)); - - BufferRight.Pack(Vs, BX2s, std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg + offset-nz, kend+ offset-nz)); - - #endif - - #if DIMENSIONS == 3 - BufferLeft.Pack(Vs, BX3s,std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+nz+1 , kend+nz+1)); - - BufferRight.Pack(Vs, BX3s, std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg + offset-nz, kend+ offset-nz)); - #endif - } - - // Send to the right - Kokkos::fence(); - - myTimer -= MPI_Wtime(); - tStart = MPI_Wtime(); -#ifdef MPI_PERSISTENT - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3)); - MPI_Waitall(2,recvRequestX3,recvStatus); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; - -#else - int procSend, procRecv; - - #ifdef MPI_NON_BLOCKING - MPI_Status sendStatus[2]; - MPI_Status recvStatus[2]; - MPI_Request sendRequest[2]; - MPI_Request recvRequest[2]; - - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, 100, - mygrid->CartComm, &sendRequest[0])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, 100, - mygrid->CartComm, &recvRequest[0])); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Isend(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, 101, - mygrid->CartComm, &sendRequest[1])); - - MPI_SAFE_CALL(MPI_Irecv(BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, 101, - mygrid->CartComm, &recvRequest[1])); - - // Wait for recv to complete (we don't care about the sends) - MPI_Waitall(2, recvRequest, recvStatus); - - #else - MPI_Status status; - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, 300, - BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, 300, - mygrid->CartComm, &status)); - - // Send to the left - // We receive from procRecv, and we send to procSend - MPI_SAFE_CALL(MPI_Cart_shift(mygrid->CartComm,2,-1,&procRecv,&procSend )); - - MPI_SAFE_CALL(MPI_Sendrecv(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, 301, - BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, 301, - mygrid->CartComm, &status)); - #endif -#endif - myTimer += MPI_Wtime(); - idfx::mpiCallsTimer += MPI_Wtime() - tStart; - // Unpack - BufferLeft=BufferRecvX3[faceLeft]; - BufferRight=BufferRecvX3[faceRight]; - - BufferLeft.ResetPointer(); - BufferRight.ResetPointer(); - - - // We fill the ghost zones - BufferLeft.Unpack(Vc, map,std::make_pair(ibeg, iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vc, map,std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+offset , kend+offset)); - // We fill the ghost zones - - if(haveVs) { - BufferLeft.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg , kend)); - - BufferRight.Unpack(Vs, BX1s, std::make_pair(ibeg, iend+1), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+offset , kend+offset)); - - #if DIMENSIONS >=2 - BufferLeft.Unpack(Vs, BX2s, std::make_pair(ibeg, iend), - std::make_pair(jbeg, jend+1), - std::make_pair(kbeg, kend)); - - BufferRight.Unpack(Vs, BX2s,std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend+1), - std::make_pair(kbeg+offset, kend+offset)); - #endif - - #if DIMENSIONS == 3 - BufferLeft.Unpack(Vs, BX3s, std::make_pair(ibeg, iend), - std::make_pair(jbeg, jend), - std::make_pair(kbeg, kend)); - - BufferRight.Unpack(Vs, BX3s,std::make_pair(ibeg , iend), - std::make_pair(jbeg , jend), - std::make_pair(kbeg+offset+1, kend+offset+1)); - #endif - } - - myTimer -= MPI_Wtime(); -#ifdef MPI_NON_BLOCKING - // Wait for the sends if they have not yet completed - MPI_Waitall(2, sendRequest, sendStatus); -#endif - -#ifdef MPI_PERSISTENT - MPI_Waitall(2, sendRequestX3, sendStatus); -#endif - myTimer += MPI_Wtime(); - bytesSentOrReceived += 4*bufferSizeX3*sizeof(real); - - idfx::popRegion(); -} - - - -void Mpi::CheckConfig() { - idfx::pushRegion("Mpi::CheckConfig"); - // compile time check - #ifdef KOKKOS_ENABLE_CUDA - #if defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT - #error Your MPI library is not CUDA Aware (check Idefix requirements). - #endif - #endif /* MPIX_CUDA_AWARE_SUPPORT */ - - // Run-time check that we can do a reduce on device arrays - IdefixArray1D src("MPIChecksrc",1); - IdefixArray1D::HostMirror srcHost = Kokkos::create_mirror_view(src); - - if(idfx::prank == 0) { - srcHost(0) = 0; - Kokkos::deep_copy(src, srcHost); - } - - if(idfx::psize > 1) { - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); - - // Capture segfaults - struct sigaction newHandler; - struct sigaction oldHandler; - memset(&newHandler, 0, sizeof(newHandler)); - newHandler.sa_flags = SA_SIGINFO; - newHandler.sa_sigaction = Mpi::SigErrorHandler; - sigaction(SIGSEGV, &newHandler, &oldHandler); - try { - // We next circulate the info round-robin accross all the nodes to check that - // MPI can exchange buffers in idefix arrays - - MPI_Status status; - int ierrSend, ierrRecv; - if(idfx::prank == 0) { - ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, idfx::prank+1, 1, MPI_COMM_WORLD); - ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::psize-1, 1, MPI_COMM_WORLD, &status); - } else { - ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::prank-1, 1, MPI_COMM_WORLD, &status); - // Add our own rank to the data - Kokkos::deep_copy(srcHost, src); - srcHost(0) += idfx::prank; - Kokkos::deep_copy(src, srcHost); - ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, (idfx::prank+1)%idfx::psize, 1, - MPI_COMM_WORLD); - } - - if(ierrSend != 0) { - char MPImsg[MPI_MAX_ERROR_STRING]; - int MPImsgLen; - MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); - throw std::runtime_error(std::string(MPImsg, MPImsgLen)); - } - if(ierrRecv != 0) { - char MPImsg[MPI_MAX_ERROR_STRING]; - int MPImsgLen; - MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); - throw std::runtime_error(std::string(MPImsg, MPImsgLen)); - } - } catch(std::exception &e) { - std::stringstream errmsg; - errmsg << "Your MPI library is unable to perform Send/Recv on Idefix arrays."; - errmsg << std::endl; - #ifdef KOKKOS_ENABLE_CUDA - errmsg << "Check that your MPI library is CUDA aware." << std::endl; - #elif defined(KOKKOS_ENABLE_HIP) - errmsg << "Check that your MPI library is RocM aware." << std::endl; - #else - errmsg << "Check your MPI library configuration." << std::endl; - #endif - errmsg << "Error: " << e.what() << std::endl; - IDEFIX_ERROR(errmsg); - } - // Restore old handlers - sigaction(SIGSEGV, &oldHandler, NULL ); - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); - } - - // Check that we have the proper end result - Kokkos::deep_copy(srcHost, src); - int64_t size = static_cast(idfx::psize); - int64_t rank = static_cast(idfx::prank); - int64_t result = rank == 0 ? size*(size-1)/2 : rank*(rank+1)/2; - - if(srcHost(0) != result) { - idfx::cout << "got " << srcHost(0) << " expected " << result << std::endl; - std::stringstream errmsg; - errmsg << "Your MPI library managed to perform MPI exchanges on Idefix Arrays, but the result "; - errmsg << "is incorrect. " << std::endl; - errmsg << "Check your MPI library configuration." << std::endl; - IDEFIX_ERROR(errmsg); - } - idfx::popRegion(); -} - -void Mpi::SigErrorHandler(int nSignum, siginfo_t* si, void* vcontext) { - std::stringstream errmsg; - errmsg << "A segmentation fault was triggered while attempting to test your MPI library."; - errmsg << std::endl; - errmsg << "Your MPI library is unable to perform reductions on Idefix arrays."; - errmsg << std::endl; - #ifdef KOKKOS_ENABLE_CUDA - errmsg << "Check that your MPI library is CUDA aware." << std::endl; - #elif defined(KOKKOS_ENABLE_HIP) - errmsg << "Check that your MPI library is RocM aware." << std::endl; - #else - errmsg << "Check your MPI library configuration." << std::endl; - #endif - IDEFIX_ERROR(errmsg); -} - -// This routine check that all of the processes are synced. -// Returns true if this is the case, false otherwise - -bool Mpi::CheckSync(real timeout) { - // If no parallelism, then we're in sync! - if(idfx::psize == 1) return(true); - - int send = idfx::prank; - int recv = 0; - MPI_Request request; - - MPI_Iallreduce(&send, &recv, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &request); - - double start = MPI_Wtime(); - int flag = 0; - MPI_Status status; - - while((MPI_Wtime()-start < timeout) && !flag) { - MPI_Test(&request, &flag, &status); - // sleep for 10 ms - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - if(!flag) { - // We did not managed to do an allreduce, so this is a failure. - return(false); - } - if(recv != idfx::psize*(idfx::psize-1)/2) { - IDEFIX_ERROR("wrong result for synchronisation"); - } - - return(true); -} diff --git a/src/mpi.hpp b/src/mpi.hpp deleted file mode 100644 index a4250a17c..000000000 --- a/src/mpi.hpp +++ /dev/null @@ -1,269 +0,0 @@ -// *********************************************************************************** -// Idefix MHD astrophysical code -// Copyright(C) Geoffroy R. J. Lesur -// and other code contributors -// Licensed under CeCILL 2.1 License, see COPYING for more information -// *********************************************************************************** - -#ifndef MPI_HPP_ -#define MPI_HPP_ - -#include -#include -#include -#include "idefix.hpp" -#include "grid.hpp" - - -class DataBlock; -class Buffer { - public: - Buffer() = default; - explicit Buffer(size_t size): pointer{0}, array{IdefixArray1D("BufferArray",size)} { }; - - void* data() { - return(array.data()); - } - - int Size() { - return(array.size()); - } - - void ResetPointer() { - this->pointer = 0; - } - - void Pack(IdefixArray3D& in, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - - auto arr = this->array; - idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, - KOKKOS_LAMBDA (int k, int j, int i) { - arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(k,j,i); - }); - - // Update pointer - this->pointer += ninjnk; - } - - void Pack(IdefixArray4D& in, - const int var, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - - auto arr = this->array; - idefix_for("LoadBuffer4D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, - KOKKOS_LAMBDA (int k, int j, int i) { - arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(var, k,j,i); - }); - - // Update pointer - this->pointer += ninjnk; - } - - void Pack(IdefixArray4D& in, - IdefixArray1D& map, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - auto arr = this->array; - - idefix_for("LoadBuffer4D",0,map.size(), - kb.first,kb.second, - jb.first,jb.second, - ib.first,ib.second, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ) = in(map(n), k,j,i); - }); - - // Update pointer - this->pointer += ninjnk*map.size(); - } - - void Unpack(IdefixArray3D& out, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - auto arr = this->array; - - idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, - KOKKOS_LAMBDA (int k, int j, int i) { - out(k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); - }); - - // Update pointer - this->pointer += ninjnk; - } - - void Unpack(IdefixArray4D& out, - const int var, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - - auto arr = this->array; - idefix_for("LoadBuffer3D",kb.first,kb.second,jb.first,jb.second,ib.first,ib.second, - KOKKOS_LAMBDA (int k, int j, int i) { - out(var,k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); - }); - - // Update pointer - this->pointer += ninjnk; - } - - void Unpack(IdefixArray4D& out, - IdefixArray1D& map, - std::pair ib, - std::pair jb, - std::pair kb) { - const int ni = ib.second-ib.first; - const int ninj = (jb.second-jb.first)*ni; - const int ninjnk = (kb.second-kb.first)*ninj; - const int ibeg = ib.first; - const int jbeg = jb.first; - const int kbeg = kb.first; - const int offset = this->pointer; - - auto arr = this->array; - idefix_for("LoadBuffer4D",0,map.size(), - kb.first,kb.second, - jb.first,jb.second, - ib.first,ib.second, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - out(map(n),k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ); - }); - - // Update pointer - this->pointer += ninjnk*map.size(); - } - - - private: - size_t pointer; - IdefixArray1D array; -}; - -class Mpi { - public: - Mpi() = default; - // MPI Exchange functions - void ExchangeAll(); ///< Exchange boundary elements in all directions (todo) - void ExchangeX1(IdefixArray4D inputVc, - IdefixArray4D inputVs = IdefixArray4D()); - ///< Exchange boundary elements in the X1 direction - void ExchangeX2(IdefixArray4D inputVc, - IdefixArray4D inputVs = IdefixArray4D()); - ///< Exchange boundary elements in the X2 direction - void ExchangeX3(IdefixArray4D inputVc, - IdefixArray4D inputVs = IdefixArray4D()); - ///< Exchange boundary elements in the X3 direction - - // Init from datablock - void Init(Grid *grid, std::vector inputMap, - int nghost[3], int nint[3], bool inputHaveVs = false ); - - // Check that MPI will work with the designated target (in particular GPU Direct) - static void CheckConfig(); - - // Check that MPI processes are synced - static bool CheckSync(real); - - - // Destructor - ~Mpi(); - - private: - // Because the MPI class initialise internal pointers, we do not allow copies of this class - // These lines should not be removed as they constitute a safeguard - Mpi(const Mpi&); - Mpi operator=(const Mpi&); - - static int nInstances; // total number of mpi instances in the code - int thisInstance; // unique number of the current instance - int nReferences; // # of references to this instance - bool isInitialized{false}; - - DataBlock *data; // pointer to datablock object - - enum {faceRight, faceLeft}; - - // Buffers for MPI calls - Buffer BufferSendX1[2]; - Buffer BufferSendX2[2]; - Buffer BufferSendX3[2]; - Buffer BufferRecvX1[2]; - Buffer BufferRecvX2[2]; - Buffer BufferRecvX3[2]; - - IdefixArray1D mapVars; - int mapNVars{0}; - - int nint[3]; //< number of internal elements of the arrays we treat - int nghost[3]; //< number of ghost zone of the arrays we treat - int ntot[3]; //< total number of cells of the arrays we treat - int beg[3]; //< begining index of the active zone - int end[3]; //< end index of the active zone - - int bufferSizeX1; - int bufferSizeX2; - int bufferSizeX3; - - bool haveVs{false}; - - // Requests for MPI persistent communications - MPI_Request sendRequestX1[2]; - MPI_Request sendRequestX2[2]; - MPI_Request sendRequestX3[2]; - MPI_Request recvRequestX1[2]; - MPI_Request recvRequestX2[2]; - MPI_Request recvRequestX3[2]; - - Grid *mygrid; - - // MPI throughput timer specific to this object - double myTimer{0}; - int64_t bytesSentOrReceived{0}; - - // Error handler used by CheckConfig - static void SigErrorHandler(int, siginfo_t* , void* ); -}; - -#endif // MPI_HPP_ diff --git a/src/mpi/CMakeLists.txt b/src/mpi/CMakeLists.txt new file mode 100644 index 000000000..6042da654 --- /dev/null +++ b/src/mpi/CMakeLists.txt @@ -0,0 +1,7 @@ +target_sources(idefix + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/buffer.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/exchanger.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/exchanger.cpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/mpi.hpp + PUBLIC ${CMAKE_CURRENT_LIST_DIR}/mpi.cpp +) diff --git a/src/mpi/buffer.hpp b/src/mpi/buffer.hpp new file mode 100644 index 000000000..ff4dff146 --- /dev/null +++ b/src/mpi/buffer.hpp @@ -0,0 +1,195 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef MPI_BUFFER_HPP_ +#define MPI_BUFFER_HPP_ + +#include "idefix.hpp" +#include "arrays.hpp" + +using BoundingBox = std::array,3>; + + +class Buffer { + public: + Buffer() = default; + explicit Buffer(size_t size): pointer{0}, array{IdefixArray1D("BufferArray",size)} {}; + + // Compute the size of a bounding box + static size_t ComputeBoxSize(BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + return(ninjnk); + } + + void* data() { + return(array.data()); + } + + int Size() { + return(array.size()); + } + + void ResetPointer() { + this->pointer = 0; + } + + void Pack(IdefixArray3D& in, BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer3D",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(k,j,i); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Pack(IdefixArray4D& in, + const int var, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("LoadBuffer4D_var",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ) = in(var, k,j,i); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Pack(IdefixArray4D& in, + IdefixArray1D& map, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + auto arr = this->array; + + idefix_for("LoadBuffer4D_map",0,map.size(), + kbeg,kend, + jbeg,jend, + ibeg,iend, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ) = in(map(n), k,j,i); + }); + + // Update pointer + this->pointer += ninjnk*map.size(); + } + + void Unpack(IdefixArray3D& out, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + auto arr = this->array; + + idefix_for("UnLoadBuffer3D",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + out(k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Unpack(IdefixArray4D& out, + const int var, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("UnLoadBuffer4D_var",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + out(var,k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset ); + }); + + // Update pointer + this->pointer += ninjnk; + } + + void Unpack(IdefixArray4D& out, + IdefixArray1D& map, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("UnLoadBuffer4D_map",0,map.size(), + kbeg,kend, + jbeg,jend, + ibeg,iend, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + out(map(n),k,j,i) = arr(i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset ); + }); + + // Update pointer + this->pointer += ninjnk*map.size(); + } + + + private: + size_t pointer; + IdefixArray1D array; +}; + +#endif // MPI_BUFFER_HPP_ diff --git a/src/mpi/exchanger.cpp b/src/mpi/exchanger.cpp new file mode 100644 index 000000000..033ce4041 --- /dev/null +++ b/src/mpi/exchanger.cpp @@ -0,0 +1,308 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#include +#include "exchanger.hpp" +#include "idefix.hpp" +#include "buffer.hpp" +#include "grid.hpp" +#include "arrays.hpp" + +//#define MPI_NON_BLOCKING +#define MPI_PERSISTENT + +int Exchanger::nInstances = 0; + +void Exchanger::Init( + Grid *grid, + int direction, + std::vector inputMap, + std::array nghost, + std::array nint, + bool inputHaveVs, + std::array overwriteBXn) { + idfx::pushRegion("Exchanger::Init"); + this->grid = grid; + this->direction = direction; + // Allocate mapVars on target and copy it from the input argument list + this->mapVars = idfx::ConvertVectorToIdefixArray(inputMap); + this->mapNVars = inputMap.size(); + this->haveVs = inputHaveVs; + + // increase the number of instances + this->thisInstance = nInstances; + + // Compute indices of arrays we will be working with + for(int dir = 0 ; dir < 3 ; dir++) { + this->nghost[dir] = nghost[dir]; + this->nint[dir] = nint[dir]; + this->ntot[dir] = nint[dir]+2*nghost[dir]; + this->beg[dir] = nghost[dir]; + this->end[dir] = nghost[dir]+nint[dir]; + } + + ///////////////////////////////////////////////////////////////////////////// + // Init exchange datasets + // Buffer size direction are for sends (i.e. buffer left is for a send from the left side) + + // Make left buffer + // Init zone to the full domain + for(int dir = 0 ; dir < 3 ; dir++) { + if(dir < direction) { + boxRecv[faceLeft][dir][0] = 0; + boxRecv[faceLeft][dir][1] = ntot[dir]; + } else if(dir > direction) { + boxRecv[faceLeft][dir][0] = beg[dir]; + boxRecv[faceLeft][dir][1] = end[dir]; + } else { + // dir == direction + boxRecv[faceLeft][dir][0] = 0; + boxRecv[faceLeft][dir][1] = nghost[dir]; + } + } + // Copy all the boxes from boxRecvLeft + boxRecv[faceRight] = boxRecv[faceLeft]; + boxSend = boxRecv; + + // Adjust the indices for send and receive in the direction of interest + boxRecv[faceRight][direction][0] = end[direction]; + boxRecv[faceRight][direction][1] = ntot[direction]; + + boxSend[faceLeft][direction][0] = beg[direction]; + boxSend[faceLeft][direction][1] = beg[direction]+nghost[direction]; + + boxSend[faceRight][direction][0] = end[direction] - nghost[direction]; + boxSend[faceRight][direction][1] = end[direction]; + + // Face-centered boxes + + // Add one element in the field direction + for(int component = 0 ; component < 3 ; component++) { + // Init as centered boxes + boxSendVs[component] = boxSend; + boxRecvVs[component] = boxRecv; + const int normalDir = component; + if(component != direction) { + for(int face = 0 ; face <2 ; face++) { + boxSendVs[component][face][normalDir][1] += 1; + boxRecvVs[component][face][normalDir][1] += 1; + } + } else { + // component == direction + if(!overwriteBXn[faceLeft]) boxSendVs[component][faceLeft][normalDir][0] += 1; + boxSendVs[component][faceLeft][normalDir][1] += 1; + + if(!overwriteBXn[faceRight]) boxRecvVs[component][faceRight][normalDir][0] += 1; + boxRecvVs[component][faceRight][normalDir][1] += 1; + } + } + + // Compute buffer sizes + for(int face=0 ; face < 2 ; face++) { + bufferSizeSend[face] = mapNVars * Buffer::ComputeBoxSize(boxSend[face]); + bufferSizeRecv[face] = mapNVars * Buffer::ComputeBoxSize(boxRecv[face]); + if(haveVs) { + for(int component = 0 ; component CartComm,direction,1,&procRecv[faceLeft],&procSend[faceRight]); + MPI_Cart_shift(grid->CartComm,direction,-1,&procRecv[faceRight],&procSend[faceLeft]); + + #ifdef MPI_PERSISTENT + + // X1-dir exchanges + // We receive from procRecv, and we send to procSend + + MPI_Send_init(BufferSend[faceRight].data(), bufferSizeSend[faceRight], realMPI, + procSend[faceRight], thisInstance*2, + grid->CartComm, &sendRequest[faceRight]); + + MPI_Recv_init(BufferRecv[faceLeft].data(), bufferSizeRecv[faceLeft], realMPI, + procRecv[faceLeft],thisInstance*2, + grid->CartComm, &recvRequest[faceLeft]); + + // Send to the left + // We receive from procRecv, and we send to procSend + + MPI_Send_init(BufferSend[faceLeft].data(), bufferSizeSend[faceLeft], realMPI, + procSend[faceLeft],thisInstance*2+1, + grid->CartComm, &sendRequest[faceLeft]); + + MPI_Recv_init(BufferRecv[faceRight].data(), bufferSizeRecv[faceRight], realMPI, + procRecv[faceRight], thisInstance*2+1, + grid->CartComm, &recvRequest[faceRight]); + + #endif // MPI_PERSISTENT + + // say this instance is initialized. + isInitialized = true; + nInstances++; + + idfx::popRegion(); +} + +Exchanger::~Exchanger() { + idfx::pushRegion("Exchanger::~Exchanger"); + if(isInitialized) { + // Properly clean up the mess + #ifdef MPI_PERSISTENT + for(int i=0 ; i< 2; i++) { + MPI_Request_free( &sendRequest[i]); + MPI_Request_free( &recvRequest[i]); + } + #endif + isInitialized = false; + } + idfx::popRegion(); +} + +void Exchanger::Exchange(IdefixArray4D Vc, IdefixArray4D Vs) { + idfx::pushRegion("Mpi::ExchangeX1"); + // Load the buffers with data + Buffer BufferLeft = BufferSend[faceLeft]; + Buffer BufferRight = BufferSend[faceRight]; + IdefixArray1D map = this->mapVars; + + bool recvRight = (procRecv[faceRight] != MPI_PROC_NULL); + bool recvLeft = (procRecv[faceLeft] != MPI_PROC_NULL); + + // If MPI Persistent, start receiving even before the buffers are filled + myTimer -= MPI_Wtime(); + double tStart = MPI_Wtime(); +#ifdef MPI_PERSISTENT + MPI_Status sendStatus[2]; + MPI_Status recvStatus[2]; + + MPI_Startall(2, recvRequest); + idfx::mpiCallsTimer += MPI_Wtime() - tStart; +#endif + myTimer += MPI_Wtime(); + + BufferLeft.ResetPointer(); + BufferRight.ResetPointer(); + + BufferLeft.Pack(Vc, map, boxSend[faceLeft]); + BufferRight.Pack(Vc, map, boxSend[faceRight]); + // Load face-centered field in the buffer + if(haveVs) { + for(int component = 0 ; component < DIMENSIONS ; component++) { + BufferLeft.Pack(Vs, component, boxSendVs[component][faceLeft]); + BufferRight.Pack(Vs, component, boxSendVs[component][faceRight]); + } + } + + // Wait for completion before sending out everything + Kokkos::fence(); + myTimer -= MPI_Wtime(); + tStart = MPI_Wtime(); +#ifdef MPI_PERSISTENT + MPI_Startall(2, sendRequest); + // Wait for buffers to be received + MPI_Waitall(2,recvRequest,recvStatus); + +#else + + #ifdef MPI_NON_BLOCKING + MPI_Status sendStatus[2]; + MPI_Status recvStatus[2]; + MPI_Request sendRequest[2]; + MPI_Request recvRequest[2]; + + // We receive from procRecv, and we send to procSend + + MPI_Isend(BufferSend[faceRight].data(), bufferSizeSend[faceRight], realMPI, + procSend[faceRight], 100, mygrid->CartComm, &sendRequest[0]); + + MPI_Irecv(BufferRecv[faceLeft].data(), bufferSizeRecv[faceLeft], realMPI, + procRecv[faceLeft], 100, mygrid->CartComm, &recvRequest[0]); + // Send to the left + // We receive from procRecv, and we send to procSend + + MPI_Isend(BufferSend[faceLeft].data(), bufferSizeSend[faceLeft], realMPI, + procSend[faceLeft], 101, mygrid->CartComm, &sendRequest[1]); + + MPI_Irecv(BufferRecv[faceRight].data(), bufferSizeRecv[faceRight], realMPI, + procRecv[faceRight], 101, mygrid->CartComm, &recvRequest[1]); + + // Wait for recv to complete (we don't care about the sends) + MPI_Waitall(2, recvRequest, recvStatus); + + #else + MPI_Status status; + // Send to the right + // We receive from procRecv, and we send to procSend + + MPI_Sendrecv(BufferSend[faceRight].data(), bufferSizeSend[faceRight], realMPI, + procSend[faceRight], 100, + BufferRecv[faceLeft].data(), bufferSizeRecv[faceLeft], realMPI, + procRecv[faceLeft], 100, + grid->CartComm, &status); + + // Send to the left + // We receive from procRecv, and we send to procSend + + MPI_Sendrecv(BufferSend[faceLeft].data(), bufferSizeSend[faceLeft], realMPI, + procSend[faceLeft], 101, + BufferRecv[faceRight].data(), bufferSizeRecv[faceRight], realMPI, + procRecv[faceRight], 101, + grid->CartComm, &status); + #endif +#endif +myTimer += MPI_Wtime(); +idfx::mpiCallsTimer += MPI_Wtime() - tStart; +// Unpack +BufferLeft=BufferRecv[faceLeft]; +BufferRight=BufferRecv[faceRight]; + +BufferLeft.ResetPointer(); +BufferRight.ResetPointer(); + +if(recvLeft) { + BufferLeft.Unpack(Vc, map, boxRecv[faceLeft]); + if(haveVs) { + for(int component = 0 ; component < DIMENSIONS ; component++) { + BufferLeft.Unpack(Vs, component, boxRecvVs[component][faceLeft]); + } + } +} +if(recvRight) { + BufferRight.Unpack(Vc, map, boxRecv[faceRight]); + if(haveVs) { + for(int component = 0 ; component < DIMENSIONS ; component++) { + BufferRight.Unpack(Vs, component, boxRecvVs[component][faceRight]); + } + } +} +myTimer -= MPI_Wtime(); +#ifdef MPI_NON_BLOCKING + // Wait for the sends if they have not yet completed + MPI_Waitall(2, sendRequest, sendStatus); +#endif + +#ifdef MPI_PERSISTENT + MPI_Waitall(2, sendRequest, sendStatus); +#endif + myTimer += MPI_Wtime(); + bytesSentOrReceived += (bufferSizeRecv[faceLeft] + +bufferSizeSend[faceLeft] + +bufferSizeRecv[faceRight] + +bufferSizeSend[faceRight])*sizeof(real); + + idfx::popRegion(); +} diff --git a/src/mpi/exchanger.hpp b/src/mpi/exchanger.hpp new file mode 100644 index 000000000..2e1affcb0 --- /dev/null +++ b/src/mpi/exchanger.hpp @@ -0,0 +1,82 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef MPI_EXCHANGER_HPP_ +#define MPI_EXCHANGER_HPP_ + +#include + +#include +#include +#include +#include "idefix.hpp" +#include "grid.hpp" +#include "buffer.hpp" +#include "arrays.hpp" + +class Grid; + +class Exchanger { + public: + Exchanger() = default; + void Init( Grid* grid, + int direction, + std::vector inputMap, + std::array nghost, + std::array nint, + bool inputHaveVs = false, + std::array overwriteBXn = {true, true}); + + void Exchange(IdefixArray4D Vc, IdefixArray4D Vs); + ~Exchanger(); + + static int nInstances; // total number of mpi instances in the code + int thisInstance; // unique number of the current instance + bool isInitialized{false}; + + // MPI throughput timer specific to this object + double myTimer{0}; + int64_t bytesSentOrReceived{0}; + + // Buffer sizes for throughput calculations + std::array bufferSizeSend; + std::array bufferSizeRecv; + + private: + enum {faceRight, faceLeft}; + + std::array boxSend, boxRecv; // bounding boxes for each face + std::array,3> boxSendVs, boxRecvVs; // 3= 3 field components + + // Buffers for MPI calls + Buffer BufferSend[2]; + Buffer BufferRecv[2]; + + int procSend[2]; // MPI process to send to in X1 direction + int procRecv[2]; // MPI process to receive from in X1 direction + + int direction; + IdefixArray1D mapVars; + int mapNVars{0}; + + int nint[3]; //< number of internal elements of the arrays we treat + int nghost[3]; //< number of ghost zone of the arrays we treat + int ntot[3]; //< total number of cells of the arrays we treat + int beg[3]; //< begining index of the active zone + int end[3]; //< end index of the active zone + + bool haveVs{false}; + + // Requests for MPI persistent communications + MPI_Request sendRequest[2]; + MPI_Request recvRequest[2]; + + Grid *grid; +}; + + +#endif // MPI_EXCHANGER_HPP_ diff --git a/src/mpi/mpi.cpp b/src/mpi/mpi.cpp new file mode 100644 index 000000000..36b4eb9a3 --- /dev/null +++ b/src/mpi/mpi.cpp @@ -0,0 +1,265 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + + +#include "mpi.hpp" +#include +#include +#include // NOLINT [build/c++11] +#include // NOLINT [build/c++11] +#include +#include +#include "idefix.hpp" +#include "dataBlock.hpp" + + +#if defined(OPEN_MPI) && OPEN_MPI +#include "mpi-ext.h" // Needed for CUDA-aware check */ +#endif + + + + +// init the number of instances +int Mpi::nInstances = 0; + +// MPI Routines exchange +void Mpi::ExchangeAll() { + IDEFIX_ERROR("Not Implemented"); +} + +/// +/// Initialise an instance of the MPI class. +/// @param grid: pointer to the grid object (needed to get the MPI neighbours) +/// @param inputMap: 1st indices of inputVc which are to be exchanged (i.e, the list of variables) +/// @param nghost: size of the ghost region in each direction +/// @param nint: size of the internal region in each direction +/// @param inputHaveVs: whether the instance should also treat face-centered variable +/// (optional, default false) +/// + +void Mpi::Init(Grid *grid, std::vector inputMap, + std::array nghost, std::array nint, + std::array lbound, + std::array rbound, + bool inputHaveVs) { + idfx::pushRegion("Mpi::Init"); + + // increase the number of instances + nInstances++; + thisInstance=nInstances; + + for(int dir=0; dir<3; dir++) { + std::array overWriteBXn = {true, true}; + if(lbound[dir] == BoundaryType::shearingbox) { + overWriteBXn[faceLeft] = false; + } + if(rbound[dir] == BoundaryType::shearingbox) { + overWriteBXn[faceRight] = false; + } + + exchanger[dir].Init(grid, dir, inputMap, + nghost, nint, + inputHaveVs, overWriteBXn); + } + + isInitialized = true; + idfx::popRegion(); +} + +// Destructor (clean up persistent communication channels) +Mpi::~Mpi() { + idfx::pushRegion("Mpi::~Mpi"); + if(isInitialized) { + if(thisInstance==1) { + int bytesSentOrReceived = 0; + double myTimer = 0; + for(int dir=0; dir<3; dir++) { + bytesSentOrReceived += exchanger[dir].bytesSentOrReceived; + myTimer += exchanger[dir].myTimer; + } + idfx::cout << "Mpi(" << thisInstance << "): measured throughput is " + << bytesSentOrReceived/myTimer/1024.0/1024.0 << " MB/s" << std::endl; + idfx::cout << "Mpi(" << thisInstance << "): message sizes were " << std::endl; + idfx::cout << " X1: " + << exchanger[IDIR].bufferSizeSend[0]*sizeof(real)/1024.0/1024.0 + << " MB" << std::endl; + idfx::cout << " X2: " + << exchanger[JDIR].bufferSizeSend[0]*sizeof(real)/1024.0/1024.0 + << " MB" << std::endl; + idfx::cout << " X3: " + << exchanger[KDIR].bufferSizeSend[0]*sizeof(real)/1024.0/1024.0 + << " MB" << std::endl; + } + isInitialized = false; + } + idfx::popRegion(); +} + +void Mpi::ExchangeX1(IdefixArray4D Vc, IdefixArray4D Vs) { + idfx::pushRegion("Mpi::ExchangeX1"); + + exchanger[IDIR].Exchange(Vc, Vs); + idfx::popRegion(); +} + +void Mpi::ExchangeX2(IdefixArray4D Vc, IdefixArray4D Vs) { + idfx::pushRegion("Mpi::ExchangeX2"); + exchanger[JDIR].Exchange(Vc, Vs); + idfx::popRegion(); +} + +void Mpi::ExchangeX3(IdefixArray4D Vc, IdefixArray4D Vs) { + idfx::pushRegion("Mpi::ExchangeX3"); + exchanger[KDIR].Exchange(Vc, Vs); + idfx::popRegion(); +} + + +void Mpi::CheckConfig() { + idfx::pushRegion("Mpi::CheckConfig"); + // compile time check + #ifdef KOKKOS_ENABLE_CUDA + #if defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT + #error Your MPI library is not CUDA Aware (check Idefix requirements). + #endif + #endif /* MPIX_CUDA_AWARE_SUPPORT */ + + // Run-time check that we can do a reduce on device arrays + IdefixArray1D src("MPIChecksrc",1); + IdefixArray1D::HostMirror srcHost = Kokkos::create_mirror_view(src); + + if(idfx::prank == 0) { + srcHost(0) = 0; + Kokkos::deep_copy(src, srcHost); + } + + if(idfx::psize > 1) { + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + + // Capture segfaults + struct sigaction newHandler; + struct sigaction oldHandler; + memset(&newHandler, 0, sizeof(newHandler)); + newHandler.sa_flags = SA_SIGINFO; + newHandler.sa_sigaction = Mpi::SigErrorHandler; + sigaction(SIGSEGV, &newHandler, &oldHandler); + try { + // We next circulate the info round-robin accross all the nodes to check that + // MPI can exchange buffers in idefix arrays + + MPI_Status status; + int ierrSend, ierrRecv; + if(idfx::prank == 0) { + ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, idfx::prank+1, 1, MPI_COMM_WORLD); + ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::psize-1, 1, MPI_COMM_WORLD, &status); + } else { + ierrRecv = MPI_Recv(src.data(), 1, MPI_INT64_T, idfx::prank-1, 1, MPI_COMM_WORLD, &status); + // Add our own rank to the data + Kokkos::deep_copy(srcHost, src); + srcHost(0) += idfx::prank; + Kokkos::deep_copy(src, srcHost); + ierrSend = MPI_Send(src.data(), 1, MPI_INT64_T, (idfx::prank+1)%idfx::psize, 1, + MPI_COMM_WORLD); + } + + if(ierrSend != 0) { + char MPImsg[MPI_MAX_ERROR_STRING]; + int MPImsgLen; + MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); + throw std::runtime_error(std::string(MPImsg, MPImsgLen)); + } + if(ierrRecv != 0) { + char MPImsg[MPI_MAX_ERROR_STRING]; + int MPImsgLen; + MPI_Error_string(ierrSend, MPImsg, &MPImsgLen); + throw std::runtime_error(std::string(MPImsg, MPImsgLen)); + } + } catch(std::exception &e) { + std::stringstream errmsg; + errmsg << "Your MPI library is unable to perform Send/Recv on Idefix arrays."; + errmsg << std::endl; + #ifdef KOKKOS_ENABLE_CUDA + errmsg << "Check that your MPI library is CUDA aware." << std::endl; + #elif defined(KOKKOS_ENABLE_HIP) + errmsg << "Check that your MPI library is RocM aware." << std::endl; + #else + errmsg << "Check your MPI library configuration." << std::endl; + #endif + errmsg << "Error: " << e.what() << std::endl; + IDEFIX_ERROR(errmsg); + } + // Restore old handlers + sigaction(SIGSEGV, &oldHandler, NULL ); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); + } + + // Check that we have the proper end result + Kokkos::deep_copy(srcHost, src); + int64_t size = static_cast(idfx::psize); + int64_t rank = static_cast(idfx::prank); + int64_t result = rank == 0 ? size*(size-1)/2 : rank*(rank+1)/2; + + if(srcHost(0) != result) { + idfx::cout << "got " << srcHost(0) << " expected " << result << std::endl; + std::stringstream errmsg; + errmsg << "Your MPI library managed to perform MPI exchanges on Idefix Arrays, but the result "; + errmsg << "is incorrect. " << std::endl; + errmsg << "Check your MPI library configuration." << std::endl; + IDEFIX_ERROR(errmsg); + } + idfx::popRegion(); +} + +void Mpi::SigErrorHandler(int nSignum, siginfo_t* si, void* vcontext) { + std::stringstream errmsg; + errmsg << "A segmentation fault was triggered while attempting to test your MPI library."; + errmsg << std::endl; + errmsg << "Your MPI library is unable to perform reductions on Idefix arrays."; + errmsg << std::endl; + #ifdef KOKKOS_ENABLE_CUDA + errmsg << "Check that your MPI library is CUDA aware." << std::endl; + #elif defined(KOKKOS_ENABLE_HIP) + errmsg << "Check that your MPI library is RocM aware." << std::endl; + #else + errmsg << "Check your MPI library configuration." << std::endl; + #endif + IDEFIX_ERROR(errmsg); +} + +// This routine check that all of the processes are synced. +// Returns true if this is the case, false otherwise + +bool Mpi::CheckSync(real timeout) { + // If no parallelism, then we're in sync! + if(idfx::psize == 1) return(true); + + int send = idfx::prank; + int recv = 0; + MPI_Request request; + + MPI_Iallreduce(&send, &recv, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &request); + + double start = MPI_Wtime(); + int flag = 0; + MPI_Status status; + + while((MPI_Wtime()-start < timeout) && !flag) { + MPI_Test(&request, &flag, &status); + // sleep for 10 ms + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + if(!flag) { + // We did not managed to do an allreduce, so this is a failure. + return(false); + } + if(recv != idfx::psize*(idfx::psize-1)/2) { + IDEFIX_ERROR("wrong result for synchronisation"); + } + + return(true); +} diff --git a/src/mpi/mpi.hpp b/src/mpi/mpi.hpp new file mode 100644 index 000000000..b8f929326 --- /dev/null +++ b/src/mpi/mpi.hpp @@ -0,0 +1,74 @@ +// *********************************************************************************** +// Idefix MHD astrophysical code +// Copyright(C) Geoffroy R. J. Lesur +// and other code contributors +// Licensed under CeCILL 2.1 License, see COPYING for more information +// *********************************************************************************** + +#ifndef MPI_MPI_HPP_ +#define MPI_MPI_HPP_ + +#include + +#include +#include +#include +#include "idefix.hpp" +#include "grid.hpp" +#include "buffer.hpp" +#include "exchanger.hpp" + + +class DataBlock; + + +class Mpi { + public: + Mpi() = default; + // MPI Exchange functions + void ExchangeAll(); ///< Exchange boundary elements in all directions (todo) + void ExchangeX1(IdefixArray4D inputVc, + IdefixArray4D inputVs = IdefixArray4D()); + ///< Exchange boundary elements in the X1 direction + void ExchangeX2(IdefixArray4D inputVc, + IdefixArray4D inputVs = IdefixArray4D()); + ///< Exchange boundary elements in the X2 direction + void ExchangeX3(IdefixArray4D inputVc, + IdefixArray4D inputVs = IdefixArray4D()); + ///< Exchange boundary elements in the X3 direction + + // Init from datablock + void Init(Grid *grid, std::vector inputMap, + std::array nghost, std::array nint, + std::array lbound, + std::array rbound, + bool inputHaveVs = false ); + + // Check that MPI will work with the designated target (in particular GPU Direct) + static void CheckConfig(); + + // Check that MPI processes are synced + static bool CheckSync(real); + + + // Destructor + ~Mpi(); + + private: + enum {faceRight, faceLeft}; + // Because the MPI class initialise internal pointers, we do not allow copies of this class + // These lines should not be removed as they constitute a safeguard + Mpi(const Mpi&); + Mpi operator=(const Mpi&); + + static int nInstances; // total number of mpi instances in the code + int thisInstance; // unique number of the current instance + int nReferences; // # of references to this instance + bool isInitialized{false}; + + std::array exchanger; ///< exchangers in each direction + // Error handler used by CheckConfig + static void SigErrorHandler(int, siginfo_t* , void* ); +}; + +#endif // MPI_MPI_HPP_ diff --git a/src/output/dump.cpp b/src/output/dump.cpp index 955e06e72..96a40b548 100644 --- a/src/output/dump.cpp +++ b/src/output/dump.cpp @@ -71,7 +71,7 @@ void Dump::CreateMPIDataType(GridBox gb, bool read) { int size[3]; int subsize[3]; - // the grid is required to now the current MPÏ domain decomposition + // the grid is required to know the current MPÏ domain decomposition Grid *grid = data->mygrid; // Dimensions for cell-centered fields diff --git a/src/output/dump.hpp b/src/output/dump.hpp index d823ddcaf..87cb7a3ba 100644 --- a/src/output/dump.hpp +++ b/src/output/dump.hpp @@ -88,13 +88,13 @@ class DumpField { h4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); return(arr3D); } else if(arrayType==Device3D) { - IdefixHostArray3D arr3D = Kokkos::create_mirror(d3Darray); + IdefixHostArray3D arr3D = Kokkos::create_mirror(Kokkos::HostSpace(), d3Darray); Kokkos::deep_copy(arr3D,d3Darray); return(arr3D); } else if(arrayType==Device4D) { IdefixArray3D arrDev3D = Kokkos::subview( d4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); - IdefixHostArray3D arr3D = Kokkos::create_mirror(arrDev3D); + IdefixHostArray3D arr3D = Kokkos::create_mirror(Kokkos::HostSpace(), arrDev3D); Kokkos::deep_copy(arr3D,arrDev3D); return(arr3D); } else { diff --git a/src/output/scalarField.hpp b/src/output/scalarField.hpp index 9af7b1986..6cbb9dd40 100644 --- a/src/output/scalarField.hpp +++ b/src/output/scalarField.hpp @@ -33,7 +33,7 @@ class ScalarField { h4Darray, var, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); return(arr3D); } else if(type==Device3D) { - IdefixHostArray3D arr3D = Kokkos::create_mirror(d3Darray); + IdefixHostArray3D arr3D = Kokkos::create_mirror(Kokkos::HostSpace(), d3Darray); Kokkos::deep_copy(arr3D,d3Darray); return(arr3D); } else if(type==Device4D) { diff --git a/src/output/xdmf.cpp b/src/output/xdmf.cpp index 23ddaa03a..4e7ff6843 100644 --- a/src/output/xdmf.cpp +++ b/src/output/xdmf.cpp @@ -286,18 +286,18 @@ int Xdmf::Write() { idfx::pushRegion("Xdmf::Write"); fs::path filename; fs::path filename_xmf; - hid_t err; + [[maybe_unused]] hid_t err; idfx::cout << "Xdmf: Write file n " << xdmfFileNumber << "..." << std::flush; timer.reset(); // Create a copy of the dataBlock on Host, and sync it. #if DIMENSIONS == 1 - int tot_dim = 1; + [[maybe_unused]] int tot_dim = 1; #elif DIMENSIONS == 2 int tot_dim = 2; #elif DIMENSIONS == 3 - int tot_dim = 3; + [[maybe_unused]] int tot_dim = 3; #endif std::stringstream ssfileName, ssfileNameXmf, ssxdmfFileNum; @@ -344,7 +344,8 @@ int Xdmf::Write() { #endif // Layout of the field data in memory - hsize_t field_data_size[3], field_data_start[3], field_data_subsize[3], stride[3]; + [[maybe_unused]] hsize_t field_data_size[3], field_data_start[3]; + [[maybe_unused]] hsize_t field_data_subsize[3], stride[3]; #ifdef WITH_MPI for(int dir = 0; dir < 3 ; dir++) { field_data_size[dir] = static_cast(this->mpi_data_size[dir]); @@ -454,11 +455,10 @@ void Xdmf::WriteHeader( hid_t tspace, tattr; hid_t unit_info, unit_attr; hid_t group; - hid_t file_access = 0; #ifdef WITH_MPI hid_t plist_id_mpiio = 0; /* for collective MPI I/O */ #endif - hid_t err; + [[maybe_unused]] hid_t err; hsize_t dimstr; @@ -814,7 +814,7 @@ void Xdmf::WriteScalar( dataset_name = var_name.c_str(); std::string dataset_label = dataset_name.c_str(); std::transform(dataset_label.begin(), dataset_label.end(), dataset_label.begin(), ::tolower); - hid_t err, dataset; + [[maybe_unused]] hid_t err, dataset; // We define the dataset that contain the fields. diff --git a/src/pydefix.cpp b/src/pydefix.cpp index f0d8bc784..1b8168088 100644 --- a/src/pydefix.cpp +++ b/src/pydefix.cpp @@ -68,7 +68,7 @@ py::array_t GatherIdefixArray(IdefixHostArray3D // np_int: size that should be copied into global // beg: offset in the incoming array where copy should begin // gbeg: offset in the global array where copy should be begin - std::array np_int,np_tot, beg, gbeg; + [[maybe_unused]] std::array np_int,np_tot, beg, gbeg; IdefixHostArray3D buf; if(rank==0) { @@ -127,7 +127,7 @@ py::array_t GatherIdefixArray(IdefixHostArray3D }// End loop on target rank for root process } else { // MPI prank >0 std::array np_int = dataHost.np_int; - std::array np_tot = dataHost.np_tot; + [[maybe_unused]] std::array np_tot = dataHost.np_tot; std::array gbeg = dataHost.gbeg; std::array beg = dataHost.beg; @@ -228,6 +228,11 @@ PYBIND11_EMBEDDED_MODULE(pydefix, m) { m.attr("BX1s") = BX1s; , m.attr("BX2s") = BX2s; , m.attr("BX3s") = BX3s; ) + #ifdef EVOLVE_VECTOR_POTENTIAL + m.attr("AX1e") = AX1e; + m.attr("AX2e") = AX2e; + m.attr("AX3e") = AX3e; + #endif #endif m.attr("IDIR") = IDIR; m.attr("JDIR") = JDIR; @@ -273,6 +278,9 @@ Pydefix::Pydefix(Input &input) { idfx::cout << "Pydefix: start Python interpreter." << std::endl; py::initialize_interpreter(); + py::exec("import sys; print(f'Pydefix: Python Version: {sys.version}')"); + py::exec("print(f'Pydefix: Executable Path: {sys.executable}')"); + py::exec("print(f'Pydefix: Sys Path: {sys.path}')"); } this->scriptFilename = input.Get("Python","script",0); if(scriptFilename.substr(scriptFilename.length() - 3, 3).compare(".py")==0) { diff --git a/src/rkl/rkl.hpp b/src/rkl/rkl.hpp index c8dd8cb96..6f22de6f5 100644 --- a/src/rkl/rkl.hpp +++ b/src/rkl/rkl.hpp @@ -198,7 +198,8 @@ RKLegendre::RKLegendre(Input &input, Fluid* hydroin) { nvarRKL = varListHost.size(); #ifdef WITH_MPI - mpi.Init(data->mygrid, varListHost, data->nghost.data(), data->np_int.data(), haveVs); + mpi.Init(data->mygrid, varListHost, data->nghost, data->np_int, + data->lbound, data->rbound, haveVs); #endif diff --git a/src/setup.cpp b/src/setup.cpp index 8c64f4447..b82f118f8 100644 --- a/src/setup.cpp +++ b/src/setup.cpp @@ -12,7 +12,9 @@ // own implementation of the constructor, initflow and destructor __attribute__((weak)) Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - IDEFIX_WARNING("Caution, this is the default Setup constructor and it does nothing!"); + #ifndef WITH_PYTHON + IDEFIX_WARNING("Caution, this is the default Setup constructor and it does nothing!"); + #endif } __attribute__((weak)) void Setup::InitFlow(DataBlock &data) { diff --git a/src/utils/column.cpp b/src/utils/column.cpp index 34d55f56b..4657abc55 100644 --- a/src/utils/column.cpp +++ b/src/utils/column.cpp @@ -56,7 +56,7 @@ Column::Column(int dir, int sign, DataBlock *data) std::vector mapVars; mapVars.push_back(ntarget); - this->mpi.Init(data->mygrid, mapVars, data->nghost.data(), data->np_int.data()); + this->mpi.Init(data->mygrid, mapVars, data->nghost, data->np_int, data->lbound, data->rbound); this->nproc = data->mygrid->nproc; #endif idfx::popRegion(); diff --git a/src/utils/lookupTable.hpp b/src/utils/lookupTable.hpp index c54fdb602..72d13a838 100644 --- a/src/utils/lookupTable.hpp +++ b/src/utils/lookupTable.hpp @@ -183,10 +183,10 @@ LookupTable::LookupTable(std::vector filenames, this->offsetDev = IdefixArray1D ("Table_offset", kDim); this->dataDev = IdefixArray1D ("Table_data", dataVector.size()); - this->xinHost = Kokkos::create_mirror_view(this->xinDev); - this->dimensionsHost = Kokkos::create_mirror_view(this->dimensionsDev); - this->offsetHost = Kokkos::create_mirror_view(this->offsetDev); - this->dataHost = Kokkos::create_mirror_view(this->dataDev); + this->xinHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->xinDev); + this->dimensionsHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dimensionsDev); + this->offsetHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->offsetDev); + this->dataHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dataDev); // Copy data in memory for(uint64_t i = 0 ; i < dataVector.size() ; i++) { @@ -346,10 +346,10 @@ LookupTable::LookupTable(std::string filename, char delimiter, bool errOOB this->offsetDev = IdefixArray1D ("Table_offset", kDim); this->dataDev = IdefixArray1D ("Table_data", size[0]*size[1]); - this->xinHost = Kokkos::create_mirror_view(this->xinDev); - this->dimensionsHost = Kokkos::create_mirror_view(this->dimensionsDev); - this->offsetHost = Kokkos::create_mirror_view(this->offsetDev); - this->dataHost = Kokkos::create_mirror_view(this->dataDev); + this->xinHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->xinDev); + this->dimensionsHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dimensionsDev); + this->offsetHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->offsetDev); + this->dataHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dataDev); // Fill the arrays with the std::vector content if(idfx::prank == 0) { @@ -458,10 +458,10 @@ LookupTable::LookupTable(Kokkos::View array, this->offsetDev = IdefixArray1D ("Table_offset", kDim); this->dataDev = IdefixArray1D ("Table_data", sizeTotal); - this->xinHost = Kokkos::create_mirror_view(this->xinDev); - this->dimensionsHost = Kokkos::create_mirror_view(this->dimensionsDev); - this->offsetHost = Kokkos::create_mirror_view(this->offsetDev); - this->dataHost = Kokkos::create_mirror_view(this->dataDev); + this->xinHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->xinDev); + this->dimensionsHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dimensionsDev); + this->offsetHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->offsetDev); + this->dataHost = Kokkos::create_mirror_view(Kokkos::HostSpace(), this->dataDev); // Copy data in memory for(uint64_t n = 0 ; n < sizeTotal ; n++) { diff --git a/test.py b/test.py new file mode 100755 index 000000000..6ded87fa0 --- /dev/null +++ b/test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import os +import sys +import pytest + +# set IDEFIX_DIR +source_dir = os.path.dirname(__file__) +os.environ["IDEFIX_DIR"] = source_dir + +# idefix test class +sys.path.append(source_dir) +from pytools.idfx_test_run import IdexPytestRunner + +# should be global so it remember state (last build) and avoids +# to build for each run if we just changed the ini file and run options. +gblIdefixPytestRunner = IdexPytestRunner(__file__) + +# define the pytest test +@pytest.mark.parametrize("config", gblIdefixPytestRunner.genTests()) +def test_idefix_build_run_check(config): + gblIdefixPytestRunner.run(config) + +# if called directly as a script +if __name__ == "__main__": + gblIdefixPytestRunner.main(all=True) diff --git a/test/Dust/DustEnergy/testme.json b/test/Dust/DustEnergy/testme.json new file mode 100644 index 000000000..9a78834b9 --- /dev/null +++ b/test/Dust/DustEnergy/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 0 + } + ] +} diff --git a/test/Dust/DustEnergy/testme.py b/test/Dust/DustEnergy/testme.py index 7187d299b..12d476b66 100755 --- a/test/Dust/DustEnergy/testme.py +++ b/test/Dust/DustEnergy/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Dust/DustyShock/testme.json b/test/Dust/DustyShock/testme.json new file mode 100644 index 000000000..d8ddad44f --- /dev/null +++ b/test/Dust/DustyShock/testme.json @@ -0,0 +1,11 @@ +{ + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/test/Dust/DustyShock/testme.py b/test/Dust/DustyShock/testme.py index 2b03d1636..622f34772 100755 --- a/test/Dust/DustyShock/testme.py +++ b/test/Dust/DustyShock/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=1e-14) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Dust/DustyWave/testme.json b/test/Dust/DustyWave/testme.json new file mode 100644 index 000000000..0050f364e --- /dev/null +++ b/test/Dust/DustyWave/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-implicit.ini"], + "noplot": true, + "reconstruction": 2, + "tolerance": 1e-14 + } + ] +} diff --git a/test/Dust/DustyWave/testme.py b/test/Dust/DustyWave/testme.py index 2b03d1636..622f34772 100755 --- a/test/Dust/DustyWave/testme.py +++ b/test/Dust/DustyWave/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=1e-14) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/FargoPlanet/testme.json b/test/HD/FargoPlanet/testme.json new file mode 100644 index 000000000..e1d735828 --- /dev/null +++ b/test/HD/FargoPlanet/testme.json @@ -0,0 +1,25 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini", "idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 1e-13 + } + ], + "when": [ + { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-12 + } + } + ] +} diff --git a/test/HD/FargoPlanet/testme.py b/test/HD/FargoPlanet/testme.py index 5890d010f..b25a74bb0 100755 --- a/test/HD/FargoPlanet/testme.py +++ b/test/HD/FargoPlanet/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/MachReflection/testme.json b/test/HD/MachReflection/testme.json new file mode 100644 index 000000000..be2d8d950 --- /dev/null +++ b/test/HD/MachReflection/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": [2, 2], + "tolerance": 0 + } + ] +} diff --git a/test/HD/MachReflection/testme.py b/test/HD/MachReflection/testme.py index ce87f3923..7e6121815 100755 --- a/test/HD/MachReflection/testme.py +++ b/test/HD/MachReflection/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp") -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/SedovBlastWave/idefix.ini b/test/HD/SedovBlastWave/idefix.ini index 51122521c..107b719cd 100644 --- a/test/HD/SedovBlastWave/idefix.ini +++ b/test/HD/SedovBlastWave/idefix.ini @@ -25,6 +25,5 @@ X3-beg periodic X3-end periodic [Output] -vtk 0.1 -xdmf 0.1 -dmp 0.1 +vtk 0.1 +dmp 0.1 diff --git a/test/HD/SedovBlastWave/testme.json b/test/HD/SedovBlastWave/testme.json new file mode 100644 index 000000000..11327598b --- /dev/null +++ b/test/HD/SedovBlastWave/testme.json @@ -0,0 +1,28 @@ +{ + "namings": "definitionFile,ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions.hpp", + "ini": ["idefix.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "definitionFile": "definitions-spherical.hpp", + "ini": ["idefix-spherical.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": true, + "dec": [2, 2, 2 ], + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/HD/SedovBlastWave/testme.py b/test/HD/SedovBlastWave/testme.py index 95fdd030a..c3ef7025f 100755 --- a/test/HD/SedovBlastWave/testme.py +++ b/test/HD/SedovBlastWave/testme.py @@ -12,7 +12,7 @@ name="dump.0001.dmp" -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/HD/ShearingBox/testme.json b/test/HD/ShearingBox/testme.json new file mode 100644 index 000000000..3e473d0ba --- /dev/null +++ b/test/HD/ShearingBox/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-fargo.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 1e-15 + } + ] +} diff --git a/test/HD/ShearingBox/testme.py b/test/HD/ShearingBox/testme.py index 3739910a0..18c792656 100755 --- a/test/HD/ShearingBox/testme.py +++ b/test/HD/ShearingBox/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','2'] diff --git a/test/HD/ViscousDisk/testme.json b/test/HD/ViscousDisk/testme.json new file mode 100644 index 000000000..df8c19d86 --- /dev/null +++ b/test/HD/ViscousDisk/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 3e-15 + } + ] +} diff --git a/test/HD/ViscousDisk/testme.py b/test/HD/ViscousDisk/testme.py index 1e8ad742b..34628fd80 100755 --- a/test/HD/ViscousDisk/testme.py +++ b/test/HD/ViscousDisk/testme.py @@ -24,7 +24,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/ViscousFlowPastCylinder/testme.json b/test/HD/ViscousFlowPastCylinder/testme.json new file mode 100644 index 000000000..7b3763436 --- /dev/null +++ b/test/HD/ViscousFlowPastCylinder/testme.json @@ -0,0 +1,23 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 3e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-8 + } + } +} diff --git a/test/HD/ViscousFlowPastCylinder/testme.py b/test/HD/ViscousFlowPastCylinder/testme.py index c831e8093..832a80e56 100755 --- a/test/HD/ViscousFlowPastCylinder/testme.py +++ b/test/HD/ViscousFlowPastCylinder/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/HD/sod-iso/testme.json b/test/HD/sod-iso/testme.json new file mode 100644 index 000000000..5016415d6 --- /dev/null +++ b/test/HD/sod-iso/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,single,reconstruction", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [2, 3], + "single": [false], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hllc-rk3.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [4], + "single": [false], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "vectPot": false, + "noplot": true, + "reconstruction": [2], + "single": [true], + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/HD/sod-iso/testme.py b/test/HD/sod-iso/testme.py index e8f0f9ae2..109109baf 100755 --- a/test/HD/sod-iso/testme.py +++ b/test/HD/sod-iso/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/sod/testme.json b/test/HD/sod/testme.json new file mode 100644 index 000000000..9128c08d4 --- /dev/null +++ b/test/HD/sod/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,single,reconstruction", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "vectPot": false, + "single": [false], + "reconstruction": [2,3], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hllc-rk3.ini"], + "noplot": true, + "vectPot": false, + "single": [false], + "reconstruction": [4], + "mpi": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hllc.ini","idefix-tvdlf.ini"], + "noplot": true, + "vectPot": false, + "reconstruction": [2], + "mpi": false, + "single": [true], + "tolerance": 0 + } + ] +} diff --git a/test/HD/sod/testme.py b/test/HD/sod/testme.py index e8f0f9ae2..109109baf 100755 --- a/test/HD/sod/testme.py +++ b/test/HD/sod/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/HD/thermalDiffusion/testme.json b/test/HD/thermalDiffusion/testme.json new file mode 100644 index 000000000..2e89ad747 --- /dev/null +++ b/test/HD/thermalDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","1","2"], + "tolerance": 0 + } + ] +} diff --git a/test/HD/thermalDiffusion/testme.py b/test/HD/thermalDiffusion/testme.py index 3806293d0..1d1a181b0 100755 --- a/test/HD/thermalDiffusion/testme.py +++ b/test/HD/thermalDiffusion/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp") -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/IO/dump/CMakeLists.txt b/test/IO/dump/CMakeLists.txt new file mode 100644 index 000000000..629aed2d1 --- /dev/null +++ b/test/IO/dump/CMakeLists.txt @@ -0,0 +1 @@ +enable_idefix_property(Idefix_MHD) diff --git a/test/IO/dump/definitions.hpp b/test/IO/dump/definitions.hpp new file mode 100644 index 000000000..a854ad9e3 --- /dev/null +++ b/test/IO/dump/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 +//#define DEBUG + +#define GEOMETRY CARTESIAN diff --git a/test/MHD/OrszagTang3D/idefix-checkrestart.ini b/test/IO/dump/idefix.ini similarity index 100% rename from test/MHD/OrszagTang3D/idefix-checkrestart.ini rename to test/IO/dump/idefix.ini diff --git a/test/IO/dump/setup.cpp b/test/IO/dump/setup.cpp new file mode 100644 index 000000000..1db56f5b0 --- /dev/null +++ b/test/IO/dump/setup.cpp @@ -0,0 +1,277 @@ +#include "idefix.hpp" +#include "setup.hpp" + +/*********************************************/ +/** +Customized random number generator +Allow one to have consistent random numbers +generators on different architectures. +**/ +/*********************************************/ + +Output* myOutput; +int outnum; +// Analysis function +// This analysis checks that the restart routines are performing as they should +void Analysis(DataBlock& data) { + + + idfx::cout << "Analysis: Checking restart routines" << std::endl; + + // Trigger dump creation + // data.SetBoundaries(); + myOutput->ForceWriteDump(data); + + // Mirror data on Host + DataBlockHost d(data); + + // Sync it + d.SyncFromDevice(); + + // Create local arrays to store the current physical state + IdefixHostArray4D myVc = IdefixHostArray4D("myVc", d.Vc.extent(0), data.np_tot[KDIR], data.np_tot[JDIR],data.np_tot[IDIR]); + IdefixHostArray4D myVs = IdefixHostArray4D("myVs", DIMENSIONS, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); + #ifdef EVOLVE_VECTOR_POTENTIAL + IdefixHostArray4D myVe = IdefixHostArray4D("myVe", AX3e+1, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); + #endif + // Transfer the datablock to myVc and myVs + for(int n = 0; n < d.Vc.extent(0) ; n++) { + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + myVc(n,k,j,i) = d.Vc(n,k,j,i); + d.Vc(n,k,j,i) = 0.0; + + } + } + } + } + + for(int n = 0; n < DIMENSIONS ; n++) { + for(int k = 0; k < d.np_tot[KDIR] + KOFFSET; k++) { + for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { + for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { + myVs(n,k,j,i) = d.Vs(n,k,j,i); + d.Vs(n,k,j,i) = 0.0; + } + } + } + } + #ifdef EVOLVE_VECTOR_POTENTIAL + for(int n = 0; n < AX3e+1 ; n++) { + for(int k = 0; k < d.np_tot[KDIR] + KOFFSET; k++) { + for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { + for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { + myVe(n,k,j,i) = d.Ve(n,k,j,i); + d.Ve(n,k,j,i) = 0.0; + } + } + } + } + #endif + + // Push our datablockHost to erase everything + d.SyncToDevice(); + // From this point, the dataBlock is full of zeros + + // Load back the restart dump + myOutput->RestartFromDump(data, outnum); + data.SetBoundaries(); + #ifdef EVOLVE_VECTOR_POTENTIAL + data.hydro->emf->ComputeMagFieldFromA(data.hydro->Ve, data.hydro->Vs); + #endif + d.SyncFromDevice(); + + // increment outnum + outnum++; + int errornum; + + errornum = 0; + idfx::cout << "Analysis: checking consistency" << std::endl; + // Check that the save/load routines have left everything unchanged. + for(int n = 0; n < d.Vc.extent(0) ; n++) { + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + if(myVc(n,k,j,i) != d.Vc(n,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Vc at (i,j,k,n) = ( " << i << ", " << j << ", " << k << ", " << n << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl + << "Original= " << myVc(n,k,j,i) << " New=" << d.Vc(n,k,j,i) << " diff=" << myVc(n,k,j,i)-d.Vc(n,k,j,i) << std::endl; + } + + } + } + } + } + + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { + if(myVs(BX1s,k,j,i) != d.Vs(BX1s,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Vs(BX1s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl + << "Original= " << myVs(BX1s,k,j,i) << " New=" << d.Vs(BX1s,k,j,i) << " diff=" << myVs(BX1s,k,j,i)-d.Vs(BX1s,k,j,i) << std::endl; + + } + + } + } + } + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + if(myVs(BX2s,k,j,i) != d.Vs(BX2s,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Vs(BX2s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; + } + + } + } + } + for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + if(myVs(BX3s,k,j,i) != d.Vs(BX3s,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Vs(BX3s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl + << "Original= " << myVs(BX3s,k,j,i) << " New=" << d.Vs(BX3s,k,j,i) << " diff=" << myVs(BX3s,k,j,i)-d.Vs(BX3s,k,j,i) << std::endl; + } + + } + } + } +#ifdef EVOLVE_VECTOR_POTENTIAL + for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { + if(myVe(AX1e,k,j,i) != d.Ve(AX1e,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Ve(AX1e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; + + } + + } + } + } + for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { + if(myVe(AX2e,k,j,i) != d.Ve(AX2e,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Ve(AX2e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; + } + + } + } + } + for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { + for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { + for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { + if(myVe(AX3e,k,j,i) != d.Ve(AX3e,k,j,i)) { + errornum++; + idfx::cout << "-----------------------------------------" << std::endl + << " Error in Ve(AX3e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl + << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl + << "Original= " << myVs(BX3s,k,j,i) << " New=" << d.Vs(BX3s,k,j,i) << " diff=" << myVs(BX3s,k,j,i)-d.Vs(BX3s,k,j,i) << std::endl; + } + + } + } + } +#endif + + idfx::cout << "Analysis: consistency check done with " << errornum << " errors " << std::endl; + if(errornum>0) { + IDEFIX_ERROR("Restart from dump failed validation"); + } +} + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { + if(input.CheckEntry("Output","analysis")>0) { + output.EnrollAnalysis(&Analysis); + myOutput = &output; + outnum=0; + } +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + real x,y,z; + IdefixHostArray4D Ve; + + #ifndef EVOLVE_VECTOR_POTENTIAL + Ve = IdefixHostArray4D("Potential vector",3, d.np_tot[KDIR]+1, d.np_tot[JDIR]+1, d.np_tot[IDIR]+1); + #else + Ve = d.Ve; + #endif + + bool haveTracer = data.hydro->haveTracer; + + real B0=1.0/sqrt(4.0*M_PI); + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + x=d.x[IDIR](i); + y=d.x[JDIR](j); + z=d.x[KDIR](k); + + d.Vc(RHO,k,j,i) = 25.0/(36.0*M_PI); + d.Vc(PRS,k,j,i) = 5.0/(12.0*M_PI); + d.Vc(VX1,k,j,i) = -sin(2.0*M_PI*y); + d.Vc(VX2,k,j,i) = sin(2.0*M_PI*x)+cos(2.0*M_PI*z); + d.Vc(VX3,k,j,i) = cos(2.0*M_PI*x); + + real xl=d.xl[IDIR](i); + real yl=d.xl[JDIR](j); + real zl=d.xl[KDIR](k); + Ve(IDIR,k,j,i) = B0/(2.0*M_PI)*(cos(2.0*M_PI*yl)); + Ve(JDIR,k,j,i) = B0/(2.0*M_PI)*sin(2.0*M_PI*xl); + Ve(KDIR,k,j,i) = B0/(2.0*M_PI)*( + cos(2.0*M_PI*yl) + cos(4.0*M_PI*xl)/2.0); + + if(haveTracer) { + d.Vc(TRG ,k,j,i) = x>0.5? 1.0:0.0; + d.Vc(TRG+1,k,j,i) = z>0.5? 1.0:0.0; + } + } + } + } + + #ifndef EVOLVE_VECTOR_POTENTIAL + d.MakeVsFromAmag(Ve); + #endif + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} + + + +// Do a specifically designed user step in the middle of the integration +void ComputeUserStep(DataBlock &data, real t, real dt) { + +} diff --git a/test/IO/dump/testme.json b/test/IO/dump/testme.json new file mode 100644 index 000000000..cb69f946f --- /dev/null +++ b/test/IO/dump/testme.json @@ -0,0 +1,28 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false, true], + "reconstruction": 2, + "single": [false], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "tolerance": 1e-13 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false], + "reconstruction": 2, + "mpi": [false, true], + "single": [true], + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/IO/dump/testme.py b/test/IO/dump/testme.py new file mode 100755 index 000000000..25e84433b --- /dev/null +++ b/test/IO/dump/testme.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +# Whether we should reset our reference run (only do that on purpose!) + +tolerance=1e-13 + +def testMe(test): + test.configure() + test.compile() + + # Check restarts + test.run("idefix.ini") + + +test=tst.idfxTest(__file__) + +# if no decomposition is specified, use that one +if not test.dec: + test.dec=["2","2","2"] + +if not test.all: + testMe(test) +else: + test.vectPot=False + test.reconstruction=2 + test.mpi=False + testMe(test) + # test in MPI mode + test.mpi=True + testMe(test) + + + # test with vector potential + test.mpi=False + test.vectPot=True + test.reconstruction=2 + testMe(test) + + test.mpi=True + testMe(test) + + # test with other precision + test.single=True + test.vectPot=False + test.reconstruction=2 + test.mpi=False + testMe(test) + # test in MPI mode + test.mpi=True + testMe(test) diff --git a/test/IO/pydefix/README.md b/test/IO/pydefix/README.md index 820a0958e..8dd3f5b4f 100644 --- a/test/IO/pydefix/README.md +++ b/test/IO/pydefix/README.md @@ -42,3 +42,10 @@ export pybind11_DIR=env/lib/python3.10/site-packages/pybind11 ``` you can then run cmake which should be able to find pybind11, and compile the code. + +If while running the code using a python interpreter in an environment, you run into errors stating that some module +installed in your environement is not present. This is because pybind11 does not always capture your venv. You need to force the following environement variables (replace XX by your python version) + +```bash +export PYTHONPATH=$VIRTUAL_ENV/lib/python3.XX/site-packages:$PYTHONPATH +``` diff --git a/test/IO/pydefix/idefix.ini b/test/IO/pydefix/idefix.ini index 57fdcfc4a..3ee42ae3e 100644 --- a/test/IO/pydefix/idefix.ini +++ b/test/IO/pydefix/idefix.ini @@ -26,5 +26,6 @@ X3-beg outflow X3-end outflow [Output] -log 10 +log 100 python 0.02 +dmp 0.5 diff --git a/test/IO/pydefix/python_requirements.txt b/test/IO/pydefix/python_requirements.txt index 6afd3019b..9489d0889 100644 --- a/test/IO/pydefix/python_requirements.txt +++ b/test/IO/pydefix/python_requirements.txt @@ -1,6 +1,7 @@ # note: version requirements are indicative and tests # should be able to run with # older versions of our dependencies, though it is not guaranteed. +# minimally require last versions available for Python 2.7 numpy>=1.16.6 matplotlib>=2.2.5 -pybind11>=2.12.0 +pybind11>=2.10.0 diff --git a/test/IO/pydefix/testme.json b/test/IO/pydefix/testme.json new file mode 100644 index 000000000..78760f724 --- /dev/null +++ b/test/IO/pydefix/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": [false, true], + "dec": ["2","2"], + "standardTest": false, + "tolerance": 1e-12 + } + ] +} diff --git a/test/IO/pydefix/testme.py b/test/IO/pydefix/testme.py new file mode 100755 index 000000000..2940e56dd --- /dev/null +++ b/test/IO/pydefix/testme.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst +tolerance=1e-12 +def testMe(test): + test.configure() + test.compile() + + test.run(inputFile="idefix.ini") + if test.init and not test.mpi: + test.makeReference(filename="dump.0001.dmp") + + test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) + + +test=tst.idfxTest(__file__) +if not test.dec: + test.dec=['2','2'] + +if not test.all: + if(test.check): + test.checkOnly(filename="dump.0001.dmp",tolerance=tolerance) + else: + testMe(test) +else: + test.noplot = True + test.vectPot = False + test.single=False + test.reconstruction=2 + test.mpi=False + testMe(test) + test.mpi=True + testMe(test) diff --git a/test/HD/SedovBlastWave/CMakeLists.txt b/test/IO/xdmf/CMakeLists.txt similarity index 50% rename from test/HD/SedovBlastWave/CMakeLists.txt rename to test/IO/xdmf/CMakeLists.txt index c4ae088b8..8ce9e0f4d 100644 --- a/test/HD/SedovBlastWave/CMakeLists.txt +++ b/test/IO/xdmf/CMakeLists.txt @@ -1 +1,2 @@ +enable_idefix_property(Idefix_MHD) enable_idefix_property(Idefix_HDF5) diff --git a/test/IO/xdmf/definitions.hpp b/test/IO/xdmf/definitions.hpp new file mode 100644 index 000000000..a854ad9e3 --- /dev/null +++ b/test/IO/xdmf/definitions.hpp @@ -0,0 +1,5 @@ +#define COMPONENTS 3 +#define DIMENSIONS 3 +//#define DEBUG + +#define GEOMETRY CARTESIAN diff --git a/test/IO/xdmf/idefix.ini b/test/IO/xdmf/idefix.ini new file mode 100644 index 000000000..250a24054 --- /dev/null +++ b/test/IO/xdmf/idefix.ini @@ -0,0 +1,26 @@ +[Grid] +X1-grid 1 0.0 32 u 1.0 +X2-grid 1 0.0 64 u 1.0 +X3-grid 1 0.0 32 u 1.0 + +[TimeIntegrator] +CFL 0.9 +tstop 0.2 +first_dt 1.e-4 +nstages 2 + +[Hydro] +solver hlld + +[Boundary] +X1-beg periodic +X1-end periodic +X2-beg periodic +X2-end periodic +X3-beg periodic +X3-end periodic + +[Output] +xdmf 0.2 +dmp 0.2 +log 10 diff --git a/test/IO/xdmf/setup.cpp b/test/IO/xdmf/setup.cpp new file mode 100644 index 000000000..8cc40857f --- /dev/null +++ b/test/IO/xdmf/setup.cpp @@ -0,0 +1,75 @@ +#include "idefix.hpp" +#include "setup.hpp" + +// Initialisation routine. Can be used to allocate +// Arrays or variables which are used later on +Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { +} + +// This routine initialize the flow +// Note that data is on the device. +// One can therefore define locally +// a datahost and sync it, if needed +void Setup::InitFlow(DataBlock &data) { + // Create a host copy + DataBlockHost d(data); + real x,y,z; + IdefixHostArray4D Ve; + + #ifndef EVOLVE_VECTOR_POTENTIAL + Ve = IdefixHostArray4D("Potential vector",3, d.np_tot[KDIR]+1, d.np_tot[JDIR]+1, d.np_tot[IDIR]+1); + #else + Ve = d.Ve; + #endif + + bool haveTracer = data.hydro->haveTracer; + + real B0=1.0/sqrt(4.0*M_PI); + + for(int k = 0; k < d.np_tot[KDIR] ; k++) { + for(int j = 0; j < d.np_tot[JDIR] ; j++) { + for(int i = 0; i < d.np_tot[IDIR] ; i++) { + x=d.x[IDIR](i); + y=d.x[JDIR](j); + z=d.x[KDIR](k); + + d.Vc(RHO,k,j,i) = 25.0/(36.0*M_PI); + d.Vc(PRS,k,j,i) = 5.0/(12.0*M_PI); + d.Vc(VX1,k,j,i) = -sin(2.0*M_PI*y); + d.Vc(VX2,k,j,i) = sin(2.0*M_PI*x)+cos(2.0*M_PI*z); + d.Vc(VX3,k,j,i) = cos(2.0*M_PI*x); + + real xl=d.xl[IDIR](i); + real yl=d.xl[JDIR](j); + real zl=d.xl[KDIR](k); + Ve(IDIR,k,j,i) = B0/(2.0*M_PI)*(cos(2.0*M_PI*yl)); + Ve(JDIR,k,j,i) = B0/(2.0*M_PI)*sin(2.0*M_PI*xl); + Ve(KDIR,k,j,i) = B0/(2.0*M_PI)*( + cos(2.0*M_PI*yl) + cos(4.0*M_PI*xl)/2.0); + + if(haveTracer) { + d.Vc(TRG ,k,j,i) = x>0.5? 1.0:0.0; + d.Vc(TRG+1,k,j,i) = z>0.5? 1.0:0.0; + } + } + } + } + + #ifndef EVOLVE_VECTOR_POTENTIAL + d.MakeVsFromAmag(Ve); + #endif + // Send it all, if needed + d.SyncToDevice(); +} + +// Analyse data to produce an output +void MakeAnalysis(DataBlock & data) { + +} + + + +// Do a specifically designed user step in the middle of the integration +void ComputeUserStep(DataBlock &data, real t, real dt) { + +} diff --git a/test/IO/xdmf/testme.json b/test/IO/xdmf/testme.json new file mode 100644 index 000000000..8198bb7bb --- /dev/null +++ b/test/IO/xdmf/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": false, + "reconstruction": 2, + "single": false, + "mpi": false, + "dec": ["2","2","2"], + "standardTest": false, + "nonRegressionTest": false, + "check_file_produced": [ "data.0001.flt.xmf", "data.0001.flt.h5" ] + } + ] +} diff --git a/test/IO/xdmf/testme.py b/test/IO/xdmf/testme.py new file mode 100755 index 000000000..6273bbe0e --- /dev/null +++ b/test/IO/xdmf/testme.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +""" + +@author: glesur +""" +import os +import sys +sys.path.append(os.getenv("IDEFIX_DIR")) + +import pytools.idfx_test as tst + +name="dump.0001.dmp" + +test=tst.idfxTest(__file__) + +def check_xdmf_exists(): + # verify that the expected XDMF sidecar and data file exist. + xdmf_file = "data.0001.flt.xmf" + h5_file = "data.0001.flt.h5" + + + if not os.path.exists(xdmf_file): + print("Missing expected XDMF output file: {}".format(xdmf_file)) + sys.exit(1) + if not os.path.exists(h5_file): + print("Missing expected XDMF data file: {}".format(h5_file)) + sys.exit(1) + +if not test.dec: + test.dec=['2','2','2'] + +if test.check: + check_xdmf_exists() + +else: + test.vectPot=False + test.single=False + test.reconstruction=2 + test.mpi=False + # Only check that the test runs. + test.configure(definitionFile="definitions.hpp") + test.compile() + test.run(inputFile="idefix.ini") + check_xdmf_exists diff --git a/test/MHD/AmbipolarCshock/testme.json b/test/MHD/AmbipolarCshock/testme.json new file mode 100644 index 000000000..2eb2ac51f --- /dev/null +++ b/test/MHD/AmbipolarCshock/testme.json @@ -0,0 +1,14 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/AmbipolarCshock/testme.py b/test/MHD/AmbipolarCshock/testme.py index 675958a7b..0e2f0815e 100755 --- a/test/MHD/AmbipolarCshock/testme.py +++ b/test/MHD/AmbipolarCshock/testme.py @@ -23,7 +23,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/AmbipolarCshock3D/idefix-rkl.ini b/test/MHD/AmbipolarCshock3D/idefix-rkl.ini index 055429557..621552b13 100644 --- a/test/MHD/AmbipolarCshock3D/idefix-rkl.ini +++ b/test/MHD/AmbipolarCshock3D/idefix-rkl.ini @@ -22,8 +22,8 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg periodic -X3-end periodic +X3-beg userdef +X3-end userdef [Output] vtk 100.0 diff --git a/test/MHD/AmbipolarCshock3D/idefix.ini b/test/MHD/AmbipolarCshock3D/idefix.ini index 984e985ef..78c7ddd23 100644 --- a/test/MHD/AmbipolarCshock3D/idefix.ini +++ b/test/MHD/AmbipolarCshock3D/idefix.ini @@ -19,8 +19,8 @@ X1-beg userdef X1-end userdef X2-beg periodic X2-end periodic -X3-beg periodic -X3-end periodic +X3-beg userdef +X3-end userdef [Output] vtk 100.0 diff --git a/test/MHD/AmbipolarCshock3D/setup.cpp b/test/MHD/AmbipolarCshock3D/setup.cpp index 5f8ec9313..abf8a0f24 100644 --- a/test/MHD/AmbipolarCshock3D/setup.cpp +++ b/test/MHD/AmbipolarCshock3D/setup.cpp @@ -79,7 +79,48 @@ void UserdefBoundary(Hydro *hydro, int dir, BoundarySide side, real t) { }); } - + if(dir ==KDIR) { + auto Vc = hydro->Vc; + auto Vs = hydro->Vs; + int jbeg = data->beg[JDIR]; + int jend = data->end[JDIR]; + int kbeg = data->beg[KDIR]; + int kend = data->end[KDIR]; + hydro->boundary->BoundaryForAll("UserdefBoundary", dir, side, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + int kref=k; + + if(side == left) { + kref = kbeg; + } else { + kref = kend -1; + } + Vc(n,k,j,i) = Vc(n,kref,j,i); + }); + + if(dir == KDIR) { + hydro->boundary->BoundaryForX1s("UserdefBoundaryX1s", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + int kref=k; + if(side == left) { + kref = kbeg; + } else { + kref = kend -1; + } + Vs(BX1s,k,j,i) = Vs(BX1s,kref,j,i); + }); + hydro->boundary->BoundaryForX2s("UserdefBoundaryX2s", dir, side, + KOKKOS_LAMBDA (int k, int j, int i) { + int kref=k; + if(side == left) { + kref = kbeg; + } else { + kref = kend -1; + } + Vs(BX2s,k,j,i) = Vs(BX2s,kref,j,i); + }); + } + } } @@ -136,8 +177,8 @@ void Setup::InitFlow(DataBlock &data) { real z = d.x[KDIR](k); real y = d.x[JDIR](j); d.Ve(AX1e,k,j,i) = B0*sin(theta)*z; - d.Ve(AX2e,k,j,i) = ZERO_F; - d.Ve(AX3e,k,j,i) = B0*cos(theta)*y; + d.Ve(AX2e,k,j,i) = -B0*cos(theta)*z; + d.Ve(AX3e,k,j,i) = 0; #else IDEFIX_ERROR("Vector potential only valid in 3 dimensions for this setup"); #endif diff --git a/test/MHD/AmbipolarCshock3D/testme.json b/test/MHD/AmbipolarCshock3D/testme.json new file mode 100644 index 000000000..fdcb11a76 --- /dev/null +++ b/test/MHD/AmbipolarCshock3D/testme.json @@ -0,0 +1,35 @@ +{ + "namings": "ini,mpi,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false], + "vectPot": [false, true], + "dec": ["2","1","1"], + "tolerance": 3e-14 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [true], + "vectPot": [false], + "dec": ["2","1","1"], + "tolerance": 3e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini", + "mpi": true + }, + "apply": { + "tolerance": 2e-10 + } + } +} diff --git a/test/MHD/AmbipolarCshock3D/testme.py b/test/MHD/AmbipolarCshock3D/testme.py index f9fae6765..94a8b0f4c 100755 --- a/test/MHD/AmbipolarCshock3D/testme.py +++ b/test/MHD/AmbipolarCshock3D/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -tolerance=2e-14 +tolerance=3e-14 def testMe(test): test.configure() test.compile() @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','1'] diff --git a/test/MHD/AxisFluxTube/testme.json b/test/MHD/AxisFluxTube/testme.json new file mode 100644 index 000000000..781d4f7b3 --- /dev/null +++ b/test/MHD/AxisFluxTube/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-coarsening.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "vectPot": false, + "standardTest": false, + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/AxisFluxTube/testme.py b/test/MHD/AxisFluxTube/testme.py index 44a53dc28..5abb46568 100755 --- a/test/MHD/AxisFluxTube/testme.py +++ b/test/MHD/AxisFluxTube/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/Coarsening/testme.json b/test/MHD/Coarsening/testme.json new file mode 100644 index 000000000..5b38ffebb --- /dev/null +++ b/test/MHD/Coarsening/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini","idefix-x2.ini","idefix-x3.ini"], + "noplot": true, + "mpi": [false,true], + "tolerance": 0 + } + ] +} diff --git a/test/MHD/Coarsening/testme.py b/test/MHD/Coarsening/testme.py index 52a083bd2..50d13c926 100755 --- a/test/MHD/Coarsening/testme.py +++ b/test/MHD/Coarsening/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/FargoMHDSpherical/testme.json b/test/MHD/FargoMHDSpherical/testme.json new file mode 100644 index 000000000..72d93f493 --- /dev/null +++ b/test/MHD/FargoMHDSpherical/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "vectPot": [false, true], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/FargoMHDSpherical/testme.py b/test/MHD/FargoMHDSpherical/testme.py index 7ee3100b2..1a3b28adb 100755 --- a/test/MHD/FargoMHDSpherical/testme.py +++ b/test/MHD/FargoMHDSpherical/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/MHD/HallWhistler/testme.json b/test/MHD/HallWhistler/testme.json new file mode 100644 index 000000000..cc71c4bf7 --- /dev/null +++ b/test/MHD/HallWhistler/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 1e-15 + } + ] +} diff --git a/test/MHD/HallWhistler/testme.py b/test/MHD/HallWhistler/testme.py index f8d633840..f8ba0a258 100755 --- a/test/MHD/HallWhistler/testme.py +++ b/test/MHD/HallWhistler/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/LinearWaveTest/idefix-entropy.ini b/test/MHD/LinearWaveTest/idefix-entropy.ini index 5a54d274b..805819477 100644 --- a/test/MHD/LinearWaveTest/idefix-entropy.ini +++ b/test/MHD/LinearWaveTest/idefix-entropy.ini @@ -29,4 +29,4 @@ epsilon 1.0e-6 [Output] dmp 1.0 vtk 1.0 -log 1 +log 10 diff --git a/test/MHD/LinearWaveTest/testme.json b/test/MHD/LinearWaveTest/testme.json new file mode 100644 index 000000000..a3f997e34 --- /dev/null +++ b/test/MHD/LinearWaveTest/testme.json @@ -0,0 +1,24 @@ +{ + "namings": "ini,reconstruction,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"], + "noplot": true, + "single": false, + "reconstruction": [2], + "mpi": [false,true], + "dec": ["2","2","2"], + "tolerance": 2e-13 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-fast.ini","idefix-slow.ini","idefix-alfven.ini","idefix-entropy.ini"], + "noplot": true, + "single": false, + "reconstruction": [3, 4], + "mpi": [false], + "dec": ["2","2","2"], + "tolerance": 2e-13 + } + ] +} diff --git a/test/MHD/LinearWaveTest/testme.py b/test/MHD/LinearWaveTest/testme.py index 8c9282862..eeaaab0fe 100755 --- a/test/MHD/LinearWaveTest/testme.py +++ b/test/MHD/LinearWaveTest/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -tolerance=1e-14 +tolerance=2e-13 def testMe(test): test.configure() test.compile() @@ -24,7 +24,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/MHD/MTI/testme.json b/test/MHD/MTI/testme.json new file mode 100644 index 000000000..3bd669cca --- /dev/null +++ b/test/MHD/MTI/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini","idefix-sl.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/MTI/testme.py b/test/MHD/MTI/testme.py index 7975c8075..0e2c9e317 100755 --- a/test/MHD/MTI/testme.py +++ b/test/MHD/MTI/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/OrszagTang/testme.json b/test/MHD/OrszagTang/testme.json new file mode 100644 index 000000000..3128b3239 --- /dev/null +++ b/test/MHD/OrszagTang/testme.json @@ -0,0 +1,74 @@ +{ + "namings": "ini,reconstruction,mpi,single,vectPot", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [false], + "single": false, + "reconstruction": [2,3,4], + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + },{ + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [false], + "single": true, + "reconstruction": [2], + "mpi": [false], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + },{ + "dumpname": "dump.0001.dmp", + "ini": [ + "idefix.ini", + "idefix-hll.ini", + "idefix-hlld-arithmetic.ini", + "idefix-hlld-hll.ini", + "idefix-hlld-hlld.ini", + "idefix-hlld-uct0.ini", + "idefix-hlld.ini", + "idefix-tvdlf.ini" + ], + "noplot": true, + "vectPot": [true], + "single": false, + "reconstruction": [2], + "mpi": [false], + "dec": ["2","2"], + "tolerance": 1e-12, + "standardTest": false + } + ], + "when": { + "conditions": { + "single": true + }, + "apply" : { + "tolerance": 1e-5 + } + } +} diff --git a/test/MHD/OrszagTang/testme.py b/test/MHD/OrszagTang/testme.py index 761505caa..41f33aba6 100755 --- a/test/MHD/OrszagTang/testme.py +++ b/test/MHD/OrszagTang/testme.py @@ -33,7 +33,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/MHD/OrszagTang3D/setup.cpp b/test/MHD/OrszagTang3D/setup.cpp index 8e7205c42..a6927a132 100644 --- a/test/MHD/OrszagTang3D/setup.cpp +++ b/test/MHD/OrszagTang3D/setup.cpp @@ -12,198 +12,10 @@ generators on different architectures. Output* myOutput; int outnum; // Analysis function -// This analysis checks that the restart routines are performing as they should -void Analysis(DataBlock& data) { - - - idfx::cout << "Analysis: Checking restart routines" << std::endl; - - // Trigger dump creation - myOutput->ForceWriteDump(data); - - // Mirror data on Host - DataBlockHost d(data); - - // Sync it - d.SyncFromDevice(); - - // Create local arrays to store the current physical state - IdefixHostArray4D myVc = IdefixHostArray4D("myVc", d.Vc.extent(0), data.np_tot[KDIR], data.np_tot[JDIR],data.np_tot[IDIR]); - IdefixHostArray4D myVs = IdefixHostArray4D("myVs", DIMENSIONS, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); - #ifdef EVOLVE_VECTOR_POTENTIAL - IdefixHostArray4D myVe = IdefixHostArray4D("myVe", AX3e+1, data.np_tot[KDIR]+KOFFSET, data.np_tot[JDIR]+JOFFSET,data.np_tot[IDIR]+IOFFSET); - #endif - // Transfer the datablock to myVc and myVs - for(int n = 0; n < d.Vc.extent(0) ; n++) { - for(int k = 0; k < d.np_tot[KDIR] ; k++) { - for(int j = 0; j < d.np_tot[JDIR] ; j++) { - for(int i = 0; i < d.np_tot[IDIR] ; i++) { - myVc(n,k,j,i) = d.Vc(n,k,j,i); - d.Vc(n,k,j,i) = 0.0; - - } - } - } - } - - for(int n = 0; n < DIMENSIONS ; n++) { - for(int k = 0; k < d.np_tot[KDIR] + KOFFSET; k++) { - for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { - for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { - myVs(n,k,j,i) = d.Vs(n,k,j,i); - d.Vs(n,k,j,i) = 0.0; - } - } - } - } - #ifdef EVOLVE_VECTOR_POTENTIAL - for(int n = 0; n < AX3e+1 ; n++) { - for(int k = 0; k < d.np_tot[KDIR] + KOFFSET; k++) { - for(int j = 0; j < d.np_tot[JDIR] + JOFFSET; j++) { - for(int i = 0; i < d.np_tot[IDIR] + IOFFSET; i++) { - myVe(n,k,j,i) = d.Ve(n,k,j,i); - d.Ve(n,k,j,i) = 0.0; - } - } - } - } - #endif - - // Push our datablockHost to erase everything - d.SyncToDevice(); - // From this point, the dataBlock is full of zeros - - // Load back the restart dump - myOutput->RestartFromDump(data, outnum); - data.SetBoundaries(); - #ifdef EVOLVE_VECTOR_POTENTIAL - data.hydro->emf->ComputeMagFieldFromA(data.hydro->Ve, data.hydro->Vs); - #endif - d.SyncFromDevice(); - - // increment outnum - outnum++; - int errornum; - - errornum = 0; - idfx::cout << "Analysis: checking consistency" << std::endl; - // Check that the save/load routines have left everything unchanged. - for(int n = 0; n < d.Vc.extent(0) ; n++) { - for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { - if(myVc(n,k,j,i) != d.Vc(n,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Vc at (i,j,k,n) = ( " << i << ", " << j << ", " << k << ", " << n << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; - } - - } - } - } - } - - for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { - if(myVs(BX1s,k,j,i) != d.Vs(BX1s,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Vs(BX1s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl - << "Original= " << myVs(BX1s,k,j,i) << " New=" << d.Vs(BX1s,k,j,i) << " diff=" << myVs(BX1s,k,j,i)-d.Vs(BX1s,k,j,i) << std::endl; - - } - - } - } - } - for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { - if(myVs(BX2s,k,j,i) != d.Vs(BX2s,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Vs(BX2s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; - } - - } - } - } - for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { - if(myVs(BX3s,k,j,i) != d.Vs(BX3s,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Vs(BX3s) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl - << "Original= " << myVs(BX3s,k,j,i) << " New=" << d.Vs(BX3s,k,j,i) << " diff=" << myVs(BX3s,k,j,i)-d.Vs(BX3s,k,j,i) << std::endl; - } - - } - } - } -#ifdef EVOLVE_VECTOR_POTENTIAL - for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR] ; i++) { - if(myVe(AX1e,k,j,i) != d.Ve(AX1e,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Ve(AX1e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; - - } - - } - } - } - for(int k = d.beg[KDIR]; k < d.end[KDIR]+KOFFSET ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR] ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { - if(myVe(AX2e,k,j,i) != d.Ve(AX2e,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Ve(AX2e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl; - } - - } - } - } - for(int k = d.beg[KDIR]; k < d.end[KDIR] ; k++) { - for(int j = d.beg[JDIR]; j < d.end[JDIR]+JOFFSET ; j++) { - for(int i = d.beg[IDIR]; i < d.end[IDIR]+IOFFSET ; i++) { - if(myVe(AX3e,k,j,i) != d.Ve(AX3e,k,j,i)) { - errornum++; - idfx::cout << "-----------------------------------------" << std::endl - << " Error in Ve(AX3e) at (i,j,k) = ( " << i << ", " << j << ", " << k << ")" << std::endl - << " Coordinates (x1,x2,x3) = ( " << d.x[IDIR](i) << ", " << d.x[JDIR](j) << ", " << d.x[KDIR](k) << ")" << std::endl - << "Original= " << myVs(BX3s,k,j,i) << " New=" << d.Vs(BX3s,k,j,i) << " diff=" << myVs(BX3s,k,j,i)-d.Vs(BX3s,k,j,i) << std::endl; - } - - } - } - } -#endif - - idfx::cout << "Analysis: consistency check done with " << errornum << " errors " << std::endl; - if(errornum>0) { - IDEFIX_ERROR("Restart from dump failed validation"); - } -} // Initialisation routine. Can be used to allocate // Arrays or variables which are used later on Setup::Setup(Input &input, Grid &grid, DataBlock &data, Output &output) { - if(input.CheckEntry("Output","analysis")>0) { - output.EnrollAnalysis(&Analysis); - myOutput = &output; - outnum=0; - } } // This routine initialize the flow diff --git a/test/MHD/OrszagTang3D/testme.json b/test/MHD/OrszagTang3D/testme.json new file mode 100644 index 000000000..a37b54164 --- /dev/null +++ b/test/MHD/OrszagTang3D/testme.json @@ -0,0 +1,34 @@ +{ + "namings": "ini,single,vectPot,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false, true], + "reconstruction": 2, + "single": [false], + "mpi": [false, true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-13 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "vectPot": [false], + "reconstruction": 2, + "mpi": [false, true], + "single": [true], + "dec": ["2","2","2"], + "standardTest": false, + "tolerance": 1e-13 + } + ], + "when": { + "conditions": { + "single": true + }, + "apply" : { + "tolerance": 1e-6 + } + } +} diff --git a/test/MHD/OrszagTang3D/testme.py b/test/MHD/OrszagTang3D/testme.py index 600a5ef5f..7a66150c5 100755 --- a/test/MHD/OrszagTang3D/testme.py +++ b/test/MHD/OrszagTang3D/testme.py @@ -27,14 +27,8 @@ def testMe(test): test.makeReference(filename="dump.0001.dmp") test.nonRegressionTest(filename="dump.0001.dmp",tolerance=tol) - # Check restarts - test.run("idefix-checkrestart.ini") - #force override the inputfile since the result should be identical - test.inifile="idefix.ini" - test.nonRegressionTest(filename="dump.0002.dmp",tolerance=tol) - -test=tst.idfxTest() +test=tst.idfxTest(__file__) # if no decomposition is specified, use that one if not test.dec: diff --git a/test/MHD/ResistiveAlfvenWave/testme.json b/test/MHD/ResistiveAlfvenWave/testme.json new file mode 100644 index 000000000..56ae122ec --- /dev/null +++ b/test/MHD/ResistiveAlfvenWave/testme.json @@ -0,0 +1,22 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-rkl.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "tolerance": 1e-14 + } + ], + "when": { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply" : { + "tolerance": 1e-10 + } + } +} diff --git a/test/MHD/ResistiveAlfvenWave/testme.py b/test/MHD/ResistiveAlfvenWave/testme.py index f9b8ae56f..ebc29ab57 100755 --- a/test/MHD/ResistiveAlfvenWave/testme.py +++ b/test/MHD/ResistiveAlfvenWave/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/ShearingBox/testme.json b/test/MHD/ShearingBox/testme.json new file mode 100644 index 000000000..71ae0ab0d --- /dev/null +++ b/test/MHD/ShearingBox/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-fargo.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","1","2"], + "tolerance": 1e-14 + } + ] +} diff --git a/test/MHD/ShearingBox/testme.py b/test/MHD/ShearingBox/testme.py index c89d24b52..ccee1a69c 100755 --- a/test/MHD/ShearingBox/testme.py +++ b/test/MHD/ShearingBox/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename="dump.0001.dmp",tolerance=mytol) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','1','2'] @@ -41,3 +41,6 @@ def testMe(test): test.reconstruction=2 test.mpi=False testMe(test) + + test.mpi=True + testMe(test) diff --git a/test/MHD/clessTDiffusion/testme.json b/test/MHD/clessTDiffusion/testme.json new file mode 100644 index 000000000..d615696f5 --- /dev/null +++ b/test/MHD/clessTDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "vectPot": false, + "tolerance": 2e-15 + } + ] +} diff --git a/test/MHD/clessTDiffusion/testme.py b/test/MHD/clessTDiffusion/testme.py index 6ed5b1648..41193e141 100755 --- a/test/MHD/clessTDiffusion/testme.py +++ b/test/MHD/clessTDiffusion/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=2e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sod-iso/testme.json b/test/MHD/sod-iso/testme.json new file mode 100644 index 000000000..9f5534fbc --- /dev/null +++ b/test/MHD/sod-iso/testme.json @@ -0,0 +1,43 @@ +{ + "namings": "ini,reconstruction,single", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "single": false, + "reconstruction": [2,3], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hlld-rk3.ini"], + "vectPot": false, + "single": false, + "reconstruction": [4], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "reconstruction": [2], + "mpi": false, + "single": true, + "standardTest": false, + "tolerance": 0 + } + ], + "when": [ + { + "conditions": { + "ini": "idefix-rkl.ini" + }, + "apply": { + "tolerance": 1e-10 + } + } + ] +} diff --git a/test/MHD/sod-iso/testme.py b/test/MHD/sod-iso/testme.py index 798aace57..c85cc4ce7 100755 --- a/test/MHD/sod-iso/testme.py +++ b/test/MHD/sod-iso/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sod/testme.json b/test/MHD/sod/testme.json new file mode 100644 index 000000000..1501e3049 --- /dev/null +++ b/test/MHD/sod/testme.json @@ -0,0 +1,33 @@ +{ + "namings": "ini,reconstruction,single", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "single": false, + "reconstruction": [2,3], + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk3.ini","idefix-hlld-rk3.ini"], + "vectPot": false, + "single": false, + "reconstruction": 4, + "mpi": false, + "standardTest": false, + "tolerance": 0 + },{ + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-hll.ini","idefix-hlld.ini","idefix-tvdlf.ini"], + "vectPot": false, + "reconstruction": 2, + "mpi": false, + "single": true, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/MHD/sod/testme.py b/test/MHD/sod/testme.py index 798aace57..c85cc4ce7 100755 --- a/test/MHD/sod/testme.py +++ b/test/MHD/sod/testme.py @@ -27,7 +27,7 @@ def testMe(test): test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sphBragTDiffusion/testme.json b/test/MHD/sphBragTDiffusion/testme.json new file mode 100644 index 000000000..c4cb9a334 --- /dev/null +++ b/test/MHD/sphBragTDiffusion/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "tolerance": 2e-15 + } + ] +} diff --git a/test/MHD/sphBragTDiffusion/testme.py b/test/MHD/sphBragTDiffusion/testme.py index 6ed5b1648..41193e141 100755 --- a/test/MHD/sphBragTDiffusion/testme.py +++ b/test/MHD/sphBragTDiffusion/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=2e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/MHD/sphBragViscosity/testme.json b/test/MHD/sphBragViscosity/testme.json new file mode 100644 index 000000000..900a34a77 --- /dev/null +++ b/test/MHD/sphBragViscosity/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 1e-15 + } + ] +} diff --git a/test/MHD/sphBragViscosity/testme.py b/test/MHD/sphBragViscosity/testme.py index 254417f44..b6c3467ea 100755 --- a/test/MHD/sphBragViscosity/testme.py +++ b/test/MHD/sphBragViscosity/testme.py @@ -29,7 +29,7 @@ def testMe(test): test.nonRegressionTest(filename=name, tolerance=1e-15) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Planet/Planet3Body/testme.json b/test/Planet/Planet3Body/testme.json new file mode 100644 index 000000000..2b2fad202 --- /dev/null +++ b/test/Planet/Planet3Body/testme.json @@ -0,0 +1,15 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "tolerance": 2e-11 + } + ] +} diff --git a/test/Planet/Planet3Body/testme.py b/test/Planet/Planet3Body/testme.py index c11d08963..6edac53ee 100755 --- a/test/Planet/Planet3Body/testme.py +++ b/test/Planet/Planet3Body/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/Planet/PlanetMigration2D/testme.json b/test/Planet/PlanetMigration2D/testme.json new file mode 100644 index 000000000..6e1e08537 --- /dev/null +++ b/test/Planet/PlanetMigration2D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetMigration2D/testme.py b/test/Planet/PlanetMigration2D/testme.py index 0921d4e62..6c435cbd3 100755 --- a/test/Planet/PlanetMigration2D/testme.py +++ b/test/Planet/PlanetMigration2D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetPlanetRK42D/testme.json b/test/Planet/PlanetPlanetRK42D/testme.json new file mode 100644 index 000000000..0181b072f --- /dev/null +++ b/test/Planet/PlanetPlanetRK42D/testme.json @@ -0,0 +1,25 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0002.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13, + "multirun": [ + { + "nonRegressionTest": true, + "standardTest": true + },{ + "restart": true, + "restart_no_overwrite": ["dump.0001.dmp", "data.0005.vtk"] + } + ] + } + ] +} diff --git a/test/Planet/PlanetPlanetRK42D/testme.py b/test/Planet/PlanetPlanetRK42D/testme.py index 26cb8ee21..9521aabac 100755 --- a/test/Planet/PlanetPlanetRK42D/testme.py +++ b/test/Planet/PlanetPlanetRK42D/testme.py @@ -38,7 +38,7 @@ def testMe(test): assert dump_mtime == os.path.getmtime("dump.0001.dmp"), "Dump was overwritten on restart" assert vtk_mtime == os.path.getmtime("data.0005.vtk"), "VTK file was overwritten on restart" -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetSpiral2D/testme.json b/test/Planet/PlanetSpiral2D/testme.json new file mode 100644 index 000000000..6e1e08537 --- /dev/null +++ b/test/Planet/PlanetSpiral2D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetSpiral2D/testme.py b/test/Planet/PlanetSpiral2D/testme.py index 0921d4e62..6c435cbd3 100755 --- a/test/Planet/PlanetSpiral2D/testme.py +++ b/test/Planet/PlanetSpiral2D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/Planet/PlanetTorque3D/testme.json b/test/Planet/PlanetTorque3D/testme.json new file mode 100644 index 000000000..085fb6c0f --- /dev/null +++ b/test/Planet/PlanetTorque3D/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2","2"], + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetTorque3D/testme.py b/test/Planet/PlanetTorque3D/testme.py index b4f04a507..4393636ea 100755 --- a/test/Planet/PlanetTorque3D/testme.py +++ b/test/Planet/PlanetTorque3D/testme.py @@ -28,7 +28,7 @@ def testMe(test): test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','2'] diff --git a/test/Planet/PlanetsIsActiveRK52D/testme.json b/test/Planet/PlanetsIsActiveRK52D/testme.json new file mode 100644 index 000000000..30848aecd --- /dev/null +++ b/test/Planet/PlanetsIsActiveRK52D/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix-rk4.ini", "idefix-rk5.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2","2"], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/Planet/PlanetsIsActiveRK52D/testme.py b/test/Planet/PlanetsIsActiveRK52D/testme.py index ce0d405c0..aca39ae53 100755 --- a/test/Planet/PlanetsIsActiveRK52D/testme.py +++ b/test/Planet/PlanetsIsActiveRK52D/testme.py @@ -26,7 +26,7 @@ def testMe(test): # test.nonRegressionTest(filename=name,tolerance=tolerance) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2'] diff --git a/test/SelfGravity/DustyCollapse/testme.json b/test/SelfGravity/DustyCollapse/testme.json new file mode 100644 index 000000000..8afbcdb6d --- /dev/null +++ b/test/SelfGravity/DustyCollapse/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/SelfGravity/DustyCollapse/testme.py b/test/SelfGravity/DustyCollapse/testme.py index 689038265..2df3893ab 100755 --- a/test/SelfGravity/DustyCollapse/testme.py +++ b/test/SelfGravity/DustyCollapse/testme.py @@ -25,7 +25,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/SelfGravity/JeansInstability/testme.json b/test/SelfGravity/JeansInstability/testme.json new file mode 100644 index 000000000..bf4819389 --- /dev/null +++ b/test/SelfGravity/JeansInstability/testme.json @@ -0,0 +1,17 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini"], + "noplot": true, + "vectPot": false, + "single": false, + "reconstruction": 2, + "mpi": [false, true], + "dec": ["2"], + "nonRegressionTest": false, + "tolerance": 1e-13 + } + ] +} diff --git a/test/SelfGravity/JeansInstability/testme.py b/test/SelfGravity/JeansInstability/testme.py index ff2f39a83..833feb262 100755 --- a/test/SelfGravity/JeansInstability/testme.py +++ b/test/SelfGravity/JeansInstability/testme.py @@ -26,7 +26,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) # if no decomposition is specified, use that one if not test.dec: diff --git a/test/SelfGravity/RandomSphere/testme.json b/test/SelfGravity/RandomSphere/testme.json new file mode 100644 index 000000000..d5cda3135 --- /dev/null +++ b/test/SelfGravity/RandomSphere/testme.json @@ -0,0 +1,14 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini","idefix-minres.ini"], + "noplot": true, + "mpi": [false, true], + "dec": ["2","2","1"], + "nonRegressionTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/SelfGravity/RandomSphere/testme.py b/test/SelfGravity/RandomSphere/testme.py index 38fc30d92..65123498b 100755 --- a/test/SelfGravity/RandomSphere/testme.py +++ b/test/SelfGravity/RandomSphere/testme.py @@ -26,7 +26,7 @@ def testMe(test): #test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.dec: test.dec=['2','2','1'] diff --git a/test/SelfGravity/RandomSphereCartesian/testme.json b/test/SelfGravity/RandomSphereCartesian/testme.json new file mode 100644 index 000000000..85f0f3889 --- /dev/null +++ b/test/SelfGravity/RandomSphereCartesian/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini","idefix-cg.ini","idefix-minres.ini","idefix-jacobi.ini"], + "noplot": true, + "nonRegressionTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/SelfGravity/RandomSphereCartesian/testme.py b/test/SelfGravity/RandomSphereCartesian/testme.py index 64b4a9ba5..598aa3f21 100755 --- a/test/SelfGravity/RandomSphereCartesian/testme.py +++ b/test/SelfGravity/RandomSphereCartesian/testme.py @@ -26,7 +26,7 @@ def testMe(test): #test.nonRegressionTest(filename=name) -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: testMe(test) else: diff --git a/test/SelfGravity/UniformCollapse/testme.json b/test/SelfGravity/UniformCollapse/testme.json new file mode 100644 index 000000000..3de00c4df --- /dev/null +++ b/test/SelfGravity/UniformCollapse/testme.json @@ -0,0 +1,16 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "noplot": true, + "single": false, + "reconstruction": 2, + "mpi": false, + "standardTest": false, + "tolerance": 0, + "nonRegressionTest": false + } + ] +} diff --git a/test/SelfGravity/UniformCollapse/testme.py b/test/SelfGravity/UniformCollapse/testme.py index 21debc0e0..69918896b 100755 --- a/test/SelfGravity/UniformCollapse/testme.py +++ b/test/SelfGravity/UniformCollapse/testme.py @@ -25,7 +25,7 @@ def testMe(test): test.standardTest() -test=tst.idfxTest() +test=tst.idfxTest(__file__) if not test.all: if(test.check): diff --git a/test/python_requirements.txt b/test/python_requirements.txt index 77ce6e9c2..9cf2383a2 100644 --- a/test/python_requirements.txt +++ b/test/python_requirements.txt @@ -8,3 +8,7 @@ scipy>=1.2.3 # note that no version of inifix supports Python older than 3.6 inifix>=0.11.2 + +# To run the test suite, we can use pytest, mostly any version +# 6.0 is the one available on system available on debian-11 +pytest >= 6.0 diff --git a/test/utils/columnDensity/testme.json b/test/utils/columnDensity/testme.json new file mode 100644 index 000000000..718413c4d --- /dev/null +++ b/test/utils/columnDensity/testme.json @@ -0,0 +1,13 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "mpi": [false,true], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/utils/columnDensity/testme.py b/test/utils/columnDensity/testme.py index 6f017d7d8..cd94eb4ef 100755 --- a/test/utils/columnDensity/testme.py +++ b/test/utils/columnDensity/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -test=tst.idfxTest() +test=tst.idfxTest(__file__) test.configure() test.compile() diff --git a/test/utils/dumpImage/testme.json b/test/utils/dumpImage/testme.json new file mode 100644 index 000000000..718413c4d --- /dev/null +++ b/test/utils/dumpImage/testme.json @@ -0,0 +1,13 @@ +{ + "namings": "ini,mpi", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini": ["idefix.ini"], + "mpi": [false,true], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/utils/dumpImage/testme.py b/test/utils/dumpImage/testme.py index 6f017d7d8..cd94eb4ef 100755 --- a/test/utils/dumpImage/testme.py +++ b/test/utils/dumpImage/testme.py @@ -9,7 +9,7 @@ sys.path.append(os.getenv("IDEFIX_DIR")) import pytools.idfx_test as tst -test=tst.idfxTest() +test=tst.idfxTest(__file__) test.configure() test.compile() diff --git a/test/utils/lookupTable/testme.json b/test/utils/lookupTable/testme.json new file mode 100644 index 000000000..8ea6f6961 --- /dev/null +++ b/test/utils/lookupTable/testme.json @@ -0,0 +1,12 @@ +{ + "namings": "ini", + "variants": [ + { + "dumpname": "dump.0001.dmp", + "ini":["idefix.ini"], + "nonRegressionTest": false, + "standardTest": false, + "tolerance": 0 + } + ] +} diff --git a/test/utils/lookupTable/testme.py b/test/utils/lookupTable/testme.py index d40884652..53f6607f2 100755 --- a/test/utils/lookupTable/testme.py +++ b/test/utils/lookupTable/testme.py @@ -30,7 +30,7 @@ def MakeNumpyFile(): -test=tst.idfxTest() +test=tst.idfxTest(__file__) MakeNumpyFile() test.configure() From dc8044464f59aef1fc6a8733b8248a9b12c084c5 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 22 Apr 2026 10:04:24 +0200 Subject: [PATCH 09/10] Upgrade Python version to 3.12 for rtd --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7c72516d6..0ebf2f25c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.12" # You can also specify other tool versions: # nodejs: "19" # rust: "1.64" From f2987b72b766a6325318d65b1b4325279fea5571 Mon Sep 17 00:00:00 2001 From: Alankar Dutta Date: Mon, 20 Jul 2026 09:29:50 +0200 Subject: [PATCH 10/10] Update src/global.cpp Co-authored-by: Geoffroy Lesur --- src/global.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/global.cpp b/src/global.cpp index b9a160420..3cf13bee4 100644 --- a/src/global.cpp +++ b/src/global.cpp @@ -119,7 +119,7 @@ void IdefixOutStream::init(int rank) { void IdefixOutStream::enableLogFile() { std::stringstream sslogFileName; - sslogFileName << idfx::logFileDir << "/./" << "idefix." << idfx::prank << ".log"; + sslogFileName << idfx::logFileDir << "/" << "idefix." << idfx::prank << ".log"; std::string logFileName(sslogFileName.str()); if(idfx::prank==0) {