From e2bb0de94b424bfc2de257119302643f857e768c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 6 Sep 2024 14:33:16 -0700 Subject: [PATCH] EBs: Compiled by Default, Controlled at Runtime (#4865) * Build Logic: Enable EBs by Default * EB: Introduce Runtime Parameter * Cleaning: Review Weiqun * Cleaning: EB w/ Clang Tidy * Formatting for Clarity Only changes spaces * CMake: Simplify `add_warpx_test` EB is not mutually exclusive anymore: we can run non-EB tests with EB compiled in. * Azure CI: EB is default-ON now --- .azure-pipelines.yml | 4 - .github/workflows/source/check_inputs.py | 2 +- CMakeLists.txt | 2 +- Docs/source/developers/testing.rst | 4 - Docs/source/install/cmake.rst | 4 +- Docs/source/usage/parameters.rst | 7 +- Examples/CMakeLists.txt | 13 -- .../beam_beam_collision/CMakeLists.txt | 1 - .../capacitive_discharge/CMakeLists.txt | 7 +- .../laser_acceleration/CMakeLists.txt | 14 -- .../laser_ion/CMakeLists.txt | 2 - .../plasma_acceleration/CMakeLists.txt | 8 - .../plasma_mirror/CMakeLists.txt | 1 - .../spacecraft_charging/CMakeLists.txt | 1 - .../uniform_plasma/CMakeLists.txt | 3 - .../Tests/accelerator_lattice/CMakeLists.txt | 3 - Examples/Tests/boosted_diags/CMakeLists.txt | 1 - Examples/Tests/boundaries/CMakeLists.txt | 1 - Examples/Tests/btd_rz/CMakeLists.txt | 1 - .../collider_relevant_diags/CMakeLists.txt | 1 - Examples/Tests/collision/CMakeLists.txt | 6 - Examples/Tests/diff_lumi_diag/CMakeLists.txt | 1 - Examples/Tests/divb_cleaning/CMakeLists.txt | 1 - Examples/Tests/dive_cleaning/CMakeLists.txt | 2 - .../electrostatic_dirichlet_bc/CMakeLists.txt | 2 - .../Tests/electrostatic_sphere/CMakeLists.txt | 5 - .../electrostatic_sphere_eb/CMakeLists.txt | 5 - .../embedded_boundary_cube/CMakeLists.txt | 3 - .../CMakeLists.txt | 1 - .../CMakeLists.txt | 1 - .../CMakeLists.txt | 2 - Examples/Tests/embedded_circle/CMakeLists.txt | 1 - .../CMakeLists.txt | 1 - Examples/Tests/field_probe/CMakeLists.txt | 1 - Examples/Tests/flux_injection/CMakeLists.txt | 2 - Examples/Tests/gaussian_beam/CMakeLists.txt | 2 - Examples/Tests/implicit/CMakeLists.txt | 4 - .../Tests/initial_distribution/CMakeLists.txt | 1 - .../initial_plasma_profile/CMakeLists.txt | 1 - Examples/Tests/ion_stopping/CMakeLists.txt | 1 - Examples/Tests/ionization/CMakeLists.txt | 3 - Examples/Tests/langmuir/CMakeLists.txt | 35 --- Examples/Tests/langmuir_fluids/CMakeLists.txt | 4 - Examples/Tests/larmor/CMakeLists.txt | 1 - Examples/Tests/laser_injection/CMakeLists.txt | 3 - .../laser_injection_from_file/CMakeLists.txt | 14 -- Examples/Tests/laser_on_fine/CMakeLists.txt | 1 - .../Tests/load_external_field/CMakeLists.txt | 6 - .../Tests/magnetostatic_eb/CMakeLists.txt | 3 - .../Tests/maxwell_hybrid_qed/CMakeLists.txt | 1 - .../Tests/nci_fdtd_stability/CMakeLists.txt | 2 - .../Tests/nci_psatd_stability/CMakeLists.txt | 17 -- .../Tests/nodal_electrostatic/CMakeLists.txt | 1 - Examples/Tests/nuclear_fusion/CMakeLists.txt | 6 - .../Tests/ohm_solver_em_modes/CMakeLists.txt | 2 - .../CMakeLists.txt | 1 - .../CMakeLists.txt | 1 - .../CMakeLists.txt | 1 - .../open_bc_poisson_solver/CMakeLists.txt | 1 - .../CMakeLists.txt | 1 - .../particle_boundary_process/CMakeLists.txt | 2 - .../particle_boundary_scrape/CMakeLists.txt | 2 - .../Tests/particle_data_python/CMakeLists.txt | 3 - .../particle_fields_diags/CMakeLists.txt | 4 +- Examples/Tests/particle_pusher/CMakeLists.txt | 1 - .../particle_thermal_boundary/CMakeLists.txt | 1 - .../Tests/particles_in_pml/CMakeLists.txt | 4 - .../pass_mpi_communicator/CMakeLists.txt | 1 - Examples/Tests/pec/CMakeLists.txt | 3 - Examples/Tests/photon_pusher/CMakeLists.txt | 1 - Examples/Tests/plasma_lens/CMakeLists.txt | 5 - Examples/Tests/pml/CMakeLists.txt | 8 - .../Tests/point_of_contact_eb/CMakeLists.txt | 2 - .../projection_divb_cleaner/CMakeLists.txt | 3 - Examples/Tests/python_wrappers/CMakeLists.txt | 1 - Examples/Tests/qed/CMakeLists.txt | 10 - .../Tests/radiation_reaction/CMakeLists.txt | 1 - Examples/Tests/reduced_diags/CMakeLists.txt | 5 - .../CMakeLists.txt | 1 - .../Tests/repelling_particles/CMakeLists.txt | 1 - Examples/Tests/resampling/CMakeLists.txt | 3 - Examples/Tests/restart/CMakeLists.txt | 9 - Examples/Tests/restart_eb/CMakeLists.txt | 4 +- Examples/Tests/rigid_injection/CMakeLists.txt | 2 - Examples/Tests/scraping/CMakeLists.txt | 2 - Examples/Tests/silver_mueller/CMakeLists.txt | 4 - Examples/Tests/single_particle/CMakeLists.txt | 1 - .../CMakeLists.txt | 2 - Examples/Tests/subcycling/CMakeLists.txt | 1 - Examples/Tests/vay_deposition/CMakeLists.txt | 2 - Source/BoundaryConditions/PML.H | 3 +- Source/BoundaryConditions/PML.cpp | 46 ++-- Source/BoundaryConditions/WarpXEvolvePML.cpp | 29 ++- .../BoundaryScrapingDiagnostics.cpp | 9 +- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 36 +-- Source/Diagnostics/WarpXIO.cpp | 2 +- Source/EmbeddedBoundary/CMakeLists.txt | 1 + Source/EmbeddedBoundary/DistanceToEB.H | 56 +++-- Source/EmbeddedBoundary/Enabled.H | 2 +- Source/EmbeddedBoundary/ParticleScraper.H | 4 +- .../EmbeddedBoundary/WarpXFaceExtensions.cpp | 80 ++++--- Source/EmbeddedBoundary/WarpXInitEB.cpp | 35 +-- Source/Evolve/WarpXEvolve.cpp | 11 +- Source/FieldSolver/ElectrostaticSolver.cpp | 98 ++++---- .../FiniteDifferenceSolver/EvolveB.cpp | 32 ++- .../FiniteDifferenceSolver/EvolveE.cpp | 56 ++--- .../FiniteDifferenceSolver/EvolveECTRho.cpp | 6 +- .../FiniteDifferenceSolver/EvolveEPML.cpp | 37 ++- .../HybridPICModel/HybridPICModel.cpp | 76 +++--- .../HybridPICSolveE.cpp | 140 +++++------ .../MacroscopicEvolveE.cpp | 31 +-- .../MagnetostaticSolver.cpp | 1 + Source/Initialization/WarpXInitData.cpp | 101 ++++---- Source/Parallelization/WarpXRegrid.cpp | 45 ++-- Source/Particles/MultiParticleContainer.cpp | 10 +- Source/Particles/ParticleBoundaryBuffer.cpp | 221 +++++++++--------- .../Particles/PhysicalParticleContainer.cpp | 15 +- Source/Particles/WarpXParticleContainer.cpp | 9 +- Source/Utils/WarpXAlgorithmSelection.cpp | 2 +- Source/WarpX.H | 6 +- Source/WarpX.cpp | 159 ++++++++----- Source/ablastr/fields/PoissonSolver.H | 184 +++++++++------ Source/ablastr/fields/VectorPoissonSolver.H | 47 ++-- setup.py | 2 +- 124 files changed, 848 insertions(+), 1060 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 6e9884966fe..1355dc2f647 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -36,10 +36,6 @@ jobs: cylindrical_rz: WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON WARPX_RZ_FFT: 'TRUE' - # embedded boundaries - embedded_boundaries: - WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_EB=ON - WARPX_RZ_FFT: 'TRUE' # single precision #single_precision: # WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE diff --git a/.github/workflows/source/check_inputs.py b/.github/workflows/source/check_inputs.py index 3cb2d8f735e..2012d6ed672 100755 --- a/.github/workflows/source/check_inputs.py +++ b/.github/workflows/source/check_inputs.py @@ -27,7 +27,7 @@ # skip lines related to other function arguments # NOTE: update range call to reflect changes # in the interface of 'add_warpx_test' - for _ in range(3): + for _ in range(2): # skip over: dims, numprocs next(f) # strip leading whitespaces, remove end-of-line comments testinput = next(f).lstrip().split(" ")[0] diff --git a/CMakeLists.txt b/CMakeLists.txt index d20de57f81c..3b4e9199f53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ include(CMakeDependentOption) option(WarpX_APP "Build the WarpX executable application" ON) option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) option(WarpX_CATALYST "Catalyst in situ diagnostics" OFF) -option(WarpX_EB "Embedded boundary support" OFF) +option(WarpX_EB "Embedded boundary support" ON) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index fd57b61fa17..8b85976c6f0 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -103,7 +103,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as test_1d_laser_acceleration # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_acceleration # inputs analysis.py # analysis diags/diag1000100 # output (plotfile) @@ -118,7 +117,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as test_2d_laser_acceleration_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_acceleration_picmi.py # inputs analysis.py # analysis diags/diag1000100 # output (plotfile) @@ -133,7 +131,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as test_3d_laser_acceleration_restart # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_acceleration_restart # inputs analysis_default_restart.py # analysis diags/diag1000100 # output (plotfile) @@ -150,7 +147,6 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as test_rz_laser_acceleration_picmi # name RZ # dims 2 # nprocs - OFF # eb "inputs_test_rz_laser_acceleration_picmi.py --test --dir 1" # inputs analysis.py # analysis diags/diag1/ # output (openPMD time series) diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 959c8cb6ad6..60d9eecc2b4 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -88,7 +88,7 @@ CMake Option Default & Values Descr ``WarpX_CATALYST`` ON/**OFF** Catalyst in situ visualization ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WarpX_DIMS`` **3**/2/1/RZ Simulation dimensionality. Use ``"1;2;RZ;3"`` for all. -``WarpX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) +``WarpX_EB`` **ON**/OFF Embedded boundary support (not supported in RZ yet) ``WarpX_IPO`` ON/**OFF** Compile WarpX with interprocedural optimization (aka LTO) ``WarpX_LIB`` ON/**OFF** Build WarpX as a library, e.g., for PICMI Python ``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) @@ -269,7 +269,7 @@ Environment Variable Default & Values Descr ============================= ============================================ ================================================================ ``WARPX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WARPX_DIMS`` ``"1;2;3;RZ"`` Simulation dimensionalities (semicolon-separated list) -``WARPX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) +``WARPX_EB`` **ON**/OFF Embedded boundary support (not supported in RZ yet) ``WARPX_MPI`` ON/**OFF** Multi-node support (message-passing) ``WARPX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 980fb1ef2e0..1d9c0c14bbf 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -590,14 +590,15 @@ In WarpX, the embedded boundary can be defined in either of two ways: A function of `x`, `y`, `z` that defines the surface of the embedded boundary. That surface lies where the function value is 0 ; the physics simulation area is where the function value is negative ; - the interior of the embeddded boundary is where the function value is positive. + the interior of the embedded boundary is where the function value is positive. - **From an STL file:** In that case, you will need to set the following parameters in the input file. * ``eb2.stl_file`` (`string`) - The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``, - in order for the file to be read by WarpX. + The path to an `STL file `__. + In addition, you also need to set ``eb2.geom_type = stl``, in order for the file to be read by WarpX. + `See the AMReX documentation for more details `__. Whether the embedded boundary is defined with an analytical function or an STL file, you can additionally define the electric potential at the embedded boundary with an analytical function: diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index f2898b557f4..7ebb1465be4 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -20,7 +20,6 @@ endif() # name: unique name of this test # dims: 1,2,RZ,3 # nprocs: 1 or 2 (maybe refactor later on to just depend on WarpX_MPI) -# eb: needs EB support? (temporary until handled as runtime parameter) # inputs: inputs file or PICMI script, WarpX_MPI decides w/ or w/o MPI # analysis: analysis script, always run without MPI # output: output file(s) to analyze @@ -30,7 +29,6 @@ function(add_warpx_test name dims nprocs - eb inputs analysis output @@ -42,17 +40,6 @@ function(add_warpx_test return() endif() - # cannot run EB tests w/o EB build - if(eb AND NOT WarpX_EB) - message(WARNING "${name}: cannot run EB tests without EB build") - return() - endif() - - # do not run no-EB tests w/ EB build - if(NOT eb AND WarpX_EB) - return() - endif() - # cannot run tests with unsupported geometry if(NOT dims IN_LIST WarpX_DIMS) return() diff --git a/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt b/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt index 793675efaba..0b34eeff865 100644 --- a/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt +++ b/Examples/Physics_applications/beam_beam_collision/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_beam_beam_collision # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_beam_beam_collision # inputs analysis_default_openpmd_regression.py # analysis diags/diag1/ # output diff --git a/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt b/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt index 4f67131556e..5af1d0a0664 100644 --- a/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt +++ b/Examples/Physics_applications/capacitive_discharge/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_background_mcc_picmi # name 1 # dims 2 # nprocs - OFF # eb "inputs_base_1d_picmi.py --test --pythonsolver" # inputs analysis_1d.py # analysis diags/diag1000050 # output @@ -16,7 +15,6 @@ add_warpx_test( test_1d_dsmc_picmi # name 1 # dims 2 # nprocs - OFF # eb "inputs_base_1d_picmi.py --test --dsmc" # inputs analysis_dsmc.py # analysis diags/diag1000050 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_background_mcc # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_background_mcc # inputs analysis_default_regression.py # analysis diags/diag1000050 # output @@ -39,8 +36,7 @@ add_warpx_test( # test_2d_background_mcc_dp_psp # name # 2 # dims # 2 # nprocs -# OFF # eb -# inputs_test_2d_background_mcc_dp_psp # inputs +## inputs_test_2d_background_mcc_dp_psp # inputs # analysis_default_regression.py # analysis # diags/diag1000050 # output # OFF # dependency @@ -50,7 +46,6 @@ add_warpx_test( test_2d_background_mcc_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_background_mcc_picmi.py # inputs analysis_2d.py # analysis diags/diag1000050 # output diff --git a/Examples/Physics_applications/laser_acceleration/CMakeLists.txt b/Examples/Physics_applications/laser_acceleration/CMakeLists.txt index 9f4a5f1dc58..1a09a669a6d 100644 --- a/Examples/Physics_applications/laser_acceleration/CMakeLists.txt +++ b/Examples/Physics_applications/laser_acceleration/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_laser_acceleration # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_acceleration # inputs analysis_default_regression.py # analysis diags/diag1000100 # output @@ -16,7 +15,6 @@ add_warpx_test( test_1d_laser_acceleration_fluid # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_acceleration_fluid # inputs analysis_1d_fluid.py # analysis diags/diag1040000 # output @@ -27,7 +25,6 @@ add_warpx_test( test_1d_laser_acceleration_fluid_boosted # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_acceleration_fluid_boosted # inputs analysis_1d_fluid_boosted.py # analysis diags/diag1000001 # output @@ -38,7 +35,6 @@ add_warpx_test( test_1d_laser_acceleration_picmi # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_acceleration_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000100 # output @@ -49,7 +45,6 @@ add_warpx_test( test_2d_laser_acceleration_boosted # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_acceleration_boosted # inputs analysis_default_regression.py # analysis diags/diag1000002 # output @@ -60,7 +55,6 @@ add_warpx_test( test_2d_laser_acceleration_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_acceleration_mr # inputs analysis_default_regression.py # analysis diags/diag1000200 # output @@ -71,7 +65,6 @@ add_warpx_test( test_2d_laser_acceleration_mr_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_acceleration_mr_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000200 # output @@ -82,7 +75,6 @@ add_warpx_test( test_2d_refined_injection # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_refined_injection # inputs analysis_refined_injection.py # analysis diags/diag1000200 # output @@ -93,7 +85,6 @@ add_warpx_test( test_3d_laser_acceleration # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_acceleration # inputs analysis_default_openpmd_regression.py # analysis diags/diag1/ # output @@ -104,7 +95,6 @@ add_warpx_test( test_3d_laser_acceleration_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_acceleration_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000100 # output @@ -115,7 +105,6 @@ add_warpx_test( test_3d_laser_acceleration_single_precision_comms # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_acceleration_single_precision_comms # inputs analysis_default_openpmd_regression.py # analysis diags/diag1/ # output @@ -126,7 +115,6 @@ add_warpx_test( test_rz_laser_acceleration # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_laser_acceleration # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -137,7 +125,6 @@ add_warpx_test( test_rz_laser_acceleration_opmd # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_laser_acceleration_opmd # inputs analysis_openpmd_rz.py # analysis diags/diag1/ # output @@ -148,7 +135,6 @@ add_warpx_test( test_rz_laser_acceleration_picmi # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_laser_acceleration_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000010 # output diff --git a/Examples/Physics_applications/laser_ion/CMakeLists.txt b/Examples/Physics_applications/laser_ion/CMakeLists.txt index ba51e4d1398..f05203de0e8 100644 --- a/Examples/Physics_applications/laser_ion/CMakeLists.txt +++ b/Examples/Physics_applications/laser_ion/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_laser_ion_acc # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_ion_acc # inputs analysis_default_openpmd_regression.py # analysis diags/diag1/ # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_laser_ion_acc_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_ion_acc_picmi.py # inputs analysis_default_openpmd_regression.py # analysis diags/diag1/ # output diff --git a/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt b/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt index ec3e4b09563..00a0f80b457 100644 --- a/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt +++ b/Examples/Physics_applications/plasma_acceleration/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_plasma_acceleration_picmi # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_plasma_acceleration_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1001000 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_plasma_acceleration_boosted # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_plasma_acceleration_boosted # inputs analysis_default_regression.py # analysis diags/diag1000020 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_plasma_acceleration_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_plasma_acceleration_mr # inputs analysis_default_regression.py # analysis diags/diag1000400 # output @@ -38,7 +35,6 @@ add_warpx_test( test_2d_plasma_acceleration_mr_momentum_conserving # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_plasma_acceleration_mr_momentum_conserving # inputs analysis_default_regression.py # analysis diags/diag1000400 # output @@ -49,7 +45,6 @@ add_warpx_test( test_3d_plasma_acceleration_boosted # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_acceleration_boosted # inputs analysis_default_regression.py # analysis diags/diag1000005 # output @@ -60,7 +55,6 @@ add_warpx_test( test_3d_plasma_acceleration_boosted_hybrid # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_acceleration_boosted_hybrid # inputs analysis_default_regression.py # analysis diags/diag1000025 # output @@ -71,7 +65,6 @@ add_warpx_test( test_3d_plasma_acceleration_mr_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_acceleration_mr_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000002 # output @@ -82,7 +75,6 @@ add_warpx_test( test_3d_plasma_acceleration_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_acceleration_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000010 # output diff --git a/Examples/Physics_applications/plasma_mirror/CMakeLists.txt b/Examples/Physics_applications/plasma_mirror/CMakeLists.txt index b90e775a4b5..073245f758a 100644 --- a/Examples/Physics_applications/plasma_mirror/CMakeLists.txt +++ b/Examples/Physics_applications/plasma_mirror/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_plasma_mirror # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_plasma_mirror # inputs analysis_default_regression.py # analysis diags/diag1000020 # output diff --git a/Examples/Physics_applications/spacecraft_charging/CMakeLists.txt b/Examples/Physics_applications/spacecraft_charging/CMakeLists.txt index 181304e9193..95349e525cc 100644 --- a/Examples/Physics_applications/spacecraft_charging/CMakeLists.txt +++ b/Examples/Physics_applications/spacecraft_charging/CMakeLists.txt @@ -6,7 +6,6 @@ add_warpx_test( test_rz_spacecraft_charging_picmi # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_spacecraft_charging_picmi.py # inputs analysis.py # analysis diags/diag1/ # output diff --git a/Examples/Physics_applications/uniform_plasma/CMakeLists.txt b/Examples/Physics_applications/uniform_plasma/CMakeLists.txt index f654dc79063..79dec989c1f 100644 --- a/Examples/Physics_applications/uniform_plasma/CMakeLists.txt +++ b/Examples/Physics_applications/uniform_plasma/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_uniform_plasma # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_uniform_plasma # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_uniform_plasma # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_uniform_plasma # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_uniform_plasma_restart # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_uniform_plasma_restart # inputs analysis_default_restart.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/accelerator_lattice/CMakeLists.txt b/Examples/Tests/accelerator_lattice/CMakeLists.txt index 7fc6b4dc8e4..f3a28d30d4a 100644 --- a/Examples/Tests/accelerator_lattice/CMakeLists.txt +++ b/Examples/Tests/accelerator_lattice/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_hard_edged_quadrupoles # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_hard_edged_quadrupoles # inputs analysis.py # analysis diags/diag1000050 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_hard_edged_quadrupoles_boosted # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_hard_edged_quadrupoles_boosted # inputs analysis.py # analysis diags/diag1000050 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_hard_edged_quadrupoles_moving # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_hard_edged_quadrupoles_moving # inputs analysis.py # analysis diags/diag1000050 # output diff --git a/Examples/Tests/boosted_diags/CMakeLists.txt b/Examples/Tests/boosted_diags/CMakeLists.txt index f0a6ceaf397..8deb7f2bee2 100644 --- a/Examples/Tests/boosted_diags/CMakeLists.txt +++ b/Examples/Tests/boosted_diags/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_laser_acceleration_btd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_acceleration_btd # inputs analysis.py # analysis diags/diag1000003 # output diff --git a/Examples/Tests/boundaries/CMakeLists.txt b/Examples/Tests/boundaries/CMakeLists.txt index 928b4b95071..fccd45e2ebf 100644 --- a/Examples/Tests/boundaries/CMakeLists.txt +++ b/Examples/Tests/boundaries/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_particle_boundaries # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_particle_boundaries # inputs analysis.py # analysis diags/diag1000008 # output diff --git a/Examples/Tests/btd_rz/CMakeLists.txt b/Examples/Tests/btd_rz/CMakeLists.txt index 15a01eb1680..6a85f653c65 100644 --- a/Examples/Tests/btd_rz/CMakeLists.txt +++ b/Examples/Tests/btd_rz/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_rz_btd # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_btd # inputs analysis.py # analysis diags/diag1000289 # output diff --git a/Examples/Tests/collider_relevant_diags/CMakeLists.txt b/Examples/Tests/collider_relevant_diags/CMakeLists.txt index ad999477507..338f66970bc 100644 --- a/Examples/Tests/collider_relevant_diags/CMakeLists.txt +++ b/Examples/Tests/collider_relevant_diags/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_collider_diagnostics # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_collider_diagnostics # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/collision/CMakeLists.txt b/Examples/Tests/collision/CMakeLists.txt index 4293ba248e7..36f8a1cb1d6 100644 --- a/Examples/Tests/collision/CMakeLists.txt +++ b/Examples/Tests/collision/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_collision_z # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_collision_z # inputs analysis_collision_1d.py # analysis diags/diag1000600 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_collision_xz # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_collision_xz # inputs analysis_collision_2d.py # analysis diags/diag1000150 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_collision_xz_picmi # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_collision_xz_picmi.py # inputs analysis_collision_2d.py # analysis diags/diag1000150 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_collision_iso # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_collision_iso # inputs analysis_collision_3d_isotropization.py # analysis diags/diag1000100 # output @@ -49,7 +45,6 @@ add_warpx_test( test_3d_collision_xyz # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_collision_xyz # inputs analysis_collision_3d.py # analysis diags/diag1000150 # output @@ -60,7 +55,6 @@ add_warpx_test( test_rz_collision # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_collision # inputs analysis_collision_rz.py # analysis diags/diag1000150 # output diff --git a/Examples/Tests/diff_lumi_diag/CMakeLists.txt b/Examples/Tests/diff_lumi_diag/CMakeLists.txt index 2385a758fb6..1651d74115e 100644 --- a/Examples/Tests/diff_lumi_diag/CMakeLists.txt +++ b/Examples/Tests/diff_lumi_diag/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_diff_lumi_diag # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_diff_lumi_diag # inputs analysis.py # analysis diags/diag1000080 # output diff --git a/Examples/Tests/divb_cleaning/CMakeLists.txt b/Examples/Tests/divb_cleaning/CMakeLists.txt index f0a8162212f..d4aae31472e 100644 --- a/Examples/Tests/divb_cleaning/CMakeLists.txt +++ b/Examples/Tests/divb_cleaning/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_divb_cleaning # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_divb_cleaning # inputs analysis.py # analysis diags/diag1000400 # output diff --git a/Examples/Tests/dive_cleaning/CMakeLists.txt b/Examples/Tests/dive_cleaning/CMakeLists.txt index 1e72305b673..c23c2aef539 100644 --- a/Examples/Tests/dive_cleaning/CMakeLists.txt +++ b/Examples/Tests/dive_cleaning/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_dive_cleaning # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_dive_cleaning # inputs analysis.py # analysis diags/diag1000128 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_dive_cleaning # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_dive_cleaning # inputs analysis.py # analysis diags/diag1000128 # output diff --git a/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt b/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt index 93e837d4b59..1325d1a6bf5 100644 --- a/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt +++ b/Examples/Tests/electrostatic_dirichlet_bc/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_dirichlet_bc # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_dirichlet_bc # inputs analysis.py # analysis diags/diag1000100 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_dirichlet_bc_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_dirichlet_bc_picmi.py # inputs analysis.py # analysis diags/diag1000100 # output diff --git a/Examples/Tests/electrostatic_sphere/CMakeLists.txt b/Examples/Tests/electrostatic_sphere/CMakeLists.txt index e80beb08e97..41a151b7884 100644 --- a/Examples/Tests/electrostatic_sphere/CMakeLists.txt +++ b/Examples/Tests/electrostatic_sphere/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_electrostatic_sphere # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_electrostatic_sphere # inputs analysis_electrostatic_sphere.py # analysis diags/diag1000030 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_electrostatic_sphere_lab_frame # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_electrostatic_sphere_lab_frame # inputs analysis_electrostatic_sphere.py # analysis diags/diag1000030 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_electrostatic_sphere_lab_frame_mr_emass_10 # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_electrostatic_sphere_lab_frame_mr_emass_10 # inputs analysis_electrostatic_sphere.py # analysis diags/diag1000002 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_electrostatic_sphere_rel_nodal # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_electrostatic_sphere_rel_nodal # inputs analysis_electrostatic_sphere.py # analysis diags/diag1000030 # output @@ -49,7 +45,6 @@ add_warpx_test( test_rz_electrostatic_sphere # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_electrostatic_sphere # inputs analysis_electrostatic_sphere.py # analysis diags/diag1000030 # output diff --git a/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt b/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt index ad5e8974225..7f7b1389119 100644 --- a/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt +++ b/Examples/Tests/electrostatic_sphere_eb/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_electrostatic_sphere_eb # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_electrostatic_sphere_eb # inputs analysis.py # analysis diags/diag1000001 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_3d_electrostatic_sphere_eb_mixed_bc # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_electrostatic_sphere_eb_mixed_bc # inputs analysis_default_regression.py # analysis diags/diag1000001 # output @@ -32,7 +30,6 @@ if(WarpX_EB) test_3d_electrostatic_sphere_eb_picmi # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_electrostatic_sphere_eb_picmi.py # inputs analysis.py # analysis diags/diag1000002 # output @@ -45,7 +42,6 @@ if(WarpX_EB) test_rz_electrostatic_sphere_eb # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_electrostatic_sphere_eb # inputs analysis_rz.py # analysis diags/diag1000001 # output @@ -58,7 +54,6 @@ if(WarpX_EB) test_rz_electrostatic_sphere_eb_mr # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_electrostatic_sphere_eb_mr # inputs analysis_rz_mr.py # analysis diags/diag1/ # output diff --git a/Examples/Tests/embedded_boundary_cube/CMakeLists.txt b/Examples/Tests/embedded_boundary_cube/CMakeLists.txt index 3fd0a0f4c3b..0044ed04ec8 100644 --- a/Examples/Tests/embedded_boundary_cube/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_cube/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_2d_embedded_boundary_cube # name 2 # dims 1 # nprocs - ON # eb inputs_test_2d_embedded_boundary_cube # inputs analysis_fields_2d.py # analysis diags/diag1000114 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_3d_embedded_boundary_cube # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_embedded_boundary_cube # inputs analysis_fields.py # analysis diags/diag1000208 # output @@ -32,7 +30,6 @@ if(WarpX_EB) test_3d_embedded_boundary_cube_macroscopic # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_embedded_boundary_cube_macroscopic # inputs analysis_fields.py # analysis diags/diag1000208 # output diff --git a/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt b/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt index d91a94b539b..6297cf1fa5c 100644 --- a/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_diffraction/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_rz_embedded_boundary_diffraction # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_embedded_boundary_diffraction # inputs analysis_fields.py # analysis diags/diag1/ # output diff --git a/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt b/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt index cf45d9d56f3..fe820c76f22 100644 --- a/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_python_api/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_embedded_boundary_picmi # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_embedded_boundary_picmi.py # inputs analysis.py # analysis diags/diag1000002 # output diff --git a/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt b/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt index c9d3b47cece..fcfe97905d8 100644 --- a/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt +++ b/Examples/Tests/embedded_boundary_rotated_cube/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_2d_embedded_boundary_rotated_cube # name 2 # dims 1 # nprocs - ON # eb inputs_test_2d_embedded_boundary_rotated_cube # inputs analysis_fields_2d.py # analysis diags/diag1000068 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_3d_embedded_boundary_rotated_cube # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_embedded_boundary_rotated_cube # inputs analysis_fields_3d.py # analysis diags/diag1000111 # output diff --git a/Examples/Tests/embedded_circle/CMakeLists.txt b/Examples/Tests/embedded_circle/CMakeLists.txt index 9eb8f23460b..4b9ee426569 100644 --- a/Examples/Tests/embedded_circle/CMakeLists.txt +++ b/Examples/Tests/embedded_circle/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_2d_embedded_circle # name 2 # dims 2 # nprocs - ON # eb inputs_test_2d_embedded_circle # inputs analysis.py # analysis diags/diag1000011 diff --git a/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt b/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt index 13012e7605b..c89d439b75e 100644 --- a/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt +++ b/Examples/Tests/energy_conserving_thermal_plasma/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_energy_conserving_thermal_plasma # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_energy_conserving_thermal_plasma # inputs analysis.py # analysis diags/diag1000500 # output diff --git a/Examples/Tests/field_probe/CMakeLists.txt b/Examples/Tests/field_probe/CMakeLists.txt index 4ef61237775..bbddbd7839e 100644 --- a/Examples/Tests/field_probe/CMakeLists.txt +++ b/Examples/Tests/field_probe/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_2d_field_probe # name 2 # dims 2 # nprocs - ON # eb inputs_test_2d_field_probe # inputs analysis.py # analysis diags/diag1000544 # output diff --git a/Examples/Tests/flux_injection/CMakeLists.txt b/Examples/Tests/flux_injection/CMakeLists.txt index 306ff2018bc..d09b83d7618 100644 --- a/Examples/Tests/flux_injection/CMakeLists.txt +++ b/Examples/Tests/flux_injection/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_flux_injection # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_flux_injection # inputs analysis_flux_injection_3d.py # analysis diags/diag1000002 # output @@ -16,7 +15,6 @@ add_warpx_test( test_rz_flux_injection # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_flux_injection # inputs analysis_flux_injection_rz.py # analysis diags/diag1000120 # output diff --git a/Examples/Tests/gaussian_beam/CMakeLists.txt b/Examples/Tests/gaussian_beam/CMakeLists.txt index 35ec08c10e3..ae0cf57ed15 100644 --- a/Examples/Tests/gaussian_beam/CMakeLists.txt +++ b/Examples/Tests/gaussian_beam/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_focusing_gaussian_beam # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_focusing_gaussian_beam # inputs analysis.py # analysis diags/diag1000000 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_gaussian_beam_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_gaussian_beam_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt index 11881ae4972..dabd4de66b8 100644 --- a/Examples/Tests/implicit/CMakeLists.txt +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_semi_implicit_picard # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_semi_implicit_picard # inputs analysis_1d.py # analysis diags/diag1000100 # output @@ -16,7 +15,6 @@ add_warpx_test( test_1d_theta_implicit_picard # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_theta_implicit_picard # inputs analysis_1d.py # analysis diags/diag1000100 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_theta_implicit_jfnk_vandb # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_theta_implicit_jfnk_vandb # inputs analysis_vandb_jfnk_2d.py # analysis diags/diag1000020 # output @@ -38,7 +35,6 @@ add_warpx_test( test_2d_theta_implicit_jfnk_vandb_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_theta_implicit_jfnk_vandb_picmi.py # inputs analysis_vandb_jfnk_2d.py # analysis diags/diag1000020 # output diff --git a/Examples/Tests/initial_distribution/CMakeLists.txt b/Examples/Tests/initial_distribution/CMakeLists.txt index 14dabd7a67c..04af9708cb2 100644 --- a/Examples/Tests/initial_distribution/CMakeLists.txt +++ b/Examples/Tests/initial_distribution/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_initial_distribution # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_initial_distribution # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/initial_plasma_profile/CMakeLists.txt b/Examples/Tests/initial_plasma_profile/CMakeLists.txt index fab15e8b97f..eb45e64dfab 100644 --- a/Examples/Tests/initial_plasma_profile/CMakeLists.txt +++ b/Examples/Tests/initial_plasma_profile/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_parabolic_channel_initialization # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_parabolic_channel_initialization # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/ion_stopping/CMakeLists.txt b/Examples/Tests/ion_stopping/CMakeLists.txt index 1f203d76fa1..83e15287e18 100644 --- a/Examples/Tests/ion_stopping/CMakeLists.txt +++ b/Examples/Tests/ion_stopping/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_ion_stopping # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_ion_stopping # inputs analysis.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/ionization/CMakeLists.txt b/Examples/Tests/ionization/CMakeLists.txt index 32da653f301..9154173ac5f 100644 --- a/Examples/Tests/ionization/CMakeLists.txt +++ b/Examples/Tests/ionization/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_ionization_boost # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_ionization_boost # inputs analysis.py # analysis diags/diag1000420 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_ionization_lab # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_ionization_lab # inputs analysis.py # analysis diags/diag1001600 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_ionization_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_ionization_picmi.py # inputs analysis.py # analysis diags/diag1001600 # output diff --git a/Examples/Tests/langmuir/CMakeLists.txt b/Examples/Tests/langmuir/CMakeLists.txt index 1223a23e4d2..3f44d364276 100644 --- a/Examples/Tests/langmuir/CMakeLists.txt +++ b/Examples/Tests/langmuir/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_langmuir_multi # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_langmuir_multi # inputs analysis_1d.py # analysis diags/diag1000080 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_langmuir_multi_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_mr # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_langmuir_multi_mr_anisotropic # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_mr_anisotropic # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -38,7 +35,6 @@ add_warpx_test( test_2d_langmuir_multi_mr_momentum_conserving # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_mr_momentum_conserving # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -50,7 +46,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_mr_psatd # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_mr_psatd # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -62,7 +57,6 @@ add_warpx_test( test_2d_langmuir_multi_nodal # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_nodal # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -73,7 +67,6 @@ add_warpx_test( test_2d_langmuir_multi_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000040 # output @@ -85,7 +78,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -98,7 +90,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_current_correction # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_current_correction # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -111,7 +102,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_current_correction_nodal # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_current_correction_nodal # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -124,7 +114,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_momentum_conserving # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_momentum_conserving # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -137,7 +126,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_multiJ # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_multiJ # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -150,7 +138,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_multiJ_nodal # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_multiJ_nodal # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -163,7 +150,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_nodal # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_nodal # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -176,7 +162,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_vay_deposition # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_vay_deposition # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -189,7 +174,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_vay_deposition_nodal # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_vay_deposition_nodal # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -202,7 +186,6 @@ if(WarpX_FFT) test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_langmuir_multi_psatd_vay_deposition_particle_shape_4 # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -214,7 +197,6 @@ add_warpx_test( test_3d_langmuir_multi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -225,7 +207,6 @@ add_warpx_test( test_3d_langmuir_multi_nodal # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_nodal # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -236,7 +217,6 @@ add_warpx_test( test_3d_langmuir_multi_picmi # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000040 # output @@ -248,7 +228,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -261,7 +240,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_current_correction # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_current_correction # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -274,7 +252,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_current_correction_nodal # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_current_correction_nodal # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -287,7 +264,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_div_cleaning # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_div_cleaning # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -300,7 +276,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_momentum_conserving # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_momentum_conserving # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -313,7 +288,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_multiJ # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_multiJ # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -326,7 +300,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_multiJ_nodal # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_multiJ_nodal # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -339,7 +312,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_nodal # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_nodal # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -352,7 +324,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_vay_deposition # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_vay_deposition # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -365,7 +336,6 @@ if(WarpX_FFT) test_3d_langmuir_multi_psatd_vay_deposition_nodal # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_langmuir_multi_psatd_vay_deposition_nodal # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -377,7 +347,6 @@ add_warpx_test( test_rz_langmuir_multi # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_langmuir_multi # inputs analysis_rz.py # analysis diags/diag1000080 # output @@ -388,7 +357,6 @@ add_warpx_test( test_rz_langmuir_multi_picmi # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_langmuir_multi_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000040 # output @@ -400,7 +368,6 @@ if(WarpX_FFT) test_rz_langmuir_multi_psatd # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_langmuir_multi_psatd # inputs analysis_rz.py # analysis diags/diag1000080 # output @@ -413,7 +380,6 @@ if(WarpX_FFT) test_rz_langmuir_multi_psatd_current_correction # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_langmuir_multi_psatd_current_correction # inputs analysis_rz.py # analysis diags/diag1000080 # output @@ -426,7 +392,6 @@ if(WarpX_FFT) test_rz_langmuir_multi_psatd_multiJ # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_langmuir_multi_psatd_multiJ # inputs analysis_rz.py # analysis diags/diag1000080 # output diff --git a/Examples/Tests/langmuir_fluids/CMakeLists.txt b/Examples/Tests/langmuir_fluids/CMakeLists.txt index 8f3ab3ebc78..054e9c80d3a 100644 --- a/Examples/Tests/langmuir_fluids/CMakeLists.txt +++ b/Examples/Tests/langmuir_fluids/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_langmuir_fluid # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_langmuir_fluid # inputs analysis_1d.py # analysis diags/diag1000080 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_langmuir_fluid # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_langmuir_fluid # inputs analysis_2d.py # analysis diags/diag1000080 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_langmuir_fluid # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_langmuir_fluid # inputs analysis_3d.py # analysis diags/diag1000040 # output @@ -38,7 +35,6 @@ add_warpx_test( test_rz_langmuir_fluid # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_langmuir_fluid # inputs analysis_rz.py # analysis diags/diag1000080 # output diff --git a/Examples/Tests/larmor/CMakeLists.txt b/Examples/Tests/larmor/CMakeLists.txt index 3ddcc394c98..6a3368a4fca 100644 --- a/Examples/Tests/larmor/CMakeLists.txt +++ b/Examples/Tests/larmor/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_larmor # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_larmor # inputs analysis_default_regression.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/laser_injection/CMakeLists.txt b/Examples/Tests/laser_injection/CMakeLists.txt index 577b8bdcebc..cec027deb70 100644 --- a/Examples/Tests/laser_injection/CMakeLists.txt +++ b/Examples/Tests/laser_injection/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_laser_injection # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_laser_injection # inputs analysis_1d.py # analysis diags/diag1000240 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_laser_injection # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_injection # inputs analysis_2d.py # analysis diags/diag1000240 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_laser_injection # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_laser_injection # inputs analysis_3d.py # analysis diags/diag1000020 # output diff --git a/Examples/Tests/laser_injection_from_file/CMakeLists.txt b/Examples/Tests/laser_injection_from_file/CMakeLists.txt index a4f09f6895d..4b4024b9029 100644 --- a/Examples/Tests/laser_injection_from_file/CMakeLists.txt +++ b/Examples/Tests/laser_injection_from_file/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_laser_injection_from_lasy_file_prepare # name 1 # dims 1 # nprocs - OFF # eb inputs_test_1d_laser_injection_from_lasy_file_prepare.py # inputs OFF # analysis OFF # output @@ -16,7 +15,6 @@ add_warpx_test( test_1d_laser_injection_from_lasy_file # name 1 # dims 1 # nprocs - OFF # eb inputs_test_1d_laser_injection_from_lasy_file # inputs analysis_1d.py # analysis diags/diag1000251 # output @@ -27,7 +25,6 @@ add_warpx_test( test_1d_laser_injection_from_lasy_file_boost_prepare # name 1 # dims 1 # nprocs - OFF # eb inputs_test_1d_laser_injection_from_lasy_file_boost_prepare.py # inputs OFF # analysis OFF # output @@ -38,7 +35,6 @@ add_warpx_test( test_1d_laser_injection_from_lasy_file_boost # name 1 # dims 1 # nprocs - OFF # eb inputs_test_1d_laser_injection_from_lasy_file_boost # inputs analysis_1d_boost.py # analysis diags/diag1000001 # output @@ -49,7 +45,6 @@ add_warpx_test( test_2d_laser_injection_from_binary_file_prepare # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_laser_injection_from_binary_file_prepare.py # inputs OFF # analysis OFF # output @@ -60,7 +55,6 @@ add_warpx_test( test_2d_laser_injection_from_binary_file # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_laser_injection_from_binary_file # inputs analysis_2d_binary.py # analysis diags/diag1000250 # output @@ -71,7 +65,6 @@ add_warpx_test( test_2d_laser_injection_from_lasy_file_prepare # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_laser_injection_from_lasy_file_prepare.py # inputs OFF # analysis OFF # output @@ -82,7 +75,6 @@ add_warpx_test( test_2d_laser_injection_from_lasy_file # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_laser_injection_from_lasy_file # inputs analysis_2d.py # analysis diags/diag1000251 # output @@ -93,7 +85,6 @@ add_warpx_test( test_3d_laser_injection_from_lasy_file_prepare # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_laser_injection_from_lasy_file_prepare.py # inputs OFF # analysis OFF # output @@ -104,7 +95,6 @@ add_warpx_test( test_3d_laser_injection_from_lasy_file # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_laser_injection_from_lasy_file # inputs analysis_3d.py # analysis diags/diag1000251 # output @@ -115,7 +105,6 @@ add_warpx_test( test_rz_laser_injection_from_lasy_file_prepare # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_laser_injection_from_lasy_file_prepare.py # inputs OFF # analysis OFF # output @@ -126,7 +115,6 @@ add_warpx_test( test_rz_laser_injection_from_lasy_file # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_laser_injection_from_lasy_file # inputs analysis_rz.py # analysis diags/diag1000252 # output @@ -137,7 +125,6 @@ add_warpx_test( test_rz_laser_injection_from_RZ_lasy_file_prepare # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_laser_injection_from_RZ_lasy_file_prepare.py # inputs OFF # analysis OFF # output @@ -148,7 +135,6 @@ add_warpx_test( test_rz_laser_injection_from_RZ_lasy_file # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_laser_injection_from_RZ_lasy_file # inputs analysis_from_RZ_file.py # analysis diags/diag1000612 # output diff --git a/Examples/Tests/laser_on_fine/CMakeLists.txt b/Examples/Tests/laser_on_fine/CMakeLists.txt index 794d5e68c66..479374137df 100644 --- a/Examples/Tests/laser_on_fine/CMakeLists.txt +++ b/Examples/Tests/laser_on_fine/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_laser_on_fine # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_laser_on_fine # inputs analysis_default_regression.py # analysis diags/diag1000050 # output diff --git a/Examples/Tests/load_external_field/CMakeLists.txt b/Examples/Tests/load_external_field/CMakeLists.txt index 93b0a1436be..0713dc877df 100644 --- a/Examples/Tests/load_external_field/CMakeLists.txt +++ b/Examples/Tests/load_external_field/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_load_external_field_grid_picmi # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_load_external_field_grid_picmi.py # inputs analysis_3d.py # analysis diags/diag1000300 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_load_external_field_particle_picmi # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_load_external_field_particle_picmi.py # inputs analysis_3d.py # analysis diags/diag1000300 # output @@ -27,7 +25,6 @@ add_warpx_test( test_rz_load_external_field_grid # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_load_external_field_grid # inputs analysis_rz.py # analysis diags/diag1000300 # output @@ -38,7 +35,6 @@ add_warpx_test( test_rz_load_external_field_grid_restart # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_load_external_field_grid_restart # inputs analysis_default_restart.py # analysis diags/diag1000300 # output @@ -49,7 +45,6 @@ add_warpx_test( test_rz_load_external_field_particles # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_load_external_field_particles # inputs analysis_rz.py # analysis diags/diag1000300 # output @@ -60,7 +55,6 @@ add_warpx_test( test_rz_load_external_field_particles_restart # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_load_external_field_particles_restart # inputs analysis_default_restart.py # analysis diags/diag1000300 # output diff --git a/Examples/Tests/magnetostatic_eb/CMakeLists.txt b/Examples/Tests/magnetostatic_eb/CMakeLists.txt index db97a6e11c2..3eb2da03136 100644 --- a/Examples/Tests/magnetostatic_eb/CMakeLists.txt +++ b/Examples/Tests/magnetostatic_eb/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_magnetostatic_eb # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_magnetostatic_eb # inputs analysis_default_regression.py # analysis diags/diag1000001 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_3d_magnetostatic_eb_picmi # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_magnetostatic_eb_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000001 # output @@ -32,7 +30,6 @@ if(WarpX_EB) test_rz_magnetostatic_eb_picmi # name RZ # dims 1 # nprocs - ON # eb inputs_test_rz_magnetostatic_eb_picmi.py # inputs analysis_rz.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt b/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt index 9e315b7536d..2c65c0a6ecb 100644 --- a/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt +++ b/Examples/Tests/maxwell_hybrid_qed/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_FFT) test_2d_maxwell_hybrid_qed_solver # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_maxwell_hybrid_qed_solver # inputs analysis.py # analysis diags/diag1000300 # output diff --git a/Examples/Tests/nci_fdtd_stability/CMakeLists.txt b/Examples/Tests/nci_fdtd_stability/CMakeLists.txt index 73d0f38beec..e58e5bfb58f 100644 --- a/Examples/Tests/nci_fdtd_stability/CMakeLists.txt +++ b/Examples/Tests/nci_fdtd_stability/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_nci_corrector # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_nci_corrector # inputs analysis_ncicorr.py # analysis diags/diag1000600 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_nci_corrector_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_nci_corrector_mr # inputs analysis_ncicorr.py # analysis diags/diag1000600 # output diff --git a/Examples/Tests/nci_psatd_stability/CMakeLists.txt b/Examples/Tests/nci_psatd_stability/CMakeLists.txt index 6a27abdc783..ed087fc4190 100644 --- a/Examples/Tests/nci_psatd_stability/CMakeLists.txt +++ b/Examples/Tests/nci_psatd_stability/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_FFT) test_2d_averaged_galilean_psatd # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_averaged_galilean_psatd # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -19,7 +18,6 @@ if(WarpX_FFT) test_2d_averaged_galilean_psatd_hybrid # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_averaged_galilean_psatd_hybrid # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -32,7 +30,6 @@ if(WarpX_FFT) test_2d_comoving_psatd_hybrid # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_comoving_psatd_hybrid # inputs analysis_default_regression.py # analysis diags/diag1000400 # output @@ -45,7 +42,6 @@ if(WarpX_FFT) test_2d_galilean_psatd # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_galilean_psatd # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -58,7 +54,6 @@ if(WarpX_FFT) test_2d_galilean_psatd_current_correction # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_galilean_psatd_current_correction # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -71,7 +66,6 @@ if(WarpX_FFT) test_2d_galilean_psatd_current_correction_psb # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_galilean_psatd_current_correction_psb # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -84,7 +78,6 @@ if(WarpX_FFT) test_2d_galilean_psatd_hybrid # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_galilean_psatd_hybrid # inputs analysis_default_regression.py # analysis diags/diag1000400 # output @@ -97,7 +90,6 @@ if(WarpX_FFT) test_3d_averaged_galilean_psatd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_averaged_galilean_psatd # inputs analysis_galilean.py # analysis diags/diag1000160 # output @@ -110,7 +102,6 @@ if(WarpX_FFT) test_3d_averaged_galilean_psatd_hybrid # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_averaged_galilean_psatd_hybrid # inputs analysis_galilean.py # analysis diags/diag1000160 # output @@ -123,7 +114,6 @@ if(WarpX_FFT) test_3d_galilean_psatd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_galilean_psatd # inputs analysis_galilean.py # analysis diags/diag1000300 # output @@ -136,7 +126,6 @@ if(WarpX_FFT) test_3d_galilean_psatd_current_correction # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_galilean_psatd_current_correction # inputs analysis_galilean.py # analysis diags/diag1000300 # output @@ -149,7 +138,6 @@ if(WarpX_FFT) test_3d_galilean_psatd_current_correction_psb # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_galilean_psatd_current_correction_psb # inputs analysis_galilean.py # analysis diags/diag1000300 # output @@ -162,7 +150,6 @@ if(WarpX_FFT) test_3d_uniform_plasma_multiJ # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_uniform_plasma_multiJ # inputs analysis_multiJ.py # analysis diags/diag1000300 # output @@ -175,7 +162,6 @@ if(WarpX_FFT) test_rz_galilean_psatd # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_galilean_psatd # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -188,7 +174,6 @@ if(WarpX_FFT) test_rz_galilean_psatd_current_correction # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_galilean_psatd_current_correction # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -201,7 +186,6 @@ if(WarpX_FFT) test_rz_galilean_psatd_current_correction_psb # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_galilean_psatd_current_correction_psb # inputs analysis_galilean.py # analysis diags/diag1000400 # output @@ -214,7 +198,6 @@ if(WarpX_FFT) test_rz_multiJ_psatd # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_multiJ_psatd # inputs analysis_default_regression.py # analysis diags/diag1000050 # output diff --git a/Examples/Tests/nodal_electrostatic/CMakeLists.txt b/Examples/Tests/nodal_electrostatic/CMakeLists.txt index 62627eb576a..915298f15ab 100644 --- a/Examples/Tests/nodal_electrostatic/CMakeLists.txt +++ b/Examples/Tests/nodal_electrostatic/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_nodal_electrostatic_solver # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_nodal_electrostatic_solver # inputs analysis.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/nuclear_fusion/CMakeLists.txt b/Examples/Tests/nuclear_fusion/CMakeLists.txt index 4ed47607c8d..c3ee8848e59 100644 --- a/Examples/Tests/nuclear_fusion/CMakeLists.txt +++ b/Examples/Tests/nuclear_fusion/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_proton_boron_fusion # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_proton_boron_fusion # inputs analysis_proton_boron_fusion.py # analysis diags/diag1000001 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_deuterium_deuterium_fusion # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_deuterium_deuterium_fusion # inputs analysis_two_product_fusion.py # analysis diags/diag1000001 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_deuterium_deuterium_fusion_intraspecies # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_deuterium_deuterium_fusion_intraspecies # inputs analysis_deuterium_deuterium_3d_intraspecies.py # analysis diags/diag1000010 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_deuterium_tritium_fusion # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_deuterium_tritium_fusion # inputs analysis_two_product_fusion.py # analysis diags/diag1000001 # output @@ -49,7 +45,6 @@ add_warpx_test( test_3d_proton_boron_fusion # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_proton_boron_fusion # inputs analysis_proton_boron_fusion.py # analysis diags/diag1000001 # output @@ -60,7 +55,6 @@ add_warpx_test( test_rz_deuterium_tritium_fusion # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_deuterium_tritium_fusion # inputs analysis_two_product_fusion.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt b/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt index ce5bed2c587..e689c83a1e4 100644 --- a/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt +++ b/Examples/Tests/ohm_solver_em_modes/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_ohm_solver_em_modes_picmi # name 1 # dims 2 # nprocs - OFF # eb "inputs_test_1d_ohm_solver_em_modes_picmi.py --test --dim 1 --bdir z" # inputs analysis.py # analysis diags/field_diag000250 # output @@ -16,7 +15,6 @@ add_warpx_test( test_rz_ohm_solver_em_modes_picmi # name RZ # dims 2 # nprocs - OFF # eb "inputs_test_rz_ohm_solver_em_modes_picmi.py --test" # inputs analysis_rz.py # analysis diags/diag1000100 # output diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt b/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt index e5017318f19..3b2d0bb794b 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_ohm_solver_landau_damping_picmi # name 2 # dims 2 # nprocs - OFF # eb "inputs_test_2d_ohm_solver_landau_damping_picmi.py --test --dim 2 --temp_ratio 0.1" # inputs analysis.py # analysis diags/diag1000100 # output diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt b/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt index a6c978ba3ef..53a9bbdeada 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt +++ b/Examples/Tests/ohm_solver_ion_beam_instability/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_ohm_solver_ion_beam_picmi # name 1 # dims 2 # nprocs - OFF # eb "inputs_test_1d_ohm_solver_ion_beam_picmi.py --test --dim 1 --resonant" # inputs analysis.py # analysis diags/diag1002500 # output diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt b/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt index 849d4c3b2a3..cef47a7d95e 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_ohm_solver_magnetic_reconnection_picmi # name 2 # dims 2 # nprocs - OFF # eb "inputs_test_2d_ohm_solver_magnetic_reconnection_picmi.py --test" # inputs analysis.py # analysis diags/diag1000020 # output diff --git a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt index 1f921ae98b2..c5ec4583da1 100644 --- a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt +++ b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_FFT) test_3d_open_bc_poisson_solver # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_open_bc_poisson_solver # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/particle_boundary_interaction/CMakeLists.txt b/Examples/Tests/particle_boundary_interaction/CMakeLists.txt index b7517ef9bc4..5bbb34c0d95 100644 --- a/Examples/Tests/particle_boundary_interaction/CMakeLists.txt +++ b/Examples/Tests/particle_boundary_interaction/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_rz_particle_boundary_interaction_picmi # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_particle_boundary_interaction_picmi.py # inputs analysis.py # analysis diags/diag1/ # output diff --git a/Examples/Tests/particle_boundary_process/CMakeLists.txt b/Examples/Tests/particle_boundary_process/CMakeLists.txt index a674c72abe3..a7081fe9090 100644 --- a/Examples/Tests/particle_boundary_process/CMakeLists.txt +++ b/Examples/Tests/particle_boundary_process/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_particle_reflection_picmi # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_particle_reflection_picmi.py # inputs analysis_reflection.py # analysis diags/diag1000010 # output @@ -17,7 +16,6 @@ if(WarpX_EB) test_3d_particle_absorption # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_particle_absorption # inputs analysis_absorption.py # analysis diags/diag1000060 # output diff --git a/Examples/Tests/particle_boundary_scrape/CMakeLists.txt b/Examples/Tests/particle_boundary_scrape/CMakeLists.txt index 361f99bfb09..9b303afcc0f 100644 --- a/Examples/Tests/particle_boundary_scrape/CMakeLists.txt +++ b/Examples/Tests/particle_boundary_scrape/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_particle_scrape # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_particle_scrape # inputs analysis_scrape.py # analysis diags/diag1000060 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_3d_particle_scrape_picmi # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_particle_scrape_picmi.py # inputs analysis_scrape.py # analysis diags/diag1000060 # output diff --git a/Examples/Tests/particle_data_python/CMakeLists.txt b/Examples/Tests/particle_data_python/CMakeLists.txt index 45bed4e9cf6..e58fe72670a 100644 --- a/Examples/Tests/particle_data_python/CMakeLists.txt +++ b/Examples/Tests/particle_data_python/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_particle_attr_access_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_particle_attr_access_picmi.py # inputs analysis.py # analysis diags/diag1000010 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_particle_attr_access_unique_picmi # name 2 # dims 2 # nprocs - OFF # eb "inputs_test_2d_particle_attr_access_picmi.py --unique" # inputs analysis.py # analysis diags/diag1000010 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_prev_positions_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_prev_positions_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/particle_fields_diags/CMakeLists.txt b/Examples/Tests/particle_fields_diags/CMakeLists.txt index b35ffe46713..a83818b6966 100644 --- a/Examples/Tests/particle_fields_diags/CMakeLists.txt +++ b/Examples/Tests/particle_fields_diags/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_particle_fields_diags # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_particle_fields_diags # inputs analysis_particle_diags.py # analysis diags/diag1000200 # output @@ -17,8 +16,7 @@ add_warpx_test( # test_3d_particle_fields_diags_single_precision # name # 3 # dims # 2 # nprocs -# OFF # eb -# inputs_test_3d_particle_fields_diags # inputs +## inputs_test_3d_particle_fields_diags # inputs # analysis_particle_diags_single.py # analysis # diags/diag1000200 # output # OFF # dependency diff --git a/Examples/Tests/particle_pusher/CMakeLists.txt b/Examples/Tests/particle_pusher/CMakeLists.txt index 583106014a5..3d8f1496587 100644 --- a/Examples/Tests/particle_pusher/CMakeLists.txt +++ b/Examples/Tests/particle_pusher/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_particle_pusher # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_particle_pusher # inputs analysis.py # analysis diags/diag1010000 # output diff --git a/Examples/Tests/particle_thermal_boundary/CMakeLists.txt b/Examples/Tests/particle_thermal_boundary/CMakeLists.txt index 26478b59c07..eeae6660e02 100644 --- a/Examples/Tests/particle_thermal_boundary/CMakeLists.txt +++ b/Examples/Tests/particle_thermal_boundary/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_particle_thermal_boundary # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_particle_thermal_boundary # inputs analysis.py # analysis diags/diag1002000 # output diff --git a/Examples/Tests/particles_in_pml/CMakeLists.txt b/Examples/Tests/particles_in_pml/CMakeLists.txt index e8f1a13601d..c1782dc4d1f 100644 --- a/Examples/Tests/particles_in_pml/CMakeLists.txt +++ b/Examples/Tests/particles_in_pml/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_particles_in_pml # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_particles_in_pml # inputs analysis_particles_in_pml.py # analysis diags/diag1000180 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_particles_in_pml_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_particles_in_pml_mr # inputs analysis_particles_in_pml.py # analysis diags/diag1000300 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_particles_in_pml # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_particles_in_pml # inputs analysis_particles_in_pml.py # analysis diags/diag1000120 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_particles_in_pml_mr # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_particles_in_pml_mr # inputs analysis_particles_in_pml.py # analysis diags/diag1000200 # output diff --git a/Examples/Tests/pass_mpi_communicator/CMakeLists.txt b/Examples/Tests/pass_mpi_communicator/CMakeLists.txt index f68986d363a..ac60636b931 100644 --- a/Examples/Tests/pass_mpi_communicator/CMakeLists.txt +++ b/Examples/Tests/pass_mpi_communicator/CMakeLists.txt @@ -9,7 +9,6 @@ add_warpx_test( test_2d_pass_mpi_comm_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pass_mpi_comm_picmi.py # inputs OFF #analysis.py # analysis OFF # output diff --git a/Examples/Tests/pec/CMakeLists.txt b/Examples/Tests/pec/CMakeLists.txt index 69c68ec5329..ec710f7d919 100644 --- a/Examples/Tests/pec/CMakeLists.txt +++ b/Examples/Tests/pec/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_pec_field # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_pec_field # inputs analysis_pec.py # analysis diags/diag1000125 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_pec_field_mr # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_pec_field_mr # inputs analysis_pec_mr.py # analysis diags/diag1000125 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_pec_particle # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_pec_particle # inputs analysis_default_regression.py # analysis diags/diag1000020 # output diff --git a/Examples/Tests/photon_pusher/CMakeLists.txt b/Examples/Tests/photon_pusher/CMakeLists.txt index 491906e0466..7926d8faeaf 100644 --- a/Examples/Tests/photon_pusher/CMakeLists.txt +++ b/Examples/Tests/photon_pusher/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_photon_pusher # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_photon_pusher # inputs analysis.py # analysis diags/diag1000050 # output diff --git a/Examples/Tests/plasma_lens/CMakeLists.txt b/Examples/Tests/plasma_lens/CMakeLists.txt index cdba552db9e..bc13ae433bc 100644 --- a/Examples/Tests/plasma_lens/CMakeLists.txt +++ b/Examples/Tests/plasma_lens/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_plasma_lens # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_lens # inputs analysis.py # analysis diags/diag1000084 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_plasma_lens_boosted # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_lens_boosted # inputs analysis.py # analysis diags/diag1000084 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_plasma_lens_hard_edged # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_lens_hard_edged # inputs analysis.py # analysis diags/diag1000084 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_plasma_lens_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_lens_picmi.py # inputs analysis.py # analysis diags/diag1000084 # output @@ -49,7 +45,6 @@ add_warpx_test( test_3d_plasma_lens_short # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_plasma_lens_short # inputs analysis.py # analysis diags/diag1000084 # output diff --git a/Examples/Tests/pml/CMakeLists.txt b/Examples/Tests/pml/CMakeLists.txt index 92847dfff24..c63412dc763 100644 --- a/Examples/Tests/pml/CMakeLists.txt +++ b/Examples/Tests/pml/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_pml_x_ckc # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_ckc # inputs analysis_pml_ckc.py # analysis diags/diag1000300 # output @@ -17,7 +16,6 @@ if(WarpX_FFT) test_2d_pml_x_galilean # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_galilean # inputs analysis_pml_psatd.py # analysis diags/diag1000300 # output @@ -30,7 +28,6 @@ if(WarpX_FFT) test_2d_pml_x_psatd # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_psatd # inputs analysis_pml_psatd.py # analysis diags/diag1000300 # output @@ -43,7 +40,6 @@ if(WarpX_FFT) test_2d_pml_x_psatd_restart # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_psatd_restart # inputs analysis_default_restart.py # analysis diags/diag1000300 # output @@ -55,7 +51,6 @@ add_warpx_test( test_2d_pml_x_yee # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_yee # inputs analysis_pml_yee.py # analysis diags/diag1000300 # output @@ -66,7 +61,6 @@ add_warpx_test( test_2d_pml_x_yee_restart # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_pml_x_yee_restart # inputs analysis_default_restart.py # analysis diags/diag1000300 # output @@ -78,7 +72,6 @@ if(WarpX_FFT) test_3d_pml_psatd_dive_divb_cleaning # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_pml_psatd_dive_divb_cleaning # inputs analysis_default_regression.py # analysis diags/diag1000100 # output @@ -91,7 +84,6 @@ if(WarpX_FFT) test_rz_pml_psatd # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_pml_psatd # inputs analysis_pml_psatd_rz.py # analysis diags/diag1000500 # output diff --git a/Examples/Tests/point_of_contact_eb/CMakeLists.txt b/Examples/Tests/point_of_contact_eb/CMakeLists.txt index 25bf4b977de..b8d7ba1131f 100644 --- a/Examples/Tests/point_of_contact_eb/CMakeLists.txt +++ b/Examples/Tests/point_of_contact_eb/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_point_of_contact_eb # name 3 # dims 2 # nprocs - ON # eb inputs_test_3d_point_of_contact_eb # inputs analysis.py # analysis diags/diag1/ # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_rz_point_of_contact_eb # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_point_of_contact_eb # inputs analysis.py # analysis diags/diag1/ # output diff --git a/Examples/Tests/projection_divb_cleaner/CMakeLists.txt b/Examples/Tests/projection_divb_cleaner/CMakeLists.txt index 91dd6bdc592..307ae7656c5 100644 --- a/Examples/Tests/projection_divb_cleaner/CMakeLists.txt +++ b/Examples/Tests/projection_divb_cleaner/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_projection_divb_cleaner_callback_picmi # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_projection_divb_cleaner_callback_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000001 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_projection_divb_cleaner_picmi # name 3 # dims 1 # nprocs - OFF # eb inputs_test_3d_projection_divb_cleaner_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000001 # output @@ -27,7 +25,6 @@ add_warpx_test( test_rz_projection_divb_cleaner # name RZ # dims 1 # nprocs - OFF # eb inputs_test_rz_projection_divb_cleaner # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/python_wrappers/CMakeLists.txt b/Examples/Tests/python_wrappers/CMakeLists.txt index 83fc6e16f7d..0045a181606 100644 --- a/Examples/Tests/python_wrappers/CMakeLists.txt +++ b/Examples/Tests/python_wrappers/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_FFT) test_2d_python_wrappers_picmi # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_python_wrappers_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000100 # output diff --git a/Examples/Tests/qed/CMakeLists.txt b/Examples/Tests/qed/CMakeLists.txt index 77690642f07..5dd786f26a1 100644 --- a/Examples/Tests/qed/CMakeLists.txt +++ b/Examples/Tests/qed/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_qed_breit_wheeler # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_qed_breit_wheeler # inputs analysis_breit_wheeler_yt.py # analysis diags/diag1000002 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_qed_breit_wheeler_opmd # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_qed_breit_wheeler_opmd # inputs analysis_breit_wheeler_opmd.py # analysis diags/diag1/ # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_qed_quantum_sync # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_qed_quantum_sync # inputs analysis_quantum_sync.py # analysis diags/diag1000002 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_qed_breit_wheeler # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_breit_wheeler # inputs analysis_breit_wheeler_yt.py # analysis diags/diag1000002 # output @@ -49,7 +45,6 @@ add_warpx_test( test_3d_qed_breit_wheeler_opmd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_breit_wheeler_opmd # inputs analysis_breit_wheeler_opmd.py # analysis diags/diag1/ # output @@ -60,7 +55,6 @@ add_warpx_test( test_3d_qed_quantum_sync # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_quantum_sync # inputs analysis_quantum_sync.py # analysis diags/diag1000002 # output @@ -71,7 +65,6 @@ add_warpx_test( test_3d_qed_schwinger_1 # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_schwinger_1 # inputs analysis_schwinger.py # analysis diags/diag1000001 # output @@ -82,7 +75,6 @@ add_warpx_test( test_3d_qed_schwinger_2 # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_schwinger_2 # inputs analysis_schwinger.py # analysis diags/diag1000001 # output @@ -93,7 +85,6 @@ add_warpx_test( test_3d_qed_schwinger_3 # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_schwinger_3 # inputs analysis_schwinger.py # analysis diags/diag1000001 # output @@ -104,7 +95,6 @@ add_warpx_test( test_3d_qed_schwinger_4 # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_qed_schwinger_4 # inputs analysis_schwinger.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/radiation_reaction/CMakeLists.txt b/Examples/Tests/radiation_reaction/CMakeLists.txt index 63814f30f29..8696cf0f9b7 100644 --- a/Examples/Tests/radiation_reaction/CMakeLists.txt +++ b/Examples/Tests/radiation_reaction/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_radiation_reaction # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_radiation_reaction # inputs analysis.py # analysis diags/diag1000064 # output diff --git a/Examples/Tests/reduced_diags/CMakeLists.txt b/Examples/Tests/reduced_diags/CMakeLists.txt index a09d5403270..cd4f6392892 100644 --- a/Examples/Tests/reduced_diags/CMakeLists.txt +++ b/Examples/Tests/reduced_diags/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_reduced_diags # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_reduced_diags # inputs analysis_reduced_diags.py # analysis diags/diag1000200 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_reduced_diags_load_balance_costs_heuristic # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_reduced_diags_load_balance_costs_heuristic # inputs analysis_reduced_diags_load_balance_costs.py # analysis diags/diag1000003 # output @@ -27,7 +25,6 @@ add_warpx_test( test_3d_reduced_diags_load_balance_costs_timers # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_reduced_diags_load_balance_costs_timers # inputs analysis_reduced_diags_load_balance_costs.py # analysis diags/diag1000003 # output @@ -38,7 +35,6 @@ add_warpx_test( test_3d_reduced_diags_load_balance_costs_timers_picmi # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_reduced_diags_load_balance_costs_timers_picmi.py # inputs analysis_reduced_diags_load_balance_costs.py # analysis diags/diag1000003 # output @@ -50,7 +46,6 @@ if(WarpX_FFT) test_3d_reduced_diags_load_balance_costs_timers_psatd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_reduced_diags_load_balance_costs_timers_psatd # inputs analysis_reduced_diags_load_balance_costs.py # analysis diags/diag1000003 # output diff --git a/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt b/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt index 9ee2a63d2d2..d89fb8b31b6 100644 --- a/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt +++ b/Examples/Tests/relativistic_space_charge_initialization/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_3d_relativistic_space_charge_initialization # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_relativistic_space_charge_initialization # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/repelling_particles/CMakeLists.txt b/Examples/Tests/repelling_particles/CMakeLists.txt index ed662b67332..056f670a860 100644 --- a/Examples/Tests/repelling_particles/CMakeLists.txt +++ b/Examples/Tests/repelling_particles/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_repelling_particles # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_repelling_particles # inputs analysis.py # analysis diags/diag1000200 # output diff --git a/Examples/Tests/resampling/CMakeLists.txt b/Examples/Tests/resampling/CMakeLists.txt index 10d51e0ea47..46e34858014 100644 --- a/Examples/Tests/resampling/CMakeLists.txt +++ b/Examples/Tests/resampling/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_resample_velocity_coincidence_thinning # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_resample_velocity_coincidence_thinning # inputs analysis_default_regression.py # analysis diags/diag1000004 # output @@ -16,7 +15,6 @@ add_warpx_test( test_1d_resample_velocity_coincidence_thinning_cartesian # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_resample_velocity_coincidence_thinning_cartesian # inputs analysis_default_regression.py # analysis diags/diag1000004 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_leveling_thinning # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_leveling_thinning # inputs analysis.py # analysis diags/diag1000008 # output diff --git a/Examples/Tests/restart/CMakeLists.txt b/Examples/Tests/restart/CMakeLists.txt index 33770495dc6..bb3e90059c9 100644 --- a/Examples/Tests/restart/CMakeLists.txt +++ b/Examples/Tests/restart/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_id_cpu_read_picmi # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_id_cpu_read_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -19,7 +18,6 @@ add_warpx_test( test_2d_runtime_components_picmi # name 2 # dims 1 # nprocs - OFF # eb inputs_test_2d_runtime_components_picmi.py # inputs OFF #analysis_default_regression.py # analysis OFF #diags/diag1000010 # output @@ -33,7 +31,6 @@ add_warpx_test( test_2d_runtime_components_picmi_restart # name 2 # dims 1 # nprocs - OFF # eb "inputs_test_2d_runtime_components_picmi.py amr.restart='../test_2d_runtime_components_picmi/diags/chk000005'" # inputs OFF #analysis_default_restart.py # analysis OFF #diags/diag1000010 # output @@ -44,7 +41,6 @@ add_warpx_test( test_3d_acceleration # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -55,7 +51,6 @@ add_warpx_test( test_3d_acceleration_restart # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration_restart # inputs analysis_default_restart.py # analysis diags/diag1000010 # output @@ -67,7 +62,6 @@ if(WarpX_FFT) test_3d_acceleration_psatd # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration_psatd # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -80,7 +74,6 @@ if(WarpX_FFT) test_3d_acceleration_psatd_restart # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration_psatd_restart # inputs analysis_default_restart.py # analysis diags/diag1000010 # output @@ -93,7 +86,6 @@ if(WarpX_FFT) test_3d_acceleration_psatd_time_avg # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration_psatd_time_avg # inputs analysis_default_regression.py # analysis diags/diag1000010 # output @@ -106,7 +98,6 @@ if(WarpX_FFT) test_3d_acceleration_psatd_time_avg_restart # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_acceleration_psatd_time_avg_restart # inputs analysis_default_restart.py # analysis diags/diag1000010 # output diff --git a/Examples/Tests/restart_eb/CMakeLists.txt b/Examples/Tests/restart_eb/CMakeLists.txt index 54d1d3ea574..50f808c3e1f 100644 --- a/Examples/Tests/restart_eb/CMakeLists.txt +++ b/Examples/Tests/restart_eb/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_3d_eb_picmi # name 3 # dims 1 # nprocs - ON # eb inputs_test_3d_eb_picmi.py # inputs analysis_default_regression.py # analysis diags/diag1000060 # output @@ -20,8 +19,7 @@ endif() # test_3d_eb_picmi_restart # name # 3 # dims # 1 # nprocs -# ON # eb -# "inputs_test_3d_eb_picmi.py amr.restart='../test_3d_eb_picmi/diags/chk000030'" # inputs +## "inputs_test_3d_eb_picmi.py amr.restart='../test_3d_eb_picmi/diags/chk000030'" # inputs # analysis_default_restart.py # analysis # diags/diag1000060 # output # test_3d_eb_picmi # dependency diff --git a/Examples/Tests/rigid_injection/CMakeLists.txt b/Examples/Tests/rigid_injection/CMakeLists.txt index 210cc86418f..21004c3248c 100644 --- a/Examples/Tests/rigid_injection/CMakeLists.txt +++ b/Examples/Tests/rigid_injection/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_rigid_injection_btd # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_rigid_injection_btd # inputs analysis_rigid_injection_btd.py # analysis diags/diag1000001 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_rigid_injection_lab # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_rigid_injection_lab # inputs analysis_rigid_injection_lab.py # analysis diags/diag1000289 # output diff --git a/Examples/Tests/scraping/CMakeLists.txt b/Examples/Tests/scraping/CMakeLists.txt index 94ec04e35d7..a0fd04b6b3f 100644 --- a/Examples/Tests/scraping/CMakeLists.txt +++ b/Examples/Tests/scraping/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_EB) test_rz_scraping # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_scraping # inputs analysis_rz.py # analysis diags/diag1000037 # output @@ -19,7 +18,6 @@ if(WarpX_EB) test_rz_scraping_filter # name RZ # dims 2 # nprocs - ON # eb inputs_test_rz_scraping_filter # inputs analysis_rz_filter.py # analysis diags/diag1000037 # output diff --git a/Examples/Tests/silver_mueller/CMakeLists.txt b/Examples/Tests/silver_mueller/CMakeLists.txt index 5b9cd278ef0..7866d23dc1f 100644 --- a/Examples/Tests/silver_mueller/CMakeLists.txt +++ b/Examples/Tests/silver_mueller/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_1d_silver_mueller # name 1 # dims 2 # nprocs - OFF # eb inputs_test_1d_silver_mueller # inputs analysis.py # analysis diags/diag1000500 # output @@ -16,7 +15,6 @@ add_warpx_test( test_2d_silver_mueller_x # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_silver_mueller_x # inputs analysis.py # analysis diags/diag1000500 # output @@ -27,7 +25,6 @@ add_warpx_test( test_2d_silver_mueller_z # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_silver_mueller_z # inputs analysis.py # analysis diags/diag1000500 # output @@ -38,7 +35,6 @@ add_warpx_test( test_rz_silver_mueller_z # name RZ # dims 2 # nprocs - OFF # eb inputs_test_rz_silver_mueller_z # inputs analysis.py # analysis diags/diag1000500 # output diff --git a/Examples/Tests/single_particle/CMakeLists.txt b/Examples/Tests/single_particle/CMakeLists.txt index b2719bee681..bee870f0b17 100644 --- a/Examples/Tests/single_particle/CMakeLists.txt +++ b/Examples/Tests/single_particle/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_bilinear_filter # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_bilinear_filter # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/space_charge_initialization/CMakeLists.txt b/Examples/Tests/space_charge_initialization/CMakeLists.txt index af07d677775..6ca1f4ad04c 100644 --- a/Examples/Tests/space_charge_initialization/CMakeLists.txt +++ b/Examples/Tests/space_charge_initialization/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_space_charge_initialization # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_space_charge_initialization # inputs analysis.py # analysis diags/diag1000001 # output @@ -16,7 +15,6 @@ add_warpx_test( test_3d_space_charge_initialization # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_space_charge_initialization # inputs analysis.py # analysis diags/diag1000001 # output diff --git a/Examples/Tests/subcycling/CMakeLists.txt b/Examples/Tests/subcycling/CMakeLists.txt index ccea031f5a4..688f54ac01c 100644 --- a/Examples/Tests/subcycling/CMakeLists.txt +++ b/Examples/Tests/subcycling/CMakeLists.txt @@ -5,7 +5,6 @@ add_warpx_test( test_2d_subcycling_mr # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_subcycling_mr # inputs analysis_default_regression.py # analysis diags/diag1000250 # output diff --git a/Examples/Tests/vay_deposition/CMakeLists.txt b/Examples/Tests/vay_deposition/CMakeLists.txt index 9ebe4ec0dba..ce8d51d3c2a 100644 --- a/Examples/Tests/vay_deposition/CMakeLists.txt +++ b/Examples/Tests/vay_deposition/CMakeLists.txt @@ -6,7 +6,6 @@ if(WarpX_FFT) test_2d_vay_deposition # name 2 # dims 2 # nprocs - OFF # eb inputs_test_2d_vay_deposition # inputs analysis.py # analysis diags/diag1000050 # output @@ -19,7 +18,6 @@ if(WarpX_FFT) test_3d_vay_deposition # name 3 # dims 2 # nprocs - OFF # eb inputs_test_3d_vay_deposition # inputs analysis.py # analysis diags/diag1000025 # output diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 8f3ca1da1d7..ba9e2c3ab5d 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -147,6 +147,7 @@ public: bool do_pml_dive_cleaning, bool do_pml_divb_cleaning, const amrex::IntVect& fill_guards_fields, const amrex::IntVect& fill_guards_current, + bool eb_enabled, int max_guard_EB, amrex::Real v_sigma_sb, amrex::IntVect do_pml_Lo = amrex::IntVect::TheUnitVector(), amrex::IntVect do_pml_Hi = amrex::IntVect::TheUnitVector()); @@ -260,7 +261,7 @@ private: } #ifdef AMREX_USE_EB - amrex::EBFArrayBoxFactory const& fieldEBFactory () const noexcept { + [[nodiscard]] amrex::EBFArrayBoxFactory const& fieldEBFactory () const noexcept { return static_cast(*pml_field_factory); } #endif diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 340005e9211..f413831c74d 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -556,6 +556,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const bool do_pml_dive_cleaning, const bool do_pml_divb_cleaning, const amrex::IntVect& fill_guards_fields, const amrex::IntVect& fill_guards_current, + bool eb_enabled, int max_guard_EB, const amrex::Real v_sigma_sb, const amrex::IntVect do_pml_Lo, const amrex::IntVect do_pml_Hi) : m_dive_cleaning(do_pml_dive_cleaning), @@ -565,6 +566,10 @@ PML::PML (const int lev, const BoxArray& grid_ba, m_geom(geom), m_cgeom(cgeom) { +#ifndef AMREX_USE_EB + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, "PML: eb_enabled is true but was not compiled in."); +#endif + // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain or fine patch(es) // (instead of extending `ncell` outside of the physical domain or fine patch(es)) // In order to implement this, we define a new reduced Box Array ensuring that it does not @@ -673,13 +678,20 @@ PML::PML (const int lev, const BoxArray& grid_ba, } #ifdef AMREX_USE_EB - pml_field_factory = amrex::makeEBFabFactory(*geom, ba, dm, - {max_guard_EB, max_guard_EB, max_guard_EB}, - amrex::EBSupport::full); -#else - amrex::ignore_unused(max_guard_EB); - pml_field_factory = std::make_unique(); + if (eb_enabled) { + pml_field_factory = amrex::makeEBFabFactory( + *geom, + ba, + dm, + {max_guard_EB, max_guard_EB, max_guard_EB}, + amrex::EBSupport::full + ); + } else #endif + { + amrex::ignore_unused(max_guard_EB); + pml_field_factory = std::make_unique(); + } // Allocate diagonal components (xx,yy,zz) only with divergence cleaning const int ncompe = (m_dive_cleaning) ? 3 : 2; @@ -707,20 +719,22 @@ PML::PML (const int lev, const BoxArray& grid_ba, WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); #ifdef AMREX_USE_EB - const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); - WarpX::AllocInitMultiFab(pml_edge_lengths[0], ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[x]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[1], ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[y]", 0.0_rt); - WarpX::AllocInitMultiFab(pml_edge_lengths[2], ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[z]", 0.0_rt); + if (eb_enabled) { + const amrex::IntVect max_guard_EB_vect = amrex::IntVect(max_guard_EB); + WarpX::AllocInitMultiFab(pml_edge_lengths[0], ba_Ex, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[x]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_edge_lengths[1], ba_Ey, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[y]", 0.0_rt); + WarpX::AllocInitMultiFab(pml_edge_lengths[2], ba_Ez, dm, WarpX::ncomps, max_guard_EB_vect, lev, "pml_edge_lengths[z]", 0.0_rt); - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || - WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || + WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - auto const eb_fact = fieldEBFactory(); + auto const eb_fact = fieldEBFactory(); - WarpX::ComputeEdgeLengths(pml_edge_lengths, eb_fact); - WarpX::ScaleEdges(pml_edge_lengths, WarpX::CellSize(lev)); + WarpX::ComputeEdgeLengths(pml_edge_lengths, eb_fact); + WarpX::ScaleEdges(pml_edge_lengths, WarpX::CellSize(lev)); + } } #endif diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index ec696689373..bbe969052a3 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -269,13 +269,16 @@ WarpX::DampJPML (int lev, PatchType patch_type) const Real* sigma_star_cumsum_fac_j_z = sigba[mfi].sigma_star_cumsum_fac[1].data(); #endif -#ifdef AMREX_USE_EB - const auto& pml_edge_lenghts = pml[lev]->Get_edge_lengths(); - - auto const& pml_lxfab = pml_edge_lenghts[0]->array(mfi); - auto const& pml_lyfab = pml_edge_lenghts[1]->array(mfi); - auto const& pml_lzfab = pml_edge_lenghts[2]->array(mfi); -#endif + amrex::Array4 pml_lxfab, pml_lyfab, pml_lzfab; + if (m_eb_enabled) { + const auto &pml_edge_lenghts = pml[lev]->Get_edge_lengths(); + + pml_lxfab = pml_edge_lenghts[0]->array(mfi); + pml_lyfab = pml_edge_lenghts[1]->array(mfi); + pml_lzfab = pml_edge_lenghts[2]->array(mfi); + } else { + amrex::ignore_unused(pml_lxfab, pml_lyfab, pml_lzfab); + } const Box& tjx = mfi.tilebox( pml_j[0]->ixType().toIntVect() ); const Box& tjy = mfi.tilebox( pml_j[1]->ixType().toIntVect() ); @@ -301,27 +304,21 @@ WarpX::DampJPML (int lev, PatchType patch_type) amrex::ParallelFor( tjx, tjy, tjz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lxfab(i, j, k) <= 0) return; -#endif + if (pml_lxfab && pml_lxfab(i, j, k) <= 0) { return; } damp_jx_pml(i, j, k, pml_jxfab, sigma_star_cumsum_fac_j_x, sigma_cumsum_fac_j_y, sigma_cumsum_fac_j_z, xs_lo,y_lo, z_lo); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lyfab(i, j, k) <= 0) return; -#endif + if (pml_lyfab && pml_lyfab(i, j, k) <= 0) { return; } damp_jy_pml(i, j, k, pml_jyfab, sigma_cumsum_fac_j_x, sigma_star_cumsum_fac_j_y, sigma_cumsum_fac_j_z, x_lo,ys_lo, z_lo); }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { -#ifdef AMREX_USE_EB - if(pml_lzfab(i, j, k)<=0) return; -#endif + if (pml_lzfab && pml_lzfab(i, j, k)<=0) { return; } damp_jz_pml(i, j, k, pml_jzfab, sigma_cumsum_fac_j_x, sigma_cumsum_fac_j_y, sigma_star_cumsum_fac_j_z, diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index da1e5fdcc00..3757082ab4d 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -6,6 +6,7 @@ */ #include "BoundaryScrapingDiagnostics.H" +#include "EmbeddedBoundary/Enabled.H" #include "ComputeDiagFunctors/ComputeDiagFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" @@ -39,11 +40,11 @@ BoundaryScrapingDiagnostics::ReadParameters () // num_buffers corresponds to the number of boundaries // (upper/lower domain boundary in each dimension) - // + the EB boundary if available m_num_buffers = AMREX_SPACEDIM*2; -#ifdef AMREX_USE_EB - m_num_buffers += 1; -#endif + + // + the EB boundary if available + bool const eb_enabled = EB::enabled(); + if (eb_enabled) { m_num_buffers += 1; } // Do a few checks #ifndef WARPX_USE_OPENPMD diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 2991831420e..190822ded9c 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -8,6 +8,7 @@ #include "ChargeOnEB.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "EmbeddedBoundary/Enabled.H" #include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -24,11 +25,13 @@ #include #include +#include #include using namespace amrex; using namespace warpx::fields; + // constructor ChargeOnEB::ChargeOnEB (const std::string& rd_name) : ReducedDiags{rd_name} @@ -44,6 +47,10 @@ ChargeOnEB::ChargeOnEB (const std::string& rd_name) "ChargeOnEB reduced diagnostics only works when compiling with EB support"); #endif + if (!EB::enabled()) { + throw std::runtime_error("ChargeOnEB reduced diagnostics only works when EBs are enabled at runtime"); + } + // resize data array m_data.resize(1, 0.0_rt); @@ -87,6 +94,9 @@ void ChargeOnEB::ComputeDiags (const int step) // Judge whether the diags should be done if (!m_intervals.contains(step+1)) { return; } + if (!EB::enabled()) { + throw std::runtime_error("ComputeDiags only works when EBs are enabled at runtime"); + } #if ((defined WARPX_DIM_3D) && (defined AMREX_USE_EB)) // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -132,9 +142,9 @@ void ChargeOnEB::ComputeDiags (const int step) // Skip boxes that do not intersect with the embedded boundary // (i.e. either fully covered or fully regular) - amrex::FabType fab_type = eb_flag[mfi].getType(box); - if (fab_type == amrex::FabType::regular) continue; - if (fab_type == amrex::FabType::covered) continue; + const amrex::FabType fab_type = eb_flag[mfi].getType(box); + if (fab_type == amrex::FabType::regular) { continue; } + if (fab_type == amrex::FabType::covered) { continue; } // Extract data for electric field const amrex::Array4 & Ex_arr = Ex.array(mfi); @@ -153,7 +163,7 @@ void ChargeOnEB::ComputeDiags (const int step) [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Only cells that are partially covered do contribute to the integral - if (eb_flag_arr(i,j,k).isRegular() || eb_flag_arr(i,j,k).isCovered()) return; + if (eb_flag_arr(i,j,k).isRegular() || eb_flag_arr(i,j,k).isCovered()) { return; } // Find nodal point which is outside of the EB // (eb_normal points towards the *interior* of the EB) @@ -164,14 +174,14 @@ void ChargeOnEB::ComputeDiags (const int step) // Find cell-centered point which is outside of the EB // (eb_normal points towards the *interior* of the EB) int i_c = i; - if ((eb_bnd_normal_arr(i,j,k,0)>0) && (eb_bnd_cent_arr(i,j,k,0)<=0)) i_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,0)<0) && (eb_bnd_cent_arr(i,j,k,0)>=0)) i_c += 1; + if ((eb_bnd_normal_arr(i,j,k,0)>0) && (eb_bnd_cent_arr(i,j,k,0)<=0)) { i_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,0)<0) && (eb_bnd_cent_arr(i,j,k,0)>=0)) { i_c += 1; } int j_c = j; - if ((eb_bnd_normal_arr(i,j,k,1)>0) && (eb_bnd_cent_arr(i,j,k,1)<=0)) j_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,1)<0) && (eb_bnd_cent_arr(i,j,k,1)>=0)) j_c += 1; + if ((eb_bnd_normal_arr(i,j,k,1)>0) && (eb_bnd_cent_arr(i,j,k,1)<=0)) { j_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,1)<0) && (eb_bnd_cent_arr(i,j,k,1)>=0)) { j_c += 1; } int k_c = k; - if ((eb_bnd_normal_arr(i,j,k,2)>0) && (eb_bnd_cent_arr(i,j,k,2)<=0)) k_c -= 1; - if ((eb_bnd_normal_arr(i,j,k,2)<0) && (eb_bnd_cent_arr(i,j,k,2)>=0)) k_c += 1; + if ((eb_bnd_normal_arr(i,j,k,2)>0) && (eb_bnd_cent_arr(i,j,k,2)<=0)) { k_c -= 1; } + if ((eb_bnd_normal_arr(i,j,k,2)<0) && (eb_bnd_cent_arr(i,j,k,2)>=0)) { k_c += 1; } // Compute contribution to the surface integral $\int dS \cdot E$) amrex::Real local_integral_contribution = 0; @@ -182,9 +192,9 @@ void ChargeOnEB::ComputeDiags (const int step) // Add weighting if requested by user if (do_parser_weighting) { // Get the 3D position of the centroid of surface element - amrex::Real x = (i + 0.5_rt + eb_bnd_cent_arr(i,j,k,0))*dx[0] + real_box.lo(0); - amrex::Real y = (j + 0.5_rt + eb_bnd_cent_arr(i,j,k,1))*dx[1] + real_box.lo(1); - amrex::Real z = (k + 0.5_rt + eb_bnd_cent_arr(i,j,k,2))*dx[2] + real_box.lo(2); + const amrex::Real x = (i + 0.5_rt + eb_bnd_cent_arr(i,j,k,0))*dx[0] + real_box.lo(0); + const amrex::Real y = (j + 0.5_rt + eb_bnd_cent_arr(i,j,k,1))*dx[1] + real_box.lo(1); + const amrex::Real z = (k + 0.5_rt + eb_bnd_cent_arr(i,j,k,2))*dx[2] + real_box.lo(2); // Apply weighting local_integral_contribution *= fun_weightingparser(x, y, z); } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index a17abe04178..4082ff7b3d5 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -393,7 +393,7 @@ WarpX::InitFromCheckpoint () } } - InitializeEBGridData(maxLevel()); + if (m_eb_enabled) { InitializeEBGridData(maxLevel()); } // Initialize particles mypc->AllocData(); diff --git a/Source/EmbeddedBoundary/CMakeLists.txt b/Source/EmbeddedBoundary/CMakeLists.txt index a102087f8c9..2fa5e3e602b 100644 --- a/Source/EmbeddedBoundary/CMakeLists.txt +++ b/Source/EmbeddedBoundary/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + Enabled.cpp WarpXInitEB.cpp WarpXFaceExtensions.cpp WarpXFaceInfoBox.H diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index 7ee47c1172c..0c13724380c 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -14,8 +14,6 @@ #include #include -#ifdef AMREX_USE_EB - namespace DistanceToEB { @@ -28,7 +26,8 @@ amrex::Real dot_product (const amrex::RealVect& a, const amrex::RealVect& b) noe AMREX_GPU_HOST_DEVICE AMREX_INLINE void normalize (amrex::RealVect& a) noexcept { - amrex::Real inv_norm = 1.0/std::sqrt(dot_product(a,a)); + using namespace amrex::literals; + amrex::Real const inv_norm = 1.0_rt / std::sqrt(dot_product(a, a)); AMREX_D_DECL(a[0] *= inv_norm, a[1] *= inv_norm, a[2] *= inv_norm); @@ -46,6 +45,7 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP amrex::Array4 const& phi, amrex::GpuArray const& dxi) noexcept { + using namespace amrex::literals; #if (defined WARPX_DIM_3D) amrex::RealVect normal{0.0, 0.0, 0.0}; @@ -53,11 +53,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int kk = 0; kk < 2; ++kk) { for (int jj=0; jj< 2; ++jj) { for (int ii = 0; ii < 2; ++ii) { - int icstart = ic + iic; - amrex::Real sign = (ii%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(jj%2); - int w2comp = static_cast(kk%2); + int const icstart = ic + iic; + amrex::Real const sign = (ii%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(jj%2); + int const w2comp = static_cast(kk%2); normal[0] += sign * phi(icstart + ii, j + jj, k + kk) * dxi[0] * Wc[0][wccomp] * W[1][w1comp] * W[2][w2comp]; } } @@ -67,11 +67,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int kk = 0; kk < 2; ++kk) { for (int ii=0; ii< 2; ++ii) { for (int jj = 0; jj < 2; ++jj) { - int jcstart = jc + iic; - amrex::Real sign = (jj%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); - int w2comp = static_cast(kk%2); + int const jcstart = jc + iic; + amrex::Real const sign = (jj%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); + int const w2comp = static_cast(kk%2); normal[1] += sign * phi(i + ii, jcstart + jj, k + kk) * dxi[1] * W[0][w1comp] * Wc[1][wccomp] * W[2][w2comp]; } } @@ -81,11 +81,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int jj = 0; jj < 2; ++jj) { for (int ii=0; ii< 2; ++ii) { for (int kk = 0; kk < 2; ++kk) { - int kcstart = kc + iic; - amrex::Real sign = (kk%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); - int w2comp = static_cast(jj%2); + int const kcstart = kc + iic; + amrex::Real const sign = (kk%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); + int const w2comp = static_cast(jj%2); normal[2] += sign * phi(i + ii, j + jj, kcstart + kk) * dxi[2] * W[0][w1comp] * W[1][w2comp] * Wc[2][wccomp]; } } @@ -97,10 +97,10 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int iic = 0; iic < 2; ++iic) { for (int jj=0; jj< 2; ++jj) { for (int ii = 0; ii < 2; ++ii) { - int icstart = ic + iic; - amrex::Real sign = (ii%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(jj%2); + int const icstart = ic + iic; + amrex::Real const sign = (ii%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(jj%2); normal[0] += sign * phi(icstart + ii, j + jj, k) * dxi[0] * Wc[0][wccomp] * W[1][w1comp]; } } @@ -108,10 +108,10 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP for (int iic = 0; iic < 2; ++iic) { for (int ii=0; ii< 2; ++ii) { for (int jj = 0; jj < 2; ++jj) { - int jcstart = jc + iic; - amrex::Real sign = (jj%2)*2. - 1.; - int wccomp = static_cast(iic%2); - int w1comp = static_cast(ii%2); + int const jcstart = jc + iic; + amrex::Real const sign = (jj%2)*2._rt - 1._rt; + int const wccomp = static_cast(iic%2); + int const w1comp = static_cast(ii%2); normal[1] += sign * phi(i + ii, jcstart + jj, k) * dxi[1] * W[0][w1comp] * Wc[1][wccomp]; } } @@ -120,13 +120,11 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP #else amrex::ignore_unused(i, j, k, ic, jc, kc, W, Wc, phi, dxi); - amrex::RealVect normal{0.0, 0.0}; + amrex::RealVect normal(0.0); WARPX_ABORT_WITH_MESSAGE("Error: interp_distance not yet implemented in 1D"); #endif return normal; } } - -#endif // AMREX_USE_EB #endif // WARPX_DISTANCETOEB_H_ diff --git a/Source/EmbeddedBoundary/Enabled.H b/Source/EmbeddedBoundary/Enabled.H index 90ea5f35101..af01272e262 100644 --- a/Source/EmbeddedBoundary/Enabled.H +++ b/Source/EmbeddedBoundary/Enabled.H @@ -7,7 +7,7 @@ #ifndef WARPX_EB_ENABLED_H_ #define WARPX_EB_ENABLED_H_ -#include +#include namespace EB { diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 1e915b39381..c5d9cc68c60 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -176,7 +176,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) return; + if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) { return; } amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); @@ -185,7 +185,7 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan amrex::Real W[AMREX_SPACEDIM][2]; ablastr::particles::compute_weights( xp, yp, zp, plo, dxi, i, j, k, W); - amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { diff --git a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp index 21c13f23845..717aa26b021 100644 --- a/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp +++ b/Source/EmbeddedBoundary/WarpXFaceExtensions.cpp @@ -6,6 +6,7 @@ */ #include "WarpXFaceInfoBox.H" +#include "EmbeddedBoundary/Enabled.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -163,43 +164,49 @@ ComputeSStab(const int i, const int j, const int k, amrex::Array1D -WarpX::CountExtFaces() { +WarpX::CountExtFaces () { amrex::Array1D sums{0, 0, 0}; #ifdef AMREX_USE_EB + if (EB::enabled()) { #ifndef WARPX_DIM_RZ #ifdef WARPX_DIM_XZ - // In 2D we change the extrema of the for loop so that we only have the case idim=1 - for(int idim = 1; idim < AMREX_SPACEDIM; ++idim) { + // In 2D we change the extrema of the for loop so that we only have the case idim=1 + for(int idim = 1; idim < AMREX_SPACEDIM; ++idim) { #elif defined(WARPX_DIM_3D) for(int idim = 0; idim < AMREX_SPACEDIM; ++idim) { #else - WARPX_ABORT_WITH_MESSAGE( - "CountExtFaces: Only implemented in 2D3V and 3D3V"); + WARPX_ABORT_WITH_MESSAGE( + "CountExtFaces: Only implemented in 2D3V and 3D3V"); #endif - amrex::ReduceOps reduce_ops; - amrex::ReduceData reduce_data(reduce_ops); - for (amrex::MFIter mfi(*m_flag_ext_face[maxLevel()][idim]); mfi.isValid(); ++mfi) { - amrex::Box const &box = mfi.validbox(); - auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); - reduce_ops.eval(box, reduce_data, - [=] AMREX_GPU_DEVICE(int i, int j, int k) -> amrex::GpuTuple { - return flag_ext_face(i, j, k); - }); - } + amrex::ReduceOps reduce_ops; + amrex::ReduceData reduce_data(reduce_ops); + for (amrex::MFIter mfi(*m_flag_ext_face[maxLevel()][idim]); mfi.isValid(); ++mfi) { + amrex::Box const &box = mfi.validbox(); + auto const &flag_ext_face = m_flag_ext_face[maxLevel()][idim]->array(mfi); + reduce_ops.eval(box, reduce_data, + [=] AMREX_GPU_DEVICE(int i, int j, int k) -> amrex::GpuTuple { + return flag_ext_face(i, j, k); + }); + } - auto r = reduce_data.value(); - sums(idim) = amrex::get<0>(r); - } + auto r = reduce_data.value(); + sums(idim) = amrex::get<0>(r); + } - amrex::ParallelDescriptor::ReduceIntSum(&(sums(0)), AMREX_SPACEDIM); + amrex::ParallelDescriptor::ReduceIntSum(&(sums(0)), AMREX_SPACEDIM); #endif + } #endif return sums; } void -WarpX::ComputeFaceExtensions(){ +WarpX::ComputeFaceExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeFaceExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB amrex::Array1D N_ext_faces = CountExtFaces(); ablastr::warn_manager::WMRecordWarning("Embedded Boundary", @@ -421,7 +428,11 @@ ComputeNBorrowEightFacesExtension(const amrex::Dim3 cell, const amrex::Real S_ex void -WarpX::ComputeOneWayExtensions() { +WarpX::ComputeOneWayExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeOneWayExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB #ifndef WARPX_DIM_RZ auto const eb_fact = fieldEBFactory(maxLevel()); @@ -452,7 +463,7 @@ WarpX::ComputeOneWayExtensions() { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; auto const &borrowing_inds_pointer = borrowing.inds_pointer.array(); auto const &borrowing_size = borrowing.size.array(); - amrex::Long ncells = box.numPts(); + amrex::Long const ncells = box.numPts(); int* borrowing_inds = borrowing.inds.data(); FaceInfoBox::Neighbours* borrowing_neigh_faces = borrowing.neigh_faces.data(); amrex::Real* borrowing_area = borrowing.area.data(); @@ -503,7 +514,7 @@ WarpX::ComputeOneWayExtensions() { for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { //This if makes sure that we don't visit the "diagonal neighbours" - if( !(i_n == j_n || i_n == -j_n)){ + if (i_n != j_n && i_n != -j_n){ // Here a face is available if it doesn't need to be extended itself and if its // area exceeds Sz_ext. Here we need to take into account if the intruded face // has given away already some area, so we use Sz_red rather than Sz. @@ -545,8 +556,14 @@ WarpX::ComputeOneWayExtensions() { void -WarpX::ComputeEightWaysExtensions() { +WarpX::ComputeEightWaysExtensions () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeEightWaysExtensions only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB + using namespace amrex::literals; + #ifndef WARPX_DIM_RZ auto const &cell_size = CellSize(maxLevel()); @@ -574,7 +591,7 @@ WarpX::ComputeEightWaysExtensions() { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; auto const &borrowing_inds_pointer = borrowing.inds_pointer.array(); auto const &borrowing_size = borrowing.size.array(); - amrex::Long ncells = box.numPts(); + amrex::Long const ncells = box.numPts(); int* borrowing_inds = borrowing.inds.data(); FaceInfoBox::Neighbours* borrowing_neigh_faces = borrowing.neigh_faces.data(); amrex::Real* borrowing_area = borrowing.area.data(); @@ -650,7 +667,7 @@ WarpX::ComputeEightWaysExtensions() { neg_face = false; for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { - if(local_avail(i_n + 1, j_n + 1)){ + if (local_avail(i_n + 1, j_n + 1) != 0_rt){ const amrex::Real patch = S_ext * GetNeigh(S, i, j, k, i_n, j_n, idim) / denom; if(GetNeigh(S_mod, i, j, k, i_n, j_n, idim) - patch <= 0) { neg_face = true; @@ -675,7 +692,7 @@ WarpX::ComputeEightWaysExtensions() { int count = 0; for (int i_n = -1; i_n < 2; i_n++) { for (int j_n = -1; j_n < 2; j_n++) { - if(local_avail(i_n + 1, j_n + 1)){ + if(local_avail(i_n + 1, j_n + 1) != 0_rt){ const amrex::Real patch = S_ext * GetNeigh(S, i, j, k, i_n, j_n, idim) / denom; borrowing_inds[ps + count] = ps + count; FaceInfoBox::addConnectedNeighbor(i_n, j_n, ps + count, @@ -703,7 +720,11 @@ WarpX::ComputeEightWaysExtensions() { } void -WarpX::ApplyBCKCorrection(const int idim) { +WarpX::ApplyBCKCorrection (const int idim) +{ + if (!EB::enabled()) { + throw std::runtime_error("ApplyBCKCorrection only works when EBs are enabled at runtime"); + } #if defined(AMREX_USE_EB) and !defined(WARPX_DIM_RZ) const std::array &cell_size = CellSize(maxLevel()); @@ -736,7 +757,8 @@ WarpX::ApplyBCKCorrection(const int idim) { } void -WarpX::ShrinkBorrowing() { +WarpX::ShrinkBorrowing () +{ for(int idim = 0; idim < AMREX_SPACEDIM; idim++) { for (amrex::MFIter mfi(*Bfield_fp[maxLevel()][idim]); mfi.isValid(); ++mfi) { auto &borrowing = (*m_borrowing[maxLevel()][idim])[mfi]; diff --git a/Source/EmbeddedBoundary/WarpXInitEB.cpp b/Source/EmbeddedBoundary/WarpXInitEB.cpp index 655bec0dc29..b3e6290ad6c 100644 --- a/Source/EmbeddedBoundary/WarpXInitEB.cpp +++ b/Source/EmbeddedBoundary/WarpXInitEB.cpp @@ -7,6 +7,7 @@ #include "WarpX.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef AMREX_USE_EB # include "Utils/Parser/ParserUtils.H" # include "Utils/TextMsg.H" @@ -57,6 +58,8 @@ namespace { ParserIF& operator= (const ParserIF& rhs) = delete; ParserIF& operator= (ParserIF&& rhs) = delete; + ~ParserIF() = default; + AMREX_GPU_HOST_DEVICE inline amrex::Real operator() (AMREX_D_DECL(amrex::Real x, amrex::Real y, amrex::Real z)) const noexcept { @@ -80,6 +83,9 @@ namespace { void WarpX::InitEB () { + if (!EB::enabled()) { + throw std::runtime_error("InitEB only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB BL_PROFILE("InitEB"); @@ -88,7 +94,7 @@ WarpX::InitEB () pp_warpx.query("eb_implicit_function", impf); if (! impf.empty()) { auto eb_if_parser = utils::parser::makeParser(impf, {"x", "y", "z"}); - ParserIF pif(eb_if_parser.compile<3>()); + ParserIF const pif(eb_if_parser.compile<3>()); auto gshop = amrex::EB2::makeShop(pif, eb_if_parser); // The last argument of amrex::EB2::Build is the maximum coarsening level // to which amrex should try to coarsen the EB. It will stop after coarsening @@ -100,13 +106,12 @@ WarpX::InitEB () } else { amrex::ParmParse pp_eb2("eb2"); if (!pp_eb2.contains("geom_type")) { - std::string geom_type = "all_regular"; + std::string const geom_type = "all_regular"; pp_eb2.add("geom_type", geom_type); // use all_regular by default } // See the comment above on amrex::EB2::Build for the hard-wired number 20. amrex::EB2::Build(Geom(maxLevel()), maxLevel(), maxLevel()+20); } - #endif } @@ -124,16 +129,16 @@ WarpX::ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& ed for (amrex::MFIter mfi(flags); mfi.isValid(); ++mfi){ #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int idim = 0; idim < 3; ++idim){ - if(idim == 1) continue; + if(idim == 1) { continue; } #elif defined(WARPX_DIM_3D) for (int idim = 0; idim < AMREX_SPACEDIM; ++idim){ #else WARPX_ABORT_WITH_MESSAGE( "ComputeEdgeLengths: Only implemented in 2D3V and 3D3V"); #endif - amrex::Box box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), - edge_lengths[idim]->nGrowVect()); - amrex::FabType fab_type = flags[mfi].getType(box); + amrex::Box const box = mfi.tilebox(edge_lengths[idim]->ixType().toIntVect(), + edge_lengths[idim]->nGrowVect()); + amrex::FabType const fab_type = flags[mfi].getType(box); auto const &edge_lengths_dim = edge_lengths[idim]->array(mfi); if (fab_type == amrex::FabType::regular) { @@ -149,7 +154,7 @@ WarpX::ComputeEdgeLengths (std::array< std::unique_ptr, 3 >& ed } else { #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) int idim_amrex = idim; - if(idim == 2) idim_amrex = 1; + if(idim == 2) { idim_amrex = 1; } auto const &edge_cent = edge_centroid[idim_amrex]->const_array(mfi); #elif defined(WARPX_DIM_3D) auto const &edge_cent = edge_centroid[idim]->const_array(mfi); @@ -207,9 +212,9 @@ WarpX::ComputeFaceAreas (std::array< std::unique_ptr, 3 >& face WARPX_ABORT_WITH_MESSAGE( "ComputeFaceAreas: Only implemented in 2D3V and 3D3V"); #endif - amrex::Box box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), - face_areas[idim]->nGrowVect()); - amrex::FabType fab_type = flags[mfi].getType(box); + amrex::Box const box = mfi.tilebox(face_areas[idim]->ixType().toIntVect(), + face_areas[idim]->nGrowVect()); + amrex::FabType const fab_type = flags[mfi].getType(box); auto const &face_areas_dim = face_areas[idim]->array(mfi); if (fab_type == amrex::FabType::regular) { // every cell in box is all regular @@ -247,7 +252,7 @@ WarpX::ScaleEdges (std::array< std::unique_ptr, 3 >& edge_lengt for (amrex::MFIter mfi(*edge_lengths[0]); mfi.isValid(); ++mfi) { #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int idim = 0; idim < 3; ++idim){ - if(idim == 1) continue; + if(idim == 1) { continue; } #elif defined(WARPX_DIM_3D) for (int idim = 0; idim < AMREX_SPACEDIM; ++idim){ #else @@ -397,7 +402,11 @@ WarpX::MarkCells(){ #endif void -WarpX::ComputeDistanceToEB () { +WarpX::ComputeDistanceToEB () +{ + if (!EB::enabled()) { + throw std::runtime_error("ComputeDistanceToEB only works when EBs are enabled at runtime"); + } #ifdef AMREX_USE_EB BL_PROFILE("ComputeDistanceToEB"); const amrex::EB2::IndexSpace& eb_is = amrex::EB2::IndexSpace::top(); diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 5a2dbdf2f30..bc18d0da75d 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -525,11 +525,12 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num } // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); - m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); - mypc->deleteInvalidParticles(); -#endif + if (m_eb_enabled) { + mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries( + *mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + mypc->deleteInvalidParticles(); + } if (sort_intervals.contains(step+1)) { if (verbose) { diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 189f2f2bb0a..80110f5eb18 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -289,11 +289,10 @@ WarpX::AddSpaceChargeFieldLabFrame () // Compute the electric field. Note that if an EB is used the electric // field will be calculated in the computePhi call. -#ifndef AMREX_USE_EB - computeE( Efield_fp, phi_fp, beta ); -#else - if ( IsPythonCallbackInstalled("poissonsolver") ) computeE( Efield_fp, phi_fp, beta ); -#endif + if (!m_eb_enabled) { computeE( Efield_fp, phi_fp, beta ); } + else { + if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp, phi_fp, beta); } + } // Compute the magnetic field computeB( Bfield_fp, phi_fp, beta ); @@ -323,64 +322,66 @@ WarpX::computePhi (const amrex::Vector >& rho, Real const required_precision, Real absolute_tolerance, int const max_iters, - int const verbosity) const -{ + int const verbosity) const { // create a vector to our fields, sorted by level - amrex::Vector sorted_rho; - amrex::Vector sorted_phi; + amrex::Vector sorted_rho; + amrex::Vector sorted_phi; for (int lev = 0; lev <= finest_level; ++lev) { sorted_rho.emplace_back(rho[lev].get()); sorted_phi.emplace_back(phi[lev].get()); } -#if defined(AMREX_USE_EB) - std::optional post_phi_calculation; - - // EB: use AMReX to directly calculate the electric field since with EB's the - // simple finite difference scheme in WarpX::computeE sometimes fails - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || - electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) +#ifdef AMREX_USE_EB + // TODO: double check no overhead occurs on "m_eb_enabled == false" + std::optional > eb_farray_box_factory; +#else + std::optional > const eb_farray_box_factory; +#endif + if (m_eb_enabled) { - // TODO: maybe make this a helper function or pass Efield_fp directly - amrex::Vector< - amrex::Array - > e_field; - for (int lev = 0; lev <= finest_level; ++lev) { - e_field.push_back( + // EB: use AMReX to directly calculate the electric field since with EB's the + // simple finite difference scheme in WarpX::computeE sometimes fails + if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame || + electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + { + // TODO: maybe make this a helper function or pass Efield_fp directly + amrex::Vector< + amrex::Array + > e_field; + for (int lev = 0; lev <= finest_level; ++lev) { + e_field.push_back( # if defined(WARPX_DIM_1D_Z) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 2) - } + amrex::Array{ + getFieldPointer(FieldType::Efield_fp, lev, 2) + } # elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 2) - } + amrex::Array{ + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 2) + } # elif defined(WARPX_DIM_3D) - amrex::Array{ - getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2) - } + amrex::Array{ + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) + } # endif - ); + ); + } + post_phi_calculation = ElectrostaticSolver::EBCalcEfromPhiPerLevel(e_field); } - post_phi_calculation = ElectrostaticSolver::EBCalcEfromPhiPerLevel(e_field); - } - std::optional > eb_farray_box_factory; - amrex::Vector< - amrex::EBFArrayBoxFactory const * - > factories; - for (int lev = 0; lev <= finest_level; ++lev) { - factories.push_back(&WarpX::fieldEBFactory(lev)); - } - eb_farray_box_factory = factories; -#else - const std::optional post_phi_calculation; - const std::optional > eb_farray_box_factory; +#ifdef AMREX_USE_EB + amrex::Vector< + amrex::EBFArrayBoxFactory const * + > factories; + for (int lev = 0; lev <= finest_level; ++lev) { + factories.push_back(&WarpX::fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; #endif + } bool const is_solver_igf_on_lev0 = WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; @@ -399,6 +400,7 @@ WarpX::computePhi (const amrex::Vector >& rho, WarpX::grid_type, this->m_poisson_boundary_handler, is_solver_igf_on_lev0, + m_eb_enabled, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp index fbc1397b413..4fe9fc76e10 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveB.cpp @@ -87,12 +87,9 @@ void FiniteDifferenceSolver::EvolveB ( } else if (m_fdtd_algo == ElectromagneticSolverAlgo::CKC) { EvolveBCartesian ( Bfield, Efield, Gfield, lev, dt ); -#ifdef AMREX_USE_EB } else if (m_fdtd_algo == ElectromagneticSolverAlgo::ECT) { - EvolveBCartesianECT(Bfield, face_areas, area_mod, ECTRhofield, Venl, flag_info_cell, borrowing, lev, dt); -#endif #endif } else { WARPX_ABORT_WITH_MESSAGE("EvolveB: Unknown algorithm"); @@ -245,9 +242,9 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( amrex::Array4 const &S = face_areas[idim]->array(mfi); amrex::Array4 const &S_mod = area_mod[idim]->array(mfi); - auto &borrowing_dim = (*borrowing[idim])[mfi]; - auto borrowing_dim_neigh_faces = borrowing_dim.neigh_faces.data(); - auto borrowing_dim_area = borrowing_dim.area.data(); + auto & borrowing_dim = (*borrowing[idim])[mfi]; + auto * borrowing_dim_neigh_faces = borrowing_dim.neigh_faces.data(); + auto * borrowing_dim_area = borrowing_dim.area.data(); auto const &borrowing_inds = (*borrowing[idim])[mfi].inds.data(); auto const &borrowing_size = (*borrowing[idim])[mfi].size.array(); @@ -259,24 +256,23 @@ void FiniteDifferenceSolver::EvolveBCartesianECT ( //Take care of the unstable cells amrex::ParallelFor(tb, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (S(i, j, k) <= 0) return; + if (S(i, j, k) <= 0) { return; } - if (!(flag_info_cell_dim(i, j, k) == 0)) - return; + if (!(flag_info_cell_dim(i, j, k) == 0)) { return; } Venl_dim(i, j, k) = Rho(i, j, k) * S(i, j, k); amrex::Real rho_enl; // First we compute the rho of the enlarged face for (int offset = 0; offset const& Ffield, int lev, amrex::Real const dt ) { -#ifdef AMREX_USE_EB if (m_fdtd_algo != ElectromagneticSolverAlgo::ECT) { amrex::ignore_unused(face_areas, ECTRhofield); } -#else - amrex::ignore_unused(face_areas, ECTRhofield); -#endif // Select algorithm (The choice of algorithm is a runtime option, // but we compile code for each algorithm, using templates) @@ -129,11 +126,12 @@ void FiniteDifferenceSolver::EvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -152,10 +150,9 @@ void FiniteDifferenceSolver::EvolveECartesian ( amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } + Ex(i, j, k) += c2 * dt * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) + T_Algo::DownwardDy(Bz, coefs_y, n_coefs_y, i, j, k) @@ -163,16 +160,15 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } #endif + Ey(i, j, k) += c2 * dt * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) + T_Algo::DownwardDz(Bx, coefs_z, n_coefs_z, i, j, k) @@ -180,10 +176,8 @@ void FiniteDifferenceSolver::EvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + if (lz && lz(i,j,k) <= 0) { return; } Ez(i, j, k) += c2 * dt * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) + T_Algo::DownwardDx(By, coefs_x, n_coefs_x, i, j, k) @@ -265,10 +259,11 @@ void FiniteDifferenceSolver::EvolveECylindrical ( Array4 const& jt = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lr, lz; + if (EB::enabled()) { + lr = edge_lengths[0]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -292,10 +287,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( amrex::ParallelFor(ter, tet, tez, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + if (lr && lr(i, j, 0) <= 0) { return; } + Real const r = rmin + (i + 0.5_rt)*dr; // r on cell-centered point (Er is cell-centered in r) Er(i, j, 0, 0) += c2 * dt*( - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) @@ -313,11 +307,10 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries // The Et field is at a node, so we need to check if the node is covered - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + if (lr && (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0)) { return; } + Real const r = rmin + i*dr; // r on a nodal grid (Et is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations Et(i, j, 0, 0) += c2 * dt*( @@ -359,10 +352,9 @@ void FiniteDifferenceSolver::EvolveECylindrical ( }, [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i, j, 0) <= 0) return; -#endif + if (lz && lz(i, j, 0) <= 0) { return; } + Real const r = rmin + i*dr; // r on a nodal grid (Ez is nodal in r) if (r != 0) { // Off-axis, regular Maxwell equations Ez(i, j, 0, 0) += c2 * dt*( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp index 95f899c98e1..8abdab71300 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveECTRho.cpp @@ -112,7 +112,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( amrex::ParallelFor(trhox, trhoy, trhoz, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sx(i, j, k) <= 0) return; + if (Sx(i, j, k) <= 0) { return; } // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_XZ @@ -122,7 +122,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( }, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sy(i, j, k) <= 0) return; + if (Sy(i, j, k) <= 0) { return; } #ifdef WARPX_DIM_XZ Rhoy(i, j, k) = (Ez(i, j, k) * lz(i, j, k) - Ez(i + 1, j, k) * lz(i + 1, j, k) + @@ -136,7 +136,7 @@ void FiniteDifferenceSolver::EvolveRhoCartesianECT ( }, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if (Sz(i, j, k) <= 0) return; + if (Sz(i, j, k) <= 0) { return; } // If we implement ECT in 1D we will need to take care of this #ifndef differently #ifndef WARPX_DIM_XZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp index 93352ce9896..a1ba6e44a8c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveEPML.cpp @@ -16,6 +16,7 @@ #else # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include "EmbeddedBoundary/Enabled.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -109,11 +110,12 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( Array4 const& By = Bfield[1]->array(mfi); Array4 const& Bz = Bfield[2]->array(mfi); -#ifdef AMREX_USE_EB - Array4 const& lx = edge_lengths[0]->array(mfi); - Array4 const& ly = edge_lengths[1]->array(mfi); - Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -132,9 +134,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - if(lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } Ex(i, j, k, PMLComp::xz) -= c2 * dt * ( T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k, PMLComp::yx) @@ -145,16 +145,15 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } #endif + Ey(i, j, k, PMLComp::yx) -= c2 * dt * ( T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k, PMLComp::zx) + T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k, PMLComp::zy) ); @@ -164,9 +163,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB - if(lz(i, j, k) <= 0) return; -#endif + if (lz && lz(i, j, k) <= 0) { return; } Ez(i, j, k, PMLComp::zy) -= c2 * dt * ( T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k, PMLComp::xy) @@ -244,13 +241,7 @@ void FiniteDifferenceSolver::EvolveEPMLCartesian ( } ); } - - } - -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - + } // MFIter } -#endif // corresponds to ifndef WARPX_DIM_RZ +#endif // ifndef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 6a72bb3569c..652dbfe7cdd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -9,6 +9,7 @@ #include "HybridPICModel.H" +#include "EmbeddedBoundary/Enabled.H" #include "FieldSolver/Fields.H" #include "WarpX.H" @@ -226,23 +227,23 @@ void HybridPICModel::InitData () // Initialize external current - note that this approach skips the check // if the current is time dependent which is what needs to be done to // write time independent fields on the first step. - for (int lev = 0; lev <= warpx.finestLevel(); ++lev) - { + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { + auto edge_lengths = std::array, 3>(); #ifdef AMREX_USE_EB - auto& edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); - auto& edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); - auto& edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); - - const auto edge_lengths = std::array< std::unique_ptr, 3 >{ - std::make_unique( - edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp()), - std::make_unique( - edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp()), - std::make_unique( - edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp()) - }; -#else - const auto edge_lengths = std::array< std::unique_ptr, 3 >(); + if (EB::enabled()) { + auto const & edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); + auto const & edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); + auto const & edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); + + edge_lengths = std::array< std::unique_ptr, 3 >{ + std::make_unique( + edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp()), + std::make_unique( + edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp()), + std::make_unique( + edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp()) + }; + } #endif GetCurrentExternal(edge_lengths, lev); } @@ -289,31 +290,26 @@ void HybridPICModel::GetCurrentExternal ( for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); - const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); - const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); - auto const& mfxfab = mfx->array(mfi); - auto const& mfyfab = mfy->array(mfi); - auto const& mfzfab = mfz->array(mfi); + auto const& mfxfab = mfx->array(mfi); + auto const& mfyfab = mfy->array(mfi); + auto const& mfzfab = mfz->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(ly); -#endif -#else - amrex::ignore_unused(edge_lengths); -#endif + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } amrex::ParallelFor (tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } + // Shift required in the x-, y-, or z- position // depending on the index type of the multifab #if defined(WARPX_DIM_1D_Z) @@ -340,9 +336,8 @@ void HybridPICModel::GetCurrentExternal ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (ly(i, j, k) <= 0) return; -#endif + if (ly && ly(i, j, k) <= 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; @@ -367,9 +362,8 @@ void HybridPICModel::GetCurrentExternal ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // skip if node is covered by an embedded boundary -#ifdef AMREX_USE_EB - if (lz(i, j, k) <= 0) return; -#endif + if (lz && lz(i, j, k) <= 0) { return; } + #if defined(WARPX_DIM_1D_Z) const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 456c542a534..5da46f23e54 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -9,6 +9,7 @@ #include "FiniteDifferenceSolver.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef WARPX_DIM_RZ # include "FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #else @@ -67,10 +68,6 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // reset Jfield Jfield[0]->setVal(0); Jfield[1]->setVal(0); @@ -95,11 +92,13 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( Array4 const& Bt = Bfield[1]->array(mfi); Array4 const& Bz = Bfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lt = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lr, lt, lz; + + if (EB::enabled()) { + lr = edge_lengths[0]->array(mfi); + lt = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -125,10 +124,8 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jr calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + if (lr && lr(i, j, 0) <= 0) { return; } // Mode m=0 Jr(i, j, 0, 0) = one_over_mu0 * ( - T_Algo::DownwardDz(Bt, coefs_z, n_coefs_z, i, j, 0, 0) @@ -151,11 +148,9 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jt calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // In RZ Jt is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(lt); - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + if (lr && (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0)) { return; } + // r on a nodal point (Jt is nodal in r) Real const r = rmin + i*dr; // Off-axis, regular curl @@ -199,10 +194,8 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCylindrical ( // Jz calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lz(i, j, 0) <= 0) return; -#endif + if (lz && lz(i, j, 0) <= 0) { return; } // r on a nodal point (Jz is nodal in r) Real const r = rmin + i*dr; // Off-axis, regular curl @@ -258,10 +251,6 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // reset Jfield Jfield[0]->setVal(0); Jfield[1]->setVal(0); @@ -272,25 +261,25 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif for ( MFIter mfi(*Jfield[0], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { amrex::Gpu::synchronize(); } auto wt = static_cast(amrex::second()); // Extract field data for this grid/tile - Array4 const& Jx = Jfield[0]->array(mfi); - Array4 const& Jy = Jfield[1]->array(mfi); - Array4 const& Jz = Jfield[2]->array(mfi); - Array4 const& Bx = Bfield[0]->const_array(mfi); - Array4 const& By = Bfield[1]->const_array(mfi); - Array4 const& Bz = Bfield[2]->const_array(mfi); - -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + Array4 const &Jx = Jfield[0]->array(mfi); + Array4 const &Jy = Jfield[1]->array(mfi); + Array4 const &Jz = Jfield[2]->array(mfi); + Array4 const &Bx = Bfield[0]->const_array(mfi); + Array4 const &By = Bfield[1]->const_array(mfi); + Array4 const &Bz = Bfield[2]->const_array(mfi); + + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -313,10 +302,9 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jx calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } + Jx(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDz(By, coefs_z, n_coefs_z, i, j, k) + T_Algo::DownwardDy(Bz, coefs_y, n_coefs_y, i, j, k) @@ -325,15 +313,13 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jy calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) // In XZ Jy is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } #endif Jy(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDx(Bz, coefs_x, n_coefs_x, i, j, k) @@ -343,10 +329,9 @@ void FiniteDifferenceSolver::CalculateCurrentAmpereCartesian ( // Jz calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + if (lz && lz(i,j,k) <= 0) { return; } + Jz(i, j, k) = one_over_mu0 * ( - T_Algo::DownwardDy(Bx, coefs_y, n_coefs_y, i, j, k) + T_Algo::DownwardDx(By, coefs_x, n_coefs_x, i, j, k) @@ -415,10 +400,6 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( int lev, HybridPICModel const* hybrid_model, const bool include_resistivity_term ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // Both steps below do not currently support m > 0 and should be // modified if such support wants to be added WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -561,11 +542,12 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& rho = rhofield->const_array(mfi); Array4 const& Pe = Pefield->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lr = edge_lengths[0]->array(mfi); - amrex::Array4 const& lt = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lr, lz; + if (EB::enabled()) { + lr = edge_lengths[0]->array(mfi); + // edge_lengths[1] is `lt` and is not needed + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_r = m_stencil_coefs_r.dataPtr(); @@ -586,10 +568,9 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Er calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lr(i, j, 0) <= 0) return; -#endif + if (lr && lr(i, j, 0) <= 0) { return; } + // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); @@ -627,11 +608,9 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Et calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // In RZ Et is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(lt); - if (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0) return; -#endif + if (lr && (lr(i, j, 0)<=0 || lr(i-1, j, 0)<=0 || lz(i, j-1, 0)<=0 || lz(i, j, 0)<=0)) { return; } + // r on a nodal grid (Et is nodal in r) Real const r = rmin + i*dr; // Mode m=0: // Ensure that Et remains 0 on axis @@ -672,10 +651,9 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/){ -#ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,0) <= 0) { return; } -#endif + if (lz && lz(i,j,0) <= 0) { return; } + // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, 0, 0); @@ -733,10 +711,6 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( int lev, HybridPICModel const* hybrid_model, const bool include_resistivity_term ) { -#ifndef AMREX_USE_EB - amrex::ignore_unused(edge_lengths); -#endif - // for the profiler amrex::LayoutData* cost = WarpX::getCosts(lev); @@ -873,11 +847,12 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& rho = rhofield->const_array(mfi); Array4 const& Pe = Pefield->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); -#endif + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } // Extract stencil coefficients Real const * const AMREX_RESTRICT coefs_x = m_stencil_coefs_x.dataPtr(); @@ -896,10 +871,9 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Ex calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } + // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); @@ -933,16 +907,14 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( }, // Ey calculation - [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB + [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Skip field solve if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) { return; } + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0)) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); @@ -980,7 +952,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) { return; } + if (lz && lz(i,j,k) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp index 3aee7697073..1a9b79a8acb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp @@ -7,6 +7,7 @@ # include "FiniteDifferenceAlgorithms/CartesianCKCAlgorithm.H" # include "FiniteDifferenceAlgorithms/FieldAccessorFunctors.H" #endif +#include "EmbeddedBoundary/Enabled.H" #include "MacroscopicProperties/MacroscopicProperties.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -139,10 +140,14 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( Array4 const& jy = Jfield[1]->array(mfi); Array4 const& jz = Jfield[2]->array(mfi); -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); + amrex::Array4 lx, ly, lz; + if (EB::enabled()) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + } +#ifdef WARPX_DIM_XZ + amrex::ignore_unused(ly); #endif // material prop // @@ -174,10 +179,9 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( // Loop over the cells and update the fields amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lx(i, j, k) <= 0) return; -#endif + if (lx && lx(i, j, k) <= 0) { return; } + // Interpolate conductivity, sigma, to Ex position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ex_stag, macro_cr, i, j, k, scomp); @@ -193,15 +197,13 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly && ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j, k)<=0 || lz(i, j-1, k)<=0) return; -#endif + if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j, k)<=0 || lz(i, j-1, k)<=0)) { return; } #endif + // Interpolate conductivity, sigma, to Ey position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ey_stag, macro_cr, i, j, k, scomp); @@ -218,10 +220,9 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian ( }, [=] AMREX_GPU_DEVICE (int i, int j, int k){ -#ifdef AMREX_USE_EB // Skip field push if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; -#endif + if (lz && lz(i,j,k) <= 0) { return; } + // Interpolate conductivity, sigma, to Ez position on the grid amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag, Ez_stag, macro_cr, i, j, k, scomp); diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 26ac1ac96c8..d715e64cdaa 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -184,6 +184,7 @@ WarpX::computeVectorPotential (const amrex::Vectordmap, this->grids, this->m_vector_poisson_boundary_handler, + m_eb_enabled, WarpX::do_single_precision_comms, this->ref_ratio, post_A_calculation, diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index a27a46b7e88..5e5ebb19921 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -717,6 +717,7 @@ WarpX::InitPML () psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, amrex::IntVect(0), amrex::IntVect(0), + m_eb_enabled, guard_cells.ng_FieldSolver.max(), v_particle_pml, do_pml_Lo[0], do_pml_Hi[0]); @@ -756,6 +757,7 @@ WarpX::InitPML () do_moving_window, pml_has_particles, do_pml_in_domain, psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, amrex::IntVect(0), amrex::IntVect(0), + m_eb_enabled, guard_cells.ng_FieldSolver.max(), v_particle_pml, do_pml_Lo[lev], do_pml_Hi[lev]); @@ -934,7 +936,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } #ifdef AMREX_USE_EB - InitializeEBGridData(lev); + if (m_eb_enabled) { InitializeEBGridData(lev); } #endif // if the input string for the B-field is "parse_b_ext_grid_function", @@ -979,11 +981,13 @@ WarpX::InitLevelData (int lev, Real /*time*/) && (lev <= maxlevel_extEMfield_init)) { #ifdef AMREX_USE_EB - // We initialize ECTRhofield consistently with the Efield - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_fdtd_solver_fp[lev]->EvolveECTRho( - Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + if (m_eb_enabled) { + // We initialize ECTRhofield consistently with the Efield + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + m_fdtd_solver_fp[lev]->EvolveECTRho( + Efield_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); + } } #endif @@ -1012,11 +1016,13 @@ WarpX::InitLevelData (int lev, Real /*time*/) 'E', lev, PatchType::coarse); #ifdef AMREX_USE_EB - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - // We initialize ECTRhofield consistently with the Efield - m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); + if (m_eb_enabled) { + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + // We initialize ECTRhofield consistently with the Efield + m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); + } } #endif } @@ -1041,7 +1047,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( ParserExecutor<3> const& zfield_parser, std::array< std::unique_ptr, 3 > const& edge_lengths, std::array< std::unique_ptr, 3 > const& face_areas, - const char field, + [[maybe_unused]] const char field, const int lev, PatchType patch_type) { @@ -1057,49 +1063,45 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); - for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); - const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); - const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); - - auto const& mfxfab = mfx->array(mfi); - auto const& mfyfab = mfy->array(mfi); - auto const& mfzfab = mfz->array(mfi); - -#ifdef AMREX_USE_EB - amrex::Array4 const& lx = edge_lengths[0]->array(mfi); - amrex::Array4 const& ly = edge_lengths[1]->array(mfi); - amrex::Array4 const& lz = edge_lengths[2]->array(mfi); - amrex::Array4 const& Sx = face_areas[0]->array(mfi); - amrex::Array4 const& Sy = face_areas[1]->array(mfi); - amrex::Array4 const& Sz = face_areas[2]->array(mfi); + for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) { + const amrex::Box &tbx = mfi.tilebox(x_nodal_flag, mfx->nGrowVect()); + const amrex::Box &tby = mfi.tilebox(y_nodal_flag, mfy->nGrowVect()); + const amrex::Box &tbz = mfi.tilebox(z_nodal_flag, mfz->nGrowVect()); + + auto const &mfxfab = mfx->array(mfi); + auto const &mfyfab = mfy->array(mfi); + auto const &mfzfab = mfz->array(mfi); + + amrex::Array4 lx, ly, lz, Sx, Sy, Sz; + if (m_eb_enabled) { + lx = edge_lengths[0]->array(mfi); + ly = edge_lengths[1]->array(mfi); + lz = edge_lengths[2]->array(mfi); + Sx = face_areas[0]->array(mfi); + Sy = face_areas[1]->array(mfi); + Sz = face_areas[2]->array(mfi); + } #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Dim3 lx_lo = amrex::lbound(lx); - const amrex::Dim3 lx_hi = amrex::ubound(lx); - const amrex::Dim3 lz_lo = amrex::lbound(lz); - const amrex::Dim3 lz_hi = amrex::ubound(lz); + amrex::Dim3 lx_lo, lx_hi, lz_lo, lz_hi; #endif - + if (m_eb_enabled) { #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(ly, Sx, Sz); -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(lx, ly, lz, Sx, Sy, Sz); -#endif - -#else - amrex::ignore_unused(edge_lengths, face_areas, field); + lx_lo = amrex::lbound(lx); + lx_hi = amrex::ubound(lx); + lz_lo = amrex::lbound(lz); + lz_hi = amrex::ubound(lz); #endif + } amrex::ParallelFor (tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { #ifdef AMREX_USE_EB #ifdef WARPX_DIM_3D - if((field=='E' and lx(i, j, k)<=0) or (field=='B' and Sx(i, j, k)<=0)) return; + if(lx && ((field=='E' and lx(i, j, k)<=0) or (field=='B' and Sx(i, j, k)<=0))) { return; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) //In XZ and RZ Ex is associated with a x-edge, while Bx is associated with a z-edge - if((field=='E' and lx(i, j, k)<=0) or (field=='B' and lz(i, j, k)<=0)) return; + if(lx && ((field=='E' and lx(i, j, k)<=0) or (field=='B' and lz(i, j, k)<=0))) { return; } #endif #endif // Shift required in the x-, y-, or z- position @@ -1129,14 +1131,15 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( [=] AMREX_GPU_DEVICE (int i, int j, int k) { #ifdef AMREX_USE_EB #ifdef WARPX_DIM_3D - if((field=='E' and ly(i, j, k)<=0) or (field=='B' and Sy(i, j, k)<=0)) return; + if(ly && ((field=='E' and ly(i, j, k)<=0) or (field=='B' and Sy(i, j, k)<=0))) { return; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) //In XZ and RZ Ey is associated with a mesh node, so we need to check if the mesh node is covered - if((field=='E' and (lx(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)<=0 + if(lx && + ((field=='E' and (lx(std::min(i , lx_hi.x), std::min(j , lx_hi.y), k)<=0 || lx(std::max(i-1, lx_lo.x), std::min(j , lx_hi.y), k)<=0 || lz(std::min(i , lz_hi.x), std::min(j , lz_hi.y), k)<=0 || lz(std::min(i , lz_hi.x), std::max(j-1, lz_lo.y), k)<=0)) or - (field=='B' and Sy(i,j,k)<=0)) return; + (field=='B' and Sy(i,j,k)<=0))) { return; } #endif #endif #if defined(WARPX_DIM_1D_Z) @@ -1164,10 +1167,10 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( [=] AMREX_GPU_DEVICE (int i, int j, int k) { #ifdef AMREX_USE_EB #ifdef WARPX_DIM_3D - if((field=='E' and lz(i, j, k)<=0) or (field=='B' and Sz(i, j, k)<=0)) return; + if(lz && ((field=='E' and lz(i, j, k)<=0) or (field=='B' and Sz(i, j, k)<=0))) { return; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) //In XZ and RZ Ez is associated with a z-edge, while Bz is associated with a x-edge - if((field=='E' and lz(i, j, k)<=0) or (field=='B' and lx(i, j, k)<=0)) return; + if(lz && ((field=='E' and lz(i, j, k)<=0) or (field=='B' and lx(i, j, k)<=0))) { return; } #endif #endif #if defined(WARPX_DIM_1D_Z) @@ -1268,9 +1271,7 @@ void WarpX::InitializeEBGridData (int lev) if (lev == maxLevel()) { // Throw a warning if EB is on and particle_shape > 1 - bool flag_eb_on = not fieldEBFactory(lev).isAllRegular(); - - if ((nox > 1 or noy > 1 or noz > 1) and flag_eb_on) + if ((nox > 1 or noy > 1 or noz > 1) and m_eb_enabled) { ablastr::warn_manager::WMRecordWarning("Particles", "when algo.particle_shape > 1, numerical artifacts will be present when\n" diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index d33bd17ccdd..112db68f488 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -215,20 +215,20 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); } -#ifdef AMREX_USE_EB - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], false); - RemakeMultiFab(m_face_areas[lev][idim], false); - if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT){ - RemakeMultiFab(Venl[lev][idim], false); - RemakeMultiFab(m_flag_info_face[lev][idim], false); - RemakeMultiFab(m_flag_ext_face[lev][idim], false); - RemakeMultiFab(m_area_mod[lev][idim], false); - RemakeMultiFab(ECTRhofield[lev][idim], false); - m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); + if (m_eb_enabled) { + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + RemakeMultiFab(m_edge_lengths[lev][idim], false); + RemakeMultiFab(m_face_areas[lev][idim], false); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + RemakeMultiFab(Venl[lev][idim], false); + RemakeMultiFab(m_flag_info_face[lev][idim], false); + RemakeMultiFab(m_flag_ext_face[lev][idim], false); + RemakeMultiFab(m_area_mod[lev][idim], false); + RemakeMultiFab(ECTRhofield[lev][idim], false); + m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); + } } } -#endif } RemakeMultiFab(F_fp[lev], true); @@ -242,18 +242,19 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); } -#ifdef AMREX_USE_EB - RemakeMultiFab(m_distance_to_eb[lev], false); - - int max_guard = guard_cells.ng_FieldSolver.max(); - m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, - {max_guard, max_guard, max_guard}, - amrex::EBSupport::full); + if (m_eb_enabled) { + RemakeMultiFab(m_distance_to_eb[lev], false); - InitializeEBGridData(lev); -#else - m_field_factory[lev] = std::make_unique(); +#ifdef AMREX_USE_EB + int const max_guard = guard_cells.ng_FieldSolver.max(); + m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, + {max_guard, max_guard, max_guard}, + amrex::EBSupport::full); #endif + InitializeEBGridData(lev); + } else { + m_field_factory[lev] = std::make_unique(); + } #ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index fc496217388..cc3cd89cd7d 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -39,10 +39,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" -#ifdef AMREX_USE_EB -# include "EmbeddedBoundary/ParticleScraper.H" -# include "EmbeddedBoundary/ParticleBoundaryProcess.H" -#endif +#include "EmbeddedBoundary/ParticleScraper.H" +#include "EmbeddedBoundary/ParticleBoundaryProcess.H" #include "WarpX.H" @@ -958,13 +956,9 @@ void MultiParticleContainer::CheckIonizationProductSpecies() void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) { -#ifdef AMREX_USE_EB for (auto& pc : allcontainers) { scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); } -#else - amrex::ignore_unused(distance_to_eb); -#endif } #ifdef WARPX_QED diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 7b2ebce92b9..bc113e8e3a3 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -6,6 +6,7 @@ */ #include "WarpX.H" +#include "EmbeddedBoundary/Enabled.H" #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Particles/MultiParticleContainer.H" @@ -23,8 +24,10 @@ #include #include #include + using namespace amrex::literals; + struct IsOutsideDomainBoundary { amrex::GpuArray m_plo; amrex::GpuArray m_phi; @@ -46,7 +49,6 @@ struct IsOutsideDomainBoundary { } }; -#ifdef AMREX_USE_EB struct FindEmbeddedBoundaryIntersection { const int m_step_index; const int m_delta_index; @@ -86,13 +88,13 @@ struct FindEmbeddedBoundaryIntersection { amrex::ParticleReal const uz = dst.m_rdata[PIdx::uz][dst_i]; // Temporary variables to avoid implicit capture - amrex::Real dt = m_dt; - amrex::Array4 phiarr = m_phiarr; - amrex::GpuArray dxi = m_dxi; - amrex::GpuArray plo = m_plo; + amrex::Real const dt = m_dt; + amrex::Array4 const phiarr = m_phiarr; + amrex::GpuArray const dxi = m_dxi; + amrex::GpuArray const plo = m_plo; // Bisection algorithm to find the point where phi(x,y,z)=0 (i.e. on the embedded boundary) - amrex::Real dt_fraction = amrex::bisect( 0.0, 1.0, + amrex::Real const dt_fraction = amrex::bisect( 0.0, 1.0, [=] (amrex::Real dt_frac) { int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; @@ -100,7 +102,7 @@ struct FindEmbeddedBoundaryIntersection { UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); ablastr::particles::compute_weights( x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); - amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -146,7 +148,7 @@ struct FindEmbeddedBoundaryIntersection { dst.m_rdata[PIdx::z][dst_i] = z_temp; dst.m_rdata[PIdx::theta][dst_i] = std::atan2(y_temp, x_temp); //save normal components - amrex::Real theta=std::atan2(y_temp, x_temp); + amrex::Real const theta = std::atan2(y_temp, x_temp); dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]*std::cos(theta); dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[0]*std::sin(theta); dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; @@ -165,7 +167,6 @@ struct FindEmbeddedBoundaryIntersection { amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; -#endif struct CopyAndTimestamp { int m_step_index; @@ -239,6 +240,8 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () constexpr auto idx_zhi = 5; #endif + bool const eb_enabled = EB::enabled(); + for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { const amrex::ParmParse pp_species(getSpeciesNames()[ispecies]); @@ -258,9 +261,9 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () pp_species.query("save_particles_at_zlo", m_do_boundary_buffer[idx_zlo][ispecies]); pp_species.query("save_particles_at_zhi", m_do_boundary_buffer[idx_zhi][ispecies]); #endif -#ifdef AMREX_USE_EB - pp_species.query("save_particles_at_eb", m_do_boundary_buffer[AMREX_SPACEDIM*2][ispecies]); -#endif + + if (eb_enabled) { pp_species.query("save_particles_at_eb", m_do_boundary_buffer[AMREX_SPACEDIM*2][ispecies]); } + // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } @@ -283,10 +286,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () m_boundary_names[idx_zlo] = "zlo"; m_boundary_names[idx_zhi] = "zhi"; #endif -#ifdef AMREX_USE_EB - m_boundary_names[AMREX_SPACEDIM*2] = "eb"; -#endif - + if (eb_enabled) { m_boundary_names[AMREX_SPACEDIM*2] = "eb"; } } void ParticleBoundaryBuffer::printNumParticles () const { @@ -306,17 +306,17 @@ void ParticleBoundaryBuffer::printNumParticles () const { } } } -#ifdef AMREX_USE_EB - auto& buffer = m_particle_containers[2*AMREX_SPACEDIM]; - for (int i = 0; i < numSpecies(); ++i) - { - const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; - amrex::Print() << Utils::TextMsg::Info( - "Species " + getSpeciesNames()[i] + " has " - + std::to_string(np) + " particles in the EB boundary buffer" - ); + + if (EB::enabled()) { + auto const & buffer = m_particle_containers[2 * AMREX_SPACEDIM]; + for (int i = 0; i < numSpecies(); ++i) { + const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; + amrex::Print() << Utils::TextMsg::Info( + "Species " + getSpeciesNames()[i] + " has " + + std::to_string(np) + " particles in the EB boundary buffer" + ); + } } -#endif } void ParticleBoundaryBuffer::redistribute () { @@ -464,107 +464,108 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) { -#ifdef AMREX_USE_EB - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); + if (EB::enabled()) { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); - using PIter = amrex::ParConstIterSoA; - const auto& warpx_instance = WarpX::GetInstance(); - const amrex::Geometry& geom = warpx_instance.Geom(0); - auto plo = geom.ProbLoArray(); - auto& buffer = m_particle_containers[m_particle_containers.size()-1]; - for (int i = 0; i < numSpecies(); ++i) - { - if (!m_do_boundary_buffer[AMREX_SPACEDIM*2][i]) continue; - const auto& pc = mypc.GetParticleContainer(i); - if (!buffer[i].isDefined()) + using PIter = amrex::ParConstIterSoA; + const auto &warpx_instance = WarpX::GetInstance(); + const amrex::Geometry &geom = warpx_instance.Geom(0); + auto plo = geom.ProbLoArray(); + + auto& buffer = m_particle_containers[m_particle_containers.size()-1]; + for (int i = 0; i < numSpecies(); ++i) { - buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("stepScraped", false); - buffer[i].AddRealComp("deltaTimeScraped", false); - buffer[i].AddRealComp("nx", false); - buffer[i].AddRealComp("ny", false); - buffer[i].AddRealComp("nz", false); + if (!m_do_boundary_buffer[AMREX_SPACEDIM*2][i]) { continue; } + const auto& pc = mypc.GetParticleContainer(i); + if (!buffer[i].isDefined()) + { + buffer[i] = pc.make_alike(); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); + buffer[i].AddRealComp("nx", false); + buffer[i].AddRealComp("ny", false); + buffer[i].AddRealComp("nz", false); - } + } - auto& species_buffer = buffer[i]; - for (int lev = 0; lev < pc.numLevels(); ++lev){ - for(PIter pti(pc, lev); pti.isValid(); ++pti){ - species_buffer.DefineAndReturnParticleTile( - lev, pti.index(), pti.LocalTileIndex()); + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev) { + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } } - } - for (int lev = 0; lev < pc.numLevels(); ++lev) - { - const auto& plevel = pc.GetParticles(lev); - auto dxi = warpx_instance.Geom(lev).InvCellSizeArray(); + for (int lev = 0; lev < pc.numLevels(); ++lev) + { + const auto& plevel = pc.GetParticles(lev); + auto dxi = warpx_instance.Geom(lev).InvCellSizeArray(); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for(PIter pti(pc, lev); pti.isValid(); ++pti) - { - auto phiarr = (*distance_to_eb[lev])[pti].array(); // signed distance function - auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; - - const auto getPosition = GetParticlePosition(pti); - auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile(lev, pti.index(), - pti.LocalTileIndex()); - const auto& ptile = plevel.at(index); - auto np = ptile.numParticles(); - if (np == 0) { continue; } - - using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; - auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) - /* NVCC 11.3.109 chokes in C++17 on this: noexcept */ - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - - amrex::Real phi_value = ablastr::particles::doGatherScalarFieldNodal( - xp, yp, zp, phiarr, dxi, plo - ); - return phi_value < 0.0 ? 1 : 0; - }; - - const auto ptile_data = ptile.getConstParticleTileData(); - - amrex::ReduceOps reduce_op; - amrex::ReduceData reduce_data(reduce_op); - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::count_out_of_boundsEB"); - reduce_op.eval(np, reduce_data, [=] AMREX_GPU_HOST_DEVICE (int ip) - { return predicate(ptile_data, ip) ? 1 : 0; }); - } + for (PIter pti(pc, lev); pti.isValid(); ++pti) { + auto phiarr = (*distance_to_eb[lev])[pti].array(); // signed distance function + auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); + if (plevel.find(index) == plevel.end()) { continue; } + + const auto getPosition = GetParticlePosition(pti); + auto &ptile_buffer = species_buffer.DefineAndReturnParticleTile(lev, pti.index(), + pti.LocalTileIndex()); + const auto &ptile = plevel.at(index); + auto np = ptile.numParticles(); + if (np == 0) { continue; } + + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + auto predicate = [=] AMREX_GPU_HOST_DEVICE(const SrcData & /*src*/, const int ip) + /* NVCC 11.3.109 chokes in C++17 on this: noexcept */ + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); - auto dst_index = ptile_buffer.numParticles(); - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); - ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); - } - auto& warpx = WarpX::GetInstance(); - const auto dt = warpx.getdt(pti.GetLevel()); - auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); - const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); - auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); - const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); - const int normal_index = string_to_index_realcomp.at("nx"); - const int step = warpx_instance.getistep(0); + amrex::Real const phi_value = ablastr::particles::doGatherScalarFieldNodal( + xp, yp, zp, phiarr, dxi, plo + ); + return phi_value < 0.0 ? 1 : 0; + }; - { - WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); - amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - FindEmbeddedBoundaryIntersection{step_scraped_index,delta_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + const auto ptile_data = ptile.getConstParticleTileData(); + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::count_out_of_boundsEB"); + reduce_op.eval(np, reduce_data, + [=] AMREX_GPU_HOST_DEVICE(int ip) { return predicate(ptile_data, ip) ? 1 : 0; }); + } + + auto dst_index = ptile_buffer.numParticles(); + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); + ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); + } + auto &warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); + const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); + auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); + const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); + const int normal_index = string_to_index_realcomp.at("nx"); + const int step = warpx_instance.getistep(0); + + { + WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); + amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, + FindEmbeddedBoundaryIntersection{step_scraped_index, + delta_index, normal_index, + step, dt, phiarr, dxi, plo}, + 0, dst_index); + + } } } } } -#else - amrex::ignore_unused(mypc, distance_to_eb); -#endif } int ParticleBoundaryBuffer::getNumParticlesInContainer( diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3a30467c20d..1ad43755464 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,6 +39,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "EmbeddedBoundary/Enabled.H" #ifdef AMREX_USE_EB # include "EmbeddedBoundary/ParticleBoundaryProcess.H" # include "EmbeddedBoundary/ParticleScraper.H" @@ -1483,8 +1484,11 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + if (EB::enabled()) + { + auto &distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticlesAtEB(*this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + } #endif // The function that calls this is responsible for redistributing particles. @@ -1980,8 +1984,11 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + if (EB::enabled()) + { + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + } #endif // Redistribute the new particles that were added to the temporary container. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 3fd70ee5795..05f8dd609f0 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -13,6 +13,7 @@ #include "Deposition/ChargeDeposition.H" #include "Deposition/CurrentDeposition.H" #include "Deposition/SharedDepositionUtils.H" +#include "EmbeddedBoundary/Enabled.H" #include "Pusher/GetAndSetPosition.H" #include "Pusher/UpdatePosition.H" #include "ParticleBoundaries_K.H" @@ -300,9 +301,11 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB - auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); - deleteInvalidParticles(); + if (EB::enabled()) { + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + deleteInvalidParticles(); + } #endif } diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 9e2360315b8..edcf5991c71 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -154,7 +154,7 @@ const std::map ReductionType_algo_to_int = { }; int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ) +GetAlgorithmInteger (const amrex::ParmParse& pp, const char* pp_search_key ) { // Read user input ; use "default" if it is not found std::string algo = "default"; diff --git a/Source/WarpX.H b/Source/WarpX.H index 4573808461e..903e97549dd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -997,6 +997,8 @@ public: */ [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;} + /** Enable embedded boundaries */ + bool m_eb_enabled = false; bool m_boundary_potential_specified = false; ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler; void ComputeSpaceChargeField (bool reset_fields); @@ -1063,7 +1065,7 @@ public: amrex::ParserExecutor<3> const& zfield_parser, std::array< std::unique_ptr, 3 > const& edge_lengths, std::array< std::unique_ptr, 3 > const& face_areas, - char field, + [[maybe_unused]] char field, int lev, PatchType patch_type); /** @@ -1130,7 +1132,7 @@ public: void BuildBufferMasksInBox ( amrex::Box tbx, amrex::IArrayBox &buffer_mask, const amrex::IArrayBox &guard_mask, int ng ); #ifdef AMREX_USE_EB - amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { + [[nodiscard]] amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept { return static_cast(*m_field_factory[lev]); } #endif diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 02f85422d12..272c7aa0aff 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -84,6 +84,7 @@ #include #include #include +#include #include #include @@ -259,7 +260,7 @@ WarpX::WarpX () BackwardCompatibility(); - InitEB(); + if (m_eb_enabled) { InitEB(); } ablastr::utils::SignalHandling::InitSignalHandling(); @@ -540,10 +541,14 @@ WarpX::ReadParameters () { const ParmParse pp_algo("algo"); electromagnetic_solver_id = static_cast(GetAlgorithmInteger(pp_algo, "maxwell_solver")); + + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT && !EB::enabled()) { + throw std::runtime_error("ECP Solver requires to enable embedded boundaries at runtime."); + } } { - const ParmParse pp_warpx("warpx"); + ParmParse const pp_warpx("warpx"); //"Synthetic" warning messages may be injected in the Warning Manager via // inputfile for debug&testing purposes. @@ -783,6 +788,14 @@ WarpX::ReadParameters () "The FFT Poisson solver is not implemented in labframe-electromagnetostatic mode yet." ); + m_eb_enabled = EB::enabled(); +#if !defined(AMREX_USE_EB) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_eb_enabled, + "Embedded boundaries are requested via warpx.eb_enabled but were not compiled!" + ); +#endif + // Parse the input file for domain boundary potentials const ParmParse pp_boundary("boundary"); bool potential_specified = false; @@ -793,16 +806,9 @@ WarpX::ReadParameters () potential_specified |= pp_boundary.query("potential_hi_y", m_poisson_boundary_handler.potential_yhi_str); potential_specified |= pp_boundary.query("potential_lo_z", m_poisson_boundary_handler.potential_zlo_str); potential_specified |= pp_boundary.query("potential_hi_z", m_poisson_boundary_handler.potential_zhi_str); -#if defined(AMREX_USE_EB) - potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); - - if (!EB::enabled()) { - throw std::runtime_error( - "Currently, users MUST use EB if it was compiled in. " - "This will change with https://github.com/ECP-WarpX/WarpX/pull/4865 ." - ); + if (m_eb_enabled) { + potential_specified |= pp_warpx.query("eb_potential(x,y,z,t)", m_poisson_boundary_handler.potential_eb_str); } -#endif m_boundary_potential_specified = potential_specified; if (potential_specified & (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC)) { ablastr::warn_manager::WMRecordWarning( @@ -2203,14 +2209,17 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d bilinear_filter.stencil_length_each_dir); + + if (m_eb_enabled) { #ifdef AMREX_USE_EB - int max_guard = guard_cells.ng_FieldSolver.max(); - m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, - {max_guard, max_guard, max_guard}, - amrex::EBSupport::full); -#else - m_field_factory[lev] = std::make_unique(); + int const max_guard = guard_cells.ng_FieldSolver.max(); + m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, + {max_guard, max_guard, max_guard}, + amrex::EBSupport::full); #endif + } else { + m_field_factory[lev] = std::make_unique(); + } if (mypc->nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { @@ -2429,51 +2438,81 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_avg_fp[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_avg_fp[z]", 0.0_rt); } -#ifdef AMREX_USE_EB - constexpr int nc_ls = 1; - amrex::IntVect ng_ls(2); - AllocInitMultiFab(m_distance_to_eb[lev], amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, lev, "m_distance_to_eb"); - - // EB info are needed only at the finest level - if (lev == maxLevel()) - { - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); - } - if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); - AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); - AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); - AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); - AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); - AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); - AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); - AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[y]"); - AllocInitMultiFab(m_flag_info_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_info_face[z]"); - AllocInitMultiFab(m_flag_ext_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[x]"); - AllocInitMultiFab(m_flag_ext_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[y]"); - AllocInitMultiFab(m_flag_ext_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[z]"); - AllocInitMultiFab(m_area_mod[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[x]"); - AllocInitMultiFab(m_area_mod[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[y]"); - AllocInitMultiFab(m_area_mod[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "m_area_mod[z]"); - m_borrowing[lev][0] = std::make_unique>(amrex::convert(ba, Bx_nodal_flag), dm); - m_borrowing[lev][1] = std::make_unique>(amrex::convert(ba, By_nodal_flag), dm); - m_borrowing[lev][2] = std::make_unique>(amrex::convert(ba, Bz_nodal_flag), dm); - AllocInitMultiFab(Venl[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[x]"); - AllocInitMultiFab(Venl[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[y]"); - AllocInitMultiFab(Venl[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "Venl[z]"); - - AllocInitMultiFab(ECTRhofield[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[x]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[y]", 0.0_rt); - AllocInitMultiFab(ECTRhofield[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, guard_cells.ng_FieldSolver, lev, "ECTRhofield[z]", 0.0_rt); + if (m_eb_enabled) { + constexpr int nc_ls = 1; + amrex::IntVect const ng_ls(2); + AllocInitMultiFab(m_distance_to_eb[lev], amrex::convert(ba, IntVect::TheNodeVector()), dm, nc_ls, ng_ls, lev, + "m_distance_to_eb"); + + // EB info are needed only at the finest level + if (lev == maxLevel()) { + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); + AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); + AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); + AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); + AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); + AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); + } + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { + AllocInitMultiFab(m_edge_lengths[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[x]"); + AllocInitMultiFab(m_edge_lengths[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[y]"); + AllocInitMultiFab(m_edge_lengths[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_edge_lengths[z]"); + AllocInitMultiFab(m_face_areas[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[x]"); + AllocInitMultiFab(m_face_areas[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[y]"); + AllocInitMultiFab(m_face_areas[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_face_areas[z]"); + AllocInitMultiFab(m_flag_info_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[x]"); + AllocInitMultiFab(m_flag_info_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[y]"); + AllocInitMultiFab(m_flag_info_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_info_face[z]"); + AllocInitMultiFab(m_flag_ext_face[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[x]"); + AllocInitMultiFab(m_flag_ext_face[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[y]"); + AllocInitMultiFab(m_flag_ext_face[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_flag_ext_face[z]"); + AllocInitMultiFab(m_area_mod[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_area_mod[x]"); + AllocInitMultiFab(m_area_mod[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_area_mod[y]"); + AllocInitMultiFab(m_area_mod[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "m_area_mod[z]"); + m_borrowing[lev][0] = std::make_unique>( + amrex::convert(ba, Bx_nodal_flag), dm); + m_borrowing[lev][1] = std::make_unique>( + amrex::convert(ba, By_nodal_flag), dm); + m_borrowing[lev][2] = std::make_unique>( + amrex::convert(ba, Bz_nodal_flag), dm); + AllocInitMultiFab(Venl[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "Venl[x]"); + AllocInitMultiFab(Venl[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "Venl[y]"); + AllocInitMultiFab(Venl[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "Venl[z]"); + + AllocInitMultiFab(ECTRhofield[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "ECTRhofield[x]", 0.0_rt); + AllocInitMultiFab(ECTRhofield[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "ECTRhofield[y]", 0.0_rt); + AllocInitMultiFab(ECTRhofield[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, + guard_cells.ng_FieldSolver, lev, "ECTRhofield[z]", 0.0_rt); + } } } -#endif int rho_ncomps = 0; if( (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 26a4c72208d..fbd8e5f14be 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -46,9 +46,7 @@ #include #include #include -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) -# include -#endif +#include #ifdef AMREX_USE_EB # include #endif @@ -85,6 +83,7 @@ namespace ablastr::fields { * \param[in] grid_type Integer that corresponds to the type of grid used in the simulation (collocated, staggered, hybrid) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels + * \param[in] eb_enabled solve with embedded boundaries * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -100,17 +99,18 @@ void computePhi (amrex::Vector const & rho, amrex::Vector & phi, std::array const beta, - amrex::Real const relative_tolerance, + amrex::Real relative_tolerance, amrex::Real absolute_tolerance, - int const max_iters, - int const verbosity, + int max_iters, + int verbosity, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, utils::enums::GridType grid_type, T_BoundaryHandler const boundary_handler, bool is_solver_igf_on_lev0, - bool const do_single_precision_comms = false, + bool eb_enabled = false, + bool do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB @@ -127,6 +127,11 @@ computePhi (amrex::Vector const & rho, rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; } +#if !defined(AMREX_USE_EB) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, + "Embedded boundary solve requested but not compiled in"); +#endif + auto const finest_level = static_cast(rho.size() - 1); // determine if rho is zero everywhere @@ -146,21 +151,17 @@ computePhi (amrex::Vector const & rho, ); } -#if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) amrex::LPInfo info; -#else - const amrex::LPInfo info; -#endif for (int lev=0; lev<=finest_level; lev++) { // Set the value of beta - amrex::Array beta_solver = + amrex::Array beta_solver = #if defined(WARPX_DIM_1D_Z) - {{ beta[2] }}; // beta_x and beta_z + {{ beta[2] }}; // beta_x and beta_z #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - {{ beta[0], beta[2] }}; // beta_x and beta_z + {{ beta[0], beta[2] }}; // beta_x and beta_z #else - {{ beta[0], beta[1], beta[2] }}; + {{ beta[0], beta[1], beta[2] }}; #endif #if !defined(ABLASTR_USE_FFT) @@ -173,13 +174,13 @@ computePhi (amrex::Vector const & rho, "The FFT Poisson solver is currently only implemented for 3D!"); #endif -#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) +#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), - geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), - geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; + geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), + geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; if ( max_norm_b == 0 ) { phi[lev]->setVal(0); } else { @@ -192,77 +193,106 @@ computePhi (amrex::Vector const & rho, // Use the Multigrid (MLMG) solver if selected or on refined patches // but first scale rho appropriately using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! - -#if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) - // Determine whether to use semi-coarsening - amrex::Array dx_scaled - {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), - geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), - geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; - int max_semicoarsening_level = 0; - int semicoarsening_direction = -1; - const auto min_dir = static_cast(std::distance(dx_scaled.begin(), - std::min_element(dx_scaled.begin(),dx_scaled.end()))); - const auto max_dir = static_cast(std::distance(dx_scaled.begin(), - std::max_element(dx_scaled.begin(),dx_scaled.end()))); - if (dx_scaled[max_dir] > dx_scaled[min_dir]) { - semicoarsening_direction = max_dir; - max_semicoarsening_level = static_cast - (std::log2(dx_scaled[max_dir]/dx_scaled[min_dir])); - } - if (max_semicoarsening_level > 0) { - info.setSemicoarsening(true); - info.setMaxSemicoarseningLevel(max_semicoarsening_level); - info.setSemicoarseningDirection(semicoarsening_direction); - } + rho[lev]->mult(-1._rt / ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; #endif -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) - // In the presence of EB or RZ: the solver assumes that the beam is - // propagating along one of the axes of the grid, i.e. that only *one* - // of the components of `beta` is non-negligible. - amrex::MLEBNodeFDLaplacian linop( {geom[lev]}, {grids[lev]}, {dmap[lev]}, info + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + amrex::Array dx_scaled + {AMREX_D_DECL(geom[lev].CellSize(0) / std::sqrt(1._rt - beta_solver[0] * beta_solver[0]), + geom[lev].CellSize(1) / std::sqrt(1._rt - beta_solver[1] * beta_solver[1]), + geom[lev].CellSize(2) / std::sqrt(1._rt - beta_solver[2] * beta_solver[2]))}; + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx_scaled.begin(), + std::min_element(dx_scaled.begin(), dx_scaled.end()))); + const auto max_dir = static_cast(std::distance(dx_scaled.begin(), + std::max_element(dx_scaled.begin(), dx_scaled.end()))); + if (dx_scaled[max_dir] > dx_scaled[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx_scaled[max_dir] / dx_scaled[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } + } + + std::unique_ptr linop; + if (eb_enabled || is_rz) { + // In the presence of EB or RZ: the solver assumes that the beam is + // propagating along one of the axes of the grid, i.e. that only *one* + // of the components of `beta` is non-negligible. + auto linop_nodelap = std::make_unique(); + if (eb_enabled) { #if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info, + amrex::Vector{eb_farray_box_factory.value()[lev]} + ); #endif - ); + } + else { + // TODO: rather use MLNodeTensorLaplacian (for RZ w/o EB) here? Semi-Coarsening would be nice here + linop_nodelap->define( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + } - // Note: this assumes that the beam is propagating along - // one of the axes of the grid, i.e. that only *one* of the - // components of `beta` is non-negligible. // we use this + // Note: this assumes that the beam is propagating along + // one of the axes of the grid, i.e. that only *one* of the + // components of `beta` is non-negligible. // we use this #if defined(WARPX_DIM_RZ) - linop.setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); + linop_nodelap->setRZ(true); + linop_nodelap->setSigma({0._rt, 1._rt-beta_solver[1]*beta_solver[1]}); #else - linop.setSigma({AMREX_D_DECL( - 1._rt-beta_solver[0]*beta_solver[0], - 1._rt-beta_solver[1]*beta_solver[1], - 1._rt-beta_solver[2]*beta_solver[2])}); + linop_nodelap->setSigma({AMREX_D_DECL( + 1._rt-beta_solver[0]*beta_solver[0], + 1._rt-beta_solver[1]*beta_solver[1], + 1._rt-beta_solver[2]*beta_solver[2])}); #endif #if defined(AMREX_USE_EB) - // if the EB potential only depends on time, the potential can be passed - // as a float instead of a callable - if (boundary_handler.phi_EB_only_t) { - linop.setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); - } - else - linop.setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); -#endif -#else - // In the absence of EB and RZ: use a more generic solver - // that can handle beams propagating in any direction - amrex::MLNodeTensorLaplacian linop( {geom[lev]}, {grids[lev]}, - {dmap[lev]}, info ); - linop.setBeta( beta_solver ); // for the non-axis-aligned solver + if (eb_enabled) { + // if the EB potential only depends on time, the potential can be passed + // as a float instead of a callable + if (boundary_handler.phi_EB_only_t) { + linop_nodelap->setEBDirichlet(boundary_handler.potential_eb_t(current_time.value())); + } else { + linop_nodelap->setEBDirichlet(boundary_handler.getPhiEB(current_time.value())); + } + } #endif + linop = std::move(linop_nodelap); + } else { + // In the absence of EB and RZ: use a more generic solver + // that can handle beams propagating in any direction + auto linop_tenslap = std::make_unique( + amrex::Vector{geom[lev]}, + amrex::Vector{grids[lev]}, + amrex::Vector{dmap[lev]}, + info + ); + linop_tenslap->setBeta(beta_solver); // for the non-axis-aligned solver + linop = std::move(linop_tenslap); + } // Solve the Poisson equation - linop.setDomainBC( boundary_handler.lobc, boundary_handler.hibc ); -#ifdef WARPX_DIM_RZ - linop.setRZ(true); -#endif - amrex::MLMG mlmg(linop); // actual solver defined here + linop->setDomainBC(boundary_handler.lobc, boundary_handler.hibc); + + amrex::MLMG mlmg(*linop); // actual solver defined here mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); @@ -287,7 +317,7 @@ computePhi (amrex::Vector const & rho, amrex::BoxArray ba = phi[lev+1]->boxArray(); const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); - const int ncomp = linop.getNComp(); + const int ncomp = linop->getNComp(); const int ng = (grid_type == utils::enums::GridType::Collocated) ? 1 : 0; amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); if (ng > 0) { diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 3ef96c30c84..f6dd2a99cf1 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -71,6 +71,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see MagnetostaticSolver::VectorPoissonBoundaryHandler + * \param[in] eb_enabled solve with embedded boundaries * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_A_calculation perform a calculation per level directly after A was calculated; required for embedded boundaries (default: none) @@ -85,15 +86,16 @@ template< void computeVectorPotential ( amrex::Vector > const & curr, amrex::Vector > & A, - amrex::Real const relative_tolerance, + amrex::Real relative_tolerance, amrex::Real absolute_tolerance, - int const max_iters, - int const verbosity, + int max_iters, + int verbosity, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, - bool const do_single_precision_comms = false, + bool eb_enabled = false, + bool do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostACalculationFunctor post_A_calculation = std::nullopt, [[maybe_unused]] std::optional current_time = std::nullopt, // only used for EB @@ -108,6 +110,11 @@ computeVectorPotential ( amrex::Vector > co rel_ref_ratio = amrex::Vector{{amrex::IntVect(AMREX_D_DECL(1, 1, 1))}}; } +#if !defined(AMREX_USE_EB) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(!eb_enabled, + "Embedded boundary solve requested but not compiled in"); +#endif + auto const finest_level = static_cast(curr.size()) - 1; // scale J appropriately; also determine if current is zero everywhere @@ -134,24 +141,18 @@ computeVectorPotential ( amrex::Vector > co // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { - amrex::MLEBNodeFDLaplacian linopx( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} -#endif - ); - amrex::MLEBNodeFDLaplacian linopy( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} -#endif - ); - amrex::MLEBNodeFDLaplacian linopz( - {geom[lev]}, {grids[lev]}, {dmap[lev]}, info -#if defined(AMREX_USE_EB) - , {eb_farray_box_factory.value()[lev]} + amrex::MLEBNodeFDLaplacian linopx, linopy, linopz; + if (eb_enabled) { +#ifdef AMREX_USE_EB + linopx.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); + linopy.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); + linopz.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info, {eb_farray_box_factory.value()[lev]}); #endif - ); + } else { + linopx.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + linopy.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + linopz.define({geom[lev]}, {grids[lev]}, {dmap[lev]}, info); + } amrex::Array linop = {&linopx,&linopy,&linopz}; amrex::Array,3> mlmg; @@ -163,9 +164,9 @@ computeVectorPotential ( amrex::Vector > co // Note: this assumes that beta is zero linop[adim]->setSigma({AMREX_D_DECL(1._rt, 1._rt, 1._rt)}); -#if defined(AMREX_USE_EB) // Set Homogeneous Dirichlet Boundary on EB - linop[adim]->setEBDirichlet(0_rt); +#if defined(AMREX_USE_EB) + if (eb_enabled) { linop[adim]->setEBDirichlet(0_rt); } #endif #ifdef WARPX_DIM_RZ diff --git a/setup.py b/setup.py index 713fb788319..efc18d900cf 100644 --- a/setup.py +++ b/setup.py @@ -195,7 +195,7 @@ def build_extension(self, ext): # consistent across platforms (especially Windows) WARPX_COMPUTE = env.pop("WARPX_COMPUTE", "OMP") WARPX_MPI = env.pop("WARPX_MPI", "OFF") -WARPX_EB = env.pop("WARPX_EB", "OFF") +WARPX_EB = env.pop("WARPX_EB", "ON") WARPX_OPENPMD = env.pop("WARPX_OPENPMD", "ON") WARPX_PRECISION = env.pop("WARPX_PRECISION", "DOUBLE") WARPX_PARTICLE_PRECISION = env.pop("WARPX_PARTICLE_PRECISION", WARPX_PRECISION)