diff --git a/.github/workflows/generate-reference-data.yml b/.github/workflows/generate-reference-data.yml index 91a2e6161..53f9e2f95 100644 --- a/.github/workflows/generate-reference-data.yml +++ b/.github/workflows/generate-reference-data.yml @@ -1,4 +1,4 @@ -name: Generate testpackage reference data +name: Generate testpackage reference data on Carrington on: # Only run when triggered manually @@ -12,8 +12,17 @@ jobs: runs-on: carrington steps: + - name: Clean workspace + run: | + RUN_STRING=$( cat << MORO + rm -rf libraries library-build testpackage + rm -f libraries.tar.zst testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt + rm -f *.xml + MORO + ) + srun -M carrington bash -c "$RUN_STRING" - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Make clean @@ -21,13 +30,16 @@ jobs: - uses: ursg/gcc-problem-matcher@master - name: Compile vlasiator (Testpackage build) run: | - export VLASIATOR_ARCH=carrington_gcc_openmpi - srun -M carrington --job-name tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 testpackage; sleep 10s' - - name: Sleep for 10 seconds - run: sleep 10s - shell: bash + srun -M carrington --job-name CI_ref_tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 testpackage; sleep 10s' + - name: Make sure the output binary is visible in lustre + uses: nick-fields/retry@v3 + with: + timeout_seconds: 15 + max_attempts: 3 + retry_on: error + command: ls vlasiator - name: Upload testpackage binary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: vlasiator-testpackage path: vlasiator @@ -45,11 +57,11 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Download testpackage binary - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: vlasiator-testpackage - name: Run testpackage @@ -60,6 +72,6 @@ jobs: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib # Fudge the run script sed -i 's/\/proj\/USERNAME\/BINARYNAME/$GITHUB_WORKSPACE\/vlasiator/' ./small_test_carrington.sh - sbatch -W -o testpackage_run_output.txt ./small_test_carrington.sh + sbatch -W -o testpackage_run_output.txt --job-name CI_ref_generate ./small_test_carrington.sh cat testpackage_run_output.txt # We don't bother to create artefacts or anything here. diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index b32f71110..de48a0953 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -9,41 +9,86 @@ on: # ... or when the workflow is started manually workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build_libraries: # Build libraries for the current version of the docker image # (to be used in any subsequent compilation and run jobs on said image) runs-on: ubuntu-latest - container: ursg/vlasiator_ci:20230220_1 + container: ursg/vlasiator_ci:20240131_1 steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run library build script run: ./build_libraries.sh - name: Build libraries tar run: tar --zstd -cvf libraries.tar.zstd libraries/ - name: Upload libraries as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: libraries path: libraries.tar.zstd retention-days: 5 if-no-files-found: error + build_libraries_appleM1: + # Build libraries on macos with M1 hardware, to test both macOS and aarch64 compatibility + runs-on: macos-14 + + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Setup MPI + uses: mpi4py/setup-mpi@v1 + with: + mpi: 'openmpi' + - name: Setup openmp-capable llvm + run: | + brew install llvm libomp + echo "CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV" + echo "OMPI_CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV" + echo "MPI_CXX=mpic++" >> "$GITHUB_ENV" + echo "CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV" + echo "OMPI_CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV" + echo "MPI_CC=mpicc" >> "$GITHUB_ENV" + - name: Run library build script + run: | + . ./build_libraries.sh appleM1 + - name: Build libraries tar + run: tar --zstd -cvf libraries-appleM1.tar.zstd libraries-appleM1 + - name: Upload libraries as artifact + uses: actions/upload-artifact@v4 + with: + name: libraries-appleM1 + path: libraries-appleM1.tar.zstd + retention-days: 5 + if-no-files-found: error + build_libraries_riscv: # Build libraries for the current version of the docker image # (to be used in any subsequent compilation and run jobs on said image) runs-on: risc-v steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + submodules: true + - name: Clean workspace + run: | + rm -rf libraries library-build libraries-arriesgado.tar.gz - name: Submit library job run: | - srun --interactive -p arriesgado-jammy -J vlasiator-libs -n 1 -c 4 --pty -t 01:00:00 bash -lc 'module load openmpi; cp ~/vlasiator/build_libraries.sh .; ./build_libraries.sh arriesgado' + #srun --interactive -p arriesgado-jammy -J vlasiator-libs -n 1 -c 4 -t 01:00:00 bash -lc 'module load openmpi; ./build_libraries.sh arriesgado' + ssh synth-hca 'source /etc/profile.d/lmod.sh; export PATH=$PATH:$HOME/bin; module load llvm/cross/EPI-0.7-development; cd '$GITHUB_WORKSPACE'; ./build_libraries.sh arriesgado' - name: Build libraries tar run: tar -czvf libraries-arriesgado.tar.gz libraries-arriesgado/ - name: Upload libraries as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: libraries-arriesgado path: libraries-arriesgado.tar.gz @@ -53,16 +98,16 @@ jobs: build_production: # Build Vlasiator with production flags runs-on: ubuntu-latest - container: ursg/vlasiator_ci:20230220_1 + container: ursg/vlasiator_ci:20240131_1 needs: build_libraries steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Download libraries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: libraries - name: Unpack libraries @@ -72,9 +117,12 @@ jobs: run: | VLASIATOR_ARCH=github_actions make clean VLASIATOR_ARCH=github_actions make -j 3 - # - name: Compile vlasiator (Release build) - # run: | - # srun -M carrington --job-name release_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make clean; make -j 16' + - name: Upload release binary + uses: actions/upload-artifact@v4 + with: + name: vlasiator-release + path: vlasiator + if-no-files-found: error build_testpackage: @@ -85,11 +133,15 @@ jobs: steps: - name: Clean workspace run: | + RUN_STRING=$( cat << MORO rm -rf libraries library-build testpackage - rm -f libraries.tar.zst dtestpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt + rm -f libraries.tar.zst testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt rm -f *.xml + MORO + ) + srun -M carrington bash -c "$RUN_STRING" - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Make clean @@ -98,18 +150,22 @@ jobs: - name: Compile vlasiator (Testpackage build) run: | export VLASIATOR_ARCH=carrington_gcc_openmpi - srun -M carrington --job-name tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 testpackage; sleep 10s' - - name: Sleep for 10 seconds - run: sleep 10s - shell: bash + srun -M carrington --job-name CI_tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 testpackage; sleep 10s' + - name: Make sure the output binary is visible in lustre + uses: nick-fields/retry@v3 + with: + timeout_seconds: 15 + max_attempts: 3 + retry_on: error + command: ls vlasiator - name: Upload testpackage binary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: vlasiator-testpackage path: vlasiator if-no-files-found: error #- name: Upload build log - # uses: actions/upload-artifact@v3 + # uses: actions/upload-artifact@v4 # with: # name: Testpackage build log # path: build.log @@ -120,29 +176,70 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true + - name: Clean workspace + run: | + rm -rf libraries library-build libraries-arriesgado.tar.gz + rm -f stdout.txt stderr.txt metrics.txt + rm -f *.xml - name: Download libraries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: libraries-arriesgado - name: Unpack libraries run: tar -xzvf libraries-arriesgado.tar.gz - - name: STOPGAP remove conflicting submoduled header libraries from the tarball extract - run: rm ./libraries-arriesgado/include/fsgrid.hpp ./libraries-arriesgado/include/dccrg.hpp - name: Make clean run: VLASIATOR_ARCH=arriesgado make clean - name: Compile vlasiator (RISC-V) run: | - srun --interactive -p arriesgado-jammy -J vlasiator_build -n 1 -c 4 --pty -t 01:00:00 bash -lc 'module load boost papi openmpi; export VLASIATOR_ARCH=arriesgado; make -j4' + #srun --interactive -p arriesgado-jammy -J vlasiator_build -n 1 -c 4 --pty -t 01:00:00 bash -lc 'module load boost papi openmpi; export VLASIATOR_ARCH=arriesgado; make -j4' + ssh synth-hca 'source /etc/profile.d/lmod.sh; export PATH=$PATH:$HOME/bin; module load llvm/cross/EPI-0.7-development; export VLASIATOR_ARCH=arriesgado; cd '$GITHUB_WORKSPACE'; make -j 12' - name: Upload riscv binary - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: vlasiator-riscv path: vlasiator if-no-files-found: error + build_appleM1: + runs-on: macos-14 + needs: build_libraries_appleM1 + + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + submodules: true + - name: Setup MPI + uses: mpi4py/setup-mpi@v1 + with: + mpi: 'openmpi' + - name: Setup openmp-capable llvm + run: | + brew install llvm libomp + echo "CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV" + echo "OMPI_CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV" + echo "CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV" + echo "OMPI_CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV" + - name: Install boost + run: | + brew install boost + ln -s /opt/homebrew/Cellar/boost/* /opt/homebrew/Cellar/boost/latest + - name: Download libraries + uses: actions/download-artifact@v4 + with: + name: libraries-appleM1 + - name: Unpack libraries + run: tar --zstd -xvf libraries-appleM1.tar.zstd + - name: Make clean + run: VLASIATOR_ARCH=appleM1 make clean + - name: Compile vlasiator (Release build) + run: | + VLASIATOR_ARCH=appleM1 make clean + VLASIATOR_ARCH=appleM1 make -j 3 + build_tools: # Build vlsvdiff and vlsvextract for testpackage use runs-on: carrington @@ -150,11 +247,15 @@ jobs: steps: - name: Clean workspace run: | + RUN_STRING=$( cat << MORO rm -rf libraries library-build testpackage - rm -f libraries.tar.zst dtestpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt + rm -f libraries.tar.zstd testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt rm -f *.xml + MORO + ) + srun -M carrington bash -c "$RUN_STRING" - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - uses: ursg/gcc-problem-matcher@master @@ -163,9 +264,9 @@ jobs: - name: Compile tools run: | export VLASIATOR_ARCH=carrington_gcc_openmpi - srun -M carrington --job-name tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 vlsvextract vlsvdiff' + srun -M carrington --job-name CI_tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 vlsvextract vlsvdiff' - name: Upload tools binaries - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: vlasiator-tools path: | @@ -181,15 +282,15 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - submodules: true + submodules: false - name: Download testpackage binary - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: vlasiator-testpackage - name: Download tools - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: vlasiator-tools - name: Run testpackage @@ -199,18 +300,35 @@ jobs: chmod +x $GITHUB_WORKSPACE/vlsv*_DP cd testpackage export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib - #srun -M carrington -t 01:30:00 --job-name CI_testpackage --interactive --nodes=1 -c 4 -n 16 --mem-per-cpu=5G -p short ./small_test_carrington_github_ci.sh sbatch -W -o testpackage_run_output.txt ./small_test_carrington_github_ci.sh + PARSE_OUTPUT_CMD=$( cat << MORO + echo "Job finished, checking output." cat testpackage_run_output.txt cat $GITHUB_STEP_SUMMARY > $GITHUB_WORKSPACE/testpackage_check_description.txt cd $GITHUB_WORKSPACE - tar -czvf testpackage-output.tar.gz testpackage_check_description.txt testpackage_output_variables.txt + ls -halB testpackage_check_description.txt + tar -czf testpackage-output.tar.gz testpackage_check_description.txt testpackage_output_variables.txt + MORO + ) + srun --job-name CI_package_results -M carrington -N 1 -c 1 --mem=1G bash -c "$PARSE_OUTPUT_CMD" if [ -f $GITHUB_WORKSPACE/testpackage_failed ]; then # Fail this step if any test failed. exit 1 fi + - name: Scancel dangling job upon cancellation + if: cancelled() + run: | + # Try accessing the job id echoed by the job script. + scancel ${{ steps.run.outputs.SLURM_JOB_ID }} + - name: Make sure the output tarball is visible in lustre + uses: nick-fields/retry@v3 + with: + timeout_seconds: 15 + max_attempts: 3 + retry_on: error + command: ls $GITHUB_WORKSPACE/testpackage-output.tar.gz - name: Upload testpackage output - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: testpackage-output @@ -219,33 +337,158 @@ jobs: # (to produce Checks against pull requests) build_ionosphereTests: - # Build IonosphereSolverTests miniApp (currently broken) - runs-on: carrington - #container: ursg/vlasiator_ci:20230220_1 - needs: [build_libraries] + # Build IonosphereSolverTests miniApp + runs-on: ubuntu-latest + container: ursg/vlasiator_ci:20240131_1 + needs: build_libraries steps: - name: Checkout source - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true # - name: Download libraries - # uses: actions/download-artifact@v3 + # uses: actions/download-artifact@v4 # with: # name: libraries # - name: Unpack libraries # run: tar --zstd -xvf libraries.tar.zstd # - uses: ursg/gcc-problem-matcher@master + - name: Download libraries + uses: actions/download-artifact@v4 + with: + name: libraries + - name: Unpack libraries + run: tar --zstd -xvf libraries.tar.zstd - name: Compile ionosphereSolverTests run: | cd mini-apps/ionosphereSolverTests/ - srun -M carrington --job-name iST_compile --interactive --nodes=1 -n 1 -c 3 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 3 main differentialFlux sigmaProfiles' - # VLASIATOR_ARCH=carrington_gcc_openmpi make -j 3 main differentialFlux sigmaProfiles + VLASIATOR_ARCH=github_actions make -j 3 - name: Upload ionosphereTest binaries - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: vlasiator-tools + name: ionosphereTests path: | mini-apps/ionosphereSolverTests/main mini-apps/ionosphereSolverTests/differentialFlux mini-apps/ionosphereSolverTests/sigmaProfiles if-no-files-found: error + + run_ionosphere_LFMtest: + runs-on: ubuntu-latest + container: ursg/vlasiator_ci:20240131_1 + needs: build_ionosphereTests + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + submodules: false + - name: Download ionosphereTests + uses: actions/download-artifact@v4 + with: + name: ionosphereTests + - name: checkout Analysator + run: | + apt update + apt -y install python3 python3-numpy python3-matplotlib gnuplot-nox + cd mini-apps/ionosphereSolverTests/ + git clone https://github.com/fmihpc/analysator + - name: Run LFMtest + run: | + chmod +x $GITHUB_WORKSPACE/main + cd mini-apps/ionosphereSolverTests/ + OMP_NUM_THREADS=1 $GITHUB_WORKSPACE/main -N 2000 -r 40 90 -r 50 90 -r 60 80 -sigma 10 -fac merkin2010 -gaugeFix equator45 -o LFMtest.vlsv -maxIter 10000 + export PYTHONPATH=$PYTHONPATH:analysator + export PTNONINTERACTIVE=1 + python3 ./lfmformat.py LFMtest.vlsv > lfmtest.dat + ./plot_LFMtest.gp + - name: Run atmospheric profile test + run: | + chmod +x $GITHUB_WORKSPACE/sigmaProfiles + cd mini-apps/ionosphereSolverTests/ + $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e7 > atmosphere.dat + $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e8 > atmosphere10.dat + $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e9 > atmosphere100.dat + ./plotAtmosphere.gp + - name: Upload ionosphereTest result images + uses: actions/upload-artifact@v4 + with: + name: ionosphereTestResults + path: | + mini-apps/ionosphereSolverTests/LFMtest.png + mini-apps/ionosphereSolverTests/atmosphere.png + if-no-files-found: error + + check_cfg_files: + runs-on: ubuntu-latest + container: ursg/vlasiator_ci:20240131_1 + needs: [build_libraries, build_production] + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + submodules: false + - name: Download libraries + uses: actions/download-artifact@v4 + with: + name: libraries + - name: Unpack libraries + run: tar --zstd -xvf libraries.tar.zstd + - name: Download release binary + uses: actions/download-artifact@v4 + with: + name: vlasiator-release + - name: Make release binary executable + run: | + chmod +x $GITHUB_WORKSPACE/vlasiator + - name: Check cfg files in projects/ + run: | + retval=0 + set +e + for cfg in `find projects -name \*.cfg`; do + echo -n "Testing cfg file $cfg... " + PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out + if [ $? != 0 ]; then + if `echo $cfg | grep -sq unsupported`; then + echo "failed (unsurprising)" + else + echo "FAILED:"; + cat cfgtest.out; + retval=127 + fi + else + echo "ok." + fi; + done + exit $retval + - name: Check cfg files in samples/ + run: | + retval=0 + set +e + for cfg in `find samples -name \*.cfg`; do + echo -n "Testing cfg file $cfg... " + PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out + if [ $? != 0 ]; then + echo "FAILED:"; + cat cfgtest.out; + retval=127 + else + echo "ok." + fi; + done + exit $retval + - name: Check cfg files in testpackage/ + run: | + retval=0 + set +e + for cfg in `find testpackage -name \*.cfg`; do + echo -n "Testing cfg file $cfg... " + PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out + if [ $? != 0 ]; then + echo "FAILED:"; + cat cfgtest.out; + retval=127 + else + echo "ok." + fi; + done + exit $retval diff --git a/.github/workflows/sync-hlrs-gitlab.yml b/.github/workflows/sync-hlrs-gitlab.yml new file mode 100644 index 000000000..2d4ca19fd --- /dev/null +++ b/.github/workflows/sync-hlrs-gitlab.yml @@ -0,0 +1,31 @@ +name: Synchronize HLRS Gitlab data + +on: + push: + branches : ["dev","master","vlasiator_gpu"] + +jobs: + push_to_gitlab: + if: github.repository_owner == 'fmihpc' + runs-on: ubuntu-latest + + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Add Gitlab Remote + run: | + git remote add gitlab https://oauth2:${token}@codehub.hlrs.de/coes/plasma-pepsc/uoh/vlasiator.git + git fetch gitlab + env: + token: ${{ secrets.GITLAB_ACCESS_TOKEN }} + - name: Update master + run: | + git push -f gitlab origin/master:master + - name: Update dev + run: | + git push -f gitlab origin/dev:dev + - name: Update Vlasiator_gpu + run: | + git push -f gitlab origin/vlasiator_gpu:vlasiator_gpu diff --git a/.gitmodules b/.gitmodules index bf2f8839e..3a7e1c23c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,17 @@ [submodule "submodules/fsgrid"] path = submodules/fsgrid - url = git@github.com:fmihpc/fsgrid.git + url = https://github.com/fmihpc/fsgrid.git [submodule "submodules/dccrg"] path = submodules/dccrg - url = git@github.com:fmihpc/dccrg.git + url = https://github.com/fmihpc/dccrg.git branch = vlasiator-version +[submodule "submodules/eigen"] + path = submodules/eigen + url = https://gitlab.com/libeigen/eigen.git + branch = master + [submodule "submodules/vectorclass"] path = submodules/vectorclass url = https://github.com/vectorclass/version2 diff --git a/MAKE/Makefile.Freezer b/MAKE/Makefile.Freezer index 5cdc5ffa3..0af052209 100644 --- a/MAKE/Makefile.Freezer +++ b/MAKE/Makefile.Freezer @@ -13,13 +13,11 @@ LNK = mpic++ ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision VECTORCLASS = VEC8F_AGNER - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ # VECTORCLASS = VEC8F_FALLBACK # INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/../vlasiator/vlasovsolver else #Double-precision VECTORCLASS = VEC4D_AGNER - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ # VECTORCLASS = VEC4D_FALLBACK # INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/../vlasiator/vlasovsolver endif @@ -70,14 +68,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv/lib LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - -#header libraries - -INC_EIGEN = -isystem $(LIBRARY_PREFIX)/eigen -#INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9 -#INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg_new_neighbours -INC_DCCRG = -I./submodules/dccrg -INC_FSGRID = -I./submodules/fsgrid - - - diff --git a/MAKE/Makefile.MN5_gpp b/MAKE/Makefile.MN5_gpp new file mode 100644 index 000000000..679acdc92 --- /dev/null +++ b/MAKE/Makefile.MN5_gpp @@ -0,0 +1,92 @@ +# -*- mode: makefile -*- +# GCC +CMP = mpicxx +LNK = mpicxx + +# Intel +#CMP = mpiicpx +#LNK = mpiicpx + +# Starting Makefile for the general partition of the MareNostrum5 supercomputer located at the BSC. + +#======== Vectorization ========== +# Set vector backend type for vlasov solvers, sets precision and length. +# NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) +# Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) + # Single-precision + VECTORCLASS = VEC16F_AGNER +else + # Double-precision + VECTORCLASS = VEC8D_AGNER +endif + +# GNU flags: +CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx2 +testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mavx + +MATHFLAGS = -ffast-math -fno-finite-math-only +testpackage: MATHFLAGS = -fno-unsafe-math-optimizations + +# GCC +LDFLAGS = -fopenmp -Wl,--as-needed +LIB_MPI = -lgomp + +# Intel +#LDFLAGS = -qopenmp -liomp -Wl,--as-needed + +#======== Allocator ========= +# Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +# Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker + +#-DNO_WRITE_AT_ALL: Define to disable write at all to +# avoid memleak (much slower IO) +#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition +# errors that come up when using +# mpi.h in c++ on Cray + +CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK +testpackage: CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK + +# BOOST_VERSION = current trilinos version +# ZOLTAN_VERSION = current trilinos verson + +#======== Libraries =========== + +# Adjust to where your own libraries/dependencies have been built. +LIBRARY_PREFIX = /gpfs/projects/bsc33/bsc033626/LIBS + +# Compiled libraries +INC_BOOST = -I/apps/GPP/BSCTOOLS/deps/boost/1.84.0/include +LIB_BOOST = -L/apps/GPP/BSCTOOLS/deps/boost/1.84.0/lib -lboost_program_options + +INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/VLASIATOR.GCC.DEV/include +LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/VLASIATOR.GCC.DEV/lib -lzoltan + +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/VLASIATOR.GCC.DEV-5.3.0/include +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/VLASIATOR.GCC.DEV-5.3.0/lib -ljemalloc + +INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv/VLASIATOR.GCC.DEV/include +LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv/VLASIATOR.GCC.DEV/lib -lvlsv + +INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/VLASIATOR.GCC.DEV/include +LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/VLASIATOR.GCC.DEV/lib -lphiprof + +# Header libraries + +INC_EIGEN = -isystem ./submodules/eigen/ +INC_DCCRG = -I./submodules/dccrg +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ +INC_FSGRID = -I./submodules/fsgrid diff --git a/MAKE/Makefile.appleM1 b/MAKE/Makefile.appleM1 new file mode 100644 index 000000000..7f8aa15e3 --- /dev/null +++ b/MAKE/Makefile.appleM1 @@ -0,0 +1,87 @@ +CMP = mpic++ +LNK = mpic++ + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + #VECTORCLASS = VEC_FALLBACK_GENERIC + #VECTORCLASS = VEC8F_AGNER + VECTORCLASS = VEC8F_FALLBACK +else +#Double-precision + #VECTORCLASS = VEC4D_AGNER + #VECTORCLASS = VEC_FALLBACK_GENERIC + VECTORCLASS = VEC8D_FALLBACK +endif + +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +#CXXFLAGS += -DPAPI_MEM # Papi does not work on MacOS, see https://stackoverflow.com/questions/69531604/installing-papi-on-macos + + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + + +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker + +#-DNO_WRITE_AT_ALL: Define to disable write at all to +# avoid memleak (much slower IO) +#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition +# errors that come up when using +# mpi.h in c++ on Cray + +FLAGS = +CXXFLAGS += -I ${GITHUB_WORKSPACE}/libraries-appleM1/include -I/opt/homebrew/opt/llvm/include -I/opt/homebrew/opt/libomp/include -march=native -O3 -std=c++17 -fopenmp -Wall -Wno-unused -g +MATHFLAGS = # -ffast-math -fno-finite-math-only +testpackage: MATHFLAGS = # -fno-unsafe-math-optimizations +LDFLAGS = -fopenmp -L/opt/homebrew/opt/llvm/lib -L/opt/homebrew/opt/libomp/lib -L/opt/homebrew/opt/llvm/lib/c++ -g +LIB_MPI = + + +#======== Libraries =========== + +LIBRARY_PREFIX = ${GITHUB_WORKSPACE}/libraries-appleM1 + +INC_BOOST = -I /opt/homebrew/Cellar/boost/latest/include +LIB_BOOST = -L /opt/homebrew/Cellar/boost/latest/lib -lboost_program_options + +INC_JEMALLOC = -I $(LIBRARY_PREFIX)/include +LIB_JEMALLOC = $(LIBRARY_PREFIX)/lib/libjemalloc.a + +INC_ZOLTAN = +LIB_ZOLTAN = -lzoltan + +INC_VLSV = -I $(LIBRARY_PREFIX)/include +LIB_VLSV = -L $(LIBRARY_PREFIX)/lib -lvlsv + +INC_SILO = +LIB_SILO = -lsiloh5 + +INC_PAPI = +LIB_PAPI = + +INC_DCCRG = -I./submodules/dccrg + +INC_FSGRID = -I./submodules/fsgrid + +LIB_PROFILE = -I $(LIBRARY_PREFIX)/include ${LIBRARY_PREFIX}/lib/libphiprof.a +INC_PROFILE = +INC_TOPO = + +INC_EIGEN = -isystem ./submodules/eigen/ + +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.arriesgado b/MAKE/Makefile.arriesgado index 96a0dda42..933ab90a6 100644 --- a/MAKE/Makefile.arriesgado +++ b/MAKE/Makefile.arriesgado @@ -2,7 +2,7 @@ # # Note: # o Load modules: -# module load boost eigen papi openmpi +# module load boost papi openmpi # o Zoltan's configure script requires a "--build=arm-linux-gnu" parameter. # (Yes, this needs to be arm, not riscv!) @@ -39,21 +39,14 @@ LIB_MPI = LIBRARY_PREFIX = libraries-arriesgado -INC_BOOST = -LIB_BOOST = ${BOOST_LIBS} -lboost_program_options +INC_BOOST = -I/apps/riscv/boost/1.77.0/include +LIB_BOOST = -L/apps/riscv/boost/1.77.0/lib -lboost_program_options INC_ZOLTAN = -I${LIBRARY_PREFIX}/include LIB_ZOLTAN = -L${LIBRARY_PREFIX}/lib -lzoltan -INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv +INC_VLSV = -I$(LIBRARY_PREFIX)/include +LIB_VLSV = -L$(LIBRARY_PREFIX)/lib -lvlsv -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - -LIB_PROFILE = -L${LIBRARY_PREFIX}/phiprof/lib -lphiprof -Wl,-rpath=${LIBRARY_PREFIX}/lib -INC_PROFILE = -I${LIBRARY_PREFIX}/phiprof/include -INC_TOPO = - -INC_EIGEN = ${EIGEN3_INCL} +LIB_PROFILE = -L${LIBRARY_PREFIX}/lib -lphiprof -Wl,-rpath=${LIBRARY_PREFIX}/lib +INC_PROFILE = -I${LIBRARY_PREFIX}/include diff --git a/MAKE/Makefile.bsc_jetsontx b/MAKE/Makefile.bsc_jetsontx index 6cdc85d3a..bd7a08e1f 100644 --- a/MAKE/Makefile.bsc_jetsontx +++ b/MAKE/Makefile.bsc_jetsontx @@ -2,7 +2,7 @@ # # Note: # o Load modules: -# module load boost eigen +# module load boost # o Zoltan's configure script requires a "--build=arm-linux-gnu" parameter. CMP = mpic++ @@ -51,12 +51,8 @@ LIB_ZOLTAN = -L${LIBRARY_PREFIX}/lib -lzoltan INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - LIB_PROFILE = -L${LIBRARY_PREFIX}/phiprof/lib -lphiprof -Wl,-rpath=${LIBRARY_PREFIX}/phiprof/lib INC_PROFILE = -I${LIBRARY_PREFIX}/phiprof/include INC_TOPO = -INC_EIGEN = ${EIGEN3_INCL} +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.carrington_gcc_openmpi b/MAKE/Makefile.carrington_gcc_openmpi index fbb7667db..1e347c9d5 100644 --- a/MAKE/Makefile.carrington_gcc_openmpi +++ b/MAKE/Makefile.carrington_gcc_openmpi @@ -110,10 +110,3 @@ INC_ZOLTAN = -isystem $(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BR #LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/lib #INC_PAPI = -isystem $(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/include LIB_PAPI = -lpapi - -#header libraries -INC_EIGEN = -isystem $(LIBRARY_PREFIX)/ -isystem $(LIBRARY_PREFIX)/Eigen/ -INC_FSGRID = -I./submodules/fsgrid/ -INC_DCCRG = -I./submodules/dccrg/ -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - diff --git a/MAKE/Makefile.cubbli20 b/MAKE/Makefile.cubbli20 index e8b74bb77..e2aa53fa2 100644 --- a/MAKE/Makefile.cubbli20 +++ b/MAKE/Makefile.cubbli20 @@ -53,11 +53,9 @@ LIB_JEMALLOC = #-L$(LIBRARY_PREFIX)/lib -ljemalloc ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ VECTORCLASS = VEC4F_FALLBACK else #Double-precision - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ VECTORCLASS = VEC4D_FALLBACK endif @@ -97,12 +95,8 @@ LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/Zoltan/build/lib -lzoltan INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-3.4.0 +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.docker b/MAKE/Makefile.docker index b4a12831c..f1fa8833c 100644 --- a/MAKE/Makefile.docker +++ b/MAKE/Makefile.docker @@ -84,10 +84,3 @@ LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof-2.0-beta/lib -lphiprof INC_PAPI = -I$(LIBRARY_PREFIX)/papi/include LIB_PAPI = -L$(LIBRARY_PREFIX)/papi/lib -lpapi - -#header libraries - -INC_EIGEN = -I/usr/include/eigen3 -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ -INC_FSGRID = -I./submodules/fsgrid diff --git a/MAKE/Makefile.github_actions b/MAKE/Makefile.github_actions index e73affa8d..5dd10cf33 100644 --- a/MAKE/Makefile.github_actions +++ b/MAKE/Makefile.github_actions @@ -47,7 +47,7 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE # mpi.h in c++ on Cray FLAGS = -CXXFLAGS += -I ${GITHUB_WORKSPACE}/libraries/include -I/usr/include/eigen3 -L ${GITHUB_WORKSPACE}/libraries/lib -march=native -O3 -std=c++17 -fopenmp -Wall -Wno-unused -fabi-version=0 -mavx -g -Wno-cast-function-type +CXXFLAGS += -I ${GITHUB_WORKSPACE}/libraries/include -L ${GITHUB_WORKSPACE}/libraries/lib -march=native -O3 -std=c++17 -fopenmp -Wall -Wno-unused -fabi-version=0 -mavx -g -Wno-cast-function-type MATHFLAGS = # -ffast-math -fno-finite-math-only testpackage: MATHFLAGS = # -fno-unsafe-math-optimizations LDFLAGS = -L ${GITHUB_WORKSPACE}/libraries/lib -Wl,-rpath=${GITHUB_WORKSPACE}/libraries/lib -g @@ -76,12 +76,12 @@ LIB_SILO = -lsiloh5 INC_PAPI = LIB_PAPI = -lpapi -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - LIB_PROFILE = -I $(LIBRARY_PREFIX)/include ${GITHUB_WORKSPACE}/libraries/lib/libphiprof.a INC_PROFILE = INC_TOPO = -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ +# Works without but as an example: arch-specific INCLUDE paths be like this: +INC_DCCRG = -I${GITHUB_WORKSPACE}/submodules/dccrg +INC_FSGRID = -I${GITHUB_WORKSPACE}/submodules/fsgrid +INC_VECTORCLASS = -isystem ${GITHUB_WORKSPACE}/submodules/vectorclass/ -isystem ${GITHUB_WORKSPACE}/submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -isystem ${GITHUB_WORKSPACE}/submodules/eigen/ diff --git a/MAKE/Makefile.hawk_gcc_mpt b/MAKE/Makefile.hawk_gcc_mpt index 6f95ff982..93d6e7149 100644 --- a/MAKE/Makefile.hawk_gcc_mpt +++ b/MAKE/Makefile.hawk_gcc_mpt @@ -22,13 +22,11 @@ LNK = mpicxx ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision VECTORCLASS = VEC8F_AGNER - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ # VECTORCLASS = VEC8F_FALLBACK # INC_VECTORCLASS = -I$(HOME)/vlasiator/vlasovsolver else #Double-precision VECTORCLASS = VEC4D_AGNER - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ # VECTORCLASS = VEC4D_FALLBACK # VECTORCLASS = VEC8D_FALLBACK # INC_VECTORCLASS = -I$(HOME)/vlasiator/vlasovsolver @@ -97,11 +95,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_B INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/include LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/lib -lpapi - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -INC_DCCRG = -I./submodules/dccrg -INC_FSGRID = -I./submodules/fsgrid -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - diff --git a/MAKE/Makefile.hawk_gcc_openmpi b/MAKE/Makefile.hawk_gcc_openmpi index f859aa156..1d3aacc2d 100644 --- a/MAKE/Makefile.hawk_gcc_openmpi +++ b/MAKE/Makefile.hawk_gcc_openmpi @@ -79,14 +79,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_B INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/include LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/lib -lpapi - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -INC_DCCRG = -I./submodules/dccrg -INC_FSGRID = -I./submodules/fsgrid -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.hawk_intel_mpt b/MAKE/Makefile.hawk_intel_mpt index f6dea4821..46047ab1d 100644 --- a/MAKE/Makefile.hawk_intel_mpt +++ b/MAKE/Makefile.hawk_intel_mpt @@ -80,14 +80,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_B INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/include LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/lib -lpapi - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -INC_DCCRG = -I./submodules/dccrg -INC_FSGRID = -I./submodules/fsgrid -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.hawk_intel_openmpi b/MAKE/Makefile.hawk_intel_openmpi index ef65a88b0..c7286832e 100644 --- a/MAKE/Makefile.hawk_intel_openmpi +++ b/MAKE/Makefile.hawk_intel_openmpi @@ -80,14 +80,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_B INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/include LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/lib -lpapi - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -INC_DCCRG = -I./submodules/dccrg -INC_FSGRID = -I./submodules/fsgrid -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.home b/MAKE/Makefile.home index ee505f66e..25c2762d9 100644 --- a/MAKE/Makefile.home +++ b/MAKE/Makefile.home @@ -27,12 +27,9 @@ LIB_VLSV = -lvlsv INC_SILO = -I${HOME}/include LIB_SILO = -L${HOME}/lib -lsilo -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - LIB_PROFILE = -L${HOME}/phiprof/lib -lphiprof INC_PROFILE = -I${HOME}/phiprof/include INC_TOPO = -INC_EIGEN = -I${HOME}/include +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.horakons b/MAKE/Makefile.horakons index b8cfa3286..4c4f07d8d 100644 --- a/MAKE/Makefile.horakons +++ b/MAKE/Makefile.horakons @@ -59,8 +59,6 @@ PAPI_FLAG = LIBRARY_PREFIX = /home/horakons/vlasiator_libs/ -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - INC_BOOST = -I/usr/include/boost/ LIB_BOOST = -L/usr/lib/x86_64-linux-gnu/ -lboost_program_options @@ -73,10 +71,6 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv INC_SILO = -I$(LIBRARY_PREFIX)/include LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - INC_PAPI = -I$(LIBRARY_PREFIX)/papi_install/include/ LIB_PAPI = -L$(LIBRARY_PREFIX)/papi_install/lib/ -lpapi @@ -86,5 +80,5 @@ LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-3.2.8/ - +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.kstppd b/MAKE/Makefile.kstppd index 02ac35f61..8a21759c1 100644 --- a/MAKE/Makefile.kstppd +++ b/MAKE/Makefile.kstppd @@ -59,8 +59,6 @@ PAPI_FLAG = LIBRARY_PREFIX = $(HOME) -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - INC_BOOST = -I/usr/include/boost/ LIB_BOOST = -L/usr/lib/x86_64-linux-gnu/ -lboost_program_options @@ -73,10 +71,6 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv INC_SILO = -I$(LIBRARY_PREFIX)/include LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - INC_PAPI = -I$(LIBRARY_PREFIX)/papi-5.4.3/src LIB_PAPI = -L$(LIBRARY_PREFIX)/papi-5.4.3/src -lpapi @@ -86,5 +80,5 @@ LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/libraries/jemalloc/lib -ljemalloc LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9 - +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.lumi_cpeGnu b/MAKE/Makefile.lumi_cpeGnu index 52d7ab766..b3001a442 100644 --- a/MAKE/Makefile.lumi_cpeGnu +++ b/MAKE/Makefile.lumi_cpeGnu @@ -84,9 +84,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX2)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - -#header libraries -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -isystem $(LIBRARY_PREFIX_HEADERS)/eigen/ -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.lumi_cray b/MAKE/Makefile.lumi_cray index b18b975a4..54ec178c9 100644 --- a/MAKE/Makefile.lumi_cray +++ b/MAKE/Makefile.lumi_cray @@ -78,10 +78,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.lumi_gnu b/MAKE/Makefile.lumi_gnu index 1fc1eab29..df4c9d0eb 100644 --- a/MAKE/Makefile.lumi_gnu +++ b/MAKE/Makefile.lumi_gnu @@ -77,10 +77,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.lumi_gnu358 b/MAKE/Makefile.lumi_gnu358 index e9f8a9cb5..ed69e4cd2 100644 --- a/MAKE/Makefile.lumi_gnu358 +++ b/MAKE/Makefile.lumi_gnu358 @@ -77,10 +77,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid/ -INC_DCCRG = -I./submodules/dccrg/ -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.mahti_gcc b/MAKE/Makefile.mahti_gcc index 876a5b887..15ed497f7 100644 --- a/MAKE/Makefile.mahti_gcc +++ b/MAKE/Makefile.mahti_gcc @@ -83,9 +83,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - -# header libraries -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -I$(LIBRARY_PREFIX)/../libraries/eigen/ -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.marconi b/MAKE/Makefile.marconi index 719e48601..0a504924d 100644 --- a/MAKE/Makefile.marconi +++ b/MAKE/Makefile.marconi @@ -83,12 +83,3 @@ LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc -Wl,-rpath,$(LIBRARY_ #system INC_PAPI = -I$(LIBRARY_PREFIX)/papi/include LIB_PAPI = -L$(LIBRARY_PREFIX)/papi/lib -lpapi -lpfm - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -I$(HEADER_LIBRARY_PREFIX)/eigen -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - diff --git a/MAKE/Makefile.perlmutter_gcc b/MAKE/Makefile.perlmutter_gcc index 7e9b16e73..56c86af12 100644 --- a/MAKE/Makefile.perlmutter_gcc +++ b/MAKE/Makefile.perlmutter_gcc @@ -2,7 +2,7 @@ CMP = CC LNK = CC # Modules loaded -# module load PrgEnv-gnu; module load eigen; module load papi +# module load PrgEnv-gnu; module load papi #======== Vectorization ========== #Set vector backend type for vlasov solvers, sets precision and length. @@ -79,10 +79,3 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.puhti_gcc b/MAKE/Makefile.puhti_gcc index 3c0dde2b1..931d42c67 100644 --- a/MAKE/Makefile.puhti_gcc +++ b/MAKE/Makefile.puhti_gcc @@ -81,15 +81,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND LIB_PAPI = -L$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib INC_PAPI = -I$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/3.3.7/ -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.puhti_intel b/MAKE/Makefile.puhti_intel index 7542dc511..bd9743efb 100644 --- a/MAKE/Makefile.puhti_intel +++ b/MAKE/Makefile.puhti_intel @@ -85,15 +85,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND LIB_PAPI = -L$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib INC_PAPI = -I$(LIBRARY_PREFIX)/hpcx-mpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/3.3.7/ -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.puhti_pgi b/MAKE/Makefile.puhti_pgi index 6d59d6ceb..91bf73351 100644 --- a/MAKE/Makefile.puhti_pgi +++ b/MAKE/Makefile.puhti_pgi @@ -84,15 +84,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_ #LIB_PAPI = -L$(LIBRARY_PREFIX)/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib #INC_PAPI = -I$(LIBRARY_PREFIX)/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/include - - -#header libraries - -INC_FSGRID = -I./submodules/fsgrid -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/3.3.7/ -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - - - - diff --git a/MAKE/Makefile.ukko_gcc_openmpi b/MAKE/Makefile.ukko_gcc_openmpi index 6c561001f..032621b2f 100644 --- a/MAKE/Makefile.ukko_gcc_openmpi +++ b/MAKE/Makefile.ukko_gcc_openmpi @@ -110,9 +110,3 @@ INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$( #LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/carrington/papi/lib #INC_PAPI = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/ukko/papi/include LIB_PAPI = -lpapi - -#header libraries -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -I$(LIBRARY_PREFIX)/Eigen/ -INC_FSGRID = -I./submodules/fsgrid -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ diff --git a/MAKE/Makefile.viper b/MAKE/Makefile.viper new file mode 100644 index 000000000..77375740e --- /dev/null +++ b/MAKE/Makefile.viper @@ -0,0 +1,91 @@ +CMP = mpic++ +LNK = mpic++ + +# Makefile for MPCDF's Viper system (https://docs.mpcdf.mpg.de/doc/computing/viper-user-guide.html) +# Modules loaded: +# module load gcc/13 openmpi +# module load papi +# module load boost + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC8F_AGNER +else +#Double-precision + VECTORCLASS = VEC4D_AGNER +endif + +FLAGS = + +#GNU flags: +CC_BRAND = gcc +CC_BRAND_VERSION = 9.3.0 +CXXFLAGS += -g -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mfma -mavx2 +testpackage: CXXFLAGS = -g -O2 -fopenmp -funroll-loops -std=c++20 -mno-fma -mno-avx2 -mno-avx + +MATHFLAGS = -ffast-math -fno-finite-math-only +testpackage: MATHFLAGS = -fno-unsafe-math-optimizations + +LDFLAGS = -lrt -fopenmp +LIB_MPI = -fopenmp + +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker + +#-DNO_WRITE_AT_ALL: Define to disable write at all to +# avoid memleak (much slower IO) + +# BOOST_VERSION = current trilinos version +# ZOLTAN_VERSION = current trilinos verson +# +#======== Libraries =========== + +LIBRARY_PREFIX = $(HOME)/vlasiator/libraries +LIBRARY_PREFIX_HEADERS = $(HOME)/vlasiator/libraries + +#compiled libraries +INC_BOOST = -I$(BOOST_HOME)/include +LIB_BOOST = -L$(BOOST_HOME)/lib -lboost_program_options -Wl,-rpath=$(BOOST_HOME)/lib + +INC_ZOLTAN = -isystem$(LIBRARY_PREFIX)/include +LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/lib -lzoltan -Wl,-rpath=$(LIBRARY_PREFIX)/lib + +INC_JEMALLOC = -isystem$(LIBRARY_PREFIX)/include +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc -Wl,-rpath=$(LIBRARY_PREFIX)/lib + +LIB_PAPI = -lpapi + +INC_VLSV = -I$(LIBRARY_PREFIX)/include +LIB_VLSV = -L$(LIBRARY_PREFIX)/lib -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/lib + +LIB_PROFILE = -L$(LIBRARY_PREFIX)/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/lib +INC_PROFILE = -I$(LIBRARY_PREFIX)/include + + +#header libraries + +INC_FSGRID = -I./submodules/fsgrid +INC_DCCRG = -I./submodules/dccrg +INC_VECTORCLASS = -isystem ./submodules/vectorclass/ -isystem ./submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -isystem ./submodules/eigen/ diff --git a/MAKE/Makefile.vorna_gcc b/MAKE/Makefile.vorna_gcc index ed8a5d821..a25225f10 100644 --- a/MAKE/Makefile.vorna_gcc +++ b/MAKE/Makefile.vorna_gcc @@ -89,12 +89,3 @@ INC_PROFILE = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$ LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/vorna/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/vorna/papi/lib INC_PAPI = -isystem $(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/$(MPT_BRAND)/$(MPT_VERSION)/vorna/papi/include - -#header libraries -INC_FSGRID = -I./submodules/fsgrid -INC_DCCRG = -I./submodules/dccrg -INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ -INC_EIGEN = -isystem $(LIBRARY_PREFIX)/ -I$(LIBRARY_PREFIX)/Eigen - - - diff --git a/MAKE/Makefile.yann b/MAKE/Makefile.yann index 68c1d0a05..4e97736cb 100644 --- a/MAKE/Makefile.yann +++ b/MAKE/Makefile.yann @@ -1,21 +1,19 @@ # For Yann's laptop #======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. +#Set vector backend type for vlasov solvers, sets precision and length. #NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: +#Options: # AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER # AVX512: VEC8D_AGNER, VEC16F_AGNER # Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - VECTORCLASS = VEC4F_FALLBACK +#Single-precision + VECTORCLASS = VEC8F_AGNER else #Double-precision - INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ - VECTORCLASS = VEC4D_FALLBACK + VECTORCLASS = VEC4D_AGNER endif @@ -29,14 +27,25 @@ PAPI_FLAG = FLAGS = #CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx -CXXFLAGS = -g3 -ggdb -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx +CXXFLAGS += -g3 -ggdb -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mfma -mavx2 -Wno-unknown-pragmas -Wno-sign-compare testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations MATHFLAGS = -ffast-math -fno-finite-math-only testpackage: MATHFLAGS = -fno-unsafe-math-optimizations LDFLAGS = -LIB_MPI = -lgomp +LIB_MPI = -lgomp -lpapi + +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======== Libraries =========== @@ -52,15 +61,9 @@ LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -INC_DCCRG = -I./submodules/dccrg - -INC_FSGRID = -I./submodules/fsgrid - INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/include LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include INC_TOPO = - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-2249f9c22fe8 diff --git a/Makefile b/Makefile index b92d8e63c..cfdb92645 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,12 @@ ifneq (,$(findstring testpackage,$(MAKECMDGOALS))) COMPFLAGS += -DIONOSPHERE_SORTED_SUMS -DINITIALIZE_ALIGNED_MALLOC_WITH_NAN endif +# Use submodules by default +# This can be overridden in architecture-specific makefiles +INC_FSGRID = -I./submodules/fsgrid/ +INC_DCCRG = -I./submodules/dccrg/ +INC_VECTORCLASS = -I ./submodules/vectorclass/ -I ./submodules/vectorclass-addon/vector3d/ +INC_EIGEN = -I ./submodules/eigen/ include MAKE/Makefile.${ARCH} @@ -156,10 +162,10 @@ OBJS = version.o memoryallocation.o backgroundfield.o quadr.o dipole.o linedipo project.o projectTriAxisSearch.o read_gaussian_population.o\ Alfven.o Diffusion.o Dispersion.o Distributions.o Firehose.o\ Flowthrough.o Fluctuations.o Harris.o KHB.o Larmor.o Magnetosphere.o MultiPeak.o\ - VelocityBox.o Riemann1.o Shock.o Template.o test_fp.o testHall.o test_trans.o\ + Riemann1.o Shock.o Template.o test_fp.o testHall.o test_trans.o\ IPShock.o object_wrapper.o\ verificationLarmor.o Shocktest.o grid.o ioread.o iowrite.o vlasiator.o logger.o\ - common.o parameters.o readparameters.o spatial_cell.o\ + common.o parameters.o readparameters.o spatial_cell_cpu.o\ vlasovmover.o $(FIELDSOLVER).o fs_common.o fs_limiters.o gridGlue.o # Include autogenerated dependency files, if they exist @@ -170,7 +176,7 @@ ifeq ($(MESH),VAMR) OBJS += cpu_moments.o else OBJS += cpu_acc_intersections.o cpu_acc_map.o cpu_acc_sort_blocks.o cpu_acc_load_blocks.o cpu_acc_semilag.o cpu_acc_transform.o \ - cpu_moments.o cpu_trans_map.o cpu_trans_map_amr.o + cpu_moments.o cpu_trans_map.o cpu_trans_map_amr.o cpu_trans_pencils.o endif # Add field solver objects diff --git a/build_libraries.sh b/build_libraries.sh index 48d35d059..d61032a60 100755 --- a/build_libraries.sh +++ b/build_libraries.sh @@ -9,6 +9,7 @@ if [[ z$1 != "z" ]]; then else PLATFORM="" fi +echo "Using platform $PLATFORM" # Clean up old library build dirs and libraries for this platform rm -rf library-build libraries${PLATFORM} @@ -22,13 +23,25 @@ cd library-build # Build phiprof git clone https://github.com/fmihpc/phiprof/ cd phiprof/src -make -j 4 CCC=mpic++ + +if [[ $PLATFORM == "-arriesgado" ]]; then + # Special workaround for missing include paths on arriesgado + make -j 4 CCC=mpic++ CCFLAGS="-I /usr/lib/gcc/riscv64-linux-gnu/11/include -fpic -O2 -std=c++17 -DCLOCK_ID=CLOCK_MONOTONIC -fopenmp -W -Wall -Wextra -pedantic" +elif [[ $PLATFORM == "-appleM1" ]]; then + make -j 4 CCC=mpic++ CC=appleLLVM CCFLAGS="-fpic -O2 -std=c++17 -DCLOCK_ID=CLOCK_MONOTONIC -fopenmp" LDFLAGS="-fopenmp" +else + make -j 4 CCC=mpic++ +fi cp ../include/* $WORKSPACE/libraries${PLATFORM}/include cp ../lib/* $WORKSPACE/libraries${PLATFORM}/lib cd ../.. # Build VLSV -git clone https://github.com/fmihpc/vlsv.git +if [[ $PLATFORM != "-appleM1" ]]; then + git clone https://github.com/fmihpc/vlsv.git +else + git clone -b appleM1Build https://github.com/ursg/vlsv.git +fi cd vlsv make cp libvlsv.a $WORKSPACE/libraries${PLATFORM}/lib @@ -36,7 +49,7 @@ cp *.h $WORKSPACE/libraries${PLATFORM}/include cd .. # Build papi -if [[ $PLATFORM != "-arriesgado" ]]; then # This fails on RISCV +if [[ $PLATFORM != "-arriesgado" && $PLATFORM != "-appleM1" ]]; then # This fails on RISCV and MacOS git clone https://github.com/icl-utk-edu/papi cd papi/src ./configure --prefix=$WORKSPACE/libraries${PLATFORM} && make -j 4 CC=gcc && make install @@ -54,12 +67,11 @@ cd .. git clone https://github.com/sandialabs/Zoltan.git mkdir zoltan-build cd zoltan-build -if [[ $PLATFORM != "-arriesgado" ]]; then - ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong && make -j 4 && make install +if [[ $PLATFORM == "-arriesgado" ]]; then + ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong --host=riscv64-unknown-linux-gnu --build=arm-linux-gnu && make -j 4 && make install +elif [[ $PLATFORM == "-appleM1" ]]; then + ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong CC=mpicc CXX=mpic++ && make -j 4 && make install else - ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong --build=arm-linux-gnu && make -j 4 && make install + ../Zoltan/configure --prefix=$WORKSPACE/libraries${PLATFORM} --enable-mpi --with-mpi-compilers --with-gnumake --with-id-type=ullong && make -j 4 && make install cd .. fi - -git clone https://gitlab.com/libeigen/eigen.git -cp -ua eigen/Eigen $WORKSPACE/libraries${PLATFORM}/include diff --git a/common.cpp b/common.cpp index 482522b2a..44ce0ad3a 100644 --- a/common.cpp +++ b/common.cpp @@ -88,15 +88,11 @@ void bailout( /*! Helper function for error handling. err_type default to 0.*/ [[ noreturn ]] void abort_mpi(const std::string str, const int err_type) { - int myRank; - MPI_Comm_rank(MPI_COMM_WORLD, &myRank); - if (myRank == MASTER_RANK) { - if (err_type == 0) { - std::cerr << str << std::endl; - } else { - std::cerr << __FILE__ << ":" << __LINE__ << ": " << str << std::endl; - } + // Single string so output isn't mangled by multiple processes + std::cerr << (err_type ? std::string(__FILE__) + ":" + std::to_string(__LINE__) + ": " + str : str) + "\n"; + MPI_Abort(MPI_COMM_WORLD, 1); - MPI_Abort(MPI_COMM_WORLD, 1); - } + // Dummy abort to convince compiler function doesn't return + // TODO replace with std::unreachable once we switch to C++23 + abort(); } diff --git a/common.h b/common.h index cc0fe58b7..7f35fa601 100644 --- a/common.h +++ b/common.h @@ -1,4 +1,4 @@ -/* +/* * This file is part of Vlasiator. * Copyright 2010-2016 Finnish Meteorological Institute * @@ -207,8 +207,8 @@ namespace CellParams { AMR_DPSQ, AMR_DBSQ, AMR_DB, - AMR_ALPHA, - AMR_JPERB, + AMR_ALPHA1, + AMR_ALPHA2, RECENTLY_REFINED, BULKV_FORCING_X, /*! Externally forced drift velocity (ex. from the ionosphere) */ BULKV_FORCING_Y, /*! Externally forced drift velocity (ex. from the ionosphere) */ diff --git a/datareduction/datareducer.cpp b/datareduction/datareducer.cpp index 7c9c8ff33..309915ae9 100644 --- a/datareduction/datareducer.cpp +++ b/datareduction/datareducer.cpp @@ -2946,9 +2946,9 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } - if(P::systemWriteAllDROs || lowercase == "vg_amr_alpha") { - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha",CellParams::AMR_ALPHA,1)); - outputReducer->addMetadata(outputReducer->size()-1,"","","$\\alpha$",""); + if(P::systemWriteAllDROs || lowercase == "vg_amr_alpha1") { + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha1",CellParams::AMR_ALPHA1,1)); + outputReducer->addMetadata(outputReducer->size()-1,"","","$\\alpha_1$",""); if(!P::systemWriteAllDROs) { continue; } @@ -2960,9 +2960,9 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti continue; } } - if(P::systemWriteAllDROs || lowercase == "vg_amr_jperb") { - outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_jperb",CellParams::AMR_JPERB,1)); - outputReducer->addMetadata(outputReducer->size()-1,"1/m","m^{-1}","J/B_{\\perp}",""); + if(P::systemWriteAllDROs || lowercase == "vg_amr_alpha2") { + outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha2",CellParams::AMR_ALPHA2,1)); + outputReducer->addMetadata(outputReducer->size()-1,"","","$\\alpha_2",""); if(!P::systemWriteAllDROs) { continue; } @@ -3207,7 +3207,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti return retval; })); - outputReducer->addMetadata(outputReducer->size()-1, "kg m^-3", "$\\mathrm{kg m^{-3}}$", "$\\rho_m$", "1.0"); + outputReducer->addMetadata(outputReducer->size()-1, "m^-3", "$\\mathrm{m^{-3}}$", "$\\n_e$", "1.0"); if(!P::systemWriteAllDROs) { continue; } diff --git a/datareduction/datareducer.h b/datareduction/datareducer.h index cd8b5d969..bda673ace 100644 --- a/datareduction/datareducer.h +++ b/datareduction/datareducer.h @@ -27,7 +27,7 @@ #include "fsgrid.hpp" #include "../definitions.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "datareductionoperator.h" #include "../sysboundary/ionosphere.h" diff --git a/datareduction/datareductionoperator.cpp b/datareduction/datareductionoperator.cpp index f3078eba9..72814d6ce 100644 --- a/datareduction/datareductionoperator.cpp +++ b/datareduction/datareductionoperator.cpp @@ -152,7 +152,13 @@ namespace DRO { lambda(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid,technicalGrid); std::array& gridSize = technicalGrid.getLocalSize(); - int vectorSize = varBuffer.size() / (gridSize[0]*gridSize[1]*gridSize[2]); + int vectorSize; + + // Check if there is anything to write (eg, we are a non-FS process) + if (gridSize[0]*gridSize[1]*gridSize[2] == 0) + vectorSize = 0; + else + vectorSize = varBuffer.size() / (gridSize[0]*gridSize[1]*gridSize[2]); if(writeAsFloat) { // Convert down to 32bit floats to save output space diff --git a/datareduction/datareductionoperator.h b/datareduction/datareductionoperator.h index 329099f26..b979035f0 100644 --- a/datareduction/datareductionoperator.h +++ b/datareduction/datareductionoperator.h @@ -29,7 +29,7 @@ #include "fsgrid.hpp" #include "../definitions.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../parameters.h" #include "../sysboundary/ionosphere.h" using namespace spatial_cell; diff --git a/fieldsolver/derivatives.cpp b/fieldsolver/derivatives.cpp index bce52d858..b1aad577c 100644 --- a/fieldsolver/derivatives.cpp +++ b/fieldsolver/derivatives.cpp @@ -806,8 +806,8 @@ void calculateScaledDeltas( cell->parameters[CellParams::AMR_DPSQ] = dPsq; cell->parameters[CellParams::AMR_DBSQ] = dBsq; cell->parameters[CellParams::AMR_DB] = dB; - cell->parameters[CellParams::AMR_ALPHA] = alpha; - cell->parameters[CellParams::AMR_JPERB] = J / (Bperp + EPS); // Epsilon in denominator so we don't get infinities + cell->parameters[CellParams::AMR_ALPHA1] = alpha; + cell->parameters[CellParams::AMR_ALPHA2] = cell->parameters[CellParams::DX] * J / (Bperp + EPS); // Epsilon in denominator so we don't get infinities } /*! \brief High-level scaled gradient calculation wrapper function. diff --git a/fieldsolver/derivatives.hpp b/fieldsolver/derivatives.hpp index e9c63a312..a076f92a5 100644 --- a/fieldsolver/derivatives.hpp +++ b/fieldsolver/derivatives.hpp @@ -24,7 +24,7 @@ #include #include "../definitions.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../sysboundary/sysboundary.h" #include "fs_limiters.h" diff --git a/fieldsolver/gridGlue.cpp b/fieldsolver/gridGlue.cpp index 433f2284a..9a0f9a7e2 100644 --- a/fieldsolver/gridGlue.cpp +++ b/fieldsolver/gridGlue.cpp @@ -1,7 +1,7 @@ #include #include #include "../grid.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../definitions.h" #include "../common.h" #include "gridGlue.hpp" @@ -105,37 +105,37 @@ void filterMoments(dccrg::Dccrg& mpiGrid, // Kernel Characteristics const int kernelOffset = 2; // offset of 5 pointstencil 3D kernel => (floor(stencilWidth/2);) - const Real kernelSum=729.0; // the total kernel's sum + const Real inverseKernelSum = 1.0 / 729.0; // the inverse of the total kernel's sum const static Real kernel[5][5][5] ={ - {{ 1, 2, 3, 2, 1}, - { 2, 4, 6, 4, 2}, - { 3, 6, 9, 6, 3}, - { 2, 4, 6, 4, 2}, - { 1, 2, 3, 2, 1}}, - - {{ 2, 4, 6, 4, 2}, - { 4, 8, 12, 8, 4}, - { 6, 12, 18, 12, 6}, - { 4, 8, 12, 8, 4}, - { 2, 4, 6, 4, 2}}, - - {{ 3, 6, 9, 6, 3}, - { 6, 12, 18, 12, 6}, - { 9, 18, 27, 18, 9}, - { 6, 12, 18, 12, 6}, - { 3, 6, 9, 6, 3}}, - - {{ 2, 4, 6, 4, 2}, - { 4, 8, 12, 8, 4}, - { 6, 12, 18, 12, 6}, - { 4, 8, 12, 8, 4}, - { 2, 4, 6, 4, 2}}, - - {{ 1, 2, 3, 2, 1}, - { 2, 4, 6, 4, 2}, - { 3, 6, 9, 6, 3}, - { 2, 4, 6, 4, 2}, - { 1, 2, 3, 2, 1}} + {{ 1 * inverseKernelSum, 2 * inverseKernelSum, 3 * inverseKernelSum, 2 * inverseKernelSum, 1 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 3 * inverseKernelSum, 6 * inverseKernelSum, 9 * inverseKernelSum, 6 * inverseKernelSum, 3 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 1 * inverseKernelSum, 2 * inverseKernelSum, 3 * inverseKernelSum, 2 * inverseKernelSum, 1 * inverseKernelSum}}, + + {{ 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 4 * inverseKernelSum, 8 * inverseKernelSum, 12 * inverseKernelSum, 8 * inverseKernelSum, 4 * inverseKernelSum}, + { 6 * inverseKernelSum, 12 * inverseKernelSum, 18 * inverseKernelSum, 12 * inverseKernelSum, 6 * inverseKernelSum}, + { 4 * inverseKernelSum, 8 * inverseKernelSum, 12 * inverseKernelSum, 8 * inverseKernelSum, 4 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}}, + + {{ 3 * inverseKernelSum, 6 * inverseKernelSum, 9 * inverseKernelSum, 6 * inverseKernelSum, 3 * inverseKernelSum}, + { 6 * inverseKernelSum, 12 * inverseKernelSum, 18 * inverseKernelSum, 12 * inverseKernelSum, 6 * inverseKernelSum}, + { 9 * inverseKernelSum, 18 * inverseKernelSum, 27 * inverseKernelSum, 18 * inverseKernelSum, 9 * inverseKernelSum}, + { 6 * inverseKernelSum, 12 * inverseKernelSum, 18 * inverseKernelSum, 12 * inverseKernelSum, 6 * inverseKernelSum}, + { 3 * inverseKernelSum, 6 * inverseKernelSum, 9 * inverseKernelSum, 6 * inverseKernelSum, 3 * inverseKernelSum}}, + + {{ 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 4 * inverseKernelSum, 8 * inverseKernelSum, 12 * inverseKernelSum, 8 * inverseKernelSum, 4 * inverseKernelSum}, + { 6 * inverseKernelSum, 12 * inverseKernelSum, 18 * inverseKernelSum, 12 * inverseKernelSum, 6 * inverseKernelSum}, + { 4 * inverseKernelSum, 8 * inverseKernelSum, 12 * inverseKernelSum, 8 * inverseKernelSum, 4 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}}, + + {{ 1 * inverseKernelSum, 2 * inverseKernelSum, 3 * inverseKernelSum, 2 * inverseKernelSum, 1 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 3 * inverseKernelSum, 6 * inverseKernelSum, 9 * inverseKernelSum, 6 * inverseKernelSum, 3 * inverseKernelSum}, + { 2 * inverseKernelSum, 4 * inverseKernelSum, 6 * inverseKernelSum, 4 * inverseKernelSum, 2 * inverseKernelSum}, + { 1 * inverseKernelSum, 2 * inverseKernelSum, 3 * inverseKernelSum, 2 * inverseKernelSum, 1 * inverseKernelSum}} }; // Update momentsGrid Ghost Cells @@ -150,7 +150,7 @@ void filterMoments(dccrg::Dccrg& mpiGrid, for (int blurPass = 0; blurPass < Parameters::maxFilteringPasses; blurPass++){ // Blurring Pass - #pragma omp parallel for collapse(2) + #pragma omp parallel for collapse(3) for (FsGridTools::FsIndex_t k = 0; k < mntDims[2]; k++){ for (FsGridTools::FsIndex_t j = 0; j < mntDims[1]; j++){ for (FsGridTools::FsIndex_t i = 0; i < mntDims[0]; i++){ @@ -169,6 +169,7 @@ void filterMoments(dccrg::Dccrg& mpiGrid, // Set Cell to zero before passing filter swap = swapGrid.get(i,j,k); + #pragma omp simd for (int e = 0; e < fsgrids::moments::N_MOMENTS; ++e) { swap->at(e)=0.0; } @@ -178,16 +179,13 @@ void filterMoments(dccrg::Dccrg& mpiGrid, for (int b=-kernelOffset; b<=kernelOffset; b++){ for (int a=-kernelOffset; a<=kernelOffset; a++){ cell = momentsGrid.get(i+a,j+b,k+c); + #pragma omp simd for (int e = 0; e < fsgrids::moments::N_MOMENTS; ++e) { swap->at(e)+=cell->at(e) *kernel[kernelOffset+a][kernelOffset+b][kernelOffset+c]; } } } }//inner filtering loop - //divide by the total kernel sum - for (int e = 0; e < fsgrids::moments::N_MOMENTS; ++e) { - swap->at(e)/=kernelSum; - } } } } //spatial loops diff --git a/fieldsolver/ldz_magnetic_field.hpp b/fieldsolver/ldz_magnetic_field.hpp index 494b933a2..9c4aea729 100644 --- a/fieldsolver/ldz_magnetic_field.hpp +++ b/fieldsolver/ldz_magnetic_field.hpp @@ -24,7 +24,7 @@ #include "../definitions.h" #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "fs_common.h" diff --git a/fieldsolver/ldz_volume.hpp b/fieldsolver/ldz_volume.hpp index 4c86915d8..bd91a3edb 100644 --- a/fieldsolver/ldz_volume.hpp +++ b/fieldsolver/ldz_volume.hpp @@ -23,7 +23,7 @@ #include #include "fs_common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" /*! \brief Top-level field averaging function. * diff --git a/fieldtracing/fieldtracing.cpp b/fieldtracing/fieldtracing.cpp index 6e1870382..78c70d349 100644 --- a/fieldtracing/fieldtracing.cpp +++ b/fieldtracing/fieldtracing.cpp @@ -129,7 +129,7 @@ namespace FieldTracing { } // Make one step along the fieldline - stepFieldLine(x,v, nodeTracingStepSize[n],fieldTracingParameters.min_tracer_dx,technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,(no.x[2] < 0)); + stepFieldLine(x,v, nodeTracingStepSize[n],fieldTracingParameters.min_tracer_dx_full_box,technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,(no.x[2] < 0)); // If we map back into the ionosphere, we obviously don't couple out to SBC::Ionosphere::downmapRadius. if(x.at(0)*x.at(0) + x.at(1)*x.at(1) + x.at(2)*x.at(2) < SBC::Ionosphere::innerRadius*SBC::Ionosphere::innerRadius) { @@ -143,10 +143,14 @@ namespace FieldTracing { const std::array x_out = x; // Take a step back and find the downmapRadius crossing point - stepFieldLine(x,v, nodeTracingStepSize[n],fieldTracingParameters.min_tracer_dx,technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,!(no.x[2] < 0)); + stepFieldLine(x,v, nodeTracingStepSize[n],fieldTracingParameters.min_tracer_dx_full_box,technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,!(no.x[2] < 0)); Real r_out = sqrt(x_out[0]*x_out[0] + x_out[1]*x_out[1] + x_out[2]*x_out[2]); Real r_in = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); Real alpha = (SBC::Ionosphere::downmapRadius-r_out)/(r_in - r_out); + alpha = std::fmax(std::fmin(alpha,1.0),0.0); + if (fabs(r_out-r_in) < 0.01*fieldTracingParameters.min_tracer_dx_full_box) { + alpha = 0.5; + } Real xi = x[0]-x_out[0]; Real yi = x[1]-x_out[1]; Real zi = x[2]-x_out[2]; @@ -336,7 +340,7 @@ namespace FieldTracing { while(x[0]*x[0]+x[1]*x[1]+x[2]*x[2] > SBC::Ionosphere::innerRadius*SBC::Ionosphere::innerRadius) { // Make one step along the fieldline - stepFieldLine(x,v, stepSize,50e3,100e3,fieldTracingParameters.tracingMethod,dipoleFieldOnly,false); + stepFieldLine(x,v, stepSize,fieldTracingParameters.min_tracer_dx_ionospere_coupling,fieldTracingParameters.max_tracer_dx_ionospere_coupling,fieldTracingParameters.tracingMethod,dipoleFieldOnly,false); // If the field lines is moving even further outwards, abort. // (this shouldn't happen under normal magnetospheric conditions, but who @@ -352,9 +356,13 @@ namespace FieldTracing { const std::array x_in = x; Real r_in = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); // Take a step back and find the innerRadius crossing point - stepFieldLine(x,v, stepSize,50e3,100e3,fieldTracingParameters.tracingMethod,dipoleFieldOnly,false); + stepFieldLine(x,v, stepSize,fieldTracingParameters.min_tracer_dx_ionospere_coupling,fieldTracingParameters.max_tracer_dx_ionospere_coupling,fieldTracingParameters.tracingMethod,dipoleFieldOnly,false); Real r_out = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); Real alpha = (SBC::Ionosphere::innerRadius - r_in)/(r_out - r_in); + alpha = std::fmax(std::fmin(alpha,1.0),0.0); + if (fabs(r_out-r_in) < 0.01*fieldTracingParameters.min_tracer_dx_ionospere_coupling) { + alpha = 0.5; + } Real xi = x[0]-x_in[0]; Real yi = x[1]-x_in[1]; Real zi = x[2]-x_in[2]; @@ -549,7 +557,7 @@ namespace FieldTracing { // Make one step along the fieldline // If the node is in the North, trace along -B (false for last argument), in the South, trace along B - stepFieldLine(x,v, nodeTracingStepSize[n],(TReal)fieldTracingParameters.min_tracer_dx,(TReal)technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,(no.x[2] < 0)); + stepFieldLine(x,v, nodeTracingStepSize[n],(TReal)fieldTracingParameters.min_tracer_dx_full_box,(TReal)technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,(no.x[2] < 0)); nodeTracingStepCount[n]++; // Look up the fsgrid cell belonging to these coordinates @@ -682,10 +690,14 @@ namespace FieldTracing { cellConnection[n] += TracingLineEndType::CLOSED; // Take a step back and find the innerRadius crossing point - stepFieldLine(x,v, cellTracingStepSize[n],(TReal)100e3,(TReal)technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,!(DIRECTION == Direction::FORWARD)); - TReal r_in = sqrt(cellTracingCoordinates[n][0]*cellTracingCoordinates[n][0] + cellTracingCoordinates[n][1]*cellTracingCoordinates[n][1] + cellTracingCoordinates[n][2]*cellTracingCoordinates[n][2]); - TReal r_out = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); - TReal alpha = (fieldTracingParameters.innerBoundaryRadius-r_in)/(r_out - r_in); + stepFieldLine(x,v, cellTracingStepSize[n],(TReal)fieldTracingParameters.min_tracer_dx_full_box,(TReal)technicalGrid.DX/2,fieldTracingParameters.tracingMethod,tracingFullField,!(DIRECTION == Direction::FORWARD)); + Real r_in = sqrt(cellTracingCoordinates[n][0]*cellTracingCoordinates[n][0] + cellTracingCoordinates[n][1]*cellTracingCoordinates[n][1] + cellTracingCoordinates[n][2]*cellTracingCoordinates[n][2]); + Real r_out = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]); + Real alpha = (fieldTracingParameters.innerBoundaryRadius-r_in)/(r_out - r_in); + alpha = std::fmax(std::fmin(alpha,1.0),0.0); + if (fabs(r_out-r_in) < 0.01*fieldTracingParameters.min_tracer_dx_full_box) { + alpha = 0.5; + } TReal xi = x[0]-cellTracingCoordinates[n][0]; TReal yi = x[1]-cellTracingCoordinates[n][1]; TReal zi = x[2]-cellTracingCoordinates[n][2]; diff --git a/fieldtracing/fieldtracing.h b/fieldtracing/fieldtracing.h index d12fb2eb7..dac55d022 100644 --- a/fieldtracing/fieldtracing.h +++ b/fieldtracing/fieldtracing.h @@ -91,7 +91,9 @@ namespace FieldTracing { TracingMethod tracingMethod; Real max_allowed_error; /*!< Maximum alowed error for the adaptive field line tracing methods */ uint32_t max_field_tracer_attempts; /*!< Max allowed attempts for the iterative field tracers */ - Real min_tracer_dx; /*!< Min allowed tracer dx to avoid getting bogged down in the archipelago */ + Real min_tracer_dx_full_box; /*!< Min allowed tracer dx for tracing in the full domain to avoid getting bogged down in the archipelago */ + const Real min_tracer_dx_ionospere_coupling=50e3; /*!< Min allowed tracer dx for tracing between the Vlasov domain and the ionosphere */ + const Real max_tracer_dx_ionospere_coupling=100e3; /*!< Max allowed tracer dx for tracing between the Vlasov domain and the ionosphere */ Real fullbox_max_incomplete_cells; /*!< Max allowed fraction of cells left unfinished before exiting tracing loop, fullbox */ Real fluxrope_max_incomplete_cells; /*!< Max allowed fraction of cells left unfinished before exiting tracing loop, fluxrope */ Real fullbox_and_fluxrope_max_distance; /*!< Max allowed tracing distance before ending tracing, fullbox and fluxrope tracing */ diff --git a/generate_version.sh b/generate_version.sh index 0a496fba2..4c0e2af02 100755 --- a/generate_version.sh +++ b/generate_version.sh @@ -19,8 +19,6 @@ curdir=$(pwd) -. /etc/profile - # if you use client/server visit and have "cd /lustre/tmp/..." in your ~/.bashrc this workaround is needed cd $curdir diff --git a/grid.cpp b/grid.cpp index 33ebfb1e9..ec1bf3c20 100644 --- a/grid.cpp +++ b/grid.cpp @@ -41,7 +41,7 @@ #include "fieldsolver/fs_common.h" #include "fieldsolver/gridGlue.hpp" #include "fieldsolver/derivatives.hpp" -#include "vlasovsolver/cpu_trans_map_amr.hpp" +#include "vlasovsolver/cpu_trans_pencils.hpp" #include "projects/project.h" #include "iowrite.h" #include "ioread.h" @@ -360,7 +360,6 @@ void initializeGrids( // If we only want the full BGB for writeout, we have it now and we can return early. if(P::writeFullBGB == true) { - phiprof::stop("Set initial state"); return; } @@ -499,15 +498,11 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S //set weights based on each cells LB weight counter const vector& cells = getLocalCells(); for (size_t i=0; iparameters[CellParams::LBWEIGHTCOUNTER]); -// else -// mpiGrid.set_cell_weight(cells[i], mpiGrid[cells[i]]->get_number_of_all_velocity_blocks()); - //reset counter - //mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER] = 0.0; + // Set cell weight. We could use different counters or number of blocks if different solvers are active. + // if (P::propagateVlasovAcceleration) + // When using the FS-SPLIT functionality, Jaro Hokkanen reported issues with using the regular + // CellParams::LBWEIGHTCOUNTER, so use of blockscounts + 1 might be required. + mpiGrid.set_cell_weight(cells[i], (Real)1 + mpiGrid[cells[i]]->parameters[CellParams::LBWEIGHTCOUNTER]); } phiprof::Timer initLBTimer {"dccrg.initialize_balance_load"}; @@ -647,10 +642,7 @@ void balanceLoad(dccrg::Dccrg& mpiGrid, S // Prepare cellIDs and pencils for AMR translation if(P::amrMaxSpatialRefLevel > 0) { - phiprof::Timer timer {"GetSeedIdsAndBuildPencils"}; - for (int dimension=0; dimension<3; dimension++) { - prepareSeedIdsAndPencils(mpiGrid,dimension); - } + prepareSeedIdsAndPencils(mpiGrid); } } @@ -675,15 +667,17 @@ bool adjustVelocityBlocks(dccrg::Dccrg& m } computeTimer.stop(); - phiprof::Timer transferTimer {"Transfer with_content_list", {"MPI"}}; - SpatialCell::set_mpi_transfer_type(Transfer::VEL_BLOCK_WITH_CONTENT_STAGE1 ); - mpiGrid.update_copies_of_remote_neighbors(NEAREST_NEIGHBORHOOD_ID); - SpatialCell::set_mpi_transfer_type(Transfer::VEL_BLOCK_WITH_CONTENT_STAGE2 ); - mpiGrid.update_copies_of_remote_neighbors(NEAREST_NEIGHBORHOOD_ID); - transferTimer.stop(); + if (doPrepareToReceiveBlocks) { + // We are in the last substep of acceleration, so need to account for neighbours + phiprof::Timer transferTimer {"Transfer with_content_list", {"MPI"}}; + SpatialCell::set_mpi_transfer_type(Transfer::VEL_BLOCK_WITH_CONTENT_STAGE1 ); + mpiGrid.update_copies_of_remote_neighbors(NEAREST_NEIGHBORHOOD_ID); + SpatialCell::set_mpi_transfer_type(Transfer::VEL_BLOCK_WITH_CONTENT_STAGE2 ); + mpiGrid.update_copies_of_remote_neighbors(NEAREST_NEIGHBORHOOD_ID); + transferTimer.stop(); + } //Adjusts velocity blocks in local spatial cells, doesn't adjust velocity blocks in remote cells. - phiprof::Timer adjustimer {"Adjusting blocks"}; #pragma omp parallel for schedule(dynamic) for (size_t i=0; i& m CellID cell_id=cellsToAdjust[i]; SpatialCell* cell = mpiGrid[cell_id]; - // gather spatial neighbor list and create vector with pointers to neighbor spatial cells - const auto* neighbors = mpiGrid.get_neighbors_of(cell_id, NEAREST_NEIGHBORHOOD_ID); - // Note: at AMR refinement boundaries this can cause blocks to propagate further than absolutely required vector neighbor_ptrs; - neighbor_ptrs.reserve(neighbors->size()); - - for ( const auto& nbrPair : *neighbors) { - CellID neighbor_id = nbrPair.first; - if (neighbor_id == 0 || neighbor_id == cell_id) { - continue; + if (doPrepareToReceiveBlocks) { + // gather spatial neighbor list and gather vector with pointers to cells + // If we are within an acceleration substep prior to the last one, + // it's enough to adjust blocks based on local data only, and in + // that case we simply pass an empty list of pointers. + const auto* neighbors = mpiGrid.get_neighbors_of(cell_id, NEAREST_NEIGHBORHOOD_ID); + // Note: at AMR refinement boundaries this can cause blocks to propagate further + // than absolutely required. Face neighbours, however, are not enough as we must + // account for diagonal propagation. + neighbor_ptrs.reserve(neighbors->size()); + for ( const auto& [neighbor_id, dir] : *neighbors) { + if (neighbor_id != 0) { + neighbor_ptrs.push_back(mpiGrid[neighbor_id]); + } } - neighbor_ptrs.push_back(mpiGrid[neighbor_id]); } if (getObjectWrapper().particleSpecies[popID].sparse_conserve_mass) { for (size_t i=0; iget_number_of_velocity_blocks(popID)*WID3; ++i) { @@ -1505,8 +1503,8 @@ bool adaptRefinement(dccrg::Dccrg& mpiGri for (CellID id : newChildren) { *mpiGrid[id] = *mpiGrid[mpiGrid.get_parent(id)]; // Irrelevant? - // mpiGrid[id]->parameters[CellParams::AMR_ALPHA] /= P::refineMultiplier; - mpiGrid[id]->parameters[CellParams::AMR_ALPHA] /= 2.0; + mpiGrid[id]->parameters[CellParams::AMR_ALPHA1] /= 2.0; + mpiGrid[id]->parameters[CellParams::AMR_ALPHA2] /= 2.0; mpiGrid[id]->parameters[CellParams::RECENTLY_REFINED] = 1; } copyChildrenTimer.stop(newChildren.size(), "Spatial cells"); diff --git a/grid.h b/grid.h index 94f047301..f124cdf99 100644 --- a/grid.h +++ b/grid.h @@ -23,7 +23,7 @@ #define GRID_H #include "definitions.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include #include #include "sysboundary/sysboundary.h" diff --git a/ioread.cpp b/ioread.cpp index 2351dddc0..ece9cb2cd 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -131,9 +131,9 @@ bool exitOnError(bool success, const string& message, MPI_Comm comm) { successInt=1; else successInt=0; - + MPI_Allreduce(&successInt,&globalSuccessInt,1,MPI_INT,MPI_MIN,comm); - + if(globalSuccessInt==1) { return true; } else { @@ -176,14 +176,14 @@ bool readCellIds(vlsv::ParallelReader & file, vector& fileCells, const i logFile << "(RESTART) ERROR: Bad vectorsize at " << __FILE__ << " " << __LINE__ << endl << write; return false; } - + // Read cell Ids: char* IDbuffer = new char[arraySize*vectorSize*byteSize]; if (file.readArrayMaster("VARIABLE",attribs,readFromFirstIndex,arraySize,IDbuffer) == false) { logFile << "(RESTART) ERROR: Failed to read cell Ids!" << endl << write; success = false; } - + // Convert global Ids into our local DCCRG 64 bit uints const uint64_t& numberOfCells = arraySize; fileCells.resize(numberOfCells); @@ -1299,8 +1299,8 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"vg_bulk_forcing_flag",CellParams::FORCING_CELL_NUM,1,mpiGrid); } if (P::refineOnRestart) { // Refinement indices alpha_1 and alpha_2 - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"vg_amr_alpha",CellParams::AMR_ALPHA,1,mpiGrid); } - if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"vg_amr_jperb",CellParams::AMR_JPERB,1,mpiGrid); } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"vg_amr_alpha1",CellParams::AMR_ALPHA1,1,mpiGrid); } + if(success) { success=readCellParamsVariable(file,fileCells,localCellStartOffset,localCells,"vg_amr_alpha2",CellParams::AMR_ALPHA2,1,mpiGrid); } } // Backround B has to be set, there are also the derivatives that should be written/read if we wanted to only read in background field @@ -1412,10 +1412,6 @@ bool readFileCells(dccrg::Dccrg& mpiGrid, bool readFsgridDecomposition(vlsv::ParallelReader& file, std::array& decomposition){ list > attribs; - uint64_t arraySize; - uint64_t vectorSize; - vlsv::datatype::type dataType; - uint64_t byteSize; int myRank; MPI_Comm_rank(MPI_COMM_WORLD,&myRank); @@ -1466,7 +1462,6 @@ bool readFsgridDecomposition(vlsv::ParallelReader& file, std::array x_corners, y_corners, z_corners; int64_t begin_rank = 0; - int i = 0; for(auto rank_size : mesh_domain_sizes){ if(myRank == MASTER_RANK){ if(file.read("MESH", mesh_attribs, begin_rank, 1, ids_ptr, false) == false){ diff --git a/ioread.h b/ioread.h index 23e73d980..33ee0ab43 100644 --- a/ioread.h +++ b/ioread.h @@ -28,7 +28,7 @@ #include "vlsv_reader_parallel.h" #include "definitions.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "datareduction/datareducer.h" diff --git a/iowrite.cpp b/iowrite.cpp index 583e6ab7a..219641907 100644 --- a/iowrite.cpp +++ b/iowrite.cpp @@ -381,8 +381,19 @@ bool writeDataReducer(const dccrg::Dccrg& } } - if( success ) { + if( dataReducer.getName(dataReducerIndex).find("fg_", 0) == 0 ) { + // Write fsgrid data + phiprof::Timer writeFsTimer {"writeFsGrid"}; + success = dataReducer.writeFsGridData(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid, technicalGrid, "fsgrid", dataReducerIndex, vlsvWriter, writeAsFloat); + writeFsTimer.stop(); + } else if( dataReducer.getName(dataReducerIndex).find("ig_", 0) == 0 ) { + // Or maybe it will be writing ionosphere data? + phiprof::Timer writeIonosphereTimer {"writeIonosphere"}; + success |= dataReducer.writeIonosphereGridData(SBC::ionosphereGrid, "ionosphere", dataReducerIndex, vlsvWriter); + writeIonosphereTimer.stop(); + } else { + // If the data reducer didn't want to write fg or ig data, maybe it will be happy writing dccrg data if( (writeAsFloat == true && dataType.compare("float") == 0) && dataSize == sizeof(double) ) { double * varBuffer_double = reinterpret_cast(varBuffer); //Declare smaller varbuffer: @@ -417,25 +428,13 @@ bool writeDataReducer(const dccrg::Dccrg& delete[] varBuffer_smaller; varBuffer_smaller = NULL; } else { - // Write reduced data to file if DROP was successful: + // Write reduced data to file if DROP was successful: phiprof::Timer writeArrayTimer {"writeArray"}; if (vlsvWriter.writeArray("VARIABLE",attribs, dataType, cells.size(), vectorSize, dataSize, varBuffer) == false) { success = false; logFile << "(MAIN) writeGrid: ERROR failed to write datareductionoperator data to file!" << endl << writeVerbose; } } - - } else { - // If the data reducer didn't want to write dccrg data, maybe it will be happy - // dumping data straight from fsgrid into our file. - phiprof::Timer writeFsTimer {"writeFsGrid"}; - success = dataReducer.writeFsGridData(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid, technicalGrid, "fsgrid", dataReducerIndex, vlsvWriter, writeAsFloat); - writeFsTimer.stop(); - - // Or maybe it will be writing ionosphere data? - phiprof::Timer writeIonosphereTimer {"writeIonosphere"}; - success |= dataReducer.writeIonosphereGridData(SBC::ionosphereGrid, "ionosphere", dataReducerIndex, vlsvWriter); - writeIonosphereTimer.stop(); } // Check if the DataReducer wants to write paramters to the output file @@ -940,8 +939,7 @@ bool writeFsGridMetadata(FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technic vlsvWriter.writeArray("MESH_DOMAIN_SIZES", xmlAttributes, 1, 2, &meshDomainSize[0]); // how many MPI ranks we wrote from - int size; - MPI_Comm_size(MPI_COMM_WORLD, &size); + int size = technicalGrid.getSize(); vlsvWriter.writeParameter("numWritingRanks", &size); // Save the FSgrid decomposition @@ -1075,6 +1073,12 @@ bool writeIonosphereGridMetadata(vlsv::Writer& vlsvWriter) { } else { vlsvWriter.writeArray("MESH", xmlAttributes, 0, 1, ionosphereGridElementsAndCorners.data()); } + + // Write different parameters of the class Ionosphere into the VLSV file + if( vlsvWriter.writeParameter("ionosphere_radius", &SBC::Ionosphere::innerRadius) == false ) { return false; } + if( vlsvWriter.writeParameter("ionosphere_downmapping_radius", &SBC::Ionosphere::downmapRadius) == false ) { return false; } + if( vlsvWriter.writeParameter("ionosphere_time_smoothing_constant", &SBC::Ionosphere::couplingTimescale) == false ) { return false; } + if( vlsvWriter.writeParameter("ionosphere_time_interval", &SBC::Ionosphere::couplingInterval) == false ) { return false; } return true; @@ -1658,8 +1662,8 @@ bool writeRestart( restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("max_fields_dt",CellParams::MAXFDT,1)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_drift",CellParams::BULKV_FORCING_X,3)); restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_bulk_forcing_flag",CellParams::FORCING_CELL_NUM,1)); - restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha",CellParams::AMR_ALPHA,1)); - restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_jperb",CellParams::AMR_JPERB,1)); + restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha1",CellParams::AMR_ALPHA1,1)); + restartReducer.addOperator(new DRO::DataReductionOperatorCellParams("vg_amr_alpha2",CellParams::AMR_ALPHA2,1)); restartReducer.addOperator(new DRO::VariableBVol); restartReducer.addMetadata(restartReducer.size()-1,"T","$\\mathrm{T}$","$B_\\mathrm{vol,vg}$","1.0"); restartReducer.addOperator(new DRO::MPIrank); diff --git a/iowrite.h b/iowrite.h index e0d23885c..cffa06bf3 100644 --- a/iowrite.h +++ b/iowrite.h @@ -29,7 +29,7 @@ #include #include "definitions.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "datareduction/datareducer.h" /*! diff --git a/mini-apps/ionosphereSolverTests/Makefile b/mini-apps/ionosphereSolverTests/Makefile index 65cba017f..335fe6dca 100644 --- a/mini-apps/ionosphereSolverTests/Makefile +++ b/mini-apps/ionosphereSolverTests/Makefile @@ -8,15 +8,21 @@ LIBS += ${LIB_JEMALLOC} LIBS += ${LIB_PROFILE} LIBS += ${LIB_ZOLTAN} LIBS += ${LIB_VLSV} - +FLAGS += -Wno-cast-function-type INC_FSGRID=-I../../submodules/fsgrid/ INC_DCCRG=-I../../submodules/dccrg/ +INC_EIGEN = -isystem ../../submodules/eigen/ + +# Default makefile target: only build the test binaries, don't run anything +# (in particular, nothing that would require python) +default: main differentialFlux sigmaProfiles -all: main differentialFlux sigmaProfiles quadrupoleTests atmosphere.pdf -.PHONY: clean quadrupoleTests +# The "all" target actually builds and runs the tests proper. +all: main differentialFlux sigmaProfiles multipoleTests LFMtest atmosphere.png +.PHONY: clean multipoleTests clean: - rm *.o main differentialFlux sigmaProfiles + -rm *.o main differentialFlux sigmaProfiles ionosphere.o: ../../sysboundary/ionosphere.h ../../sysboundary/ionosphere.cpp ../../backgroundfield/backgroundfield.h ../../projects/project.h ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c ../../sysboundary/ionosphere.cpp ${INC_DCCRG} ${INC_FSGRID} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_VECTORCLASS} ${INC_PROFILE} ${INC_JEMALLOC} -Wno-comment @@ -42,8 +48,8 @@ object_wrapper.o: ../../object_wrapper.h ../../object_wrapper.cpp particle_species.o: ../../particle_species.h ../../particle_species.cpp $(CMP) $(CXXFLAGS) $(FLAGS) -c ../../particle_species.cpp -spatial_cell.o: ../../spatial_cell.cpp - $(CMP) $(CXXFLAGS) ${MATHFLAGS} $(FLAGS) -c ../../spatial_cell.cpp $(INC_BOOST) ${INC_DCCRG} ${INC_EIGEN} ${INC_ZOLTAN} ${INC_VECTORCLASS} ${INC_FSGRID} ${INC_PROFILE} ${INC_JEMALLOC} +spatial_cell.o: ../../spatial_cell_cpu.cpp + $(CMP) $(CXXFLAGS) ${MATHFLAGS} $(FLAGS) -c ../../spatial_cell_cpu.cpp -o spatial_cell.o $(INC_BOOST) ${INC_DCCRG} ${INC_EIGEN} ${INC_ZOLTAN} ${INC_VECTORCLASS} ${INC_FSGRID} ${INC_PROFILE} ${INC_JEMALLOC} cpu_moments.o: ../../vlasovsolver/cpu_moments.cpp ${CMP} ${CXXFLAGS} ${FLAG_OPENMP} ${MATHFLAGS} ${FLAGS} -c ../../vlasovsolver/cpu_moments.cpp ${INC_DCCRG} ${INC_BOOST} ${INC_ZOLTAN} ${INC_PROFILE} ${INC_FSGRID} ${INC_JEMALLOC} @@ -85,10 +91,55 @@ differentialFlux: differentialFlux.cpp sigmaProfiles: sigmaProfiles.cpp -atmosphere.pdf: sigmaProfiles plotAtmosphere.gp +atmosphere.png: sigmaProfiles plotAtmosphere.gp ./sigmaProfiles 1e6 1.16046e7 > atmosphere.dat + ./sigmaProfiles 1e6 1.16046e8 > atmosphere10.dat + ./sigmaProfiles 1e6 1.16046e9 > atmosphere100.dat ./plotAtmosphere.gp -quadrupoleTests: main plot_ionosphere.py - ./main -N 1024 -r 40 90 -r 50 80 -sigma 100 -fac hexadecapole -gaugeFix equator - ./plot_ionosphere.py output.vlsv +MULTIPOLES := $(shell seq 1 6) +MULTIPOLEJOBS := $(addprefix multipole,${MULTIPOLES}) +multipole-1-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 1 $* -gaugeFix pole -o multipole-1-$*.vlsv +multipole-2-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 2 $* -gaugeFix pole -o multipole-2-$*.vlsv +multipole-3-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 3 $* -gaugeFix pole -o multipole-3-$*.vlsv +multipole-4-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 4 $* -gaugeFix pole -o multipole-4-$*.vlsv +multipole-5-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 5 $* -gaugeFix pole -o multipole-5-$*.vlsv +multipole-6-%.vlsv: main + OMP_NUM_THREADS=1 ./main -N 1024 -r 40 90 -r 50 80 -sigma identity -fac multipole 6 $* -gaugeFix pole -o multipole-6-$*.vlsv + +${MULTIPOLEJOBS}: multipole%: main plot_ionosphere.py multipole-%--1.vlsv multipole-%-1.vlsv multipole-%-0.vlsv + echo "Running multipoles with L=$*" + for m in `seq -$* $*`; do \ + make multipole-$*-$$m.vlsv; \ + ./plot_ionosphere.py multipole-$*-$$m.vlsv; \ + echo "Running ./plot_ionosphere.py multipole-$*-$$m.vlsv"; \ + done; + +convergence%.dat: main + #for nodecount in 50 100 200 500 1000 2000 5000; do \ + # for i in {1..500}; do j=$(( $$RANDOM % (2*$i) - $i )); \ + # echo -n "$i $j "; + # OMP_NUM_THREADS=4 ./main -q -N $nodecount -sigma identity -fac multipole $i $i -gaugeFix pole -o multipole-$i-$j.vlsv; \ + # done | tee multipole$nodecount.dat; \ + #done + zsh -c 'i=$* j=$$(($$RANDOM%(2*$$i)-$$i)); \ + for nodecount in {20..2000}; do \ + echo -n "$$nodecount $$i $$j "; \ + OMP_NUM_THREADS=4 ./main -q -N $$nodecount -sigma identity -fac multipole $$i $$i -gaugeFix pole -o multipole-$$i-$$j.vlsv; \ + done | tee convergence$$i.dat' +multipoleTests: ${MULTIPOLEJOBS} + +LFMtest.vlsv: main + #./main -N 2000 -sigma 10 -fac merkin2010 -gaugeFix equator45 -o LFMtest0.vlsv -maxIter 10000 + OMP_NUM_THREADS=1 ./main -N 2000 -r 40 90 -r 50 90 -r 60 80 -sigma 10 -fac merkin2010 -gaugeFix equator45 -o LFMtest.vlsv -maxIter 10000 + +lfmtest.dat: LFMtest.vlsv + ./lfmformat.py LFMtest.vlsv > lfmtest.dat +LFMtest: lfmtest.dat + ./plot_LFMtest.gp + #./plot_ionosphereLFM.py LFMtest.vlsv diff --git a/mini-apps/ionosphereSolverTests/lfmformat.py b/mini-apps/ionosphereSolverTests/lfmformat.py new file mode 100755 index 000000000..787bc356d --- /dev/null +++ b/mini-apps/ionosphereSolverTests/lfmformat.py @@ -0,0 +1,19 @@ +#!/usr/bin/python3 + +import pytools as pt +import numpy as np +import sys + +infile = sys.argv[1] + +f = pt.vlsvfile.VlsvReader(infile) + +coords = f.get_ionosphere_node_coords() +print(np.shape(coords)) +lat = np.arctan2(coords[:,2], np.sqrt(coords[:,1]*coords[:,1] + coords[:,0]*coords[:,0])) +lon = np.arctan2(coords[:,1], coords[:,0]) +phi = f.read_ionosphere_variable("ig_potential"); + +for i in range(len(lat)): + if np.cos(lon[i]) > 0.02: + print(str(lat[i]) + " " + str(lon[i]) + " " + str(phi[i]) + " " + str(phi[i] / np.cos(lon[i]))) diff --git a/mini-apps/ionosphereSolverTests/main.cpp b/mini-apps/ionosphereSolverTests/main.cpp index 447fcbb99..235447364 100644 --- a/mini-apps/ionosphereSolverTests/main.cpp +++ b/mini-apps/ionosphereSolverTests/main.cpp @@ -106,9 +106,14 @@ int main(int argc, char** argv) { std::string facString="constant"; std::string gaugeFixString="pole"; std::string inputFile; + std::string outputFilename("output.vlsv"); std::vector> refineExtents; Ionosphere::solverMaxIterations = 1000; bool doPrecondition = true; + bool writeSolverMtarix = false; + bool quiet = false; + int multipoleL = 0; + int multipolem = 0; if(argc ==1) { cerr << "Running with default options. Run main --help to see available settings." << endl; } @@ -129,6 +134,12 @@ int main(int argc, char** argv) { } if(!strcmp(argv[i], "-fac")) { facString = argv[++i]; + + // Special handling for multipoles + if(facString == "multipole") { + multipoleL = atoi(argv[++i]); + multipolem = atoi(argv[++i]); + } continue; } if(!strcmp(argv[i], "-gaugeFix")) { @@ -147,38 +158,57 @@ int main(int argc, char** argv) { Ionosphere::solverMaxIterations = atoi(argv[++i]); continue; } + if(!strcmp(argv[i], "-o")) { + outputFilename = argv[++i]; + continue; + } + if(!strcmp(argv[i], "-matrix")) { + writeSolverMtarix = true; + continue; + } + if(!strcmp(argv[i], "-q")) { + quiet = true; + continue; + } cerr << "Unknown command line option \"" << argv[i] << "\"" << endl; cerr << endl; - cerr << "main [-N num] [-r ] [-sigma (identity|random|35|53|file)] [-fac (constant|dipole|quadrupole|octopole|hexadecapole||file)] [-facfile ] [-gaugeFix equator|equator40|equator60|pole|integral|none] [-np]" << endl; + cerr << "main [-N num] [-r ] [-sigma (identity|random|35|53|file)] [-fac (constant|dipole|quadrupole|octopole|hexadecapole||file)] [-facfile ] [-gaugeFix equator|equator40|equator45|equator60|pole|integral|none] [-np]" << endl; cerr << "Paramters:" << endl; - cerr << " -N: Number of ionosphere mesh nodes (default: 64)" << endl; - cerr << " -r: Refine grid between the given latitudes (can be specified multiple times)" << endl; - cerr << " -sigma: Conductivity matrix contents (default: identity)" << endl; - cerr << " options are:" << endl; - cerr << " identity - identity matrix w/ conductivity 1" << endl; - cerr << " ponly - Constant pedersen conductivitu"<< endl; - cerr << " 35 - Sigma_H = 3, Sigma_P = 5" << endl; - cerr << " 53 - Sigma_H = 5, Sigma_P = 3" << endl; - cerr << " 100 - Sigma_H = 100, Sigma_P=20" << endl; - cerr << " file - Read from vlsv input file " << endl; - cerr << " -fac: FAC pattern on the sphere (default: constant)" << endl; - cerr << " options are:" << endl; - cerr << " constant - Constant value of 1" << endl; - cerr << " dipole - north/south dipole" << endl; - cerr << " quadrupole - east/west quadrupole (L=2, m=1)" << endl; - cerr << " octopole - octopole (L=3, m=2)" << endl; - cerr << " hexadecapole - hexadecapole (L=4, m=3)" << endl; - cerr << " file - read FAC distribution from vlsv input file" << endl; - cerr << " -infile: Read FACs from this input file" << endl; - cerr << " -gaugeFix: Solver gauge fixing method (default: pole)" << endl; - cerr << " options are:" << endl; - cerr << " pole - Fix potential in a single node at the north pole" << endl; - cerr << " equator - Fix potential on all nodes +- 10 degrees of the equator" << endl; - cerr << " equator40 - Fix potential on all nodes +- 40 degrees of the equator" << endl; - cerr << " equator60 - Fix potential on all nodes +- 60 degrees of the equator" << endl; - cerr << " -np: DON'T use the matrix preconditioner (default: do)" << endl; - cerr << " -maxIter: Maximum number of solver iterations" << endl; - + cerr << " -N: Number of ionosphere mesh nodes (default: 64)" << endl; + cerr << " -r: Refine grid between the given latitudes (can be specified multiple times)" << endl; + cerr << " -sigma: Conductivity matrix contents (default: identity)" << endl; + cerr << " options are:" << endl; + cerr << " identity - identity matrix w/ conductivity 1" << endl; + cerr << " ponly - Constant pedersen conductivitu"<< endl; + cerr << " 10 - Sigma_H = 0, Sigma_P = 10" << endl; + cerr << " 35 - Sigma_H = 3, Sigma_P = 5" << endl; + cerr << " 53 - Sigma_H = 5, Sigma_P = 3" << endl; + cerr << " 100 - Sigma_H = 100, Sigma_P=20" << endl; + cerr << " file - Read from vlsv input file " << endl; + cerr << " -fac: FAC pattern on the sphere (default: constant)" << endl; + cerr << " options are:" << endl; + cerr << " constant - Constant value of 1" << endl; + cerr << " dipole - north/south dipole" << endl; + cerr << " quadrupole - east/west quadrupole (L=2, m=1)" << endl; + cerr << " octopole - octopole (L=3, m=2)" << endl; + cerr << " hexadecapole - hexadecapole (L=4, m=3)" << endl; + cerr << " multipole - generic multipole, L and m given separately." << endl; + cerr << " merkin2010 - eq13 of Merkin et al (2010)" << endl; + cerr << " file - read FAC distribution from vlsv input file" << endl; + cerr << " -infile: Read FACs from this input file" << endl; + cerr << " -gaugeFix: Solver gauge fixing method (default: pole)" << endl; + cerr << " options are:" << endl; + cerr << " pole - Fix potential in a single node at the north pole" << endl; + cerr << " equator - Fix potential on all nodes +- 10 degrees of the equator" << endl; + cerr << " equator40 - Fix potential on all nodes +- 40 degrees of the equator" << endl; + cerr << " equator45 - Fix potential on all nodes +- 45 degrees of the equator" << endl; + cerr << " equator60 - Fix potential on all nodes +- 60 degrees of the equator" << endl; + cerr << " -np: DON'T use the matrix preconditioner (default: do)" << endl; + cerr << " -maxIter: Maximum number of solver iterations" << endl; + cerr << " -o : Output filename (default: \"output.vlsv\")" << endl; + cerr << " -matrix: Write solver dependency matrix to solverMatrix.txt (default: don't.)" << endl; + cerr << " -q: Quiet mode (only output residual value" << endl; + return 1; } @@ -197,6 +227,9 @@ int main(int argc, char** argv) { } else if (gaugeFixString == "equator40") { ionosphereGrid.gaugeFixing = SphericalTriGrid::Equator; Ionosphere::shieldingLatitude = 40.; + } else if (gaugeFixString == "equator45") { + ionosphereGrid.gaugeFixing = SphericalTriGrid::Equator; + Ionosphere::shieldingLatitude = 45.; } else if (gaugeFixString == "equator60") { ionosphereGrid.gaugeFixing = SphericalTriGrid::Equator; Ionosphere::shieldingLatitude = 60.; @@ -206,7 +239,7 @@ int main(int argc, char** argv) { cerr << "Unknown gauge fixing method " << gaugeFixString << endl; return 1; } - + // Refine the base shape to acheive desired resolution auto refineBetweenLatitudes = [](Real phi1, Real phi2) -> void { uint numElems=ionosphereGrid.elements.size(); @@ -253,16 +286,22 @@ int main(int argc, char** argv) { // If that doesn't exist, reconstruct it from the sigmaH and sigmaP components // (This assumes that the input file was run with the "GUMICS" conductivity model. Which is reasonable, // because the others don't work very well) - cerr << "Reading conductivity tensor from ig_sigmah, ig_sigmap." << endl; + if(!quiet) { + cerr << "Reading conductivity tensor from ig_sigmah, ig_sigmap." << endl; + } readIonosphereNodeVariable(inVlsv, "ig_sigmah", ionosphereGrid, ionosphereParameters::SIGMAH); readIonosphereNodeVariable(inVlsv, "ig_sigmap", ionosphereGrid, ionosphereParameters::SIGMAP); //readIonosphereNodeVariable(inVlsv, "ig_sigmaparallel", ionosphereGrid, ionosphereParameters::SIGMAPARALLEL); assignConductivityTensorFromLoadedData(nodes); - } + } } else if(sigmaString == "ponly") { Real sigmaP=3.; Real sigmaH=0.; assignConductivityTensor(nodes, sigmaP, sigmaH); + } else if(sigmaString == "10") { + Real sigmaP=10.; + Real sigmaH=0.; + assignConductivityTensor(nodes, sigmaP, sigmaH); } else if(sigmaString == "35") { Real sigmaP=3.; Real sigmaH=5.; @@ -271,6 +310,10 @@ int main(int argc, char** argv) { Real sigmaP=5.; Real sigmaH=3.; assignConductivityTensor(nodes, sigmaP, sigmaH); + } else if(sigmaString == "10") { + Real sigmaP=10.; + Real sigmaH=0.; + assignConductivityTensor(nodes, sigmaP, sigmaH); } else if(sigmaString == "100") { Real sigmaP=20.; Real sigmaH=100.; @@ -338,6 +381,44 @@ int main(int argc, char** argv) { nodes[n].parameters[ionosphereParameters::SOURCE] = sph_legendre(4,3,theta) * cos(3*phi) * area; } + } else if(facString == "multipole") { + for(uint n=0; n= theta_0 && fabs(theta) < theta_0 + deltaTheta) { + j_parallel = j_0 * sin(M_PI/2 - theta) * sin(phi); + } + nodes[n].parameters[ionosphereParameters::SOURCE] = j_parallel * area; + } } else if(facString == "file") { vlsv::ParallelReader inVlsv; inVlsv.open(inputFile,MPI_COMM_WORLD,masterProcessID); @@ -357,45 +438,83 @@ int main(int argc, char** argv) { ionosphereGrid.initSolver(true); - // Write solver dependency matrix to stdout. - ofstream matrixOut("solverMatrix.txt"); - for(uint n=0; n::max(), minPotentialN, minPotentialS, maxPotentialN, maxPotentialS; ionosphereGrid.solve(iterations, nRestarts, residual, minPotentialN, maxPotentialN, minPotentialS, maxPotentialS); - cout << "Ionosphere solver: iterations " << iterations << " restarts " << nRestarts - << " residual " << std::scientific << residual << std::defaultfloat - << " potential min N = " << minPotentialN << " S = " << minPotentialS - << " max N = " << maxPotentialN << " S = " << maxPotentialS - << " difference N = " << maxPotentialN - minPotentialN << " S = " << maxPotentialS - minPotentialS - << endl; + if(!quiet) { + cout << "Ionosphere solver: iterations " << iterations << " restarts " << nRestarts + << " residual " << std::scientific << residual << std::defaultfloat + << " potential min N = " << minPotentialN << " S = " << minPotentialS + << " max N = " << maxPotentialN << " S = " << maxPotentialS + << " difference N = " << maxPotentialN - minPotentialN << " S = " << maxPotentialS - minPotentialS + << endl; + } else { + if(multipoleL == 0) { + cout << std::scientific << residual << std::defaultfloat << std::endl; + } else { + // Actually corellate with our input multipole + Real correlate=0; + Real selfNorm=0; + Real sphNorm =0; + Real totalArea = 0; + for(uint n=0; n({"potato potato"}); @@ -452,8 +571,10 @@ int main(int argc, char** argv) { } outputFile.close(); - cout << "--- OUTPUT WRITTEN TO output.vlsv ---" << endl; + if(!quiet) { + cout << "--- OUTPUT WRITTEN TO " << outputFilename << " ---" << endl; + } - cout << "--- DONE. ---" << endl; + //cout << "--- DONE. ---" << endl; return 0; } diff --git a/mini-apps/ionosphereSolverTests/plotAtmosphere.gp b/mini-apps/ionosphereSolverTests/plotAtmosphere.gp index 170b6957a..0779bdc12 100755 --- a/mini-apps/ionosphereSolverTests/plotAtmosphere.gp +++ b/mini-apps/ionosphereSolverTests/plotAtmosphere.gp @@ -1,14 +1,15 @@ -#!/usr/bin/gnuplot +#!/usr/bin/gnuplot # # Before running this, do: -# +# # make sigmaProfiles # ./sigmaProfiles 1e6 1e7 > atmosphere.dat # -set title "Ionosphere with incoming maxwellian electrons n=10^6 m^{-3}, T=1keV" -set terminal pdfcairo size 6,10 -set output "atmosphere.pdf" +#set terminal pdfcairo size 4.8,8 dashed +#set output "atmosphere.pdf" +set terminal pngcairo size 960,1600 dashed +set output "atmosphere.png" set multiplot @@ -19,13 +20,26 @@ set format x "10^{%T}" set size 1,0.33 -plot [1e-10:1e5] [0:300] 'atmosphere.dat' using 3:1 with lines title "Pedersen conductivity", '' using 4:1 with lines title "Hall conductivity", '' using 5:1 with lines title "Parallel conductivity" +plot [1e-10:1e5] [0:250] 'atmosphere.dat' using 3:1 with lines lc 1 dt 1 title "Pedersen conductivity",\ + '' using 4:1 with lines lc 1 dt 2 title "Hall conductivity",\ + '' using 5:1 with lines lc 1 dt 3 title "Parallel conductivity",\ + 'atmosphere10.dat' using 3:1 with lines lc 2 dt 1 title "T=10keV",\ + '' using 4:1 with lines lc 2 dt 2 notitle,\ + '' using 5:1 with lines lc 2 dt 3 notitle,\ + 'atmosphere100.dat' using 3:1 with lines lc 3 dt 1 title "T=100keV",\ + '' using 4:1 with lines lc 3 dt 2 notitle,\ + '' using 5:1 with lines lc 3 dt 3 notitle set origin 0,.33 set xlabel "Electron density (m^{-3})" -plot [1e9:1e13] [0:300] 'atmosphere.dat' using 2:1 with lines title "Free electron density" +plot [1e9:1e13] [0:250] 'atmosphere.dat' using 2:1 with lines title "Free electron density, T=1kev",\ + 'atmosphere10.dat' using 2:1 with lines title "T=10keV",\ + 'atmosphere100.dat' using 2:1 with lines title "T=100keV" set origin 0,.66 set xlabel "Production rate (m^{-3} s^{-1})" -plot [1e8:1e13] [0:300] 'atmosphere.dat' using 6:1 with lines title "Electron production rate" +set title "Ionosphere with incoming maxwellian electrons n=10^6 m^{-3}" +plot [1e8:1e13] [0:250] 'atmosphere.dat' using 6:1 with lines title "Electron production rate. T=1keV",\ + 'atmosphere10.dat' using 6:1 with lines title "T=10keV",\ + 'atmosphere100.dat' using 6:1 with lines title "T=100keV" diff --git a/mini-apps/ionosphereSolverTests/plot_LFMtest.gp b/mini-apps/ionosphereSolverTests/plot_LFMtest.gp new file mode 100755 index 000000000..6c7e5da46 --- /dev/null +++ b/mini-apps/ionosphereSolverTests/plot_LFMtest.gp @@ -0,0 +1,13 @@ +#!/usr/bin/gnuplot -persist + +set terminal pngcairo size 500,500 +set output "LFMtest.png" + +set xlabel "Latitude θ" +set ylabel "Φ(θ, φ) / sin(φ) (kV)" + +set key bottom center + +plot [45:90] [0:120] \ + 'lfmtest.dat' using ($1/3.14159*180):($4/1e3*1.05) pt 0 title "our result", \ + 'merkinReference.dat' using (90 - $1):2 with lines title "Merkin et al (2010)" diff --git a/mini-apps/ionosphereSolverTests/plot_ionosphere.py b/mini-apps/ionosphereSolverTests/plot_ionosphere.py new file mode 100755 index 000000000..8dd99c7a4 --- /dev/null +++ b/mini-apps/ionosphereSolverTests/plot_ionosphere.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 +import sys +import pytools as pt +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.inset_locator import inset_axes +import matplotlib.patches as patches + +#for step in range(int(sys.argv[1]), int(sys.argv[2])): +filename = sys.argv[1] +#what = sys.argv[2] +#pole = sys.argv[3] +#if pole == "south": +# pole = -1 +#else: +# pole = 1 + +pt.plot.plot_ionosphere(filename=filename,outputdir="./ionosphereplot/",var="ig_fac",vmin=-.5, vmax=.5, lin=True, wmark="NE", minlatitude=40) +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_sigmah",vmin=0, vmax=1e-3, colormap="viridis", lin=True, wmark="NE") +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_sigmap",vmin=0, vmax=1e-3, colormap="viridis", lin=True, wmark="NE") +pt.plot.plot_ionosphere(filename=filename,outputdir="./ionosphereplot/",var="ig_potential", colormap="PuOr", vmin=-1e12, vmax=1e12, lin=True, wmark="NE", symmetric=True, minlatitude=40) +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_rhon", colormap="turbo", vmin=0, vmax=1e8, wmark="NE") diff --git a/mini-apps/ionosphereSolverTests/plot_ionosphereLFM.py b/mini-apps/ionosphereSolverTests/plot_ionosphereLFM.py new file mode 100755 index 000000000..98f97d4c0 --- /dev/null +++ b/mini-apps/ionosphereSolverTests/plot_ionosphereLFM.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 +import sys +import pytools as pt +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1.inset_locator import inset_axes +import matplotlib.patches as patches + +#for step in range(int(sys.argv[1]), int(sys.argv[2])): +filename = sys.argv[1] +#what = sys.argv[2] +#pole = sys.argv[3] +#if pole == "south": +# pole = -1 +#else: +# pole = 1 + +pt.plot.plot_ionosphere(filename=filename,outputdir="./ionosphereplot/",var="ig_fac",vmin=-1e-6, vmax=1e-6, lin=True, wmark="NE", minlatitude=40) +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_sigmah",vmin=0, vmax=1e-3, colormap="viridis", lin=True, wmark="NE") +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_sigmap",vmin=0, vmax=1e-3, colormap="viridis", lin=True, wmark="NE") +pt.plot.plot_ionosphere(filename=filename,outputdir="./ionosphereplot/",var="ig_potential", colormap="PuOr", vmin=-100e3, vmax=100e3, lin=True, wmark="NE", symmetric=True, minlatitude=40) +#pt.plot.plot_ionosphere(filename=inputfile,outputdir="./ionosphereplot/",var="ig_rhon", colormap="turbo", vmin=0, vmax=1e8, wmark="NE") diff --git a/mini-apps/ionosphereSolverTests/sigmaProfiles.cpp b/mini-apps/ionosphereSolverTests/sigmaProfiles.cpp index d1b4171a7..b2c8b8603 100644 --- a/mini-apps/ionosphereSolverTests/sigmaProfiles.cpp +++ b/mini-apps/ionosphereSolverTests/sigmaProfiles.cpp @@ -186,8 +186,8 @@ int main(int argc, char** argv) { // GUMICS version - const double gyro = (1.6e-19*Bval/(31*1.66e-27)); - const double rho = atmosphere[h].nui/gyro; + //const double gyro = (1.6e-19*Bval/(31*1.66e-27)); + //const double rho = atmosphere[h].nui/gyro; //atmosphere[h].pedersencoeff = (1.6e-19/Bval)*(rho/(1+rho*rho)); //atmosphere[h].hallcoeff = rho * atmosphere[h].pedersencoeff; } @@ -227,7 +227,7 @@ int main(int argc, char** argv) { for(int h=0; h P::amrBoxHalfWidthX; @@ -278,7 +284,7 @@ bool P::addParameters() { RP::add("project", "Specify the name of the project to use. Supported to date (20150610): Alfven Diffusion Dispersion " "Distributions Firehose Flowthrough Fluctuations Harris KHB Larmor Magnetosphere Multipeak Riemann1 Shock " - "Shocktest Template test_fp testHall test_trans VelocityBox verificationLarmor", + "Shocktest Template test_fp testHall test_trans verificationLarmor", string("")); RP::add("restart.write_as_float", "If true, write restart fields in floats instead of doubles", false); @@ -402,7 +408,7 @@ bool P::addParameters() { "ig_electrontemp ig_solverinternals ig_upmappednodecoords ig_upmappedb ig_openclosed ig_potential "+ "ig_precipitation ig_deltaphi "+ "ig_inplanecurrent ig_b ig_e vg_drift vg_ionospherecoupling vg_connection vg_fluxrope fg_curvature "+ - "vg_amr_drho vg_amr_du vg_amr_dpsq vg_amr_dbsq vg_amr_db vg_amr_alpha vg_amr_reflevel vg_amr_jperb "+ + "vg_amr_drho vg_amr_du vg_amr_dpsq vg_amr_dbsq vg_amr_db vg_amr_alpha1 vg_amr_reflevel vg_amr_alpha2 "+ "vg_amr_translate_comm vg_gridcoordinates fg_gridcoordinates "); RP::addComposing( @@ -480,7 +486,13 @@ bool P::addParameters() { RP::add("AMR.alpha2_coarsen_threshold","Determines the maximum value of alpha_2 to unrefine cells, default half of the refine threshold", -1.0); RP::add("AMR.refine_cadence","Refine every nth load balance", 5); RP::add("AMR.refine_after","Start refinement after this many simulation seconds", 0.0); - RP::add("AMR.refine_radius","Maximum distance from Earth to refine", LARGE_REAL); + RP::add("AMR.refine_radius","Maximum distance from origin to allow refinement within. Only induced refinement allowed outside this radius.", LARGE_REAL); + RP::add("AMR.refinement_min_x", "Refinement minimum X coordinate, no refinement at x < this value (m) except induced refinement.", -LARGE_REAL); + RP::add("AMR.refinement_min_y", "Refinement minimum Y coordinate, no refinement at y < this value (m) except induced refinement.", -LARGE_REAL); + RP::add("AMR.refinement_min_z", "Refinement minimum Z coordinate, no refinement at z < this value (m) except induced refinement.", -LARGE_REAL); + RP::add("AMR.refinement_max_x", "Refinement maximum X coordinate, no refinement at x > this value (m) except induced refinement.", LARGE_REAL); + RP::add("AMR.refinement_max_y", "Refinement maximum Y coordinate, no refinement at y > this value (m) except induced refinement.", LARGE_REAL); + RP::add("AMR.refinement_max_z", "Refinement maximum Z coordinate, no refinement at z > this value (m) except induced refinement.", LARGE_REAL); RP::add("AMR.alpha1_drho_weight","Multiplier for delta rho in alpha calculation", 1.0); RP::add("AMR.alpha1_du_weight","Multiplier for delta U in alpha calculation", 1.0); RP::add("AMR.alpha1_dpsq_weight","Multiplier for delta p squared in alpha calculation", 1.0); @@ -633,6 +645,22 @@ void Parameters::getParameters() { } } + bool includefSaved = false; + for(uint i=0; i 0 || + P::systemWriteDistributionWriteYlineStride[i] > 0 || + P::systemWriteDistributionWriteZlineStride[i] > 0) { + includefSaved = true; + } + } + for(uint i=0; i 0) { + includefSaved = true; + } + } + + vector mpiioKeys, mpiioValues; RP::get("io.system_write_mpiio_hint_key", mpiioKeys); RP::get("io.system_write_mpiio_hint_value", mpiioValues); @@ -748,25 +776,25 @@ void Parameters::getParameters() { RP::get("AMR.refine_on_restart",P::refineOnRestart); RP::get("AMR.force_refinement",P::forceRefinement); RP::get("AMR.should_filter",P::shouldFilter); - RP::get("AMR.use_alpha1",P::useAlpha); - RP::get("AMR.alpha1_refine_threshold",P::alphaRefineThreshold); - RP::get("AMR.alpha1_coarsen_threshold",P::alphaCoarsenThreshold); - if (P::useAlpha && P::alphaCoarsenThreshold < 0) { - P::alphaCoarsenThreshold = P::alphaRefineThreshold / 2.0; + RP::get("AMR.use_alpha1",P::useAlpha1); + RP::get("AMR.alpha1_refine_threshold",P::alpha1RefineThreshold); + RP::get("AMR.alpha1_coarsen_threshold",P::alpha1CoarsenThreshold); + if (P::useAlpha1 && P::alpha1CoarsenThreshold < 0) { + P::alpha1CoarsenThreshold = P::alpha1RefineThreshold / 2.0; } - if (P::useAlpha && P::alphaRefineThreshold < 0) { + if (P::useAlpha1 && P::alpha1RefineThreshold < 0) { if (myRank == MASTER_RANK) { cerr << "ERROR invalid alpha_1 refine threshold" << endl; } MPI_Abort(MPI_COMM_WORLD, 1); } - RP::get("AMR.use_alpha2",P::useJPerB); - RP::get("AMR.alpha2_refine_threshold",P::jperbRefineThreshold); - RP::get("AMR.alpha2_coarsen_threshold",P::jperbCoarsenThreshold); - if (P::useJPerB && P::jperbCoarsenThreshold < 0) { - P::jperbCoarsenThreshold = P::jperbRefineThreshold / 2.0; + RP::get("AMR.use_alpha2",P::useAlpha2); + RP::get("AMR.alpha2_refine_threshold",P::alpha2RefineThreshold); + RP::get("AMR.alpha2_coarsen_threshold",P::alpha2CoarsenThreshold); + if (P::useAlpha2 && P::alpha2CoarsenThreshold < 0) { + P::alpha2CoarsenThreshold = P::alpha2RefineThreshold / 2.0; } - if (P::useJPerB && P::jperbRefineThreshold < 0) { + if (P::useAlpha2 && P::alpha2RefineThreshold < 0) { if (myRank == MASTER_RANK) { cerr << "ERROR invalid alpha_2 refine threshold" << endl; } @@ -776,6 +804,12 @@ void Parameters::getParameters() { RP::get("AMR.refine_cadence",P::refineCadence); RP::get("AMR.refine_after",P::refineAfter); RP::get("AMR.refine_radius",P::refineRadius); + RP::get("AMR.refinement_min_x", P::refinementMinX); + RP::get("AMR.refinement_min_y", P::refinementMinY); + RP::get("AMR.refinement_min_z", P::refinementMinZ); + RP::get("AMR.refinement_max_x", P::refinementMaxX); + RP::get("AMR.refinement_max_y", P::refinementMaxY); + RP::get("AMR.refinement_max_z", P::refinementMaxZ); RP::get("AMR.alpha1_drho_weight", P::alphaDRhoWeight); RP::get("AMR.alpha1_du_weight", P::alphaDUWeight); RP::get("AMR.alpha1_dpsq_weight", P::alphaDPSqWeight); @@ -946,6 +980,11 @@ void Parameters::getParameters() { RP::get("variables.output", P::outputVariableList); RP::get("variables.diagnostic", P::diagnosticVariableList); + // Insert vg_f_saved to the list if necessary + if(includefSaved) { + P::outputVariableList.push_back("vg_f_saved"); + } + // Filter duplicate variable names set dummy(P::outputVariableList.begin(), P::outputVariableList.end()); P::outputVariableList.clear(); @@ -973,7 +1012,7 @@ void Parameters::getParameters() { RP::get("fieldtracing.fieldLineTracer", tracerString); RP::get("fieldtracing.tracer_max_allowed_error", FieldTracing::fieldTracingParameters.max_allowed_error); RP::get("fieldtracing.tracer_max_attempts", FieldTracing::fieldTracingParameters.max_field_tracer_attempts); - RP::get("fieldtracing.tracer_min_dx", FieldTracing::fieldTracingParameters.min_tracer_dx); + RP::get("fieldtracing.tracer_min_dx", FieldTracing::fieldTracingParameters.min_tracer_dx_full_box); RP::get("fieldtracing.fullbox_max_incomplete_cells", FieldTracing::fieldTracingParameters.fullbox_max_incomplete_cells); RP::get("fieldtracing.fluxrope_max_incomplete_cells", FieldTracing::fieldTracingParameters.fluxrope_max_incomplete_cells); RP::get("fieldtracing.fullbox_and_fluxrope_max_absolute_distance_to_trace", FieldTracing::fieldTracingParameters.fullbox_and_fluxrope_max_distance); diff --git a/parameters.h b/parameters.h index 6fc3556e9..87838ef9f 100644 --- a/parameters.h +++ b/parameters.h @@ -192,12 +192,12 @@ struct Parameters { static bool refineOnRestart; static bool forceRefinement; static bool shouldFilter; - static bool useAlpha; - static Real alphaRefineThreshold; - static Real alphaCoarsenThreshold; - static bool useJPerB; - static Real jperbRefineThreshold; - static Real jperbCoarsenThreshold; + static bool useAlpha1; + static Real alpha1RefineThreshold; + static Real alpha1CoarsenThreshold; + static bool useAlpha2; + static Real alpha2RefineThreshold; + static Real alpha2CoarsenThreshold; static uint refineCadence; static Real refineAfter; static Real refineRadius; @@ -206,6 +206,12 @@ struct Parameters { static Real alphaDPSqWeight; static Real alphaDBSqWeight; static Real alphaDBWeight; + static Real refinementMinX; /*!< Do not refine at x coordinates below this value. */ + static Real refinementMinY; /*!< Do not refine at y coordinates below this value. */ + static Real refinementMinZ; /*!< Do not refine at z coordinates below this value. */ + static Real refinementMaxX; /*!< Do not refine at x coordinates above this value. */ + static Real refinementMaxY; /*!< Do not refine at y coordinates above this value. */ + static Real refinementMaxZ; /*!< Do not refine at z coordinates above this value. */ static int maxFilteringPasses; static int amrBoxNumber; static std::vector amrBoxHalfWidthX; diff --git a/projects/Alfven/Alfven.cfg b/projects/Alfven/Alfven.cfg index 20a687b48..c7a6d744e 100644 --- a/projects/Alfven/Alfven.cfg +++ b/projects/Alfven/Alfven.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = Alfven [io] @@ -24,6 +25,10 @@ y_min = 0.0 y_max = 200000.0 z_min = 0.0 z_max = 2500.0 +timestep_max = 1200 +dt = 2.0 + +[proton_vspace] vx_min = -350.0 vx_max = 350.0 vy_min = -350.0 @@ -33,8 +38,6 @@ vz_max = 350.0 vx_length = 10 vy_length = 10 vz_length = 10 -timestep_max = 1200 -dt = 2.0 [boundaries] periodic_x = yes @@ -52,8 +55,10 @@ B0 = 1e-10 Bx_guiding = 0.866025 By_guiding = 0.5 Bz_guiding = 0 -rho = 1e8 Wavelength = 100000.0 -Temperature = 0.86456498092 A_mag = 0.01 + +[proton_Alfven] +rho = 1e8 +Temperature = 0.86456498092 A_vel = 0.01 diff --git a/projects/Alfven/Alfven.cpp b/projects/Alfven/Alfven.cpp index b5756244b..485e555ad 100644 --- a/projects/Alfven/Alfven.cpp +++ b/projects/Alfven/Alfven.cpp @@ -118,16 +118,16 @@ namespace projects { } void Alfven::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { - Real* cellParams = cell->get_cell_parameters(); - creal x = cellParams[CellParams::XCRD]; - creal dx = cellParams[CellParams::DX]; - creal y = cellParams[CellParams::YCRD]; - creal dy = cellParams[CellParams::DY]; - - Real ksi = ((x + 0.5 * dx) * cos(this->ALPHA) + (y + 0.5 * dy) * sin(this->ALPHA)) / this->WAVELENGTH; - Real dBxavg = sin(2.0 * M_PI * ksi); - Real dByavg = sin(2.0 * M_PI * ksi); - Real dBzavg = cos(2.0 * M_PI * ksi); + //Real* cellParams = cell->get_cell_parameters(); + //creal x = cellParams[CellParams::XCRD]; + //creal dx = cellParams[CellParams::DX]; + //creal y = cellParams[CellParams::YCRD]; + //creal dy = cellParams[CellParams::DY]; + // + //Real ksi = ((x + 0.5 * dx) * cos(this->ALPHA) + (y + 0.5 * dy) * sin(this->ALPHA)) / this->WAVELENGTH; + //Real dBxavg = sin(2.0 * M_PI * ksi); + //Real dByavg = sin(2.0 * M_PI * ksi); + //Real dBzavg = cos(2.0 * M_PI * ksi); } void Alfven::setProjectBField( diff --git a/projects/Diffusion/Diffusion.cfg b/projects/Diffusion/Diffusion.cfg index d9af16e09..cf93fb663 100644 --- a/projects/Diffusion/Diffusion.cfg +++ b/projects/Diffusion/Diffusion.cfg @@ -1,5 +1,11 @@ +ParticlePopulations = proton project = Diffusion +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 0 @@ -23,6 +29,10 @@ y_min = -1000000 y_max = +1000000 z_min = -25000 z_max = +25000 +timestep_max = 10 +dt = 0.010 + +[proton_vspace] vx_min = -1000000 vx_max = +1000000 vy_min = -1000000 @@ -32,8 +42,6 @@ vz_max = +1000000 vx_length = 10 vy_length = 10 vz_length = 10 -timestep_max = 10 -dt = 0.010 [boundaries] periodic_x = no @@ -44,7 +52,7 @@ periodic_z = yes algorithm = RCB tolerance = 1.05 -[sparse] +[proton_sparse] minValue = 1.0e-12 [variables] @@ -54,7 +62,10 @@ diagnostic = populations_vg_blocks [Diffusion] B0 = 0.0 + +[proton_Diffusion] rho = 1.0e7 Temperature = 2.0e6 Scale_x = 100000.0 Scale_y = 100000.0 + diff --git a/projects/Dispersion/Dispersion.cfg b/projects/Dispersion/Dispersion.cfg index 846275888..b7f2dd3be 100644 --- a/projects/Dispersion/Dispersion.cfg +++ b/projects/Dispersion/Dispersion.cfg @@ -1,5 +1,7 @@ dynamic_timestep = 0 project = Dispersion +ParticlePopulations = proton + [io] diagnostic_write_interval = 1 @@ -24,15 +26,6 @@ y_min = 0.0 y_max = 1.0e5 z_min = 0.0 z_max = 1.0e5 -vx_min = -150000 -vx_max = +150000 -vy_min = -150000 -vy_max = +150000 -vz_min = -150000 -vz_max = +150000 -vx_length = 20 -vy_length = 20 -vz_length = 20 timestep_max = 1000000 dt = 0.04 t_max = 1000.0 @@ -51,21 +44,34 @@ diagnostic = populations_vg_blocks algorithm = RCB tolerance = 1.05 -[sparse] +[proton_vspace] +vx_min = -150000 +vx_max = +150000 +vy_min = -150000 +vy_max = +150000 +vz_min = -150000 +vz_max = +150000 +vx_length = 20 +vy_length = 20 +vz_length = 20 + +[proton_sparse] minValue = 1.0e-12 [Dispersion] B0 = 1.0e-9 +angleXY = 1.0 +angleXZ = 1.0 +magXPertAbsAmp = 1.0e-10 +magYPertAbsAmp = 1.0e-10 +magZPertAbsAmp = 1.0e-10 +maxwCutoff = 0.0 + +[proton_Dispersion] VX0 = 0.0 VY0 = 0.0 VZ0 = 0.0 -angleXY = 1.0 -angleXZ = 1.0 rho = 1.0e3 Temperature = 1.0e5 densityPertRelAmp = 0.1 -magXPertAbsAmp = 1.0e-10 -magYPertAbsAmp = 1.0e-10 -magZPertAbsAmp = 1.0e-10 velocityPertAbsAmp = 1000.0 -maxwCutoff = 0.0 diff --git a/projects/Dispersion/Dispersion.cpp b/projects/Dispersion/Dispersion.cpp index a8306f2b3..8508cb100 100644 --- a/projects/Dispersion/Dispersion.cpp +++ b/projects/Dispersion/Dispersion.cpp @@ -193,13 +193,13 @@ namespace projects { } void Dispersion::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { - Real* cellParams = cell->get_cell_parameters(); - creal x = cellParams[CellParams::XCRD]; - creal dx = cellParams[CellParams::DX]; - creal y = cellParams[CellParams::YCRD]; - creal dy = cellParams[CellParams::DY]; - creal z = cellParams[CellParams::ZCRD]; - creal dz = cellParams[CellParams::DZ]; + //Real* cellParams = cell->get_cell_parameters(); + //creal x = cellParams[CellParams::XCRD]; + //creal dx = cellParams[CellParams::DX]; + //creal y = cellParams[CellParams::YCRD]; + //creal dy = cellParams[CellParams::DY]; + //creal z = cellParams[CellParams::ZCRD]; + //creal dz = cellParams[CellParams::DZ]; std::default_random_engine rndState; setRandomCellSeed(cell,rndState); diff --git a/projects/Distributions/Distributions.cfg b/projects/Distributions/Distributions.cfg index c71196e73..cc2c72aae 100644 --- a/projects/Distributions/Distributions.cfg +++ b/projects/Distributions/Distributions.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = Distributions [io] @@ -23,6 +24,10 @@ y_min = -10.0e6 y_max = 10.0e6 z_min = -1.0e6 z_max = 1.0e6 +timestep_max = 0 +t_max = 0.0 + +[proton_vspace] vx_min = -1.0e6 vx_max = +1.0e6 vy_min = -1.0e6 @@ -32,8 +37,6 @@ vz_max = +1.0e6 vx_length = 100 vy_length = 100 vz_length = 100 -timestep_max = 0 -t_max = 0.0 [vlasovsolver] #minCFL = 0.4 @@ -58,10 +61,8 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-22 [Distributions] diff --git a/projects/Firehose/Firehose.cfg b/projects/Firehose/Firehose.cfg index 9dcadd509..229453742 100644 --- a/projects/Firehose/Firehose.cfg +++ b/projects/Firehose/Firehose.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = Firehose [io] @@ -23,6 +24,10 @@ y_min = 0.0 y_max = 1.0e3 z_min = 0 z_max = 1.0e2 +timestep_max = 10 +dt = 0.0 + +[proton_vspace] vx_min = -500000.0 vx_max = +500000.0 vy_min = -500000.0 @@ -32,8 +37,6 @@ vz_max = +500000.0 vx_length = 20 vy_length = 20 vz_length = 20 -timestep_max = 10 -dt = 0.0 [boundaries] @@ -52,10 +55,10 @@ output = vg_rank diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -[sparse] +[proton_sparse] minValue = 1.0e-15 -[Firehose] +[proton_Firehose] Vx1 = 0.0 Vy1 = 0.0 Vz1 = 0.0 @@ -72,6 +75,7 @@ Ty2 = 500000.0 Tz2 = 500000.0 rho2 = 50000.0 +[Firehose] Bx = 1.0e-9 By = 0.0 Bz = 0.0 diff --git a/projects/Flowthrough/Flowthrough.cpp b/projects/Flowthrough/Flowthrough.cpp index c1d71caf9..7f7f97bf7 100644 --- a/projects/Flowthrough/Flowthrough.cpp +++ b/projects/Flowthrough/Flowthrough.cpp @@ -62,6 +62,7 @@ namespace projects { RP::add("Flowthrough.emptyBox","Is the simulation domain empty initially?",false); RP::add("Flowthrough.densityModel","Plasma density model, 'Maxwellian' or 'SheetMaxwellian'",string("Maxwellian")); RP::add("Flowthrough.densityWidth","Width of signal around origin",6.e7); + RP::add("Flowthrough.rescaleDensity","Rescale VDF to match spatial ",false); RP::add("Flowthrough.Bx", "Magnetic field x component (T)", 0.0); RP::add("Flowthrough.By", "Magnetic field y component (T)", 0.0); RP::add("Flowthrough.Bz", "Magnetic field z component (T)", 0.0); @@ -100,6 +101,7 @@ namespace projects { exit(1); } RP::get("Flowthrough.densityWidth",this->densityWidth); + RP::get("Flowthrough.rescaleDensity",this->rescaleDensityFlag); // Per-population parameters for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { @@ -116,6 +118,60 @@ namespace projects { speciesParams.push_back(sP); } } + Real Flowthrough::getCorrectNumberDensity(spatial_cell::SpatialCell* cell,const uint popID) const { + const FlowthroughSpeciesParameters& sP = speciesParams[popID]; + Real x,y,z; + Real rvalue; + x = cell->parameters[CellParams::XCRD]; + y = cell->parameters[CellParams::YCRD]; + z = cell->parameters[CellParams::ZCRD]; + switch (densityModel) { + case Maxwellian: + rvalue = sP.rho; + break; + case SheetMaxwellian: + rvalue = sqrt(x*x + y*y + z*z); + if (rvalue <= 0.5*densityWidth) { + rvalue = 4*sP.rho; + } else { + rvalue = 0; + } + break; + case Square: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4*sP.rho; + } else { + rvalue = 4*sP.rhoBase; + //rvalue = 0; + } + break; + case Triangle: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4; + rvalue *= ( sP.rhoBase + (sP.rho-sP.rhoBase) * (1.-abs(x) / (0.5*densityWidth))); + } else { + rvalue = 4*sP.rhoBase; + //rvalue = 0; + } + break; + case Sinewave: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4; + rvalue *= ( sP.rhoBase + (sP.rho-sP.rhoBase) * (0.5 + 0.5*cos(M_PI * x / (0.5*densityWidth)))); + } else { + rvalue = 4*sP.rhoBase; + //rvalue = 0; + } + break; + default: + rvalue = sP.rho; + break; + + } + + return rvalue; + + } Real Flowthrough::getDistribValue(creal& x,creal& y, creal& z, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz, const uint popID) const { @@ -186,8 +242,6 @@ namespace projects { Real Flowthrough::calcPhaseSpaceDensity(creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz,const uint popID) const { - const FlowthroughSpeciesParameters& sP = speciesParams[popID]; - if (emptyBox == true) return 0.0; return getDistribValue(x+0.5*dx,y+0.5*dy,z+0.5*dz,vx+0.5*dvx,vy+0.5*dvy,vz+0.5*dvz,dvx,dvy,dvz,popID); } diff --git a/projects/Flowthrough/Flowthrough.h b/projects/Flowthrough/Flowthrough.h index 4685d4d78..94eb9e3ac 100644 --- a/projects/Flowthrough/Flowthrough.h +++ b/projects/Flowthrough/Flowthrough.h @@ -53,6 +53,11 @@ namespace projects { ); protected: + bool rescalesDensity(const uint popID) const { + return this->rescaleDensityFlag; + }; + Real getCorrectNumberDensity(spatial_cell::SpatialCell* cell,const uint popID) const; + Real getDistribValue( creal& x,creal& y, creal& z, creal& vx, creal& vy, creal& vz, @@ -77,6 +82,7 @@ namespace projects { * and matter will flow in only through the boundaries.*/ Real densityWidth; + bool rescaleDensityFlag; Real Bx; Real By; Real Bz; diff --git a/projects/Fluctuations/Fluctuations.cfg b/projects/Fluctuations/Fluctuations.cfg index b56428bfc..a6df3d07f 100644 --- a/projects/Fluctuations/Fluctuations.cfg +++ b/projects/Fluctuations/Fluctuations.cfg @@ -33,15 +33,6 @@ y_min = 0.0 y_max = 1.3e7 z_min = 0.0 z_max = 1.3e7 -vx_min = -1000000 -vx_max = +1000000 -vy_min = -1000000 -vy_max = +1000000 -vz_min = -1000000 -vz_max = +1000000 -vx_length = 10 -vy_length = 10 -vz_length = 10 t_max = 20 [boundaries] @@ -58,6 +49,17 @@ diagnostic = populations_vg_blocks algorithm = RCB tolerance = 1.05 +[proton_vspace] +vx_min = -1000000 +vx_max = +1000000 +vy_min = -1000000 +vy_max = +1000000 +vz_min = -1000000 +vz_max = +1000000 +vx_length = 10 +vy_length = 10 +vz_length = 10 + [proton_sparse] minValue = 1.0e-12 diff --git a/projects/Fluctuations/Fluctuations.cpp b/projects/Fluctuations/Fluctuations.cpp index 280fa2bd6..b2dae645c 100644 --- a/projects/Fluctuations/Fluctuations.cpp +++ b/projects/Fluctuations/Fluctuations.cpp @@ -135,17 +135,6 @@ namespace projects { } void Fluctuations::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { - Real* cellParams = cell->get_cell_parameters(); - creal x = cellParams[CellParams::XCRD]; - creal dx = cellParams[CellParams::DX]; - creal y = cellParams[CellParams::YCRD]; - creal dy = cellParams[CellParams::DY]; - creal z = cellParams[CellParams::ZCRD]; - creal dz = cellParams[CellParams::DZ]; - - CellID cellID = (int) ((x - Parameters::xmin) / dx) + - (int) ((y - Parameters::ymin) / dy) * Parameters::xcells_ini + - (int) ((z - Parameters::zmin) / dz) * Parameters::xcells_ini * Parameters::ycells_ini; std::default_random_engine rndState; setRandomCellSeed(cell,rndState); diff --git a/projects/Harris/Harris.cfg b/projects/Harris/Harris.cfg index 28717463c..184c3bde0 100644 --- a/projects/Harris/Harris.cfg +++ b/projects/Harris/Harris.cfg @@ -1,4 +1,5 @@ +ParticlePopulations = proton propagate_field = 1 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 1 @@ -25,6 +26,9 @@ y_min = -10000 y_max = +10000 z_min = -10000 z_max = +10000 +t_max = 100.0 + +[proton_vspace] vx_min = -1000000 vx_max = +1000000 vy_min = -1000000 @@ -34,15 +38,14 @@ vz_max = +1000000 vx_length = 20 vy_length = 20 vz_length = 20 -t_max = 1.0 [fieldsolver] ohmHallTerm = 2 ohmGradPeTerm = 0 electronTemperature = 4.0e6 -[sparse] -minValue = 1.0e-13 +[proton_sparse] +minValue = 1.0e-15 [boundaries] periodic_x = no @@ -53,6 +56,8 @@ boundary = Maxwellian [maxwellian] face = x+ face = x- + +[proton_maxwellian] file_x- = wall-.dat file_x+ = wall+.dat @@ -72,5 +77,7 @@ Scale_size = 150000.0 BX0 = 8.33061003094e-8 BY0 = 0.0 BZ0 = 0.0 + +[proton_Harris] Temperature = 2.0e6 rho = 1.0e7 diff --git a/projects/Harris/Harris.cpp b/projects/Harris/Harris.cpp index f57e2e7a7..9a5485315 100644 --- a/projects/Harris/Harris.cpp +++ b/projects/Harris/Harris.cpp @@ -99,7 +99,6 @@ namespace projects { creal& vx,creal& vy,creal& vz, creal& dvx,creal& dvy,creal& dvz,const uint popID ) const { - const HarrisSpeciesParameters& sP = speciesParams[popID]; return getDistribValue(x+0.5*dx, y+0.5*dy, z+0.5*dz, vx+0.5*dvx, vy+0.5*dvy, vz+0.5*dvz, dvx, dvy, dvz, popID); } diff --git a/projects/IPShock/IPShock.cfg b/projects/IPShock/IPShock.cfg index e0055142b..862f907a6 100644 --- a/projects/IPShock/IPShock.cfg +++ b/projects/IPShock/IPShock.cfg @@ -92,10 +92,11 @@ output = vg_f_saved #output = populations_vg_effectivesparsitythreshold diagnostic = populations_vg_blocks -dr_backstream_radius = 81264 -dr_backstream_vx = -250000 -dr_backstream_vy = 0.0 -dr_backstream_vz = -196561 +[proton_thermal] +vx = -250000. +vy = 0 +vz = -196561 +radius = 81264 [loadBalance] rebalanceInterval = 25 @@ -120,9 +121,9 @@ face = x- vlasovScheme_face_x- = None [maxwellian] -dynamic = 0 face = x+ precedence = 2 + [proton_maxwellian] file_x+ = upstream.dat diff --git a/projects/KHB/KHB.cfg b/projects/KHB/KHB.cfg index 051c0d5a9..b058ed3b8 100644 --- a/projects/KHB/KHB.cfg +++ b/projects/KHB/KHB.cfg @@ -1,63 +1,65 @@ -project = KHB - -[io] -diagnostic_write_interval = 1 -write_initial_state = 0 -restart_walltime_interval = 21000 -number_of_restarts = 1000 - -system_write_t_interval = 1 -system_write_file_name = bulk -system_write_distribution_stride = 0 -system_write_distribution_xline_stride = 0 -system_write_distribution_yline_stride = 0 -system_write_distribution_zline_stride = 0 - -[gridbuilder] -x_length = 20 -y_length = 1 -z_length = 20 -x_min = -5.0e6 -x_max = 5.0e6 -y_min = 0.0 -y_max = 5.0e5 -z_min = 0.0 -z_max = 1.0e7 -vx_min = -600000.0 -vx_max = +600000.0 -vy_min = -600000.0 -vy_max = +600000.0 -vz_min = -600000.0 -vz_max = +600000.0 -vx_length = 10 -vy_length = 10 -vz_length = 10 - -timestep_max = 10000 - - -[vlasovsolver] -maxCFL = 0.9 -minCFL = 0.7 - -[fieldsolver] -maxAlfvenVelocity = 1.0e9 - -[boundaries] -periodic_x = no -periodic_y = yes -periodic_z = yes -boundary = Maxwellian - -[maxwellian] -dynamic = 0 -face = x- -face = x+ -file_x- = mxm.dat -file_x+ = mxp.dat -precedence = 3 - -[variables] +ParticlePopulations = proton +project = KHB + +[io] +diagnostic_write_interval = 1 +write_initial_state = 0 +restart_walltime_interval = 21000 +number_of_restarts = 1000 + +system_write_t_interval = 1 +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[gridbuilder] +x_length = 20 +y_length = 20 +z_length = 1 +x_min = -5.0e6 +x_max = 5.0e6 +y_min = 0.0 +y_max = 1.0e7 +z_min = 0.0 +z_max = 5.0e5 +timestep_max = 10000 + +[proton_vspace] +vx_min = -600000.0 +vx_max = +600000.0 +vy_min = -600000.0 +vy_max = +600000.0 +vz_min = -600000.0 +vz_max = +600000.0 +vx_length = 10 +vy_length = 10 +vz_length = 10 + + + +[vlasovsolver] +maxCFL = 0.9 +minCFL = 0.7 + + +[boundaries] +periodic_x = no +periodic_y = yes +periodic_z = yes +boundary = Maxwellian + +[maxwellian] +face = x- +face = x+ +precedence = 3 + +[proton_maxwellian] +file_x- = mxm.dat +file_x+ = mxp.dat + +[variables] output = populations_vg_rho output = fg_b output = vg_pressure @@ -65,31 +67,31 @@ output = populations_vg_v output = populations_vg_blocks output = vg_boundarytype diagnostic = populations_vg_blocks - -[sparse] -minValue = 1.0e-16 - -[KHB] -Vx1 = 0.0 -Vy1 = 0.0 -Vz1 = 4.0e5 -Bx1 = 0.0 -By1 = 1.0e-9 -Bz1 = 0.0 -T1 = 1.0e5 -rho1 = 1.0e6 - -Vx2 = 0.0 -Vy2 = 0.0 -Vz2 = -4.0e5 -Bx2 = 0.0 -By2 = 1.0e-9 -Bz2 = 0.0 -T2 = 1.0e5 -rho2 = 2.0e6 - -lambda = 3.3333333e6 -amp = 0.0 -offset = 0.0 -transitionWidth = 1.0e6 - + +[proton_sparse] +minValue = 1.0e-16 + +[KHB] +P = 2.765276873575132E-10 + +Vx1 = 0.0 +Vy1 = 4.0e5 +Vz1 = 0.0 +Bx1 = 0.0 +By1 = 0.0 +Bz1 = 1.0e-9 +rho1 = 1.0e6 + +Vx2 = 0.0 +Vy2 = -4.0e5 +Vz2 = 0.0 +Bx2 = 0.0 +By2 = 0.0 +Bz2 = 1.0e-9 +rho2 = 2.0e6 + +lambda = 4.0e3 +amp = 0.0 +offset = 0.0 +transitionWidth = 1.0e6 + diff --git a/projects/KHB/KHB.cpp b/projects/KHB/KHB.cpp index c1b2f30bd..06bfef17b 100644 --- a/projects/KHB/KHB.cpp +++ b/projects/KHB/KHB.cpp @@ -33,17 +33,16 @@ namespace projects { using namespace std; - KHB::KHB(): Project() { } + KHB::KHB(): TriAxisSearch() { } KHB::~KHB() { } bool KHB::initialize(void) {return Project::initialize();} void KHB::addParameters() { typedef Readparameters RP; + RP::add("KHB.P", "Constant total pressure (thermal+magnetic), used to determine the temperature profile (Pa)", 0.0); RP::add("KHB.rho1", "Number density, this->TOP state (m^-3)", 0.0); RP::add("KHB.rho2", "Number density, this->BOTTOM state (m^-3)", 0.0); - RP::add("KHB.T1", "Temperature, this->TOP state (K)", 0.0); - RP::add("KHB.T2", "Temperature, this->BOTTOM state (K)", 0.0); RP::add("KHB.Vx1", "Bulk velocity x component, this->TOP state (m/s)", 0.0); RP::add("KHB.Vx2", "Bulk velocity x component, this->BOTTOM state (m/s)", 0.0); RP::add("KHB.Vy1", "Bulk velocity y component, this->TOP state (m/s)", 0.0); @@ -57,9 +56,11 @@ namespace projects { RP::add("KHB.Bz1", "Magnetic field z component, this->TOP state (T)", 0.0); RP::add("KHB.Bz2", "Magnetic field z component, this->BOTTOM state (T)", 0.0); RP::add("KHB.lambda", "Initial perturbation wavelength (m)", 0.0); - RP::add("KHB.amp", "Initial perturbation amplitude (m)", 0.0); + RP::add("KHB.amp", "Initial velocity perturbation amplitude (m s^-1)", 0.0); RP::add("KHB.offset", "Boundaries offset from 0 (m)", 0.0); RP::add("KHB.transitionWidth", "Width of tanh transition for all changing values", 0.0); + RP::add("KHB.harmonics", "Number of harmonics of lambda included in the initial perturbation", 0); + RP::add("KHB.randomPhase", "If true, set a random phase for each mode of the initial perturbation. Seed set via project_common.seed", 0); } void KHB::getParameters() { @@ -71,10 +72,9 @@ namespace projects { abort(); } + RP::get("KHB.P", this->P); RP::get("KHB.rho1", this->rho[this->TOP]); RP::get("KHB.rho2", this->rho[this->BOTTOM]); - RP::get("KHB.T1", this->T[this->TOP]); - RP::get("KHB.T2", this->T[this->BOTTOM]); RP::get("KHB.Vx1", this->Vx[this->TOP]); RP::get("KHB.Vx2", this->Vx[this->BOTTOM]); RP::get("KHB.Vy1", this->Vy[this->TOP]); @@ -91,37 +91,80 @@ namespace projects { RP::get("KHB.amp", this->amp); RP::get("KHB.offset", this->offset); RP::get("KHB.transitionWidth", this->transitionWidth); + RP::get("KHB.harmonics", this->harmonics); + RP::get("KHB.randomPhase", this->randomPhase); } - Real KHB::profile(creal top, creal bottom, creal x, creal z) const { + Real KHB::profile(creal top, creal bottom, creal x) const { if(top == bottom) { return top; } if(this->offset != 0.0) { return 0.5 * ((top-bottom) * ( - tanh((x + this->offset + this->amp * cos(2.0*M_PI*z/this->lambda))/this->transitionWidth) - - tanh((x-(this->offset + this->amp * cos(2.0*M_PI*z/this->lambda)))/this->transitionWidth) -1) + top+bottom); + tanh((x + this->offset)/this->transitionWidth) - + tanh((x - this->offset)/this->transitionWidth) -1) + top+bottom); } else { return 0.5 * ((top-bottom) * tanh(x/this->transitionWidth) + top+bottom); } } - Real KHB::getDistribValue(creal& x, creal& z, creal& vx, creal& vy, creal& vz, const uint popID) const { - creal mass = physicalconstants::MASS_PROTON; - creal kb = physicalconstants::K_B; - Real rho = profile(this->rho[this->BOTTOM], this->rho[this->TOP], x, z); - Real T = profile(this->T[this->BOTTOM], this->T[this->TOP], x, z); - Real Vx = profile(this->Vx[this->BOTTOM], this->Vx[this->TOP], x, z); - Real Vy = profile(this->Vy[this->BOTTOM], this->Vy[this->TOP], x, z); - Real Vz = profile(this->Vz[this->BOTTOM], this->Vz[this->TOP], x, z); - - return rho * pow(mass / (2.0 * M_PI * kb * T), 1.5) * - exp(- mass * (pow(vx - Vx, 2.0) + pow(vy - Vy, 2.0) + pow(vz - Vz, 2.0)) / (2.0 * kb * T)); + inline vector > KHB::getV0( + creal x, + creal y, + creal z, + const uint popID + ) const { + Real Vx = profile(this->Vx[this->BOTTOM], this->Vx[this->TOP], x); + Real Vy = profile(this->Vy[this->BOTTOM], this->Vy[this->TOP], x); + Real Vz = profile(this->Vz[this->BOTTOM], this->Vz[this->TOP], x); + + // add an initial velocity perturbation to Vx + // initialize RNG for calculating random phases for the initial perturbation + std::default_random_engine rndState; + setRandomSeed(0,rndState); + Real phase = 0.0; + + // add each mode to the initial perturbation + for (int i=0; i<=this->harmonics; i++) { + if (this->randomPhase) { + phase = 2.0 * M_PI * getRandomNumber(rndState); + } + + if (this->offset != 0.0) { + Vx += this->amp * sin(2.0 * (i + 1) * M_PI * y / this->lambda + phase) * (exp(-pow((x + this->offset) / this->transitionWidth,2)) + exp(-pow((x - this->offset) / this->transitionWidth,2))); + } else { + Vx += this->amp * sin(2.0 * (i + 1) * M_PI * y / this->lambda + phase) * exp(-pow(x / this->transitionWidth,2)); + } + } + + vector > centerPoints; + std::array V0 {{Vx,Vy,Vz}}; + centerPoints.push_back(V0); + return centerPoints; + } + + inline Real KHB::getDistribValue(creal& x, creal& y, creal& z, creal& vx, creal& vy, creal& vz, const uint popID) const { + creal mass = getObjectWrapper().particleSpecies[popID].mass; + Real rho = profile(this->rho[this->BOTTOM], this->rho[this->TOP], x); + std::array initV0 = this->getV0(x, y, z, popID)[0]; + Real Vx = initV0[0]; + Real Vy = initV0[1]; + Real Vz = initV0[2]; + + // calculate the temperature such that the total pressure is constant across the domain + Real mu0 = physicalconstants::MU_0; + Real Bx = profile(this->Bx[this->BOTTOM], this->Bx[this->TOP], x); + Real By = profile(this->By[this->BOTTOM], this->By[this->TOP], x); + Real Bz = profile(this->Bz[this->BOTTOM], this->Bz[this->TOP], x); + Real kbT = (this->P - 0.5 * (Bx * Bx + By * By + Bz * Bz) / mu0) / rho; + + return rho * pow(mass / (2.0 * M_PI * kbT), 1.5) * + exp(- mass * (pow(vx - Vx, 2.0) + pow(vy - Vy, 2.0) + pow(vz - Vz, 2.0)) / (2.0 * kbT)); } Real KHB::calcPhaseSpaceDensity(creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz,const uint popID) const { - return getDistribValue(x+0.5*dx, z+0.5*dz, vx+0.5*dvx, vy+0.5*dvy, vz+0.5*dvz, popID); + return getDistribValue(x+0.5*dx, y+0.5*dy, z+0.5*dz, vx+0.5*dvx, vy+0.5*dvy, vz+0.5*dvz, popID); } void KHB::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } @@ -143,9 +186,9 @@ namespace projects { const std::array xyz = perBGrid.getPhysicalCoords(x, y, z); std::array* cell = perBGrid.get(x, y, z); - cell->at(fsgrids::bfield::PERBX) = profile(this->Bx[this->BOTTOM], this->Bx[this->TOP], xyz[0]+0.5*perBGrid.DX, xyz[2]+0.5*perBGrid.DZ); - cell->at(fsgrids::bfield::PERBY) = profile(this->By[this->BOTTOM], this->By[this->TOP], xyz[0]+0.5*perBGrid.DX, xyz[2]+0.5*perBGrid.DZ); - cell->at(fsgrids::bfield::PERBZ) = profile(this->Bz[this->BOTTOM], this->Bz[this->TOP], xyz[0]+0.5*perBGrid.DX, xyz[2]+0.5*perBGrid.DZ); + cell->at(fsgrids::bfield::PERBX) = profile(this->Bx[this->BOTTOM], this->Bx[this->TOP], xyz[0]+0.5*perBGrid.DX); + cell->at(fsgrids::bfield::PERBY) = profile(this->By[this->BOTTOM], this->By[this->TOP], xyz[0]+0.5*perBGrid.DX); + cell->at(fsgrids::bfield::PERBZ) = profile(this->Bz[this->BOTTOM], this->Bz[this->TOP], xyz[0]+0.5*perBGrid.DX); } } } diff --git a/projects/KHB/KHB.h b/projects/KHB/KHB.h index 122501835..512f61195 100644 --- a/projects/KHB/KHB.h +++ b/projects/KHB/KHB.h @@ -26,10 +26,10 @@ #include #include "../../definitions.h" -#include "../project.h" +#include "../projectTriAxisSearch.h" namespace projects { - class KHB: public Project { + class KHB: public TriAxisSearch { public: KHB(); virtual ~KHB(); @@ -38,6 +38,12 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); + virtual std::vector > getV0( + creal x, + creal y, + creal z, + const uint popID + ) const; virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, @@ -52,17 +58,17 @@ namespace projects { ); protected: Real getDistribValue( - creal& x, creal& z, + creal& x, creal& y, creal& z, creal& vx, creal& vy, creal& vz, const uint popID) const; - Real profile(creal top, creal bottom, creal x, creal z) const; + Real profile(creal top, creal bottom, creal x) const; enum { TOP, BOTTOM }; + Real P; Real rho[2]; - Real T[2]; Real Vx[2]; Real Vy[2]; Real Vz[2]; @@ -73,6 +79,8 @@ namespace projects { Real amp; Real offset; Real transitionWidth; + int harmonics; + bool randomPhase; }; // class KHB } // namespace #endif diff --git a/projects/KHB/mxm.dat b/projects/KHB/mxm.dat index ba68210cf..b222f11c0 100644 --- a/projects/KHB/mxm.dat +++ b/projects/KHB/mxm.dat @@ -1 +1 @@ -0.0 1.0e6 1.0e7 0.0 0.0 4.0e6 1.0e-11 1.0e-7 1.0e-11 +0.0 1.0e6 2.0e7 0.0 4.0e6 0.0 0.0 0.0 1.0e-9 diff --git a/projects/KHB/mxp.dat b/projects/KHB/mxp.dat index bb64e30cd..6d7bd746a 100644 --- a/projects/KHB/mxp.dat +++ b/projects/KHB/mxp.dat @@ -1 +1 @@ -0.0 4.0e6 1.0e7 0.0 0.0 -4.0e6 1.0e-11 1.0e-7 1.0e-11 +0.0 2.0e6 1.0e7 0.0 -4.0e6 0.0 0.0 0.0 1.0e-9 diff --git a/projects/Larmor/Larmor.cfg b/projects/Larmor/Larmor.cfg index 1f076be62..10031fd27 100644 --- a/projects/Larmor/Larmor.cfg +++ b/projects/Larmor/Larmor.cfg @@ -1,3 +1,5 @@ + +ParticlePopulations = proton propagate_field = 0 project = Larmor @@ -24,6 +26,10 @@ x_min = 0.0 x_max = 1.0e7 z_min = 0.0 z_max = 5.0e4 +timestep_max = 100000 +dt = 0.1 + +[proton_vspace] vx_min = -150000 vx_max = +150000 vy_min = -150000 @@ -33,8 +39,6 @@ vz_max = +150000 vx_length = 10 vy_length = 10 vz_length = 10 -timestep_max = 100000 -dt = 0.1 [boundaries] periodic_x = yes @@ -52,7 +56,7 @@ diagnostic = populations_vg_blocks algorithm = RCB tolerance = 1.05 -[sparse] +[proton_sparse] minValue = 1.0e-14 [Larmor] diff --git a/projects/Magnetosphere/Magnetosphere.cfg b/projects/Magnetosphere/Magnetosphere.cfg index 1088cc723..6cdc7bb24 100644 --- a/projects/Magnetosphere/Magnetosphere.cfg +++ b/projects/Magnetosphere/Magnetosphere.cfg @@ -94,7 +94,6 @@ minValue = 1.0e-13 constBgBX = 0.0 constBgBY = 0.0 constBgBZ = -1.0e-9 -noDipoleInSW = 0.0 refine_L3radius = 6.371e7 # 10 RE @@ -178,8 +177,6 @@ output = vg_f_saved output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction [boundaries] periodic_x = no @@ -202,7 +199,6 @@ face = z- face = z+ [maxwellian] -dynamic = 0 face = x+ precedence = 4 diff --git a/projects/Magnetosphere/Magnetosphere.cpp b/projects/Magnetosphere/Magnetosphere.cpp index 86a881796..ed19b673b 100644 --- a/projects/Magnetosphere/Magnetosphere.cpp +++ b/projects/Magnetosphere/Magnetosphere.cpp @@ -117,12 +117,13 @@ namespace projects { RP::get("Magnetosphere.dipoleType", this->dipoleType); - /* Enforce no dipole in solar wind with dipole type 4 */ - if ((this->dipoleType == 4) && (!this->noDipoleInSW)) { + /* Enforce "dipole" (incl. correction terms) in solar wind with dipole type 4. */ + if ((this->dipoleType == 4) && (this->noDipoleInSW)) { if(myRank == MASTER_RANK) { - std::cerr<<"Note: Initializing Magnetosphere with dipole type 4, enforcing no dipole in solar wind!"<noDipoleInSW = true; + this->noDipoleInSW = false; } /** Read inner boundary parameters from either ionospheric or copysphere sysboundary condition */ @@ -262,8 +263,6 @@ namespace projects { creal& vx,creal& vy,creal& vz,creal& dvx,creal& dvy, creal& dvz,const uint popID) const { - const MagnetosphereSpeciesParameters& sP = this->speciesParams[popID]; - return getDistribValue(x+0.5*dx,y+0.5*dy,z+0.5*dz,vx+0.5*dvx,vy+0.5*dvy,vz+0.5*dvz,dvx,dvy,dvz,popID); } diff --git a/projects/Magnetosphere/Magnetosphere_BCH-like.cfg b/projects/Magnetosphere/Magnetosphere_BCH-like.cfg index f3e38481e..c6f862317 100644 --- a/projects/Magnetosphere/Magnetosphere_BCH-like.cfg +++ b/projects/Magnetosphere/Magnetosphere_BCH-like.cfg @@ -147,7 +147,6 @@ face = z- face = z+ [maxwellian] -dynamic = 0 face = x+ precedence = 4 diff --git a/projects/MultiPeak/MultiPeak.cfg b/projects/MultiPeak/MultiPeak.cfg index 47523cc77..7bfc9a932 100644 --- a/projects/MultiPeak/MultiPeak.cfg +++ b/projects/MultiPeak/MultiPeak.cfg @@ -1,9 +1,16 @@ +ParticlePopulations = proton + propagate_field = 0 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 0 dynamic_timestep = 0 project = MultiPeak +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 1 @@ -27,6 +34,10 @@ y_min = 0.0 y_max = 1.0e3 z_min = 0.0 z_max = 1.0e3 +t_max = 3600.0 +dt = 15.0 + +[proton_vspace] vx_min = -2.0e6 vx_max = +2.0e6 vy_min = -2.0e6 @@ -36,8 +47,6 @@ vz_max = +2.0e6 vx_length = 12 vy_length = 12 vz_length = 12 -t_max = 3600.0 -dt = 15.0 [vlasovsolver] #minCFL = 0.4 @@ -62,13 +71,26 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-22 [MultiPeak] +Bx = 0.0 +By = 0.0 +Bz = 1.82206867e-10 + +dBx = 0.0 +dBy = 0.0e-10 +dBz = 0.0e-10 + +magXPertAbsAmp = 0.0 +magYPertAbsAmp = 0.0 +magZPertAbsAmp = 0.0 + +lambda = 120.0e5 + +[proton_MultiPeak] n = 2 Vx = -5.0e5 @@ -89,17 +111,3 @@ Tz = 5.0e5 rho = 1.0e4 rhoPertAbsAmp = 0.0 -Bx = 0.0 -By = 0.0 -Bz = 1.82206867e-10 - -dBx = 0.0 -dBy = 0.0e-10 -dBz = 0.0e-10 - -magXPertAbsAmp = 0.0 -magYPertAbsAmp = 0.0 -magZPertAbsAmp = 0.0 - -lambda = 120.0e5 - diff --git a/projects/MultiPeak/MultiPeak.cpp b/projects/MultiPeak/MultiPeak.cpp index e53ddc758..1fbbf6926 100644 --- a/projects/MultiPeak/MultiPeak.cpp +++ b/projects/MultiPeak/MultiPeak.cpp @@ -146,8 +146,6 @@ namespace projects { creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz, const uint popID) const { - const MultiPeakSpeciesParameters& sP = speciesParams[popID]; - Real rhoFactor = 1.0; switch (densityModel) { case Uniform: diff --git a/projects/MultiPeak/MultiPeak120.cfg b/projects/MultiPeak/MultiPeak120.cfg index 0b54be541..b9febd3f5 100644 --- a/projects/MultiPeak/MultiPeak120.cfg +++ b/projects/MultiPeak/MultiPeak120.cfg @@ -1,9 +1,16 @@ +ParticlePopulations = proton + propagate_field = 1 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 1 dynamic_timestep = 1 project = MultiPeak +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 1 @@ -42,6 +49,10 @@ y_min = 0.0 y_max = 1.0e3 z_min = 0.0 z_max = 1.0e3 +timestep_max = 0 +t_max = 1000.0 + +[proton_vspace] vx_min = -1.0e6 vx_max = +1.0e6 vy_min = -1.0e6 @@ -51,8 +62,6 @@ vz_max = +1.0e6 vx_length = 10 vy_length = 10 vz_length = 10 -timestep_max = 0 -t_max = 1000.0 [vlasovsolver] #minCFL = 0.4 @@ -77,13 +86,26 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-22 [MultiPeak] +Bx = 5.0e-9 +By = 0.0 +Bz = 0.0 + +dBx = 0.0 +dBy = 0.0e-10 +dBz = 0.0e-10 + +magXPertAbsAmp = 0.0 +magYPertAbsAmp = 0.0 +magZPertAbsAmp = 0.0 + +lambda = 120.0e5 + +[proton_MultiPeak] n = 2 Vx = -5.0e5 @@ -104,16 +126,3 @@ Tz = 5.0e5 rho = 1.0e4 rhoPertAbsAmp = 0.0 -Bx = 5.0e-9 -By = 0.0 -Bz = 0.0 - -dBx = 0.0 -dBy = 0.0e-10 -dBz = 0.0e-10 - -magXPertAbsAmp = 0.0 -magYPertAbsAmp = 0.0 -magZPertAbsAmp = 0.0 - -lambda = 120.0e5 diff --git a/projects/MultiPeak/MultiPeak2.cfg b/projects/MultiPeak/MultiPeak2.cfg index 2b4569949..c51d1f243 100644 --- a/projects/MultiPeak/MultiPeak2.cfg +++ b/projects/MultiPeak/MultiPeak2.cfg @@ -1,8 +1,16 @@ +ParticlePopulations = proton + propagate_field = 0 -propagate_vlasov = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 dynamic_timestep = 1 project = MultiPeak +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 1 @@ -34,6 +42,9 @@ y_min = 0.0 y_max = 2.4e4 z_min = 0.0 z_max = 2.4e4 +t_max = 1300.0 + +[proton_vspace] vx_min = -1.0e6 vx_max = +1.0e6 vy_min = -1.0e6 @@ -43,7 +54,6 @@ vz_max = +1.0e6 vx_length = 50 vy_length = 50 vz_length = 50 -t_max = 1300.0 [vlasovsolver] #minCFL = 0.4 @@ -68,13 +78,22 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-16 [MultiPeak] +Bx = 5.0e-9 +By = 0.0 +Bz = 0.0 + +dBx = 0.0 +dBy = 5.0e-10 +dBz = 5.0e-10 + +lambda = 1.2e7 + +[proton_MultiPeak] n = 2 Vx = -5.0e5 @@ -95,12 +114,3 @@ Tz = 5.0e5 rho = 1.0e4 rhoPertAbsAmp = 0.0 -Bx = 5.0e-9 -By = 0.0 -Bz = 0.0 - -dBx = 0.0 -dBy = 5.0e-10 -dBz = 5.0e-10 - -lambda = 1.2e7 diff --git a/projects/MultiPeak/MultiPeak9.cfg b/projects/MultiPeak/MultiPeak9.cfg index 4015ecb74..e4e56b8a0 100644 --- a/projects/MultiPeak/MultiPeak9.cfg +++ b/projects/MultiPeak/MultiPeak9.cfg @@ -1,8 +1,16 @@ +ParticlePopulations = proton + propagate_field = 1 -propagate_vlasov = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 dynamic_timestep = 1 project = MultiPeak +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 1 @@ -26,6 +34,9 @@ y_min = 0.0 y_max = 6.0e7 z_min = 0.0 z_max = 8.57e5 +t_max = 1000.0 + +[proton_vspace] vx_min = -1.0e6 vx_max = +1.0e6 vy_min = -1.0e6 @@ -35,7 +46,6 @@ vz_max = +1.0e6 vx_length = 50 vy_length = 50 vz_length = 50 -t_max = 1000.0 [vlasovsolver] #minCFL = 0.4 @@ -60,13 +70,22 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-16 [MultiPeak] +Bx = 5.0e-9 +By = 5.0e-11 +Bz = 0.0 + +dBx = 0.0 +dBy = 0.0 +dBz = 0.0 + +lambda = 1.2e7 + +[proton_MultiPeak] n = 2 Vx = -5.0e5 @@ -87,12 +106,3 @@ Tz = 5.0e5 rho = 1.0e4 rhoPertAbsAmp = 0.0 -Bx = 5.0e-9 -By = 5.0e-11 -Bz = 0.0 - -dBx = 0.0 -dBy = 0.0 -dBz = 0.0 - -lambda = 1.2e7 diff --git a/projects/Riemann1/Riemann1.cfg b/projects/Riemann1/Riemann1.cfg index b82e7d4d7..2b587d7bf 100644 --- a/projects/Riemann1/Riemann1.cfg +++ b/projects/Riemann1/Riemann1.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = Riemann1 [io] @@ -23,6 +24,10 @@ y_min = -10000.0 y_max = +10000.0 z_min = -10000.0 z_max = +10000.0 +timestep_max = 20000 +dt = 0.004 + +[proton_vspace] vx_min = -500000.0 vx_max = +500000.0 vy_min = -500000.0 @@ -32,8 +37,6 @@ vz_max = +500000.0 vx_length = 15 vy_length = 15 vz_length = 15 -timestep_max = 20000 -dt = 0.004 [variables] output = populations_vg_rho @@ -42,7 +45,7 @@ output = vg_pressure output = populations_vg_v diagnostic = populations_vg_blocks -[sparse] +[proton_sparse] minValue = 1.0e-15 [boundaries] @@ -51,13 +54,15 @@ periodic_y = yes periodic_z = yes [maxwellian] -dynamic = 0 face = x- -file_x- = Riemann_x-.dat face = x+ -file_x+ = Riemann_x+.dat precedence = 4 +[proton_maxwellian] +dynamic = 0 +file_x- = Riemann_x-.dat +file_x+ = Riemann_x+.dat + [Riemann] Vx1 = 400000.0 Vy1 = 40000.0 diff --git a/projects/Riemann1/Riemann1.cpp b/projects/Riemann1/Riemann1.cpp index 044571bba..8f4cbe35c 100644 --- a/projects/Riemann1/Riemann1.cpp +++ b/projects/Riemann1/Riemann1.cpp @@ -117,8 +117,8 @@ namespace projects { const std::array xyz = perBGrid.getPhysicalCoords(x, y, z); std::array* cell = perBGrid.get(x, y, z); - Real Bxavg, Byavg, Bzavg; - Bxavg = Byavg = Bzavg = 0.0; + //Real Bxavg, Byavg, Bzavg; + //Bxavg = Byavg = Bzavg = 0.0; cell->at(fsgrids::bfield::PERBX) = (xyz[0] < 0.0) ? this->Bx[this->LEFT] : this->Bx[this->RIGHT]; cell->at(fsgrids::bfield::PERBY) = (xyz[0] < 0.0) ? this->By[this->LEFT] : this->By[this->RIGHT]; cell->at(fsgrids::bfield::PERBZ) = (xyz[0] < 0.0) ? this->Bz[this->LEFT] : this->Bz[this->RIGHT]; diff --git a/projects/Shock/Shock.cfg b/projects/Shock/Shock.cfg index 7e9a77d28..7916c579a 100644 --- a/projects/Shock/Shock.cfg +++ b/projects/Shock/Shock.cfg @@ -2,6 +2,7 @@ propagate_field = 0 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 1 project = Shock +ParticlePopulations = proton [io] diagnostic_write_interval = 1 @@ -26,15 +27,6 @@ x_min = 0.0 x_max = 0.2e7 z_min = 0.0 z_max = 5.0e4 -vx_min = -150000 -vx_max = +150000 -vy_min = -150000 -vy_max = +150000 -vz_min = -150000 -vz_max = +150000 -vx_length = 10 -vy_length = 10 -vz_length = 10 timestep_max = 20000 dt = 0.05 @@ -54,7 +46,18 @@ diagnostic = populations_vg_blocks algorithm = RCB tolerance = 1.05 -[sparse] +[proton_vspace] +vx_min = -150000 +vx_max = +150000 +vy_min = -150000 +vy_max = +150000 +vz_min = -150000 +vz_max = +150000 +vx_length = 10 +vy_length = 10 +vz_length = 10 + +[proton_sparse] minValue = 1.0e-14 [Shock] diff --git a/projects/Shocktest/Shocktest.cfg b/projects/Shocktest/Shocktest.cfg index da9d0e7d7..706a0b431 100644 --- a/projects/Shocktest/Shocktest.cfg +++ b/projects/Shocktest/Shocktest.cfg @@ -1,3 +1,5 @@ + +ParticlePopulations = proton project = Shocktest [io] @@ -27,6 +29,9 @@ y_min = -1.0e5 y_max = +1.0e5 z_min = -1.0e5 z_max = +1.0e5 +timestep_max = 300 + +[proton_vspace] vx_min = -1000000.0 vx_max = +1000000.0 vy_min = -1000000.0 @@ -37,7 +42,6 @@ vx_length = 50 vy_length = 50 vz_length = 50 -timestep_max = 300 [vlasovsolver] maxSlAccelerationSubcycles = 2 @@ -49,15 +53,18 @@ periodic_z = yes boundary = Maxwellian [outflow] -face = x+ precedence = 3 +[proton_outflow] +face = x+ [maxwellian] -dynamic = 0 face = x- -file_x- = Shocktest_x-.dat precedence = 4 +[proton_maxwellian] +file_x- = Shocktest_x-.dat +dynamic = 0 + [variables] output = populations_vg_rho output = populations_vg_v @@ -71,10 +78,9 @@ output = vg_boundarylayer output = vg_pressure diagnostic = populations_vg_blocks -diagnostic = populations_vg_rho diagnostic = populations_vg_rho_loss_adjust -[sparse] +[proton_sparse] minValue = 1.0e-15 [Shocktest] diff --git a/projects/Shocktest/Shocktest.cpp b/projects/Shocktest/Shocktest.cpp index 4630c8e7b..ef9e31f19 100644 --- a/projects/Shocktest/Shocktest.cpp +++ b/projects/Shocktest/Shocktest.cpp @@ -28,7 +28,7 @@ #include "../../object_wrapper.h" #include "Shocktest.h" -#include "../../spatial_cell.hpp" +#include "../../spatial_cell_wrapper.hpp" #include "../../common.h" #include "../project.h" #include "../../parameters.h" @@ -186,8 +186,6 @@ namespace projects { const std::array xyz = perBGrid.getPhysicalCoords(x, y, z); std::array* cell = perBGrid.get(x, y, z); - Real Bxavg, Byavg, Bzavg; - Bxavg = Byavg = Bzavg = 0.0; cell->at(fsgrids::bfield::PERBX) = (xyz[0] < 0.0) ? this->Bx[this->LEFT] : this->Bx[this->RIGHT]; cell->at(fsgrids::bfield::PERBY) = (xyz[0] < 0.0) ? this->By[this->LEFT] : this->By[this->RIGHT]; cell->at(fsgrids::bfield::PERBZ) = (xyz[0] < 0.0) ? this->Bz[this->LEFT] : this->Bz[this->RIGHT]; diff --git a/projects/Shocktest/Shocktest.h b/projects/Shocktest/Shocktest.h index 3f5e102fb..a661460bd 100644 --- a/projects/Shocktest/Shocktest.h +++ b/projects/Shocktest/Shocktest.h @@ -26,7 +26,7 @@ #include #include "../../definitions.h" -#include "../../spatial_cell.hpp" +#include "../../spatial_cell_wrapper.hpp" #include "../project.h" #include "../projectTriAxisSearch.h" diff --git a/projects/Template/Template.cfg b/projects/Template/Template.cfg index 29f14c785..d6d95907b 100644 --- a/projects/Template/Template.cfg +++ b/projects/Template/Template.cfg @@ -1,3 +1,5 @@ +ParticlePopulations = proton + propagate_field = 1 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 1 @@ -5,6 +7,11 @@ dynamic_timestep = 1 project = Template +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + [io] diagnostic_write_interval = 1 write_initial_state = 0 @@ -28,6 +35,9 @@ y_min = 0.0 y_max = 6.0e7 z_min = 0.0 z_max = 8.57e5 +t_max = 2000.0 + +[proton_vspace] vx_min = -1.0e6 vx_max = +1.0e6 vy_min = -1.0e6 @@ -37,7 +47,6 @@ vz_max = +1.0e6 vx_length = 50 vy_length = 50 vz_length = 50 -t_max = 2000.0 [boundaries] periodic_x = yes @@ -57,10 +66,8 @@ output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks diagnostic = populations_vg_rho_loss_adjust -diagnostic = MaxDistributionFunction -diagnostic = MinDistributionFunction -[sparse] +[proton_sparse] minValue = 1.0e-16 [Template] diff --git a/projects/VelocityBox/VelocityBox.cfg b/projects/VelocityBox/VelocityBox.cfg deleted file mode 100644 index 13f662340..000000000 --- a/projects/VelocityBox/VelocityBox.cfg +++ /dev/null @@ -1,76 +0,0 @@ -propagate_field = 0 -propagate_vlasov_acceleration = 1 -propagate_vlasov_translation = 1 -dynamic_timestep = 0 -project = VelocityBox - -[io] -diagnostic_write_interval = 1 -write_initial_state = 1 - -system_write_t_interval = 360.0 -system_write_file_name = fullf -system_write_distribution_stride = 1 -system_write_distribution_xline_stride = 0 -system_write_distribution_yline_stride = 0 -system_write_distribution_zline_stride = 0 - - - -[gridbuilder] -x_length = 1 -y_length = 1 -z_length = 1 -x_min = 0.0 -x_max = 1.0e3 -y_min = 0.0 -y_max = 1.0e3 -z_min = 0 -z_max = 1.0e2 -vx_min = -500000.0 -vx_max = +500000.0 -vy_min = -500000.0 -vy_max = +500000.0 -vz_min = -500000.0 -vz_max = +500000.0 -vx_length = 25 -vy_length = 25 -vz_length = 25 -timestep_max = 3600 -dt = 1.0 - - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = populations_vg_rho -output = fg_b -output = vg_pressure -output = populations_vg_v -output = fg_e -output = populations_vg_ptensor -output = vg_rank -diagnostic = populations_vg_blocks -diagnostic = populations_vg_rho -diagnostic = populations_vg_rho_loss_adjust - -[sparse] -minValue = 1.0e-15 - -[VelocityBox] -rho = 1e-10 - -Vx1 = -100e3 -Vx2 = 100e3 -Vy1 = -100e3 -Vy2 = 100e3 -Vz1 = -400e3 -Vz2 = 400e3 - -#1.82206867e-10 gives a period of 360s, useful for testing... -Bx = 1.82206867e-10 -By = 0.0 -Bz = 0.0 diff --git a/projects/VelocityBox/VelocityBox.cpp b/projects/VelocityBox/VelocityBox.cpp deleted file mode 100644 index 010b8eb73..000000000 --- a/projects/VelocityBox/VelocityBox.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include - -#include "../../common.h" -#include "../../readparameters.h" -#include "../../backgroundfield/backgroundfield.h" -#include "../../backgroundfield/constantfield.hpp" -#include "../../object_wrapper.h" - -#include "VelocityBox.h" - -using namespace std; -using namespace spatial_cell; - -namespace projects { - VelocityBox::VelocityBox(): Project() { } - VelocityBox::~VelocityBox() { } - - - bool VelocityBox::initialize(void) {return Project::initialize();} - - void VelocityBox::addParameters(){ - typedef Readparameters RP; - RP::add("VelocityBox.rho", "Number density in full 6 dimensions (m^-6 s^3)", 0.0); - RP::add("VelocityBox.Vx1", "Box min x (m/s)", 0.0); - RP::add("VelocityBox.Vx2", "Box max x (m/s)", 0.0); - RP::add("VelocityBox.Vy1", "Box min y (m/s)", 0.0); - RP::add("VelocityBox.Vy2", "Box max y (m/s)", 0.0); - RP::add("VelocityBox.Vz1", "Box min z (m/s)", 0.0); - RP::add("VelocityBox.Vz2", "Box max z (m/s)", 0.0); - RP::add("VelocityBox.Bx", "Magnetic field x component (T)", 0.0); - RP::add("VelocityBox.By", "Magnetic field y component (T)", 0.0); - RP::add("VelocityBox.Bz", "Magnetic field z component (T)", 0.0); - } - - void VelocityBox::getParameters(){ - Project::getParameters(); - typedef Readparameters RP; - - if(getObjectWrapper().particleSpecies.size() > 1) { - std::cerr << "The selected project does not support multiple particle populations! Aborting in " << __FILE__ << " line " << __LINE__ << std::endl; - abort(); - } - RP::get("VelocityBox.rho", this->rho); - RP::get("VelocityBox.Vx1", this->Vx[0]); - RP::get("VelocityBox.Vx2", this->Vx[1]); - RP::get("VelocityBox.Vy1", this->Vy[0]); - RP::get("VelocityBox.Vy2", this->Vy[1]); - RP::get("VelocityBox.Vz1", this->Vz[0]); - RP::get("VelocityBox.Vz2", this->Vz[1]); - RP::get("VelocityBox.Bx", this->Bx); - RP::get("VelocityBox.By", this->By); - RP::get("VelocityBox.Bz", this->Bz); - } - - Real VelocityBox::getDistribValue(creal& vx, creal& vy, creal& vz, const uint popID) const { - if (vx >= this->Vx[0] && vx <= this->Vx[1] && - vy >= this->Vy[0] && vy <= this->Vy[1] && - vz >= this->Vz[0] && vz <= this->Vz[1]) - return this->rho; - else - return 0.0; - } - - - - Real VelocityBox::calcPhaseSpaceDensity( - creal& x, creal& y, creal& z, - creal& dx, creal& dy, creal& dz, - creal& vx, creal& vy, creal& vz, - creal& dvx, creal& dvy, creal& dvz,const uint popID - ) const { - return getDistribValue(vx+0.5*dvx, vy+0.5*dvy, vz+0.5*dvz, popID); - } - - - - void VelocityBox::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } - - void VelocityBox::setProjectBField( - FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, - FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, - FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid - ) { - ConstantField bgField; - bgField.initialize(this->Bx, - this->By, - this->Bz); - - setBackgroundField(bgField, BgBGrid); - } - -}// namespace projects diff --git a/projects/VelocityBox/VelocityBox.h b/projects/VelocityBox/VelocityBox.h deleted file mode 100644 index d04a21a1c..000000000 --- a/projects/VelocityBox/VelocityBox.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - - -#ifndef VELOCITYBOX_H -#define VELOCITYBOX_H - -#include "../../definitions.h" -#include "../project.h" - -namespace projects { - class VelocityBox: public Project { - public: - VelocityBox(); - virtual ~VelocityBox(); - - virtual bool initialize(void); - static void addParameters(void); - virtual void getParameters(void); - virtual void setProjectBField( - FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, - FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, - FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid - ); - protected: - Real getDistribValue(creal& vx, creal& vy, creal& vz, const uint popID) const; - virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); - virtual Real calcPhaseSpaceDensity( - creal& x, creal& y, creal& z, - creal& dx, creal& dy, creal& dz, - creal& vx, creal& vy, creal& vz, - creal& dvx, creal& dvy, creal& dvz, - const uint popID - ) const; - - Real rho; - Real Vx[2]; - Real Vy[2]; - Real Vz[2]; - Real Bx; - Real By; - Real Bz; - }; // class VelocityBox -} // namespace projects - - - -#endif - diff --git a/projects/project.cpp b/projects/project.cpp index a3f2cc1b8..a4b174fec 100644 --- a/projects/project.cpp +++ b/projects/project.cpp @@ -41,7 +41,6 @@ #include "Larmor/Larmor.h" #include "Magnetosphere/Magnetosphere.h" #include "MultiPeak/MultiPeak.h" -#include "VelocityBox/VelocityBox.h" #include "Riemann1/Riemann1.h" #include "Shock/Shock.h" #include "IPShock/IPShock.h" @@ -116,7 +115,6 @@ namespace projects { projects::Larmor::addParameters(); projects::Magnetosphere::addParameters(); projects::MultiPeak::addParameters(); - projects::VelocityBox::addParameters(); projects::Riemann1::addParameters(); projects::Shock::addParameters(); projects::IPShock::addParameters(); @@ -571,7 +569,7 @@ namespace projects { MPI_Comm_rank(MPI_COMM_WORLD,&myRank); int refines {0}; - if (!P::useAlpha && !P::useJPerB) { + if (!P::useAlpha1 && !P::useAlpha2) { if (myRank == MASTER_RANK) { std::cout << "WARNING All refinement indices disabled" << std::endl; } @@ -592,26 +590,69 @@ namespace projects { // Skip refining, touching boundaries during runtime breaks everything mpiGrid.dont_refine(id); mpiGrid.dont_unrefine(id); - } else if (r2 < r_max2) { - // We don't care about cells that are too far from the ionosphere - const Real alphaTwo {cell->parameters[CellParams::AMR_JPERB] * cell->parameters[CellParams::DX]}; - bool shouldRefine = (P::useAlpha ? cell->parameters[CellParams::AMR_ALPHA] > P::alphaRefineThreshold : false) || (P::useJPerB ? alphaTwo > P::jperbRefineThreshold : false); - bool shouldUnrefine = (P::useAlpha ? cell->parameters[CellParams::AMR_ALPHA] < P::alphaCoarsenThreshold : true) && (P::useJPerB ? alphaTwo < P::jperbCoarsenThreshold : true); + } else { + // Evaluate possible refinement or unrefinement for this cell + + // Cells too far from the ionosphere should be unrefined but + // induced refinement still possible just beyond this r_max2 limit. + bool shouldRefine {(r2 < r_max2) && ((P::useAlpha1 ? cell->parameters[CellParams::AMR_ALPHA1] > P::alpha1RefineThreshold : false) || (P::useAlpha2 ? cell->parameters[CellParams::AMR_ALPHA2] > P::alpha2RefineThreshold : false))}; + bool shouldUnrefine {(r2 > r_max2) || ((P::useAlpha1 ? cell->parameters[CellParams::AMR_ALPHA1] < P::alpha1CoarsenThreshold : true) && (P::useAlpha2 ? cell->parameters[CellParams::AMR_ALPHA2] < P::alpha2CoarsenThreshold : true))}; + + if(shouldRefine + // If this cell is planned to be refined, but is outside the allowed refinement region, cancel that refinement. + // Induced refinement still possible just beyond that limit. + && ((xyz[0] < P::refinementMinX) || (xyz[0] > P::refinementMaxX) + || (xyz[1] < P::refinementMinY) || (xyz[1] > P::refinementMaxY) + || (xyz[2] < P::refinementMinZ) || (xyz[2] > P::refinementMaxZ))) { + shouldRefine = false; + } + if(!shouldUnrefine + // If this cell is planned to remain at the current refinement level, but is outside the allowed refinement region, + // attempt to unrefine it instead. (If it is already at the lowest refinement level, DCCRG should not go belly-up.) + // Induced refinement still possible just beyond that limit. + && ((xyz[0] < P::refinementMinX) || (xyz[0] > P::refinementMaxX) + || (xyz[1] < P::refinementMinY) || (xyz[1] > P::refinementMaxY) + || (xyz[2] < P::refinementMinZ) || (xyz[2] > P::refinementMaxZ))) { + shouldUnrefine = true; + } // Finally, check neighbors int refined_neighbors {0}; int coarser_neighbors {0}; for (const auto& [neighbor, dir] : mpiGrid.get_face_neighbors_of(id)) { + // Evaluate all face neighbors of the current cell + std::array neighborXyz {mpiGrid.get_center(neighbor)}; + Real neighborR2 {pow(neighborXyz[0], 2) + pow(neighborXyz[1], 2) + pow(neighborXyz[2], 2)}; const int neighborRef = mpiGrid.get_refinement_level(neighbor); - const Real neighborAlphaTwo {mpiGrid[neighbor]->parameters[CellParams::AMR_JPERB] * mpiGrid[neighbor]->parameters[CellParams::DX]}; - if (neighborRef > refLevel) { + // Induced refinement still possible just beyond the r_max2 limit. + bool shouldRefineNeighbor {(neighborR2 < r_max2) && ((P::useAlpha1 ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA1] > P::alpha1RefineThreshold : false) || (P::useAlpha2 ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA2] > P::alpha2RefineThreshold : false))}; + if(shouldRefineNeighbor && + // If the neighbor is planned to be refined, but is outside the allowed refinement region, cancel that refinement. + // Induced refinement still possible just beyond that limit. + ((neighborXyz[0] < P::refinementMinX) || (neighborXyz[0] > P::refinementMaxX) + || (neighborXyz[1] < P::refinementMinY) || (neighborXyz[1] > P::refinementMaxY) + || (neighborXyz[2] < P::refinementMinZ) || (neighborXyz[2] > P::refinementMaxZ))) { + shouldRefineNeighbor = false; + } + // Induced refinement still possible just beyond the r_max2 limit. + bool shouldUnrefineNeighbor {(neighborR2 > r_max2) || ((P::useAlpha1 ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA1] < P::alpha1CoarsenThreshold : true) && (P::useAlpha2 ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA2] < P::alpha2CoarsenThreshold : true))}; + if(!shouldUnrefineNeighbor && + // If the neighbor is planned to remain at the current refinement level, but is outside the allowed refinement region, + // consider it as unrefining instead for purposes of evaluating the neighbors of this cell. + // Induced refinement still possible just beyond that limit. + ((neighborXyz[0] < P::refinementMinX) || (neighborXyz[0] > P::refinementMaxX) + || (neighborXyz[1] < P::refinementMinY) || (neighborXyz[1] > P::refinementMaxY) + || (neighborXyz[2] < P::refinementMinZ) || (neighborXyz[2] > P::refinementMaxZ))) { + shouldUnrefineNeighbor = true; + } + if (neighborRef > refLevel && !shouldUnrefineNeighbor) { ++refined_neighbors; - } else if (neighborRef < refLevel) { + } else if (neighborRef < refLevel && !shouldRefineNeighbor) { ++coarser_neighbors; - } else if ((P::useAlpha ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA] > P::alphaRefineThreshold : false) || (P::useJPerB ? neighborAlphaTwo > P::jperbRefineThreshold : false)) { + } else if (shouldRefineNeighbor) { // If neighbor refines, 4 of its children will be this cells refined neighbors refined_neighbors += 4; - } else if ((P::useAlpha ? mpiGrid[neighbor]->parameters[CellParams::AMR_ALPHA] < P::alphaCoarsenThreshold : true) && (P::useJPerB ? neighborAlphaTwo < P::jperbCoarsenThreshold : true)) { + } else if (shouldUnrefineNeighbor) { ++coarser_neighbors; } } @@ -660,21 +701,21 @@ namespace projects { CellID id = cellPair.first; // To preserve the mean, we must only consider refined cells int refLevel = mpiGrid.get_refinement_level(id); - std::vector refinedNeighbours; - for (auto& neighbour : *mpiGrid.get_neighbors_of(id, NEAREST_NEIGHBORHOOD_ID)) { - if (mpiGrid[neighbour.first]->parameters[CellParams::RECENTLY_REFINED] && mpiGrid.get_refinement_level(neighbour.first) == refLevel) { - refinedNeighbours.push_back(neighbour.first); + std::vector refinedNeighbors; + for (auto& neighbor : *mpiGrid.get_neighbors_of(id, NEAREST_NEIGHBORHOOD_ID)) { + if (mpiGrid[neighbor.first]->parameters[CellParams::RECENTLY_REFINED] && mpiGrid.get_refinement_level(neighbor.first) == refLevel) { + refinedNeighbors.push_back(neighbor.first); } } - if (refinedNeighbours.size() == 7) { - continue; // Simple heuristic, in these cases all neighbours are from the same parent cell, ergo are identical + if (refinedNeighbors.size() == 7) { + continue; // Simple heuristic, in these cases all neighbors are from the same parent cell, ergo are identical } - // In boxcar filter, we take the average of each of the neighbours and the cell itself. For each missing neighbour, add the cell one more time - Real fluffiness = (Real) refinedNeighbours.size() / 27.0; + // In boxcar filter, we take the average of each of the neighbors and the cell itself. For each missing neighbour, add the cell one more time + Real fluffiness = (Real) refinedNeighbors.size() / 27.0; for (uint popID=0; popID -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include #include #include "fsgrid.hpp" diff --git a/projects/projects_common.h b/projects/projects_common.h index e80bcf680..1945885f3 100644 --- a/projects/projects_common.h +++ b/projects/projects_common.h @@ -22,7 +22,7 @@ #ifndef PROJECTS_COMMON_H #define PROJECTS_COMMON_H -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include #include diff --git a/projects/testHall/testHall.cfg b/projects/testHall/testHall.cfg index a54117077..6000c6b95 100644 --- a/projects/testHall/testHall.cfg +++ b/projects/testHall/testHall.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = testHall [io] @@ -26,6 +27,10 @@ y_min = -7500 y_max = +7500 z_min = -7500 z_max = +7500 +timestep_max = 0 +dt = 0.002 + +[proton_vspace] vx_min = -1000000 vx_max = +1000000 vy_min = -1000000 @@ -35,8 +40,6 @@ vz_max = +1000000 vx_length = 10 vy_length = 10 vz_length = 10 -timestep_max = 0 -dt = 0.002 [boundaries] periodic_x = yes diff --git a/projects/test_fp/test_fp.cfg b/projects/test_fp/test_fp.cfg index eabac537b..e82cd253e 100644 --- a/projects/test_fp/test_fp.cfg +++ b/projects/test_fp/test_fp.cfg @@ -2,6 +2,7 @@ # dt = 0.05 and timestep_max = 160 gives two full periods, 1st is at # 80 steps and 2nd at 160 +ParticlePopulations = proton propagate_field = 1 propagate_vlasov_acceleration = 0 propagate_vlasov_translation = 0 @@ -34,15 +35,6 @@ y_min = -1.0e4 y_max = +1.0e4 z_min = -1.0e4 z_max = +1.0e4 -vx_min = -2.0e4 -vx_max = +2.0e4 -vy_min = -2.0e4 -vy_max = +2.0e4 -vz_min = -2.0e4 -vz_max = +2.0e4 -vx_length = 9 -vy_length = 9 -vz_length = 9 dt = 0.05 t_max = 4.0 #timestep_max = 160 @@ -53,6 +45,17 @@ t_max = 4.0 #dt = 0.00625 #timestep_max = 1280 +[proton_vspace] +vx_min = -2.0e4 +vx_max = +2.0e4 +vy_min = -2.0e4 +vy_max = +2.0e4 +vz_min = -2.0e4 +vz_max = +2.0e4 +vx_length = 9 +vy_length = 9 +vz_length = 9 + [boundaries] periodic_x = yes periodic_y = yes @@ -68,7 +71,7 @@ output = populations_vg_v output = populations_vg_blocks diagnostic = populations_vg_blocks -[sparse] +[proton_sparse] minValue = 1e-15 [test_fp] diff --git a/projects/test_trans/test_trans.cfg b/projects/test_trans/test_trans.cfg index 61c4d0b7b..bc9b9b54e 100644 --- a/projects/test_trans/test_trans.cfg +++ b/projects/test_trans/test_trans.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton project = test_trans propagate_field = 0 propagate_vlasov_acceleration = 0 @@ -31,8 +32,7 @@ timestep_max = 20 #dt = 0.171428572 dt = 0.09 -[velocitymesh] -name = IonMesh +[proton_vspace] vx_min = -1.0 vx_max = +1.0 vy_min = -1.0 @@ -48,21 +48,12 @@ periodic_x = yes periodic_y = yes periodic_z = yes -[sparse] +[proton_sparse] minValue = 1e-7 -[ParticlePopulation] -name = avgs -mass_units = PROTON -mass = 1.0 -charge = 1 -sparse_min_value = 1e-7 -mesh = IonMesh - [variables] output = populations_vg_rho output = populations_vg_blocks -diagnostic = populations_vg_rho diagnostic = populations_vg_blocks [test_trans] diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cfg b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cfg deleted file mode 100644 index 58b9b6579..000000000 --- a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cfg +++ /dev/null @@ -1,69 +0,0 @@ -system_write_t_interval = 15 -diagnostic_write_interval = 1 -restart_write_t_interval = 500 -propagate_field = 1 -propagate_vlasov = 1 -project = KelvinHelmholtz - - -[gridbuilder] -x_length = 20 -y_length = 1 -z_length = 10 -x_min = 0.0 -x_max = 1.0e7 -y_min = 0.0 -y_max = 5.0e5 -z_min = -2.5e6 -z_max = +2.5e6 -vx_min = -600000.0 -vx_max = +600000.0 -vy_min = -600000.0 -vy_max = +600000.0 -vz_min = -600000.0 -vz_max = +600000.0 -vx_length = 10 -vy_length = 10 -vz_length = 10 -timestep_max = 10 -dt = 0.01 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = B -output = Pressure -output = RhoV -diagnostic = Blocks - -[sparse] -minValue = 1.0e-15 -minAvgValue = 5.0e-16 - -[KelvinHelmholtz] -Vx1 = 400000.0 -Vy1 = 0.0 -Vz1 = 0.0 -Bx1 = 2.0e-09 -By1 = 2.0e-10 -Bz1 = 0.0 -T1 = 100000.0 -rho1 = 1000000.0 - -Vx2 = 0.0 -Vy2 = 0.0 -Vz2 = 0.0 -Bx2 = 2.0e-08 -By2 = 2.0e-09 -Bz2 = 0.0 -T2 = 100000.0 -rho2 = 5000000.0 - -lambda = 3.3333333e6 -amp = 5.0e5 -offset = 1.0e6 -transitionWidth = 1.0e6 diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cpp b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cpp deleted file mode 100644 index b756f7e24..000000000 --- a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "../../common.h" -#include "../../readparameters.h" -#include "../../backgroundfield/backgroundfield.h" - -#include "KelvinHelmholtz.h" - -namespace projects { - using namespace std; - KelvinHelmholtz::KelvinHelmholtz(): Project() { } - KelvinHelmholtz::~KelvinHelmholtz() { } - - bool KelvinHelmholtz::initialize(void) {return true;} - - void KelvinHelmholtz::addParameters() { - typedef Readparameters RP; - RP::add("KelvinHelmholtz.rho1", "Number density, this->TOP state (m^-3)", 0.0); - RP::add("KelvinHelmholtz.rho2", "Number density, this->BOTTOM state (m^-3)", 0.0); - RP::add("KelvinHelmholtz.T1", "Temperature, this->TOP state (K)", 0.0); - RP::add("KelvinHelmholtz.T2", "Temperature, this->BOTTOM state (K)", 0.0); - RP::add("KelvinHelmholtz.Vx1", "Bulk velocity x component, this->TOP state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Vx2", "Bulk velocity x component, this->BOTTOM state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Vy1", "Bulk velocity y component, this->TOP state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Vy2", "Bulk velocity y component, this->BOTTOM state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Vz1", "Bulk velocity z component, this->TOP state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Vz2", "Bulk velocity z component, this->BOTTOM state (m/s)", 0.0); - RP::add("KelvinHelmholtz.Bx1", "Magnetic field x component, this->TOP state (T)", 0.0); - RP::add("KelvinHelmholtz.Bx2", "Magnetic field x component, this->BOTTOM state (T)", 0.0); - RP::add("KelvinHelmholtz.By1", "Magnetic field y component, this->TOP state (T)", 0.0); - RP::add("KelvinHelmholtz.By2", "Magnetic field y component, this->BOTTOM state (T)", 0.0); - RP::add("KelvinHelmholtz.Bz1", "Magnetic field z component, this->TOP state (T)", 0.0); - RP::add("KelvinHelmholtz.Bz2", "Magnetic field z component, this->BOTTOM state (T)", 0.0); - RP::add("KelvinHelmholtz.lambda", "Initial perturbation wavelength (m)", 0.0); - RP::add("KelvinHelmholtz.amp", "Initial perturbation amplitude (m)", 0.0); - RP::add("KelvinHelmholtz.offset", "Boundaries offset from 0 (m)", 0.0); - RP::add("KelvinHelmholtz.transitionWidth", "Width of tanh transition for all changing values", 0.0); - } - - void KelvinHelmholtz::getParameters() { - typedef Readparameters RP; - RP::get("KelvinHelmholtz.rho1", this->rho[this->TOP]); - RP::get("KelvinHelmholtz.rho2", this->rho[this->BOTTOM]); - RP::get("KelvinHelmholtz.T1", this->T[this->TOP]); - RP::get("KelvinHelmholtz.T2", this->T[this->BOTTOM]); - RP::get("KelvinHelmholtz.Vx1", this->Vx[this->TOP]); - RP::get("KelvinHelmholtz.Vx2", this->Vx[this->BOTTOM]); - RP::get("KelvinHelmholtz.Vy1", this->Vy[this->TOP]); - RP::get("KelvinHelmholtz.Vy2", this->Vy[this->BOTTOM]); - RP::get("KelvinHelmholtz.Vz1", this->Vz[this->TOP]); - RP::get("KelvinHelmholtz.Vz2", this->Vz[this->BOTTOM]); - RP::get("KelvinHelmholtz.Bx1", this->Bx[this->TOP]); - RP::get("KelvinHelmholtz.Bx2", this->Bx[this->BOTTOM]); - RP::get("KelvinHelmholtz.By1", this->By[this->TOP]); - RP::get("KelvinHelmholtz.By2", this->By[this->BOTTOM]); - RP::get("KelvinHelmholtz.Bz1", this->Bz[this->TOP]); - RP::get("KelvinHelmholtz.Bz2", this->Bz[this->BOTTOM]); - RP::get("KelvinHelmholtz.lambda", this->lambda); - RP::get("KelvinHelmholtz.amp", this->amp); - RP::get("KelvinHelmholtz.offset", this->offset); - RP::get("KelvinHelmholtz.transitionWidth", this->transitionWidth); - } - - - Real KelvinHelmholtz::profile(creal top, creal bottom, creal x, creal z) { - if(top == bottom) { - return top; - } - if(this->offset != 0.0) { - return 0.5 * ((top-bottom) * ( - tanh((z + this->offset + this->amp * cos(2.0*M_PI*x/this->lambda))/this->transitionWidth) - - tanh((z-(this->offset + this->amp * cos(2.0*M_PI*x/this->lambda)))/this->transitionWidth) -1) + top+bottom); - } else { - return 0.5 * ((top-bottom) * tanh(z/this->transitionWidth) + top+bottom); - } - } - - Real KelvinHelmholtz::getDistribValue(creal& x, creal& z, creal& vx, creal& vy, creal& vz){ - creal mass = physicalconstants::MASS_PROTON; - creal kb = physicalconstants::K_B; - Real rho = profile(this->rho[this->BOTTOM], this->rho[this->TOP], x, z); - Real T = profile(this->T[this->BOTTOM], this->T[this->TOP], x, z); - Real Vx = profile(this->Vx[this->BOTTOM], this->Vx[this->TOP], x, z); - Real Vy = profile(this->Vy[this->BOTTOM], this->Vy[this->TOP], x, z); - Real Vz = profile(this->Vz[this->BOTTOM], this->Vz[this->TOP], x, z); - - return rho * pow(mass / (2.0 * M_PI * kb * T), 1.5) * - exp(- mass * (pow(vx - Vx, 2.0) + pow(vy - Vy, 2.0) + pow(vz - Vz, 2.0)) / (2.0 * kb * T)); - } - - Real KelvinHelmholtz::calcPhaseSpaceDensity(creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz) { - return getDistribValue(x+0.5*dx, z+0.5*dz, vx+0.5*dvx, vy+0.5*dvy, vz+0.5*dvz); - } - - void KelvinHelmholtz::calcCellParameters(Real* cellParams,creal& t) { - cellParams[CellParams::EX ] = 0.0; - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - } - - void KelvinHelmholtz::setCellBackgroundField(SpatialCell* cell) { - creal x = cell->parameters[CellParams::XCRD]; - creal dx = cell->parameters[CellParams::DX]; - creal z = cell->parameters[CellParams::ZCRD]; - creal dz = cell->parameters[CellParams::DZ]; - - cell->parameters[CellParams::BGBX ] = profile(this->Bx[this->BOTTOM], this->Bx[this->TOP], x+0.5*dx, z+0.5*dz); - cell->parameters[CellParams::BGBY ] = profile(this->By[this->BOTTOM], this->By[this->TOP], x+0.5*dx, z+0.5*dz); - cell->parameters[CellParams::BGBZ ] = profile(this->Bz[this->BOTTOM], this->Bz[this->TOP], x+0.5*dx, z+0.5*dz); - } -} // namespace projects diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.h b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.h deleted file mode 100644 index 3dc1bb1d8..000000000 --- a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef KELVINHELMHOLTZ_H -#define KELVINHELMHOLTZ_H - -#include - -#include "../../definitions.h" -#include "../project.h" - -namespace projects { - class KelvinHelmholtz: public Project { - public: - KelvinHelmholtz(); - virtual ~KelvinHelmholtz(); - - virtual bool initialize(void); - static void addParameters(void); - virtual void getParameters(void); - virtual void calcCellParameters(Real* cellParams,creal& t); - virtual void setCellBackgroundField(SpatialCell* cell); - virtual Real calcPhaseSpaceDensity( - creal& x, creal& y, creal& z, - creal& dx, creal& dy, creal& dz, - creal& vx, creal& vy, creal& vz, - creal& dvx, creal& dvy, creal& dvz - ); - protected: - Real getDistribValue( - creal& x, creal& z, - creal& vx, creal& vy, creal& vz); - Real profile(creal top, creal bottom, creal x, creal z); - - enum { - TOP, - BOTTOM - }; - Real rho[2]; - Real T[2]; - Real Vx[2]; - Real Vy[2]; - Real Vz[2]; - Real Bx[2]; - Real By[2]; - Real Bz[2]; - Real lambda; - Real amp; - Real offset; - Real transitionWidth; - }; // class KelvinHelmholtz -} // namespace -#endif diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.ods b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.ods deleted file mode 100644 index 306baa35d..000000000 Binary files a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz.ods and /dev/null differ diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.pdf b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.pdf deleted file mode 100644 index 238412c3a..000000000 Binary files a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.pdf and /dev/null differ diff --git a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.tex b/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.tex deleted file mode 100644 index dea7b69c7..000000000 --- a/projects/unsupported/KelvinHelmholtz/KelvinHelmholtz_technical_info.tex +++ /dev/null @@ -1,51 +0,0 @@ -\documentclass[a4paper,10pt]{scrartcl} -\usepackage[utf8x]{inputenc} -\usepackage[left=1.5cm, right=1.5cm, top=1.5cm, bottom=2.5cm]{geometry} -\usepackage[autolanguage,np]{numprint} -\usepackage{tabularx} - -\usepackage{hyperref} -\hyperbaseurl{.} - -%opening -\title{ -\Huge{Vlasiator test cases technical information} \\ -\LARGE{KelvinHelmholtz} -} -\author{Yann Kempf} -\date{Updated on \today} - -\begin{document} - -\maketitle - -\begin{abstract} - This document gives technical information on the KelvinHelmholtz test case. -\end{abstract} - -\section{Purpose} -Try to produce the Kelvin-Helmholtz instability in Vlasiator. - - -\section{Implementation} -Code originally copied from Riemann1, there is here an \verb=enum= for the \verb=TOP=/\verb=BOTTOM= states. One region centred on $z=0$ (\verb=TOP=) gets a distinct velocity and density state (\textit{e.\ g}.\ high velocity, low density), separated by a boundary which can be straight or have sinusoidal perturbations. The offset of the boundary from the $x$-axis is user-set. - -\section{Options} -The options available in the \verb=cfg= file are: - -\begin{tabularx}{\textwidth}{lX} - \verb=rho[12]= & Number density (m\textsuperscript{-3}) \\ - \verb=T[12]= & Temperature (K) \\ - \verb=V[xyz][12]= & Velocity (m/s) \\ - \verb=B[xyz][12]= & Magnetic field (T) \\ - \verb=lambda= & Boundary perturbation wavelength (m) \\ - \verb=amp= & Boundary perturbation amplitude (m) \\ - \verb=offset= & Boundary offset from the $x$-axis (m) \\ - \verb=nSpaceSamples= & Number of sampling points along spatial dimensions within a spatial cell, includes the corners (minimum 2) \\ - \verb=nVelocitySamples= & Number of sampling points along velocity dimensions within a velocity cell, includes the corners (minimum 2) -\end{tabularx} - - - - -\end{document} \ No newline at end of file diff --git a/projects/unsupported/harm1D/harm1D.cfg b/projects/unsupported/harm1D/harm1D.cfg deleted file mode 100644 index 0f17f15e2..000000000 --- a/projects/unsupported/harm1D/harm1D.cfg +++ /dev/null @@ -1,41 +0,0 @@ -system_write_t_interval = 0.025 -diagnostic_write_interval = 1 -restart_write_t_interval = 0.025 -propagate_field = 0 -propagate_vlasov = 1 -project = harm1D - -[gridbuilder] -x_length = 10 -y_length = 1 -z_length = 1 -x_min = -1.2 -x_max = +1.2 -y_min = -1.2 -y_max = +1.2 -z_min = -1.2 -z_max = +1.2 -vx_min = -1.0 -vx_max = +1.0 -vy_min = -1.0 -vy_max = +1.0 -vz_min = -1.0 -vz_max = +1.0 -vx_length = 20 -vy_length = 20 -vz_length = 20 -q = 1.0 -m = 1.0 -timestep_max = 50 -dt = 0.025 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks diff --git a/projects/unsupported/harm1D/harm1D.cpp b/projects/unsupported/harm1D/harm1D.cpp deleted file mode 100644 index b10f7c3ca..000000000 --- a/projects/unsupported/harm1D/harm1D.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "../../common.h" -#include "../../readparameters.h" -#include "../../backgroundfield/backgroundfield.h" - -#include "harm1D.h" - -namespace projects { - harm1D::harm1D(): Project() { } - harm1D::~harm1D() { } - - using namespace std; - - bool harm1D::initialize(void) {return true;} - void harm1D::addParameters() { } - void harm1D::getParameters() { } - - Real harm1D::calcPhaseSpaceDensity( - creal& x,creal& y,creal& z, - creal& dx,creal& dy,creal& dz, - creal& vx,creal& vy,creal& vz, - creal& dvx,creal& dvy,creal& dvz - ) { - /* - creal VX0 = 0.5; - creal VY0 = 0.0; - creal VZ0 = 0.0; - creal SIGMA = 0.4714; - creal INVSIG2 = 1.0/(SIGMA*SIGMA); - return 1.5*exp(-INVSIG2*(vx-VX0)*(vx-VX0))*exp(-INVSIG2*(vy-VY0)*(vy-VY0))*exp(-INVSIG2*(vz-VZ0)*(vz-VZ0)); - */ - creal X0 = 1.0/14.0; - creal Y0 = 1.0/14.0; - - creal VX0 = -0.4; - creal VY0 = -0.4; - creal VZ0 = 0.0; - creal DVX = 0.1; - creal DVY = 0.1; - creal DVZ = 0.1; - creal VSIGMA = 0.2; - creal INVVSIG2 = 1.0/(VSIGMA*VSIGMA); - - if (fabs(x + 0.6) > dx) return 1e-10; - if (fabs(vx) > 0.051) return 1e-10; - if (fabs(vy) > 0.8) return 1e-10; - if (fabs(vz) > 0.8) return 1e-10; - //if (fabs(x) > X0 || fabs(y) > Y0) return 0.0; - //if (fabs(vy-VY0) > DVY) return 0.0; - //if (fabs(vz-VZ0) > DVZ) return 0.0; - //return 5.0*exp(-INVVSIG2*(vx-VX0)*(vx-VX0))*exp(-INVVSIG2*(vy-VY0)*(vy-VY0))*exp(-INVVSIG2*(vz-VZ0)*(vz-VZ0)); - return 1.0; - } - - void harm1D::calcCellParameters(Real* cellParams,creal& t) { - creal x = cellParams[CellParams::XCRD]; - creal dx = cellParams[CellParams::DX]; - - // Setting these is not needed for correct propagation, - // but may be a good idea for visualization: - cellParams[CellParams::EX ] = -1.0*(x+0.5*dx); - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - cellParams[CellParams::BGBX ] = 0.0; - cellParams[CellParams::BGBY ] = 0.0; - cellParams[CellParams::BGBZ ] = 0.0; - } -} // namespace projects diff --git a/projects/unsupported/testAmr/testAmr.cfg b/projects/unsupported/testAmr/testAmr.cfg deleted file mode 100644 index 5cf3ce52e..000000000 --- a/projects/unsupported/testAmr/testAmr.cfg +++ /dev/null @@ -1,104 +0,0 @@ -dynamic_timestep = 1 -project = testAmr -ParticlePopulations = proton -propagate_field = 0 -propagate_vlasov_acceleration = 0 -propagate_vlasov_translation = 1 - -[proton_properties] -mass = 1 -mass_units = PROTON -charge = 1 - -[io] -diagnostic_write_interval = 1 -write_initial_state = 1 - -system_write_t_interval = 0.01 -system_write_file_name = fullf -system_write_distribution_stride = 1 -system_write_distribution_xline_stride = 0 -system_write_distribution_yline_stride = 0 -system_write_distribution_zline_stride = 0 - -[AMR] -max_spatial_level = 2 -number_of_boxes = 1 -box_max_level = 1 -box_half_width_x = 1 -box_half_width_y = 1 -box_half_width_z = 1 -box_center_x = 1.0e6 -box_center_y = 1.0e6 -box_center_z = 1.0e6 - -[gridbuilder] -x_length = 8 -y_length = 8 -z_length = 8 -x_min = -1.0e6 -x_max = 1.0e6 -y_min = -1.0e6 -y_max = 1.0e6 -z_min = -1.0e6 -z_max = 1.0e6 -timestep_max = 1 - -[proton_vspace] -vx_min = -2.0e6 -vx_max = +2.0e6 -vy_min = -2.0e6 -vy_max = +2.0e6 -vz_min = -2.0e6 -vz_max = +2.0e6 -vx_length = 2 -vy_length = 2 -vz_length = 2 -max_refinement_level = 1 -[proton_sparse] -minValue = 1.0e-16 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = populations_vg_rho -output = fg_b -output = vg_pressure -output = populations_vg_v -output = fg_e -output = vg_rank -output = populations_vg_blocks -#output = populations_vg_acceleration_subcycles - -diagnostic = populations_vg_blocks -#diagnostic = vg_pressure -#diagnostic = populations_vg_rho -#diagnostic = populations_vg_rho_loss_adjust - -[testAmr] -#magnitude of 1.82206867e-10 gives a period of 360s, useful for testing... -Bx = 1.2e-10 -By = 0.8e-10 -Bz = 1.1135233442526334e-10 -magXPertAbsAmp = 0 -magYPertAbsAmp = 0 -magZPertAbsAmp = 0 -densityModel = uniform - -[proton_testAmr] -n = 1 -Vx = 5e5 -Vy = 5e5 -Vz = 0.0 -Tx = 500000.0 -Ty = 500000.0 -Tz = 500000.0 -rho = 1.0e6 -rhoPertAbsAmp = 0.0 - -[loadBalance] -#algorithm = RCB -algorithm = RANDOM diff --git a/projects/unsupported/testAmr/testAmr.cpp b/projects/unsupported/testAmr/testAmr.cpp deleted file mode 100644 index e56d5884f..000000000 --- a/projects/unsupported/testAmr/testAmr.cpp +++ /dev/null @@ -1,276 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include - -#include "../../common.h" -#include "../../readparameters.h" -#include "../../backgroundfield/backgroundfield.h" -#include "../../backgroundfield/constantfield.hpp" -#include "../../object_wrapper.h" - -#include "testAmr.h" - -using namespace std; -using namespace spatial_cell; - -Real projects::testAmr::rhoRnd; - -namespace projects { - testAmr::testAmr(): TriAxisSearch() { } - - testAmr::~testAmr() { } - - bool testAmr::initialize(void) { - return Project::initialize(); - } - - void testAmr::addParameters(){ - typedef Readparameters RP; - - RP::add("testAmr.Bx", "Magnetic field x component (T)", 0.0); - RP::add("testAmr.By", "Magnetic field y component (T)", 0.0); - RP::add("testAmr.Bz", "Magnetic field z component (T)", 0.0); - RP::add("testAmr.dBx", "Magnetic field x component cosine perturbation amplitude (T)", 0.0); - RP::add("testAmr.dBy", "Magnetic field y component cosine perturbation amplitude (T)", 0.0); - RP::add("testAmr.dBz", "Magnetic field z component cosine perturbation amplitude (T)", 0.0); - RP::add("testAmr.magXPertAbsAmp", "Absolute amplitude of the random magnetic perturbation along x (T)", 1.0e-9); - RP::add("testAmr.magYPertAbsAmp", "Absolute amplitude of the random magnetic perturbation along y (T)", 1.0e-9); - RP::add("testAmr.magZPertAbsAmp", "Absolute amplitude of the random magnetic perturbation along z (T)", 1.0e-9); - RP::add("testAmr.lambda", "B cosine perturbation wavelength (m)", 1.0); - RP::add("testAmr.densityModel","Which spatial density model is used?",string("uniform")); - - // Per-population parameters - for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { - const std::string& pop = getObjectWrapper().particleSpecies[i].name; - RP::add(pop+"_testAmr.n", "Number of peaks to create", 0); - RP::addComposing(pop+"_testAmr.rho", "Number density (m^-3)"); - RP::addComposing(pop+"_testAmr.Tx", "Temperature (K)"); - RP::addComposing(pop+"_testAmr.Ty", "Temperature"); - RP::addComposing(pop+"_testAmr.Tz", "Temperature"); - RP::addComposing(pop+"_testAmr.Vx", "Bulk velocity x component (m/s)"); - RP::addComposing(pop+"_testAmr.Vy", "Bulk velocity y component (m/s)"); - RP::addComposing(pop+"_testAmr.Vz", "Bulk velocity z component (m/s)"); - RP::addComposing(pop+"_testAmr.rhoPertAbsAmp", "Absolute amplitude of the density perturbation"); - } - } - - void testAmr::getParameters(){ - - typedef Readparameters RP; - Project::getParameters(); - RP::get("testAmr.Bx", this->Bx); - RP::get("testAmr.By", this->By); - RP::get("testAmr.Bz", this->Bz); - RP::get("testAmr.magXPertAbsAmp", this->magXPertAbsAmp); - RP::get("testAmr.magYPertAbsAmp", this->magYPertAbsAmp); - RP::get("testAmr.magZPertAbsAmp", this->magZPertAbsAmp); - RP::get("testAmr.dBx", this->dBx); - RP::get("testAmr.dBy", this->dBy); - RP::get("testAmr.dBz", this->dBz); - RP::get("testAmr.lambda", this->lambda); - RP::get("testAmr.maxSpatialRefinementLevel", this->maxSpatialRefinementLevel); - - // Per-population parameters - for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { - const std::string& pop = getObjectWrapper().particleSpecies[i].name; - - testAmrSpeciesParameters sP; - RP::get(pop + "_testAmr.n", sP.numberOfPeaks); - RP::get(pop + "_testAmr.rho",sP.rho); - RP::get(pop + "_testAmr.Tx", sP.Tx); - RP::get(pop + "_testAmr.Ty", sP.Ty); - RP::get(pop + "_testAmr.Tz", sP.Tz); - RP::get(pop + "_testAmr.Vx", sP.Vx); - RP::get(pop + "_testAmr.Vy", sP.Vy); - RP::get(pop + "_testAmr.Vz", sP.Vz); - - RP::get(pop + "_testAmr.rhoPertAbsAmp", sP.rhoPertAbsAmp); - - if(!sP.isConsistent()) { - cerr << "You should define all parameters (testAmr.rho, testAmr.Tx, testAmr.Ty, testAmr.Tz, testAmr.Vx, testAmr.Vy, testAmr.Vz, testAmr.rhoPertAbsAmp) for all " << sP.numberOfPeaks << " peaks of population " << pop << "." << endl; - abort(); - } - - speciesParams.push_back(sP); - } - - string densModelString; - RP::get("testAmr.densityModel",densModelString); - - if (densModelString == "uniform") densityModel = Uniform; - else if (densModelString == "testcase") densityModel = TestCase; - } - - Real testAmr::getDistribValue(creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz,const uint popID) const { - const testAmrSpeciesParameters& sP = speciesParams[popID]; - creal mass = getObjectWrapper().particleSpecies[popID].mass; - creal kb = physicalconstants::K_B; - - Real value = 0.0; - - for (uint i=0; i::min(),avg * static_cast(1e-6)); - Real avgAccum = avgTotal / (avg + N3_sum); - Real avgCurrent = avg / (N*N*N); - if (fabs(avgCurrent-avgAccum)/(avgAccum+eps) < 0.01) ok = true; - else if (avg < avgLimit) ok = true; - else if (N > 10) { - ok = true; - } - - avgTotal += avg; - N3_sum += N*N*N; - ++N; - } while (ok == false); - - return avgTotal / N3_sum; - } - - void testAmr::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { - std::default_random_engine rndState; - setRandomCellSeed(cell,rndState); - rhoRnd = 0.5 - getRandomNumber(rndState); - } - - void testAmr::setProjectBField( - FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, - FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, - FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid - ) { - ConstantField bgField; - bgField.initialize(this->Bx, - this->By, - this->Bz); - - setBackgroundField(bgField, BgBGrid); - - if(!P::isRestart) { - auto localSize = perBGrid.getLocalSize().data(); - - #pragma omp parallel for collapse(3) - for (FsGridTools::FsIndex_t x = 0; x < localSize[0]; ++x) { - for (FsGridTools::FsIndex_t y = 0; y < localSize[1]; ++y) { - for (FsGridTools::FsIndex_t z = 0; z < localSize[2]; ++z) { - const std::array xyz = perBGrid.getPhysicalCoords(x, y, z); - std::array* cell = perBGrid.get(x, y, z); - - const int64_t cellid = perBGrid.GlobalIDForCoords(x, y, z); - - std::default_random_engine rndState; - setRandomSeed(cellid,rndState); - - if (this->lambda != 0.0) { - cell->at(fsgrids::bfield::PERBX) = this->dBx*cos(2.0 * M_PI * xyz[0] / this->lambda); - cell->at(fsgrids::bfield::PERBY) = this->dBy*sin(2.0 * M_PI * xyz[0] / this->lambda); - cell->at(fsgrids::bfield::PERBZ) = this->dBz*cos(2.0 * M_PI * xyz[0] / this->lambda); - } - - cell->at(fsgrids::bfield::PERBX) += this->magXPertAbsAmp * (0.5 - getRandomNumber(rndState)); - cell->at(fsgrids::bfield::PERBY) += this->magYPertAbsAmp * (0.5 - getRandomNumber(rndState)); - cell->at(fsgrids::bfield::PERBZ) += this->magZPertAbsAmp * (0.5 - getRandomNumber(rndState)); - } - } - } - } - } - - std::vector > testAmr::getV0( - creal x, - creal y, - creal z, - const uint popID - ) const { - const testAmrSpeciesParameters& sP = speciesParams[popID]; - vector > centerPoints; - for(uint i=0; i point {{sP.Vx[i], sP.Vy[i], sP.Vz[i]}}; - centerPoints.push_back(point); - } - return centerPoints; - } - -}// namespace projects diff --git a/projects/unsupported/testAmr/testAmr.h b/projects/unsupported/testAmr/testAmr.h deleted file mode 100644 index 2dbc325fe..000000000 --- a/projects/unsupported/testAmr/testAmr.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef TESTAMR_H -#define TESTAMR_H - -#include - -#include "../../definitions.h" -#include "../projectTriAxisSearch.h" - -namespace projects { - struct testAmrSpeciesParameters { - uint numberOfPeaks; - std::vector rho; - std::vector Tx; - std::vector Ty; - std::vector Tz; - std::vector Vx; - std::vector Vy; - std::vector Vz; - std::vector rhoPertAbsAmp; - - // Test whether parameters have been set up for all peaks - bool isConsistent() { - return rho.size() == Tx.size() && - Tx.size() == Ty.size() && - Ty.size() == Tz.size() && - Tz.size() == Vx.size() && - Vx.size() == Vy.size() && - Vy.size() == Vz.size() && - Vz.size() == rhoPertAbsAmp.size() && - rhoPertAbsAmp.size() == rho.size() && - rho.size() == numberOfPeaks; - } - }; - class testAmr: public TriAxisSearch { - public: - testAmr(); - virtual ~testAmr(); - - virtual bool initialize(void); - static void addParameters(void); - virtual void getParameters(void); - virtual void setProjectBField( - FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, - FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, - FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid - ); - protected: - Real getDistribValue( - creal& x,creal& y, creal& z, - creal& vx, creal& vy, creal& vz, - const uint popID) const; - virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); - virtual Real calcPhaseSpaceDensity( - creal& x, creal& y, creal& z, - creal& dx, creal& dy, creal& dz, - creal& vx, creal& vy, creal& vz, - creal& dvx, creal& dvy, creal& dvz, - const uint popID) const; - virtual std::vector > getV0( - creal x, - creal y, - creal z, - const uint popID - ) const; - - static Real rhoRnd; //static as it has to be threadprivate - #pragma omp threadprivate(rhoRnd) - Real Bx; - Real By; - Real Bz; - Real dBx; - Real dBy; - Real dBz; - Real magXPertAbsAmp; - Real magYPertAbsAmp; - Real magZPertAbsAmp; - Real lambda; - int maxSpatialRefinementLevel; - std::vector speciesParams; - - enum densitymodel { - Uniform, - TestCase - } densityModel; - - }; // class testAmr -} // namespace projects - -#endif - diff --git a/projects/unsupported/test_acc/test_acc.cfg b/projects/unsupported/test_acc/test_acc.cfg deleted file mode 100644 index 7808208c6..000000000 --- a/projects/unsupported/test_acc/test_acc.cfg +++ /dev/null @@ -1,46 +0,0 @@ -system_write_t_interval = 0.1 -diagnostic_write_interval = 1 -restart_write_t_interval = 0.1 -propagate_field = 0 -propagate_vlasov = 1 - -[gridbuilder] -x_length = 1 -y_length = 1 -z_length = 1 -x_min = -1.0 -x_max = +1.0 -y_min = -1.0 -y_max = +1.0 -z_min = -1.0 -z_max = +1.0 -vx_min = -1.0 -vx_max = +1.0 -vy_min = -1.0 -vy_max = +1.0 -vz_min = -1.0 -vz_max = +1.0 -vx_length = 5 -vy_length = 5 -vz_length = 5 -q = 1.0 -m = 1.0 -timestep_max = 6 -#dt = 0.171428572 -dt = 0.1 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[test_acc] -SPEED = 0.5 -v_min = 0.2 -v_max = 0.3 - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks diff --git a/projects/unsupported/test_acc/test_acc.cpp b/projects/unsupported/test_acc/test_acc.cpp deleted file mode 100644 index e03b782c7..000000000 --- a/projects/unsupported/test_acc/test_acc.cpp +++ /dev/null @@ -1,185 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "spatial_cell.hpp" -#include "common.h" -#include "project.h" -#include "parameters.h" -#include "readparameters.h" -#include "vlasovmover.h" - -using namespace std; - -typedef test_accParameters taP; -Real taP::SPEED = NAN; -Real taP::v_min = NAN; -Real taP::v_max = NAN; - -bool initializeProject(void) {return true;} - -bool addProjectParameters(){ - typedef Readparameters RP; - RP::add("test_acc.SPEED", "Acceleration value", 0.5); - RP::add("test_acc.v_min", "Inner corner of initial blobs", 0.2); - RP::add("test_acc.v_max", "Outer corner of initial blobs", 0.3); - return true; -} -bool getProjectParameters(){ - typedef Readparameters RP; - RP::get("test_acc.SPEED", taP::SPEED); - RP::get("test_acc.v_min", taP::v_min); - RP::get("test_acc.v_max", taP::v_max); - return true; -} - -/** Integrate the distribution function over the given six-dimensional phase-space cell. - * @param x Starting value of the x-coordinate of the cell. - * @param y Starting value of the y-coordinate of the cell. - * @param z Starting value of the z-coordinate of the cell. - * @param dx The size of the cell in x-direction. - * @param dy The size of the cell in y-direction. - * @param dz The size of the cell in z-direction. - * @param vx Starting value of the vx-coordinate of the cell. - * @param vy Starting value of the vy-coordinate of the cell. - * @param vz Starting value of the vz-coordinate of the cell. - * @param dvx The size of the cell in vx-direction. - * @param dvy The size of the cell in vy-direction. - * @param dvz The size of the cell in vz-direction. - * @return The volume average of the distribution function in the given phase space cell. - * The physical unit of this quantity is 1 / (m^3 (m/s)^3). - */ -Real calcPhaseSpaceDensity(creal& x,creal& y,creal& z,creal& dx,creal& dy,creal& dz, - creal& vx,creal& vy,creal& vz,creal& dvx,creal& dvy,creal& dvz) { - Real Vx = vx + 0.5 * dvx; - Real Vy = vy + 0.5 * dvy; - Real Vz = vz + 0.5 * dvz; - if (Vz >= taP::v_min && Vz < taP::v_max) { - if (Vx >= taP::v_min && Vx < taP::v_max) { - if (Vy >= -taP::v_max && Vy < -taP::v_min) return 1.0; - if (Vy >= taP::v_min && Vy < taP::v_max) return 1.0; - } - if (Vx >= -taP::v_max && Vx < -taP::v_min) { - if (Vy >= -taP::v_max && Vy < -taP::v_min) return 1.0; - if (Vy >= taP::v_min && Vy < taP::v_max) return 1.0; - } - } - if (Vz >= -taP::v_max && Vz < -taP::v_min) { - if (Vx >= taP::v_min && Vx < taP::v_max) { - if (Vy >= -taP::v_max && Vy < -taP::v_min) return 1.0; - if (Vy >= taP::v_min && Vy < taP::v_max) return 1.0; - } - if (Vx >= -taP::v_max && Vx < -taP::v_min) { - if (Vy >= -taP::v_max && Vy < -taP::v_min) return 1.0; - if (Vy >= taP::v_min && Vy < taP::v_max) return 1.0; - } - } - return 0.0; -} - -/** Calculate parameters for the given spatial cell at the given time. - * Here you need to set values for the following array indices: - * CellParams::EX, CellParams::EY, CellParams::EZ, CellParams::BX, CellParams::BY, and CellParams::BZ. - * - * The following array indices contain the coordinates of the "lower left corner" of the cell: - * CellParams::XCRD, CellParams::YCRD, and CellParams::ZCRD. - * The cell size is given in the following array indices: CellParams::DX, CellParams::DY, and CellParams::DZ. - * @param cellParams Array containing cell parameters. - * @param t The current value of time. This is passed as a convenience. If you need more detailed information - * of the state of the simulation, you can read it from Parameters. - */ -void calcCellParameters(Real* cellParams,creal& t) { - cellParams[CellParams::EX ] = 0.0; - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - cellParams[CellParams::BGBX ] = 0.0; - cellParams[CellParams::BGBY ] = 0.0; - cellParams[CellParams::BGBZ ] = 0.0; - - cellParams[CellParams::EXVOL] = 0.0; - cellParams[CellParams::EYVOL] = 0.0; - cellParams[CellParams::EZVOL] = 0.0; - cellParams[CellParams::BXVOL] = 0.0; - cellParams[CellParams::BYVOL] = 0.0; - cellParams[CellParams::BZVOL] = 0.0; -} - -void setProjectCell(SpatialCell* cell) { - // Set up cell parameters: - calcCellParameters(&((*cell).parameters[0]), 0.0); - - cell->parameters[CellParams::RHOLOSSADJUST] = 0.0; - cell->parameters[CellParams::RHOLOSSVELBOUNDARY] = 0.0; - - // Go through each velocity block in the velocity phase space grid. - // Set the initial state and block parameters: - creal dvx_block = SpatialCell::block_dvx; // Size of a block in vx-direction - creal dvy_block = SpatialCell::block_dvy; // vy - creal dvz_block = SpatialCell::block_dvz; // vz - creal dvx_blockCell = SpatialCell::cell_dvx; // Size of one cell in a block in vx-direction - creal dvy_blockCell = SpatialCell::cell_dvy; // vy - creal dvz_blockCell = SpatialCell::cell_dvz; // vz - - for (uint kv=0; kvparameters[CellParams::XCRD], - cell->parameters[CellParams::YCRD], - cell->parameters[CellParams::ZCRD], - cell->parameters[CellParams::DX], - cell->parameters[CellParams::DY], - cell->parameters[CellParams::DZ], - vx_cell,vy_cell,vz_cell, - dvx_blockCell,dvy_blockCell,dvz_blockCell); - - if(average!=0.0){ - creal vx_cell_center = vx_block + (ic+convert(0.5))*dvx_blockCell; - creal vy_cell_center = vy_block + (jc+convert(0.5))*dvy_blockCell; - creal vz_cell_center = vz_block + (kc+convert(0.5))*dvz_blockCell; - cell->set_value(vx_cell_center,vy_cell_center,vz_cell_center,average); - } - } - } - calculateCellVelocityMoments(cell); - - //let's get rid of blocks not fulfilling the criteria here to save memory. - cell->adjustSingleCellVelocityBlocks(); -} - diff --git a/projects/unsupported/test_acc/test_acc.h b/projects/unsupported/test_acc/test_acc.h deleted file mode 100644 index 76651ee4d..000000000 --- a/projects/unsupported/test_acc/test_acc.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef TEST_ACC_H -#define TEST_ACC_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -#include "dccrg.hpp" - -struct test_accParameters { - static Real SPEED; - static Real v_min; - static Real v_max; -} ; - -/** - * Initialize project. Can be used, e.g., to read in parameters from the input file - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); - -template void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams -) { - ax = blockParams[BlockParams::VXCRD] + (I+convert(0.5))*blockParams[BlockParams::DVX]; - ay = blockParams[BlockParams::VYCRD] + (J+convert(0.5))*blockParams[BlockParams::DVY]; - az = blockParams[BlockParams::VZCRD] + (K+convert(0.5))*blockParams[BlockParams::DVZ]; - -// const REAL SPEED = convert(0.5) + convert(0.5)*(convert(1.0) / convert(6.0)); - - if (ax > convert(0.0)) ax = test_accParameters::SPEED; - else ax = -test_accParameters::SPEED; - if (ay > convert(0.0)) ay = test_accParameters::SPEED; - else ay = -test_accParameters::SPEED; - if (az > convert(0.0)) az = test_accParameters::SPEED; - else az = -test_accParameters::SPEED; -} - -template void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I,const UINT& J,const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams -) { - ax = blockParams[BlockParams::VXCRD] + (I+convert(0.5))*blockParams[BlockParams::DVX]; - ay = blockParams[BlockParams::VYCRD] + (J+convert(0.5))*blockParams[BlockParams::DVY]; - az = blockParams[BlockParams::VZCRD] + (K+convert(0.5))*blockParams[BlockParams::DVZ]; - -// const REAL SPEED = convert(0.5) + convert(0.5)*(convert(1.0) / convert(6.0)); - - if (ax > convert(0.0)) ax = test_accParameters::SPEED; - else ax = -test_accParameters::SPEED; - if (ay > convert(0.0)) ay = test_accParameters::SPEED; - else ay = -test_accParameters::SPEED; - if (az > convert(0.0)) az = test_accParameters::SPEED; - else az = -test_accParameters::SPEED; -} - -template void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I,const UINT& J,const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams -) { - ax = blockParams[BlockParams::VXCRD] + (I+convert(0.5))*blockParams[BlockParams::DVX]; - ay = blockParams[BlockParams::VYCRD] + (J+convert(0.5))*blockParams[BlockParams::DVY]; - az = blockParams[BlockParams::VZCRD] + (K+convert(0.5))*blockParams[BlockParams::DVZ]; - -// const REAL SPEED = convert(0.5) + convert(0.5)*(convert(1.0) / convert(6.0)); - - if (ax > convert(0.0)) ax = test_accParameters::SPEED; - else ax = -test_accParameters::SPEED; - if (ay > convert(0.0)) ay = test_accParameters::SPEED; - else ay = -test_accParameters::SPEED; - if (az > convert(0.0)) az = test_accParameters::SPEED; - else az = -test_accParameters::SPEED; -} - -#endif - diff --git a/projects/unsupported/test_acc/test_acc_technical_info.pdf b/projects/unsupported/test_acc/test_acc_technical_info.pdf deleted file mode 100644 index ade746939..000000000 Binary files a/projects/unsupported/test_acc/test_acc_technical_info.pdf and /dev/null differ diff --git a/projects/unsupported/test_acc/test_acc_technical_info.tex b/projects/unsupported/test_acc/test_acc_technical_info.tex deleted file mode 100644 index 1b08f8cfe..000000000 --- a/projects/unsupported/test_acc/test_acc_technical_info.tex +++ /dev/null @@ -1,44 +0,0 @@ -\documentclass[a4paper,10pt]{scrartcl} -\usepackage[utf8x]{inputenc} -\usepackage[left=1.5cm, right=1.5cm, top=1.5cm, bottom=2.5cm]{geometry} -\usepackage[autolanguage,np]{numprint} -\usepackage{tabularx} - -\usepackage{hyperref} -\hyperbaseurl{.} - -%opening -\title{ -\Huge{Vlasiator test cases technical information} \\ -\LARGE{test\_acc} -} -\author{Yann Kempf} -\date{Updated on \today} - -\begin{document} - -\maketitle - -\begin{abstract} - This document gives technical information on the test\_acc test case. -\end{abstract} - -\section{Purpose} -Test the isotropy of the acceleration part of the Vlasov solver. Can be used for isotropy and order of accuracy. - -\section{Implementation} -Uses a single spatial cell in periodic boundaries without field solver. Initialises symmetrically in each octant a cube with non-zero density, imposes an artificial acceleration along the diagonals of the space. One can then check the result visually or by calculating norms using VisIt (done on my MSc thesis for instance). The position of the initial cells can be set so that one can also generate reference final states for comparison. The acceleration is artificial in that it is hand-coded into the \verb=calcAccFace[XYZ]= functions in the header file. - -\section{Options} -The options available in the \verb=cfg= file are: - -\begin{tabularx}{\textwidth}{lX} - \verb=SPEED= & Artificially imposed acceleration value \\ - \verb=v_min= & Minimum value of the cubes initialised with non-zero density in velocity space \\ - \verb=v_max= & Maximum value of the cubes initialised with non-zero density in velocity space -\end{tabularx} - - - - -\end{document} \ No newline at end of file diff --git a/projects/unsupported/test_trans2/test_trans2.cfg b/projects/unsupported/test_trans2/test_trans2.cfg deleted file mode 100644 index ccc804d17..000000000 --- a/projects/unsupported/test_trans2/test_trans2.cfg +++ /dev/null @@ -1,58 +0,0 @@ -system_write_t_interval = 0.25 -diagnostic_write_interval = 1 -restart_write_t_interval = 0.25 -propagate_field = 0 -propagate_vlasov = 1 - -[gridbuilder] -x_length = 100 -y_length = 100 -z_length = 1 -#x_length = 10 -#y_length = 10 -#z_length = 5 -x_min = -3.0 -x_max = +3.0 -y_min = -3.0 -y_max = +3.0 -z_min = -1.0 -z_max = +1.0 -vx_min = -3.0 -vx_max = +3.0 -vy_min = -3.0 -vy_max = +3.0 -vz_min = -0.75 -vz_max = +1.25 -vx_length = 25 -vy_length = 25 -vz_length = 1 -q = 1.0 -m = 1.0 -timestep_max = 500 -#dt = 0.171428572 -dt = 0.01 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks - -[loadBalance] -algorithm = RCB -tolerance = 1.05 -rebalanceInterval = 10 - -[sparse] -minValue = 1.0e-7 -minAvgValue = 5.0e-8 -blockAdjustmentInterval = 1 - -[test_trans2] -radLimitInf = 0.97 -radLimitSup = 1.03 diff --git a/projects/unsupported/test_trans2/test_trans2.cpp b/projects/unsupported/test_trans2/test_trans2.cpp deleted file mode 100644 index 281f17d82..000000000 --- a/projects/unsupported/test_trans2/test_trans2.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#include -#include -#include - -#include "spatial_cell.hpp" -#include "common.h" -#include "project.h" -#include "parameters.h" -#include "readparameters.h" -#include "vlasovmover.h" - -using namespace std; - -typedef testTrans2Parameters TTP; -Real TTP::radLimitInf = NAN; -Real TTP::radLimitSup = NAN; - -bool initializeProject(void) {return true;} - -bool addProjectParameters(){ - typedef Readparameters RP; - RP::add("test_trans2.radLimitInf", "radial inner radius in space", 0.2); - RP::add("test_trans2.radLimitSup", "radial outer radius", 0.5); - return true; -} - -bool getProjectParameters(){ - typedef Readparameters RP; - RP::get("test_trans2.radLimitInf", TTP::radLimitInf); - RP::get("test_trans2.radLimitSup", TTP::radLimitSup); - return true; -} - -/** Integrate the distribution function over the given six-dimensional phase-space cell. - * @param x Starting value of the x-coordinate of the cell. - * @param y Starting value of the y-coordinate of the cell. - * @param z Starting value of the z-coordinate of the cell. - * @param dx The size of the cell in x-direction. - * @param dy The size of the cell in y-direction. - * @param dz The size of the cell in z-direction. - * @param vx Starting value of the vx-coordinate of the cell. - * @param vy Starting value of the vy-coordinate of the cell. - * @param vz Starting value of the vz-coordinate of the cell. - * @param dvx The size of the cell in vx-direction. - * @param dvy The size of the cell in vy-direction. - * @param dvz The size of the cell in vz-direction. - * @return The volume average of the distribution function in the given phase space cell. - * The physical unit of this quantity is 1 / (m^3 (m/s)^3). - */ -Real calcPhaseSpaceDensity(creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz) { - creal X = x + 0.5*dx; - creal Y = y + 0.5*dy; - creal vX = vx + 0.5*dvx; - creal vY = vy + 0.5*dvy; - - creal radius = sqrt(X*X + Y*Y); - creal cell2dID = (int) (X / dx) + - (int) (Y / dy) * Parameters::xcells_ini; - creal vel2dID = (int) (vX / dvx) + - (int) (vY / dvy) * Parameters::vxblocks_ini * 4; - - if (cell2dID == vel2dID && - vz + 0.5 * dvz >= -0.4 * dvz && - vz + 0.5 * dvz < 0.4 * dvz && - radius <= TTP::radLimitSup && - radius >= TTP::radLimitInf - ) { - return 1.0; - } - return 0.0; -} - -void calcBlockParameters(Real* blockParams) { } - -/** Calculate parameters for the given spatial cell at the given time. - * Here you need to set values for the following array indices: - * CellParams::EX, CellParams::EY, CellParams::EZ, CellParams::BX, CellParams::BY, and CellParams::BZ. - * - * The following array indices contain the coordinates of the "lower left corner" of the cell: - * CellParams::XCRD, CellParams::YCRD, and CellParams::ZCRD. - * The cell size is given in the following array indices: CellParams::DX, CellParams::DY, and CellParams::DZ. - * @param cellParams Array containing cell parameters. - * @param t The current value of time. This is passed as a convenience. If you need more detailed information - * of the state of the simulation, you can read it from Parameters. - */ -void calcCellParameters(Real* cellParams,creal& t) { - cellParams[CellParams::EX ] = 0.0; - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - - cellParams[CellParams::EXVOL] = 0.0; - cellParams[CellParams::EYVOL] = 0.0; - cellParams[CellParams::EZVOL] = 0.0; - cellParams[CellParams::BXVOL] = 0.0; - cellParams[CellParams::BYVOL] = 0.0; - cellParams[CellParams::BZVOL] = 0.0; -} - -void setProjectCell(SpatialCell* cell) { - // Set up cell parameters: - calcCellParameters(&((*cell).parameters[0]), 0.0); - - cell->parameters[CellParams::RHOLOSSADJUST] = 0.0; - cell->parameters[CellParams::RHOLOSSVELBOUNDARY] = 0.0; - - // Go through each velocity block in the velocity phase space grid. - // Set the initial state and block parameters: - creal dvx_block = SpatialCell::block_dvx; // Size of a block in vx-direction - creal dvy_block = SpatialCell::block_dvy; // vy - creal dvz_block = SpatialCell::block_dvz; // vz - creal dvx_blockCell = SpatialCell::cell_dvx; // Size of one cell in a block in vx-direction - creal dvy_blockCell = SpatialCell::cell_dvy; // vy - creal dvz_blockCell = SpatialCell::cell_dvz; // vz - - for (uint kv=0; kvparameters[CellParams::XCRD], - cell->parameters[CellParams::YCRD], - cell->parameters[CellParams::ZCRD], - cell->parameters[CellParams::DX], - cell->parameters[CellParams::DY], - cell->parameters[CellParams::DZ], - vx_cell,vy_cell,vz_cell, - dvx_blockCell,dvy_blockCell,dvz_blockCell); - - if(average!=0.0){ - creal vx_cell_center = vx_block + (ic+convert(0.5))*dvx_blockCell; - creal vy_cell_center = vy_block + (jc+convert(0.5))*dvy_blockCell; - creal vz_cell_center = vz_block + (kc+convert(0.5))*dvz_blockCell; - cell->set_value(vx_cell_center,vy_cell_center,vz_cell_center,average); - } - } - } - calculateCellVelocityMoments(cell); - - //let's get rid of blocks not fulfilling the criteria here to save memory. - cell->adjustSingleCellVelocityBlocks(); -} - diff --git a/projects/unsupported/test_trans2/test_trans2.h b/projects/unsupported/test_trans2/test_trans2.h deleted file mode 100644 index 77148082d..000000000 --- a/projects/unsupported/test_trans2/test_trans2.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#ifndef TESTTRANS2_H -#define TESTTRANS2_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -struct testTrans2Parameters { - static Real radLimitInf; - static Real radLimitSup; -} ; - -/** - * Initialize project. Can be used, e.g., to read in parameters from the input file - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + I*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + (J+HALF)*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + (K+HALF)*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + (I+HALF)*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + J*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + (K+HALF)*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + (I+HALF)*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + (J+HALF)*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + K*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} - -#endif diff --git a/projects/unsupported/test_trans2/test_trans2_technical_info.pdf b/projects/unsupported/test_trans2/test_trans2_technical_info.pdf deleted file mode 100644 index a423979c2..000000000 Binary files a/projects/unsupported/test_trans2/test_trans2_technical_info.pdf and /dev/null differ diff --git a/projects/unsupported/test_trans2/test_trans2_technical_info.tex b/projects/unsupported/test_trans2/test_trans2_technical_info.tex deleted file mode 100644 index fb159b19b..000000000 --- a/projects/unsupported/test_trans2/test_trans2_technical_info.tex +++ /dev/null @@ -1,44 +0,0 @@ -\documentclass[a4paper,10pt]{scrartcl} -\usepackage[utf8x]{inputenc} -\usepackage[left=1.5cm, right=1.5cm, top=1.5cm, bottom=2.5cm]{geometry} -\usepackage[autolanguage,np]{numprint} -\usepackage{tabularx} - -\usepackage{hyperref} -\hyperbaseurl{.} - -%opening -\title{ -\Huge{Vlasiator test cases technical information} \\ -\LARGE{test\_trans2} -} -\author{Yann Kempf} -\date{Updated on \today} - -\begin{document} - -\maketitle - -\begin{abstract} - This document gives technical information on the test\_trans2 test case. -\end{abstract} - -\section{Purpose} -Investigate spatial/velocity resolution issues more deeply after the appearance of \textit{blocky} features in the Diffusion test case. - - -\section{Implementation} -A ring is initialised with radial velocity in the $xy$-plane, set in periodic boundary conditions. Due to the finite resolution box-cars appear so that the ring is not smooth. The definition of a uniform initial ring with exact radial velocity is tricky, the best is to match the spatial and velocity grids so that cell indices can be matched. This is implemented as well as a velocity condition to select zero $z$ velocity. - -\section{Options} -The options available in the \verb=cfg= file are: - -\begin{tabularx}{\textwidth}{lX} - \verb=radLimitInf= & radial inner radius in space \\ - \verb=radLimitSup= & radial outer radius in space -\end{tabularx} - - - - -\end{document} \ No newline at end of file diff --git a/projects/unsupported/test_trans_1D/test_trans_1D.cfg b/projects/unsupported/test_trans_1D/test_trans_1D.cfg deleted file mode 100644 index 5251bc8fd..000000000 --- a/projects/unsupported/test_trans_1D/test_trans_1D.cfg +++ /dev/null @@ -1,44 +0,0 @@ -system_write_t_interval = 0.1 -diagnostic_write_interval = 1 -restart_write_t_interval = 1000 -propagate_field = 0 -propagate_vlasov = 1 - -[gridbuilder] -x_length = 10 -y_length = 1 -z_length = 1 -#x_length = 10 -#y_length = 10 -#z_length = 5 -x_min = -2.0 -x_max = +2.0 -y_min = -0.2 -y_max = +0.2 -z_min = -1.0 -z_max = +1.0 -vx_min = +0.625 -vx_max = +1.625 -vy_min = -0.75 -vy_max = +0.75 -vz_min = -0.75 -vz_max = +0.75 -vx_length = 1 -vy_length = 1 -vz_length = 1 -q = 1.0 -m = 1.0 -timestep_max = 20 -#dt = 0.171428572 -dt = 0.1 - -[boundaries] -periodic_x = no -periodic_y = no -periodic_z = no - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks diff --git a/projects/unsupported/test_trans_1D/test_trans_1D.cpp b/projects/unsupported/test_trans_1D/test_trans_1D.cpp deleted file mode 100644 index b5c2e5499..000000000 --- a/projects/unsupported/test_trans_1D/test_trans_1D.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "spatial_cell.hpp" -#include "common.h" -#include "project.h" -#include "parameters.h" -#include "readparameters.h" -#include "vlasovmover.h" - -using namespace std; - -bool initializeProject(void) {return true;} - -bool addProjectParameters(){ -// typedef Readparameters RP; -// RP::add("test_trans.cellPosition", "Position of the centre of the cells initiaited (same used in velocity and space).", 1.5); - return true; -} - -bool getProjectParameters(){ -// typedef Readparameters RP; -// RP::get("test_trans.cellPosition", ttP::cellPosition); - return true; -} - -/** Integrate the distribution function over the given six-dimensional phase-space cell. - * @param x Starting value of the x-coordinate of the cell. - * @param y Starting value of the y-coordinate of the cell. - * @param z Starting value of the z-coordinate of the cell. - * @param dx The size of the cell in x-direction. - * @param dy The size of the cell in y-direction. - * @param dz The size of the cell in z-direction. - * @param vx Starting value of the vx-coordinate of the cell. - * @param vy Starting value of the vy-coordinate of the cell. - * @param vz Starting value of the vz-coordinate of the cell. - * @param dvx The size of the cell in vx-direction. - * @param dvy The size of the cell in vy-direction. - * @param dvz The size of the cell in vz-direction. - * @return The volume average of the distribution function in the given phase space cell. - * The physical unit of this quantity is 1 / (m^3 (m/s)^3). - */ -Real calcPhaseSpaceDensity(creal& x,creal& y,creal& z,creal& dx,creal& dy,creal& dz, - creal& vx,creal& vy,creal& vz,creal& dvx,creal& dvy,creal& dvz) { - - Real vxcrd, xcrd; - vxcrd = vx + 0.5 * dvx; - xcrd = x + 0.5 * dx; - - if(vxcrd > 0.9 && vxcrd < 1.1 && - xcrd > -0.2 && xcrd < 0.2) { - return 1.0;} - - return 0.0; -} - -/** Calculate parameters for the given spatial cell at the given time. - * Here you need to set values for the following array indices: - * CellParams::EX, CellParams::EY, CellParams::EZ, CellParams::BX, CellParams::BY, and CellParams::BZ. - * - * The following array indices contain the coordinates of the "lower left corner" of the cell: - * CellParams::XCRD, CellParams::YCRD, and CellParams::ZCRD. - * The cell size is given in the following array indices: CellParams::DX, CellParams::DY, and CellParams::DZ. - * @param cellParams Array containing cell parameters. - * @param t The current value of time. This is passed as a convenience. If you need more detailed information - * of the state of the simulation, you can read it from Parameters. - */ -void calcCellParameters(Real* cellParams,creal& t) { - cellParams[CellParams::EX ] = 0.0; - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - - cellParams[CellParams::EXVOL] = 0.0; - cellParams[CellParams::EYVOL] = 0.0; - cellParams[CellParams::EZVOL] = 0.0; - cellParams[CellParams::BXVOL] = 0.0; - cellParams[CellParams::BYVOL] = 0.0; - cellParams[CellParams::BZVOL] = 0.0; -} - -void setProjectCell(SpatialCell* cell) { - // Set up cell parameters: - calcCellParameters(&((*cell).parameters[0]), 0.0); - - cell->parameters[CellParams::RHOLOSSADJUST] = 0.0; - cell->parameters[CellParams::RHOLOSSVELBOUNDARY] = 0.0; - - // Go through each velocity block in the velocity phase space grid. - // Set the initial state and block parameters: - creal dvx_block = SpatialCell::block_dvx; // Size of a block in vx-direction - creal dvy_block = SpatialCell::block_dvy; // vy - creal dvz_block = SpatialCell::block_dvz; // vz - creal dvx_blockCell = SpatialCell::cell_dvx; // Size of one cell in a block in vx-direction - creal dvy_blockCell = SpatialCell::cell_dvy; // vy - creal dvz_blockCell = SpatialCell::cell_dvz; // vz - - for (uint kv=0; kvparameters[CellParams::XCRD], - cell->parameters[CellParams::YCRD], - cell->parameters[CellParams::ZCRD], - cell->parameters[CellParams::DX], - cell->parameters[CellParams::DY], - cell->parameters[CellParams::DZ], - vx_cell,vy_cell,vz_cell, - dvx_blockCell,dvy_blockCell,dvz_blockCell); - - if(average!=0.0){ - creal vx_cell_center = vx_block + (ic+convert(0.5))*dvx_blockCell; - creal vy_cell_center = vy_block + (jc+convert(0.5))*dvy_blockCell; - creal vz_cell_center = vz_block + (kc+convert(0.5))*dvz_blockCell; - cell->set_value(vx_cell_center,vy_cell_center,vz_cell_center,average); - } - } - } - calculateCellVelocityMoments(cell); - - //let's get rid of blocks not fulfilling the criteria here to save memory. - cell->adjustSingleCellVelocityBlocks(); -} - diff --git a/projects/unsupported/test_trans_1D/test_trans_1D.h b/projects/unsupported/test_trans_1D/test_trans_1D.h deleted file mode 100644 index 34d6251d2..000000000 --- a/projects/unsupported/test_trans_1D/test_trans_1D.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef TEST_TRANS_1D_H -#define TEST_TRANS_1D_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "parameters.h" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -#include "dccrg.hpp" - -/** - * Initialize project. Can be used, e.g., to read in parameters from the input file - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); - -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + I*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + (J+HALF)*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + (K+HALF)*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + (I+HALF)*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + J*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + (K+HALF)*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} -// WARNING Lorentz force not fixed in this project (cf. JXB term in the acceleration)!!! -template -void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - const REAL HALF = 0.5; - const REAL VX = blockParams[BlockParams::VXCRD] + (I+HALF)*blockParams[BlockParams::DVX]; - const REAL VY = blockParams[BlockParams::VYCRD] + (J+HALF)*blockParams[BlockParams::DVY]; - const REAL VZ = blockParams[BlockParams::VZCRD] + K*blockParams[BlockParams::DVZ]; - ax = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EX] + - VY*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ]) - - VZ*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY])); - ay = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EY] + - VZ*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX]) - - VX*(cellParams[CellParams::PERBZ]+cellParams[CellParams::BGBZ])); - az = physicalconstants::CHARGE/physicalconstants::MASS_PROTON * (cellParams[CellParams::EZ] + - VX*(cellParams[CellParams::PERBY]+cellParams[CellParams::BGBY]) - - VY*(cellParams[CellParams::PERBX]+cellParams[CellParams::BGBX])); -} - - -#endif diff --git a/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.pdf b/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.pdf deleted file mode 100644 index f1d81ab70..000000000 Binary files a/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.pdf and /dev/null differ diff --git a/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.tex b/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.tex deleted file mode 100644 index bfffc14ad..000000000 --- a/projects/unsupported/test_trans_1D/test_trans_1D_technical_info.tex +++ /dev/null @@ -1,38 +0,0 @@ -\documentclass[a4paper,10pt]{scrartcl} -\usepackage[utf8x]{inputenc} -\usepackage[left=1.5cm, right=1.5cm, top=1.5cm, bottom=2.5cm]{geometry} -\usepackage[autolanguage,np]{numprint} -\usepackage{tabularx} - -\usepackage{hyperref} -\hyperbaseurl{.} - -%opening -\title{ -\Huge{Vlasiator test cases technical information} \\ -\LARGE{test\_trans\_1D} -} -\author{Yann Kempf} -\date{Updated on \today} - -\begin{document} - -\maketitle - -\begin{abstract} - This document gives technical information on the test\_trans\_1D test case. -\end{abstract} - -\section{Purpose} -One-dimensional adaptation of test\_trans to make it faster/possible to compute for accuracy tests with high spatial resolutions. - - -\section{Implementation} -Straight copy of test\_trans reduced to 1D, making it similar to the grid-aligned test\_fp reduced to 1D. - -\section{Options} -The parameters are hard-coded, only the \verb=x_length= should be varied to perform accuracy analysis. - - - -\end{document} \ No newline at end of file diff --git a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cfg b/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cfg deleted file mode 100644 index 5114e1533..000000000 --- a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cfg +++ /dev/null @@ -1,36 +0,0 @@ -system_write_t_interval = 0.5 -diagnostic_write_interval = 1 -restart_write_t_interval = 1000 - -[gridbuilder] -vx_min = -1e6 -vx_max = 1e6 -vy_min = -1e6 -vy_max = 1e6 -vz_min = -1e6 -vz_max = 1e6 -vx_length = 10 -vy_length = 10 -vz_length = 10 -x_min = 0 -x_max = 1 -y_min = 0 -y_max = 1 -z_min = 0 -z_max = 1 -x_length = 1 -y_length = 1 -z_length = 1 -timestep_max = 100 -dt = 0.1 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks diff --git a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cpp b/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cpp deleted file mode 100644 index f738807a7..000000000 --- a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "spatial_cell.hpp" -#include "common.h" -#include "project.h" -#include "parameters.h" -#include "readparameters.h" -#include "vlasovmover.h" - -using namespace std; - -bool initializeProject(void) {return true;} -bool addProjectParameters(){return true;} -bool getProjectParameters(){return true;} - -/** Integrate the distribution function over the given six-dimensional phase-space cell. - * @param x Starting value of the x-coordinate of the cell. - * @param y Starting value of the y-coordinate of the cell. - * @param z Starting value of the z-coordinate of the cell. - * @param dx The size of the cell in x-direction. - * @param dy The size of the cell in y-direction. - * @param dz The size of the cell in z-direction. - * @param vx Starting value of the vx-coordinate of the cell. - * @param vy Starting value of the vy-coordinate of the cell. - * @param vz Starting value of the vz-coordinate of the cell. - * @param dvx The size of the cell in vx-direction. - * @param dvy The size of the cell in vy-direction. - * @param dvz The size of the cell in vz-direction. - * @return The integral of the distribution function over the given six-dimensional phase-space cell. - */ -Real calcPhaseSpaceDensity(creal& x,creal& y,creal& z,creal& dx,creal& dy,creal& dz, creal& vx,creal& vy,creal& vz,creal& dvx,creal& dvy,creal& dvz) { - #define AMP (1.0 / dx / dy / dz / dvx / dvy / dvz) - #define DELTAX 50000 - #define rad0 (1e6 / 2) - - /*double phi = atan2(vy, vx); - double rad = sqrt(vx * vx + vy * vy); - - if (phi < 3 * M_PI / 4 && phi > M_PI / 4) { - return AMP * exp(-(rad - rad0) * (rad - rad0) / DELTAX / DELTAX); - } else { - return 0; - }*/ - creal sw_vx = -5e5; - creal sw_vy = 5e5; - creal sw_vz = 5e5; - if (fabs(vx - sw_vx) <= dvx / 2 - && fabs(vy - sw_vy) <= dvy / 2 - && fabs(vz - sw_vz) <= dvz / 2) { - return 1e6; - } else { - return 0; - } -} - -/** Calculate parameters for the given spatial cell at the given time. - * Here you need to set values for the following array indices: - * CellParams::EX, CellParams::EY, CellParams::EZ, CellParams::BX, CellParams::BY, and CellParams::BZ. - * - * The following array indices contain the coordinates of the "lower left corner" of the cell: - * CellParams::XCRD, CellParams::YCRD, and CellParams::ZCRD. - * The cell size is given in the following array indices: CellParams::DX, CellParams::DY, and CellParams::DZ. - * @param cellParams Array containing cell parameters. - * @param t The current value of time. This is passed as a convenience. If you need more detailed information - * of the state of the simulation, you can read it from Parameters. - */ -void calcCellParameters(Real* cellParams, creal& /*t*/) { - cellParams[CellParams::EX] = 0.0; - cellParams[CellParams::EY] = 0.0; - cellParams[CellParams::EZ] = 0.0; - cellParams[CellParams::PERBX] = 0.0; - cellParams[CellParams::PERBY] = 0.0; - cellParams[CellParams::PERBZ] = 0; - cellParams[CellParams::BGBX] = 1e-9; - cellParams[CellParams::BGBY] = 0.0; - cellParams[CellParams::BGBZ] = 0; -} - -void setProjectCell(SpatialCell* cell) { - // Set up cell parameters: - calcCellParameters(&((*cell).parameters[0]), 0.0); - - cell->parameters[CellParams::RHOLOSSADJUST] = 0.0; - cell->parameters[CellParams::RHOLOSSVELBOUNDARY] = 0.0; - - // Go through each velocity block in the velocity phase space grid. - // Set the initial state and block parameters: - creal dvx_block = SpatialCell::block_dvx; // Size of a block in vx-direction - creal dvy_block = SpatialCell::block_dvy; // vy - creal dvz_block = SpatialCell::block_dvz; // vz - creal dvx_blockCell = SpatialCell::cell_dvx; // Size of one cell in a block in vx-direction - creal dvy_blockCell = SpatialCell::cell_dvy; // vy - creal dvz_blockCell = SpatialCell::cell_dvz; // vz - - for (uint kv=0; kvparameters[CellParams::XCRD], - cell->parameters[CellParams::YCRD], - cell->parameters[CellParams::ZCRD], - cell->parameters[CellParams::DX], - cell->parameters[CellParams::DY], - cell->parameters[CellParams::DZ], - vx_cell,vy_cell,vz_cell, - dvx_blockCell,dvy_blockCell,dvz_blockCell); - - if(average!=0.0){ - creal vx_cell_center = vx_block + (ic+convert(0.5))*dvx_blockCell; - creal vy_cell_center = vy_block + (jc+convert(0.5))*dvy_blockCell; - creal vz_cell_center = vz_block + (kc+convert(0.5))*dvz_blockCell; - cell->set_value(vx_cell_center,vy_cell_center,vz_cell_center,average); - } - } - } - calculateCellVelocityMoments(cell); - - //let's get rid of blocks not fulfilling the criteria here to save memory. - cell->adjustSingleCellVelocityBlocks(); -} diff --git a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.h b/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.h deleted file mode 100644 index 527478b9e..000000000 --- a/projects/unsupported/velocity_rotation_1+3d/velocity_rotation_1+3d.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef VELOCITY_ROTATION_H -#define VELOCITY_ROTATION_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -#include "dccrg.hpp" - -/** - * Initialize project. Can be used, e.g., to read in parameters from the input file - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); - -template void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceX(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceY(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceZ(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -#endif diff --git a/projects/unsupported/velrot2+2/velrot2+2.cfg b/projects/unsupported/velrot2+2/velrot2+2.cfg deleted file mode 100644 index 5e0e13b8c..000000000 --- a/projects/unsupported/velrot2+2/velrot2+2.cfg +++ /dev/null @@ -1,39 +0,0 @@ -system_write_t_interval = 0.628318525 -diagnostic_write_interval = 1 -restart_write_t_interval = 251.32741 - -[gridbuilder] -x_length = 1 -y_length = 1 -z_length = 1 -x_min = -1.2 -x_max = +1.2 -y_min = -1.2 -y_max = +1.2 -z_min = -1.2 -z_max = +1.2 -vx_min = -1.0 -vx_max = +1.0 -vy_min = -1.0 -vy_max = +1.0 -vz_min = -1.0 -vz_max = +1.0 -vx_length = 10 -vy_length = 10 -vz_length = 1 -q = 1.0 -m = 1.0 -timestep_max = 2000 -dt = 0.025132741 -#dt = 0.012566371 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = B -diagnostic = FluxB -diagnostic = Blocks diff --git a/projects/unsupported/velrot2+2/velrot2+2.cpp b/projects/unsupported/velrot2+2/velrot2+2.cpp deleted file mode 100644 index b54056795..000000000 --- a/projects/unsupported/velrot2+2/velrot2+2.cpp +++ /dev/null @@ -1,237 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -#include "spatial_cell.hpp" -#include "common.h" -#include "project.h" -#include "parameters.h" -#include "readparameters.h" -#include "vlasovmover.h" - -using namespace std; - -bool initializeProject(void) {return true;} -bool addProjectParameters(){return true;} -bool getProjectParameters(){return true;} - -bool cellParametersChanged(creal& t) {return false;} - -Real getDistribValue(creal& OMEGA,creal& x,creal& y,creal& vx,creal& vy,creal& dvx,creal& dvy) { - - creal NORM = 1.0; // Norm. constant - creal R0 = 0.6; // Center point of distribution in r - creal SIGMA = 0.15; // Sigma of r-distribution - creal SIGMA2 = SIGMA*SIGMA; - - // Calculate r,phi coords. corresponding to x,y. Check that - // (r,phi) has non-zero distribution function value: - creal r = sqrt(x*x+y*y); - Real phi = acos(x/r); - if (y < 0.0) phi = 2.0*M_PI - phi; - - creal PHI_CENT = 320 / 180.0*M_PI; - creal PHI_WIDTH = 60.0 / 180.0*M_PI; - creal PHI0 = PHI_CENT - 0.5*PHI_WIDTH; - creal PHI1 = PHI_CENT + 0.5*PHI_WIDTH; - if (phi < PHI0 || phi > PHI1) return 0.0; - - // Calculate v,theta coordinates for (r,phi): - creal v = OMEGA*r; - Real theta = phi - M_PI/2.0; - while (theta < 0.0) theta += 2.0*M_PI; - - // Calculate vx',vy' corresponding to (v,theta), and - // check that vx_dot,vy_dot is inside the given velocity cell: - creal vx_dot = v*cos(theta); - creal vy_dot = v*sin(theta); - if (vx_dot < vx || vx_dot > vx+dvx) return 0.0; - if (vy_dot < vy || vy_dot > vy+dvy) return 0.0; - - creal arg = (r-R0)*(r-R0)/SIGMA2; - return NORM*exp(-1.0*arg); - - /* - creal V = sqrt(vx*vx+vy*vy); - if (V < 0.1) return 0.0; - - creal NORM = 1.0; - creal V0 = 0.6; - creal SIGMA = 0.15; - - creal THETA_CENT = 320 / 180.0*M_PI; - creal THETA_WIDTH = 60.0 / 180.0*M_PI; - creal THETA0 = THETA_CENT - 0.5*THETA_WIDTH; - creal THETA1 = THETA_CENT + 0.5*THETA_WIDTH; - - Real theta = acos(vx/V); - if (vy < 0.0) theta = 2.0*M_PI-theta; - //if (V < V0-DELTA || V > V0+DELTA) return 0.0; - if (theta < THETA0 || theta > THETA1) return 0.0; - - creal SIGMA2 = SIGMA*SIGMA; - creal arg = (V-V0)*(V-V0)/SIGMA2; - - return NORM * exp(-1.0*arg)/V; - */ -} - -/** Integrate the distribution function over the given six-dimensional phase-space cell. - * @param x Starting value of the x-coordinate of the cell. - * @param y Starting value of the y-coordinate of the cell. - * @param z Starting value of the z-coordinate of the cell. - * @param dx The size of the cell in x-direction. - * @param dy The size of the cell in y-direction. - * @param dz The size of the cell in z-direction. - * @param vx Starting value of the vx-coordinate of the cell. - * @param vy Starting value of the vy-coordinate of the cell. - * @param vz Starting value of the vz-coordinate of the cell. - * @param dvx The size of the cell in vx-direction. - * @param dvy The size of the cell in vy-direction. - * @param dvz The size of the cell in vz-direction. - * @return The volume average of the distribution function in the given phase space cell. - * The physical unit of this quantity is 1 / (m^3 (m/s)^3). - */ -Real calcPhaseSpaceDensity(creal& x,creal& y,creal& z,creal& dx,creal& dy,creal& dz, - creal& vx,creal& vy,creal& vz,creal& dvx,creal& dvy,creal& dvz) { - // Calculate (r,phi) corresponding to (x,y), and check that - // the coordinates are within the wedge containing nonzero distrib.f. values: - creal r = sqrt(x*x+y*y); - Real phi = acos(x/r); - creal EPSPHI = acos(dx/r); - if (y < 0.0) phi = 2.0*M_PI - phi; - - creal PHI_CENT = 320 / 180.0*M_PI; - creal PHI_WIDTH = 60.0 / 180.0*M_PI; - creal PHI0 = PHI_CENT - 0.5*PHI_WIDTH; - creal PHI1 = PHI_CENT + 0.5*PHI_WIDTH; - if (phi < PHI0-EPSPHI || phi > PHI1+EPSPHI) return 0.0; - - // Calculate (v,theta) corresponding to (vx,vy), and check that - // the coordinates are within the wedge of nonzero distrib. f.: - creal v = sqrt(vx*vx+vy*vy); - Real theta = acos(vx/v); - if (vy < 0.0) theta = 2.0*M_PI - theta; - //if (vz < 0.0) theta = 2.0*M_PI - theta; - creal EPSTHETA = acos(dvx/v); - - Real THETA0 = PHI0 - M_PI/2.0; - Real THETA1 = PHI1 - M_PI/2.0; - if (THETA0 < 0.0) THETA0 += 2.0*M_PI; - if (THETA1 < 0.0) THETA1 += 2.0*M_PI; - if (theta < THETA0-EPSTHETA || theta > THETA1+EPSTHETA) return 0.0; - - // Sample the 4D distribution function. - creal OMEGA = 1.0; - cuint N_samples = 100; - - creal d_x = dx / (N_samples+1); - creal d_y = dy / (N_samples+1); - creal d_z = dz / (N_samples+1); - Real avg = 0.0; - for (uint i=0; iparameters[CellParams::RHOLOSSADJUST] = 0.0; - cell->parameters[CellParams::RHOLOSSVELBOUNDARY] = 0.0; - - // Go through each velocity block in the velocity phase space grid. - // Set the initial state and block parameters: - creal dvx_block = SpatialCell::block_dvx; // Size of a block in vx-direction - creal dvy_block = SpatialCell::block_dvy; // vy - creal dvz_block = SpatialCell::block_dvz; // vz - creal dvx_blockCell = SpatialCell::cell_dvx; // Size of one cell in a block in vx-direction - creal dvy_blockCell = SpatialCell::cell_dvy; // vy - creal dvz_blockCell = SpatialCell::cell_dvz; // vz - - for (uint kv=0; kvparameters[CellParams::XCRD], - cell->parameters[CellParams::YCRD], - cell->parameters[CellParams::ZCRD], - cell->parameters[CellParams::DX], - cell->parameters[CellParams::DY], - cell->parameters[CellParams::DZ], - vx_cell,vy_cell,vz_cell, - dvx_blockCell,dvy_blockCell,dvz_blockCell); - - if(average!=0.0){ - creal vx_cell_center = vx_block + (ic+convert(0.5))*dvx_blockCell; - creal vy_cell_center = vy_block + (jc+convert(0.5))*dvy_blockCell; - creal vz_cell_center = vz_block + (kc+convert(0.5))*dvz_blockCell; - cell->set_value(vx_cell_center,vy_cell_center,vz_cell_center,average); - } - } - } - calculateCellVelocityMoments(cell); - - //let's get rid of blocks not fulfilling the criteria here to save memory. - cell->adjustSingleCellVelocityBlocks(); -} diff --git a/projects/unsupported/velrot2+2/velrot2+2.h b/projects/unsupported/velrot2+2/velrot2+2.h deleted file mode 100644 index 1cc8e2313..000000000 --- a/projects/unsupported/velrot2+2/velrot2+2.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef VELROT22_H -#define VELROT22_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -#include "dccrg.hpp" - -/** - * Initialize project. Can be used, e.g., to read in parameters from the input file - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); - -template void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceX(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceY(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceZ(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -#endif - diff --git a/projects/unsupported/verification-Larmor/test_job.sh b/projects/unsupported/verification-Larmor/test_job.sh deleted file mode 100644 index 00b4258eb..000000000 --- a/projects/unsupported/verification-Larmor/test_job.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -l -## Generated by cscrun -#PBS -l mppwidth=120 -#PBS -l mppdepth=1 -#PBS -q short -#PBS -V -#PBS -N ver-Lmr -#PBS -l walltime=0:15:0 -cd $PBS_O_WORKDIR - -#increase some variabls to avoid mpi problems -export MPICH_PTL_OTHER_EVENTS=4096 -export MPICH_UNEX_BUFFER_SIZE=240000000 -export MPICH_MAX_NUM_THREADS=1 -aprun -n 120 -d 1 ./vlasiator_meteo_DP_verification-Larmor --run_config=verification-Larmor.cfg - -#compute diff between steps separated by one full gyroperiod -rm diff.dat -for ((i=0;i<1000;i+=50)); - do j=$((i+1000)); - filea=$( printf 'grid.%07d.vlsv' $i ); - fileb=$( printf 'grid.%07d.vlsv' $j ); - diff=$( vlsvdiff_DP $filea $fileb rho 0 |grep "The absolute 1-distance" |gawk '{print $8}' ); - echo $i $j $diff >> diff.dat ; -done - diff --git a/projects/unsupported/verification-Larmor/verification-Larmor.cfg b/projects/unsupported/verification-Larmor/verification-Larmor.cfg deleted file mode 100644 index 0f4a7e9fe..000000000 --- a/projects/unsupported/verification-Larmor/verification-Larmor.cfg +++ /dev/null @@ -1,69 +0,0 @@ -system_write_t_interval = 3.27972353731 -diagnostic_write_interval = 10 -restart_write_t_interval = 1311.88941492 -propagate_field = 0 -propagate_vlasov = 1 - - -[gridbuilder] -y_length = 100 -x_length = 100 -z_length = 1 -y_min = 0.0 -y_max = 2.5e6 -x_min = 0.0 -x_max = 2.5e6 -z_min = 0.0 -z_max = 5.0e4 -vx_min = -120000 -vx_max = +120000 -vy_min = -120000 -vy_max = +120000 -vz_min = -120000 -vz_max = +120000 -vx_length = 20 -vy_length = 20 -vz_length = 20 -timestep_max = 2000 - -#dt to give 1000steps per period -dt = 0.06559447074625 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[variables] -output = Rho -output = RhoV -output = B -output = VolB -output = MPIrank -output = Blocks -diagnostic = FluxB -diagnostic = Blocks - -[loadBalance] -algorithm = RCB -tolerance = 1.05 - -[sparse] -minValue = 1.0e-14 -minAvgValue = 5.0e-15 - -[Larmor] -BX0 = 0.0 -BY0 = 0.0 -BZ0 = 1.0e-9 - -VX0 = 80000 -VY0 = 0.0 -VZ0 = 0.0 - -X0 = 1.25e6 -Y0 = 2.0e6 -Z0 = 0.0 - -rho = 1.0e3 - diff --git a/projects/unsupported/verification-Larmor/verification-Larmor.cpp b/projects/unsupported/verification-Larmor/verification-Larmor.cpp deleted file mode 100644 index 259747017..000000000 --- a/projects/unsupported/verification-Larmor/verification-Larmor.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include - -#include "../../common.h" -#include "../../readparameters.h" -#include "../../backgroundfield/backgroundfield.h" - -#include "verificationLarmor.h" - -namespace projects { - verificationLarmor::verificationLarmor(): Project() { } - verificationLarmor::~verificationLarmor() { } - bool verificationLarmor::initialize(void) {return true;} - - void verificationLarmor::addParameters() { - typedef Readparameters RP; - RP::add("Larmor.BX0", "Background field value (T)", 0.0); - RP::add("Larmor.BY0", "Background field value (T)", 0.0); - RP::add("Larmor.BZ0", "Background field value (T)", 0.0); - RP::add("Larmor.VX0", "Bulk velocity in x", 0.0); - RP::add("Larmor.VY0", "Bulk velocity in y", 0.0); - RP::add("Larmor.VZ0", "Bulk velocity in z", 0.0); - RP::add("Larmor.X0", "Initial Position", 0.0); - RP::add("Larmor.Y0", "Initial Position", 0.0); - RP::add("Larmor.Z0", "Initial Position", 0.0); - RP::add("Larmor.rho", "Number density (m^-3)", 1.0e7); - } - - void verificationLarmor::getParameters() { - typedef Readparameters RP; - RP::get("Larmor.BX0", this->BX0); - RP::get("Larmor.BY0", this->BY0); - RP::get("Larmor.BZ0", this->BZ0); - RP::get("Larmor.VX0", this->VX0); - RP::get("Larmor.VY0", this->VY0); - RP::get("Larmor.VZ0", this->VZ0); - RP::get("Larmor.X0", this->X0); - RP::get("Larmor.Y0", this->Y0); - RP::get("Larmor.Z0", this->Z0); - RP::get("Larmor.rho", this->DENSITY); - } - - Real verificationLarmor::calcPhaseSpaceDensity(creal& x, creal& y, creal& z, creal& dx, creal& dy, creal& dz, creal& vx, creal& vy, creal& vz, creal& dvx, creal& dvy, creal& dvz) { - - static bool isSet=false; - //static variables should be threadprivate - #pragma omp threadprivate(isSet) - - if(vx < Parameters::vxmin + 0.5 * dvx || - vy < Parameters::vymin + 0.5 * dvy || - vz < Parameters::vzmin + 0.5 * dvz || - vx > Parameters::vxmax - 1.5 * dvx || - vy > Parameters::vymax - 1.5 * dvy || - vz > Parameters::vzmax - 1.5 * dvz - ) return 0.0; - - if(isSet) - return 0.0; //exactly one value to be set - - if( fabs(vx-this->VX0)VY0)VZ0)X0)Y0)Z0)DENSITY/(dvx*dvy*dvz); - } - - return 0.0; - - } - - - void verificationLarmor::calcCellParameters(Real* cellParams,creal& t) { - creal x = cellParams[CellParams::XCRD]; - creal dx = cellParams[CellParams::DX]; - creal y = cellParams[CellParams::YCRD]; - creal dy = cellParams[CellParams::DY]; - creal z = cellParams[CellParams::ZCRD]; - creal dz = cellParams[CellParams::DZ]; - - cellParams[CellParams::EX ] = 0.0; - cellParams[CellParams::EY ] = 0.0; - cellParams[CellParams::EZ ] = 0.0; - cellParams[CellParams::PERBX ] = 0.0; - cellParams[CellParams::PERBY ] = 0.0; - cellParams[CellParams::PERBZ ] = 0.0; - cellParams[CellParams::BGBX ] = this->BX0; - cellParams[CellParams::BGBY ] = this->BY0; - cellParams[CellParams::BGBZ ] = this->BZ0; - - } - -} //namespace projects diff --git a/projects/unsupported/verification-Larmor/verification-Larmor.h b/projects/unsupported/verification-Larmor/verification-Larmor.h deleted file mode 100644 index ead947df1..000000000 --- a/projects/unsupported/verification-Larmor/verification-Larmor.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * This file is part of Vlasiator. - * Copyright 2010-2016 Finnish Meteorological Institute - * - * For details of usage, see the COPYING file and read the "Rules of the Road" - * at http://www.physics.helsinki.fi/vlasiator/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef LARMOR_H -#define LARMOR_H - -#include "definitions.h" -#include "spatial_cell.hpp" -#include "projects/projects_common.h" -#include "projects/projects_vlasov_acceleration.h" - -#include "dccrg.hpp" - -struct larmorParameters { - static Real BX0; - static Real BY0; - static Real BZ0; - static Real VX0; - static Real VY0; - static Real VZ0; - static Real X0; - static Real Y0; - static Real Z0; - static Real DENSITY; - -} ; - -/** - * Initialize project. - */ -bool initializeProject(void); - -/** Register parameters that should be read in - */ -bool addProjectParameters(void); -/** Get the value that was read in - */ -bool getProjectParameters(void); - -/*!\brief Set the fields and distribution of a cell according to the default simulation settings. - * This is used for the NOT_SYSBOUNDARY cells and some other system boundary conditions (e.g. Outflow). - * \param cell Pointer to the cell to set. - */ -void setProjectCell(SpatialCell* cell); - -template void calcAccFaceX( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceX(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceY( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceY(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -template void calcAccFaceZ( - REAL& ax, REAL& ay, REAL& az, - const UINT& I, const UINT& J, const UINT& K, - const REAL* const cellParams, - const REAL* const blockParams, - const REAL* const cellBVOLDerivatives -) { - lorentzForceFaceZ(ax,ay,az,I,J,K,cellParams,blockParams,cellBVOLDerivatives); -} - -#endif diff --git a/projects/verificationLarmor/verificationLarmor.cfg b/projects/verificationLarmor/verificationLarmor.cfg index acda9868d..d6f26763d 100644 --- a/projects/verificationLarmor/verificationLarmor.cfg +++ b/projects/verificationLarmor/verificationLarmor.cfg @@ -1,3 +1,4 @@ +ParticlePopulations = proton propagate_field = 0 propagate_vlasov_acceleration = 1 propagate_vlasov_translation = 1 @@ -27,6 +28,12 @@ x_min = 0.0 x_max = 2.5e6 z_min = 0.0 z_max = 5.0e4 +timestep_max = 2000 + +#dt to give 1000steps per period +dt = 0.06559447074625 + +[proton_vspace] vx_min = -120000 vx_max = +120000 vy_min = -120000 @@ -36,10 +43,6 @@ vz_max = +120000 vx_length = 20 vy_length = 20 vz_length = 20 -timestep_max = 2000 - -#dt to give 1000steps per period -dt = 0.06559447074625 [boundaries] periodic_x = yes @@ -59,7 +62,7 @@ diagnostic = populations_vg_blocks algorithm = RCB tolerance = 1.05 -[sparse] +[proton_sparse] minValue = 1.0e-14 [VerificationLarmor] diff --git a/samples/IPshock/IPshock.cfg b/samples/IPshock/IPshock.cfg new file mode 100644 index 000000000..682a537a1 --- /dev/null +++ b/samples/IPshock/IPshock.cfg @@ -0,0 +1,177 @@ +# Sample cfg file for an interplanetary shock simulation +# +# The user has to determine shock conditions themselves, +# e.g. by using a Rankine-Hugoniot condition solver such as +# htts://github.com/fmihpc/analysator/scripts/obliqueshock.py +# +# The upstream to downstream transition is done using a +# smootherstep() profile. +# +# Note that the code assumes all relevant components to be +# in the X and Z components, Y components will be discarded. +# +# This example configuration uses a 2D box and low velocity +# resolution to run tens of seconds in a matter of minutes +# on 4 nodes on LUMI-C. + +project = IPShock +dynamic_timestep = 1 +propagate_field = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 + +hallMinimumRho = 1e3 + +ParticlePopulations = proton + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[io] +diagnostic_write_interval = 10 +#restart_write_t_interval = 15 +write_initial_state = 0 +restart_walltime_interval = 85000 #1500 +number_of_restarts = 1 +write_restart_stripe_factor = 5 + +system_write_t_interval = 5 +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 10 +system_write_distribution_yline_stride = 1 +system_write_distribution_zline_stride = 10 + +# The grid can easily be reduced to 1D along X +# or expanded to 3D along Y. +[gridbuilder] +x_length = 750 +y_length = 1 +z_length = 50 +x_min = -150e6 +x_max = 150e6 +y_min = -2e5 +y_max = 2e5 +z_min = -1e7 +z_max = 1e7 +t_max = 500 + +[proton_vspace] +vx_min = -8.04e6 +vx_max = +8.04e6 +vy_min = -8.04e6 +vy_max = +8.04e6 +vz_min = -8.04e6 +vz_max = +8.04e6 +vx_length = 67 +vy_length = 67 +vz_length = 67 + +[boundaries] +periodic_x = no +periodic_y = yes +periodic_z = yes +boundary = Outflow +boundary = Maxwellian + +[bailout] +min_dt = 1.e-7 + +[variables] +output = populations_vg_rho +output = populations_vg_ptensor +output = populations_vg_v +output = fg_e +output = vg_e_gradpe +output = fg_b +output = vg_b_vol +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +output = vg_f_saved +output = populations_vg_moments_nonthermal +output = populations_vg_moments_thermal +diagnostic = populations_vg_blocks + +[loadBalance] +rebalanceInterval = 25 + +[fieldsolver] +ohmHallTerm = 2 +minCFL = 0.4 +maxCFL = 0.45 +maxSubcycles = 50 + +# This set of options parameterises and activates the +# electron pressure gradient term of the electric field. +# At resolutions like htis example or finer it does start +# to be non-negligible relatively to the main E components. +electronTemperature = 0.5e6 # inflow +electronDensity = 1.0e6 #inflow +electronPTindex = 1.666667 # adiabatic +ohmGradPeTerm = 1 # active + + +[vlasovsolver] +minCFL = 0.8 +maxCFL = 0.99 +maxSlAccelerationRotation = 22 +maxSlAccelerationSubcycles = 2 + +[proton_sparse] +minValue = 1.e-16 +dynamicAlgorithm = 0 + +[outflow] +precedence = 3 + +# Note the use of None, preserving the initial state instead of +# using a copy/Neumann condition. +[proton_outflow] +face = x- +vlasovScheme_face_x- = None + +[maxwellian] +face = x+ +precedence = 2 + +[proton_maxwellian] +dynamic = 0 +file_x+ = upstream.dat + +[IPShock] +BX0u = 7.47e-9 +BY0u = 0.0 +BZ0u = 4.31e-9 +BX0d = 7.47e-09 +BY0d = 0.0 +BZ0d = 1.64749743e-08 +Width = 1.0e6 + +[proton_IPShock] +VX0u = -750000. +VY0u = 0 +VZ0u = -432730.92369478 +rhou = 1.0e+06 +Temperatureu = 500000. +VX0d = -222384.89635475 +VY0d = 0 +VZ0d = 490466.59202853 +rhod = 3372531.1938612536 +Temperatured = 11621882.668058667 + +maxwCutoff = 1.0e-16 + +# Parameterisation for the thermal vs. nonthermal data reducers. +# Set the origin and radius of a sphere in velocity-space coordinates (m/s) +# encompassing the upstream solar wind core. During the computation of +# moments, phase-space cells inside the radius centred on the (vx,vy,vz) +# below will be assumed "thermal" and cells outside the radius, "nonthermal". +# Note that this only yields sensible results upstream in the foreshock! +[proton_thermal] +vx = -750000. +vy = 0 +vz = -432730.92369478 +radius = 650000. diff --git a/samples/IPshock/upstream.dat b/samples/IPshock/upstream.dat new file mode 100644 index 000000000..ee58ca7a6 --- /dev/null +++ b/samples/IPshock/upstream.dat @@ -0,0 +1 @@ +0.0 1e+06 500000. -750000. 0 -432730.92369478 7.47e-9 0.0 4.31e-9 diff --git a/samples/KHB/KHB.cfg b/samples/KHB/KHB.cfg new file mode 100644 index 000000000..724f7b868 --- /dev/null +++ b/samples/KHB/KHB.cfg @@ -0,0 +1,153 @@ +# Sample Vlasiator configuration file +# Low-resolution local 2D simulation of the Kelvin-Helmholtz instability in transverse geometry (velocity shear perp. to magnetic field) +# This simulation can be run e.g., on 1 node with 2x AMD Rome 7H12 each (2x64 cores per node) +# In 10 minutes (1321 timesteps, 350 seconds of physical simulation) +# Using 10 GiB resident memory and 11 GiB high water mark memory + + + +# Simulation type, corresponding to a specific type of setup and dedicated class defined under projects/ in the source +project = KHB + +# Let the simulation adapt the time step according to the CFL condition +dynamic_timestep = 1 + +# Arbitrary name given to the simulated ion population. This project does not support multiple concurrent ion populations. +ParticlePopulations = proton + +# Specifications of the ion population +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +# Field solver options +[fieldsolver] +ohmHallTerm = 2 +maxSubcycles = 50 + +# Set of options shared by all projects +[Project_common] +# Seed used by a random number generator for setting random phases for the initial perturbation +seed = 124 + +# Set of options specifying conditions under which the run will exit gracefully to avoid potential crashing +[bailout] +max_memory = 250 # set based on system + +# Input/Output +[io] +diagnostic_write_interval = 10 +write_initial_state = 0 +#restart_walltime_interval = 9999999.0 +#number_of_restarts = 1 + +system_write_t_interval = 5.0 +system_write_file_name = bulk + +# With these set to zero, no velocity distribution functions are saved in the bulk file +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +# Spatial grid and run completion parameters +[gridbuilder] +x_length = 42 +y_length = 28 +z_length = 1 + +x_min = -10500.0E+3 +x_max = +10500.0E+3 +y_min = -7000.0E+3 +y_max = +7000.0E+3 +z_min = -250.0E+3 +z_max = +250.0E+3 + +timestep_max = 10000 +t_max = 350.0 + +# Velocity space parameters for the ion population declared at the top +[proton_vspace] +vx_min = -3000.0E+3 +vx_max = +3000.0E+3 +vy_min = -3000.0E+3 +vy_max = +3000.0E+3 +vz_min = -3000.0E+3 +vz_max = +3000.0E+3 +vx_length = 30 +vy_length = 30 +vz_length = 30 + +# Velocity sparsity parameters for the ion population declared at the top +[proton_sparse] +minValue = 1.0E-15 + +# Simulation boundary conditions; periodicity and declaration of boundary types that will be required +[boundaries] +periodic_x = no +periodic_y = yes +periodic_z = yes +boundary = Maxwellian + +# Uniform Maxwellian distribution on set faces of the simulation domain +[maxwellian] +face = x- +face = x+ + +# Files containing the Maxwellian properties at the set faces of the simulation domain +[proton_maxwellian] +dynamic = 0 +file_x- = mxm.dat +file_x+ = mxp.dat + +# Variables (data reduction operators) to be written to the output VLSV files and to the diagnostic.txt file +[variables] +output = populations_vg_rho +output = populations_vg_v +output = populations_vg_ptensor +output = fg_e +output = fg_b + +# Options specific to the project +[KHB] + +# width of the velocity shear layer as tanh(x/transitionWidth), in m +transitionWidth = 1000.0E+3 + +# amplitude of the initial perturbation, in m/s +amp = 3.5E+3 + +# wavelength of the initial perturbation, in m +lambda = 14000.0E+3 + +# number of harmonic modes in the initial perturbation, harmonics=0 includes only lambda, 1 includes lambda and lambda/2 etc. +harmonics = 0 + +# if true, set a random phase for each mode in the initial perturbation +randomPhase = 0 + +# if non-zero, include two velocity shear layers in the simulation domain displaced by +-offset m from x=0 +offset = 0.0 + +# total (thermal+magnetic) pressure across the domain, in Pa. Used to set a pressure balance across the domain +# /!\ to ensure a stable setup, verify that the pressure value gives the asymptotic temperature values set in mxm.dat/mxp.dat +# P = rho * kB * T + 0.5 * B * B / mu_0 +P = 2.229736594818919E-10 + +# ion number density, bulk velocity and magnetic field components at the -x (1) and +x (2) faces of the simulation domain +rho1 = 1.0E+6 +rho2 = 2.0E+6 +Vx1 = 0.0 +Vx2 = 0.0 +Vy1 = -187.5E+3 +Vy2 = +187.5E+3 +Vz1 = 0.0 +Vz2 = 0.0 +Bx1 = 0.0 +Bx2 = 0.0 +By1 = 0.0 +By2 = 0.0 +Bz1 = +12.0E-9 +Bz2 = +12.0E-9 + diff --git a/samples/KHB/mxm.dat b/samples/KHB/mxm.dat new file mode 100644 index 000000000..07fb74aae --- /dev/null +++ b/samples/KHB/mxm.dat @@ -0,0 +1,2 @@ +# time [s] / rho [m^-3] / T [K] / vx, vy, vz [m s^-1] / Bx, By, Bz [T] +0.0 1.0E+6 12.0E+6 0.0 -187.5E+3 0.0 0.0 0.0 +12.0E-9 diff --git a/samples/KHB/mxp.dat b/samples/KHB/mxp.dat new file mode 100644 index 000000000..8a298f20a --- /dev/null +++ b/samples/KHB/mxp.dat @@ -0,0 +1,2 @@ +# time [s] / rho [m^-3] / T [K] / vx, vy, vz [m s^-1] / Bx, By, Bz [T] +0.0 2.0E+6 6.0E+6 0.0 +187.5E+3 0.0 0.0 0.0 +12.0E-9 diff --git a/samples/Magnetosphere3D/Magnetosphere3D.cfg b/samples/Magnetosphere3D/Magnetosphere3D.cfg index 66a2a026e..443b81c1c 100644 --- a/samples/Magnetosphere3D/Magnetosphere3D.cfg +++ b/samples/Magnetosphere3D/Magnetosphere3D.cfg @@ -5,20 +5,30 @@ # Using 8 x 31 GiB resident memory and 8 x 36 GiB high water mark memory # Suitable for simple benchmarking of performance + +# After a handful of top-level options, remnaining options are grouped in sections prefixed by [group] in this file. + + +# Simulation type, corresponding to a specific type of setup and a dedicated class defined under projects/ in the source project = Magnetosphere + +# Let the simulation adapt the time step according to the CFL conditions dynamic_timestep = 1 + +# Arbitrary name given to at least one ion population. See [proton_properties] for its specification. +# all option groups specific to this population will be prefixed with that name, i.e. proton_* in this case. ParticlePopulations = proton +# Set of options specifying conditions under which the run will exit gracefully to avoid potential crashing. [bailout] max_memory = 58 # set based on system +# Adaptive Mesh Refinement [AMR] max_spatial_level = 3 - - refine_radius = 2e8 - +# Input/Output [io] diagnostic_write_interval = 10 write_initial_state = 0 @@ -29,7 +39,7 @@ write_initial_state = 0 # write_as_float = 1 system_write_t_interval = 10 -system_write_file_name = bulk10 +system_write_file_name = bulk system_write_distribution_stride = 0 system_write_distribution_xline_stride = 10 system_write_distribution_yline_stride = 10 @@ -38,7 +48,8 @@ system_write_distribution_zline_stride = 10 #system_write_distribution_shell_radius = 31e6 #system_write_distribution_shell_stride = 1 -[gridbuilder] # Highest resolution 2250 km cubed +# Spatial grid and run completion parameters +[gridbuilder] # Highest resolution 2250 km cubed (see [AMR]) x_length = 51 y_length = 40 z_length = 40 @@ -52,11 +63,13 @@ z_max = 3.6e8 #timestep_max = 175 t_max = 20.0 +# Specifications of ion population(s) declared at the top [proton_properties] mass = 1 mass_units = PROTON charge = 1 +# Velocity space parameters for ion population(s) declared at the top [proton_vspace] vx_min = -4.0e6 vx_max = +4.0e6 @@ -68,6 +81,7 @@ vx_length = 50 # in blocks of 4x4x4 v-space cells vy_length = 50 # gives 40 km/s vz_length = 50 +# Velocity space sparsity parameters for ion population(s) declared at the top [proton_sparse] minValue = 1.0e-15 # dynamicAlgorithm = 0 @@ -76,11 +90,11 @@ minValue = 1.0e-15 # dynamicMinValue1 = 1.0e-15 # dynamicMinValue2 = 1.0e-13 +# Options specific to the project [Magnetosphere] constBgBX = 0.0 constBgBY = 0.0 constBgBZ = -5.0e-9 -noDipoleInSW = 0 #dipole type 4, set to 0 (default) dipoleType = 4 dipoleTiltPhi = 0.0 @@ -103,14 +117,7 @@ refine_L3tailwidth = 8.0e7 refine_L3tailxmin = -30.0e7 refine_L3tailxmax = -5.0e7 -[copysphere] -centerX = 0.0 -centerY = 0.0 -centerZ = 0.0 -radius = 38.1e6 -precedence = 2 -reapplyUponRestart = 1 - +# Options specific to the project and the ion population [proton_Magnetosphere] T = 0.5e6 rho = 1.0e6 @@ -122,6 +129,46 @@ VZ0 = 0.0 taperInnerRadius = 5e7 taperOuterRadius = 1e8 +# Simulation boundary conditions; periodicity and declaration of the boundary types that will be required +[boundaries] +periodic_x = no +periodic_y = no +periodic_z = no +boundary = Outflow +boundary = Maxwellian +boundary = Copysphere + +# Outflow options +# Precedence sets the order of precedence for overlapping boundary types, should not be changed. +[outflow] +precedence = 3 + +[proton_outflow] +face = x- +face = y- +face = y+ +face = z- +face = z+ + +# Uniform Maxwellian distribution on set face(s) of the simulation domain +[maxwellian] +face = x+ +precedence = 4 +reapplyUponRestart = 1 + +[proton_maxwellian] # inflow file +dynamic = 0 +file_x+ = sw1.dat + +# Spherical inner boundary with static velocity distribution and copy-condition for perturbed magnetic field +[copysphere] +centerX = 0.0 +centerY = 0.0 +centerZ = 0.0 +radius = 38.1e6 +precedence = 2 +reapplyUponRestart = 1 + [proton_copysphere] # increased temperature and density close to inner boundary to add load balance challenge T = 1.5e6 @@ -130,11 +177,19 @@ VX0 = 0.0 VY0 = 0.0 VZ0 = 0.0 +# MPI domain load balancing options [loadBalance] -algorithm = RCB +algorithm = RCB # RIB or HYPERGRAPH may yield good performance too +# This times the approximate timestep length should be about the same or less than the fastest +# dynamics so that the computations are well-distributed across MPI ranks. rebalanceInterval = 50 +# Closer to 1.0 -> more even distribution -> but it takes longer to obtain the load balance from Zoltan tolerance = 1.2 +# Variables (data reduction operators) to be written to the output VLSV files and to the diagnostic.txt file +# fg_* variables on the finest-resolution field solver grid (/!\ can be several GB per component in large runs) +# vg_* variables on the adaptive mesh storing the plasma distribution and moments +# populations_* variables are stored separately for each specified ion population as /vg_* [variables] #output = fg_rank output = fg_b @@ -156,7 +211,6 @@ output = populations_vg_precipitationdifferentialflux #output = populations_vg_maxdt_translation #output = populations_vg_rho_loss_adjust output = vg_b_vol -output = vg_b_vol_derivatives output = vg_e_vol output = vg_e_gradpe output = vg_boundarytype @@ -167,41 +221,17 @@ diagnostic = populations_vg_blocks #diagnostic = populations_vg_rho #diagnostic = populations_vg_rho_loss_adjust +# Options for the population-specific energy density data reducer [proton_energydensity] solarwindspeed = 7.5e5 +# Options for the population-specific precipitation data reducer [proton_precipitation] -nChannels = 10 +nChannels = 9 emin = 500 # These are eV emax = 50000 -[boundaries] -periodic_x = no -periodic_y = no -periodic_z = no -boundary = Outflow -boundary = Maxwellian -boundary = Copysphere - -[outflow] -precedence = 3 - -[proton_outflow] -face = x- -face = y- -face = y+ -face = z- -face = z+ - -[maxwellian] -face = x+ -precedence = 4 -reapplyUponRestart = 1 - -[proton_maxwellian] # inflow file -dynamic = 0 -file_x+ = sw1.dat - +# Field solver options [fieldsolver] maxSubcycles = 50 ohmHallTerm = 2 @@ -214,9 +244,11 @@ electronDensity = 1.0e6 #inflow electronPTindex = 1.666667 # adiabatic ohmGradPeTerm = 1 # active +# Vlasov solver (translation and acceleration) options [vlasovsolver] minCFL = 0.8 maxCFL = 0.99 -maxSlAccelerationRotation = 22 +# The angle x number of subcycles should not exceed 45 deg +maxSlAccelerationRotation = 22 # in deg maxSlAccelerationSubcycles = 2 diff --git a/spatial_cell.cpp b/spatial_cell_cpu.cpp similarity index 99% rename from spatial_cell.cpp rename to spatial_cell_cpu.cpp index 20226b1af..b7f0f2c0d 100644 --- a/spatial_cell.cpp +++ b/spatial_cell_cpu.cpp @@ -22,7 +22,7 @@ #include -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "velocity_blocks.h" #include "object_wrapper.h" @@ -742,8 +742,8 @@ namespace spatial_cell { // Refinement parameters if ((SpatialCell::mpi_transfer_type & Transfer::REFINEMENT_PARAMETERS)){ - displacements.push_back(reinterpret_cast(this->parameters.data() + CellParams::AMR_ALPHA) - reinterpret_cast(this)); - block_lengths.push_back(sizeof(Real) * (CellParams::AMR_JPERB - CellParams::AMR_ALPHA + 1)); // This is just 2, but let's be explicit + displacements.push_back(reinterpret_cast(this->parameters.data() + CellParams::AMR_ALPHA1) - reinterpret_cast(this)); + block_lengths.push_back(sizeof(Real) * (CellParams::AMR_ALPHA2 - CellParams::AMR_ALPHA1 + 1)); // This is just 2, but let's be explicit } // Copy random number generator state variables @@ -1378,7 +1378,6 @@ namespace spatial_cell { * @return True on success.*/ bool SpatialCell::shrink_to_fit() { bool success = true; - return success; for (size_t p=0; p - -#include "../../definitions.h" -#include "../project.h" - -namespace projects { - class harm1D: public Project { - public: - harm1D(); - virtual ~harm1D(); - - virtual bool initialize(void); - static void addParameters(void); - virtual void getParameters(void); - - protected: - virtual void calcCellParameters(Real* cellParams,creal& t); - virtual Real calcPhaseSpaceDensity( - creal& x, creal& y, creal& z, - creal& dx, creal& dy, creal& dz, - creal& vx, creal& vy, creal& vz, - creal& dvx, creal& dvy, creal& dvz - ); - }; // class harm1D -} // namespace projects +// Can be used to include different variants of spatial cell class +#include "spatial_cell_cpu.hpp" #endif diff --git a/submodules/dccrg b/submodules/dccrg index 28e5d5bbd..cad44660c 160000 --- a/submodules/dccrg +++ b/submodules/dccrg @@ -1 +1 @@ -Subproject commit 28e5d5bbd7aa448d203d194ee7a0ca648141b4ff +Subproject commit cad44660c86a904ecf35061c92cc744e9ad4f37e diff --git a/submodules/eigen b/submodules/eigen new file mode 160000 index 000000000..3147391d9 --- /dev/null +++ b/submodules/eigen @@ -0,0 +1 @@ +Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/submodules/fsgrid b/submodules/fsgrid index fcc137671..ddaff3e01 160000 --- a/submodules/fsgrid +++ b/submodules/fsgrid @@ -1 +1 @@ -Subproject commit fcc1376712a706c5c819050acac55b35b9b5f291 +Subproject commit ddaff3e0129d3e82da484a60aaacc14c1cafd6b6 diff --git a/sysboundary/copysphere.cpp b/sysboundary/copysphere.cpp index b31ff210f..68f8e172c 100644 --- a/sysboundary/copysphere.cpp +++ b/sysboundary/copysphere.cpp @@ -737,7 +737,6 @@ namespace SBC { const uint popID, const bool calculate_V_moments ) { - phiprof::Timer timer {"vlasovBoundaryCondition (Copysphere)"}; this->vlasovBoundaryFluffyCopyFromAllCloseNbrs(mpiGrid, cellID, popID, calculate_V_moments, this->speciesParams[popID].fluffiness); } @@ -758,7 +757,6 @@ namespace SBC { // Loop over particle species for (uint popID=0; popIDspeciesParams[popID]; const vector blocksToInitialize = findBlocksToInitialize(templateCell,popID); Realf* data = templateCell.get_data(popID); diff --git a/sysboundary/copysphere.h b/sysboundary/copysphere.h index cf7d30499..8562b75de 100644 --- a/sysboundary/copysphere.h +++ b/sysboundary/copysphere.h @@ -26,7 +26,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" using namespace projects; diff --git a/sysboundary/donotcompute.cpp b/sysboundary/donotcompute.cpp index ed12a8806..6e7d55939 100644 --- a/sysboundary/donotcompute.cpp +++ b/sysboundary/donotcompute.cpp @@ -76,12 +76,6 @@ namespace SBC { cell->parameters[CellParams::VY_DT2] = 0.0; cell->parameters[CellParams::VZ_DT2] = 0.0; cell->parameters[CellParams::RHOQ_DT2] = 0.0; - - //let's get rid of blocks not fulfilling the criteria here to save - //memory. - for (uint popID=0; popIDadjustSingleCellVelocityBlocks(popID,true); - } } } diff --git a/sysboundary/donotcompute.h b/sysboundary/donotcompute.h index f9e73e1ff..ba46706c8 100644 --- a/sysboundary/donotcompute.h +++ b/sysboundary/donotcompute.h @@ -26,7 +26,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" using namespace projects; diff --git a/sysboundary/inflow.h b/sysboundary/inflow.h index 4033ddb88..55507d723 100644 --- a/sysboundary/inflow.h +++ b/sysboundary/inflow.h @@ -26,7 +26,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" namespace SBC { diff --git a/sysboundary/ionosphere.cpp b/sysboundary/ionosphere.cpp index a2ebfdebd..9a161c29b 100644 --- a/sysboundary/ionosphere.cpp +++ b/sysboundary/ionosphere.cpp @@ -882,7 +882,6 @@ namespace SBC { if(!refillTensorAtRestart) { // Ranks that don't participate in ionosphere solving skip this function outright if(!isCouplingInwards && !isCouplingOutwards) { - phiprof::stop("ionosphere-calculateConductivityTensor"); return; } @@ -3108,8 +3107,6 @@ namespace SBC { const uint popID, const bool calculate_V_moments ) { - phiprof::Timer timer {"vlasovBoundaryCondition (Ionosphere)"}; - // TODO Make this a more elegant solution // Now it's hacky as the counter is incremented in vlasiator.cpp if(globalflags::ionosphereJustSolved) { // else we don't update this boundary diff --git a/sysboundary/ionosphere.h b/sysboundary/ionosphere.h index ce3354250..867a52c0f 100644 --- a/sysboundary/ionosphere.h +++ b/sysboundary/ionosphere.h @@ -28,7 +28,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" #include "../backgroundfield/fieldfunction.hpp" #include "../fieldsolver/fs_common.h" diff --git a/sysboundary/outflow.h b/sysboundary/outflow.h index e4c5f8c71..a947d748b 100644 --- a/sysboundary/outflow.h +++ b/sysboundary/outflow.h @@ -26,7 +26,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" namespace SBC { diff --git a/sysboundary/setmaxwellian.h b/sysboundary/setmaxwellian.h index b26ce0df3..8df6031df 100644 --- a/sysboundary/setmaxwellian.h +++ b/sysboundary/setmaxwellian.h @@ -26,7 +26,7 @@ #include #include "../definitions.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "inflow.h" #include "sysboundarycondition.h" diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index 3bd7dbf56..17f4c145f 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -669,7 +669,7 @@ void SysBoundary::applySysBoundaryVlasovConditions( /*Transfer along boundaries*/ // First the small stuff without overlapping in an extended neighbourhood: -#warning TODO This now communicates in the wider neighbourhood for both layers, could be reduced to smaller neighbourhood for layer 1, larger neighbourhood for layer 2. +// TODO This now communicates in the wider neighbourhood for both layers, could be reduced to smaller neighbourhood for layer 1, larger neighbourhood for layer 2. SpatialCell::set_mpi_transfer_type(Transfer::CELL_PARAMETERS | Transfer::POP_METADATA | Transfer::CELL_SYSBOUNDARYFLAG, true); mpiGrid.update_copies_of_remote_neighbors(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID); @@ -695,10 +695,13 @@ void SysBoundary::applySysBoundaryVlasovConditions( cuint sysBoundaryType = mpiGrid[localCells[i]]->sysBoundaryFlag; this->getSysBoundary(sysBoundaryType)->vlasovBoundaryCondition(mpiGrid, localCells[i], popID, calculate_V_moments); } - if (calculate_V_moments) { - calculateMoments_V(mpiGrid, localCells, true); - } else { - calculateMoments_R(mpiGrid, localCells, true); + if (popID==getObjectWrapper().particleSpecies.size()-1) { + // Only calculate moments when handling last population + if (calculate_V_moments) { + calculateMoments_V(mpiGrid, localCells, true); + } else { + calculateMoments_R(mpiGrid, localCells, true); + } } computeInnerTimer.stop(); @@ -716,10 +719,13 @@ void SysBoundary::applySysBoundaryVlasovConditions( cuint sysBoundaryType = mpiGrid[boundaryCells[i]]->sysBoundaryFlag; this->getSysBoundary(sysBoundaryType)->vlasovBoundaryCondition(mpiGrid, boundaryCells[i], popID, calculate_V_moments); } - if (calculate_V_moments) { - calculateMoments_V(mpiGrid, boundaryCells, true); - } else { - calculateMoments_R(mpiGrid, boundaryCells, true); + if (popID==getObjectWrapper().particleSpecies.size()-1) { + // Only calculate moments when handling last population + if (calculate_V_moments) { + calculateMoments_V(mpiGrid, boundaryCells, true); + } else { + calculateMoments_R(mpiGrid, boundaryCells, true); + } } computeBoundaryTimer.stop(); diff --git a/sysboundary/sysboundary.h b/sysboundary/sysboundary.h index 35b9752c0..c268490d4 100644 --- a/sysboundary/sysboundary.h +++ b/sysboundary/sysboundary.h @@ -32,7 +32,7 @@ #include "../definitions.h" #include "../parameters.h" #include "../readparameters.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "sysboundarycondition.h" diff --git a/sysboundary/sysboundarycondition.h b/sysboundary/sysboundarycondition.h index 3055246fc..b06592e7d 100644 --- a/sysboundary/sysboundarycondition.h +++ b/sysboundary/sysboundarycondition.h @@ -30,7 +30,7 @@ #include #include "../common.h" #include "../definitions.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../projects/project.h" using namespace spatial_cell; diff --git a/testpackage/run_tests.sh b/testpackage/run_tests.sh index dec195415..67dbed40d 100755 --- a/testpackage/run_tests.sh +++ b/testpackage/run_tests.sh @@ -2,6 +2,11 @@ ## code, no need to touch ## ##--------------------------------------------------## +# define tab interval sequence so that we have aligned output +# this is now covering at least up proton/vg_ptensor_nonthermal_offdiagonal_0 and numbers printed at setprecision(3) with negative mantissa and exponent +tabseq="1,46,62,78,94,110" +tabs $tabseq &> /dev/null # suppress special character output, list matches expand below + ## add absolute paths to folder names, filenames reference_dir=$( readlink -f $reference_dir ) @@ -83,15 +88,15 @@ do ###copy new reference data to correct folder if [ $create_verification_files == 1 ] then - result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} - if [ -e $result_dir ] + reference_result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} + if [ -e $reference_result_dir ] then - echo "remove old results" - rm -rf $result_dir + echo "Removing previous reference results" + rm -rf $reference_result_dir fi - mkdir -p $result_dir - cp * $result_dir + mkdir -p $reference_result_dir + cp * $reference_result_dir fi cd $base_dir @@ -105,7 +110,7 @@ do echo "--------------------------------------------------------------------------------------------" echo "${test_name[$run]} - Verifying ${revision}_$solveropts against $reference_revision" echo "--------------------------------------------------------------------------------------------" - result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} + reference_result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} #print header @@ -113,9 +118,9 @@ do echo "------------------------------------------------------------" echo " ref-time | new-time | speedup |" echo "------------------------------------------------------------" - if [ -e ${result_dir}/${comparison_phiprof[$run]} ] + if [ -e ${reference_result_dir}/${comparison_phiprof[$run]} ] then - refPerf=$(grep "Propagate " ${result_dir}/${comparison_phiprof[$run]} |gawk '(NR==1){print $11}') + refPerf=$(grep "Propagate " ${reference_result_dir}/${comparison_phiprof[$run]} |gawk '(NR==1){print $11}') else refPerf="NA" fi @@ -127,44 +132,71 @@ do fi #print speedup if both refPerf and newPerf are numerical values speedup=$( echo $refPerf $newPerf |gawk '{if($2 == $2 + 0 && $1 == $1 + 0 ) print $1/$2; else print "NA"}') - echo "$refPerf $newPerf $speedup" + + tabs 1,14,33,59 &> /dev/null # match next line + echo -e " $refPerf\t| $newPerf\t| $speedup\t|" | expand -t 1,14,33,59 # match previous line echo "------------------------------------------------------------" - echo " variable | absolute diff | relative diff | " + tabs $tabseq &> /dev/null # reset for other printouts + echo -e " variable\t| absolute diff\t| relative diff |" | expand -t $tabseq # list matches tabs above echo "------------------------------------------------------------" variables=(${variable_names[$run]// / }) indices=(${variable_components[$run]// / }) - for i in ${!variables[*]} + for vlsv in ${comparison_vlsv[$run]} do - if [[ "${variables[$i]}" == "fg_"* ]] - then - relativeValue=$($run_command_tools $diffbin --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) -#print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - - elif [[ "${variables[$i]}" == "ig_"* ]] - then - relativeValue=$($run_command_tools $diffbin --meshname=ionosphere ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin --meshname=ionosphere ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) -#print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - - elif [ ! "${variables[$i]}" == "proton" ] - then - relativeValue=$($run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) -#print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - elif [ "${variables[$i]}" == "proton" ] - then + if [ ! -f "${vlsv_dir}/${vlsv}" ]; then + echo "Output file ${vlsv_dir}/${vlsv} not found!" echo "--------------------------------------------------------------------------------------------" - echo " Distribution function diff " + continue + fi + if [ ! -f "${reference_result_dir}/${vlsv}" ]; then + echo "Reference file ${reference_result_dir}/${vlsv} not found!" echo "--------------------------------------------------------------------------------------------" - $run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} proton 0 + continue fi - done # loop over variables - + echo "Comparing file ${vlsv_dir}/${vlsv} against reference" + for i in ${!variables[*]} + do + if [[ "${variables[$i]}" == "fg_"* ]] + then + A=$( $run_command_tools $diffbin --meshname=fsgrid ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + #print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq #list matches tabs above + elif [[ "${variables[$i]}" == "ig_"* ]] + then + A=$( $run_command_tools $diffbin --meshname=ionosphere ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + #print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq # list matches tabs above + elif [ ! "${variables[$i]}" == "proton" ] + then # Regular vg_ variable + A=$( $run_command_tools $diffbin ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + #print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq # list matches tabs above + elif [ "${variables[$i]}" == "proton" ] + then + echo "--------------------------------------------------------------------------------------------" + echo " Distribution function diff " + echo "--------------------------------------------------------------------------------------------" + $run_command_tools $diffbin ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} proton 0 + fi + done # loop over variables + + # Print also time difference, if it is not zero + timeDiff=$(grep "delta t" <<< $A |gawk '{print $8}' ) + if (( $(awk 'BEGIN{print ('$timeDiff'!= 0.0)?1:0}') )) + then + echo "WARNING! VLSV file timestamps differ by ${timeDiff}s." + else + echo "VLSV file timestamps match." + fi + echo "--------------------------------------------------------------------------------------------" + done # loop over vlsv files to compare echo "--------------------------------------------------------------------------------------------" fi done # loop over tests diff --git a/testpackage/small_test_carrington.sh b/testpackage/small_test_carrington.sh index 4337e7d98..ccca6391f 100755 --- a/testpackage/small_test_carrington.sh +++ b/testpackage/small_test_carrington.sh @@ -65,7 +65,7 @@ umask 007 echo "Running $exec on $tasks mpi tasks, with $t threads per task on $nodes nodes ($ht threads per physical core)" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/small_test_carrington_github_ci.sh b/testpackage/small_test_carrington_github_ci.sh index 740caff5c..09d02d5b5 100755 --- a/testpackage/small_test_carrington_github_ci.sh +++ b/testpackage/small_test_carrington_github_ci.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -t 01:30:00 # Run time (hh:mm:ss) #SBATCH --job-name=CI_testpackage -##SBATCH -A spacephysics +##SBATCH -A spacephysics #SBATCH -M carrington # test short medium 20min1d 3d #SBATCH -p short @@ -16,7 +16,7 @@ # if 0 then we check the v1 create_verification_files=0 -# folder for all reference data +# folder for all reference data reference_dir="/proj/group/spacephysics/vlasiator_testpackage/" cd $SLURM_SUBMIT_DIR #cd $reference_dir # don't run on /proj @@ -40,6 +40,9 @@ module load OpenMPI/4.1.1-GCC-11.2.0 module load PMIx/4.1.0-GCCcore-11.2.0 module load PAPI/6.0.0.1-GCCcore-11.2.0 +# send JOB ID to output usable by CI eg to scancel this job +echo "SLURM_JOB_ID=$SLURM_JOB_ID" >> $GITHUB_OUTPUT + #-------------------------------------------------------------------- #---------------------DO NOT TOUCH----------------------------------- nodes=$SLURM_NNODES @@ -54,10 +57,13 @@ tasks=$(echo $total_units $t | gawk '{print $1/$2}') tasks_per_node=$(echo $units_per_node $t | gawk '{print $1/$2}') export OMP_NUM_THREADS=$t +# With this the code won't print the warning, so we have a shorter report +export OMPI_MCA_io="^ompio" + #command for running stuff run_command="mpirun --mca btl self -mca pml ^vader,tcp,openib,uct,yalla -x UCX_NET_DEVICES=mlx5_0:1 -x UCX_TLS=rc,sm -x UCX_IB_ADDR_TYPE=ib_global -np $tasks" small_run_command="mpirun --mca btl self -mca pml ^vader,tcp,openib,uct,yalla -x UCX_NET_DEVICES=mlx5_0:1 -x UCX_TLS=rc,sm -x UCX_IB_ADDR_TYPE=ib_global -n 1 -N 1" -run_command_tools="srun --mpi=pmix_v3 -n 1 " +run_command_tools="mpirun -np 1 " umask 007 # Launch the OpenMP job to the allocated compute node @@ -67,7 +73,7 @@ echo "Running $exec on $tasks mpi tasks, with $t threads per task on $nodes node hostname # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait @@ -76,6 +82,11 @@ if [ $create_verification_files == 1 ]; then exit 1 fi +# define tab interval sequence so that we have aligned output +# this is now covering at least up proton/vg_ptensor_nonthermal_offdiagonal_0 and numbers printed at setprecision(3) with negative mantissa and exponent +tabseq="1,46,62,78,94,110" +tabs $tabseq &> /dev/null # suppress special character output, list matches expand below + # Note we are *not* using run_tests.sh here, as we are creating JUnit XML output. # Get absolute paths @@ -90,6 +101,10 @@ flags=$( $run_command $bin --version |grep CXXFLAGS) solveropts=$(echo $flags|sed 's/[-+]//g' | gawk '{for(i = 1;i<=NF;i++) { if( $i=="DDP" || $i=="DFP" || index($i,"PF")|| index($i,"DVEC") || index($i,"SEMILAG") ) printf "__%s", $(i) }}') revision=$( $run_command $bin --version |gawk '{if(flag==1) {print $1;flag=0}if ($3=="log") flag=1;}' ) +echo "----------" +echo "This will be verifying ${revision}_$solveropts against $reference_revision" +echo "----------" + #$small_run_command $bin --version > VERSION.txt 2> $GITHUB_WORKSPACE/stderr.txt echo -e "### Testpackage output:\n" >> $GITHUB_STEP_SUMMARY @@ -101,7 +116,6 @@ FAILEDTESTS=0 # loop over different test cases for run in ${run_tests[*]}; do - echo "running ${test_name[$run]} " # directory for test results vlsv_dir=${run_dir}/${test_name[$run]} cfg_dir=${test_dir}/${test_name[$run]} @@ -130,10 +144,6 @@ for run in ${run_tests[*]}; do # Store error return value RUN_ERROR=${PIPESTATUS[0]} - # Fore set to error if output file does not exist - if [ ! -f ${vlsv_dir}/${comparison_vlsv[$run]} ]; then - RUN_ERROR=1 - fi if [[ $RUN_ERROR != 0 ]]; then echo -e "
:red_circle: ${test_name[$run]}: Failed to run or died with an error.\n" >> $GITHUB_STEP_SUMMARY @@ -153,19 +163,20 @@ for run in ${run_tests[*]}; do ##Compare test case with right solutions { { - echo "--------------------------------------------------------------------------------------------" - echo "${test_name[$run]} - Verifying ${revision}_$solveropts against $reference_revision" - echo "--------------------------------------------------------------------------------------------" + echo -e "\n" + echo "----------" + echo "running ${test_name[$run]} " + echo "----------" } 2>&1 1>&3 3>&- | tee -a $GITHUB_WORKSPACE/stderr.txt;} 3>&1 1>&2 | tee -a $GITHUB_WORKSPACE/stdout.txt - result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} + reference_result_dir=${reference_dir}/${reference_revision}/${test_name[$run]} { { - echo "------------------------------------------------------------" - echo " ref-time | new-time | speedup |" - echo "------------------------------------------------------------" + echo "--------------------------------" + echo " ref-time | new-time | speedup |" + echo "--------------------------------" } 2>&1 1>&3 3>&- | tee -a $GITHUB_WORKSPACE/stderr.txt;} 3>&1 1>&2 | tee -a $GITHUB_WORKSPACE/stdout.txt - if [ -e ${result_dir}/${comparison_phiprof[$run]} ]; then - refPerf=$(grep "Propagate " ${result_dir}/${comparison_phiprof[$run]} | gawk '(NR==1){print $11}') + if [ -e ${reference_result_dir}/${comparison_phiprof[$run]} ]; then + refPerf=$(grep "Propagate " ${reference_result_dir}/${comparison_phiprof[$run]} | gawk '(NR==1){print $11}') else refPerf="NA" fi @@ -174,122 +185,184 @@ for run in ${run_tests[*]}; do else newPerf="NA" fi - + #print speedup if both refPerf and newPerf are numerical values speedup=$( echo $refPerf $newPerf |gawk '{if($2 == $2 + 0 && $1 == $1 + 0 ) print $1/$2; else print "NA"}') { { - echo "$refPerf $newPerf $speedup" - echo "------------------------------------------------------------" - echo " variable | absolute diff | relative diff | " - echo "------------------------------------------------------------" + tabs 1,12,23 &> /dev/null # match next line + echo -e " $refPerf\t$newPerf\t$speedup" | expand -t 1,12,23 # match previous line + echo "-------------------------------------------" + tabs $tabseq &> /dev/null # reset for other printouts + echo -e " variable\t| absolute diff\t| relative diff |" | expand -t $tabseq # list matches tabs above + echo "-------------------------------------------" } 2>&1 1>&3 3>&- | tee -a $GITHUB_WORKSPACE/stderr.txt;} 3>&1 1>&2 | tee -a $GITHUB_WORKSPACE/stdout.txt { MAXERR=0. # Absolute error MAXREL=0. # Relative error + MAXDT=0. # Output time difference MAXERRVAR="" # Variable with max absolute error MAXRELVAR="" # Variable with max relative error - + COMPAREDFILES=0 # How many files we successfully compare + TOCOMPAREFILES=0 # How many files we were supposed to compare + # Save initial values in case we call continue in the loops below + echo $COMPAREDFILES > $RUNNER_TEMP/COMPAREDFILES.txt + echo $TOCOMPAREFILES > $RUNNER_TEMP/TOCOMPAREFILES.txt variables=(${variable_names[$run]// / }) indices=(${variable_components[$run]// / }) - for i in ${!variables[*]} + for vlsv in ${comparison_vlsv[$run]} do - if [[ "${variables[$i]}" == "fg_"* ]] - then - relativeValue=$($run_command_tools $diffbin --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) - #print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - - # Also log to metrics file - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt - - # Check if we have a new maximum error - if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXERR=$absoluteValue - MAXERRVAR=${variables[$i]} - fi - # ... or new max relative error - if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXREL=$relativeValue - MAXRELVAR=${variables[$i]} - fi - - elif [[ "${variables[$i]}" == "ig_"* ]] - then - relativeValue=$($run_command_tools $diffbin --meshname=ionosphere ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin --meshname=ionosphere ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) - # print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - - # Also log to metrics file - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt - - # Check if we have a new maximum error - if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXERR=$absoluteValue - MAXERRVAR=${variables[$i]} - fi - # ... or new max relative error - if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXREL=$relativeValue - MAXRELVAR=${variables[$i]} - fi - - elif [ ! "${variables[$i]}" == "proton" ] - then - relativeValue=$($run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) - absoluteValue=$($run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) - #print the results - echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " - - # Also log to metrics file - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt - echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt - - # Check if we have a new maximum error - if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXERR=$absoluteValue - MAXERRVAR=${variables[$i]} - fi - # ... or new max relative error - if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - MAXREL=$relativeValue - MAXRELVAR=${variables[$i]} - fi - - elif [ "${variables[$i]}" == "proton" ] - then - echo "--------------------------------------------------------------------------------------------" - echo " Distribution function diff " - echo "--------------------------------------------------------------------------------------------" - $run_command_tools $diffbin ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} proton 0 + TOCOMPAREFILES=$((TOCOMPAREFILES+1)) + echo $TOCOMPAREFILES > $RUNNER_TEMP/TOCOMPAREFILES.txt + if [ ! -f "${vlsv_dir}/${vlsv}" ]; then + echo "Output file ${vlsv_dir}/${vlsv} not found!" + echo "----------" + continue fi + if [ ! -f "${reference_result_dir}/${vlsv}" ]; then + echo "Reference file ${reference_result_dir}/${vlsv} not found!" + echo "----------" + continue + fi + echo "Comparing file ${vlsv_dir}/${vlsv} against reference" + COMPAREDFILES=$((COMPAREDFILES+1)) + echo $COMPAREDFILES > $RUNNER_TEMP/COMPAREDFILES.txt + + for i in ${!variables[*]} + do + if [[ "${variables[$i]}" == "fg_"* ]] + then + A=$( $run_command_tools $diffbin --meshname=fsgrid ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + #print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq #list matches tabs above + + # Also log to metrics file + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt + + # Check if we have a new maximum error + if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXERR=$absoluteValue + MAXERRVAR=${variables[$i]} + fi + # ... or new max relative error + if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXREL=$relativeValue + MAXRELVAR=${variables[$i]} + fi + + elif [[ "${variables[$i]}" == "ig_"* ]] + then + A=$( $run_command_tools $diffbin --meshname=ionosphere ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + # print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq # list matches tabs above + + # Also log to metrics file + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt + + # Check if we have a new maximum error + if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXERR=$absoluteValue + MAXERRVAR=${variables[$i]} + fi + # ... or new max relative error + if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXREL=$relativeValue + MAXRELVAR=${variables[$i]} + fi + + elif [ ! "${variables[$i]}" == "proton" ] + then # Regular vg_ variable + A=$( $run_command_tools $diffbin ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} ${variables[$i]} ${indices[$i]} ) + relativeValue=$(grep "The relative 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + absoluteValue=$(grep "The absolute 0-distance between both datasets" <<< $A |gawk '{print $8}' ) + #print the results + echo -e " ${variables[$i]}_${indices[$i]}\t ${absoluteValue}\t ${relativeValue}" | expand -t $tabseq # list matches tabs above + + # Also log to metrics file + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"absolute\"} $absoluteValue" >> $GITHUB_WORKSPACE/metrics.txt + echo "test_carrington{test=\"${test_name[$run]}\",var=\"${variables[$i]}\",index=\"${indices[$i]}\",diff=\"relative\"} $relativeValue" >> $GITHUB_WORKSPACE/metrics.txt + + # Check if we have a new maximum error + if (( $( echo "$absoluteValue $MAXERR" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXERR=$absoluteValue + MAXERRVAR=${variables[$i]} + fi + # ... or new max relative error + if (( $( echo "$relativeValue $MAXREL" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXREL=$relativeValue + MAXRELVAR=${variables[$i]} + fi + + elif [ "${variables[$i]}" == "proton" ] + then + echo "----------" + echo "Distribution function diff" + echo "----------" + $run_command_tools $diffbin ${reference_result_dir}/${vlsv} ${vlsv_dir}/${vlsv} proton 0 + fi + + done # loop over variables + + # Check if dt is nonzero + timeDiff=$(grep "delta t" <<< $A |gawk '{print $8}' ) + if (( $(awk 'BEGIN{print ('$timeDiff'!= 0.0)?1:0}') )); then + if (( $( echo "${timeDiff#-} $MAXDT" | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + MAXDT=$timeDiff + fi + echo "WARNING! VLSV file timestamps differ by ${timeDiff}s." + else + echo "VLSV file timestamps match." + fi + echo "----------" # This loop runs in a subshell (because of the stdout and stderr capture below), # so we save the variables to temp files echo $MAXERR > $RUNNER_TEMP/MAXERR.txt echo $MAXREL > $RUNNER_TEMP/MAXREL.txt + echo $MAXDT > $RUNNER_TEMP/MAXDT.txt echo $MAXERRVAR > $RUNNER_TEMP/MAXERRVAR.txt echo $MAXRELVAR > $RUNNER_TEMP/MAXRELVAR.txt echo $speedup > $RUNNER_TEMP/speedup.txt + echo $COMPAREDFILES > $RUNNER_TEMP/COMPAREDFILES.txt + echo $TOCOMPAREFILES > $RUNNER_TEMP/TOCOMPAREFILES.txt + done 2>&1 1>&3 3>&- | tee -a $GITHUB_WORKSPACE/stderr.txt; } 3>&1 1>&2 | tee -a $GITHUB_WORKSPACE/stdout.txt + # end loop over vlsvfiles # Recover error variables - MAXERR=`cat $RUNNER_TEMP/MAXERR.txt` - MAXERRVAR=`cat $RUNNER_TEMP/MAXERRVAR.txt` - MAXREL=`cat $RUNNER_TEMP/MAXREL.txt` - MAXRELVAR=`cat $RUNNER_TEMP/MAXRELVAR.txt` - speedup=`cat $RUNNER_TEMP/speedup.txt` + COMPAREDFILES=`cat $RUNNER_TEMP/COMPAREDFILES.txt` + TOCOMPAREFILES=`cat $RUNNER_TEMP/TOCOMPAREFILES.txt` + if [[ $COMPAREDFILES -eq 0 ]]; then + MAXERR=-42 + MAXERRVAR="n/a" + MAXREL=42 + MAXRELVAR="n/a" + MAXDT=0 + speedup=0 + else + MAXERR=`cat $RUNNER_TEMP/MAXERR.txt` + MAXERRVAR=`cat $RUNNER_TEMP/MAXERRVAR.txt` + MAXREL=`cat $RUNNER_TEMP/MAXREL.txt` + MAXRELVAR=`cat $RUNNER_TEMP/MAXRELVAR.txt` + MAXDT=`cat $RUNNER_TEMP/MAXDT.txt` + speedup=`cat $RUNNER_TEMP/speedup.txt` + fi # Output CI step annotation - if (( $( echo "$MAXERR 0." | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then - echo -e "
:large_orange_diamond: ${test_name[$run]}: Nonzero diffs: : \`$MAXERRVAR\` has absolute error $MAXERR, \`$MAXRELVAR\` has relative error $MAXREL. Speedup: $speedup\n" >> $GITHUB_STEP_SUMMARY + if [[ $COMPAREDFILES -ne $TOCOMPAREFILES ]]; then + echo -e "
:red_square: ${test_name[$run]}: Comparison failure, accessed \`$COMPAREDFILES\` out of \`$TOCOMPAREFILES\` files: \`$MAXERRVAR\` has absolute error $MAXERR, \`$MAXRELVAR\` has relative error $MAXREL. Max timestamp difference is $MAXDT. Speedup: $speedup\n" >> $GITHUB_STEP_SUMMARY + NONZEROTESTS=$((NONZEROTESTS+1)) + elif (( $( echo "$MAXERR 0." | awk '{ if($1 > $2) print 1; else print 0 }' ) )) || (( $( echo "$MAXDT 0." | awk '{ if($1 > $2) print 1; else print 0 }' ) )); then + echo -e "
:large_orange_diamond: ${test_name[$run]}: Nonzero diffs: \`$MAXERRVAR\` has absolute error $MAXERR, \`$MAXRELVAR\` has relative error $MAXREL. Max timestamp difference is $MAXDT. Speedup: $speedup\n" >> $GITHUB_STEP_SUMMARY NONZEROTESTS=$((NONZEROTESTS+1)) - else + else echo -e "
:heavy_check_mark: ${test_name[$run]}: Ran with zero diffs. Speedup: $speedup\n" >> $GITHUB_STEP_SUMMARY ZEROTESTS=$((ZEROTESTS+1)) fi @@ -353,4 +426,3 @@ elif [[ $NONZEROTESTS > 0 ]]; then else echo "conclusion=success" >> $GITHUB_WORKSPACE/testpackage_output_variables.txt fi - diff --git a/testpackage/small_test_definitions.sh b/testpackage/small_test_definitions.sh deleted file mode 100644 index a71e329e3..000000000 --- a/testpackage/small_test_definitions.sh +++ /dev/null @@ -1,155 +0,0 @@ - -## Define test and runs - -# Assume binaries are set in job script -#bin=vlasiator -#diffbin=vlsvdiff_DP - -if [ ! -f $bin ] -then - echo Executable $bin does not exist - exit -fi - -# where the tests are run -run_dir="run" - -# where the directories for different tests, including cfg and other needed data files are located -test_dir="tests" - -# choose tests to run -run_tests=( 1 2 3 4 5 6 7 8 9 10 11 12 13 14 16 17 18 19 ) - -# acceleration test -test_name[1]="acctest_2_maxw_500k_100k_20kms_10deg" -comparison_vlsv[1]="fullf.0000001.vlsv" -comparison_phiprof[1]="phiprof_0.txt" -variable_names[1]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" -variable_components[1]="0 0 1 2" -single_cell[1]=1 - -# acceleration test w/ substepping -test_name[2]="acctest_3_substeps" -comparison_vlsv[2]="fullf.0000001.vlsv" -comparison_phiprof[2]="phiprof_0.txt" -variable_names[2]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" -variable_components[2]="0 0 1 2" -single_cell[2]=1 - -# translation test -test_name[3]="transtest_2_maxw_500k_100k_20kms_20x20" -comparison_vlsv[3]="fullf.0000001.vlsv" -comparison_phiprof[3]="phiprof_0.txt" -variable_names[3]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" -variable_components[3]="0 0 1 2" - -test_name[4]="acctest_4_helium" -comparison_vlsv[4]="fullf.0000001.vlsv" -comparison_phiprof[4]="phiprof_0.txt" -variable_names[4]="helium/vg_rho helium/vg_v helium/vg_v helium/vg_v" -variable_components[4]="0 0 1 2" -single_cell[4]=1 - -# Gyration test with protons and antiprotons -test_name[5]="acctest_5_proton_antiproton" -comparison_vlsv[5]="fullf.0000001.vlsv" -comparison_phiprof[5]="phiprof_0.txt" -variable_names[5]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" -variable_components[5]="0 0 1 2" -single_cell[5]=1 - -# Restart tests. Writing and reading -test_name[6]="restart_write" -comparison_vlsv[6]="bulk.0000001.vlsv" -comparison_phiprof[6]="phiprof_0.txt" -variable_names[6]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[6]="0 0 1 2 0 1 2 0 1 2" -test_name[7]="restart_read" -comparison_vlsv[7]="initial-grid.0000000.vlsv" -comparison_phiprof[7]="phiprof_0.txt" -variable_names[7]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[7]="0 0 1 2 0 1 2 0 1 2" - -#Very small ecliptic magnetosphere, no subcycling in ACC or FS -test_name[8]="Magnetosphere_small" -comparison_vlsv[8]="bulk.0000001.vlsv" -comparison_phiprof[8]="phiprof_0.txt" -variable_names[8]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton" -variable_components[8]="0 0 1 2 0 1 2 0 1 2" - -#Very small polar magnetosphere, with subcycling in ACC or FS -test_name[9]="Magnetosphere_polar_small" -comparison_vlsv[9]="bulk.0000001.vlsv" -comparison_phiprof[9]="phiprof_0.txt" -variable_names[9]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton/vg_v_nonthermal proton/vg_v_nonthermal proton/vg_v_nonthermal proton/vg_ptensor_nonthermal_diagonal proton/vg_ptensor_nonthermal_diagonal proton/vg_ptensor_nonthermal_diagonal proton" -variable_components[9]="0 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2" - -# Field solver test -test_name[10]="test_fp_fsolver_only_3D" -comparison_vlsv[10]="fullf.0000001.vlsv" -comparison_phiprof[10]="phiprof_0.txt" -variable_names[10]="fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[10]="0 1 2 0 1 2" - -# Field solver test w/ subcycles -test_name[11]="test_fp_substeps" -comparison_vlsv[11]="fullf.0000001.vlsv" -comparison_phiprof[11]="phiprof_0.txt" -variable_names[11]="fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[11]="0 1 2 0 1 2" - -# Flowthrough tests -test_name[12]="Flowthrough_trans_periodic" -comparison_vlsv[12]="bulk.0000001.vlsv" -comparison_phiprof[12]="phiprof_0.txt" -variable_names[12]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[12]="0 0 1 2 0 1 2 0 1 2" - -test_name[13]="Flowthrough_x_inflow_y_outflow" -comparison_vlsv[13]="bulk.0000001.vlsv" -comparison_phiprof[13]="phiprof_0.txt" -variable_names[13]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[13]="0 0 1 2 0 1 2 0 1 2" - -test_name[14]="Flowthrough_x_inflow_y_outflow_acc" -comparison_vlsv[14]="bulk.0000001.vlsv" -comparison_phiprof[14]="phiprof_0.txt" -variable_names[14]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[14]="0 0 1 2 0 1 2 0 1 2" - -# Self-consistent wave generation test -test_name[15]="Selfgen_Waves_Periodic" -comparison_vlsv[15]="fullf.0000001.vlsv" -comparison_phiprof[15]="phiprof_0.txt" -variable_names[15]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton" -variable_components[15]="0 0 1 2 0 1 2 0 1 2" - -## Spatial AMR tests -# Flowthrough test -test_name[16]="Flowthrough_amr" -comparison_vlsv[16]="bulk.0000001.vlsv" -comparison_phiprof[16]="phiprof_0.txt" -variable_names[16]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[16]="0 0 1 2 0 1 2 0 1 2" - -# Magnetosphere 3D -test_name[17]="Magnetosphere_3D_small" -comparison_vlsv[17]="bulk.0000001.vlsv" -comparison_phiprof[17]="phiprof_0.txt" -variable_names[17]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[17]="0 0 1 2 0 1 2 0 1 2" - -# Ionosphere 3D -test_name[18]="Ionosphere_small" -comparison_vlsv[18]="bulk.0000001.vlsv" -comparison_phiprof[18]="phiprof_0.txt" -variable_names[18]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e ig_upmappedarea ig_fac ig_rhon ig_potential" -variable_components[18]="0 0 1 2 0 1 2 0 1 2 0 0 0 0" - -# Flowthrough with timevarying inflow -test_name[19]="Flowthrough_1D_timevarying" -comparison_vlsv[19]="bulk.0000001.vlsv" -comparison_phiprof[19]="phiprof_0.txt" -variable_names[19]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[19]="0 0 1 2 0 1 2 0 1 2 0 0" -single_cell[19]=1 diff --git a/testpackage/small_test_docker.sh b/testpackage/small_test_docker.sh index ad4997453..522848666 100755 --- a/testpackage/small_test_docker.sh +++ b/testpackage/small_test_docker.sh @@ -33,7 +33,7 @@ reference_dir="/home/vlasiator/testpackage/" reference_revision="current" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/small_test_lumi.sh b/testpackage/small_test_lumi.sh index 2089d63b5..55ef0fe45 100755 --- a/testpackage/small_test_lumi.sh +++ b/testpackage/small_test_lumi.sh @@ -58,7 +58,7 @@ umask 007 echo "Running $exec on $tasks mpi tasks, with $t threads per task on $nodes nodes ($ht threads per physical core)" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/small_test_mahti.sh b/testpackage/small_test_mahti.sh index 2f0c9a61b..68cd1e9b3 100755 --- a/testpackage/small_test_mahti.sh +++ b/testpackage/small_test_mahti.sh @@ -63,7 +63,7 @@ reference_revision="current" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/small_test_puhti.sh b/testpackage/small_test_puhti.sh index 279712e67..3af8fdec9 100755 --- a/testpackage/small_test_puhti.sh +++ b/testpackage/small_test_puhti.sh @@ -63,7 +63,7 @@ reference_revision="current" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/small_test_vorna.sh b/testpackage/small_test_vorna.sh index 7eee3ebc0..30f3970a0 100755 --- a/testpackage/small_test_vorna.sh +++ b/testpackage/small_test_vorna.sh @@ -58,7 +58,7 @@ umask 007 echo "Running $exec on $tasks mpi tasks, with $t threads per task on $nodes nodes ($ht threads per physical core)" # Define test -source small_test_definitions.sh +source test_definitions_small.sh wait # Run tests source run_tests.sh diff --git a/testpackage/test_definitions_small.sh b/testpackage/test_definitions_small.sh new file mode 100644 index 000000000..8bf8e7c52 --- /dev/null +++ b/testpackage/test_definitions_small.sh @@ -0,0 +1,262 @@ + +## Define test and runs + +# Assume binaries are set in job script +#bin=vlasiator +#diffbin=vlsvdiff_DP + +if [ ! -f $bin ] +then + echo Executable $bin does not exist + exit +fi + +# where the tests are run +run_dir="run" + +# where the directories for different tests, including cfg and other needed data files are located +test_dir="tests" + +# Counter for creating tests +index=1 + +####### +# ACCELERATION TESTS (1..5) +####### + +# 1 basic multipeak acceleration test (fixed timestep) +test_name[${index}]="acctest_1_maxw_500k_100k_20kms_10deg" +comparison_vlsv[${index}]="fullf.0000000.vlsv fullf.0000001.vlsv fullf.0000002.vlsv fullf.0000020.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" +variable_components[${index}]="0 0 1 2" +single_cell[${index}]=1 +((index+=1)) + +# 2 basic multipeak acceleration test (dynamic timestep) +test_name[${index}]="acctest_2_maxw_500k_100k_20kms_10deg" +comparison_vlsv[${index}]="fullf.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" +variable_components[${index}]="0 0 1 2" +single_cell[${index}]=1 +((index+=1)) + +# 3 acceleration test w/ substepping +test_name[${index}]="acctest_3_substeps" +comparison_vlsv[${index}]="fullf.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" +variable_components[${index}]="0 0 1 2" +single_cell[${index}]=1 +((index+=1)) + +# 4 Helium acceleration +test_name[${index}]="acctest_4_helium" +comparison_vlsv[${index}]="fullf.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="helium/vg_rho helium/vg_v helium/vg_v helium/vg_v" +variable_components[${index}]="0 0 1 2" +single_cell[${index}]=1 +((index+=1)) + +# 5 Gyration test with protons and antiprotons (multipop) +test_name[${index}]="acctest_5_proton_antiproton" +comparison_vlsv[${index}]="fullf.0000000.vlsv fullf.0000001.vlsv fullf.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" +variable_components[${index}]="0 0 1 2" +single_cell[${index}]=1 +((index+=1)) + +####### +# 1D/2D TRANSLATION TESTS (6..9) +####### + +# 6 Flowthrough tests (no boundaries, includes empty cells) +test_name[${index}]="Flowthrough_trans_periodic" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv bulk.0000003.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 7 Inflow and outflow boundaries +test_name[${index}]="Flowthrough_x_inflow_y_outflow" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv bulk.0000003.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 8 Inflow and outflow boundaries together with acceleration +test_name[${index}]="Flowthrough_x_inflow_y_outflow_acc" +comparison_vlsv[${index}]="bulk.0000002.vlsv bulk.0000003.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 9 Flowthrough with timevarying inflow +test_name[${index}]="Flowthrough_1D_timevarying" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 0" +single_cell[${index}]=1 +((index+=1)) + +####### +# 3D AMR TRANSLATION TESTS (10..13) +####### + +# 10 AMR translation, single v-cell, triangle signal +test_name[${index}]="transtest_1_amr_triangle" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000010.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v" +variable_components[${index}]="0 0 1 2" +((index+=1)) + +# 11 AMR translation, single v-cell, sinewave signal +test_name[${index}]="transtest_2_amr_sinewave" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000010.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v" +variable_components[${index}]="0 0 1 2" +((index+=1)) + +# 12 AMR translation, single v-cell, square signal +test_name[${index}]="transtest_3_amr_square" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000010.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v" +variable_components[${index}]="0 0 1 2" +((index+=1)) + +# 13 Large AMR translation flowthrough test +test_name[${index}]="Flowthrough_amr" +comparison_vlsv[${index}]="bulk.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 14 Large (dynamic) AMR translation flowthrough test +test_name[${index}]="Flowthrough_damr" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv bulk.0000003.vlsv bulk.0000004.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v vg_amr_alpha1 vg_amr_alpha2 fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 0 0 1 2 0 1 2" +((index+=1)) + +# TBA: Large flowthrough test with dAMR + +####### +# RESTARTING TESTS (15..18) +####### + +# Restart tests. Writing and reading +# 14 Restart write with translation only +test_name[${index}]="restart_write" +comparison_vlsv[${index}]="bulk.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 15 Restart read and propagate with translation only +test_name[${index}]="restart_read" +comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 16 Restart write with translation, acceleration, and fieldsolver +test_name[${index}]="restart_write_acc" +comparison_vlsv[${index}]="bulk.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 17 Restart read and propagate with translation, acceleration, and fieldsolver +test_name[${index}]="restart_read_acc" +comparison_vlsv[${index}]="initial-grid.0000000.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +####### +# FIELDSOLVER TESTS (19..20) +####### + +# 18 3D Field solver test +test_name[${index}]="test_fp_fsolver_only_3D" +comparison_vlsv[${index}]="fullf.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 1 2 0 1 2" +((index+=1)) + +# 19 3D Field solver test w/ subcycles +test_name[${index}]="test_fp_substeps" +comparison_vlsv[${index}]="fullf.0000001.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 1 2 0 1 2" +((index+=1)) + +####### +# GLOBAL 2D TESTS (21..22) +####### + +# 20 Very small ecliptic magnetosphere, no subcycling in ACC or FS +test_name[${index}]="Magnetosphere_small" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 21 Very small polar magnetosphere, with subcycling in ACC or FS +test_name[${index}]="Magnetosphere_polar_small" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton/vg_v_nonthermal proton/vg_v_nonthermal proton/vg_v_nonthermal proton/vg_ptensor_nonthermal_diagonal proton/vg_ptensor_nonthermal_diagonal proton/vg_ptensor_nonthermal_diagonal proton" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2" +((index+=1)) + +####### +# GLOBAL 3D TESTS (23..24) +####### + +# 22 Magnetosphere 3D, very small, 40 timesteps +test_name[${index}]="Magnetosphere_3D_small" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2" +((index+=1)) + +# 23 Ionosphere 3D (not a very physical or successful test at the moment but verifies some things about IG grid and outputs all datareducers) +test_name[${index}]="Ionosphere_small" +comparison_vlsv[${index}]="bulk.0000001.vlsv bulk.0000002.vlsv" +comparison_phiprof[${index}]="phiprof_0.txt" +variable_names[${index}]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e ig_upmappedarea ig_fac ig_rhon ig_potential" +variable_components[${index}]="0 0 1 2 0 1 2 0 1 2 0 0 0 0" +((index+=1)) + + +# choose tests to run (default: all tests) +run_tests=( ) +for (( c=1; c<$index; c++ )) +do + run_tests+=($c) +done + +# Alternatively, set tests manually, e.g. +# run_tests=( 1 6 9 ) + diff --git a/testpackage/tests/Flowthrough_1D_timevarying/Flowthrough_1D_timevarying.cfg b/testpackage/tests/Flowthrough_1D_timevarying/Flowthrough_1D_timevarying.cfg index 932ff653b..83b867ffb 100644 --- a/testpackage/tests/Flowthrough_1D_timevarying/Flowthrough_1D_timevarying.cfg +++ b/testpackage/tests/Flowthrough_1D_timevarying/Flowthrough_1D_timevarying.cfg @@ -36,7 +36,7 @@ y_min = -0.5 y_max = 0.5 z_min = -0.5 z_max = 0.5 -t_max = 10.0 +t_max = 20.0 dt = 0.5 [proton_properties] diff --git a/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg b/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg index 9a74b9b5c..61f1c5a9f 100644 --- a/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg +++ b/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg @@ -33,7 +33,7 @@ y_max = 4e7 z_min = -4e7 z_max = 4e7 t_max = 182.0 -dt = 2.0 +#dt = 2.0 [proton_vspace] vx_min = -1.92e6 diff --git a/testpackage/tests/Flowthrough_damr/Flowthrough_damr.cfg b/testpackage/tests/Flowthrough_damr/Flowthrough_damr.cfg new file mode 100644 index 000000000..1b1805aa7 --- /dev/null +++ b/testpackage/tests/Flowthrough_damr/Flowthrough_damr.cfg @@ -0,0 +1,116 @@ +ParticlePopulations = proton + +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 +dynamic_timestep = 0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[AMR] +max_spatial_level = 2 +adapt_refinement = 1 +use_alpha1 = 1 +use_alpha2 = 1 +refine_cadence = 2 +refinement_min_x = -3e7 +refinement_min_y = 0 +refinement_max_z = 0 + +[gridbuilder] +x_length = 16 +y_length = 2 +z_length = 2 +x_min = -8e7 +x_max = 8e7 +y_min = -1e7 +y_max = 1e7 +z_min = -1e7 +z_max = 1e7 +t_max = 402.0 +dt = 0.8 + +[proton_vspace] +vx_min = -1.92e6 +vx_max = +1.92e6 +vy_min = -1.92e6 +vy_max = +1.92e6 +vz_min = -1.92e6 +vz_max = +1.92e6 +vx_length = 60 +vy_length = 60 +vz_length = 60 + +[io] +write_initial_state = 1 + +system_write_t_interval = 100.0 +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = populations_vg_rho +output = populations_vg_v +output = fg_e +output = fg_b +output = vg_boundarytype +output = vg_boundarylayer +output = vg_rank +output = populations_vg_blocks +output = vg_amr_alpha1 +output = vg_amr_alpha2 +diagnostic = populations_vg_blocks + +[boundaries] +periodic_x = no +periodic_y = yes +periodic_z = yes +boundary = Outflow +boundary = Maxwellian + +[outflow] +precedence = 3 + +[proton_outflow] +face = x+ +#face = y- +#face = y+ +#face = z- +#face = z+ + +[maxwellian] +precedence = 4 +face = x- + +[proton_maxwellian] +dynamic = 0 +file_x- = sw1.dat + +[proton_sparse] +minValue = 1.0e-17 + +[Flowthrough] +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 + +[proton_Flowthrough] +T = 1.0e5 +rho = 1.0e6 +VX0 = 1e5 +VY0 = 0 +VZ0 = 0 + +[loadBalance] +algorithm = RCB + +[bailout] +velocity_space_wall_block_margin = 0 + diff --git a/testpackage/tests/Flowthrough_damr/sw1.dat b/testpackage/tests/Flowthrough_damr/sw1.dat new file mode 100644 index 000000000..0496edeb9 --- /dev/null +++ b/testpackage/tests/Flowthrough_damr/sw1.dat @@ -0,0 +1 @@ +0.0 2.0e6 2.0e6 2.0e5 0.0 0.0 0.0 0.0 0.0 diff --git a/testpackage/tests/Flowthrough_trans_periodic/Flowthrough_trans_periodic.cfg b/testpackage/tests/Flowthrough_trans_periodic/Flowthrough_trans_periodic.cfg index c924a548c..2372724ad 100644 --- a/testpackage/tests/Flowthrough_trans_periodic/Flowthrough_trans_periodic.cfg +++ b/testpackage/tests/Flowthrough_trans_periodic/Flowthrough_trans_periodic.cfg @@ -9,7 +9,7 @@ ParticlePopulations = proton [io] write_initial_state = 0 -system_write_t_interval = 649.0 +system_write_t_interval = 200.0 system_write_file_name = bulk system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 @@ -39,8 +39,8 @@ y_min = -1.3e8 y_max = 1.3e8 z_min = -6.5e6 z_max = 6.5e6 -t_max = 650 -dt = 2.0 +t_max = 600 +#dt = 2.0 [proton_properties] mass = 1 diff --git a/testpackage/tests/Flowthrough_x_inflow_y_outflow/Flowthrough_x_inflow_y_outflow.cfg b/testpackage/tests/Flowthrough_x_inflow_y_outflow/Flowthrough_x_inflow_y_outflow.cfg index 727bfd61a..0ea679c2b 100644 --- a/testpackage/tests/Flowthrough_x_inflow_y_outflow/Flowthrough_x_inflow_y_outflow.cfg +++ b/testpackage/tests/Flowthrough_x_inflow_y_outflow/Flowthrough_x_inflow_y_outflow.cfg @@ -9,7 +9,7 @@ ParticlePopulations = proton [io] write_initial_state = 0 -system_write_t_interval = 649.0 +system_write_t_interval = 200.0 system_write_file_name = bulk system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 @@ -38,8 +38,8 @@ y_min = -1.3e8 y_max = 1.3e8 z_min = -6.5e6 z_max = 6.5e6 -t_max = 650 -dt = 2.0 +t_max = 600 +#dt = 2.0 [proton_properties] mass = 1 diff --git a/testpackage/tests/Flowthrough_x_inflow_y_outflow_acc/Flowthrough_x_inflow_y_outflow_acc.cfg b/testpackage/tests/Flowthrough_x_inflow_y_outflow_acc/Flowthrough_x_inflow_y_outflow_acc.cfg index 969f4a2f5..015846112 100644 --- a/testpackage/tests/Flowthrough_x_inflow_y_outflow_acc/Flowthrough_x_inflow_y_outflow_acc.cfg +++ b/testpackage/tests/Flowthrough_x_inflow_y_outflow_acc/Flowthrough_x_inflow_y_outflow_acc.cfg @@ -9,7 +9,7 @@ ParticlePopulations = proton [io] write_initial_state = 0 -system_write_t_interval = 649.0 +system_write_t_interval = 200.0 system_write_file_name = bulk system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 @@ -38,9 +38,9 @@ y_min = -1.3e8 y_max = 1.3e8 z_min = -6.5e6 z_max = 6.5e6 +t_max = 600 +#dt = 2.0 -t_max = 650 -dt = 2.0 [proton_properties] mass = 1 mass_units = PROTON diff --git a/testpackage/tests/Ionosphere_small/Ionosphere_small.cfg b/testpackage/tests/Ionosphere_small/Ionosphere_small.cfg index 274746535..62d0a1904 100644 --- a/testpackage/tests/Ionosphere_small/Ionosphere_small.cfg +++ b/testpackage/tests/Ionosphere_small/Ionosphere_small.cfg @@ -11,7 +11,7 @@ charge = 1 diagnostic_write_interval = 1 write_initial_state = 0 -system_write_t_interval = 3 +system_write_t_interval = 15 system_write_file_name = bulk system_write_distribution_stride = 0 system_write_distribution_xline_stride = 10 @@ -31,7 +31,7 @@ y_min = -3.3e8 y_max = 3.3e8 z_min = -3.3e8 z_max = 3.3e8 -t_max = 3.1 +t_max = 30.1 #timestep_max = 100 [AMR] @@ -147,7 +147,7 @@ file_x+ = sw1.dat constBgBX = 0.0 constBgBY = 0.0 constBgBZ = -5.0e-9 -noDipoleInSW = 0 # Use 0 with dipole type 4 +dipoleScalingFactor = 0.1 # smaller dipole for faster testing dipoleType = 4 dipoleTiltPhi = 0.0 diff --git a/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg b/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg index edaa6cfa0..9bbcc78a1 100644 --- a/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg +++ b/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg @@ -29,7 +29,7 @@ y_min = -3.3e8 y_max = 3.3e8 z_min = -3.3e8 z_max = 3.3e8 -t_max = 3.1 +t_max = 6.1 #timestep_max = 100 [AMR] @@ -114,7 +114,7 @@ file_x+ = sw1.dat constBgBX = 0.0 constBgBY = 0.0 constBgBZ = -5.0e-9 -noDipoleInSW = 0 # Use 0 with dipole type 4 +dipoleScalingFactor = 0.1 # smaller dipole for faster testing dipoleType = 4 dipoleTiltPhi = 0.0 diff --git a/testpackage/tests/Magnetosphere_polar_small/Magnetosphere_polar_small.cfg b/testpackage/tests/Magnetosphere_polar_small/Magnetosphere_polar_small.cfg index 369d29aea..224b438f8 100644 --- a/testpackage/tests/Magnetosphere_polar_small/Magnetosphere_polar_small.cfg +++ b/testpackage/tests/Magnetosphere_polar_small/Magnetosphere_polar_small.cfg @@ -33,7 +33,7 @@ y_min = -5.0e6 y_max = 5.0e6 z_min = -250.0e6 z_max = 250.0e6 -t_max = 20.05 +t_max = 40.05 [proton_vspace] @@ -125,6 +125,7 @@ constBgBZ = -3.53553e-9 noDipoleInSW = 1.0 dipoleType = 2 dipoleMirrorLocationX = 625.0e6 +dipoleScalingFactor = 0.1 # smaller dipole for faster testing [proton_thermal] # Pretty much bogus values, just so that the reducer has something diff --git a/testpackage/tests/Magnetosphere_small/Magnetosphere_small.cfg b/testpackage/tests/Magnetosphere_small/Magnetosphere_small.cfg index fad589d1d..ca3f5ad0e 100644 --- a/testpackage/tests/Magnetosphere_small/Magnetosphere_small.cfg +++ b/testpackage/tests/Magnetosphere_small/Magnetosphere_small.cfg @@ -29,7 +29,7 @@ y_min = -2e8 y_max = 2e8 z_min = -4e6 z_max = 4e6 -t_max = 10.1 +t_max = 20.1 #timestep_max = 100 [proton_vspace] @@ -84,11 +84,12 @@ centerX = 0.0 centerY = 0.0 centerZ = 0.0 radius = 38.2e6 +zeroPerB = 1 precedence = 2 [proton_copysphere] rho = 1.0e6 -T=100000.0 +T = 100000.0 [outflow] precedence = 3 @@ -108,6 +109,7 @@ file_x+ = sw1.dat constBgBX = -3.5355339e-9 constBgBY = 3.5355339e-9 noDipoleInSW = 1.0 +dipoleScalingFactor = 0.1 # smaller dipole for faster testing [proton_Magnetosphere] T = 100000.0 diff --git a/testpackage/tests/transtest_2_maxw_500k_100k_20kms_20x20/transtest_2_maxw_500k_100k_20kms_20x20.cfg b/testpackage/tests/acctest_1_maxw_500k_100k_20kms_10deg/acctest_1_maxw_500k_100k_20kms_10deg.cfg similarity index 80% rename from testpackage/tests/transtest_2_maxw_500k_100k_20kms_20x20/transtest_2_maxw_500k_100k_20kms_20x20.cfg rename to testpackage/tests/acctest_1_maxw_500k_100k_20kms_10deg/acctest_1_maxw_500k_100k_20kms_10deg.cfg index 4c8b87447..a03883316 100644 --- a/testpackage/tests/transtest_2_maxw_500k_100k_20kms_20x20/transtest_2_maxw_500k_100k_20kms_20x20.cfg +++ b/testpackage/tests/acctest_1_maxw_500k_100k_20kms_10deg/acctest_1_maxw_500k_100k_20kms_10deg.cfg @@ -1,38 +1,55 @@ -dynamic_timestep = 1 +dynamic_timestep = 0 project = MultiPeak -ParticlePopulations = proton propagate_field = 0 -propagate_vlasov_acceleration = 0 -propagate_vlasov_translation = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 0 -[proton_properties] -mass = 1 -mass_units = PROTON -charge = 1 +ParticlePopulations = proton [io] diagnostic_write_interval = 1 write_initial_state = 0 -system_write_t_interval = 9.4 +system_write_t_interval = 180 system_write_file_name = fullf system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 system_write_distribution_yline_stride = 0 system_write_distribution_zline_stride = 0 +[variables] +output = vg_rhom +output = fg_b +output = vg_pressure +output = populations_vg_v +output = fg_e +output = vg_rank +output = populations_vg_blocks +output = populations_vg_rho +#output = populations_vg_acceleration_subcycles + +diagnostic = populations_vg_blocks +#diagnostic = vg_pressure +#diagnostic = populations_vg_rho +#diagnostic = populations_vg_rho_loss_adjust [gridbuilder] -x_length = 20 -y_length = 20 +x_length = 1 +y_length = 1 z_length = 1 x_min = 0.0 x_max = 1.0e6 y_min = 0.0 y_max = 1.0e6 z_min = 0 -z_max = 50000.0 -timestep_max = 200 +z_max = 1.0e6 +t_max = 3600 +dt = 10.0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 [proton_vspace] vx_min = -2.0e6 @@ -44,7 +61,6 @@ vz_max = +2.0e6 vx_length = 50 vy_length = 50 vz_length = 50 - [proton_sparse] minValue = 1.0e-16 @@ -53,21 +69,6 @@ periodic_x = yes periodic_y = yes periodic_z = yes -[variables] -output = populations_vg_rho -output = fg_b -output = vg_pressure -output = populations_vg_v -output = fg_e -output = vg_rank -output = populations_vg_blocks -#output = populations_vg_acceleration_subcycles - -diagnostic = populations_vg_blocks -#diagnostic = vg_pressure -#diagnostic = populations_vg_rho -#diagnostic = populations_vg_rho_loss_adjust - [MultiPeak] #magnitude of 1.82206867e-10 gives a period of 360s, useful for testing... Bx = 1.2e-10 @@ -78,15 +79,24 @@ magYPertAbsAmp = 0 magZPertAbsAmp = 0 [proton_MultiPeak] -n = 1 -Vx = 5e5 +n = 2 +Vx = 0.0 Vy = 5e5 Vz = 0.0 Tx = 500000.0 Ty = 500000.0 Tz = 500000.0 -rho = 1000000.0 -rhoPertAbsAmp = 10000 +rho = 2000000.0 +rhoPertAbsAmp = 0 + +Vx = 0.0 +Vy = -5e5 +Vz = 0.0 +Tx = 100000.0 +Ty = 100000.0 +Tz = 100000.0 +rho = 2000000 +rhoPertAbsAmp = 0 [bailout] velocity_space_wall_block_margin = 0 diff --git a/testpackage/tests/acctest_1_maxw_500k_30kms_1deg/acctest_1_maxw_500k_30kms_1deg.cfg b/testpackage/tests/acctest_1_maxw_500k_30kms_1deg/acctest_1_maxw_500k_30kms_1deg.cfg deleted file mode 100644 index 8ec3bd351..000000000 --- a/testpackage/tests/acctest_1_maxw_500k_30kms_1deg/acctest_1_maxw_500k_30kms_1deg.cfg +++ /dev/null @@ -1,129 +0,0 @@ -project = MultiPeak -dynamic_timestep = 0 -propagate_field = 0 -propagate_vlasov_acceleration = 1 -propagate_vlasov_translation = 0 - -ParticlePopulations = proton - -[io] -diagnostic_write_interval = 10 -write_initial_state = 0 - -system_write_t_interval = 3600 -system_write_file_name = fullf -system_write_distribution_stride = 1 -system_write_distribution_xline_stride = 0 -system_write_distribution_yline_stride = 0 -system_write_distribution_zline_stride = 0 - -[variables] -output = fg_b -output = fg_b_background -output = fg_b_perturbed -output = fg_e -output = vg_rhom -output = vg_rhoq -output = populations_vg_rho -output = vg_v -output = populations_vg_v -output = populations_vg_moments_nonthermal -output = populations_vg_moments_thermal -output = populations_vg_effectivesparsitythreshold -output = populations_vg_rho_loss_adjust -output = vg_loadbalance_weight -output = vg_maxdt_acceleration -output = vg_maxdt_translation -output = populations_vg_maxdt_acceleration -output = populations_vg_maxdt_translation -output = fg_maxdt_fieldsolver -output = vg_rank -output = fg_rank -output = fg_boundarytype -output = vg_boundarytype -output = vg_boundarylayer -output = populations_vg_blocks -output = vg_f_saved -output = populations_vg_acceleration_subcycles -output = fg_e_hall -output = vg_e_gradpe -output = vg_b_vol -output = vg_b_background_vol -output = vg_b_perturbed_vol -output = vg_pressure -output = populations_vg_ptensor -output = vg_b_vol_derivatives -output = vg_b_vol_derivatives -output = vg_gridcoordinates -output = MeshData -#output = populations_vg_acceleration_subcycles - -diagnostic = populations_vg_blocks -diagnostic = vg_rhom -diagnostic = populations_vg_rho_loss_adjust -diagnostic = vg_loadbalance_weight -diagnostic = vg_maxdt_acceleration -diagnostic = vg_maxdt_translation -diagnostic = populations_MaxDistributionFunction -diagnostic = populations_MinDistributionFunction - -[gridbuilder] -x_length = 1 -y_length = 1 -z_length = 1 -x_min = 0.0 -x_max = 1.0e6 -y_min = 0.0 -y_max = 1.0e6 -z_min = 0 -z_max = 1.0e6 -t_max = 3600 -dt = 1.0 - -[proton_properties] -mass = 1 -mass_units = PROTON -charge = 1 - -[proton_vspace] -vx_min = -3.0e6 -vx_max = +3.0e6 -vy_min = -3.0e6 -vy_max = +3.0e6 -vz_min = -3.0e6 -vz_max = +3.0e6 -vx_length = 50 -vy_length = 50 -vz_length = 50 - -[proton_sparse] -minValue = 1.0e-16 - -[boundaries] -periodic_x = yes -periodic_y = yes -periodic_z = yes - -[MultiPeak] -#magnitude of 1.82206867e-10 gives a period of 360s, useful for testing... -Bx = 1.2e-10 -By = 0.8e-10 -Bz = 1.1135233442526334e-10 -lambda = 10 -magXPertAbsAmp = 0 -magYPertAbsAmp = 0 -magZPertAbsAmp = 0 - -[proton_MultiPeak] -n = 1 -Vx = 0.0 -Vy = 5e5 -Vz = 0.0 -Tx = 500000.0 -Ty = 500000.0 -Tz = 500000.0 -rho = 2000000.0 -rhoPertAbsAmp = 0 - -[bailout] -velocity_space_wall_block_margin = 0 diff --git a/testpackage/tests/acctest_2_maxw_500k_100k_20kms_10deg/acctest_2_maxw_500k_100k_20kms_10deg.cfg b/testpackage/tests/acctest_2_maxw_500k_100k_20kms_10deg/acctest_2_maxw_500k_100k_20kms_10deg.cfg index 05c7fb025..d3e9d3363 100644 --- a/testpackage/tests/acctest_2_maxw_500k_100k_20kms_10deg/acctest_2_maxw_500k_100k_20kms_10deg.cfg +++ b/testpackage/tests/acctest_2_maxw_500k_100k_20kms_10deg/acctest_2_maxw_500k_100k_20kms_10deg.cfg @@ -1,4 +1,4 @@ -dynamic_timestep = 0 +dynamic_timestep = 1 project = MultiPeak propagate_field = 0 propagate_vlasov_acceleration = 1 @@ -44,7 +44,7 @@ y_max = 1.0e6 z_min = 0 z_max = 1.0e6 t_max = 3600 -dt = 10.0 +#dt = 10.0 [proton_properties] mass = 1 diff --git a/testpackage/tests/acctest_5_proton_antiproton/acctest_5_proton_antiproton.cfg b/testpackage/tests/acctest_5_proton_antiproton/acctest_5_proton_antiproton.cfg index 990468e37..afb494770 100644 --- a/testpackage/tests/acctest_5_proton_antiproton/acctest_5_proton_antiproton.cfg +++ b/testpackage/tests/acctest_5_proton_antiproton/acctest_5_proton_antiproton.cfg @@ -12,7 +12,7 @@ ParticlePopulations = antiproton diagnostic_write_interval = 0 write_initial_state = 0 -system_write_t_interval = 90 +system_write_t_interval = 45 system_write_file_name = fullf system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 diff --git a/testpackage/tests/restart_read/restart_read.cfg b/testpackage/tests/restart_read/restart_read.cfg index c1b1802cc..73d7d3b5d 100644 --- a/testpackage/tests/restart_read/restart_read.cfg +++ b/testpackage/tests/restart_read/restart_read.cfg @@ -2,17 +2,18 @@ project = Flowthrough propagate_field = 0 propagate_vlasov_acceleration = 0 propagate_vlasov_translation = 1 -dynamic_timestep = 1 +dynamic_timestep = 0 ParticlePopulations = proton [restart] +#write_as_float = 1 filename = restart.vlsv [io] write_initial_state = 1 -system_write_t_interval = 1 +system_write_t_interval = 10.0 system_write_file_name = bulk system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 @@ -41,7 +42,7 @@ y_min = -1.3e8 y_max = 1.3e8 z_min = -6500000.0 z_max = 6500000.0 -t_max = 30 +t_max = 20 dt = 2.0 [proton_properties] @@ -76,7 +77,7 @@ precedence = 3 [proton_outflow] reapplyFaceUponRestart = x+ reapplyFaceUponRestart = y+ -vlasovScheme_face_x+ = None +vlasovScheme_face_x+ = Copy vlasovScheme_face_y+ = Copy face = x+ face = y+ diff --git a/testpackage/tests/restart_read_acc/restart_read_acc.cfg b/testpackage/tests/restart_read_acc/restart_read_acc.cfg new file mode 100644 index 000000000..99c5644b6 --- /dev/null +++ b/testpackage/tests/restart_read_acc/restart_read_acc.cfg @@ -0,0 +1,112 @@ +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 +dynamic_timestep = 1 + +ParticlePopulations = proton + +[restart] +#write_as_float = 1 +filename = restart.vlsv + +[io] +write_initial_state = 1 +restart_walltime_interval = 1000 # setting to nonzero activates restart writing on exit + +system_write_t_interval = 10.0 +system_write_file_name = bulk +system_write_distribution_stride = 1 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = vg_rhom +output = fg_e +output = fg_b +output = vg_pressure +output = populations_vg_v +output = populations_vg_rho +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +diagnostic = populations_vg_blocks + +[gridbuilder] +x_length = 20 +y_length = 20 +z_length = 1 +x_min = -1.3e8 +x_max = 1.3e8 +y_min = -1.3e8 +y_max = 1.3e8 +z_min = -6500000.0 +z_max = 6500000.0 +t_max = 20 +#dt = 2.0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[proton_vspace] +vx_min = -640000.0 +vx_max = +640000.0 +vy_min = -640000.0 +vy_max = +640000.0 +vz_min = -640000.0 +vz_max = +640000.0 +vx_length = 32 +vy_length = 32 +vz_length = 32 + +[proton_sparse] +minValue = 1.0e-15 + +[boundaries] +periodic_x = no +periodic_y = no +periodic_z = yes +boundary = Outflow +boundary = Maxwellian + +[outflow] +precedence = 3 + +[proton_outflow] +reapplyFaceUponRestart = x+ +reapplyFaceUponRestart = y+ +vlasovScheme_face_x+ = Copy +vlasovScheme_face_y+ = Copy +face = x+ +face = y+ + +[maxwellian] +face = x- +face = y- +precedence = 2 + +[proton_maxwellian] +dynamic = 0 +file_x- = sw1.dat +file_y- = sw1.dat + +[Flowthrough] +#emptyBox = 1 +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 +densityModel = Maxwellian + +[proton_Flowthrough] +# half of the inflow values +T = 0.5e5 +rho = 0.5e6 +VX0 = 4e5 +VY0 = 0 +VZ0 = 0 + +[bailout] +velocity_space_wall_block_margin = 0 diff --git a/testpackage/tests/restart_read_acc/sw1.dat b/testpackage/tests/restart_read_acc/sw1.dat new file mode 100644 index 000000000..1440559e3 --- /dev/null +++ b/testpackage/tests/restart_read_acc/sw1.dat @@ -0,0 +1 @@ +0.0 1.0e6 1.0e5 +5.0e5 +2.5e5 0.0 0.0e-9 0.0 0.0 diff --git a/testpackage/tests/restart_read_acc/test_prelude.sh b/testpackage/tests/restart_read_acc/test_prelude.sh new file mode 100755 index 000000000..d8e4bd81f --- /dev/null +++ b/testpackage/tests/restart_read_acc/test_prelude.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +LAST_RESTART=$(ls -t ../restart_write_acc/restart*.vlsv | head -n 1) +test -e $LAST_RESTART && ln -s $LAST_RESTART ./restart.vlsv diff --git a/testpackage/tests/restart_write/restart_write.cfg b/testpackage/tests/restart_write/restart_write.cfg index 24d299ed1..21942fa44 100644 --- a/testpackage/tests/restart_write/restart_write.cfg +++ b/testpackage/tests/restart_write/restart_write.cfg @@ -2,26 +2,24 @@ project = Flowthrough propagate_field = 0 propagate_vlasov_acceleration = 0 propagate_vlasov_translation = 1 -dynamic_timestep = 1 +dynamic_timestep = 0 ParticlePopulations = proton -[io] +[restart] #write_as_float = 1 + +[io] write_initial_state = 0 -restart_walltime_interval = 1000 +restart_walltime_interval = 1000 # setting to nonzero activates restart writing on exit -system_write_t_interval = 30.0 +system_write_t_interval = 10.0 system_write_file_name = bulk system_write_distribution_stride = 1 system_write_distribution_xline_stride = 0 system_write_distribution_yline_stride = 0 system_write_distribution_zline_stride = 0 -[restart] -#write_as_float = 1 - - [variables] output = vg_rhom output = fg_e @@ -44,7 +42,7 @@ y_min = -1.3e8 y_max = 1.3e8 z_min = -6500000.0 z_max = 6500000.0 -t_max = 30 +t_max = 10 dt = 2.0 [proton_properties] diff --git a/testpackage/tests/restart_write_acc/restart_write_acc.cfg b/testpackage/tests/restart_write_acc/restart_write_acc.cfg new file mode 100644 index 000000000..068175678 --- /dev/null +++ b/testpackage/tests/restart_write_acc/restart_write_acc.cfg @@ -0,0 +1,111 @@ +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 1 +propagate_vlasov_translation = 1 +dynamic_timestep = 1 + +ParticlePopulations = proton + +[restart] +#write_as_float = 1 + +[io] +write_initial_state = 0 +restart_walltime_interval = 1000 # setting to nonzero activates restart writing on exit + +system_write_t_interval = 10.0 +system_write_file_name = bulk +system_write_distribution_stride = 1 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = vg_rhom +output = fg_e +output = fg_b +output = vg_pressure +output = populations_vg_v +output = populations_vg_rho +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +diagnostic = populations_vg_blocks + +[gridbuilder] +x_length = 20 +y_length = 20 +z_length = 1 +x_min = -1.3e8 +x_max = 1.3e8 +y_min = -1.3e8 +y_max = 1.3e8 +z_min = -6500000.0 +z_max = 6500000.0 +t_max = 10 +#dt = 2.0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[proton_vspace] +vx_min = -640000.0 +vx_max = +640000.0 +vy_min = -640000.0 +vy_max = +640000.0 +vz_min = -640000.0 +vz_max = +640000.0 +vx_length = 32 +vy_length = 32 +vz_length = 32 + +[proton_sparse] +minValue = 1.0e-15 + +[boundaries] +periodic_x = no +periodic_y = no +periodic_z = yes +boundary = Outflow +boundary = Maxwellian + +[outflow] +precedence = 3 + +[proton_outflow] +reapplyFaceUponRestart = x+ +reapplyFaceUponRestart = y+ +vlasovScheme_face_x+ = Copy +vlasovScheme_face_y+ = Copy +face = x+ +face = y+ + +[maxwellian] +face = x- +face = y- +precedence = 2 + +[proton_maxwellian] +dynamic = 0 +file_x- = sw1.dat +file_y- = sw1.dat + +[Flowthrough] +#emptyBox = 1 +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 +densityModel = Maxwellian + +[proton_Flowthrough] +# half of the inflow values +T = 0.5e5 +rho = 0.5e6 +VX0 = 4e5 +VY0 = 0 +VZ0 = 0 + +[bailout] +velocity_space_wall_block_margin = 0 diff --git a/testpackage/tests/restart_write_acc/sw1.dat b/testpackage/tests/restart_write_acc/sw1.dat new file mode 100644 index 000000000..1440559e3 --- /dev/null +++ b/testpackage/tests/restart_write_acc/sw1.dat @@ -0,0 +1 @@ +0.0 1.0e6 1.0e5 +5.0e5 +2.5e5 0.0 0.0e-9 0.0 0.0 diff --git a/testpackage/tests/restart_write_acc/test_postproc.sh b/testpackage/tests/restart_write_acc/test_postproc.sh new file mode 100755 index 000000000..430126005 --- /dev/null +++ b/testpackage/tests/restart_write_acc/test_postproc.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +LAST_RESTART=$(ls -t restart*.vlsv | head -n 1) +test -e $LAST_RESTART && ln -s $LAST_RESTART ./restart.vlsv diff --git a/testpackage/tests/test_fp_fsolver_only_3D/test_fp_fsolver_only_3D.cfg b/testpackage/tests/test_fp_fsolver_only_3D/test_fp_fsolver_only_3D.cfg index 979936d54..f3f1f442c 100644 --- a/testpackage/tests/test_fp_fsolver_only_3D/test_fp_fsolver_only_3D.cfg +++ b/testpackage/tests/test_fp_fsolver_only_3D/test_fp_fsolver_only_3D.cfg @@ -35,7 +35,7 @@ y_min = -1.0e4 y_max = +1.0e4 z_min = -1.0e4 z_max = +1.0e4 -dt = 0.05 +#dt = 0.05 t_max = 6.92820323 [proton_vspace] diff --git a/testpackage/tests/test_fp_substeps/test_fp_substeps.cfg b/testpackage/tests/test_fp_substeps/test_fp_substeps.cfg index 10638e883..bcfb9e30f 100644 --- a/testpackage/tests/test_fp_substeps/test_fp_substeps.cfg +++ b/testpackage/tests/test_fp_substeps/test_fp_substeps.cfg @@ -35,7 +35,7 @@ y_min = -1.0e4 y_max = +1.0e4 z_min = -1.0e4 z_max = +1.0e4 -dt = 0.05 +#dt = 0.05 t_max = 6.92820323 [proton_vspace] diff --git a/testpackage/tests/transtest_1_amr_triangle/transtest_1_amr_triangle.cfg b/testpackage/tests/transtest_1_amr_triangle/transtest_1_amr_triangle.cfg new file mode 100644 index 000000000..d2a2288c9 --- /dev/null +++ b/testpackage/tests/transtest_1_amr_triangle/transtest_1_amr_triangle.cfg @@ -0,0 +1,102 @@ +ParticlePopulations = proton + +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 0 +propagate_vlasov_translation = 1 +dynamic_timestep = 0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[AMR] +max_spatial_level = 2 +number_of_boxes = 1 +box_max_level = 2 +box_half_width_x = 1 +box_half_width_y = 1 +box_half_width_z = 1 +box_center_x = 0 +box_center_y = 0 +box_center_z = 0 + +[gridbuilder] +x_length = 20 +y_length = 1 +z_length = 1 +x_min = -1e6 +x_max = 1e6 +y_min = -0.5e5 +y_max = 0.5e5 +z_min = -0.5e5 +z_max = 0.5e5 +t_max = 400.0 #perform 10 cycles +# reflevel 0 cell size 100 km +dt = 0.04 #CFL limnit 0.8 leads to 1.6s timestep at AMR level 0, 0.4s at level 2 + +[proton_vspace] +# single v-cell only, thus only 1 block as well +vx_min = +49e3 +vx_max = +57e3 +vy_min = -1e3 +vy_max = +7e3 +vz_min = -1e3 +vz_max = +7e3 +vx_length = 1 +vy_length = 1 +vz_length = 1 + +[io] +write_initial_state = 0 + +system_write_t_interval = 40.0 # write once per full cycle +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = populations_vg_rho +output = populations_vg_v +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +#output = fg_e +#output = fg_b +#output = fg_rhom +#output = fg_amr_level +diagnostic = populations_vg_blocks + +[boundaries] +periodic_x = yes +periodic_y = yes +periodic_z = yes + +[proton_sparse] +minValue = 1.0e-5 +conserve_mass = 1 + +[Flowthrough] +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 +densityModel = Triangle +densityWidth = 1.5e6 + +[proton_Flowthrough] +T = 1.0 +rho = 2.0e6 +rhoBase = 1.0e6 +VX0 = 50e3 # just a single velocity cell +VY0 = 0 +VZ0 = 0 + +[loadBalance] +algorithm = RCB + +[bailout] +velocity_space_wall_block_margin = 0 + diff --git a/testpackage/tests/transtest_2_amr_sinewave/transtest_2_amr_sinewave.cfg b/testpackage/tests/transtest_2_amr_sinewave/transtest_2_amr_sinewave.cfg new file mode 100644 index 000000000..c54bbfd9a --- /dev/null +++ b/testpackage/tests/transtest_2_amr_sinewave/transtest_2_amr_sinewave.cfg @@ -0,0 +1,102 @@ +ParticlePopulations = proton + +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 0 +propagate_vlasov_translation = 1 +dynamic_timestep = 0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[AMR] +max_spatial_level = 2 +number_of_boxes = 1 +box_max_level = 2 +box_half_width_x = 1 +box_half_width_y = 1 +box_half_width_z = 1 +box_center_x = 0 +box_center_y = 0 +box_center_z = 0 + +[gridbuilder] +x_length = 20 +y_length = 1 +z_length = 1 +x_min = -1e6 +x_max = 1e6 +y_min = -0.5e5 +y_max = 0.5e5 +z_min = -0.5e5 +z_max = 0.5e5 +t_max = 400.0 #perform 10 cycles +# reflevel 0 cell size 100 km +dt = 0.04 #CFL limnit 0.8 leads to 1.6s timestep at AMR level 0, 0.4s at level 2 + +[proton_vspace] +# single v-cell only, thus only 1 block as well +vx_min = +49e3 +vx_max = +57e3 +vy_min = -1e3 +vy_max = +7e3 +vz_min = -1e3 +vz_max = +7e3 +vx_length = 1 +vy_length = 1 +vz_length = 1 + +[io] +write_initial_state = 0 + +system_write_t_interval = 40.0 # write once per full cycle +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = populations_vg_rho +output = populations_vg_v +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +#output = fg_e +#output = fg_b +#output = fg_rhom +#output = fg_amr_level +diagnostic = populations_vg_blocks + +[boundaries] +periodic_x = yes +periodic_y = yes +periodic_z = yes + +[proton_sparse] +minValue = 1.0e-5 +conserve_mass = 1 + +[Flowthrough] +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 +densityModel = Sinewave +densityWidth = 1.5e6 + +[proton_Flowthrough] +T = 1.0 +rho = 2.0e6 +rhoBase = 1.0e6 +VX0 = 50e3 # just a single velocity cell +VY0 = 0 +VZ0 = 0 + +[loadBalance] +algorithm = RCB + +[bailout] +velocity_space_wall_block_margin = 0 + diff --git a/testpackage/tests/transtest_3_amr_square/transtest_3_amr_square.cfg b/testpackage/tests/transtest_3_amr_square/transtest_3_amr_square.cfg new file mode 100644 index 000000000..ba42edaa9 --- /dev/null +++ b/testpackage/tests/transtest_3_amr_square/transtest_3_amr_square.cfg @@ -0,0 +1,102 @@ +ParticlePopulations = proton + +project = Flowthrough +propagate_field = 1 +propagate_vlasov_acceleration = 0 +propagate_vlasov_translation = 1 +dynamic_timestep = 0 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[AMR] +max_spatial_level = 2 +number_of_boxes = 1 +box_max_level = 2 +box_half_width_x = 1 +box_half_width_y = 1 +box_half_width_z = 1 +box_center_x = 0 +box_center_y = 0 +box_center_z = 0 + +[gridbuilder] +x_length = 20 +y_length = 1 +z_length = 1 +x_min = -1e6 +x_max = 1e6 +y_min = -0.5e5 +y_max = 0.5e5 +z_min = -0.5e5 +z_max = 0.5e5 +t_max = 400.0 #perform 10 cycles +# reflevel 0 cell size 100 km +dt = 0.04 #CFL limnit 0.8 leads to 1.6s timestep at AMR level 0, 0.4s at level 2 + +[proton_vspace] +# single v-cell only, thus only 1 block as well +vx_min = +49e3 +vx_max = +57e3 +vy_min = -1e3 +vy_max = +7e3 +vz_min = -1e3 +vz_max = +7e3 +vx_length = 1 +vy_length = 1 +vz_length = 1 + +[io] +write_initial_state = 0 + +system_write_t_interval = 40.0 # write once per full cycle +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 0 +system_write_distribution_yline_stride = 0 +system_write_distribution_zline_stride = 0 + +[variables] +output = populations_vg_rho +output = populations_vg_v +output = vg_boundarytype +output = vg_rank +output = populations_vg_blocks +#output = fg_e +#output = fg_b +#output = fg_rhom +#output = fg_amr_level +diagnostic = populations_vg_blocks + +[boundaries] +periodic_x = yes +periodic_y = yes +periodic_z = yes + +[proton_sparse] +minValue = 1.0e-5 +conserve_mass = 1 + +[Flowthrough] +Bx = 1.0e-9 +By = 1.0e-9 +Bz = 1.0e-9 +densityModel = Square +densityWidth = 1.5e6 + +[proton_Flowthrough] +T = 1.0 +rho = 2.0e6 +rhoBase = 1.0e6 +VX0 = 50e3 # just a single velocity cell +VY0 = 0 +VZ0 = 0 + +[loadBalance] +algorithm = RCB + +[bailout] +velocity_space_wall_block_margin = 0 + diff --git a/tools/check_vlasiator_cfg.sh b/tools/check_vlasiator_cfg.sh index 489db51d4..77508c7f4 100755 --- a/tools/check_vlasiator_cfg.sh +++ b/tools/check_vlasiator_cfg.sh @@ -1,9 +1,17 @@ #!/bin/bash -mpirun_cmd=$(which aprun &> /dev/null && echo "aprun" || echo "mpirun") +# use this when running locally/on a frontend +mpirun_cmd=" " +# or pick the launcher you need +#mpirun_cmd="srun" +#mpirun_cmd="mpirun" +#mpirun_cmd="aprun" + vlasiator=$1 cfg=$2 +# Return a nonzero exit code if anything invalid was found +retval=0 if [ $# -ne 2 ] then @@ -13,21 +21,21 @@ Prints out differences between parameters in a cfg file and the options that the Usage: $0 vlasiator_executable cfg_file EOF - exit + exit 127 fi if [ ! -x $vlasiator ] then echo "ERROR: Vlasiator executable $vlasiator does not exist or is not executable" - exit + exit 127 fi if [ ! -e $cfg ] then echo "ERROR: cfg file $cfg does not exist" - exit + exit 127 fi @@ -120,20 +128,22 @@ output_update=`expr match "$( cat .vlasiator_variables | grep "variables.output" diagnostic_update=`expr match "$( cat .vlasiator_variables | grep "variables.diagnostic" )" '.*\([0-9]\{8\}\).*'` -echo "------------------------------------------------------------------------------------------------------------" -echo "Available unused options" -echo "------------------------------------------------------------------------------------------------------------" -comm -23 .vlasiator_variable_names .cfg_variable_names | grep -v "\." > .unused_variables -comm -23 .vlasiator_variable_names .cfg_variable_names | grep -f .allowed_prefixes >> .unused_variables -grep -f .unused_variables .vlasiator_variable_names_default_val -echo "------------------------------------------------------------------------------------------------------------" +if [ z$PRINT_ONLY_ERRORS == "z" ]; then + echo "------------------------------------------------------------------------------------------------------------" + echo "Available unused options" + echo "------------------------------------------------------------------------------------------------------------" + comm -23 .vlasiator_variable_names .cfg_variable_names | grep -v "\." > .unused_variables + comm -23 .vlasiator_variable_names .cfg_variable_names | grep -f .allowed_prefixes >> .unused_variables + grep -f .unused_variables .vlasiator_variable_names_default_val + echo "------------------------------------------------------------------------------------------------------------" -echo "------------------------------------------------------------------------------------------------------------" -echo "Available unused output and diagnostic variables (as of "$output_update" resp. "$diagnostic_update")" -echo "------------------------------------------------------------------------------------------------------------" -comm -23 .vlasiator_output_variable_names .cfg_output_variable_names -comm -23 .vlasiator_diagnostic_variable_names .cfg_diagnostic_variable_names -echo "------------------------------------------------------------------------------------------------------------" + echo "------------------------------------------------------------------------------------------------------------" + echo "Available unused output and diagnostic variables (as of "$output_update" resp. "$diagnostic_update")" + echo "------------------------------------------------------------------------------------------------------------" + comm -23 .vlasiator_output_variable_names .cfg_output_variable_names + comm -23 .vlasiator_diagnostic_variable_names .cfg_diagnostic_variable_names + echo "------------------------------------------------------------------------------------------------------------" +fi output=$( comm -13 .vlasiator_variable_names .cfg_variable_names ) if [ ${#output} -ne 0 ] @@ -142,6 +152,7 @@ then echo "------------------------------------------------------------------------------------------------------------" comm -13 .vlasiator_variable_names .cfg_variable_names echo "------------------------------------------------------------------------------------------------------------" + retval=1 else echo "------------------------------------------------------------------------------------------------------------" echo "No invalid options" @@ -164,6 +175,7 @@ then comm -13 .vlasiator_diagnostic_variable_names .cfg_diagnostic_variable_names fi echo "------------------------------------------------------------------------------------------------------------" + retval=2 else echo "------------------------------------------------------------------------------------------------------------" echo "No invalid output or diagnostic variables (as of "$output_update" resp. "$diagnostic_update")" @@ -171,5 +183,6 @@ else fi +rm -f .cfg_variables .cfg_variable_names .vlasiator_variables .vlasiator_variable_names .allowed_prefixes .unused_variables .vlasiator_variable_names_default_val .cfg_output_variable_names .cfg_diagnostic_variable_names .vlasiator_diagnostic_variable_names .vlasiator_output_variable_names -rm .cfg_variables .cfg_variable_names .vlasiator_variables .vlasiator_variable_names .allowed_prefixes .unused_variables .vlasiator_variable_names_default_val .cfg_output_variable_names .cfg_diagnostic_variable_names .vlasiator_diagnostic_variable_names .vlasiator_output_variable_names +exit $retval diff --git a/tools/fileHandling/glueChunksTogether.sh b/tools/fileHandling/glueChunksTogether.sh index 4bc3c5792..2bcc1155b 100755 --- a/tools/fileHandling/glueChunksTogether.sh +++ b/tools/fileHandling/glueChunksTogether.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ ! $# -eq 5 ] +if [ ! $# -eq 6 ] then cat <> dd_chunk.err done -echo $(date) "Transferring ${file} of size $fileSize, with a total of $chunkNumber chunks of $chunkSize bytes. Done handling chunks $start to $end." +echo $(date) "Transferring ${file}, with a total of $chunkNumber chunks of $chunkSize bytes. Done handling chunks $start to $end." diff --git a/tools/update_project_configfiles.py b/tools/update_project_configfiles.py index e3f60f81f..b0da7ed42 100644 --- a/tools/update_project_configfiles.py +++ b/tools/update_project_configfiles.py @@ -25,8 +25,6 @@ 'test_fp', 'testHall', 'test_trans', - 'unsupported', - 'VelocityBox', 'verificationLarmor' ] diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index 00f256fd3..d6e17d60c 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -718,6 +718,7 @@ bool convertSILO(const string fileName, const uint compToExtract, map * orderedData, unordered_map& cellOrder, + Real& time, const bool& storeCellOrder=false) { bool success = true; @@ -747,6 +748,9 @@ bool convertSILO(const string fileName, return false; } } + + vlsvReader.readParameter("time", time); + vlsvReader.close(); return success; } @@ -935,11 +939,11 @@ bool outputDistance(const Real p, { if(verboseOutput == true) { if(shiftedAverage == false) { - cout << "The absolute " << p << "-distance between both datasets is " << *absolute << endl; - cout << "The relative " << p << "-distance between both datasets is " << *relative << endl; + cout << "The absolute " << p << "-distance between both datasets is " << setprecision(3) << *absolute << endl; + cout << "The relative " << p << "-distance between both datasets is " << setprecision(3) << *relative << endl; } else { - cout << "The average-shifted absolute " << p << "-distance between both datasets is " << *absolute << endl; - cout << "The average-shifted relative " << p << "-distance between both datasets is " << *relative << endl; + cout << "The average-shifted absolute " << p << "-distance between both datasets is " << setprecision(3) << *absolute << endl; + cout << "The average-shifted relative " << p << "-distance between both datasets is " << setprecision(3) << *relative << endl; } } else { static vector fileOutputData; @@ -948,7 +952,7 @@ bool outputDistance(const Real p, if(lastCall == true) { vector::const_iterator it; for(it = fileOutputData.begin(); it != fileOutputData.end(); it++) { - cout << *it << "\t"; + cout << setprecision(3) << *it << "\t"; } fileOutputData.clear(); return 0; @@ -960,6 +964,36 @@ bool outputDistance(const Real p, return 0; } +/*! In verbose mode print delta t, in non-verbose store them for later output when lastCall is true + * \param dt delta t + * \param verboseOutput Boolean parameter telling whether the output is verbose or compact + * \param lastCall Boolean parameter telling whether this is the last call to the function + */ +bool outputDt( + const Real dt, + const bool verboseOutput, + const bool lastCall +) { + if(verboseOutput == true) { + cout << "The delta t between both datasets is " << dt << endl; + } else { + static vector fileOutputData; + static uint fileNumber = 0; + + if(lastCall == true) { + vector::const_iterator it; + for(auto f : fileOutputData) { + cout << f << "\t"; + } + fileOutputData.clear(); + return 0; + } + + fileOutputData.push_back(dt); + } + return 0; +} + /*! Compute statistics on a single file * \param size Return argument pointer, dataset size * \param mini Return argument pointer, dataset minimum @@ -1099,6 +1133,7 @@ bool printNonVerboseData() // last argument (lastCall) is true to get the output of the whole stored dataset outputStats(NULL, NULL, NULL, NULL, NULL, false, true); outputDistance(0, NULL, NULL, false, false, true); + outputDt(0, false, true); return 0; } @@ -1570,17 +1605,23 @@ bool compareAvgs( const string fileName1, minDiff = *it; } } - + + Real time1 {0.0}; + Real time2 {0.0}; + vlsvReader1.readParameter("time", time1); + vlsvReader2.readParameter("time", time2); + const double relativeSumDiff = sumDiff / totalAbsAvgs; cout << "File names: " << fileName1 << " & " << fileName2 << endl << + setprecision(3) << "NonIdenticalBlocks: " << numOfNonIdenticalBlocks << endl << "IdenticalBlocks: " << numOfIdenticalBlocks << endl << "Absolute_Error: " << totalAbsDiff << endl << "Mean-Absolute-Error: " << totalAbsDiff / numOfRelevantCells << endl << "Max-Absolute-Error: " << maxDiff << endl << "Absolute-log-Error: " << totalAbsLog10Diff << endl << - "Mean-Absolute-log-Error: " << totalAbsLog10Diff / numOfRelevantCells << endl; - + "Mean-Absolute-log-Error: " << totalAbsLog10Diff / numOfRelevantCells << endl << + "Delta-t: " << time2 - time1 << endl; return true; } @@ -1613,19 +1654,23 @@ bool process2Files(const string fileName1, cellIds1.push_back(compToExtract); cellIds2.push_back(compToExtract2); // Compare files: - if( compareAvgs(fileName1, fileName2, verboseOutput, cellIds1, cellIds2) == false ) { return false; } + if (compareAvgs(fileName1, fileName2, verboseOutput, cellIds1, cellIds2) == false) { + return false; + } } else { unordered_map cellOrder; bool success = true; - success = convertSILO(fileName1, varToExtract, compToExtract, &orderedData1, cellOrder, true); + Real time1 {0.0}; + success = convertSILO(fileName1, varToExtract, compToExtract, &orderedData1, cellOrder, time1, true); if( success == false ) { cerr << "ERROR Data import error with " << fileName1 << endl; return 1; } - success = convertSILO(fileName2, varToExtract, compToExtract, &orderedData2, cellOrder, false); + Real time2 {0.0}; + success = convertSILO(fileName2, varToExtract, compToExtract, &orderedData2, cellOrder, time2, false); if( success == false ) { cerr << "ERROR Data import error with " << fileName2 << endl; @@ -1687,6 +1732,8 @@ bool process2Files(const string fileName1, pDistance(orderedData1, orderedData2, 2, &absolute, &relative, true, cellOrder,outputFile,attributes["--meshname"],"d2_sft_"+varName); outputDistance(2, &absolute, &relative, true, verboseOutput, false); + outputDt(time2 - time1, verboseOutput, false); + outputFile.close(); } @@ -1918,47 +1965,36 @@ int main(int argn,char* args[]) { DIR* dir1 = opendir(fileName1.c_str()); DIR* dir2 = opendir(fileName2.c_str()); - if (dir1 == NULL && dir2 == NULL) { + if (dir1 == nullptr && dir2 == nullptr) { cout << "INFO Reading in two files." << endl; // Process two files with verbose output (last argument true) process2Files(fileName1, fileName2, varToExtract, compToExtract, true, compToExtract2); - //CONTINUE - - closedir(dir1); - closedir(dir2); - } - else if (dir1 == NULL || dir2 == NULL) - { + } else if (dir1 == nullptr || dir2 == nullptr) { // Mixed file and directory cout << "#INFO Reading in one file and one directory." << endl; set fileList; - set::iterator it; - if(dir1 == NULL){ + if(dir1 == nullptr){ //file in 1, directory in 2 processDirectory(dir2, &fileList); - for(it = fileList.begin(); it != fileList.end();++it){ + for(auto f : fileList) { // Process two files with non-verbose output (last argument false), give full path to the file processor - process2Files(fileName1,fileName2 + "/" + *it, varToExtract, compToExtract, false, compToExtract2); + process2Files(fileName1,fileName2 + "/" + f, varToExtract, compToExtract, false, compToExtract2); } + closedir(dir2); } - if(dir2 == NULL){ + if(dir2 == nullptr){ //directory in 1, file in 2 processDirectory(dir1, &fileList); - for(it = fileList.begin(); it != fileList.end();++it){ + for(auto f : fileList) { // Process two files with non-verbose output (last argument false), give full path to the file processor - process2Files(fileName1+"/"+*it,fileName2, varToExtract, compToExtract, false, compToExtract2); + process2Files(fileName1 + "/" + f,fileName2, varToExtract, compToExtract, false, compToExtract2); } + closedir(dir1); } - - closedir(dir1); - closedir(dir2); - return 1; - } - else if (dir1 != NULL && dir2 != NULL) - { + } else if (dir1 && dir2) { // Process two folders, files of the same rank compared, first folder is reference in relative distances cout << "#INFO Reading in two directories." << endl; set fileList1, fileList2; @@ -1968,20 +2004,15 @@ int main(int argn,char* args[]) { processDirectory(dir2, &fileList2); // Basic consistency check - if(fileList1.size() != fileList2.size()) - { + if(fileList1.size() != fileList2.size()) { cerr << "ERROR Folders have different number of files." << endl; return 1; } - set::iterator it1, it2; - for(it1 = fileList1.begin(), it2 = fileList2.begin(); - it1 != fileList2.end(), it2 != fileList2.end(); - it1++, it2++) - { - // Process two files with non-verbose output (last argument false), give full path to the file processor - process2Files(fileName1 + "/" + *it1, - fileName2 + "/" + *it2, varToExtract, compToExtract, false, compToExtract2); + // TODO zip these once we're using C++23 + for (auto it1 = fileList1.begin(), it2 = fileList2.begin(); it1 != fileList2.end(), it2 != fileList2.end(); it1++, it2++) { + // Process two files with non-verbose output (last argument false), give full path to the file processor + process2Files(fileName1 + "/" + *it1, fileName2 + "/" + *it2, varToExtract, compToExtract, false, compToExtract2); } closedir(dir1); diff --git a/tools/vlsvreaderinterface.h b/tools/vlsvreaderinterface.h index 9d14e1166..f9bddfc0c 100644 --- a/tools/vlsvreaderinterface.h +++ b/tools/vlsvreaderinterface.h @@ -23,8 +23,6 @@ #ifndef VLSVREADER_INTERFACE_H #define VLSVREADER_INTERFACE_H -#define TOOL_NOT_PARALLEL - #include #include #include diff --git a/vlasiator.cpp b/vlasiator.cpp index d8175d691..c8f1df0ac 100644 --- a/vlasiator.cpp +++ b/vlasiator.cpp @@ -39,7 +39,7 @@ #include "logger.h" #include "parameters.h" #include "readparameters.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "datareduction/datareducer.h" #include "sysboundary/sysboundary.h" #include "fieldtracing/fieldtracing.h" @@ -293,7 +293,7 @@ int main(int argn,char* args[]) { const creal DT_EPSILON=1e-12; typedef Parameters P; Real newDt; - bool dtIsChanged; + bool dtIsChanged {false}; // Before MPI_Init we hardwire some settings, if we are in OpenMPI int required=MPI_THREAD_FUNNELED; @@ -334,7 +334,13 @@ int main(int argn,char* args[]) { exit(1); } if (myRank == MASTER_RANK) { - cout << mpiioMessage.str(); + const char* mpiioenv = std::getenv("OMPI_MCA_io"); + if(mpiioenv != nullptr) { + std::string mpiioenvstr(mpiioenv); + if(mpiioenvstr.find("^ompio") == std::string::npos) { + cout << mpiioMessage.str(); + } + } } phiprof::initialize(); @@ -592,9 +598,8 @@ int main(int argn,char* args[]) { ) { cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl; } - - phiprof::stop("Initialization"); - phiprof::stop("main"); + initTimer.stop(); + mainTimer.stop(); phiprof::print(MPI_COMM_WORLD,"phiprof"); @@ -748,6 +753,8 @@ int main(int argn,char* args[]) { if (P::dynamicTimestep == true && dtIsChanged == true) { // Only actually update the timestep if dynamicTimestep is on P::dt=newDt; + } else { + dtIsChanged = false; } computeDtimer.stop(); @@ -1053,6 +1060,12 @@ int main(int argn,char* args[]) { //TODO - add LB measure and do LB if it exceeds threshold if(((P::tstep % P::rebalanceInterval == 0 && P::tstep > P::tstep_min) || overrideRebalanceNow)) { logFile << "(LB): Start load balance, tstep = " << P::tstep << " t = " << P::t << endl << writeVerbose; + + phiprof::Timer shrinkTimer {"Shrink_to_fit"}; + // * shrink to fit before LB * // + shrink_to_fit_grid_data(mpiGrid); + shrinkTimer.stop(); + if (refineNow || (!dtIsChanged && P::adaptRefinement && P::tstep % (P::rebalanceInterval * P::refineCadence) == 0 && P::t > P::refineAfter)) { logFile << "(AMR): Adapting refinement!" << endl << writeVerbose; refineNow = false; @@ -1091,10 +1104,6 @@ int main(int argn,char* args[]) { // This now uses the block-based count just copied between the two refinement calls above. balanceLoad(mpiGrid, sysBoundaryContainer); addTimedBarrier("barrier-end-load-balance"); - phiprof::Timer shrinkTimer {"Shrink_to_fit"}; - // * shrink to fit after LB * // - shrink_to_fit_grid_data(mpiGrid); - shrinkTimer.stop(); logFile << "(LB): ... done!" << endl << writeVerbose; P::prepareForRebalance = false; diff --git a/vlasovmover.h b/vlasovmover.h index c097314ed..9289d8396 100644 --- a/vlasovmover.h +++ b/vlasovmover.h @@ -26,7 +26,7 @@ #include #include "definitions.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" using namespace spatial_cell; #include diff --git a/vlasovsolver/cpu_1d_ppm_nonuniform.hpp b/vlasovsolver/cpu_1d_ppm_nonuniform.hpp index 93eac5014..3ca3732c6 100644 --- a/vlasovsolver/cpu_1d_ppm_nonuniform.hpp +++ b/vlasovsolver/cpu_1d_ppm_nonuniform.hpp @@ -29,13 +29,14 @@ #include "cmath" #include "cpu_slope_limiters.hpp" #include "cpu_face_estimates.hpp" +#include "../definitions.h" using namespace std; /* Compute parabolic reconstruction with an explicit scheme */ -inline void compute_ppm_coeff_nonuniform(const Vec * const dv, const Vec * const values, face_estimate_order order, uint k, Vec a[3], const Realv threshold){ +inline void compute_ppm_coeff_nonuniform(const Realf * const dv, const Vec * const values, face_estimate_order order, uint k, Vec a[3], const Realv threshold){ Vec fv_l; /*left face value*/ Vec fv_r; /*right face value*/ compute_filtered_face_values_nonuniform(dv, values, k, order, fv_l, fv_r, threshold); diff --git a/vlasovsolver/cpu_acc_intersections.cpp b/vlasovsolver/cpu_acc_intersections.cpp index b0ae43385..c31b02074 100644 --- a/vlasovsolver/cpu_acc_intersections.cpp +++ b/vlasovsolver/cpu_acc_intersections.cpp @@ -32,7 +32,7 @@ #include #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "cpu_acc_intersections.hpp" using namespace std; diff --git a/vlasovsolver/cpu_acc_intersections.hpp b/vlasovsolver/cpu_acc_intersections.hpp index 19d3cbf5f..b75c69ab3 100644 --- a/vlasovsolver/cpu_acc_intersections.hpp +++ b/vlasovsolver/cpu_acc_intersections.hpp @@ -25,7 +25,7 @@ #include #include "../definitions.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" Eigen::Matrix line_plane_intersection(const Eigen::Matrix& l_point, const Eigen::Matrix& l_direction, diff --git a/vlasovsolver/cpu_acc_load_blocks.hpp b/vlasovsolver/cpu_acc_load_blocks.hpp index 87e94a2fc..e4e01e2d1 100644 --- a/vlasovsolver/cpu_acc_load_blocks.hpp +++ b/vlasovsolver/cpu_acc_load_blocks.hpp @@ -2,7 +2,7 @@ #define CPU_ACC_LOAD_BLOCKS_H #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "vec.h" diff --git a/vlasovsolver/cpu_acc_map.hpp b/vlasovsolver/cpu_acc_map.hpp index e20462fdd..579face38 100644 --- a/vlasovsolver/cpu_acc_map.hpp +++ b/vlasovsolver/cpu_acc_map.hpp @@ -24,7 +24,7 @@ #define CPU_ACC_MAP_H #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "vec.h" using namespace spatial_cell; diff --git a/vlasovsolver/cpu_acc_semilag.cpp b/vlasovsolver/cpu_acc_semilag.cpp index dcb66c250..8215a2068 100644 --- a/vlasovsolver/cpu_acc_semilag.cpp +++ b/vlasovsolver/cpu_acc_semilag.cpp @@ -91,12 +91,9 @@ void cpu_accelerate_cell(SpatialCell* spatial_cell, //vmesh::VelocityBlockContainer& blockContainer = spatial_cell->get_velocity_blocks(popID); // compute transform, forward in time and backward in time - phiprof::Timer transformTimer {"compute-transform"}; - //compute the transform performed in this acceleration Transform fwd_transform= compute_acceleration_transformation(spatial_cell,popID,dt); Transform bwd_transform= fwd_transform.inverse(); - transformTimer.stop(); const uint8_t refLevel = 0; Real intersection_z,intersection_z_di,intersection_z_dj,intersection_z_dk; diff --git a/vlasovsolver/cpu_acc_semilag.hpp b/vlasovsolver/cpu_acc_semilag.hpp index ef4e6fcde..62832fc85 100644 --- a/vlasovsolver/cpu_acc_semilag.hpp +++ b/vlasovsolver/cpu_acc_semilag.hpp @@ -24,7 +24,7 @@ #define CPU_ACC_SEMILAG_H #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" void prepareAccelerateCell(spatial_cell::SpatialCell* spatial_cell, const uint popID); uint getAccelerationSubcycles(spatial_cell::SpatialCell* spatial_cell, Real dt, const uint popID); diff --git a/vlasovsolver/cpu_acc_sort_blocks.cpp b/vlasovsolver/cpu_acc_sort_blocks.cpp index 8bffb12f0..f906d6142 100644 --- a/vlasovsolver/cpu_acc_sort_blocks.cpp +++ b/vlasovsolver/cpu_acc_sort_blocks.cpp @@ -42,7 +42,7 @@ inline bool paircomparator( const std::pair & l, const std::pair& vmesh, const uint dimension, diff --git a/vlasovsolver/cpu_acc_sort_blocks.hpp b/vlasovsolver/cpu_acc_sort_blocks.hpp index fd3f9ecf3..da67e4f99 100644 --- a/vlasovsolver/cpu_acc_sort_blocks.hpp +++ b/vlasovsolver/cpu_acc_sort_blocks.hpp @@ -27,7 +27,7 @@ #include #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" void sortBlocklistByDimension( //const spatial_cell::SpatialCell* spatial_cell, const vmesh::VelocityMesh& vmesh, diff --git a/vlasovsolver/cpu_acc_transform.hpp b/vlasovsolver/cpu_acc_transform.hpp index ba2d3614f..3a2413f90 100644 --- a/vlasovsolver/cpu_acc_transform.hpp +++ b/vlasovsolver/cpu_acc_transform.hpp @@ -27,7 +27,7 @@ #include #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" Eigen::Transform compute_acceleration_transformation( spatial_cell::SpatialCell* spatial_cell,const uint popID,const Real& dt); diff --git a/vlasovsolver/cpu_face_estimates.hpp b/vlasovsolver/cpu_face_estimates.hpp index 05d0f3d28..367aa8867 100644 --- a/vlasovsolver/cpu_face_estimates.hpp +++ b/vlasovsolver/cpu_face_estimates.hpp @@ -28,6 +28,7 @@ #include "algorithm" #include "cmath" #include "cpu_slope_limiters.hpp" +#include "../definitions.h" /*enum for setting face value and derivative estimates. Implicit ones not supported in the solver, so they are now not listed*/ @@ -177,7 +178,7 @@ inline void compute_h4_left_face_value(const Vec * const values, uint k, Vec &fv \param fv_l Face value on left face of cell i \param h Array with cell widths. Can be in abritrary units since they always cancel. Maybe 1/refinement ratio? */ -inline void compute_h4_left_face_value_nonuniform(const Vec * const h, const Vec * const u, uint k, Vec &fv_l) { +inline void compute_h4_left_face_value_nonuniform(const Realf * const h, const Vec * const u, uint k, Vec &fv_l) { fv_l = ( 1.0 / ( h[k - 2] + h[k - 1] + h[k] + h[k + 1] ) @@ -338,7 +339,7 @@ inline void compute_filtered_face_values(const Vec * const values,uint k, face_e } -inline void compute_filtered_face_values_nonuniform(const Vec * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ +inline void compute_filtered_face_values_nonuniform(const Realf * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ switch(order){ case h4: compute_h4_left_face_value_nonuniform(dv, values, k, fv_l); @@ -391,7 +392,7 @@ inline void compute_filtered_face_values_nonuniform(const Vec * const dv, const } } -inline Vec get_D2aLim(const Vec * h, const Vec * values, uint k, const Vec C, Vec & fv) { +inline Vec get_D2aLim(const Realf * h, const Vec * values, uint k, const Vec C, Vec & fv) { // Colella & Sekora, eq. 18 Vec invh2 = 1.0 / (h[k] * h[k]); @@ -410,7 +411,7 @@ inline Vec get_D2aLim(const Vec * h, const Vec * values, uint k, const Vec C, Ve } -inline pair constrain_face_values(const Vec * h,const Vec * values,uint k,Vec & fv_l, Vec & fv_r) { +inline pair constrain_face_values(const Realf * h,const Vec * values,uint k,Vec & fv_l, Vec & fv_r) { const Vec C = 1.25; Vec invh2 = 1.0 / (h[k] * h[k]); @@ -457,7 +458,7 @@ inline pair constrain_face_values(const Vec * h,const Vec * values,uint return faceInterpolants; } -inline void compute_filtered_face_values_nonuniform_conserving(const Vec * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ +inline void compute_filtered_face_values_nonuniform_conserving(const Realf * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ switch(order){ case h4: compute_h4_left_face_value_nonuniform(dv, values, k, fv_l); diff --git a/vlasovsolver/cpu_moments.cpp b/vlasovsolver/cpu_moments.cpp index ee53decf7..995dc12f5 100644 --- a/vlasovsolver/cpu_moments.cpp +++ b/vlasovsolver/cpu_moments.cpp @@ -28,8 +28,8 @@ using namespace std; -/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the - * given spatial cell. The calculated moments include contributions from +/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the + * given spatial cell. The calculated moments include contributions from * all existing particle populations. This function is VAMR safe. * @param cell Spatial cell. * @param computeSecond If true, second velocity moments are calculated. @@ -62,13 +62,20 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, if (skipMoments == false) { for (uint popID=0; popID& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; - + Population & pop = cell->get_population(popID); + if (blockContainer.size() == 0) { + pop.RHO = 0; + for (int i=0; i<3; ++i) { + pop.V[i]=0; + pop.P[i]=0; + } + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; const Real charge = getObjectWrapper().particleSpecies[popID].charge; - + // Temporary array for storing moments Real array[4]; for (int i=0; i<4; ++i) array[i] = 0.0; @@ -79,14 +86,13 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, blockParams+blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS, array); } - - Population & pop = cell->get_population(popID); + pop.RHO = array[0]; pop.V[0] = divideIfNonZero(array[1], array[0]); pop.V[1] = divideIfNonZero(array[2], array[0]); pop.V[2] = divideIfNonZero(array[3], array[0]); - - if(!computePopulationMomentsOnly) { + + if (!computePopulationMomentsOnly) { // Store species' contribution to bulk velocity moments cell->parameters[CellParams::RHOM ] += array[0]*mass; cell->parameters[CellParams::VX] += array[1]*mass; @@ -95,8 +101,8 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, cell->parameters[CellParams::RHOQ ] += array[0]*charge; } } // for-loop over particle species - - if(!computePopulationMomentsOnly) { + + if (!computePopulationMomentsOnly) { cell->parameters[CellParams::VX] = divideIfNonZero(cell->parameters[CellParams::VX], cell->parameters[CellParams::RHOM]); cell->parameters[CellParams::VY] = divideIfNonZero(cell->parameters[CellParams::VY], cell->parameters[CellParams::RHOM]); cell->parameters[CellParams::VZ] = divideIfNonZero(cell->parameters[CellParams::VZ], cell->parameters[CellParams::RHOM]); @@ -104,20 +110,25 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, } // Compute second moments only if requested - if (computeSecond == false) return; - + if (computeSecond == false) { + return; + } + // Loop over all particle species for (uint popID=0; popID& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; - + if (blockContainer.size() == 0) { + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; - + // Temporary array for storing moments Real array[3]; - for (int i=0; i<3; ++i) array[i] = 0.0; + for (int i=0; i<3; ++i) { + array[i] = 0.0; + } // Calculate species' contribution to second velocity moments Population & pop = cell->get_population(popID); @@ -129,13 +140,13 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, cell->parameters[CellParams::VZ], array); } - + // Store species' contribution to bulk velocity moments pop.P[0] = mass*array[0]; pop.P[1] = mass*array[1]; pop.P[2] = mass*array[2]; - - if(!computePopulationMomentsOnly) { + + if (!computePopulationMomentsOnly) { cell->parameters[CellParams::P_11] += pop.P[0]; cell->parameters[CellParams::P_22] += pop.P[1]; cell->parameters[CellParams::P_33] += pop.P[2]; @@ -143,9 +154,9 @@ void calculateCellMoments(spatial_cell::SpatialCell* cell, } // for-loop over particle species } -/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the - * given spatial cell. The calculated moments include - * contributions from all existing particle populations. The calculated moments +/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the + * given spatial cell. The calculated moments include + * contributions from all existing particle populations. The calculated moments * are stored to SpatialCell::parameters in _R variables. This function is VAMR safe. * @param mpiGrid Parallel grid library. * @param cells Vector containing the spatial cells to be calculated. @@ -154,18 +165,18 @@ void calculateMoments_R( dccrg::Dccrg& mpiGrid, const std::vector& cells, const bool& computeSecond) { - + phiprof::Timer momentsTimer {"compute-moments-n"}; for (uint popID=0; popIDsysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) { continue; } - + // Clear old moments to zero value if (popID == 0) { cell->parameters[CellParams::RHOM_R ] = 0.0; @@ -179,7 +190,15 @@ void calculateMoments_R( } vmesh::VelocityBlockContainer& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; + Population & pop = cell->get_population(popID); + if (blockContainer.size() == 0) { + pop.RHO_R = 0; + for (int i=0; i<3; ++i) { + pop.V_R[i]=0; + pop.P_R[i]=0; + } + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; @@ -211,12 +230,11 @@ void calculateMoments_R( } // for-loop over velocity blocks // Store species' contribution to bulk velocity moments - Population & pop = cell->get_population(popID); pop.RHO_R = array[0]; pop.V_R[0] = divideIfNonZero(array[1], array[0]); pop.V_R[1] = divideIfNonZero(array[2], array[0]); pop.V_R[2] = divideIfNonZero(array[3], array[0]); - + cell->parameters[CellParams::RHOM_R ] += array[0]*mass; cell->parameters[CellParams::VX_R] += array[1]*mass; cell->parameters[CellParams::VY_R] += array[2]*mass; @@ -224,7 +242,7 @@ void calculateMoments_R( cell->parameters[CellParams::RHOQ_R ] += array[0]*charge; } // for-loop over spatial cells } // for-loop over particle species - + #pragma omp parallel for for (size_t c=0; csysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) { continue; } - + vmesh::VelocityBlockContainer& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; + if (blockContainer.size() == 0) { + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; // Temporary array where species' contribution to 2nd moments is accumulated Real array[3]; - for (int i=0; i<3; ++i) array[i] = 0.0; + for (int i=0; i<3; ++i) { + array[i] = 0.0; + } // Calculate species' contribution to second velocity moments Population & pop = cell->get_population(popID); @@ -275,7 +297,7 @@ void calculateMoments_R( pop.P_R[0] = mass*array[0]; pop.P_R[1] = mass*array[1]; pop.P_R[2] = mass*array[2]; - + cell->parameters[CellParams::P_11_R] += pop.P_R[0]; cell->parameters[CellParams::P_22_R] += pop.P_R[1]; cell->parameters[CellParams::P_33_R] += pop.P_R[2]; @@ -284,10 +306,10 @@ void calculateMoments_R( } -/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the - * given spatial cell. Additionally, for each species, calculate the maximum - * spatial time step so that CFL(spatial)=1. The calculated moments include - * contributions from all existing particle populations. The calculated moments +/** Calculate zeroth, first, and (possibly) second bulk velocity moments for the + * given spatial cell. Additionally, for each species, calculate the maximum + * spatial time step so that CFL(spatial)=1. The calculated moments include + * contributions from all existing particle populations. The calculated moments * are stored to SpatialCell::parameters in _V variables. This function is VAMR safe. * @param mpiGrid Parallel grid library. * @param cells Vector containing the spatial cells to be calculated. @@ -296,19 +318,19 @@ void calculateMoments_V( dccrg::Dccrg& mpiGrid, const std::vector& cells, const bool& computeSecond) { - + phiprof::Timer momentsTimer {"Compute _V moments"}; - + // Loop over all particle species for (uint popID=0; popIDsysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) { continue; } - + // Clear old moments to zero value if (popID == 0) { cell->parameters[CellParams::RHOM_V ] = 0.0; @@ -322,7 +344,15 @@ void calculateMoments_V( } vmesh::VelocityBlockContainer& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; + Population & pop = cell->get_population(popID); + if (blockContainer.size() == 0) { + pop.RHO_V = 0; + for (int i=0; i<3; ++i) { + pop.V_V[i]=0; + pop.P_V[i]=0; + } + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; @@ -338,14 +368,13 @@ void calculateMoments_V( blockParams+blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS, array); } - + // Store species' contribution to bulk velocity moments - Population & pop = cell->get_population(popID); pop.RHO_V = array[0]; pop.V_V[0] = divideIfNonZero(array[1], array[0]); pop.V_V[1] = divideIfNonZero(array[2], array[0]); pop.V_V[2] = divideIfNonZero(array[3], array[0]); - + cell->parameters[CellParams::RHOM_V ] += array[0]*mass; cell->parameters[CellParams::VX_V] += array[1]*mass; cell->parameters[CellParams::VY_V] += array[2]*mass; @@ -353,7 +382,7 @@ void calculateMoments_V( cell->parameters[CellParams::RHOQ_V ] += array[0]*charge; } // for-loop over spatial cells } // for-loop over particle species - + #pragma omp parallel for for (size_t c=0; csysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) { continue; } vmesh::VelocityBlockContainer& blockContainer = cell->get_velocity_blocks(popID); - if (blockContainer.size() == 0) continue; + if (blockContainer.size() == 0) { + continue; + } const Realf* data = blockContainer.getData(); const Real* blockParams = blockContainer.getParameters(); const Real mass = getObjectWrapper().particleSpecies[popID].mass; // Temporary array where moments are stored Real array[3]; - for (int i=0; i<3; ++i) array[i] = 0.0; + for (int i=0; i<3; ++i) { + array[i] = 0.0; + } // Calculate species' contribution to second velocity moments Population & pop = cell->get_population(popID); @@ -400,16 +433,16 @@ void calculateMoments_V( cell->parameters[CellParams::VZ_V], array); } // for-loop over velocity blocks - + // Store species' contribution to 2nd bulk velocity moments pop.P_V[0] = mass*array[0]; pop.P_V[1] = mass*array[1]; pop.P_V[2] = mass*array[2]; - + cell->parameters[CellParams::P_11_V] += pop.P_V[0]; cell->parameters[CellParams::P_22_V] += pop.P_V[1]; cell->parameters[CellParams::P_33_V] += pop.P_V[2]; - + } // for-loop over spatial cells } // for-loop over particle species } diff --git a/vlasovsolver/cpu_moments.h b/vlasovsolver/cpu_moments.h index c5f112896..ce98931cb 100644 --- a/vlasovsolver/cpu_moments.h +++ b/vlasovsolver/cpu_moments.h @@ -30,7 +30,7 @@ #include "../definitions.h" #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" using namespace spatial_cell; diff --git a/vlasovsolver/cpu_slope_limiters.hpp b/vlasovsolver/cpu_slope_limiters.hpp index 8b49d1892..abc4a6376 100644 --- a/vlasovsolver/cpu_slope_limiters.hpp +++ b/vlasovsolver/cpu_slope_limiters.hpp @@ -58,7 +58,7 @@ inline Vec slope_limiter_sb(const Vec& l,const Vec& m, const Vec& r) { */ inline Vec slope_limiter_minmod(const Vec& l,const Vec& m, const Vec& r) { - Vec sign; + //Vec sign; Vec a=r-m; Vec b=m-l; return minmod(a,b); @@ -69,7 +69,7 @@ inline Vec slope_limiter_minmod(const Vec& l,const Vec& m, const Vec& r) { */ inline Vec slope_limiter_mc(const Vec& l,const Vec& m, const Vec& r) { - Vec sign; + //Vec sign; Vec a=r-m; Vec b=m-l; Vec minval=min(two*abs(a),two*abs(b)); diff --git a/vlasovsolver/cpu_trans_map.cpp b/vlasovsolver/cpu_trans_map.cpp index 05fa233fa..b5d667886 100644 --- a/vlasovsolver/cpu_trans_map.cpp +++ b/vlasovsolver/cpu_trans_map.cpp @@ -36,6 +36,7 @@ #include "cpu_1d_ppm_nonuniform.hpp" #include "cpu_1d_pqm.hpp" #include "cpu_trans_map.hpp" +#include "cpu_trans_pencils.hpp" // for do_translate_cell using namespace std; using namespace spatial_cell; @@ -55,15 +56,6 @@ using namespace spatial_cell; //#define i_trans_pt_blockv(j, k, b_k) ( ( (j) * WID + (k) * WID2 + ((b_k) + 1 ) * WID3) / VECL ) #define i_trans_pt_blockv(planeVectorIndex, planeIndex, blockIndex) ( planeVectorIndex + planeIndex * VEC_PER_PLANE + (blockIndex + 1) * VEC_PER_BLOCK) -//Is cell translated? It is not translated if DO_NO_COMPUTE or if it is sysboundary cell and not in first sysboundarylayer -bool do_translate_cell(SpatialCell* SC){ - if(SC->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE || - (SC->sysBoundaryLayer != 1 && SC->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY)) - return false; - else - return true; -} - /* * return INVALID_CELLID if the spatial neighbor does not exist, or if * it is a cell that is not computed. If the diff --git a/vlasovsolver/cpu_trans_map.hpp b/vlasovsolver/cpu_trans_map.hpp index 3f993cf7b..c65e61239 100644 --- a/vlasovsolver/cpu_trans_map.hpp +++ b/vlasovsolver/cpu_trans_map.hpp @@ -26,7 +26,7 @@ #include "vec.h" #include "../common.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" void compute_spatial_source_neighbors(const dccrg::Dccrg& mpiGrid, const CellID& cellID,const uint dimension,SpatialCell **neighbors); diff --git a/vlasovsolver/cpu_trans_map_amr.cpp b/vlasovsolver/cpu_trans_map_amr.cpp index e18b6edb4..61d33724a 100644 --- a/vlasovsolver/cpu_trans_map_amr.cpp +++ b/vlasovsolver/cpu_trans_map_amr.cpp @@ -5,7 +5,7 @@ #include "../object_wrapper.h" #include "../memoryallocation.h" #include "cpu_trans_map_amr.hpp" -#include "cpu_trans_map.hpp" +#include "cpu_trans_pencils.hpp" // use DCCRG version Nov 8th 2018 01482cfba8 @@ -13,695 +13,17 @@ using namespace std; using namespace spatial_cell; // indices in padded source block, which is of type Vec with VECL -// element sin each vector. b_k is the block index in z direction in -// ordinary space [- VLASOV_STENCIL_WIDTH to VLASOV_STENCIL_WIDTH], -// i,j,k are the cell ids inside on block (i in vector elements). -// Vectors with same i,j,k coordinates, but in different spatial cells, are consequtive -#define i_trans_ps_blockv(planeVectorIndex, planeIndex, blockIndex) ( (blockIndex) + VLASOV_STENCIL_WIDTH + ( (planeVectorIndex) + (planeIndex) * VEC_PER_PLANE ) * ( 1 + 2 * VLASOV_STENCIL_WIDTH) ) - -// indices in padded target block, which is of type Vec with VECL -// element sin each vector. b_k is the block index in z direction in -// ordinary space, i,j,k are the cell ids inside on block (i in vector -// elements). -#define i_trans_pt_blockv(planeVectorIndex, planeIndex, blockIndex) ( planeVectorIndex + planeIndex * VEC_PER_PLANE + (blockIndex + 1) * VEC_PER_BLOCK) - -#define i_trans_ps_blockv_pencil(planeVectorIndex, planeIndex, blockIndex, lengthOfPencil) ( (blockIndex) + VLASOV_STENCIL_WIDTH + ( (planeVectorIndex) + (planeIndex) * VEC_PER_PLANE ) * ( lengthOfPencil + 2 * VLASOV_STENCIL_WIDTH) ) - - -/* Get the one-dimensional neighborhood index for a given direction and neighborhood size. - * - * @param dimension spatial dimension of neighborhood - * @param stencil neighborhood size in cells - * @return neighborhood index that can be passed to DCCRG functions - */ -int getNeighborhood(const uint dimension, const uint stencil) { - - int neighborhood = 0; - - if (stencil == 1) { - switch (dimension) { - case 0: - neighborhood = VLASOV_SOLVER_TARGET_X_NEIGHBORHOOD_ID; - break; - case 1: - neighborhood = VLASOV_SOLVER_TARGET_Y_NEIGHBORHOOD_ID; - break; - case 2: - neighborhood = VLASOV_SOLVER_TARGET_Z_NEIGHBORHOOD_ID; - break; - } - } - - if (stencil > 1) { - switch (dimension) { - case 0: - neighborhood = VLASOV_SOLVER_X_NEIGHBORHOOD_ID; - break; - case 1: - neighborhood = VLASOV_SOLVER_Y_NEIGHBORHOOD_ID; - break; - case 2: - neighborhood = VLASOV_SOLVER_Z_NEIGHBORHOOD_ID; - break; - } - } - - return neighborhood; - -} - -void flagSpatialCellsForAmrCommunication(const dccrg::Dccrg& mpiGrid, - const vector& localPropagatedCells) { - - // Only flag/unflag cells if AMR is active - if (mpiGrid.get_maximum_refinement_level()==0) return; - - // return if there's no cells to flag - if(localPropagatedCells.size() == 0) { - std::cerr<<"No cells!"<SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = false; - - // In dimension, check iteratively if any neighbors up to VLASOV_STENCIL_WIDTH distance away are on a different process - const auto* NbrPairs = mpiGrid.get_neighbors_of(c, neighborhood); - - // Create list of unique distances - std::set< int > distancesplus; - std::set< int > distancesminus; - std::set foundNeighborsP; - std::set foundNeighborsM; - /** Using sets of cells as well, we should only get one distance per - (potentially less refined) cell. This should result in safe behaviour - as long as the neighborhood of a cell does not contain cells with a - refinement level more than 1 level apart from the cell itself. - */ - for (const auto& nbrPair : *NbrPairs) { - if(nbrPair.second[dimension] > 0) { - if (foundNeighborsP.find(nbrPair.first) == foundNeighborsP.end()) { - distancesplus.insert(nbrPair.second[dimension]); - foundNeighborsP.insert(nbrPair.first); - } - } - if(nbrPair.second[dimension] < 0) { - if (foundNeighborsM.find(nbrPair.first) == foundNeighborsM.end()) { - distancesminus.insert(-nbrPair.second[dimension]); - foundNeighborsM.insert(nbrPair.first); - } - } - } - - foundNeighborsP.clear(); - foundNeighborsM.clear(); - - int iSrc = VLASOV_STENCIL_WIDTH - 1; - // Iterate through positive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (auto it = distancesplus.begin(); it != distancesplus.end(); ++it) { - if (ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] == true) iSrc = -1; - if (iSrc < 0) break; // found enough elements - // Check all neighbors at distance *it - for (const auto& nbrPair : *NbrPairs) { - SpatialCell *ncell = mpiGrid[nbrPair.first]; - if (!ncell) continue; - int distanceInRefinedCells = nbrPair.second[dimension]; - if (distanceInRefinedCells == *it) { - if (foundNeighborsP.find(nbrPair.first) != foundNeighborsP.end()) continue; - foundNeighborsP.insert(nbrPair.first); - if (!mpiGrid.is_local(nbrPair.first)) { - ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = true; - break; - } - } - } // end loop over neighbors - iSrc--; - } // end loop over positive distances - - iSrc = VLASOV_STENCIL_WIDTH - 1; - // Iterate through negtive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (auto it = distancesminus.begin(); it != distancesminus.end(); ++it) { - if (ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] == true) iSrc = -1; - if (iSrc < 0) break; // found enough elements - // Check all neighbors at distance *it - for (const auto& nbrPair : *NbrPairs) { - SpatialCell *ncell = mpiGrid[nbrPair.first]; - if (!ncell) continue; - int distanceInRefinedCells = -nbrPair.second[dimension]; - if (distanceInRefinedCells == *it) { - if (foundNeighborsM.find(nbrPair.first) != foundNeighborsM.end()) continue; - foundNeighborsM.insert(nbrPair.first); - if (!mpiGrid.is_local(nbrPair.first)) { - ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = true; - break; - } - } - } // end loop over neighbors - iSrc--; - } // end loop over negative distances - } // end loop over local propagated cells - } // end loop over dimensions - return; -} - -/* Get pointers to spatial cells that are considered source cells for a pencil. - * Source cells are cells that the pencil reads data from to compute polynomial - * fits that are used for propagation in the vlasov solver. All cells included - * in the pencil + VLASOV_STENCIL_WIDTH cells on both ends are source cells. - * Invalid cells are replaced by closest good cells. - * Boundary cells are included. - * - * @param [in] mpiGrid DCCRG grid object - * @param [in] pencils pencil data struct - * @param [in] ipencil index of a pencil in the pencils data struct - * @param [in] dimension spatial dimension - * @param [out] sourceCells pointer to an array of pointers to SpatialCell objects for the source cells - */ -void computeSpatialSourceCellsForPencil(const dccrg::Dccrg& mpiGrid, - setOfPencils& pencils, - const uint iPencil, - const uint dimension, - SpatialCell **sourceCells){ - - // L = length of the pencil iPencil - int L = pencils.lengthOfPencils[iPencil]; - vector ids = pencils.getIds(iPencil); - - // These neighborhoods now include the AMR addition beyond the regular vlasov stencil - int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); - - // Get pointers for each cell id of the pencil - for (int i = 0; i < L; ++i) { - sourceCells[i + VLASOV_STENCIL_WIDTH] = mpiGrid[ids[i]]; - } - - // Insert pointers for neighbors of ids.front() and ids.back() - const auto* frontNbrPairs = mpiGrid.get_neighbors_of(ids.front(), neighborhood); - const auto* backNbrPairs = mpiGrid.get_neighbors_of(ids.back(), neighborhood); - - // Create list of unique distances in the negative direction from the first cell in pencil - std::set< int > distances; - for (const auto& nbrPair : *frontNbrPairs) { - if(nbrPair.second[dimension] < 0) { - // gather positive distance values - distances.insert(-nbrPair.second[dimension]); - } - } - - int iSrc = VLASOV_STENCIL_WIDTH - 1; - - // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (auto it = distances.begin(); it != distances.end(); ++it) { - if (iSrc < 0) break; // found enough elements - - // Collect all neighbors at distance *it to a vector - std::vector< CellID > neighbors; - for (const auto& nbrPair : *frontNbrPairs) { - int distanceInRefinedCells = -nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); - } - // Get rid of duplicate neighbor cells at single distance - neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); - - int refLvl = mpiGrid.get_refinement_level(ids.front()); - if (neighbors.size() == 1) { - if (sourceCells[iSrc+1] == mpiGrid[neighbors.at(0)]) continue; // already found this cell for different distance - sourceCells[iSrc--] = mpiGrid[neighbors.at(0)]; - } else if ( pencils.path[iPencil][refLvl] < neighbors.size() ) { - if (sourceCells[iSrc+1] == mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]) continue; // already found this cell for different distance (should not happen) - sourceCells[iSrc--] = mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]; - // Code for alternate approach to verify that multiple neighbors are in correct ordering (z-y-x) - // int ix=0,iy=0; - // switch(dimension) { - // case 0: - // ix = 1; - // iy = 2; - // break; - // case 1: - // ix = 0; - // iy = 2; - // break; - // case 2: - // ix = 0; - // iy = 1; - // break; - // } - // bool accept = false; - // std::array parentCoords = mpiGrid.get_center(ids.front()); - // for (CellID n : neighbors) { - // std::array myCoords = mpiGrid.get_center(n); - // switch (pencils.path[iPencil][refLvl]) { - // case 0: - // if (myCoords[ix] < parentCoords[ix] && myCoords[iy] < parentCoords[iy]) accept=true; - // break; - // case 1: - // if (myCoords[ix] > parentCoords[ix] && myCoords[iy] < parentCoords[iy]) accept=true; - // break; - // case 2: - // if (myCoords[ix] < parentCoords[ix] && myCoords[iy] > parentCoords[iy]) accept=true; - // break; - // case 3: - // if (myCoords[ix] > parentCoords[ix] && myCoords[iy] > parentCoords[iy]) accept=true; - // break; - // } - // if (accept) { - // sourceCells[iSrc--] = mpiGrid[n]; - // break; - // } - // } - } else { - std::cerr<<"error too few neighbors for path! "< 0) { - distances.insert(nbrPair.second[dimension]); - } - } - - // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - // Distances are positive here so smallest distance has smallest value. - for (auto it = distances.begin(); it != distances.end(); ++it) { - if (iSrc >= L+2*VLASOV_STENCIL_WIDTH) break; // Found enough cells - - // Collect all neighbors at distance *it to a vector - std::vector< CellID > neighbors; - for (const auto& nbrPair : *backNbrPairs) { - int distanceInRefinedCells = nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); - } - // Get rid of duplicate neighbor cells at single distance - neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); - - int refLvl = mpiGrid.get_refinement_level(ids.back()); - if (neighbors.size() == 1) { - if (sourceCells[iSrc-1] == mpiGrid[neighbors.at(0)]) continue; // already found this cell for different distance - sourceCells[iSrc++] = mpiGrid[neighbors.at(0)]; - } else if ( pencils.path[iPencil][refLvl] < neighbors.size() ) { - if (sourceCells[iSrc-1] == mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]) continue; // already found this cell for different distance (should not happen) - sourceCells[iSrc++] = mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]; - } else { - std::cerr<<"error too few neighbors for path!"<= 0 ;--i){ - if(sourceCells[i] == NULL || sourceCells[i]->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) - sourceCells[i] = lastGoodCell; - else - lastGoodCell = sourceCells[i]; - } - /*loop to positive side and replace all invalid cells with the closest good cell*/ - lastGoodCell = mpiGrid[ids.back()]; - for(int i = VLASOV_STENCIL_WIDTH + L; i < (L + 2*VLASOV_STENCIL_WIDTH); ++i){ - if(sourceCells[i] == NULL || sourceCells[i]->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) - sourceCells[i] = lastGoodCell; - else - lastGoodCell = sourceCells[i]; - } -} - - -/* Get pointers to spatial cells that are considered target cells for all pencils. - * Target cells are cells that the pencil writes data into after translation by - * the vlasov solver. All cells included in the pencil + 1 cells on both ends - * are source cells. Boundary cells are not included. - * Now uses get_face_neighbors_of(). - * - * @param [in] mpiGrid DCCRG grid object - * @param [in] pencils pencil data struct - * @param [in] dimension spatial dimension - * @param [out] targetCells pointer to an array of pointers to SpatialCell objects for the target cells - * - */ -void computeSpatialTargetCellsForPencilsWithFaces(const dccrg::Dccrg& mpiGrid, - setOfPencils& pencils, - const uint dimension, - SpatialCell **targetCells){ - - uint GID = 0; - // Loop over pencils - for(uint iPencil = 0; iPencil < pencils.N; iPencil++){ - int L = pencils.lengthOfPencils[iPencil]; - vector ids = pencils.getIds(iPencil); - - // Get pointers for each cell id of the pencil - for (int i = 0; i < L; ++i) { - targetCells[GID + i + 1] = mpiGrid[ids[i]]; - } - - int refLvl; - vector frontNeighborIds; - vector backNeighborIds; - const auto& frontNeighbors = mpiGrid.get_face_neighbors_of(ids.front()); - if (frontNeighbors.size() > 0) { - for (const auto& [neighbor, dir] : frontNeighbors) { - if(dir == (-((int)dimension + 1))) { - frontNeighborIds.push_back(neighbor); - } - } - refLvl = mpiGrid.get_refinement_level(ids.front()); - - if (frontNeighborIds.size() == 0) { - std::cerr<<"abort frontNeighborIds.size() == 0 at "< 0) { - for (const auto& [neighbor, dir] : backNeighbors) { - if(dir == ((int)dimension + 1)) { - backNeighborIds.push_back(neighbor); - } - } - refLvl = mpiGrid.get_refinement_level(ids.back()); - if (backNeighborIds.size() == 0) { - std::cerr<<"abort backNeighborIds.size() == 0 at "<sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY ) { - targetCells[i] = NULL; - } - } -} - -/* Select one nearest neighbor of a cell on the + side in a given dimension. If the neighbor - * has a higher level of refinement, a path variable is needed to make the selection. - * Returns INVALID_CELLID if the nearest neighbor is not local to this process. - * - * @param grid DCCRG grid object - * @param id DCCRG cell id - * @param dimension spatial dimension - * @param path index of the desired face neighbor - * @return neighbor DCCRG cell id of the neighbor - */ -CellID selectNeighbor(const dccrg::Dccrg &grid, - CellID id, int dimension = 0, uint path = 0) { - - //int neighborhood = getNeighborhood(dimension,1); - //const auto* nbrPairs = grid.get_neighbors_of(id, neighborhood); - - vector < CellID > myNeighbors; - CellID neighbor = INVALID_CELLID; - - // Iterate through neighbor ids in the positive direction of the chosen dimension, - // select the neighbor indicated by path, if it is local to this process. - for (const auto& [neighbor, dir] : grid.get_face_neighbors_of(id)) { - if (dir == ((int)dimension + 1)) { - myNeighbors.push_back(neighbor); - } - } - - if( myNeighbors.size() == 0 ) { - return neighbor; - } - - int neighborIndex = 0; - if (myNeighbors.size() > 1) { - neighborIndex = path; - } - - if (grid.is_local(myNeighbors[neighborIndex])) { - neighbor = myNeighbors[neighborIndex]; - } - - return neighbor; -} - -/* Recursive function for building one-dimensional pencils to cover local DCCRG cells. - * Starts from a given seedID and proceeds finding the nearest neighbor in the given dimension - * and adding it to the pencil until no neighbors are found or an endId is met. When a higher - * refinement level (ie. multiple nearest neighbors) is met, the pencil splits into four - * copies to remain at a width of 1 cell. This is done by the function calling itself recursively - * and passing as inputs the cells added so far. The cell selected by each copy of the function - * at a split is stored in the path variable, the same path has to be followed if a refinement - * level is encoutered multiple times. - * - * @param [in] grid DCCRG grid object - * @param [out] pencils Pencil data struct - * @param [in] seedId DCCRG cell id where we start building the pencil. - * The pencil will continue in the + direction in the given dimension until an end condition is met - * @param [in] dimension Spatial dimension - * @param [in] path Integer value that determines which neighbor is added to the pencil when a higher refinement level is met - * @param [in] endIds Prescribed end conditions for the pencil. If any of these cell ids is about to be added to the pencil, - * the builder terminates. - */ -setOfPencils buildPencilsWithNeighbors( const dccrg::Dccrg &grid, - setOfPencils &pencils, const CellID seedId, - vector ids, const uint dimension, - vector path, const vector &endIds) { - - const bool debug = false; - CellID nextNeighbor; - CellID id = seedId; - int startingRefLvl = grid.get_refinement_level(id); - bool periodic = false; - // If this is a new pencil (instead of being a result of a pencil being split - if( ids.size() == 0 ) - ids.push_back(seedId); - - // If the cell where we start is refined, we need to figure out which path - // to follow in future refined cells. This is a bit hacky but we have to - // use the order or the children of the parent cell to figure out which - // corner we are in. - - std::array coordinates = grid.get_center(seedId); - int startingPathSize = path.size(); - auto it = path.end(); - if( startingRefLvl > startingPathSize ) { - - CellID myId = seedId; - - for ( int i = path.size(); i < startingRefLvl; ++i) { - - //CellID parentId = grid.mapping.get_parent(myId); - CellID parentId = grid.get_parent(myId); - - auto myCoords = grid.get_center(myId); - auto parentCoords = grid.get_center(parentId); - - int ix=0,iy=0; - - switch(dimension) { - case 0: - ix = 1; - iy = 2; - break; - case 1: - ix = 0; - iy = 2; - break; - case 2: - ix = 0; - iy = 1; - break; - } - //int ix = (dimension + 1) % 3; // incorrect for DCCRG - //int iy = (dimension + 2) % 3; - - int step = -1; - - if (myCoords[ix] < parentCoords[ix] && myCoords[iy] < parentCoords[iy]) { - step = 0; - } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] < parentCoords[iy]) { - step = 1; - } else if (myCoords[ix] < parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { - step = 2; - } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { - step = 3; - } - - it = path.insert(it, step); - - myId = parentId; - } - } - - while (id != INVALID_CELLID) { - - periodic = false; - bool neighborExists = false; - int refLvl = 0; - - // Find the refinement level in the neighboring cell. Check all possible neighbors - // in case some of them are remote. - for (int tmpPath = 0; tmpPath < 4; ++tmpPath) { - nextNeighbor = selectNeighbor(grid,id,dimension,tmpPath); - if(nextNeighbor != INVALID_CELLID) { - refLvl = max(refLvl,grid.get_refinement_level(nextNeighbor)); - neighborExists = true; - } - } - - // If there are no neighbors, we can stop. - if (!neighborExists) - break; - - if (refLvl > 0) { - - // If we have encountered this refinement level before and stored - // the path this builder follows, we will just take the same path - // again. - if ( static_cast(path.size()) >= refLvl ) { - - if(debug) { - std::cout << "I am cell " << id << ". "; - std::cout << "I have seen refinement level " << refLvl << " before. Path is "; - for (auto k = path.begin(); k != path.end(); ++k) - std::cout << *k << " "; - std::cout << std::endl; - } - - nextNeighbor = selectNeighbor(grid,id,dimension,path[refLvl - 1]); - coordinates = grid.get_center(nextNeighbor); - - } else { - - if(debug) { - std::cout << "I am cell " << id << ". "; - std::cout << "I have NOT seen refinement level " << refLvl << " before. Path is "; - for (auto k = path.begin(); k != path.end(); ++k) - std::cout << *k << ' '; - std::cout << std::endl; - } - - // New refinement level, create a path through each neighbor cell - for ( uint i : {0,1,2,3} ) { - - vector < uint > myPath = path; - myPath.push_back(i); - - nextNeighbor = selectNeighbor(grid,id,dimension,myPath.back()); - - if ( i == 3 ) { - - // This builder continues with neighbor 3 - path = myPath; - coordinates = grid.get_center(nextNeighbor); +// elements in each vector. - } else { - - // Spawn new builders for neighbors 0,1,2 - buildPencilsWithNeighbors(grid,pencils,id,ids,dimension,myPath,endIds); - - } - - } - - } - - } else { - if(debug) { - std::cout << "I am cell " << id << ". "; - std::cout << " This pencil has reached refinement level 0." << std::endl; - } - }// Closes if (refLvl == 0) - - // If we found a neighbor, add it to the list of ids for this pencil. - if(nextNeighbor != INVALID_CELLID) { - if (debug) { - std::cout << " Next neighbor is " << nextNeighbor << "." << std::endl; - } +#define i_trans_ps_blockv_pencil(planeVectorIndex, planeIndex, blockIndex, lengthOfPencil) ( (blockIndex) + ( (planeVectorIndex) + (planeIndex) * VEC_PER_PLANE ) * ( lengthOfPencil) ) - if ( std::any_of(endIds.begin(), endIds.end(), [nextNeighbor](uint i){return i == nextNeighbor;}) || - !do_translate_cell(grid[nextNeighbor])) { - - nextNeighbor = INVALID_CELLID; - } else { - ids.push_back(nextNeighbor); - } - } - - id = nextNeighbor; - } // Closes while loop - - // Get the x,y - coordinates of the pencil (in the direction perpendicular to the pencil) - double x,y; - int ix=0,iy=0; - - switch(dimension) { - case 0: - ix = 1; - iy = 2; - break; - case 1: - ix = 0; - iy = 2; - break; - case 2: - ix = 0; - iy = 1; - break; - } - //ix = (dimension + 1) % 3; // incorrect for DCCRG - //iy = (dimension + 2) % 3; - - x = coordinates[ix]; - y = coordinates[iy]; - - pencils.addPencil(ids,x,y,periodic,path); - - // TODO why do we have both return value and the argument modified in place? Could be made consistent. - return pencils; - -} - -bool check_skip_remapping(Vec* values) { - for (int index=0; index<2*VLASOV_STENCIL_WIDTH+1; ++index) { +inline bool check_skip_remapping(Vec* values) { + for (int index=-VLASOV_STENCIL_WIDTH; index Vec(0))) return false; } return true; } + /* Propagate a given velocity block in all spatial cells of a pencil by a time step dt using a PPM reconstruction. * * @param dz Width of spatial cells in the direction of the pencil, vector datatype @@ -713,15 +35,17 @@ bool check_skip_remapping(Vec* values) { * @param lengthOfPencil Number of cells in the pencil */ void propagatePencil( - Vec* dz, - Vec* values, - Vec* targetValues, // thread-owned aligned-allocated + Realf* dz, + Vec* values, // Vec-ordered block data values for pencils const uint dimension, const uint blockGID, const Realv dt, const vmesh::VelocityMesh &vmesh, const uint lengthOfPencil, - const Realv threshold + const Realv threshold, + Realf** blockDataPointer, // Spacing is for sources, but will be written into + Realf* targetRatios, // Vector holding target ratios + const unsigned int* const cellid_transpose ) { // Get velocity data from vmesh that we need later to calculate the translation velocity_block_indices_t block_indices; @@ -729,35 +53,34 @@ void propagatePencil( vmesh.getIndices(blockGID,refLevel, block_indices[0], block_indices[1], block_indices[2]); Realv dvz = vmesh.getCellSize(refLevel)[dimension]; Realv vz_min = vmesh.getMeshMinLimits()[dimension]; - + // Assuming 1 neighbor in the target array because of the CFL condition // In fact propagating to > 1 neighbor will give an error // Also defined in the calling function for the allocation of targetValues - const uint nTargetNeighborsPerPencil = 1; - - for (uint i = 0; i < (lengthOfPencil + 2 * nTargetNeighborsPerPencil) * WID3 / VECL; i++) { - - // init target_values - targetValues[i] = Vec(0.0); - - } - - // Go from 0 to length here to propagate all the cells in the pencil - for (uint i = 0; i < lengthOfPencil; i++){ - - // The source array is padded by VLASOV_STENCIL_WIDTH on both sides. - uint i_source = i + VLASOV_STENCIL_WIDTH; - + // Go over length of propagated cells + for (int i = VLASOV_STENCIL_WIDTH; i < (int)lengthOfPencil-VLASOV_STENCIL_WIDTH; i++){ + // Get pointers to block data used for output. + // CUDATODO: use blockGID to get pointers here + Realf* block_data_m1 = blockDataPointer[i - 1]; + Realf* block_data = blockDataPointer[i]; + Realf* block_data_p1 = blockDataPointer[i + 1]; + // Cells which shouldn't be written to (e.g. sysboundary cells) have a targetRatio of 0 + // Also need to check if pointer is valid, because a cell can be missing an elsewhere propagated block + Realf areaRatio_m1 = targetRatios[i - 1]; + Realf areaRatio = targetRatios[i]; + Realf areaRatio_p1 = targetRatios[i + 1]; + + Realf vector[VECL]; + // Loop over planes for (uint k = 0; k < WID; ++k) { - const Realv cell_vz = (block_indices[dimension] * WID + k + 0.5) * dvz + vz_min; //cell centered velocity - const Vec z_translation = cell_vz * dt / dz[i_source]; // how much it moved in time dt (reduced units) + const Vec z_translation = cell_vz * dt / dz[i]; // how much it moved in time dt (reduced units) // Determine direction of translation // part of density goes here (cell index change along spatial direcion) Vecb positiveTranslationDirection = (z_translation > Vec(0.0)); - + // Calculate normalized coordinates in current cell. // The coordinates (scaled units from 0 to 1) between which we will // integrate to put mass in the target neighboring cell. @@ -772,284 +95,94 @@ void propagatePencil( // std::cout << "Exiting\n"; // std::exit(1); // } - - for (uint planeVector = 0; planeVector < VEC_PER_PLANE; planeVector++) { + // Loop over Vec's in current plance + for (uint planeVector = 0; planeVector < VEC_PER_PLANE; planeVector++) { // Check if all values are 0: - if (check_skip_remapping(values + i_trans_ps_blockv_pencil(planeVector, k, i-VLASOV_STENCIL_WIDTH, lengthOfPencil))) continue; + if (check_skip_remapping(values + i_trans_ps_blockv_pencil(planeVector, k, i, lengthOfPencil))) continue; // Compute polynomial coefficients Vec a[3]; - // Dz: is a padded array, pointer can point to the beginning, i + VLASOV_STENCIL_WIDTH will get the right cell. - // values: transpose function adds VLASOV_STENCIL_WIDTH to the block index, therefore we substract it here, then - // i + VLASOV_STENCIL_WIDTH will point to the right cell. Complicated! Why! Sad! MVGA! - compute_ppm_coeff_nonuniform(dz + i, - values + i_trans_ps_blockv_pencil(planeVector, k, i-VLASOV_STENCIL_WIDTH, lengthOfPencil), + // Silly indexing into coefficient calculation necessary due to built-in assumptions of unsigned indexing. + compute_ppm_coeff_nonuniform(dz + i - VLASOV_STENCIL_WIDTH, + values + i_trans_ps_blockv_pencil(planeVector, k, i, lengthOfPencil) - VLASOV_STENCIL_WIDTH, h4, VLASOV_STENCIL_WIDTH, a, threshold); - + // Compute integral const Vec ngbr_target_density = z_2 * ( a[0] + z_2 * ( a[1] + z_2 * a[2] ) ) - z_1 * ( a[0] + z_1 * ( a[1] + z_1 * a[2] ) ); - + // Store mapped density in two target cells - // in the neighbor cell we will put this density - targetValues[i_trans_pt_blockv(planeVector, k, i + 1)] += select( positiveTranslationDirection, - ngbr_target_density - * dz[i_source] / dz[i_source + 1], - Vec(0.0)); - targetValues[i_trans_pt_blockv(planeVector, k, i - 1 )] += select(!positiveTranslationDirection, - ngbr_target_density - * dz[i_source] / dz[i_source - 1], - Vec(0.0)); - // in the current original cells we will put the rest of the original density - targetValues[i_trans_pt_blockv(planeVector, k, i)] += - values[i_trans_ps_blockv_pencil(planeVector, k, i, lengthOfPencil)] - ngbr_target_density; - } - } - } -} - -/* Determine which cells in the local DCCRG mesh should be starting points for pencils. - * If a neighbor cell is non-local, across a periodic boundary, or in non-periodic boundary layer 1 - * then we use this cell as a seed for pencils - * - * @param [in] mpiGrid DCCRG grid object - * @param [in] localPropagatedCells List of local cells that get propagated - * ie. not boundary or DO_NOT_COMPUTE - * @param [in] dimension Spatial dimension - * @param [out] seedIds list of cell ids that will be starting points for pencils - */ -void getSeedIds(const dccrg::Dccrg& mpiGrid, - const vector &localPropagatedCells, - const uint dimension, - vector &seedIds) { - - const bool debug = false; - int myRank; - if (debug) MPI_Comm_rank(MPI_COMM_WORLD,&myRank); - - // These neighborhoods now include the AMR addition beyond the regular vlasov stencil - int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); - -#pragma omp parallel for - for (uint i=0; i pow(2,mpiGrid.get_maximum_refinement_level()) || - !mpiGrid.is_local(neighbor) || - !do_translate_cell(mpiGrid[neighbor]) ) - { - addToSeedIds = true; - break; - } - } - } // finish check A - if ( addToSeedIds ) { -#pragma omp critical - seedIds.push_back(celli); - continue; - } - myRefLevel = mpiGrid.get_refinement_level(celli); - if (mpiGrid.get_maximum_refinement_level() == myRefLevel) continue; - - // Gather neighbours in neighbourhood stencil - const auto* nbrPairs = mpiGrid.get_neighbors_of(celli, neighborhood); - // Create list of unique neighbour distances in both directions - std::set< int > distancesplus; - std::set< int > distancesminus; - for (const auto& nbrPair : *nbrPairs) { - if(nbrPair.second[dimension] > 0) { - distancesplus.insert(nbrPair.second[dimension]); - } - if(nbrPair.second[dimension] < 0) { - // gather positive distance values - distancesminus.insert(-nbrPair.second[dimension]); - } - } - /* Proceed with B, checking if the next positive neighbour has the same refinement level as ccell, but the - second neighbour a higher one. Iterate through positive distances for VLASOV_STENCIL_WIDTH elements - starting from the smallest distance. */ - int iSrc = VLASOV_STENCIL_WIDTH-1; - for (auto it = distancesplus.begin(); it != distancesplus.end(); ++it) { - if (iSrc < 0) break; // found enough elements - for (const auto& nbrPair : *nbrPairs) { - int distanceInRefinedCells = nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) { - // Break search if we are not at the final entry, and have different refinement level - if (iSrc!=0 && mpiGrid.get_refinement_level(nbrPair.first)!=myRefLevel) { - iSrc = -1; - break; - } - // Flag as seed id if VLASOV_STENCIL_WIDTH positive neighbour is at higher refinement level - if (iSrc==0 && mpiGrid.get_refinement_level(nbrPair.first)>myRefLevel) { - addToSeedIds = true; - break; + if (areaRatio && block_data) { + const Vec selfContribution = (values[i_trans_ps_blockv_pencil(planeVector, k, i, lengthOfPencil)] - ngbr_target_density) * areaRatio; + selfContribution.store(vector); + // Loop over 3rd (vectorized) vspace dimension + #pragma omp simd + for (uint iv = 0; iv < VECL; iv++) { + block_data[cellid_transpose[iv + planeVector * VECL + k * WID2]] += vector[iv]; } } - } - iSrc--; - } // Finish B check - - if ( addToSeedIds ) { - #pragma omp critical - seedIds.push_back(celli); - continue; - } - /* Proceed with C, checking if the next two negative neighbours have the same refinement level as ccell, but the - third neighbour a higher one. Iterate through negative distances for VLASOV_STENCIL_WIDTH+1 elements - starting from the smallest distance. */ - iSrc = VLASOV_STENCIL_WIDTH; - for (auto it = distancesminus.begin(); it != distancesminus.end(); ++it) { - if (iSrc < 0) break; // found enough elements - for (const auto& nbrPair : *nbrPairs) { - int distanceInRefinedCells = -nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) { - // Break search if we are not at the final entry, and have different refinement level - if (iSrc!=0 && mpiGrid.get_refinement_level(nbrPair.first)!=myRefLevel) { - iSrc = -1; - break; + if (areaRatio_p1 && block_data_p1) { + const Vec p1Contribution = select(positiveTranslationDirection, ngbr_target_density + * dz[i] / dz[i + 1], Vec(0.0)) * areaRatio_p1; + p1Contribution.store(vector); + // Loop over 3rd (vectorized) vspace dimension + #pragma omp simd + for (uint iv = 0; iv < VECL; iv++) { + block_data_p1[cellid_transpose[iv + planeVector * VECL + k * WID2]] += vector[iv]; } - // Flag as seed id if VLASOV_STENCIL_WIDTH+1 positive neighbour is at higher refinement level - if (iSrc==0 && mpiGrid.get_refinement_level(nbrPair.first)>myRefLevel) { - addToSeedIds = true; - break; + } + if (areaRatio_m1 && block_data_m1) { + const Vec m1Contribution = select(!positiveTranslationDirection, ngbr_target_density + * dz[i] / dz[i - 1], Vec(0.0)) * areaRatio_m1; + m1Contribution.store(vector); + // Loop over 3rd (vectorized) vspace dimension + #pragma omp simd + for (uint iv = 0; iv < VECL; iv++) { + block_data_m1[cellid_transpose[iv + planeVector * VECL + k * WID2]] += vector[iv]; } } } - iSrc--; - } // Finish C check - - if ( addToSeedIds ) { -#pragma omp critical - seedIds.push_back(celli); } } - - if(debug) { - cout << "Rank " << myRank << ", Seed ids are: "; - for (const auto seedId : seedIds) { - cout << seedId << " "; - } - cout << endl; - } } - - - -/* Copy the data to the temporary values array, so that the - * dimensions are correctly swapped. Also, copy the same block for - * then neighboring spatial cells (in the dimension). neighbors - * generated with compute_spatial_neighbors_wboundcond). - * Adapted from copy_trans_block_data to be suitable for use with - * AMR and pencils. +/* Copy the pencil source data to the temporary values array, so that the + * dimensions are correctly swapped. * * This function must be thread-safe. * - * @param source_neighbors Array containing the VLASOV_STENCIL_WIDTH closest - * spatial neighbors of this cell in the propagated dimension. - * @param blockGID Global ID of the velocity block. - * @param int lengthOfPencil Number of spatial cells in pencil - * @param values Vector where loaded data is stored. + * @param blockDataPointer Vector of pre-prepared pointers to input (cell) block data + * @param start Index from blockDataPointer to start at + * @param int lengthOfPencil Number of spatial cells in pencil (not inclusive 2*VLASOV_STENCIL_WIDTH + * @param values Vector into which the data should be loaded * @param cellid_transpose * @param popID ID of the particle species. */ bool copy_trans_block_data_amr( - SpatialCell** source_neighbors, - const vmesh::GlobalID blockGID, - int lengthOfPencil, - Vec* values, - const unsigned char* const cellid_transpose, - const uint popID) { - - // Allocate data pointer for all blocks in pencil. Pad on both ends by VLASOV_STENCIL_WIDTH - Realf* blockDataPointer[lengthOfPencil + 2 * VLASOV_STENCIL_WIDTH]; - - int nonEmptyBlocks = 0; - - for (int b = -VLASOV_STENCIL_WIDTH; b < lengthOfPencil + VLASOV_STENCIL_WIDTH; b++) { - // Get cell pointer and local block id - SpatialCell* srcCell = source_neighbors[b + VLASOV_STENCIL_WIDTH]; - - const vmesh::LocalID blockLID = srcCell->get_velocity_block_local_id(blockGID,popID); - if (blockLID != srcCell->invalid_local_id()) { - // Get data pointer - blockDataPointer[b + VLASOV_STENCIL_WIDTH] = srcCell->get_data(blockLID,popID); - nonEmptyBlocks++; - // //prefetch storage pointers to L1 - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]), _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 64, _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 128, _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 192, _MM_HINT_T0); - // if(VPREC == 8) { - // //prefetch storage pointers to L1 - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 256, _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 320, _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 384, _MM_HINT_T0); - // _mm_prefetch((char *)(blockDataPointer[b + VLASOV_STENCIL_WIDTH]) + 448, _MM_HINT_T0); - // } - - } else { - blockDataPointer[b + VLASOV_STENCIL_WIDTH] = NULL; - } - } - - if(nonEmptyBlocks == 0) { - return false; - } - + Realf** pencilBlockData, + const int lengthOfPencil, + Vec* values, + const unsigned int* const cellid_transpose, + const uint popID) { + // Copy volume averages of this block from all spatial cells: - for (int b = -VLASOV_STENCIL_WIDTH; b < lengthOfPencil + VLASOV_STENCIL_WIDTH; b++) { - if(blockDataPointer[b + VLASOV_STENCIL_WIDTH] != NULL) { + for (int b = 0; b < lengthOfPencil; b++) { + if(pencilBlockData[b] != NULL) { Realf blockValues[WID3]; - const Realf* block_data = blockDataPointer[b + VLASOV_STENCIL_WIDTH]; + Realf* block_data = pencilBlockData[b]; // Copy data to a temporary array and transpose values so that mapping is along k direction. - // spatial source_neighbors already taken care of when - // creating source_neighbors table. If a normal spatial cell does not - // simply have the block, its value will be its null_block which - // is fine. This null_block has a value of zero in data, and that - // is thus the velocity space boundary + #pragma omp simd for (uint i=0; i& mpiGrid, - setOfPencils& pencils, - uint dimension) { - - const bool debug = false; - int neighborhoodId = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); - - int myRank; - if(debug) { - MPI_Comm_rank(MPI_COMM_WORLD,&myRank); - } - - std::vector idsToSplit; - -// Thread this loop here -#pragma omp parallel for - for (uint pencili = 0; pencili < pencils.N; ++pencili) { - - if(pencils.periodic[pencili]) continue; - - auto ids = pencils.getIds(pencili); - - // It is possible that the pencil has already been refined by the pencil building algorithm - // and is on a higher refinement level than the refinement level of any of the cells it contains - // due to e.g. process boundaries. - int maxPencilRefLvl = pencils.path[pencili].size(); - int maxNbrRefLvl = 0; - - const auto* frontNeighbors = mpiGrid.get_neighbors_of(ids.front(),neighborhoodId); - const auto* backNeighbors = mpiGrid.get_neighbors_of(ids.back() ,neighborhoodId); - - - // Create list of unique distances in the negative direction from the first cell in pencil - std::set< int > distances; - for (const auto& nbrPair : *frontNeighbors) { - if(nbrPair.second[dimension] < 0) { - // gather positive values - distances.insert(-nbrPair.second[dimension]); - } - } - int foundcells = 0; - CellID lastcell = INVALID_CELLID; - // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (auto it = distances.begin(); it != distances.end(); ++it) { - for (const auto& nbrPair : *frontNeighbors) { - if (nbrPair.first==lastcell) continue; - int distanceInRefinedCells = -nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) { - maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); - lastcell = nbrPair.first; - foundcells++; - continue; - } - } - if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances - } - - // Create list of unique distances in the positive direction from the last cell in pencil - distances.clear(); - for (const auto& nbrPair : *backNeighbors) { - if(nbrPair.second[dimension] > 0) { - distances.insert(nbrPair.second[dimension]); - } - } - foundcells = 0; - lastcell = INVALID_CELLID; - for (auto it = distances.begin(); it != distances.end(); ++it) { - for (const auto& nbrPair : *backNeighbors) { - if (nbrPair.first==lastcell) continue; - int distanceInRefinedCells = nbrPair.second[dimension]; - if(distanceInRefinedCells == *it) { - maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); - lastcell = nbrPair.first; - foundcells++; - continue; - } - } - if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances - } - - // Old version which can check needlessly far - // for (const auto nbrPair: *frontNeighbors) { - // maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); - // } - // for (const auto nbrPair: *backNeighbors) { - // maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); - // } - - - if (maxNbrRefLvl > maxPencilRefLvl) { - if(debug) { - std::cout << "I am rank " << myRank << ". "; - std::cout << "Found refinement level " << maxNbrRefLvl << " in one of the ghost cells of pencil " << pencili << ". "; - std::cout << "Highest refinement level in this pencil is " << maxPencilRefLvl; - std::cout << ". Splitting pencil " << pencili << endl; - } - // Let's avoid modifying pencils while we are looping over it. Write down the indices of pencils - // that need to be split and split them later. -#pragma omp critical - { - idsToSplit.push_back(pencili); - } - } - } - -// No threading here, probably more efficient to thread inside the splitting - for (auto pencili: idsToSplit) { - - Realv dx = 0.0; - Realv dy = 0.0; - // TODO: Double-check that this gives you the right dimensions! - auto ids = pencils.getIds(pencili); - switch(dimension) { - case 0: - dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DY]; - dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; - break; - case 1: - dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; - dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; - break; - case 2: - dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; - dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DY]; - break; - } - -// WARNING threading inside this function - pencils.split(pencili,dx,dy); - - } -} - -/* Checks that each local spatial cell appears in pencils at least 1 time. - * - * @param mpiGrid DCCRG grid object - * @param cells Local spatial cells - * @param pencils Pencil data struct - */ -bool checkPencils( - const dccrg::Dccrg& mpiGrid, - const std::vector &cells, - const setOfPencils& pencils -) { - - bool correct = true; - - for (auto id : cells) { - - if (mpiGrid[id]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY ) { - - int myCount = std::count(pencils.ids.begin(), pencils.ids.end(), id); - - if( myCount == 0) { - - std::cerr << "ERROR: Cell ID " << id << " Appears in pencils " << myCount << " times!"<< std::endl; - correct = false; - } - - } - - } - - for (uint ipencil = 0; ipencil < pencils.N; ++ipencil) { - cint nPencilsThroughThisCell = pow(pow(2,pencils.path[ipencil].size()),2); - auto ids = pencils.getIds(ipencil); - - for (auto id : ids) { - - cint myCount = std::count(pencils.ids.begin(), pencils.ids.end(), id); - - if (myCount > nPencilsThroughThisCell) { - - std::cerr << "ERROR: Cell ID " << id << " Appears in pencils " << myCount << " times!"<< std::endl; - std::cerr << " It should not appear more than " << nPencilsThroughThisCell << " times." << std::endl; - correct = false; - - } - - } - - } - - return correct; - -} - -/* Debugging function, prints the list of cells in each pencil - * - * @param pencils Pencil data struct - * @param dimension Spatial dimension - * @param myRank MPI rank - */ -void printPencilsFunc(const setOfPencils& pencils, const uint dimension, const int myRank) { - - // Print out ids of pencils (if needed for debugging) - uint ibeg = 0; - uint iend = 0; - std::cout << "I am rank " << myRank << ", I have " << pencils.N << " pencils along dimension " << dimension << ":\n"; - MPI_Barrier(MPI_COMM_WORLD); - if(myRank == MASTER_RANK) { - std::cout << "t, N, mpirank, dimension (x, y): indices {path} " << std::endl; - std::cout << "-----------------------------------------------------------------" << std::endl; - } - MPI_Barrier(MPI_COMM_WORLD); - for (uint i = 0; i < pencils.N; i++) { - iend += pencils.lengthOfPencils[i]; - std::cout << P::t << ", "; - std::cout << i << ", "; - std::cout << myRank << ", "; - std::cout << dimension << ", "; - std::cout << "(" << pencils.x[i] << ", " << pencils.y[i] << "): "; - for (auto j = pencils.ids.begin() + ibeg; j != pencils.ids.begin() + iend; ++j) { - std::cout << *j << " "; - } - ibeg = iend; - - std::cout << "{"; - for (auto step : pencils.path[i]) { - std::cout << step << ", "; - } - std::cout << "}"; - - std::cout << std::endl; - } - - MPI_Barrier(MPI_COMM_WORLD); -} - -/* Wrapper function for calling seed ID selection and pencil generation, per dimension. - * Includes threading and gathering of pencils into thread-containers. - * - * @param [in] mpiGrid DCCRG grid object - * @param [in] dimension Spatial dimension - */ -void prepareSeedIdsAndPencils(const dccrg::Dccrg& mpiGrid, - const uint dimension) { - - const bool printPencils = false; - int myRank; - if(printPencils) MPI_Comm_rank(MPI_COMM_WORLD,&myRank); - - const vector& localCells = getLocalCells(); - vector localPropagatedCells; - // Figure out which spatial cells are translated, - // result independent of particle species. - for (size_t c=0; c seedIds; - getSeedIds(mpiGrid, localPropagatedCells, dimension, seedIds); - getSeedsTimer.stop(); - - phiprof::Timer buildPencilsTimer {"buildPencils"}; - // Output vectors for ready pencils - //setOfPencils pencils; - - // Clear previous set - DimensionPencils[dimension].removeAllPencils(); - -#pragma omp parallel - { - // Empty vectors for internal use of buildPencilsWithNeighbors. Could be default values but - // default vectors are complicated. Should overload buildPencilsWithNeighbors like suggested here - // https://stackoverflow.com/questions/3147274/c-default-argument-for-vectorint - vector ids; - vector path; - // thread-internal pencil set to be accumulated at the end - setOfPencils thread_pencils; - // iterators used in the accumulation - std::vector::iterator ibeg, iend; - -#pragma omp for schedule(guided) - for (uint i=0; i pencilIds(ibeg, iend); - DimensionPencils[dimension].addPencil(pencilIds,thread_pencils.x[i],thread_pencils.y[i],thread_pencils.periodic[i],thread_pencils.path[i]); - } - } - } - - phiprof::Timer checkGhostsTimer {"check_ghost_cells"}; - // Check refinement of two ghost cells on each end of each pencil - check_ghost_cells(mpiGrid,DimensionPencils[dimension],dimension); - checkGhostsTimer.stop(); - - // **************************************************************************** - - if(printPencils) { - printPencilsFunc(DimensionPencils[dimension],dimension,myRank); - } - buildPencilsTimer.stop(); -} - /* Map velocity blocks in all local cells forward by one time step in one spatial dimension. * This function uses 1-cell wide pencils to update cells in-place to avoid allocating large * temporary buffers. @@ -1394,46 +211,34 @@ void prepareSeedIdsAndPencils(const dccrg::Dccrg& mpiGrid, +bool trans_map_1d_amr(const dccrg::Dccrg& mpiGrid, const vector& localPropagatedCells, const vector& remoteTargetCells, std::vector& nPencils, const uint dimension, const Realv dt, const uint popID) { - phiprof::Timer setupTimer {"setup"}; - uint cell_indices_to_id[3]; /*< used when computing id of target cell in block*/ - unsigned char cellid_transpose[WID3]; /*< defines the transpose for the solver internal (transposed) id: i + j*WID + k*WID2 to actual one*/ + /***********************/ + phiprof::Timer setupTimer {"trans-amr-setup"}; + /***********************/ // return if there's no cells to propagate if(localPropagatedCells.size() == 0) { - cout << "Returning because of no cells" << endl; return false; } - // Vector with all cell ids - vector allCells(localPropagatedCells); - allCells.insert(allCells.end(), remoteTargetCells.begin(), remoteTargetCells.end()); - - // Vectors of pointers to the cell structs - std::vector allCellsPointer(allCells.size()); - - // Initialize allCellsPointer - #pragma omp parallel for - for(uint celli = 0; celli < allCells.size(); celli++){ - allCellsPointer[celli] = mpiGrid[allCells[celli]]; - } - + uint cell_indices_to_id[3]; /*< used when computing id of target cell in block*/ + unsigned int cellid_transpose[WID3]; /*< defines the transpose for the solver internal (transposed) id: i + j*WID + k*WID2 to actual one*/ // Fiddle indices x,y,z in VELOCITY SPACE switch (dimension) { case 0: - // set values in array that is used to convert block indices + // set values in array that is used to convert block indices // to global ID using a dot product. cell_indices_to_id[0]=WID2; cell_indices_to_id[1]=WID; cell_indices_to_id[2]=1; break; case 1: - // set values in array that is used to convert block indices + // set values in array that is used to convert block indices // to global ID using a dot product cell_indices_to_id[0]=1; cell_indices_to_id[1]=WID2; @@ -1451,14 +256,24 @@ bool trans_map_1d_amr(const dccrg::Dccrg& abort(); break; } - - // **************************************************************************** - // compute pencils => set of pencils (shared datastructure) - // prepareSeedIdsAndPencils(mpiGrid,dimension); // moved to grid.cpp + // Assuming 1 neighbor in the target array because of the CFL condition + // In fact propagating to > 1 neighbor will give an error + + // Vector with all cell ids + vector allCells(localPropagatedCells); + allCells.insert(allCells.end(), remoteTargetCells.begin(), remoteTargetCells.end()); + // Vectors of pointers to the cell structs + std::vector allCellsPointer(allCells.size()); + + // Initialize allCellsPointer + #pragma omp parallel for schedule(static) + for(uint celli = 0; celli < allCells.size(); celli++){ + allCellsPointer[celli] = mpiGrid[allCells[celli]]; + } // init cellid_transpose (moved here to take advantage of the omp parallel region) -#pragma omp parallel for collapse(2) + #pragma omp parallel for collapse(2) schedule(static) for (uint k=0; k& } } - // Warning: checkPencils fails to understand situations where pencils reach across 3 levels of refinement. - // if(!checkPencils(mpiGrid, localPropagatedCells, pencils)) { - // std::cerr<<"abort checkpencils"<& nPencils[nPencils.size()-1] += myPencilCount; } } - + // Get a pointer to the velocity mesh of the first spatial cell const vmesh::VelocityMesh& vmesh = allCellsPointer[0]->get_velocity_mesh(popID); - phiprof::Timer buildBlockListimer {"buildBlockList"}; + phiprof::Timer buildBlockListimer {"trans-amr-buildBlockList"}; // Get a unique sorted list of blockids that are in any of the - // propagated cells. First use set for this, then add to vector (may not - // be the most nice way to do this and in any case we could do it along - // dimension for data locality reasons => copy acc map column code, TODO: FIXME - // TODO: Do this separately for each pencil? + // propagated cells. std::vector unionOfBlocks; std::unordered_set unionOfBlocksSet; -// unionOfBlocks.reserve(unionOfBlocksSet.size()); #pragma omp parallel { std::unordered_set thread_unionOfBlocksSet; - #pragma omp for for(unsigned int i=0; i& cvmesh = (*cell)->get_velocity_mesh(popID); + const vmesh::VelocityMesh& cvmesh = (*cell)->get_velocity_mesh(popID); for (vmesh::LocalID block_i=0; block_i< cvmesh.size(); ++block_i) { thread_unionOfBlocksSet.insert(cvmesh.getGlobalID(block_i)); } } - #pragma omp critical { unionOfBlocksSet.insert(thread_unionOfBlocksSet.begin(), thread_unionOfBlocksSet.end()); } // pragma omp critical } // pragma omp parallel - unionOfBlocks.insert(unionOfBlocks.end(), unionOfBlocksSet.begin(), unionOfBlocksSet.end()); - - buildBlockListimer.stop(); - // **************************************************************************** - - // Assuming 1 neighbor in the target array because of the CFL condition - // In fact propagating to > 1 neighbor will give an error - const uint nTargetNeighborsPerPencil = 1; - - // Compute spatial neighbors for target cells. - // For targets we need the local cells, plus a padding of 1 cell at both ends - phiprof::Timer computeTargetsTimer {"computeSpatialTargetCellsForPencils"}; - std::vector targetCells(DimensionPencils[dimension].sumOfLengths + DimensionPencils[dimension].N * 2 * nTargetNeighborsPerPencil ); - computeSpatialTargetCellsForPencilsWithFaces(mpiGrid, DimensionPencils[dimension], dimension, targetCells.data()); - computeTargetsTimer.stop(); - + buildBlockListimer.stop(); + /***********************/ setupTimer.stop(); - - int mappingId {phiprof::initializeTimer("mapping")}; - int storeId {phiprof::initializeTimer("store")}; - - #pragma omp parallel + /***********************/ + + int mappingTimerId = phiprof::initializeTimer("trans-amr-mapping"); + int loadTimerId = phiprof::initializeTimer("trans-amr-load source data"); + int memsetTimerId = phiprof::initializeTimer("trans-amr-MemSet"); + int propagateTimerId = phiprof::initializeTimer("trans-amr-propagatePencil"); + +#pragma omp parallel { - // declarations for variables needed by the threads - std::vector> targetBlockData((DimensionPencils[dimension].sumOfLengths + 2 * nTargetNeighborsPerPencil * DimensionPencils[dimension].N) * WID3); - std::vector> pencilSourceCells; - - // Allocate aligned vectors which are needed once per pencil to avoid reallocating once per block loop + pencil loop iteration - std::vector>> pencilTargetValues; - std::vector>> pencilSourceVecData; - std::vector>> pencildz; - - for(uint pencili = 0; pencili < DimensionPencils[dimension].N; ++pencili) { - - cint L = DimensionPencils[dimension].lengthOfPencils[pencili]; - cuint sourceLength = L + 2 * VLASOV_STENCIL_WIDTH; - - // Vector buffer where we write data, initialized to 0*/ - std::vector> targetValues((L + 2 * nTargetNeighborsPerPencil) * WID3 / VECL); - pencilTargetValues.push_back(targetValues); - // Allocate source data: sourcedata> sourceVecData(sourceLength * WID3 / VECL); - pencilSourceVecData.push_back(sourceVecData); - - // Compute spatial neighbors for the source cells of the pencil. In - // source cells we have a wider stencil and take into account boundaries. - std::vector sourceCells(sourceLength); - computeSpatialSourceCellsForPencil(mpiGrid, DimensionPencils[dimension], pencili, dimension, sourceCells.data()); - pencilSourceCells.push_back(sourceCells); - - // dz is the cell size in the direction of the pencil - std::vector> dz(sourceLength); - for(uint i = 0; i < sourceCells.size(); ++i) { - dz[i] = sourceCells[i]->parameters[CellParams::DX+dimension]; - } - pencildz.push_back(dz); - } - - // Loop over velocity space blocks. Thread this loop (over vspace blocks) with OpenMP. - #pragma omp for schedule(guided,8) - for(uint blocki = 0; blocki < unionOfBlocks.size(); blocki++) { + // Vector of pointers to cell block data, used for both reading and writing + std::vector blockDataBuffer(DimensionPencils[dimension].sumOfLengths*WID3/VECL); + std::vector cellBlockData(DimensionPencils[dimension].sumOfLengths); + std::vector pencilBlocksCount(DimensionPencils[dimension].N); + // Loop over velocity space blocks (threaded). +#pragma omp for schedule(guided,8) + for(uint blocki = 0; blocki < unionOfBlocks.size(); blocki++) { // Get global id of the velocity block vmesh::GlobalID blockGID = unionOfBlocks[blocki]; - phiprof::Timer mappingTimer {mappingId}; - - // Loop over pencils - uint totalTargetLength = 0; - for(uint pencili = 0; pencili < DimensionPencils[dimension].N; ++pencili){ -// for ( auto pencili : unionOfBlocksMapToPencilIds.at(blockGID) ) { - - int L = DimensionPencils[dimension].lengthOfPencils[pencili]; - uint targetLength = L + 2 * nTargetNeighborsPerPencil; - - // load data(=> sourcedata) / (proper xy reconstruction in future) - bool pencil_has_data = copy_trans_block_data_amr(pencilSourceCells[pencili].data(), blockGID, L, pencilSourceVecData[pencili].data(), - cellid_transpose, popID); - - if(!pencil_has_data) { - totalTargetLength += targetLength; - continue; - } - - // Dz and sourceVecData are both padded by VLASOV_STENCIL_WIDTH - // Dz has 1 value/cell, sourceVecData has WID3 values/cell - propagatePencil(pencildz[pencili].data(), pencilSourceVecData[pencili].data(), pencilTargetValues[pencili].data(), dimension, blockGID, dt, vmesh, L, pencilSourceCells[pencili][0]->getVelocityBlockMinValue(popID)); - - // sourceVecData => targetBlockData[this pencil]) - - // Loop over cells in pencil - for (uint icell = 0; icell < targetLength; icell++) { - // Loop over 1st vspace dimension - for (uint k=0; ksysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { - - // Get local velocity block id - const vmesh::LocalID blockLID = spatial_cell->get_velocity_block_local_id(blockGID, popID); - - // Check for invalid block id - if (blockLID != vmesh::VelocityMesh::invalidLocalID()) { - - // Get a pointer to the block data - Realf* blockData = spatial_cell->get_data(blockLID, popID); - - // Loop over velocity block cells - for(int i = 0; i < WID3; i++) { - blockData[i] = 0.0; - } - } + phiprof::Timer mappingTimer {mappingTimerId}; // mapping (top-level) + + // Load data for pencils. + phiprof::Timer loadTimer {loadTimerId}; + for (uint pencili = 0; pencili < DimensionPencils[dimension].N; ++pencili){ + int nonEmptyBlocks = 0; + int L = DimensionPencils[dimension].lengthOfPencils[pencili]; + int start = DimensionPencils[dimension].idsStart[pencili]; + // Loop over cells in pencil + for (int b = 0; b < L; b++) { + // Get cell pointer and local block id + SpatialCell* srcCell = mpiGrid[DimensionPencils[dimension].ids[start + b]]; + const vmesh::LocalID blockLID = srcCell->get_velocity_block_local_id(blockGID,popID); + // Store block data pointer for both loading of data and writing back to the cell + if (blockLID != srcCell->invalid_local_id()) { + // Get data pointer + cellBlockData[start + b] = srcCell->get_data(blockLID,popID); + nonEmptyBlocks++; + } else { + cellBlockData[start + b] = NULL; } } - - // store_data(target_data => targetCells) :Aggregate data for blockid to original location - // Loop over pencils again - totalTargetLength = 0; - for(uint pencili = 0; pencili < DimensionPencils[dimension].N; pencili++){ - - uint targetLength = DimensionPencils[dimension].lengthOfPencils[pencili] + 2 * nTargetNeighborsPerPencil; - - // store values from targetBlockData array to the actual blocks - // Loop over cells in the pencil, including the padded cells of the target array - for ( uint celli = 0; celli < targetLength; celli++ ) { - - uint GID = celli + totalTargetLength; - SpatialCell* targetCell = targetCells[GID]; - - if(targetCell) { // this check also skips sysboundary cells - - const vmesh::LocalID blockLID = targetCell->get_velocity_block_local_id(blockGID, popID); - - // Check for invalid block id - if( blockLID == vmesh::VelocityMesh::invalidLocalID() ) { - continue; - } - - Realf* blockData = targetCell->get_data(blockLID, popID); - - // areaRatio is the reatio of the cross-section of the spatial cell to the cross-section of the pencil. - int diff = targetCell->SpatialCell::parameters[CellParams::REFINEMENT_LEVEL] - DimensionPencils[dimension].path[pencili].size(); - int ratio; - Realf areaRatio; - if(diff>=0) { - ratio = 1 << diff; - areaRatio = ratio*ratio; - } else { - ratio = 1 << -diff; - areaRatio = 1.0 / (ratio*ratio); - } - - for(int i = 0; i < WID3 ; i++) { - blockData[i] += targetBlockData[GID * WID3 + i] * areaRatio; - } - } + if(nonEmptyBlocks == 0) { + continue; + } + pencilBlocksCount.at(pencili) = nonEmptyBlocks; + // Transpose and copy block data from cells to source buffer + Vec* blockDataSource = blockDataBuffer.data() + start*WID3/VECL; + Realf** pencilBlockData = cellBlockData.data() + start; + copy_trans_block_data_amr(pencilBlockData, L, blockDataSource, + cellid_transpose, popID); + } + loadTimer.stop(); + + phiprof::Timer memsetTimer {memsetTimerId}; + // reset blocks in all non-sysboundary neighbor spatial cells for this block id + for (CellID target_cell_id: DimensionTargetCells[dimension]) { + SpatialCell* target_cell = mpiGrid[target_cell_id]; + if (target_cell) { + // Get local velocity block id + const vmesh::LocalID blockLID = target_cell->get_velocity_block_local_id(blockGID, popID); + // Check for invalid block id + if (blockLID != vmesh::VelocityMesh::invalidLocalID()) { + // Get a pointer to the block data + Realf* blockData = target_cell->get_data(blockLID, popID); + memset(blockData, 0, WID3*sizeof(Realf)); } + } + } + memsetTimer.stop(); - totalTargetLength += targetLength; - - } // closes loop over pencils - - storeTimer.stop(); + phiprof::Timer propagateTimer {propagateTimerId}; + for(uint pencili = 0; pencili < DimensionPencils[dimension].N; ++pencili){ + // Skip pencils without blocks + if (pencilBlocksCount.at(pencili) == 0) { + continue; + } + // sourceVecData => targetBlockData[this pencil]) + int L = DimensionPencils[dimension].lengthOfPencils[pencili]; + int start = DimensionPencils[dimension].idsStart[pencili]; + // Dz and sourceVecData are both padded by VLASOV_STENCIL_WIDTH + // Dz has 1 value/cell, sourceVecData has WID3 values/cell + // vmesh is required just for general indexes and accessors + Realv scalingthreshold = mpiGrid[DimensionPencils[dimension].ids[start + VLASOV_STENCIL_WIDTH]]->getVelocityBlockMinValue(popID); + Realf* pencilDZ = DimensionPencils[dimension].sourceDZ.data() + start; + Realf* pencilRatios = DimensionPencils[dimension].targetRatios.data() + start; + Realf** pencilBlockData = cellBlockData.data() + start; + Vec* blockDataSource = blockDataBuffer.data() +start*WID3/VECL; + propagatePencil(pencilDZ, + blockDataSource, + dimension, + blockGID, + dt, + vmesh, + L, + scalingthreshold, + pencilBlockData, + pencilRatios, + cellid_transpose + ); + } + propagateTimer.stop(); + + mappingTimer.stop(); // mapping (top-level) } // Closes loop over blocks } // closes pragma omp parallel @@ -1724,11 +441,10 @@ bool trans_map_1d_amr(const dccrg::Dccrg& int get_sibling_index(dccrg::Dccrg& mpiGrid, const CellID& cellid) { const int NO_SIBLINGS = 0; - if(mpiGrid.get_refinement_level(cellid) == 0) { return NO_SIBLINGS; } - + //CellID parent = mpiGrid.mapping.get_parent(cellid); CellID parent = mpiGrid.get_parent(cellid); @@ -1748,13 +464,13 @@ int get_sibling_index(dccrg::Dccrg& mpiGr abort(); } return index; - + } /* This function communicates the mapping on process boundaries, and then updates the data to their correct values. * When sending data between neighbors of different refinement levels, special care has to be taken to ensure that * The sending and receiving ranks allocate the correct size arrays for neighbor_block_data. - * This is partially due to DCCRG defining neighborhood size relative to the host cell. For details, see + * This is partially due to DCCRG defining neighborhood size relative to the host cell. For details, see * https://github.com/fmihpc/dccrg/issues/12 * * @param mpiGrid DCCRG grid object @@ -1768,16 +484,16 @@ void update_remote_mapping_contribution_amr( int direction, const uint popID) { - const vector& local_cells = getLocalCells(); + const vector local_cells = mpiGrid.get_local_cells_on_process_boundary(VLASOV_SOLVER_NEIGHBORHOOD_ID); const vector remote_cells = mpiGrid.get_remote_cells_on_process_boundary(VLASOV_SOLVER_NEIGHBORHOOD_ID); vector receive_cells; set send_cells; - + vector receive_origin_cells; vector receive_origin_index; int neighborhood = 0; - + //normalize and set neighborhoods if(direction > 0) { direction = 1; @@ -1836,12 +552,12 @@ void update_remote_mapping_contribution_amr( } } } - + vector receiveBuffers; vector sendBuffers; - + for (auto c : local_cells) { - + SpatialCell *ccell = mpiGrid[c]; if (!ccell) continue; @@ -1858,18 +574,18 @@ void update_remote_mapping_contribution_amr( n_nbrs.push_back(neighbor); } } - + uint sendIndex = 0; uint recvIndex = 0; int mySiblingIndex = get_sibling_index(mpiGrid,c); - + // Set up sends if any neighbor cells in p_nbrs are non-local. if (!all_of(p_nbrs.begin(), p_nbrs.end(), [&mpiGrid](CellID i){return mpiGrid.is_local(i);})) { // ccell adds a neighbor_block_data block for each neighbor in the positive direction to its local data for (const auto nbr : p_nbrs) { - + //Send data in nbr target array that we just mapped to, if // 1) it is a valid target, // 2) the source cell in center was translated, @@ -1883,49 +599,49 @@ void update_remote_mapping_contribution_amr( 3) Ref_c > Ref_nbr , index = c sibling index 4) Ref_c < Ref_nbr , index = nbr sibling index */ - + if(mpiGrid.get_refinement_level(c) >= mpiGrid.get_refinement_level(nbr)) { sendIndex = mySiblingIndex; } else { sendIndex = get_sibling_index(mpiGrid,nbr); } - + SpatialCell *pcell = mpiGrid[nbr]; - + // 4) it exists and is not a boundary cell, if(pcell && pcell->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { ccell->neighbor_number_of_blocks.at(sendIndex) = pcell->get_number_of_velocity_blocks(popID); - + if(send_cells.find(nbr) == send_cells.end()) { // 5 We have not already sent data from this rank to this cell. - + ccell->neighbor_block_data.at(sendIndex) = pcell->get_data(popID); send_cells.insert(nbr); - + } else { // The receiving cell can't know which cell is sending the data from this rank. // Therefore, we have to send 0's from other cells in the case where multiple cells // from one rank are sending to the same remote cell so that all sent cells can be // summed for the correct result. - + ccell->neighbor_block_data.at(sendIndex) = - (Realf*) aligned_malloc(ccell->neighbor_number_of_blocks.at(sendIndex) * WID3 * sizeof(Realf), 64); + (Realf*) aligned_malloc(ccell->neighbor_number_of_blocks.at(sendIndex) * WID3 * sizeof(Realf), WID3); sendBuffers.push_back(ccell->neighbor_block_data.at(sendIndex)); for (uint j = 0; j < ccell->neighbor_number_of_blocks.at(sendIndex) * WID3; ++j) { ccell->neighbor_block_data.at(sendIndex)[j] = 0.0; - + } // closes for(uint j = 0; j < ccell->neighbor_number_of_blocks.at(sendIndex) * WID3; ++j) - + } // closes if(send_cells.find(nbr) == send_cells.end()) - + } // closes if(pcell && pcell->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) - + } // closes if(nbr != INVALID_CELLID && do_translate_cell(ccell) && !mpiGrid.is_local(nbr)) - + } // closes for(uint i_nbr = 0; i_nbr < nbrs_to.size(); ++i_nbr) - + } // closes if(!all_of(nbrs_to.begin(), nbrs_to.end(),[&mpiGrid](CellID i){return mpiGrid.is_local(i);})) // Set up receives if any neighbor cells in n_nbrs are non-local. @@ -1933,7 +649,7 @@ void update_remote_mapping_contribution_amr( // ccell adds a neighbor_block_data block for each neighbor in the positive direction to its local data for (const auto nbr : n_nbrs) { - + if (nbr != INVALID_CELLID && !mpiGrid.is_local(nbr) && ccell->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { //Receive data that ncell mapped to this local cell data array, @@ -1953,28 +669,28 @@ void update_remote_mapping_contribution_amr( 3) Ref_nbr > Ref_c , index = nbr sibling index 4) Ref_nbr < Ref_c , index = c sibling index */ - + if(mpiGrid.get_refinement_level(nbr) >= mpiGrid.get_refinement_level(c)) { // Allocate memory for one sibling at recvIndex. - + recvIndex = get_sibling_index(mpiGrid,nbr); ncell->neighbor_number_of_blocks.at(recvIndex) = ccell->get_number_of_velocity_blocks(popID); ncell->neighbor_block_data.at(recvIndex) = - (Realf*) aligned_malloc(ncell->neighbor_number_of_blocks.at(recvIndex) * WID3 * sizeof(Realf), 64); + (Realf*) aligned_malloc(ncell->neighbor_number_of_blocks.at(recvIndex) * WID3 * sizeof(Realf), WID3); receiveBuffers.push_back(ncell->neighbor_block_data.at(recvIndex)); - + } else { recvIndex = mySiblingIndex; - + // std::array siblingarr = mpiGrid.mapping.get_all_children(mpiGrid.mapping.get_parent(c)); // vector mySiblings(siblingarr.begin(), siblingarr.end()); auto mySiblings = mpiGrid.get_all_children(mpiGrid.get_parent(c)); auto myIndices = mpiGrid.mapping.get_indices(c); - - // Allocate memory for each sibling to receive all the data sent by coarser ncell. + + // Allocate memory for each sibling to receive all the data sent by coarser ncell. // only allocate blocks for face neighbors. for (uint i_sib = 0; i_sib < MAX_NEIGHBORS_PER_DIM; ++i_sib) { @@ -1982,7 +698,7 @@ void update_remote_mapping_contribution_amr( auto sibIndices = mpiGrid.mapping.get_indices(sibling); auto* scell = mpiGrid[sibling]; - + // Only allocate siblings that are remote face neighbors to ncell // Also take care to have these consistent with the sending process neighbor checks! if(sibling != INVALID_CELLID @@ -1991,39 +707,39 @@ void update_remote_mapping_contribution_amr( && myIndices.at(dimension) == sibIndices.at(dimension) && ncell->neighbor_number_of_blocks.at(i_sib) != scell->get_number_of_velocity_blocks(popID) && scell->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { - - + + ncell->neighbor_number_of_blocks.at(i_sib) = scell->get_number_of_velocity_blocks(popID); ncell->neighbor_block_data.at(i_sib) = - (Realf*) aligned_malloc(ncell->neighbor_number_of_blocks.at(i_sib) * WID3 * sizeof(Realf), 64); + (Realf*) aligned_malloc(ncell->neighbor_number_of_blocks.at(i_sib) * WID3 * sizeof(Realf), WID3); receiveBuffers.push_back(ncell->neighbor_block_data.at(i_sib)); } } } - + receive_cells.push_back(c); receive_origin_cells.push_back(nbr); receive_origin_index.push_back(recvIndex); - + } // closes (nbr != INVALID_CELLID && !mpiGrid.is_local(nbr) && ...) - + } // closes for(uint i_nbr = 0; i_nbr < nbrs_of.size(); ++i_nbr) - + } // closes if(!all_of(nbrs_of.begin(), nbrs_of.end(),[&mpiGrid](CellID i){return mpiGrid.is_local(i);})) - + } // closes for (auto c : local_cells) { MPI_Barrier(MPI_COMM_WORLD); - + // Do communication SpatialCell::setCommunicatedSpecies(popID); SpatialCell::set_mpi_transfer_type(Transfer::NEIGHBOR_VEL_BLOCK_DATA); mpiGrid.update_copies_of_remote_neighbors(neighborhood); MPI_Barrier(MPI_COMM_WORLD); - - // Reduce data: sum received data in the data array to - // the target grid in the temporary block container + + // Reduce data: sum received data in the data array to + // the target grid in the temporary block container //#pragma omp parallel { for (size_t c = 0; c < receive_cells.size(); ++c) { @@ -2033,23 +749,23 @@ void update_remote_mapping_contribution_amr( if(!receive_cell || !origin_cell) { continue; } - + Realf *blockData = receive_cell->get_data(popID); Realf *neighborData = origin_cell->neighbor_block_data[receive_origin_index[c]]; - //#pragma omp for - for(uint vCell = 0; vCell < VELOCITY_BLOCK_LENGTH * receive_cell->get_number_of_velocity_blocks(popID); ++vCell) { + //#pragma omp for + for(uint vCell = 0; vCell < WID3 * receive_cell->get_number_of_velocity_blocks(popID); ++vCell) { blockData[vCell] += neighborData[vCell]; } } - + // send cell data is set to zero. This is to avoid double copy if // one cell is the neighbor on bot + and - side to the same process for (auto c : send_cells) { SpatialCell* spatial_cell = mpiGrid[c]; Realf * blockData = spatial_cell->get_data(popID); //#pragma omp for nowait - for(unsigned int vCell = 0; vCell < VELOCITY_BLOCK_LENGTH * spatial_cell->get_number_of_velocity_blocks(popID); ++vCell) { + for(unsigned int vCell = 0; vCell < WID3 * spatial_cell->get_number_of_velocity_blocks(popID); ++vCell) { // copy received target data to temporary array where target data is stored. blockData[vCell] = 0; } diff --git a/vlasovsolver/cpu_trans_map_amr.hpp b/vlasovsolver/cpu_trans_map_amr.hpp index f95859be0..a4d7f12fb 100644 --- a/vlasovsolver/cpu_trans_map_amr.hpp +++ b/vlasovsolver/cpu_trans_map_amr.hpp @@ -23,169 +23,9 @@ #define CPU_TRANS_MAP_AMR_H #include - #include "vec.h" #include "../common.h" -#include "../spatial_cell.hpp" - - -struct setOfPencils { - - uint N; // Number of pencils in the set - uint sumOfLengths; - std::vector< uint > lengthOfPencils; // Lengths of pencils - std::vector< CellID > ids; // List of cells - std::vector< uint > idsStart; // List of where a pencil's CellIDs start in the ids array - std::vector< Realv > x,y; // x,y - position - std::vector< bool > periodic; - std::vector< std::vector > path; // Path taken through refinement levels - - setOfPencils() { - - N = 0; - sumOfLengths = 0; - } - - void removeAllPencils() { - - N = 0; - sumOfLengths = 0; - sumOfLengths = 0; - lengthOfPencils.clear(); - idsStart.clear(); - ids.clear(); - x.clear(); - y.clear(); - periodic.clear(); - path.clear(); - } - - - void addPencil(std::vector idsIn, Real xIn, Real yIn, bool periodicIn, std::vector pathIn) { - - N++; - sumOfLengths += idsIn.size(); - lengthOfPencils.push_back(idsIn.size()); - idsStart.push_back(ids.size()); - ids.insert(ids.end(),idsIn.begin(),idsIn.end()); - x.push_back(xIn); - y.push_back(yIn); - periodic.push_back(periodicIn); - path.push_back(pathIn); - } - - void removePencil(const uint pencilId) { - - x.erase(x.begin() + pencilId); - y.erase(y.begin() + pencilId); - periodic.erase(periodic.begin() + pencilId); - path.erase(path.begin() + pencilId); - - uint ibeg = idsStart[pencilId]; - ids.erase(ids.begin() + ibeg, ids.begin() + ibeg + lengthOfPencils[pencilId]); - idsStart.erase(idsStart.begin() + pencilId); - - N--; - sumOfLengths -= lengthOfPencils[pencilId]; - lengthOfPencils.erase(lengthOfPencils.begin() + pencilId); - - } - - std::vector getIds(const uint pencilId) const { - - if (pencilId >= N) { - std::vector idsEmpty; - return idsEmpty; - } - - // Use vector range constructor - std::vector::const_iterator ibeg = ids.begin() + idsStart[pencilId]; - std::vector::const_iterator iend = ibeg + lengthOfPencils[pencilId]; - std::vector idsOut(ibeg, iend); - - return idsOut; - } - - // Split one pencil into up to four pencils covering the same space. - // dx and dy are the dimensions of the original pencil. - void split(const uint myPencilId, const Realv dx, const Realv dy) { - - auto myIds = this->getIds(myPencilId); - - // Find paths that members of this pencil may have in other pencils (can happen) - // so that we don't add duplicates. - std::vector existingSteps; - -#pragma omp parallel for - for (uint theirPencilId = 0; theirPencilId < this->N; ++theirPencilId) { - if(theirPencilId == myPencilId) continue; - auto theirIds = this->getIds(theirPencilId); - for (auto theirId : theirIds) { - for (auto myId : myIds) { - if (myId == theirId) { - std::vector theirPath = this->path.at(theirPencilId); - std::vector myPath = this->path.at(myPencilId); - if(theirPath.size() > myPath.size()) { - bool samePath = true; - for (uint i = 0; i < myPath.size(); ++i) { - if(myPath.at(i) != theirPath.at(i)) { - samePath = false; - } - } - - if(samePath) { - uint theirStep = theirPath.at(myPath.size()); -#pragma omp critical - { - existingSteps.push_back(theirStep); - } - } - } - } - } - } - } - - bool firstPencil = true; - const auto copy_of_path = path.at(myPencilId); - const auto copy_of_x = x.at(myPencilId); - const auto copy_of_y = y.at(myPencilId); - - // Add those pencils whose steps dont already exist in the pencils struct - for (int step = 0; step < 4; ++step) { - if (std::any_of(existingSteps.begin(), existingSteps.end(), [step](int i){return step == i;})) { - continue; - } - - Realv signX = 1.0; - Realv signY = 1.0; - - if(step < 2) { - signY = -1.0; - } - - if(step % 2 == 0) { - signX = -1.0; - } - - auto myX = copy_of_x + signX * 0.25 * dx; - auto myY = copy_of_y + signY * 0.25 * dy; - - if(firstPencil) { - //TODO: set x and y correctly. Right now they are not used anywhere. - path.at(myPencilId).push_back(step); - x.at(myPencilId) = myX; - y.at(myPencilId) = myY; - firstPencil = false; - } else { - auto myPath = copy_of_path; - myPath.push_back(step); - addPencil(myIds, myX, myY, periodic.at(myPencilId), myPath); - } - } - } -}; - +#include "../spatial_cell_wrapper.hpp" bool trans_map_1d_amr(const dccrg::Dccrg& mpiGrid, const std::vector& localPropagatedCells, @@ -201,14 +41,4 @@ void update_remote_mapping_contribution_amr(dccrg::Dccrg& mpiGrid, - const uint dimension); - -// pencils used for AMR translation -static std::array DimensionPencils; - -void flagSpatialCellsForAmrCommunication(const dccrg::Dccrg& mpiGrid, - const std::vector& localPropagatedCells); - #endif diff --git a/vlasovsolver/cpu_trans_pencils.cpp b/vlasovsolver/cpu_trans_pencils.cpp new file mode 100644 index 000000000..38cfc40ee --- /dev/null +++ b/vlasovsolver/cpu_trans_pencils.cpp @@ -0,0 +1,1182 @@ +// use DCCRG version Nov 8th 2018 01482cfba8 +#include "../grid.h" +using namespace std; +using namespace spatial_cell; + +#include "cpu_trans_pencils.hpp" + +std::array DimensionPencils; +std::array,3> DimensionTargetCells; + +//Is cell translated? It is not translated if DO_NO_COMPUTE or if it is sysboundary cell and not in first sysboundarylayer +bool do_translate_cell(SpatialCell* SC){ + if(SC->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE || + (SC->sysBoundaryLayer != 1 && SC->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY)) + return false; + else + return true; +} + +/* Get the one-dimensional neighborhood index for a given direction and neighborhood size. + * + * @param dimension spatial dimension of neighborhood + * @param stencil neighborhood size in cells + * @return neighborhood index that can be passed to DCCRG functions + */ +int getNeighborhood(const uint dimension, const uint stencil) { + + int neighborhood = 0; + + if (stencil == 1) { + switch (dimension) { + case 0: + neighborhood = VLASOV_SOLVER_TARGET_X_NEIGHBORHOOD_ID; + break; + case 1: + neighborhood = VLASOV_SOLVER_TARGET_Y_NEIGHBORHOOD_ID; + break; + case 2: + neighborhood = VLASOV_SOLVER_TARGET_Z_NEIGHBORHOOD_ID; + break; + } + } + + if (stencil > 1) { + switch (dimension) { + case 0: + neighborhood = VLASOV_SOLVER_X_NEIGHBORHOOD_ID; + break; + case 1: + neighborhood = VLASOV_SOLVER_Y_NEIGHBORHOOD_ID; + break; + case 2: + neighborhood = VLASOV_SOLVER_Z_NEIGHBORHOOD_ID; + break; + } + } + + return neighborhood; + +} + +void flagSpatialCellsForAmrCommunication(const dccrg::Dccrg& mpiGrid, + const vector& localPropagatedCells) { + + // Only flag/unflag cells if AMR is active + if (mpiGrid.get_maximum_refinement_level()==0) return; + + // return if there's no cells to flag + if(localPropagatedCells.size() == 0) { + return; + } + + for (int dimension=0; dimension<3; dimension++) { + // These neighborhoods now include the AMR addition beyond the regular vlasov stencil + int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); + + // Set flags: loop over local cells +#pragma omp parallel for + for (uint i=0; iSpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = false; + + // In dimension, check iteratively if any neighbors up to VLASOV_STENCIL_WIDTH distance away are on a different process + const auto* NbrPairs = mpiGrid.get_neighbors_of(c, neighborhood); + + // Create list of unique distances + std::set< int > distancesplus; + std::set< int > distancesminus; + std::set foundNeighborsP; + std::set foundNeighborsM; + /** Using sets of cells as well, we should only get one distance per + (potentially less refined) cell. This should result in safe behaviour + as long as the neighborhood of a cell does not contain cells with a + refinement level more than 1 level apart from the cell itself. + */ + for (const auto& nbrPair : *NbrPairs) { + if(nbrPair.second[dimension] > 0) { + if (foundNeighborsP.find(nbrPair.first) == foundNeighborsP.end()) { + distancesplus.insert(nbrPair.second[dimension]); + foundNeighborsP.insert(nbrPair.first); + } + } + if(nbrPair.second[dimension] < 0) { + if (foundNeighborsM.find(nbrPair.first) == foundNeighborsM.end()) { + distancesminus.insert(-nbrPair.second[dimension]); + foundNeighborsM.insert(nbrPair.first); + } + } + } + + foundNeighborsP.clear(); + foundNeighborsM.clear(); + + int iSrc = VLASOV_STENCIL_WIDTH - 1; + // Iterate through positive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (auto it = distancesplus.begin(); it != distancesplus.end(); ++it) { + if (ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] == true) iSrc = -1; + if (iSrc < 0) break; // found enough elements + // Check all neighbors at distance *it + for (const auto& nbrPair : *NbrPairs) { + SpatialCell *ncell = mpiGrid[nbrPair.first]; + if (!ncell) continue; + int distanceInRefinedCells = nbrPair.second[dimension]; + if (distanceInRefinedCells == *it) { + if (foundNeighborsP.find(nbrPair.first) != foundNeighborsP.end()) continue; + foundNeighborsP.insert(nbrPair.first); + if (!mpiGrid.is_local(nbrPair.first)) { + ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = true; + break; + } + } + } // end loop over neighbors + iSrc--; + } // end loop over positive distances + + iSrc = VLASOV_STENCIL_WIDTH - 1; + // Iterate through negtive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (auto it = distancesminus.begin(); it != distancesminus.end(); ++it) { + if (ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] == true) iSrc = -1; + if (iSrc < 0) break; // found enough elements + // Check all neighbors at distance *it + for (const auto& nbrPair : *NbrPairs) { + SpatialCell *ncell = mpiGrid[nbrPair.first]; + if (!ncell) continue; + int distanceInRefinedCells = -nbrPair.second[dimension]; + if (distanceInRefinedCells == *it) { + if (foundNeighborsM.find(nbrPair.first) != foundNeighborsM.end()) continue; + foundNeighborsM.insert(nbrPair.first); + if (!mpiGrid.is_local(nbrPair.first)) { + ccell->SpatialCell::parameters[CellParams::AMR_TRANSLATE_COMM_X+dimension] = true; + break; + } + } + } // end loop over neighbors + iSrc--; + } // end loop over negative distances + } // end loop over local propagated cells + } // end loop over dimensions + return; +} + +/* Get cellIDs for spatial cells that are considered target / source cells for a pencil. + * + * Source cells are cells that the pencil reads data from to compute polynomial + * fits that are used for propagation in the vlasov solver. All cells included + * in the pencil proper + VLASOV_STENCIL_WIDTH cells on both ends are source cells. + * Invalid cells are replaced by closest good cells. + * Boundary cells are included. + * + * Target cells are cells that the pencil writes data into after translation by + * the vlasov solver. All cells included in the pencil proper + 1 cells on both ends + * are target cells. + * + * There is only one list of cellIDs for the pencil, where each source cell has also + * their width stored, and for target cells, the relative contribution is stored. If + * the cell in question is not a target cell, the contribution is set to zero. + * + * @param [in] mpiGrid DCCRG grid object + * @param [inout] ids pointer to subsection of vector of ids for actual pencil + * @param [in] L length of pencil (including stencil cells) + * @param [in] dimension spatial dimension + * @param [in] path index of the desired face neighbor when going to a higher refinement level + * @param [out] source pointer to subsection of vector storing cell widths + * @param [out] targetRatios pointer to subsection of vector storing relative contribution of pencil to target cells + */ +void computeSpatialSourceCellsForPencil(const dccrg::Dccrg& mpiGrid, + CellID *ids, + const uint L, + const uint dimension, + std::vector path, + Realf* sourceDZ, + Realf* targetRatios + ){ + + // These neighborhoods now include the AMR addition beyond the regular vlasov stencil + int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); + stringstream ss; + for (uint j = 0; j < L; ++j) { + ss<< ids[j] << " "; + } + + // Insert pointers for neighbors of ids.front() and ids.back() + const auto* frontNbrPairs = mpiGrid.get_neighbors_of(ids[VLASOV_STENCIL_WIDTH], neighborhood); + const auto* backNbrPairs = mpiGrid.get_neighbors_of(ids[L-VLASOV_STENCIL_WIDTH-1], neighborhood); + // Create list of unique distances in the negative direction from the first cell in pencil + std::set< int > distances; + for (const auto& nbrPair : *frontNbrPairs) { + if(nbrPair.second[dimension] < 0) { + // gather absolute distance values + distances.insert(-nbrPair.second[dimension]); + } + } + + int iSrc = VLASOV_STENCIL_WIDTH - 1; + // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (auto it = distances.begin(); it != distances.end(); ++it) { + if (iSrc < 0) break; // found enough elements + + // Collect all neighbors at distance *it to a vector + std::vector< CellID > neighbors; + for (const auto& nbrPair : *frontNbrPairs) { + int distanceInRefinedCells = -nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); + } + // Get rid of duplicate neighbor cells at single distance + std::sort(neighbors.begin(), neighbors.end()); + neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); + + // Find source cells (VLASOV_STENCIL_WIDTH at each end) + int refLvl = mpiGrid.get_refinement_level(ids[VLASOV_STENCIL_WIDTH]); + if (neighbors.size() == 1) { + if (ids[iSrc+1] == neighbors.at(0)) continue; // already found this cell for different distance + ids[iSrc--] = neighbors.at(0); + } else if ( path[refLvl] < neighbors.size() ) { + if (ids[iSrc+1] == neighbors.at(path[refLvl])) continue; // already found this cell for different distance (should not happen) + ids[iSrc--] = neighbors.at(path[refLvl]); + } else { + ss<<"error too few front neighbors for path! cellid "< 0) { + distances.insert(nbrPair.second[dimension]); + } + } + + // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + // Distances are positive here so smallest distance has smallest value. + iSrc = L - VLASOV_STENCIL_WIDTH; + for (auto it = distances.begin(); it != distances.end(); ++it) { + if (iSrc >= (int)L) break; // Found enough cells + + // Collect all neighbors at distance *it to a vector + std::vector< CellID > neighbors; + for (const auto& nbrPair : *backNbrPairs) { + int distanceInRefinedCells = nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); + } + // Get rid of duplicate neighbor cells at single distance + std::sort(neighbors.begin(), neighbors.end()); + neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); + + int refLvl = mpiGrid.get_refinement_level(ids[L-VLASOV_STENCIL_WIDTH-1]); + if (neighbors.size() == 1) { + if (ids[iSrc-1] == neighbors.at(0)) continue; // already found this cell for different distance + ids[iSrc++] = neighbors.at(0); + } else if ( path[refLvl] < neighbors.size() ) { + if (ids[iSrc-1] == neighbors.at(path[refLvl])) continue; // already found this cell for different distance (should not happen) + ids[iSrc++] = neighbors.at(path[refLvl]); + } else { + ss<<"error too few back neighbors for path! cellid "<= 0 ;--i){ + bool isGood = false; + if (ids[i]!=0) { + if (mpiGrid[ids[i]] != NULL) { + if (mpiGrid[ids[i]]->sysBoundaryFlag != sysboundarytype::DO_NOT_COMPUTE) { + isGood = true; + } + } + } + if (!isGood) { + ids[i] = lastGoodCell; + } else { + lastGoodCell = ids[i]; + } + } + + /*loop to positive side and replace all invalid cells with the closest good cell*/ + lastGoodCell = ids[L - VLASOV_STENCIL_WIDTH - 1]; + for(int i = (int)L - VLASOV_STENCIL_WIDTH; i < (int)L; ++i){ + bool isGood = false; + if (ids[i]!=0) { + if (mpiGrid[ids[i]] != NULL) { + if (mpiGrid[ids[i]]->sysBoundaryFlag != sysboundarytype::DO_NOT_COMPUTE) { + isGood = true; + } + } + } + if (!isGood) { + ids[i] = lastGoodCell; + } else { + lastGoodCell = ids[i]; + } + } + + // Loop over all cells and store widths in translation direction + for (int i = 0; i < (int)L; ++i) { + sourceDZ[i] = mpiGrid[ids[i]]->parameters[CellParams::DX+dimension]; + } + + // Loop over all cells and store pencil-to-cell cross-sectional area for target cells + for (int i = 0; i < (int)L; ++i) { + if ((i < VLASOV_STENCIL_WIDTH-1) || (i > (int)L-VLASOV_STENCIL_WIDTH)) { + // Source cell, not a target cell + targetRatios[i]=0.0; + continue; + } + if (ids[i]) { + SpatialCell* tc = mpiGrid[ids[i]]; + if (tc && tc->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { + // areaRatio is the ratio of the cross-section of the spatial cell to the cross-section of the pencil. + const int diff = tc->SpatialCell::parameters[CellParams::REFINEMENT_LEVEL] - path.size(); + if(diff>0) { + // Undefined behaviour! Cell is smaller than pencil (higher reflevel than path size) + std::cerr<<"Error in path size to cell size: __FILE__:__LINE__"< &grid, + CellID id, int dimension = 0, uint path = 0) { + // If face neighbours are at a higher refinement level, only returns the one which + // which has a neighbor index matching the input path + + //int neighborhood = getNeighborhood(dimension,1); + //const auto* nbrPairs = grid.get_neighbors_of(id, neighborhood); + + vector < CellID > myNeighbors; + CellID neighbor = INVALID_CELLID; + + // Iterate through neighbor ids in the positive direction of the chosen dimension, + // select the neighbor indicated by path, if it is local to this process. + for (const auto& [neighbor, dir] : grid.get_face_neighbors_of(id)) { + if (dir == ((int)dimension + 1)) { + myNeighbors.push_back(neighbor); + } + } + // TODO Verify: are neighbours always in the same order? Let's sort based + // on CellID to be sure. + std::sort(myNeighbors.begin(), myNeighbors.end()); + + if( myNeighbors.size() == 0 ) { + return neighbor; // == INVALID_CELLID + } + + int neighborIndex = 0; + if (myNeighbors.size() > 1) { + neighborIndex = path; + } + + if (grid.is_local(myNeighbors[neighborIndex])) { + neighbor = myNeighbors[neighborIndex]; + } + + return neighbor; +} + +/* Recursive function for building one-dimensional pencils to cover local DCCRG cells. + * Starts from a given seedID and proceeds finding the nearest neighbor in the given dimension + * and adding it to the pencil until no neighbors are found or an endId is met. When a higher + * refinement level (ie. multiple nearest neighbors) is met, the pencil splits into four + * copies to remain at a width of 1 cell. This is done by the function calling itself recursively + * and passing as inputs the cells added so far. The cell selected by each copy of the function + * at a split is stored in the path variable, the same path has to be followed if a refinement + * level is encoutered multiple times. + * + * @param [in] grid DCCRG grid object + * @param [out] pencils Pencil data struct + * @param [in] seedId DCCRG cell id where we start building the pencil. + * The pencil will continue in the + direction in the given dimension until an end condition is met + * @param [in] dimension Spatial dimension + * @param [in] path Integer value that determines which neighbor is added to the pencil when a higher refinement level is met + * @param [in] endIds Prescribed end conditions for the pencil. If any of these cell ids is about to be added to the pencil, + * the builder terminates. + */ +void buildPencilsWithNeighbors( const dccrg::Dccrg &grid, + setOfPencils &pencils, const CellID seedId, + vector ids, const uint dimension, + vector path, const vector &endIds) { + + const bool debug = false; + CellID nextNeighbor; + CellID id = seedId; + int startingRefLvl = grid.get_refinement_level(id); + bool periodic = false; + // If this is a new pencil (instead of being a result of a pencil being split + if( ids.size() == 0 ) { + ids.push_back(seedId); + } + // If the cell where we start is refined, we need to figure out which path + // to follow in future refined cells. This is a bit hacky but we have to + // use the order or the children of the parent cell to figure out which + // corner we are in. + + std::array coordinates = grid.get_center(seedId); + int startingPathSize = path.size(); + + // Find the "pre-existing" path for new pencils starting at higher reflevels + if( startingRefLvl > startingPathSize ) { + CellID myId = seedId; + for ( int i = path.size(); i < startingRefLvl; ++i) { + //CellID parentId = grid.mapping.get_parent(myId); + CellID parentId = grid.get_parent(myId); + + auto myCoords = grid.get_center(myId); + auto parentCoords = grid.get_center(parentId); + int ix=0, iy=0; + switch(dimension) { + case 0: + ix = 1; + iy = 2; + break; + case 1: + ix = 0; + iy = 2; + break; + case 2: + ix = 0; + iy = 1; + break; + } + //int ix = (dimension + 1) % 3; // incorrect for DCCRG + //int iy = (dimension + 2) % 3; + + int step = -1; + + if (myCoords[ix] < parentCoords[ix] && myCoords[iy] < parentCoords[iy]) { + step = 0; + } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] < parentCoords[iy]) { + step = 1; + } else if (myCoords[ix] < parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { + step = 2; + } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { + step = 3; + } + + // path needs to end up in reflevel order, whereas this loop goes in reverse order + path.insert(path.begin(), step); + myId = parentId; + } + } + + // Now start loop for gathering ids into pencil. Break when next found cell is rejected. + while (id != INVALID_CELLID) { + + periodic = false; + bool neighborExists = false; + int refLvl = 0; + + // Find the refinement level in the neighboring (local) cell. Check all possible neighbors + // in case some of them are remote. + for (int tmpPath = 0; tmpPath < 4; ++tmpPath) { + nextNeighbor = selectNeighbor(grid,id,dimension,tmpPath); + if(nextNeighbor != INVALID_CELLID) { + refLvl = max(refLvl,grid.get_refinement_level(nextNeighbor)); + neighborExists = true; + } + } + + // If there are no local acceptable neighbors, we can stop. This is not an error. + if (!neighborExists) { + break; + } + + // Do we need to consider refinement? + if (refLvl > 0) { + // If we have encountered this refinement level before and stored + // the path this builder follows, we will just take the same path + // again. + if ( static_cast(path.size()) >= refLvl ) { + + if(debug) { + std::cout << "I am cell " << id << ". "; + std::cout << "I have seen refinement level " << refLvl << " before. Path is "; + for (auto k = path.begin(); k != path.end(); ++k) + std::cout << *k << " "; + std::cout << std::endl; + } + + nextNeighbor = selectNeighbor(grid,id,dimension,path[refLvl - 1]); + if(nextNeighbor != INVALID_CELLID) { + coordinates = grid.get_center(nextNeighbor); + } + } else { + // We encounter a new refinement level. + if(debug) { + std::cout << "I am cell " << id << ". "; + std::cout << "I have NOT seen refinement level " << refLvl << " before. Path is "; + for (auto k = path.begin(); k != path.end(); ++k) + std::cout << *k << ' '; + std::cout << std::endl; + } + + // Create a path through each neighbor cell + for ( uint newPath : {0,1,2,3} ) { + vector < uint > myPath = path; + // Extend the path to cover the new reflevel + myPath.push_back(newPath); + nextNeighbor = selectNeighbor(grid,id,dimension,newPath); + + if ( newPath == 3 ) { + // This builder continues with neighbor 3 with an extended path + path = myPath; + if(nextNeighbor != INVALID_CELLID) { + coordinates = grid.get_center(nextNeighbor); + } + } else { + // Spawn recursive new builders for neighbors 0,1,2 + buildPencilsWithNeighbors(grid,pencils,id,ids,dimension,myPath,endIds); + } + } + } + } // Closes if (refLvl == 0) + + // If we found a neighbor, let's verify if it should be translated + if(nextNeighbor != INVALID_CELLID) { + if (debug) { + std::cout << " Next neighbor is " << nextNeighbor << "." << std::endl; + } + // Non-local, non-translated, and ids belonging to other pencils are not included + if ( std::any_of(endIds.begin(), endIds.end(), [nextNeighbor](uint i){return i == nextNeighbor;}) || + !do_translate_cell(grid[nextNeighbor])) { + nextNeighbor = INVALID_CELLID; + } else { + // Yep, this goes in this pencil. + ids.push_back(nextNeighbor); + } + } + + id = nextNeighbor; + } // Closes while loop - end of pencil reached. + + // Get the x,y - coordinates of the pencil (in the direction perpendicular to the pencil) + double x,y; + int ix=0, iy=0; + + switch(dimension) { + case 0: + ix = 1; + iy = 2; + break; + case 1: + ix = 0; + iy = 2; + break; + case 2: + ix = 0; + iy = 1; + break; + } + //ix = (dimension + 1) % 3; // incorrect for DCCRG + //iy = (dimension + 2) % 3; + + x = coordinates[ix]; + y = coordinates[iy]; + + pencils.addPencil(ids,x,y,periodic,path); + return; +} + +/* Determine which cells in the local DCCRG mesh should be starting points for pencils. + * If a neighbor cell is non-local, across a periodic boundary, or in non-periodic boundary layer 1 + * then we use this cell as a seed for pencils + * + * @param [in] mpiGrid DCCRG grid object + * @param [in] localPropagatedCells List of local cells that get propagated + * ie. not L2-boundary or DO_NOT_COMPUTE + * @param [in] dimension Spatial dimension + * @param [out] seedIds list of cell ids that will be starting points for pencils + */ +void getSeedIds(const dccrg::Dccrg& mpiGrid, + const vector &localPropagatedCells, + const uint dimension, + vector &seedIds) { + + const bool debug = false; + int myRank; + if (debug) MPI_Comm_rank(MPI_COMM_WORLD,&myRank); + + // These neighborhoods now include the AMR addition beyond the regular vlasov stencil + int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); + +#pragma omp parallel for + for (uint i=0; i distancesplus; + std::set< int > distancesminus; + for (const auto& nbrPair : *nbrPairs) { + if(nbrPair.second[dimension] > 0) { + distancesplus.insert(nbrPair.second[dimension]); + } + if(nbrPair.second[dimension] < 0) { + // gather absolute distance values for correct order + distancesminus.insert(-nbrPair.second[dimension]); + } + } + int iSrc = VLASOV_STENCIL_WIDTH-1; + for (auto it = distancesplus.begin(); it != distancesplus.end(); ++it) { + if (iSrc < 0) break; // found enough elements + for (const auto& nbrPair : *nbrPairs) { + int distanceInRefinedCells = nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) { + // Break search if we are not at the final entry, and have different refinement level + if (iSrc!=0 && mpiGrid.get_refinement_level(nbrPair.first)!=myRefLevel) { + iSrc = -1; + break; + } + // Flag as seed id if VLASOV_STENCIL_WIDTH positive neighbour is at higher refinement level + if (iSrc==0 && mpiGrid.get_refinement_level(nbrPair.first)>myRefLevel) { + addToSeedIds = true; + break; + } + } + } + iSrc--; + } // Finish B check + + if ( addToSeedIds ) { +#pragma omp critical + seedIds.push_back(celli); + continue; + } + /* Proceed with C, checking if the next two negative neighbours have the same refinement level as ccell, but the + third neighbour a higher one. Iterate through negative distances for VLASOV_STENCIL_WIDTH+1 elements + starting from the smallest distance. */ + iSrc = VLASOV_STENCIL_WIDTH; + for (auto it = distancesminus.begin(); it != distancesminus.end(); ++it) { + if (iSrc < 0) break; // found enough elements + for (const auto& nbrPair : *nbrPairs) { + int distanceInRefinedCells = -nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) { + // Break search if we are not at the final entry, and have different refinement level + if (iSrc!=0 && mpiGrid.get_refinement_level(nbrPair.first)!=myRefLevel) { + iSrc = -1; + break; + } + // Flag as seed id if VLASOV_STENCIL_WIDTH+1 positive neighbour is at higher refinement level + if (iSrc==0 && mpiGrid.get_refinement_level(nbrPair.first)>myRefLevel) { + addToSeedIds = true; + break; + } + } + } + iSrc--; + } // Finish C check + + if ( addToSeedIds ) { +#pragma omp critical + seedIds.push_back(celli); + } + } + + if(debug) { + cout << "Rank " << myRank << ", Seed ids are: "; + for (const auto seedId : seedIds) { + cout << seedId << " "; + } + cout << endl; + } +} + +/* Check whether the ghost cells around the pencil contain higher refinement than the pencil does. + * If they do, the pencil must be split to match the finest refined ghost cell. + * + * @param mpiGrid DCCRG grid object + * @param pencils Pencil data struct + * @param dimension Spatial dimension + */ +void check_ghost_cells(const dccrg::Dccrg& mpiGrid, + setOfPencils& pencils, + uint dimension) { + + const bool debug = false; + int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); + + int myRank; + if(debug) { + MPI_Comm_rank(MPI_COMM_WORLD,&myRank); + } + + std::vector pencilIdsToSplit; + +#pragma omp parallel for + for (uint pencili = 0; pencili < pencils.N; ++pencili) { + + // This check isn't in use at the moment, because no pencils are ever flagged periodic.. + if (pencils.periodic[pencili]) { + continue; + } + + // This returns a list of only the central cells, excluding the stencil + auto ids = pencils.getIds(pencili); + + // It is possible that the pencil has already been refined by the pencil building algorithm + // and is on a higher refinement level than the refinement level of any of the cells it contains + // due to e.g. process boundaries. Example: x is the pencil, N is non-local cells. + /* ----------------------------------------- + | | | | |_|_|_|_| N | N | N | N | + |xxx|xxx|xxx|xxx|N|N|N|N| | | | | + ----------------------------------------- + */ + int maxPencilRefLvl = pencils.path[pencili].size(); + int maxNbrRefLvl = 0; + + const auto* frontNeighbors = mpiGrid.get_neighbors_of(ids.front(),neighborhood); + const auto* backNeighbors = mpiGrid.get_neighbors_of(ids.back(),neighborhood); + + // Create list of unique distances in the negative direction from the first cell in pencil + std::set< int > distances; // is sorted + for (const auto& nbrPair : *frontNeighbors) { + if(nbrPair.second[dimension] < 0) { + // gather absolute distance values + distances.insert(-nbrPair.second[dimension]); + } + } + int foundcells = 0; + CellID lastcell = INVALID_CELLID; + // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (auto it = distances.begin(); it != distances.end(); ++it) { + for (const auto& nbrPair : *frontNeighbors) { + if (nbrPair.first==lastcell) continue; + int distanceInRefinedCells = -nbrPair.second[dimension]; + if (distanceInRefinedCells == *it) { + maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + lastcell = nbrPair.first; + foundcells++; + continue; + } + } + if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances + } + + // Create list of unique distances in the positive direction from the last cell in pencil + distances.clear(); + for (const auto& nbrPair : *backNeighbors) { + if(nbrPair.second[dimension] > 0) { + distances.insert(nbrPair.second[dimension]); + } + } + foundcells = 0; + lastcell = INVALID_CELLID; + for (auto it = distances.begin(); it != distances.end(); ++it) { + for (const auto& nbrPair : *backNeighbors) { + if (nbrPair.first==lastcell) continue; + int distanceInRefinedCells = nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) { + maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + lastcell = nbrPair.first; + foundcells++; + continue; + } + } + if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances + } + + if (maxNbrRefLvl > maxPencilRefLvl) { + if(debug) { + std::cout << "I am rank " << myRank << ". "; + std::cout << "Found refinement level " << maxNbrRefLvl << " in one of the ghost cells of pencil " << pencili << ". "; + std::cout << "Highest refinement level in this pencil is " << maxPencilRefLvl; + std::cout << ". Splitting pencil " << pencili << endl; + } + // Let's avoid modifying pencils while we are looping over it. Write down the indices of pencils + // that need to be split and split them later. +#pragma omp critical + { + pencilIdsToSplit.push_back(pencili); + } + } + } + + // No threading here! Splitting requires knowledge of all + // already existing pencils. + for (auto pencili: pencilIdsToSplit) { + + Real dx = 0.0; + Real dy = 0.0; + auto ids = pencils.getIds(pencili); + switch(dimension) { + case 0: + dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DY]; + dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; + break; + case 1: + dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; + dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; + break; + case 2: + dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; + dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DY]; + break; + } + +// WARNING threading inside this function + pencils.split(pencili,dx,dy); + + } +} + +/* Checks that each local spatial cell appears in pencils at least 1 time. + * + * @param mpiGrid DCCRG grid object + * @param cells Local spatial cells + * @param pencils Pencil data struct + */ +bool checkPencils( + const dccrg::Dccrg& mpiGrid, + const std::vector &cells, + const setOfPencils& pencils +) { + bool correct = true; + for (auto id : cells) { + if (mpiGrid[id]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY ) { + int myCount = std::count(pencils.ids.begin(), pencils.ids.end(), id); + if( myCount == 0) { + std::cerr << "ERROR: Cell ID " << id << " Appears in pencils " << myCount << " times!"<< std::endl; + correct = false; + } + } + } + for (uint ipencil = 0; ipencil < pencils.N; ++ipencil) { + cint nPencilsThroughThisCell = pow(pow(2,pencils.path[ipencil].size()),2); + auto ids = pencils.getIds(ipencil); + for (auto id : ids) { + cint myCount = std::count(pencils.ids.begin(), pencils.ids.end(), id); + if (myCount > nPencilsThroughThisCell) { + std::cerr << "ERROR: Cell ID " << id << " Appears in pencils " << myCount << " times!"<< std::endl; + std::cerr << " It should not appear more than " << nPencilsThroughThisCell << " times." << std::endl; + correct = false; + } + } + } + return correct; +} + +/* Debugging function, prints the list of cells in each pencil + * + * @param pencils Pencil data struct + * @param dimension Spatial dimension + * @param myRank MPI rank + */ +void printPencilsFunc(const setOfPencils& pencils, const uint dimension, const int myRank,const dccrg::Dccrg& mpiGrid) { + +// Print out ids of pencils (if needed for debugging) + uint ibeg = 0; + uint iend = 0; + stringstream ss; + ss << "I am rank " << myRank << ", I have " << pencils.N << " pencils along dimension " << dimension << ":\n"; + MPI_Barrier(MPI_COMM_WORLD); + if(myRank == MASTER_RANK) { + ss << "(D=DO_NOT_COMPUTE, S=Sysboundary L2, L=Sysboundary L1, N=Non-sysboundary L2, G=Ghost cell)" << std::endl; + ss << "t, N, mpirank, dimension, length (x, y): indices {path} DZs AreaRatios" << std::endl; + ss << "----------------------------------------------------------------------" << std::endl; + } + MPI_Barrier(MPI_COMM_WORLD); + for (uint i = 0; i < pencils.N; i++) { + const uint L = pencils.lengthOfPencils[i]; + iend = ibeg + L; + ss << P::t << ", "; + ss << i << ", "; + ss << myRank << ", "; + ss << dimension << ", "; + ss << L << ", "; + ss << "(" << pencils.x[i] << ", " << pencils.y[i] << "): "; + for (auto j = pencils.ids.begin() + ibeg; j != pencils.ids.begin() + iend; ++j) { + ss << *j; + if (*j && mpiGrid[*j]) { + SpatialCell* c = mpiGrid[*j]; + if (c->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) ss<<"D"; + if (c->sysBoundaryLayer != 1 && c->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY) ss<<"S"; + if (c->sysBoundaryLayer == 1 && c->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY) ss<<"L"; + if (c->sysBoundaryLayer == 2 && c->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) ss<<"N"; + if (!mpiGrid.is_local(*j)) ss<<"G"; + } + ss<< " "; + } + + ss << "{"; + for (auto step : pencils.path[i]) { + ss << step << ", "; + } + ss << "}"; + + ss << "source DZs: "; + for (auto j = pencils.sourceDZ.begin() + ibeg; j != pencils.sourceDZ.begin() + iend; ++j) { + ss << *j << " "; + } + + ss << "target Ratios: "; + for (auto j = pencils.targetRatios.begin() + ibeg; j != pencils.targetRatios.begin() + iend; ++j) { + ss << *j << " "; + } + + ibeg = iend; + ss << std::endl; + } + MPI_Barrier(MPI_COMM_WORLD); + if(myRank == MASTER_RANK) { + ss << "-----------------------------------------------------------------" << std::endl; + } + std::cout<& mpiGrid) { + phiprof::Timer timer {"GetSeedIdsAndBuildPencils"}; + // Remove all old pencils now + for (int dimension=0; dimension<3; dimension++) { + DimensionPencils[dimension].removeAllPencils(); + } + for (int dimension=0; dimension<3; dimension++) { + prepareSeedIdsAndPencils(mpiGrid, dimension); + } +} + +/* Wrapper function for calling seed ID selection and pencil generation, per dimension. + * Includes threading and gathering of pencils into thread-containers. + * + * @param [in] mpiGrid DCCRG grid object + * @param [in] dimension Spatial dimension + */ +void prepareSeedIdsAndPencils(const dccrg::Dccrg& mpiGrid, + const uint dimension) { + + // Optional heavy printouts for debugging + const bool printPencils = false; + const bool printSeeds = false; + int myRank, mpi_size; + if(printPencils || printSeeds) { + MPI_Comm_rank(MPI_COMM_WORLD,&myRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + } + + switch (dimension) { + case 0: + if(P::xcells_ini == 1) return; + break; + case 1: + if(P::ycells_ini == 1) return; + break; + case 2: + if(P::zcells_ini == 1) return; + break; + default: + std::cerr<<"Error in dimension: __FILE__:__LINE__"<& localCells = getLocalCells(); + vector localPropagatedCells; + // Figure out which spatial cells are translated, + // result independent of particle species. + for (size_t c=0; c seedIds; + getSeedIds(mpiGrid, localPropagatedCells, dimension, seedIds); + getSeedIdsTimer.stop(); + + if (printSeeds) { + for (int rank=0; ranksysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) ss<<"D"; + if (c->sysBoundaryLayer != 1 && c->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY) ss<<"S"; + if (c->sysBoundaryLayer == 1 && c->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY) ss<<"L"; + if (c->sysBoundaryLayer == 2 && c->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) ss<<"N"; + if (!mpiGrid.is_local(seedIds.at(i))) ss<<"G"; + } + ss<<" "; + } + ss< ids; + vector path; + // thread-internal pencil set to be accumulated at the end + setOfPencils thread_pencils; + // iterators used in the accumulation + std::vector::iterator ibeg, iend; + +#pragma omp for schedule(guided,8) + for (uint i=0; i pencilIds(ibeg, iend); + DimensionPencils[dimension].addPencil(pencilIds,thread_pencils.x[i],thread_pencils.y[i],thread_pencils.periodic[i],thread_pencils.path[i]); + } + } + } + + phiprof::Timer checkGhostCellsTimer {"check_ghost_cells"}; + // Check refinement of two ghost cells on each end of each pencil + // in case pencil needs to be split. + // This function contains threading. + check_ghost_cells(mpiGrid,DimensionPencils[dimension],dimension); + checkGhostCellsTimer.stop(); + + phiprof::Timer findSourceRatiosTimer {"Find_source_cells_ratios_dz"}; + // Compute also the stencil around the pencil (source cells), and + // Store source cell widths and target cell contribution ratios. +#pragma omp parallel for schedule(guided) + for (uint i=0; i0.0)) { +#pragma omp critical + { + DimensionTargetCells[dimension].insert(targ); + } + } + } + + // Warning: checkPencils fails to understand situations where pencils reach across 3 levels of refinement. + // if(!checkPencils(mpiGrid, localPropagatedCells, pencils)) { + // std::cerr<<"abort checkpencils"< +#include "vec.h" +#include "../common.h" +#include "../spatial_cell_wrapper.hpp" + +struct setOfPencils { + + uint N; // Number of pencils in the set + uint sumOfLengths; + std::vector< uint > lengthOfPencils; // Lengths of pencils (including stencil cells) + std::vector< CellID > ids; // List of pencil cells (including stencil cells) + std::vector< uint > idsStart; // List of where a pencil's CellIDs start in the ids array + std::vector< Realf > sourceDZ; // Widths of source cells + std::vector< Realf > targetRatios; // Pencil to target cell area ratios of target cells + std::vector< Real > x,y; // x,y - position + std::vector< bool > periodic; + std::vector< std::vector > path; // Path taken through refinement levels + + setOfPencils() { + N = 0; + sumOfLengths = 0; + } + + void removeAllPencils() { + N = 0; + sumOfLengths = 0; + sumOfLengths = 0; + lengthOfPencils.clear(); + idsStart.clear(); + ids.clear(); + sourceDZ.clear(); + targetRatios.clear(); + x.clear(); + y.clear(); + periodic.clear(); + path.clear(); + } + + void addPencil(std::vector idsIn, Real xIn, Real yIn, bool periodicIn, std::vector pathIn) { + N++; + // If necessary, add the zero cells to the beginning and end + if (idsIn.front() != 0) { + idsIn.insert(idsIn.begin(),VLASOV_STENCIL_WIDTH,0); + } + if (idsIn.back() != 0) { + for (int i = 0; i < VLASOV_STENCIL_WIDTH; i++) + { + idsIn.push_back(0); + } + } + sumOfLengths += idsIn.size(); + lengthOfPencils.push_back(idsIn.size()); + idsStart.push_back(ids.size()); + ids.insert(ids.end(),idsIn.begin(),idsIn.end()); + sourceDZ.resize(sumOfLengths); + targetRatios.resize(sumOfLengths); + x.push_back(xIn); + y.push_back(yIn); + periodic.push_back(periodicIn); + path.push_back(pathIn); + } + + void removePencil(const uint pencilId) { + x.erase(x.begin() + pencilId); + y.erase(y.begin() + pencilId); + periodic.erase(periodic.begin() + pencilId); + path.erase(path.begin() + pencilId); + + uint ibeg = idsStart[pencilId]; + ids.erase(ids.begin() + ibeg, ids.begin() + ibeg + lengthOfPencils[pencilId] + 2*VLASOV_STENCIL_WIDTH); + targetRatios.erase(targetRatios.begin() + ibeg, targetRatios.begin() + ibeg + lengthOfPencils[pencilId] + 2*VLASOV_STENCIL_WIDTH); + sourceDZ.erase(sourceDZ.begin() + ibeg, sourceDZ.begin() + ibeg + lengthOfPencils[pencilId] + 2*VLASOV_STENCIL_WIDTH); + idsStart.erase(idsStart.begin() + pencilId); + + N--; + sumOfLengths -= lengthOfPencils[pencilId]; + lengthOfPencils.erase(lengthOfPencils.begin() + pencilId); + } + + std::vector getIds(const uint pencilId) const { + if (pencilId >= N) { + std::vector idsEmpty; + return idsEmpty; + } + // Use vector range constructor. Only return actual pencil ids, not the stencils at the ends + std::vector::const_iterator ibeg = ids.begin() + idsStart[pencilId] + VLASOV_STENCIL_WIDTH; + std::vector::const_iterator iend = ibeg + lengthOfPencils[pencilId] - 2*VLASOV_STENCIL_WIDTH; + std::vector idsOut(ibeg, iend); + return idsOut; + } + + // Split one pencil into up to four pencils covering the same space. + // dx and dy are the dimensions of the original pencil. + void split(const uint myPencilId, const Real dx, const Real dy) { + auto myIds = this->getIds(myPencilId); + + // Find paths that members of this pencil may have in other pencils (can happen) + // so that we don't add duplicates. + std::vector existingSteps; + +#pragma omp parallel for + for (uint theirPencilId = 0; theirPencilId < this->N; ++theirPencilId) { + if(theirPencilId == myPencilId) continue; + auto theirIds = this->getIds(theirPencilId); + for (auto theirId : theirIds) { + for (auto myId : myIds) { + if (myId == theirId) { + std::vector theirPath = this->path.at(theirPencilId); + std::vector myPath = this->path.at(myPencilId); + if(theirPath.size() > myPath.size()) { + bool samePath = true; + for (uint i = 0; i < myPath.size(); ++i) { + if(myPath.at(i) != theirPath.at(i)) { + samePath = false; + } + } + + if(samePath) { + uint theirStep = theirPath.at(myPath.size()); +#pragma omp critical + { + existingSteps.push_back(theirStep); + } + } + } + } + } + } + } // end parallel region + + bool firstPencil = true; + const auto copy_of_path = path.at(myPencilId); + const auto copy_of_x = x.at(myPencilId); + const auto copy_of_y = y.at(myPencilId); + + // Add those pencils whose steps dont already exist in the pencils struct + for (int step = 0; step < 4; ++step) { + if (std::any_of(existingSteps.begin(), existingSteps.end(), [step](int i){return step == i;})) { + continue; + } + + Real signX = 1.0; + Real signY = 1.0; + + if(step < 2) { + signY = -1.0; + } + + if(step % 2 == 0) { + signX = -1.0; + } + + auto myX = copy_of_x + signX * 0.25 * dx; + auto myY = copy_of_y + signY * 0.25 * dy; + + if (firstPencil) { + //TODO: set x and y correctly. Right now they are not used anywhere. + path.at(myPencilId).push_back(step); + x.at(myPencilId) = myX; + y.at(myPencilId) = myY; + firstPencil = false; + } else { + auto myPath = copy_of_path; + myPath.push_back(step); + addPencil(myIds, myX, myY, periodic.at(myPencilId), myPath); + } + } + } +}; +// Note: Splitting does not handle target or source cells, as those are computed after all pencil splitting has concluded. + +bool do_translate_cell(spatial_cell::SpatialCell* SC); + +// grid.cpp calls this function to both find seed cells and build pencils for all dimensions +void prepareSeedIdsAndPencils(const dccrg::Dccrg& mpiGrid); +// find seed cells and build pencils for one dimension +void prepareSeedIdsAndPencils(const dccrg::Dccrg& mpiGrid, + const uint dimension); + +// pencils used for AMR translation +extern std::array DimensionPencils; +extern std::array,3> DimensionTargetCells; + +void flagSpatialCellsForAmrCommunication(const dccrg::Dccrg& mpiGrid, + const std::vector& localPropagatedCells); + +#endif diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index 7cc32dbba..0332df94f 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -32,7 +32,7 @@ #include #include -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../vlasovmover.h" #include "../grid.h" #include "../definitions.h" @@ -55,13 +55,13 @@ creal ONE = 1.0; creal TWO = 2.0; creal EPSILON = 1.0e-25; -/** Propagates the distribution function in spatial space. - +/** Propagates the distribution function in spatial space. + Based on SLICE-3D algorithm: Zerroukat, M., and T. Allen. "A three‐dimensional monotone and conservative semi‐Lagrangian scheme (SLICE‐3D) for transport problems." Quarterly Journal of the Royal Meteorological Society 138.667 (2012): 1640-1651. - + */ void calculateSpatialTranslation( dccrg::Dccrg& mpiGrid, @@ -81,7 +81,7 @@ void calculateSpatialTranslation( if (P::amrMaxSpatialRefLevel > 0) AMRtranslationActive = true; double t1; - + int myRank; MPI_Comm_rank(MPI_COMM_WORLD,&myRank); @@ -198,7 +198,7 @@ void calculateSpatialTranslation( if(P::amrMaxSpatialRefLevel == 0) { trans_map_1d(mpiGrid,local_propagated_cells, remoteTargetCellsy, 1,dt,popID); // map along y// } else { - trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencils, 1,dt,popID); // map along y// + trans_map_1d_amr(mpiGrid,local_propagated_cells, remoteTargetCellsy, nPencils, 1,dt,popID); // map along y// } computeTimer.stop(); time += MPI_Wtime() - t1; @@ -227,9 +227,9 @@ void calculateSpatialTranslation( } /*! - - Propagates the distribution function in spatial space. - + + Propagates the distribution function in spatial space. + Based on SLICE-3D algorithm: Zerroukat, M., and T. Allen. "A three‐dimensional monotone and conservative semi‐Lagrangian scheme (SLICE‐3D) for transport problems." Quarterly Journal of the Royal @@ -253,8 +253,8 @@ void calculateSpatialTranslation( vector local_target_cells; vector nPencils; Real time=0.0; - - // If dt=0 we are either initializing or distribution functions are not translated. + + // If dt=0 we are either initializing or distribution functions are not translated. // In both cases go to the end of this function and calculate the moments. if (dt == 0.0) { calculateMoments_R(mpiGrid,localCells,true); @@ -265,7 +265,7 @@ void calculateSpatialTranslation( remoteTargetCellsx = mpiGrid.get_remote_cells_on_process_boundary(VLASOV_SOLVER_TARGET_X_NEIGHBORHOOD_ID); remoteTargetCellsy = mpiGrid.get_remote_cells_on_process_boundary(VLASOV_SOLVER_TARGET_Y_NEIGHBORHOOD_ID); remoteTargetCellsz = mpiGrid.get_remote_cells_on_process_boundary(VLASOV_SOLVER_TARGET_Z_NEIGHBORHOOD_ID); - + // Figure out which spatial cells are translated, // result independent of particle species. for (size_t c=0; cget_max_v_dt(popID); - + //compute subcycle dt. The length is maxVdt on all steps //except the last one. This is to keep the neighboring //spatial cells in sync, so that two neighboring cells with @@ -382,7 +384,7 @@ void calculateAcceleration(const uint popID,const uint globalMaxSubcycles,const subcycleDt = maxVdt; } if (dt<0) subcycleDt = -subcycleDt; - + //generate pseudo-random order which is always the same irrespective of parallelization, restarts, etc. std::default_random_engine rndState; // set seed, initialise generator and get value. The order is the same @@ -390,7 +392,7 @@ void calculateAcceleration(const uint popID,const uint globalMaxSubcycles,const rndState.seed(P::tstep); uint map_order=std::uniform_int_distribution<>(0,2)(rndState); - phiprof::Timer semilagAccTimer {"cell-semilag-acc"}; + phiprof::Timer semilagAccTimer {timerId}; cpu_accelerate_cell(mpiGrid[cellID],popID,map_order,subcycleDt); semilagAccTimer.stop(); } @@ -406,23 +408,23 @@ void calculateAcceleration(const uint popID,const uint globalMaxSubcycles,const if(step < (globalMaxSubcycles - 1)) adjustVelocityBlocks(mpiGrid, propagatedCells, false, popID); } -/** Accelerate all particle populations to new time t+dt. +/** Accelerate all particle populations to new time t+dt. * This function is AMR safe. * @param mpiGrid Parallel grid library. * @param dt Time step.*/ void calculateAcceleration(dccrg::Dccrg& mpiGrid, Real dt - ) { + ) { typedef Parameters P; const vector& cells = getLocalCells(); int myRank; MPI_Comm_rank(MPI_COMM_WORLD,&myRank); - + if (dt == 0.0 && P::tstep > 0) { - - // Even if acceleration is turned off we need to adjust velocity blocks - // because the boundary conditions may have altered the velocity space, + + // Even if acceleration is turned off we need to adjust velocity blocks + // because the boundary conditions may have altered the velocity space, // and to update changes in no-content blocks during translation. for (uint popID=0; popID& cells = getLocalCells(); - + //Iterate through all local cells #pragma omp parallel for for (size_t c=0; c #include diff --git a/vlasovsolver_amr/cpu_acc_map_amr.hpp b/vlasovsolver_amr/cpu_acc_map_amr.hpp index c4f1aabc7..e8ca5df37 100644 --- a/vlasovsolver_amr/cpu_acc_map_amr.hpp +++ b/vlasovsolver_amr/cpu_acc_map_amr.hpp @@ -28,7 +28,7 @@ #include "cmath" #include "utility" #include "common.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "cpu_acc_sort_blocks.hpp" #include "cpu_1d_pqm.hpp" #include "cpu_1d_ppm.hpp" diff --git a/vlasovsolver_amr/cpu_acc_semilag.hpp b/vlasovsolver_amr/cpu_acc_semilag.hpp index ebfc4a420..381ce1cfa 100644 --- a/vlasovsolver_amr/cpu_acc_semilag.hpp +++ b/vlasovsolver_amr/cpu_acc_semilag.hpp @@ -32,7 +32,7 @@ #include "boost/unordered_map.hpp" #include "common.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include #include diff --git a/vlasovsolver_amr/cpu_acc_sort_blocks.hpp b/vlasovsolver_amr/cpu_acc_sort_blocks.hpp index c4b498266..a4de8a82f 100644 --- a/vlasovsolver_amr/cpu_acc_sort_blocks.hpp +++ b/vlasovsolver_amr/cpu_acc_sort_blocks.hpp @@ -27,7 +27,7 @@ #include "cmath" #include "utility" #include "common.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" diff --git a/vlasovsolver_amr/cpu_acc_transform.hpp b/vlasovsolver_amr/cpu_acc_transform.hpp index 876b01009..7e77a24c9 100644 --- a/vlasovsolver_amr/cpu_acc_transform.hpp +++ b/vlasovsolver_amr/cpu_acc_transform.hpp @@ -23,7 +23,7 @@ #define CPU_ACC_TRANSFORM_H #include "common.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include #include diff --git a/vlasovsolver_amr/cpu_trans_map_amr.hpp b/vlasovsolver_amr/cpu_trans_map_amr.hpp index 014a10b52..772b78557 100644 --- a/vlasovsolver_amr/cpu_trans_map_amr.hpp +++ b/vlasovsolver_amr/cpu_trans_map_amr.hpp @@ -31,7 +31,7 @@ #include "cmath" #include "utility" #include "common.h" -#include "spatial_cell.hpp" +#include "spatial_cell_wrapper.hpp" #include "cpu_1d_plm.hpp" #include "cpu_1d_ppm.hpp" #include "cpu_1d_pqm.hpp" diff --git a/vlasovsolver_amr/vlasovmover.cpp b/vlasovsolver_amr/vlasovmover.cpp index 7362173bc..2bd8a3590 100644 --- a/vlasovsolver_amr/vlasovmover.cpp +++ b/vlasovsolver_amr/vlasovmover.cpp @@ -33,7 +33,7 @@ #include #include "../vlasovmover.h" -#include "../spatial_cell.hpp" +#include "../spatial_cell_wrapper.hpp" #include "../grid.h" #include "../definitions.h" #include "../iowrite.h"