Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Compilation Tests for ARW #2115

Merged
merged 10 commits into from
Oct 16, 2024
Merged

Compilation Tests for ARW #2115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
718ccc6
Simplify regex for intel classic
islas Sep 26, 2024
2389aea
Add workflow tests for these compilers
islas Sep 26, 2024
f41871a
Add workflow tests to ci reg tests
islas Sep 26, 2024
5cf1961
Break out make compile tests into smaller portions since they cannot …
islas Sep 26, 2024
9756220
Use submit_options key for valid syntax
islas Sep 27, 2024
4067767
Updates to testing infrastructure
islas Sep 28, 2024
6e92963
Add better naming for test logs and jobs
islas Sep 30, 2024
e8acf48
Adjust timings required
islas Sep 30, 2024
2911df9
Parallelize copy of test directories
islas Sep 30, 2024
69152b4
Define tests variable and parallel rm
islas Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci/hpc-workflows
Open in desktop
Submodule hpc-workflows updated 13 files
+8 −0 .ci/Step.py
+26 −8 .ci/SubmitAction.py
+10 −0 .ci/SubmitCommon.py
+57 −48 .ci/SubmitOptions.py
+4 −0 .ci/Test.py
+0 −6 .ci/example.json
+16 −1 .ci/runner.py
+4 −2 tutorials/AdvancedRunOption_join_hpc.ipynb
+30 −21 tutorials/AdvancedRunOption_join_hpc.md
+11 −8 tutorials/AdvancedTestConfig_host_specific.md
+50 −50 tutorials/AdvancedTestConfig_hpc_argpacks.md
+17 −17 tutorials/AdvancedTestConfig_regex_argpacks.md
+13 −13 tutorials/BasicTestConfig.md
92 changes: 57 additions & 35 deletions .ci/wrf_compilation_tests-make.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,53 +17,75 @@
"queue" : "main",
"hpc_arguments" :
{
"node_select" : { "-l " : { "select" : 1, "ncpus" : 16 } },
"node_select" : { "-l " : { "select" : 1, "ncpus" : 8 } },
"priority" : { "-l " : { "job_priority" : "economy" } }
},
"arguments" :
{
"base_env_numprocs" : [ "-e", "NUM_PROCS=16" ],
"base_env_numprocs" : [ "-e", "NUM_PROCS=8" ],
"very_last_modules" : [ "netcdf" ],
".*gnu.*::test_modules" : [ "gcc" ],
".*intel(?!-llvm).*::test_modules" : [ "intel-classic" ],
".*intel-classic.*::test_modules" : [ "intel-classic" ],
".*intel-llvm.*::test_modules" : [ "intel-oneapi" ],
".*pgi.*::test_modules" : [ "nvhpc" ],
".*dm.*::test_mpi_module" : [ "cray-mpich" ]
}
}
},
"make-gnu" :
{
"steps" :
{
"serial" :
{
"command" : ".ci/tests/build.sh",
"arguments" : [ "-c", "32" ]
},
"sm" :
{
"command" : ".ci/tests/build.sh",
"arguments" : [ "-c", "33" ],
"dependencies" : { "serial" : "afterany" }
}
}

"make-gnu-serial" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "32" ] } } },
"make-gnu-sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "33" ] } } },
"make-gnu-dm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "34" ] } } },
"make-gnu-dm+sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "35" ] } } },

"make-intel-classic-serial" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "13" ] } }
},
"make-gnu-mpi" :
{
"steps" :
{
"dm" :
{
"command" : ".ci/tests/build.sh",
"arguments" : [ "-c", "34" ]
},
"dm+sm" :
{
"command" : ".ci/tests/build.sh",
"arguments" : [ "-c", "35" ],
"dependencies" : { "dm" : "afterany" }
}
}
"make-intel-classic-sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "14" ] } }
},
"make-intel-classic-dm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "15" ] } }
},
"make-intel-classic-dm+sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "16" ] } }
},

"make-intel-llvm-serial" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "76" ] } }
},
"make-intel-llvm-sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "77" ] } }
},
"make-intel-llvm-dm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "78" ] } }
},
"make-intel-llvm-dm+sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "79" ] } }
},

"make-pgi-serial" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "1" ] } }
},
"make-pgi-sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "2" ] } }
},
"make-pgi-dm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "3" ] } }
},
"make-pgi-dm+sm" : {
"submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
"steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "4" ] } }
}
}
18 changes: 16 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,22 @@ jobs:
tpool : 1
mkdirs : true
tests :
- make-gnu
- make-gnu-mpi
- make-gnu-serial
- make-gnu-sm
- make-gnu-dm
- make-gnu-dm+sm
- make-intel-classic-serial
- make-intel-classic-sm
- make-intel-classic-dm
- make-intel-classic-dm+sm
- make-intel-llvm-serial
- make-intel-llvm-sm
- make-intel-llvm-dm
- make-intel-llvm-dm+sm
- make-pgi-serial
- make-pgi-sm
- make-pgi-dm
- make-pgi-dm+sm
# add new compilation tests here

uses : ./.github/workflows/test_workflow.yml
Expand Down
20 changes: 12 additions & 8 deletions .github/workflows/test_workflow.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,15 @@ jobs:
if : ${{ inputs.mkdirs }}
id : cpTestDirs
run : |
for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do
echo "Creating duplicate directory for $testDir"
# Remove if it exists to get a fresh start
rm -rf $testDir
cp -Rp main/ $testDir
done
# Limit parallel ops to 4 at a time to not thrash login nodes

# Remove if it exists to get a fresh start
echo "Cleaning old test directories..."
tests="${{ join( fromJson( inputs.tests ), ' ' ) }}"
time printf "%s\n" $tests | xargs -i -P 4 rm -rf {}

echo "Creating duplicate directory for $tests"
time printf "%s\n" $tests | xargs -i -P 4 cp -Rp main/ {}

- name: Test ${{ inputs.name }}
id : runTest
Expand All @@ -87,6 +90,7 @@ jobs:
-t ${{ join( fromJson( inputs.tests ), ' ' ) }} \
-a "${{ inputs.account }}" \
-p ${{ inputs.pool}} -tp ${{ inputs.tpool }} \
-jn ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }} \
${{ inputs.args }} $ALT_DIRS


Expand Down Expand Up @@ -129,8 +133,8 @@ jobs:
# *documented* functionality doesn't work as expected. Wow, bravo
# can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex
# and expands to nothing
name: ${{ github.event_name == 'push' && 'master' || github.event.number }}-${{ inputs.id }}_logfiles
path: ${{ inputs.archive }}/${{ github.event_name == 'push' && 'master' || github.event.number }}/${{ inputs.id }}/
name: ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }}_logfiles
path: ${{ inputs.archive }}/${{ github.event_name == 'push' && github.ref_name || github.event.number }}/${{ inputs.id }}/

# As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target
# and strictly controlled usage of the GH token
Expand Down