diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml new file mode 100644 index 000000000..5bcf5051c --- /dev/null +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -0,0 +1,18 @@ +name: Nightly 01 Directory Setup + +runs: + using: "composite" + steps: + - name: Nightly 01 Directory Setup + shell: bash + run: | + # Write RUNID to file so that following steps use the + # same date and don't pick up the next day's date + RUNID=`date +"%Y%m%d"` + echo "${RUNID}" > RUNID_SAVE.log + # Get day of week for later use and write to file + DOW=$(date +%u) + echo "${DOW}" > DOW_SAVE.log + # + cd util/weekly_build + ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml new file mode 100644 index 000000000..268adaf29 --- /dev/null +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 02 Get Spack-Stack + +runs: + using: "composite" + steps: + - name: Nightly 02 Get Spack-Stack + shell: bash + run: | + RUNID=$(``: Set this variable to ON in order to enable the corresponding test under util/weekly_build/apptests/. + - ``$BUILD_CACHE``: Name of or path to (beginning with '\file://') build cache directory. Defaults to 'local-binary' mirror defined in site's mirrors.yaml. + - ``$SOURCE_CACHE``: Name of or path to (beginning with '\file://') source cache directory. Defaults to 'local-source' mirror defined in site's mirrors.yaml. + - ``$PADDED_LENGTH``: Padded length setting for Spack build cache generation. It should be as long as possible without the build failing. + - ``$KEEP_WEEKLY_BUILD_DIR``: Set to 'YES' to use a persistent directory structure for automated builds. + +------------------------------ +Setting up weekly builds: cron +------------------------------ + +Prior to running, obtain a current copy of the build scripts by cloning the spack-stack repository to the platform of interest (/path/to/spack-stack-auto-build/). + +If /path/to/spack-stack is the local spack-stack root directory containing spack-stack-x.y.z installations, run ``mkdir -p /path/to/spack-stack/weekly_build/logs``. + +In crontab, add: + +.. code-block:: console + + 0 1 * * SUN /path/to/spack-stack-auto-build/util/weekly_build/SpackStackBuildCache_AllSteps.sh $(date +\%y\%m\%d) /path/to/spack-stack/weekly_build acorn > /path/to/spack-stack/weekly_build/logs/cron.$(date +\%y\%m\%d).out 2>&1 diff --git a/doc/source/index.rst b/doc/source/index.rst index e084a1ed1..7a67d65f2 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -19,6 +19,7 @@ Table of contents Utilities KnownIssues MaintainersSection + AutomatedBuilds Documentation Acronyms Glossary diff --git a/util/weekly_build/01_DirectorySetup.sh b/util/weekly_build/01_DirectorySetup.sh new file mode 100755 index 000000000..f7a2af74a --- /dev/null +++ b/util/weekly_build/01_DirectorySetup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +set -ex + +echo Base directory: ${RUNDIR:?} + +mkdir -p $RUNDIR diff --git a/util/weekly_build/02_GetSpackStack.sh b/util/weekly_build/02_GetSpackStack.sh new file mode 100755 index 000000000..f1eab34eb --- /dev/null +++ b/util/weekly_build/02_GetSpackStack.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +set -ex + +cd $RUNDIR +if [ -d $RUNID ]; then + cd $RUNID + git pull +else + git clone --depth 1 --recurse-submodules --shallow-submodules ${SPACK_STACK_URL:-https://github.com/JCSDA/spack-stack} -b ${SPACK_STACK_BRANCH:-develop} $RUNID +fi diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh new file mode 100755 index 000000000..a98fa6cdd --- /dev/null +++ b/util/weekly_build/03_SetupEnv.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +set -ex + +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + echo "Setting up environment $envname in $envdir" + rm -rf $envdir + spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler + cd $envdir + spack env activate . + spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" + # Check for duplicates and fail before doing the "real" concretization: + spack_wrapper log.concretize concretize --fresh + ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf +# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. +# spack concretize --force --fresh --test all | tee log.concretize_test + done +done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh new file mode 100755 index 000000000..bddf21f42 --- /dev/null +++ b/util/weekly_build/04_SpackInstall.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -ex + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/${RUNID} + +set +x +. setup.sh +set -x + +if [ "$REUSE_BUILD_CACHE" == YES ]; then + cache_flag="--no-check-signature" +else + cache_flag="--no-cache" +fi + +INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + echo "Building environment $envname in $envdir" + cd $envdir + spack env activate . + if [ "${SOURCE_CACHE::7}" == "file://" ]; then + mirrorpath=${SOURCE_CACHE} + else + mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") + fi + mirrorpath=${mirrorpath#file://} + spack_wrapper log.fetch mirror create --dependencies \ + --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \ + ${PACKAGES_TO_INSTALL:---all} + # Just install the packages we're testing (+dependencies): + if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then + spack_install_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + fi + # Install the rest of the stack as usual: + spack_install_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL + done +done diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh new file mode 100755 index 000000000..2799c0464 --- /dev/null +++ b/util/weekly_build/05_BuildCache.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + cd $envdir + spack env activate . + spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL + spack buildcache rebuild-index ${BUILD_CACHE} + done +done diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh new file mode 100755 index 000000000..a8ee6ffb1 --- /dev/null +++ b/util/weekly_build/06_AppTests.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + cd $envdir + spack env activate . + spack module lmod refresh -y + spack stack setup-meta-modules + done +done + +if [ "$TEST_UFSWM" == ON ]; then + . $(dirname $0)/apptests/test_ufswm.sh +fi diff --git a/util/weekly_build/07_Cleanup.sh b/util/weekly_build/07_Cleanup.sh new file mode 100755 index 000000000..2d7a1f84e --- /dev/null +++ b/util/weekly_build/07_Cleanup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +if [ "$KEEP_WEEKLY_BUILD_DIR" != YES ]; then + /usr/bin/rm -rf ${RUNDIR:?}/${RUNID:?} +fi diff --git a/util/weekly_build/README b/util/weekly_build/README new file mode 100644 index 000000000..31232331b --- /dev/null +++ b/util/weekly_build/README @@ -0,0 +1 @@ +Concretization for deployments should be done with `spack concretize --fresh` so as to ensure that the stack would concretize the same way if the build cache weren't there. diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh new file mode 100644 index 000000000..c516e6fb3 --- /dev/null +++ b/util/weekly_build/ShellSetup.sh @@ -0,0 +1,42 @@ +echo Run ID: ${1?"First arg: Unique run ID"} +echo Run directory: ${2?"Second arg: base directory for build"} +echo Platform name: ${3?"Third arg: platform name ('hera', 'hercules', etc.)"} + +RUNID=$1 +RUNDIR=$2 +PLATFORM=$3 + +if [ ${RUNDIR::1} != "/" ]; then + echo "FATAL ERROR: Directory should be an absolute path!" + exit 1 +fi + +# Defaults to empty, can be overwritten in site config +PACKAGES_TO_TEST="" + +function alert_cmd { + echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." +} + +function spack_wrapper { + logfile=$1 + shift + set -o pipefail + spack $* 2>&1 | tee -a $logfile +} + +function spack_install_wrapper { + logfile=$1 + shift + set -o pipefail + spack $* 2>&1 | tee -a $logfile +} + +# Include platform-dependent configuration +. $(dirname $0)/sites/${PLATFORM}.sh + +SOURCE_CACHE=${SOURCE_CACHE:-local-source} +BUILD_CACHE=${BUILD_CACHE:-local-binary} + +echo "Source cache mirror name/directory: ${SOURCE_CACHE}" +echo "Build cache mirror name/directory: ${BUILD_CACHE}" diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh new file mode 100755 index 000000000..73a7931c4 --- /dev/null +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +echo "build host: $(hostname)" + +set -exa + +cd $(dirname $0) + +. ShellSetup.sh $* +export SETUPDONE=YES + +function trap_and_run { + trap ERROR ERR + scriptname=$1 + function ERROR { alert_cmd $scriptname ; exit 1;} + $* +} + +trap_and_run ./01_DirectorySetup.sh $* +trap_and_run ./02_GetSpackStack.sh $* +trap_and_run ./03_SetupEnv.sh $* +trap_and_run ./04_SpackInstall.sh $* +trap_and_run ./05_BuildCache.sh $* +trap_and_run ./06_AppTests.sh $* +trap_and_run ./07_Cleanup.sh $* diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh new file mode 100755 index 000000000..030135295 --- /dev/null +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Do some stuff, and exit non-zero if UFSWM cannot be successfully built and +# tested (ideally, have a discernably different error condition if there are +# numerical differences) + +echo "Current directory, in which ufs-weather-model will be cloned: $PWD" + +UFSWM_BRANCH=${UFSWM_BRANCH:-develop} +UFSWM_URL=${UFSWM_URL:-"https://github.com/ufs-community/ufs-weather-model.git"} + +if [ ! -d ufs-weather-model ]; then + git clone --recurse-submodules --single-branch --depth 1 --shallow-submodules ${UFSWM_URL} -b ${UFSWM_BRANCH} +fi +cd ufs-weather-model/tests + +# rt.sh will parse arguments passed to it +./rt.sh $RT_ARGS -a ${BATCHACCOUNT:?} -n 'control_c48 intel' + +rc = $? +return rc diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh new file mode 100644 index 000000000..45420c7cf --- /dev/null +++ b/util/weekly_build/sites/acorn.sh @@ -0,0 +1,29 @@ +module load gcc/11.2.0 python/3.11.7 +COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} +TEMPLATES=${TEMPLATES:-"unified-dev"} +function spack_install_wrapper { + logfile=$1 + shift +# set +e +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# wait +# rc=$? +# set -e +# cat spack-build-cache-${RUNID}* +# return $rc +## cp ${SPACK_STACK_DIR:?}/util/acorn/{build.pbs,spackinstall.sh} ${SPACK_ENV}/. +## /opt/pbs/bin/qsub -Wblock=true ${SPACK_ENV}/build.pbs +## spack $* | tee -a log.install 2>&1 + shift 1 + ${SPACK_STACK_DIR}/util/parallel_install.sh 3 4 $* +} +function alert_cmd { + module purge # annoying libstdc++ issue + mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") +} +PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" +PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" +PADDED_LENGTH=140 +TEST_UFSWM=ON +BATCHACCOUNT=NCEPLIBS-DEV diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh new file mode 100644 index 000000000..3c411475a --- /dev/null +++ b/util/weekly_build/sites/nautilus.sh @@ -0,0 +1,10 @@ +COMPILERS=${COMPILERS:-"intel oneapi gcc"} +#TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"} +TEMPLATES=${TEMPLATES:-"neptune-dev"} +module purge +umask 0022 + +SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack +SPACK_STACK_BRANCH=feature/weekly_build_nautilus +KEEP_WEEKLY_BUILD_DIR="YES" + diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh new file mode 100644 index 000000000..9f202a9f5 --- /dev/null +++ b/util/weekly_build/sites/orion.sh @@ -0,0 +1,28 @@ + +COMPILERS=${COMPILERS:-"intel"} +TEMPLATES=${TEMPLATES:-"unified-dev"} + +function alert_cmd { + mail -s 'spack-stack weekly build failure' richard.grubin@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") +} + +module --force purge +umask 0022 + +SPACK_STACK_URL=https://github.com/JCSDA/spack-stack.git +SPACK_STACK_BRANCH=develop + +KEEP_WEEKLY_BUILD_DIR="YES" +PADDED_LENGTH=200 + +# package test / install settings (future use) +# PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" +# PACKAGES_TO_INSTALL="ufs-weather-model-env" + +TEST_UFSWM=ON +UFSWM_BRANCH=develop +UFSWM_URL="https://github.com/ufs-community/ufs-weather-model.git" + +# rt.sh parameters / arguments +BATCHACCOUNT=epic +RT_ARGS="-k -r"