diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml
index 1eca69ef5..417dd8029 100644
--- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml
@@ -50,7 +50,7 @@ jobs:
# Set up spack-stack
source ./setup.sh
- export ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache
+ export ENVNAME=ue-oneifx-2024.2.0-buildcache
export ENVDIR=$PWD/envs/${ENVNAME}
spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi
spack env activate ${ENVDIR}
@@ -138,8 +138,8 @@ jobs:
spack config add "modules:default:tcl:exclude:[ecflow]"
# Concretize and check for duplicates
- spack concretize 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0-buildcache
- ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl
+ spack concretize 2>&1 | tee log.concretize.oneifx-2024.2.0-buildcache
+ ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneifx-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl
# Add and update source cache
spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
@@ -161,22 +161,22 @@ jobs:
# base-env
echo "base-env ..."
- spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.base-env
+ spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.base-env
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
# jedi-base-env
echo "jedi-base-env ..."
- spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-base-env
+ spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.jedi-base-env
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
# jedi-ufs-env
echo "jedi-ufs-env ..."
- spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-ufs-env
+ spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.jedi-ufs-env
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
# the rest
echo "unified-env ..."
- spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.unified-env
+ spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneifx-2024.2.0-buildcache.unified-env
spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
# Remove binary cache for next round of concretization
@@ -197,16 +197,16 @@ jobs:
run: |
# Set up spack-stack
source ./setup.sh
- export BUILDCACHE_ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache
+ export BUILDCACHE_ENVNAME=ue-oneifx-2024.2.0-buildcache
export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
- export ENVNAME=ue-oneapi-ifx-2024.2.0
+ export ENVNAME=ue-oneifx-2024.2.0
export ENVDIR=$PWD/envs/${ENVNAME}
rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
spack env activate ${ENVDIR}
# Concretize and check for duplicates
- spack concretize --force 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0
- ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0 -i fms -i crtm -i esmf -i mapl
+ spack concretize --force 2>&1 | tee log.concretize.oneifx-2024.2.0
+ ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneifx-2024.2.0 -i fms -i crtm -i esmf -i mapl
# Add binary cache back in
spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
@@ -214,7 +214,7 @@ jobs:
spack buildcache list
# Install from cache
- spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0.unified-env
+ spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneifx-2024.2.0.unified-env
# Check shared libraries
${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
@@ -230,7 +230,7 @@ jobs:
source /etc/profile.d/modules.sh
module use /home/ubuntu/spack-stack/modulefiles
- export ENVNAME=ue-oneapi-ifx-2024.2.0
+ export ENVNAME=ue-oneifx-2024.2.0
export ENVDIR=$PWD/envs/${ENVNAME}
ls -l ${ENVDIR}/install/modulefiles/Core
diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml
index 9bcc1c866..cf83dc500 100644
--- a/configs/containers/docker-ubuntu-clang-mpich.yaml
+++ b/configs/containers/docker-ubuntu-clang-mpich.yaml
@@ -16,7 +16,7 @@ spack:
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
flags: {}
- operating_system: ubuntu22.04
+ operating_system: ubuntu24.04
modules: []
environment:
prepend_path:
@@ -30,7 +30,7 @@ spack:
require: '%clang'
target: [x86_64]
providers:
- mpi: [mpich@4.2.1]
+ mpi: [mpich@4.2.3]
compiler: [clang@14.0.0]
gcc:
buildable: false
@@ -56,9 +56,9 @@ spack:
mpich:
buildable: false
externals:
- - spec: mpich@4.2.1
- prefix: /opt/mpich-4.2.1
- version: [4.2.1]
+ - spec: mpich@4.2.3
+ prefix: /opt/mpich-4.2.3
+ version: [4.2.3]
diffutils:
buildable: false
externals:
@@ -82,7 +82,7 @@ spack:
mysql:
buildable: false
externals:
- - spec: mysql@8.0.39
+ - spec: mysql@8.0.40
prefix: /usr
qt:
buildable: false
@@ -93,9 +93,9 @@ spack:
wget:
buildable: false
externals:
- - spec: wget@1.21.2
+ - spec: wget@1.21.4
prefix: /usr
- version: [1.21.2]
+ version: [1.21.4]
specs: []
@@ -117,7 +117,7 @@ spack:
# Sets the base images for the stages where Spack builds the
# software or where the software gets installed after being built..
images:
- os: ubuntu:22.04
+ os: ubuntu:24.04
spack:
url: https://github.com/jcsda/spack
ref: spack-stack-dev
@@ -197,15 +197,15 @@ spack:
# Register spack-stack extension repo
RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
# Set default binaries for clang-14 and GNU-12.
- RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \
- ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \
- ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \
- ln -svf /usr/bin/g++-12 /usr/bin/g++ && \
- ln -svf /usr/bin/clang-14 /usr/bin/clang && \
- ln -svf /usr/bin/clang++-14 /usr/bin/clang++ && \
- ln -svf /usr/bin/clang-cpp-14 /usr/bin/clang-cpp
+ RUN rm /usr/bin/cpp && ln -s /usr/bin/cpp-12 /usr/bin/cpp && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 100 && \
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 100 && \
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 100 && \
+ update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-14 100
# Build mpich outside of spack-stack
- ENV MPICH_VERSION=4.2.1
+ ENV MPICH_VERSION=4.2.3
ENV CC=clang
ENV CXX=clang++
ENV FC=gfortran
@@ -231,23 +231,23 @@ spack:
# Set environment variables for installing tzdata
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=Etc/UTC
- ENV MPICH_VERSION=4.2.1
+ ENV MPICH_VERSION=4.2.3
ENV PATH=/opt/mpich-${MPICH_VERSION}/bin:${PATH}
ENV CPATH=/opt/mpich-${MPICH_VERSION}/include:${CPATH}
ENV LD_LIBRARY_PATH=/opt/mpich-${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}
final: |
# Copy spack find output from builder
COPY --from=builder /root/spack_find.out /root/spack_find.out
- ENV MPICH_VERSION=4.2.1
+ ENV MPICH_VERSION=4.2.3
# Set the GNU 12.x toolchain as the default compilers.
# Set default binaries for clang-14 and GNU-12.
- RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \
- ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \
- ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \
- ln -svf /usr/bin/g++-12 /usr/bin/g++ && \
- ln -svf /usr/bin/clang-14 /usr/bin/clang && \
- ln -svf /usr/bin/clang++-14 /usr/bin/clang++ && \
- ln -svf /usr/bin/clang-cpp-14 /usr/bin/clang-cpp
+ RUN rm /usr/bin/cpp && ln -s /usr/bin/cpp-12 /usr/bin/cpp && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 100 && \
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 100 && \
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 100 && \
+ update-alternatives --install /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-14 100
# Copy mpich-${MPICH_VERSION} installation from builder
COPY --from=builder /opt/mpich-${MPICH_VERSION} /opt/mpich-${MPICH_VERSION}
# Make a non-root user:nonroot / group:nonroot for running MPI
diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
index 85827df7c..9a1ed0eb9 100644
--- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml
+++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
@@ -9,14 +9,14 @@ spack:
compilers:
- compiler:
- spec: gcc@12.3.0
+ spec: gcc@13.3.0
paths:
- cc: /usr/bin/gcc-12
- cxx: /usr/bin/g++-12
- f77: /usr/bin/gfortran-12
- fc: /usr/bin/gfortran-12
+ cc: /usr/bin/gcc-13
+ cxx: /usr/bin/g++-13
+ f77: /usr/bin/gfortran-13
+ fc: /usr/bin/gfortran-13
flags: {}
- operating_system: ubuntu22.04
+ operating_system: ubuntu24.04
modules: []
environment: {}
extra_rpaths: []
@@ -29,16 +29,16 @@ spack:
target: [x86_64]
providers:
mpi: [openmpi@5.0.5]
- compiler: [gcc@12.3.0]
+ compiler: [gcc@13.3.0]
gcc:
buildable: false
externals:
- - spec: gcc@12.3.0
+ - spec: gcc@13.3.0
prefix: /usr
gcc-runtime:
buildable: false
externals:
- - spec: gcc-runtime@12.3.0
+ - spec: gcc-runtime@13.3.0
prefix: /usr
diffutils:
buildable: false
@@ -55,15 +55,15 @@ spack:
externals:
- spec: git-lfs@3.0.2
prefix: /usr
- grep:
+ llvm:
buildable: false
externals:
- - spec: grep@3.7
+ - spec: llvm@14.0.6
prefix: /usr
mysql:
buildable: false
externals:
- - spec: mysql@8.0.39
+ - spec: mysql@8.0.40
prefix: /usr
qt:
buildable: false
@@ -76,7 +76,6 @@ spack:
externals:
- spec: wget@1.21.2
prefix: /usr
- version: [1.21.2]
specs: []
@@ -98,7 +97,7 @@ spack:
# Sets the base images for the stages where Spack builds the
# software or where the software gets installed after being built..
images:
- os: ubuntu:22.04
+ os: ubuntu:24.04
spack:
url: https://github.com/jcsda/spack
ref: spack-stack-dev
@@ -111,12 +110,13 @@ spack:
os_packages:
build:
- bc
- - cpp-12
- - g++-12
- - gcc-12
- - gfortran-12
+ - cpp-13
+ - g++-13
+ - gcc-13
+ - gfortran-13
- git
- git-lfs
+ - llvm-14
- make
- mysql-server
- qtbase5-dev
@@ -127,12 +127,13 @@ spack:
final:
- bc
- - cpp-12
- - g++-12
- - gcc-12
- - gfortran-12
+ - cpp-13
+ - g++-13
+ - gcc-13
+ - gfortran-13
- git
- git-lfs
+ - llvm-14
- make
- mysql-server
- qtbase5-dev
@@ -157,12 +158,9 @@ spack:
# Copy the spack-stack extension repo into the spack code
COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack
# Register spack-stack extension repo
- RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
- # Set the GNU 12.x toolchain as the default compilers.
- RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \
- ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \
- ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \
- ln -svf /usr/bin/g++-12 /usr/bin/g++
+ RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100
pre_final: |
# Set environment variables for installing tzdata
ENV DEBIAN_FRONTEND=noninteractive
@@ -178,11 +176,6 @@ spack:
final: |
# Copy spack find output from builder
COPY --from=builder /root/spack_find.out /root/spack_find.out
- # Set the GNU 12.x toolchain as the default compilers.
- RUN ln -svf /usr/bin/gfortran-12 /usr/bin/gfortran && \
- ln -svf /usr/bin/cpp-12 /usr/bin/cpp && \
- ln -svf /usr/bin/gcc-12 /usr/bin/gcc && \
- ln -svf /usr/bin/g++-12 /usr/bin/g++
# Make a non-root user:nonroot / group:nonroot for running MPI
RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \
echo "ulimit -s unlimited" >> /home/nonroot/.bashrc && \
@@ -190,6 +183,8 @@ spack:
echo "export CC=gcc" >> /home/nonroot/.bashrc && \
echo "export CXX=g++" >> /home/nonroot/.bashrc && \
echo "export FC=gfortran" >> /home/nonroot/.bashrc && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \
mkdir /home/nonroot/.pmix && \
echo "rmaps_default_mapping_policy=:oversubscribe" >> /home/nonroot/.pmix/mca-params.conf && \
diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml
index c7a1c5e2c..1cbdd9620 100644
--- a/configs/containers/docker-ubuntu-intel-impi.yaml
+++ b/configs/containers/docker-ubuntu-intel-impi.yaml
@@ -9,21 +9,47 @@ spack:
compilers:
- compiler:
- spec: intel@2021.6.0
+ spec: intel@2021.10.0
paths:
- cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc
- cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc
- f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort
- fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort
+ cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc
+ cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc
+ f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
+ fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
flags: {}
- operating_system: ubuntu22.04
+ operating_system: ubuntu24.04
modules: []
environment:
prepend_path:
PATH: /usr/bin
- LD_LIBRARY_PATH: /usr/lib64:/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin
+ LD_LIBRARY_PATH: /opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin
CPATH: /usr/include
extra_rpaths: []
+ - compiler:
+ spec: gcc@=11.4.0
+ paths:
+ cc: /usr/bin/gcc
+ cxx: /usr/bin/g++
+ f77: /usr/bin/gfortran
+ fc: /usr/bin/gfortran
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: oneapi@=2023.2.3
+ paths:
+ cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icx
+ cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/icpx
+ f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx
+ fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/ifx
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
# Basic package config from configs/common/packages.yaml
# Additional package config for container
@@ -31,30 +57,51 @@ spack:
all:
require: '%intel'
target: [core2]
+ compiler: [intel@2021.10.0, gcc@11.4.0]
providers:
- mpi: [intel-oneapi-mpi@2021.6.0]
- compiler: [intel@2021.6.0]
- intel:
- buildable: false
+ mpi: [intel-oneapi-mpi@2021.10.0]
+ autoconf:
externals:
- - spec: intel@2021.6.0
- prefix: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64
- intel-oneapi-mpi:
- buildable: false
+ - spec: autoconf@2.71
+ prefix: /usr
+ automake:
externals:
- - spec: intel-oneapi-mpi@2021.6.0
- prefix: /opt/intel/oneapi
- # Comment out for now so that fftw-api uses fftw and not mkl
- #intel-oneapi-mkl:
- # buildable: false
- # externals:
- # - spec: intel-oneapi-mkl@2022.1.0
- # prefix: /opt/intel/oneapi
+ - spec: automake@1.16.5
+ prefix: /usr
+ binutils:
+ externals:
+ - spec: binutils@2.38
+ prefix: /usr
+ boost:
+ require:
+ - '%gcc'
+ coreutils:
+ externals:
+ - spec: coreutils@8.32
+ prefix: /usr
diffutils:
buildable: false
externals:
- spec: diffutils@3.8
prefix: /usr
+ ecflow:
+ require:
+ - '%gcc'
+ ectrans:
+ require:
+ - '+mkl ~fftw'
+ findutils:
+ externals:
+ - spec: findutils@4.8.0
+ prefix: /usr
+ flex:
+ externals:
+ - spec: flex@2.6.4+lex
+ prefix: /usr
+ gawk:
+ externals:
+ - spec: gawk@5.1.0
+ prefix: /usr
git:
buildable: false
externals:
@@ -65,6 +112,51 @@ spack:
externals:
- spec: git-lfs@3.0.2
prefix: /usr
+ gmake:
+ externals:
+ - spec: gmake@4.3
+ prefix: /usr
+ groff:
+ externals:
+ - spec: groff@1.22.4
+ prefix: /usr
+ gsibec:
+ require::
+ - '@1.2.1 ~mkl'
+ intel-oneapi-mkl:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-mkl@2023.2.0
+ prefix: /opt/intel/oneapi
+ intel-oneapi-mpi:
+ buildable: false
+ externals:
+ # This spec is required in order set paths allowing execution of mpiicx.
+ - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names
+ prefix: /opt/intel/oneapi
+ intel-oneapi-tbb:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-tbb@2021.10.0
+ prefix: /opt/intel/oneapi
+ libtool:
+ externals:
+ - spec: libtool@2.4.6
+ prefix: /usr
+ mpi:
+ buildable: False
+ m4:
+ externals:
+ - spec: m4@1.4.18
+ prefix: /usr
+ lcms:
+ externals:
+ - spec: lcms@2.13.1
+ prefix: /usr
+ llvm:
+ buildable: false
+ externals:
+ - spec: llvm@14.0.6
grep:
buildable: false
externals:
@@ -73,33 +165,54 @@ spack:
mysql:
buildable: false
externals:
- - spec: mysql@8.0.39
+ - spec: mysql@8.0.40
+ prefix: /usr
+ pixman:
+ require:
+ - '%gcc'
+ perl:
+ externals:
+ - spec: perl@5.34.0~cpanm+opcode+open+shared+threads
+ prefix: /usr
+ pkg-config:
+ externals:
+ - spec: pkg-config@0.29.2
+ prefix: /usr
+ py-pynacl:
+ buildable: false
+ externals:
+ - spec: py-pynacl@1.5.0
prefix: /usr
+ py-numpy:
+ require:
+ - '^intel-oneapi-mkl'
+ # Turn off crypt, because libxcrypt doesn't
+ # build with Intel.
+ python:
+ variants: ~crypt
qt:
buildable: false
externals:
- spec: qt@5.15.3
prefix: /usr
version: [5.15.3]
+ sed:
+ externals:
+ - spec: sed@4.8
+ prefix: /usr
+ tar:
+ externals:
+ - spec: tar@1.34
+ prefix: /usr
wget:
buildable: false
externals:
- spec: wget@1.21.2
prefix: /usr
- version: [1.21.2]
gettext:
externals:
- spec: gettext@0.21
prefix: /usr
- py-pynacl:
- buildable: false
- externals:
- - spec: py-pynacl@1.5.0
- prefix: /usr
- # Turn off crypt, because libxcrypt doesn't
- # build with Intel.
- python:
- variants: ~crypt
specs: []
@@ -121,7 +234,7 @@ spack:
# Sets the base images for the stages where Spack builds the
# software or where the software gets installed after being built..
images:
- os: ubuntu:22.04
+ os: ubuntu:24.04
spack:
url: https://github.com/jcsda/spack
ref: spack-stack-dev
@@ -133,41 +246,52 @@ spack:
## Additional system packages that are needed at runtime
os_packages:
build:
+ - autoconf
+ - automake
- autopoint
- bc
- - cpp
- - g++
- - gcc
+ - cpp-11
+ - cmake
+ - flex
+ - g++-11
+ - gcc-11
- gettext
- - gfortran
+ - gfortran-11
- git
- git-lfs
- gpg
- - make
+ - libgomp1
+ - llvm-14
- mysql-server
+ - pkg-config
- qtbase5-dev
- qt5-qmake
- libqt5svg5-dev
- qt5dxcb-plugin
+ - libcurl4-openssl-dev
- wget
final:
- autopoint
- bc
- - cpp
- - g++
- - gcc
+ - cpp-11
+ - g++-11
+ - gcc-11
- gettext
- - gfortran
+ - gfortran-11
- git
- git-lfs
- gpg
+ - libgomp1
+ - llvm-14
- make
- mysql-server
+ - pkg-config
- qtbase5-dev
- qt5-qmake
- libqt5svg5-dev
- qt5dxcb-plugin
+ - libcurl4-openssl-dev
- wget
# Implicitly included in build step
- build-essential
@@ -197,7 +321,13 @@ spack:
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
apt update && \
- apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \
+ apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \
+ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
rm -rf /var/lib/apt/lists/*
pre_final: |
# Set environment variables for installing tzdata
@@ -217,7 +347,13 @@ spack:
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
apt update && \
- apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2022.1.0 intel-oneapi-compiler-fortran-2022.1.0 intel-oneapi-mkl-devel-2022.1.0 intel-oneapi-mpi-devel-2021.6.0 -y && \
+ apt install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.3 intel-oneapi-compiler-fortran-2023.2.3 intel-oneapi-mkl-devel-2023.2.0 intel-oneapi-mpi-devel-2021.10.0 -y && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 && \
+ rm /usr/bin/cpp && ln -s /usr/bin/cpp-11 /usr/bin/cpp && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
rm -rf /var/lib/apt/lists/* && \
/opt/views/view/bin/pip3 install pynacl
# Copy spack find output from builder
@@ -229,8 +365,7 @@ spack:
echo "export CC=icc" >> /home/nonroot/.bashrc && \
echo "export CXX=icpc" >> /home/nonroot/.bashrc && \
echo "export FC=ifort" >> /home/nonroot/.bashrc && \
- echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /home/nonroot/.bashrc && \
- echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /home/nonroot/.bashrc && \
+ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \
printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \
chown -R nonroot:nonroot /home/nonroot/.gitconfig
# Replicate settings for root user
@@ -240,8 +375,6 @@ spack:
echo "export CXX=icpc" >> /root/.bashrc && \
echo "export FC=ifort" >> /root/.bashrc && \
echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/intel-*/jedi-cmake-*)" >> /root/.bashrc && \
- echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \
- echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \
printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig
# Labels for the image
diff --git a/configs/containers/docker-ubuntu-oneapi-impi.yaml b/configs/containers/docker-ubuntu-oneapi-impi.yaml
new file mode 100644
index 000000000..5aec25af1
--- /dev/null
+++ b/configs/containers/docker-ubuntu-oneapi-impi.yaml
@@ -0,0 +1,315 @@
+spack:
+ concretizer:
+ unify: true
+
+ config:
+ checksum: false
+ build_jobs: 2
+ connect_timeout: 60
+
+ compilers:
+ - compiler:
+ spec: gcc@=13.4.0
+ paths:
+ cc: /usr/bin/gcc
+ cxx: /usr/bin/g++
+ f77: /usr/bin/gfortran
+ fc: /usr/bin/gfortran
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+ - compiler:
+ spec: oneapi@=2024.2.1
+ paths:
+ cc: /opt/intel/oneapi/compiler/2024.2/bin/icx
+ cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx
+ #f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx
+ #fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx
+ f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort
+ fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment:
+ # change to append_path once relevant PR is merged
+ append_path:
+ CPATH: /opt/intel/oneapi/compiler/2024.2/opt/compiler/include/intel64
+ extra_rpaths: []
+
+ # Basic package config from configs/common/packages.yaml
+ # Additional package config for container
+ packages:
+ all:
+ require: '%oneapi'
+ target: [core2]
+ compiler: [oneapi@2024.2.1, gcc@13.4.0]
+ providers:
+ mpi: [intel-oneapi-mpi@2021.13]
+ autoconf:
+ externals:
+ - spec: autoconf@2.71
+ prefix: /usr
+ automake:
+ externals:
+ - spec: automake@1.16.5
+ prefix: /usr
+ binutils:
+ externals:
+ - spec: binutils@2.38
+ prefix: /usr
+ bison:
+ require:
+ - '%gcc'
+ coreutils:
+ externals:
+ - spec: coreutils@8.32
+ prefix: /usr
+ diffutils:
+ buildable: false
+ externals:
+ - spec: diffutils@3.8
+ prefix: /usr
+ ectrans:
+ require:
+ - '+mkl ~fftw'
+ git:
+ buildable: false
+ externals:
+ - spec: git@2.34.1~tcltk
+ prefix: /usr
+ git-lfs:
+ buildable: false
+ externals:
+ - spec: git-lfs@3.0.2
+ prefix: /usr
+ gmake:
+ externals:
+ - spec: gmake@4.3
+ prefix: /usr
+ gsibec:
+ require:
+ - '+mkl'
+ # Building icu4c with oneapi intel causes segfault.
+ icu4c:
+ require:
+ - '%gcc'
+ intel-oneapi-mkl:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-mkl@2024.2
+ prefix: /opt/intel/oneapi
+ intel-oneapi-mpi:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1
+ prefix: /opt/intel/oneapi
+ intel-oneapi-tbb:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-tbb@2021.13
+ prefix: /opt/intel/oneapi
+ mpi:
+ buildable: False
+ libmd:
+ require:
+ - '%gcc'
+ libbsd:
+ require:
+ - '%gcc'
+ llvm:
+ buildable: false
+ externals:
+ - spec: llvm@14.0.6
+ mysql:
+ buildable: false
+ externals:
+ - spec: mysql@8.0.40
+ prefix: /usr
+ pkg-config:
+ externals:
+ - spec: pkg-config@0.29.2
+ prefix: /usr
+ py-pynacl:
+ buildable: false
+ externals:
+ - spec: py-pynacl@1.5.0
+ prefix: /usr
+ py-numpy:
+ require:
+ - '^intel-oneapi-mkl'
+ tar:
+ externals:
+ - spec: tar@1.34
+ prefix: /usr
+ wget:
+ buildable: false
+ externals:
+ - spec: wget@1.21.2
+ prefix: /usr
+ gettext:
+ externals:
+ - spec: gettext@0.21
+ prefix: /usr
+
+ specs: []
+
+ container:
+
+ # Select the format of the recipe e.g. docker,
+ # singularity or anything else that is currently supported
+ format: docker
+
+ # Sets the base images for the stages where Spack builds and installs.
+ images:
+ os: ubuntu:24.04
+ spack:
+ url: https://github.com/jcsda/spack
+ ref: spack-stack-dev
+ resolve_sha: true
+
+ # Whether or not to strip binaries
+ strip: false
+
+ ## Additional system packages that are needed at runtime
+ os_packages:
+ build:
+ - autoconf
+ - automake
+ - autopoint
+ - bc
+ - cpp-13
+ - cmake
+ - flex
+ - g++-13
+ - gcc-13
+ - gettext
+ - gfortran-13
+ - git
+ - git-lfs
+ - gpg
+ - libgomp1
+ - llvm-14
+ - mysql-server
+ - pkg-config
+ - qtbase5-dev
+ - qt5-qmake
+ - libqt5svg5-dev
+ - qt5dxcb-plugin
+ - libcurl4-openssl-dev
+ - wget
+
+ final:
+ - autopoint
+ - bc
+ - cpp-13
+ - g++-13
+ - gcc-13
+ - gettext
+ - gfortran-13
+ - git
+ - git-lfs
+ - gpg
+ - libgomp1
+ - llvm-14
+ - make
+ - mysql-server
+ - pkg-config
+ - qtbase5-dev
+ - qt5-qmake
+ - libqt5svg5-dev
+ - qt5dxcb-plugin
+ - libcurl4-openssl-dev
+ - wget
+ # Implicitly included in build step
+ - build-essential
+ - ca-certificates
+ - curl
+ - file
+ - gnupg2
+ - iproute2
+ - locales
+ - python3
+ - python3-pip
+ - python3-setuptools
+ - unzip
+ - vim
+ - nano
+ - less
+
+ # Extra instructions
+ extra_instructions:
+ pre_build: |
+ # Copy the spack-stack extension repo into the spack code
+ COPY spack-ext-SPACK_STACK_HASH/repos/spack-stack $SPACK_ROOT/var/spack/repos/spack-stack
+ # Register spack-stack extension repo
+ RUN spack repo add $SPACK_ROOT/var/spack/repos/spack-stack --scope defaults
+ # Install Intel compilers and MPI library
+ RUN apt update && apt install apt-utils && \
+ wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
+ apt update && \
+ apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y && \
+ # Intel package versions reported by apt cache policy
+ # compiler - 2024.2.1
+ # fortran - 2024.2.1
+ # mpi - 2021.13.1
+ # tbb - 2021.13.1
+ # mkl - 2024.2.2
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \
+ rm /usr/bin/cpp && ln -s /usr/bin/cpp-13 /usr/bin/cpp && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
+ rm -rf /var/lib/apt/lists/*
+ pre_final: |
+ # Set environment variables for installing tzdata
+ ENV DEBIAN_FRONTEND=noninteractive
+ ENV TZ=Etc/UTC
+ ENV CC=icx
+ ENV CXX=icpx
+ ENV FC=ifort
+ build: |
+ # Put output of spack find into a file
+ RUN cd /opt/spack-environment && \
+ spack env activate -d . && \
+ spack find 2>&1 | tee /root/spack_find.out
+ final: |
+ # Install Intel compilers and MPI library and pynacl
+ RUN apt update && apt install apt-utils && \
+ wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+ echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
+ apt update && \
+ apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
+ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \
+ rm /usr/bin/cpp && ln -s /usr/bin/cpp-13 /usr/bin/cpp && \
+ # Needed by py-llvmlite
+ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \
+ rm -rf /var/lib/apt/lists/* && \
+ /opt/views/view/bin/pip3 install pynacl
+ # Copy spack find output from builder
+ COPY --from=builder /root/spack_find.out /root/spack_find.out
+ # Make a non-root user:nonroot / group:nonroot for running MPI
+ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 && \
+ echo "source /opt/intel/oneapi/setvars.sh" >> /etc/bash.bashrc && \
+ echo "ulimit -s unlimited" >> /etc/bash.bashrc && \
+ echo "ulimit -v unlimited" >> /etc/bash.bashrc && \
+ echo "export CC=icx" >> /etc/bash.bashrc && \
+ echo "export CXX=icpx" >> /etc/bash.bashrc && \
+ echo "export FC=ifort" >> /etc/bash.bashrc && \
+ echo "export jedi_cmake_ROOT=$(ls -d /opt/software/linux-ubuntu*/oneapi-*/jedi-cmake-*)" >> /etc/bash.bashrc && \
+ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig && \
+ # Replicate gitconfig for nonroot user.
+ printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig && \
+ chown -R nonroot:nonroot /home/nonroot/.gitconfig
+
+ # Labels for the image
+ labels:
+ app: ""
+ mpi: "intel-oneapi-mpi"
diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml
index 25f17dc6f..1f4061e0d 100644
--- a/configs/containers/specs/jedi-ci.yaml
+++ b/configs/containers/specs/jedi-ci.yaml
@@ -15,7 +15,7 @@
eckit@1.28.3,
ecmwf-atlas@0.40.0 +fckit +trans +tesselation +fftw,
fiat@1.4.1,
- ectrans@1.2.0 +fftw,
+ ectrans@1.2.0,
eigen@3.4.0,
fckit@0.13.2,
fms@2024.02,
@@ -38,7 +38,6 @@
py-f90nml,
py-gitpython,
py-h5py,
- py-numpy,
py-pandas,
py-pip,
py-pyyaml,
@@ -54,7 +53,7 @@
zstd,
odc@1.5.2,
shumlib@macos_clang_linux_intel_port,
- awscli-v2@2.13.22,
+ awscli-v2,
py-globus-cli@3.16.0,
py-fortranformat@2.0.0,
# Added for new CI system 2024/04/30
diff --git a/configs/sites/tier2/aws-rocky8/README.md b/configs/sites/tier2/aws-rocky8/README.md
new file mode 100644
index 000000000..9705277ea
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/README.md
@@ -0,0 +1,452 @@
+## spack-stack AMI (Rocky8)
+
+This document is to go over the the running and usage of this specific AMI for Rocky8. This image has two environments: gnu (gcc-11.2), and intel (intel@2021.10.0). You can use either one of these environments for development purposes.
+
+### Using the AMI
+
+Here is an example `aws cli` command-line to run this AMI with your designated key-pair.
+
+```bash
+aws ec2 run-instances \
+ --image-id "ami-02391db2758465a87" --instance-type "m6i.4xlarge" \
+ --key-name YOUR-KEYPAIR \
+ --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \
+ --network-interfaces '{"SubnetId":"subnet-00575f6b8ccc2005d","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0436b207bc220df08"]}' \
+ --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"rocky8-spack-stack-gcc-intel"},{"Key":"User","Value":$(whoami)}]}' \
+ --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \
+ --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \
+ --count "1"
+```
+
+## Using this Site Config
+
+JCSDA publishes a fully configured and built installation of spack stack derived
+from this config as an Amazon Machine Images (AMI). The easiest way to use this
+configuration of spack stack is to launch a VM using that AMI. The instructions
+below are included for maintainance of the history of this site config and
+their possible relevance to debugging issues should they arise.
+
+### Base Instance
+
+This AMI was built on an instance with these properties:
+
+* AMI Name: Rocky-8-EC2-Base-8.9-20231119.0.x86_64-d6577ceb-8ea8-4e0e-84c6-f098fc302e82
+* AMI ID: ami-02391db2758465a87
+* Instance m6i.4xlarge (uses Intel Xeon processor)
+* 300GB of gp3 storage as /
+
+## Pre-requisites (For All Compilers)
+
+
+Tmux Usage
+
+### Tmux
+
+Rocky 8 doesn't have screen so here's a guide to tmux.
+
+```bash
+# Start a session
+tmux new -s my-session-name
+
+# Detach from a session
+ctr-b d
+
+# list sessions
+tmux ls
+
+# Attach to existing session
+tmux attach -t my-session-name
+```
+
+More information on how to use `tmux`:
+
+
+
+### Installing Packages
+
+```bash
+# Update system software and start a tmux session.
+sudo dnf -y update
+sudo dnf -y install tmux
+tmux new -s setup
+sudo su -
+
+# Compilers
+dnf install gcc-toolset-11-gcc-c++ \
+ gcc-toolset-11-gcc-gfortran \
+ gcc-toolset-11-gdb
+
+#Install other requirements.
+dnf install binutils-devel \
+ m4 \
+ wget \
+ git \
+ git-lfs \
+ bash-completion \
+ bzip2 bzip2-devel \
+ libcurl libcurl-devel \
+ unzip \
+ patch \
+ automake \
+ xorg-x11-xauth \
+ xterm \
+ perl-IPC-Cmd \
+ pearl-core \
+ gettext-devel \
+ texlive \
+ tcl-devel \
+ nano \
+ vim \
+ bison \
+ qt5-qtbase \
+ qt5-qtbase-devel \
+ qt5-qtsvg \
+ qt5-qtsvg-devel \
+ autoconf2.7x \
+ libxml2 libxml2-devel
+
+# Python develop.
+dnf install python3-devel
+
+# Enable gcc toolset. This is needed for later builds. Once lmod is installed
+# this will be configured as a pass-through module.
+scl enable gcc-toolset-11 bash
+
+# Configure git credential caching and git lfs for the rocky user and root.
+git config --global credential.helper 'cache --timeout=3600'
+git lfs install
+
+# Configure x11 forwarding.
+echo "X11Forwarding yes" >> /etc/ssh/sshd_config
+service sshd restart
+```
+
+### Install MySQL Community Server
+
+```bash
+dnf config-manager --set-enabled crb
+
+dnf install mysql-server mysql-devel
+sudo systemctl start mysqld.service
+sudo systemctl enable mysqld
+
+# Use the mysql server.
+mysql -u root
+exit
+```
+
+### Install Lmod
+
+```bash
+# Enable gcc using system module
+scl enable gcc-toolset-11 bash
+
+# Install lua/lmod manually because apt only has older versions
+# that are not compatible with the modern lua modules spack produces
+# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz
+sudo su -
+mkdir -p /opt/lua/5.1.4.9/src && cd $_
+wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2
+tar -xvf lua-5.1.4.9.tar.bz2
+cd lua-5.1.4.9
+./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config
+make VERBOSE=1 2>&1 | tee log.make
+make install 2>&1 | tee log.install
+
+cat << 'EOF' >> /etc/profile.d/02-lua.sh
+# Set environment variables for lua
+export PATH="/opt/lua/5.1.4.9/bin:$PATH"
+export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH"
+export CPATH="/opt/lua/5.1.4.9/include:$CPATH"
+export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH"
+EOF
+
+source /etc/profile.d/02-lua.sh
+mkdir -p /opt/lmod/8.7/src
+cd /opt/lmod/8.7/src
+wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2
+tar -xvf Lmod-8.7.tar.bz2
+cd Lmod-8.7
+# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically
+./configure --prefix=/opt/ \
+ --with-lmodConfigDir=/opt/lmod/8.7/config \
+ 2>&1 | tee log.config
+make install 2>&1 | tee log.install
+ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh
+ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh
+ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish
+
+# Add custom module for system gcc
+cat << 'EOF' >> /opt/rh/gcc-toolset-11/gcc-toolset.lua
+--%Module
+family("compiler")
+help([[Wrapper module for gcc-toolset-11 using Software Collections]])
+whatis("Description: GCC Toolset 11")
+
+-- Execute 'scl enable' to load gcc-toolset-11 environment
+execute {cmd="scl enable gcc-toolset-11 bash", modeA={"load"}}
+EOF
+
+# Add a number of default module locations to the lmod startup script.
+cat << 'EOF' >> /etc/profile.d/z01_lmod.sh
+module use /opt/rh/gcc-toolset-11
+EOF
+
+# Log out completely, ssh back into the instance and check if lua modules work
+exit
+exit
+```
+
+### Clone `spack-stack`
+
+```bash
+# if in sudo su -
+cd /opt
+git clone -b develop --recursive https://github.com/jcsda/spack-stack.git
+```
+
+## Install Steps by Compiler
+
+
+GCC Installation
+
+```bash
+sudo su -
+
+cd /opt/spack-stack
+source setup.sh
+
+scl enable gcc-toolset-11 bash
+
+# Swap default module type for default linux.
+sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml
+spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc
+cd envs/unified-env-gcc
+spack env activate -p .
+export SPACK_SYSTEM_CONFIG_PATH="$PWD/site"
+spack external find --scope system \
+ --exclude cmake \
+ --exclude curl --exclude openssl \
+ --exclude openssh --exclude python
+spack external find --scope system wget
+spack external find --scope system mysql
+spack external find --scope system grep
+spack compiler find --scope system
+unset SPACK_SYSTEM_CONFIG_PATH
+
+# ACTION: Edit the site/packages.yaml and add these packages
+# If not present.
+cat << 'EOF' >> $PWD/site/packages.yaml
+ gcc:
+ buildable: false
+ externals:
+ - spec: gcc@11.2.1
+ prefix: /usr
+ gcc-runtime:
+ buildable: false
+ externals:
+ - spec: gcc-runtime@11.2.1
+ prefix: /usr
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+ version: [5.15.3]
+EOF
+
+# Continue configuration.
+spack config add "packages:all:compiler:[gcc@11.2.1]"
+spack config add "packages:all:providers:mpi:[openmpi@5.0.5]"
+spack config add "packages:fontconfig:variants:+pic"
+spack config add "packages:pixman:variants:+pic"
+spack config add "packages:cairo:variants:+pic"
+spack config add "packages:ewok-env:variants:+mysql"
+
+# Concretize and install
+spack concretize 2>&1 | tee log.concretize
+${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d -c log.concretize
+spack install --fail-fast -j 16 2>&1 | tee log.install
+
+# Install modules
+spack module lmod refresh
+spack stack setup-meta-modules
+
+cat << 'EOF' >> /etc/profile.d/z01_lmod.sh
+module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core
+EOF
+```
+
+
+
+
+Intel Installation
+
+#### Install Intel Compiler
+
+```bash
+rm -rf /opt/intel
+rm -rf /var/intel
+
+mkdir -p /opt/intel/src
+pushd /opt/intel/src
+
+# Download Intel install assets.
+wget -O cpp-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d85fbeee-44ec-480a-ba2f-13831bac75f7/l_dpcpp-cpp-compiler_p_2023.2.3.12_offline.sh
+wget -O fortran-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0ceccee5-353c-4fd2-a0cc-0aecb7492f87/l_fortran-compiler_p_2023.2.3.13_offline.sh
+wget -O tbb.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95cd995-586b-4688-b7e8-2d4485a1b5bf/l_tbb_oneapi_p_2021.10.0.49543_offline.sh
+wget -O mpi.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4f5871da-0533-4f62-b563-905edfb2e9b7/l_mpi_oneapi_p_2021.10.0.49374_offline.sh
+wget -O math.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/adb8a02c-4ee7-4882-97d6-a524150da358/l_onemkl_p_2023.2.0.49497_offline.sh
+
+# Install the Intel assets.
+sh cpp-compiler.sh -a --silent --eula accept 2>&1 | tee install.cpp-compiler.log
+sh fortran-compiler.sh -a --silent --eula accept | tee install.fortran-compiler.log
+sh tbb.sh -a --silent --eula accept | tee install.tbb.log
+sh mpi.sh -a --silent --eula accept | tee install.mpi.log
+sh math.sh -a --silent --eula accept | tee install.math.log
+
+popd
+```
+
+#### Create Intel Environment
+```bash
+tmux -s intel
+sudo su -
+module load gcc-toolset
+
+source /opt/intel/oneapi/compiler/2023.2.3/env/vars.sh
+source /opt/intel/oneapi/mpi/2021.10.0/env/vars.sh
+source /opt/intel/oneapi/setvars.sh
+
+cd /opt/spack-stack
+source ./setup.sh
+
+spack stack create env --site linux.default --template unified-dev --name unified-env-intel --compiler intel
+cd envs/unified-env-intel
+spack env activate -p .
+
+export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site"
+
+spack external find --scope system --exclude python --exclude curl
+spack external find --scope system perl
+spack external find --scope system wget
+spack external find --scope system texlive
+spack external find --scope system mysql
+spack external find --scope system grep
+
+# No external find for pre-installed intel-oneapi-mpi (from pcluster AMI),
+# and no way to add object entry to list using "spack config add".
+cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+ intel-oneapi-mpi:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
+ prefix: /opt/intel/oneapi
+EOF
+
+# Can't find qt5 because qtpluginfo is broken,
+# and no way to add object entry to list using "spack config add".
+cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+EOF
+
+spack compiler find --scope system
+
+export -n SPACK_SYSTEM_CONFIG_PATH
+
+# spack config add "packages:mpi:buildable:False"
+spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]"
+spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.2.1]"
+
+# Edit envs/unified-intel/spack.yaml.
+# 1) Find this line:
+# compilers: ['%aocc', '%apple-clang', '%gcc', '%intel']
+# 2) Delete all compilers except for your target compiler. In the case of intel
+# the line should look like this:
+# compilers: ['%intel']
+
+# edit envs/unified-env/site/compilers.yaml and replace the following line in the **Intel** compiler section:
+# environment: {}
+# -->
+# environment:
+# prepend_path:
+# PATH: /opt/rh/gcc-toolset-11/root/usr/bin
+# LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin:/usr/lib:/usr/lib64'
+# CPATH: /opt/rh/gcc-toolset-11/root/usr/include
+
+spack concretize 2>&1 | tee log.concretize
+${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize
+spack install -j 12 --verbose 2>&1 | tee log.install
+spack module lmod refresh
+spack stack setup-meta-modules
+
+cat << 'EOF' >> /etc/profile.d/z01_lmod.sh
+module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core
+EOF
+```
+
+
+
+## Test Installation
+
+
+GCC
+
+```bash
+# Example given for building jedi-bundle
+module use /opt/spack-stack/envs/unified-dev-gcc/install/modulefiles/Core
+module load stack-gcc/11.2.1
+module load stack-openmpi/5.0.5
+module load base-env
+module load jedi-mpas-env
+module load jedi-fv3-env
+module load ewok-env
+module load sp
+
+mkdir /opt/jedi
+cd /opt/jedi
+git clone https://github.com/JCSDA-internal/jedi-bundle.git
+cd jedi-bundle
+mkdir build && cd build
+ecbuild ../
+make update
+make -j10
+ctest
+```
+
+
+
+
+Intel
+
+```bash
+# Example given for building jedi-bundle
+module use /opt/spack-stack/envs/unified-dev-intel/install/modulefiles/Core
+module load stack-intel/2021.10.0
+module load stack-intel-oneapi-mpi/2021.10.0
+module load base-env
+module load jedi-mpas-env
+module load jedi-fv3-env
+module load ewok-env
+module load sp
+
+mkdir /opt/jedi
+cd /opt/jedi
+git clone https://github.com/JCSDA-internal/jedi-bundle.git
+cd jedi-bundle
+mkdir build && cd build
+ecbuild ../
+make update
+make -j10
+ctest
+```
+
+**Note**: If the `make -j10` shows any errors, re-run it and it should be successful.
+
+
+
+The installation and configuration is now complete for the instance.
diff --git a/configs/sites/tier2/aws-rocky8/compilers.yaml b/configs/sites/tier2/aws-rocky8/compilers.yaml
new file mode 100644
index 000000000..f52755104
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/compilers.yaml
@@ -0,0 +1,32 @@
+compilers:
+- compiler:
+ spec: intel@=2021.10.0
+ paths:
+ cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc
+ cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc
+ f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
+ fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
+ flags: {}
+ operating_system: rocky8
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ PATH: /opt/rh/gcc-toolset-11/root/user/bin
+ LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin:/usr/lib64:/usr/lib'
+ CPATH: /opt/rh/gcc-toolset-11/root/user/include
+ extra_rpaths: []
+- compiler:
+ spec: gcc@11.2.1
+ paths:
+ cc: /opt/rh/gcc-toolset-11/root/user/bin/gcc
+ cxx: /opt/rh/gcc-toolset-11/root/user/bin/g++
+ f77: /opt/rh/gcc-toolset-11/root/user/bin/gfortran
+ fc: /opt/rh/gcc-toolset-11/root/user/bin/gfortran
+ flags: {}
+ operating_system: rocky8
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+
diff --git a/configs/sites/tier2/aws-rocky8/config.yaml b/configs/sites/tier2/aws-rocky8/config.yaml
new file mode 100644
index 000000000..cf54e6360
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/config.yaml
@@ -0,0 +1,9 @@
+config:
+ build_jobs: 8
+
+ # Overrides for spack build and staging areas to speed up builds
+ # by using a local directory instead of the EFS shared filesystem
+ build_stage: /tmp/spack-stack/cache/build_stage
+ test_stage: /tmp/spack-stack/cache/test_stage
+ source_cache: /tmp/spack-stack/cache/source_cache
+ misc_cache: /tmp/spack-stack/cache/misc_cache
diff --git a/configs/sites/tier2/aws-rocky8/modules.yaml b/configs/sites/tier2/aws-rocky8/modules.yaml
new file mode 100644
index 000000000..65801706f
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/modules.yaml
@@ -0,0 +1,7 @@
+modules:
+ default:
+ enable::
+ - lmod
+ lmod:
+ include:
+ - python
diff --git a/configs/sites/tier2/aws-rocky8/packages.yaml b/configs/sites/tier2/aws-rocky8/packages.yaml
new file mode 100644
index 000000000..6158d0119
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/packages.yaml
@@ -0,0 +1,94 @@
+packages:
+ mysql:
+ externals:
+ - spec: mysql@8.0.37
+ prefix: /usr
+ wget:
+ externals:
+ - spec: wget@1.21.2
+ prefix: /usr
+ binutils:
+ externals:
+ - spec: binutils@2.38
+ prefix: /usr
+ coreutils:
+ externals:
+ - spec: coreutils@8.32
+ prefix: /usr
+ diffutils:
+ externals:
+ - spec: diffutils@3.8
+ prefix: /usr
+ findutils:
+ externals:
+ - spec: findutils@4.8.0
+ prefix: /usr
+ gawk:
+ externals:
+ - spec: gawk@5.1.0
+ prefix: /usr
+ tar:
+ externals:
+ - spec: tar@1.34
+ prefix: /usr
+ autoconf:
+ externals:
+ - spec: autoconf@2.71
+ prefix: /usr
+ automake:
+ externals:
+ - spec: automake@1.16.5
+ prefix: /usr
+ flex:
+ externals:
+ - spec: flex@2.6.4+lex
+ prefix: /usr
+ gettext:
+ externals:
+ - spec: gettext@0.21
+ prefix: /usr
+ git:
+ externals:
+ - spec: git@2.34.1~tcltk
+ prefix: /usr
+ git-lfs:
+ externals:
+ - spec: git-lfs@3.0.2
+ prefix: /usr
+ gmake:
+ externals:
+ - spec: gmake@4.3
+ prefix: /usr
+ groff:
+ externals:
+ - spec: groff@1.22.4
+ prefix: /usr
+ libtool:
+ externals:
+ - spec: libtool@2.4.6
+ prefix: /usr
+ m4:
+ externals:
+ - spec: m4@1.4.18
+ prefix: /usr
+ perl:
+ externals:
+ - spec: perl@5.34.0~cpanm+opcode+open+shared+threads
+ prefix: /usr
+ pkg-config:
+ externals:
+ - spec: pkg-config@0.29.2
+ prefix: /usr
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+ sed:
+ externals:
+ - spec: sed@4.8
+ prefix: /usr
+ lcms:
+ externals:
+ - spec: lcms@2.13.1
+ prefix: /usr
diff --git a/configs/sites/tier2/aws-rocky8/packages_gcc.yaml b/configs/sites/tier2/aws-rocky8/packages_gcc.yaml
new file mode 100644
index 000000000..8ee1dc22b
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/packages_gcc.yaml
@@ -0,0 +1,5 @@
+packages:
+ all:
+ compiler:: [gcc@11.2.1]
+ providers:
+ mpi:: [openmpi@4.1.6]
diff --git a/configs/sites/tier2/aws-rocky8/packages_intel.yaml b/configs/sites/tier2/aws-rocky8/packages_intel.yaml
new file mode 100644
index 000000000..97c09dcb0
--- /dev/null
+++ b/configs/sites/tier2/aws-rocky8/packages_intel.yaml
@@ -0,0 +1,29 @@
+packages:
+ all:
+ compiler:: [intel@2021.10.0,gcc@11.2.1]
+ providers:
+ mpi:: [intel-oneapi-mpi@2021.10.0]
+ # Remove the next three lines to switch to intel-oneapi-mkl
+ blas:: [openblas]
+ fftw-api:: [fftw]
+ lapack:: [openblas]
+ intel-oneapi-mpi:
+ buildable: false
+ externals:
+ - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
+ prefix: /opt/intel/oneapi
+ intel-oneapi-mkl:
+ # Remove buildable: False and configure version+uncomment externals section below to use intel-oneapi-mkl
+ buildable: False
+ #externals:
+ #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.10.0
+ # prefix: /opt/intel/oneapi
+ ectrans:
+ require::
+ - '@1.2.0 ~mkl +fftw'
+ gsibec:
+ require::
+ - '@1.2.1 ~mkl'
+ py-numpy:
+ require::
+ - '@:1.23 ^openblas'
diff --git a/configs/sites/tier2/aws-ubuntu2404/README.md b/configs/sites/tier2/aws-ubuntu2404/README.md
new file mode 100644
index 000000000..2fddc5d2f
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/README.md
@@ -0,0 +1,384 @@
+## spack-stack AMI (Ubuntu 24.04)
+
+This document is to go over the the running and usage of this specific AMI for Ubuntu 24.04 LTS. This image has two environments: gnu (gcc-12.3), and intel (intel@2021.10.0). You can use either one of these environments for development purposes.
+
+### Using the Snapshot
+
+Here is an example `aws cli` command-line to run a pre-built snapshot with your designated key-pair.
+
+```bash
+aws ec2 run-instances \
+ --image-id "ami-0ea3c35c5c3284d82" --instance-type "m6i.4xlarge" \
+ --key-name YOUR-KEYPAIR \
+ --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \
+ --network-interfaces '{"SubnetId":"subnet-072fb62ff85b32a7a","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0091fa8e748fbe355"]}' \
+ --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"ubuntu2404-spack-stack-gcc-intel"},{"Key":"User","Value":$(whoami)}]}' \
+ --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \
+ --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \
+ --count "1"
+```
+
+## Using this Site Config
+
+JCSDA publishes a fully configured and built installation of spack stack derived
+from this config as an AWS Snapshot. The easiest way to use this
+configuration of spack stack is to launch a VM using that AMI. The instructions
+below are included for maintainance of the history of this site config and
+their possible relevance to debugging issues should they arise.
+
+### Base Instance
+
+This AMI was built on an instance with these properties:
+
+* AMI Name: ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-20240927
+* AMI ID: ami-0ea3c35c5c3284d82
+* Instance m6i.4xlarge (uses Intel Xeon processor)
+* 300GB of gp3 storage as /
+
+## Pre-requisites (For All Compilers)
+
+### Installing Packages
+
+```bash
+# Update system software and start a tmux session.
+sudo su -
+apt update
+apt upgrade
+
+# Build tools
+apt install -y build-essential g++-11 gcc-11 gfortran-11 make cmake automake autoconf
+
+#Install other requirements.
+apt install -y cpp-11 libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev
+
+# Editors
+apt install -y vim nano
+
+# Python develop.
+apt install -y python3 python3-pip python3-setuptools
+
+# Configure git credential caching and git lfs for the rocky user and root.
+git config --global credential.helper 'cache --timeout=3600'
+git lfs install
+
+# Change the gcc, g++, and gfortran version to 11 and give it the highest priority
+update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100
+update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100
+
+exit # Exit root access
+```
+
+**Important**: Running the `update-alternatives` changes defaults, and if you need to use a different version of gcc/g++/gfortran you can run: `update-alternatives --config [gcc|g++|gfortran]` and select the version you want that is installed.
+
+### Install Lmod
+
+```bash
+# Install lua/lmod manually because apt only has older versions
+# that are not compatible with the modern lua modules spack produces
+# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz
+sudo su -
+mkdir -p /opt/lua/5.1.4.9/src && cd $_
+wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2
+tar -xvf lua-5.1.4.9.tar.bz2
+cd lua-5.1.4.9
+./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config
+make VERBOSE=1 2>&1 | tee log.make
+make install 2>&1 | tee log.install
+
+cat << 'EOF' >> /etc/profile.d/02-lua.sh
+# Set environment variables for lua
+export PATH="/opt/lua/5.1.4.9/bin:$PATH"
+export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH"
+export CPATH="/opt/lua/5.1.4.9/include:$CPATH"
+export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH"
+EOF
+
+source /etc/profile.d/02-lua.sh
+mkdir -p /opt/lmod/8.7/src
+cd /opt/lmod/8.7/src
+wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2
+tar -xvf Lmod-8.7.tar.bz2
+cd Lmod-8.7
+# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically
+./configure --prefix=/opt/ \
+ --with-lmodConfigDir=/opt/lmod/8.7/config \
+ 2>&1 | tee log.config
+make install 2>&1 | tee log.install
+ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh
+ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh
+ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish
+
+# Log out completely, ssh back into the instance and check if lua modules work
+exit
+exit
+```
+
+### Setup MySQL Sercer
+
+```bash
+sudo systemctl start mysqld.service
+sudo systemctl enable mysqld
+
+# Use the mysql server.
+mysql -u root
+exit
+```
+
+### Clone `spack-stack`
+
+```bash
+cd /opt
+sudo git clone -b develop --depth 1 --recursive https://github.com/jcsda/spack-stack.git
+```
+
+## Install Spack-Stack Steps by Compiler
+
+
+GCC Installation
+
+```bash
+sudo su -
+
+cd /opt/spack-stack
+source setup.sh
+# Swap default module type for default linux.
+sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml
+spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc
+cd envs/unified-env-gcc
+spack env activate -p .
+export SPACK_SYSTEM_CONFIG_PATH="$PWD/site"
+spack external find --scope system \
+ --exclude cmake \
+ --exclude curl --exclude openssl \
+ --exclude openssh --exclude python
+spack external find --scope system wget
+spack external find --scope system mysql
+spack external find --scope system grep
+spack compiler find --scope system
+unset SPACK_SYSTEM_CONFIG_PATH
+# ACTION: Edit the site/compilers.yaml with the following.
+# 1) Delete or comment gcc-13 refs and preserve only gcc-12
+# 2) Delete or comment clang refs.
+# ACTION: Edit the site/packages.yaml and add these packages
+# If not present.
+cat << 'EOF' >> $PWD/site/packages.yaml
+ gcc:
+ buildable: false
+ externals:
+ - spec: gcc@11.4.0
+ prefix: /usr
+ gcc-runtime:
+ buildable: false
+ externals:
+ - spec: gcc-runtime@11.4.0
+ prefix: /usr
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+ version: [5.15.3]
+EOF
+
+# Continue configuration.
+spack config add "packages:all:compiler:[gcc@11.4.0]"
+spack config add "packages:all:providers:mpi:[openmpi@5.0.5]"
+spack config add "packages:fontconfig:variants:+pic"
+spack config add "packages:pixman:variants:+pic"
+spack config add "packages:cairo:variants:+pic"
+spack config add "packages:ewok-env:variants:+mysql"
+
+# Concretize and install
+spack concretize 2>&1 | tee log.concretize
+${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d -c log.concretize
+spack install --verbose --fail-fast 2>&1 | tee log.install
+
+# Install modules
+spack module lmod refresh
+spack stack setup-meta-modules
+
+# Add a number of default module locations to the lmod startup script.
+cat << 'EOF' >> /etc/profile.d/z01_lmod.sh
+module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core
+EOF
+```
+
+
+
+
+Intel Installation
+
+#### Install Intel Compiler
+
+```bash
+sudo su -
+
+mkdir -p /opt/intel/src
+pushd /opt/intel/src
+
+# Download Intel install assets.
+wget -O cpp-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d85fbeee-44ec-480a-ba2f-13831bac75f7/l_dpcpp-cpp-compiler_p_2023.2.3.12_offline.sh
+wget -O fortran-compiler.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/0ceccee5-353c-4fd2-a0cc-0aecb7492f87/l_fortran-compiler_p_2023.2.3.13_offline.sh
+wget -O tbb.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/c95cd995-586b-4688-b7e8-2d4485a1b5bf/l_tbb_oneapi_p_2021.10.0.49543_offline.sh
+wget -O mpi.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/4f5871da-0533-4f62-b563-905edfb2e9b7/l_mpi_oneapi_p_2021.10.0.49374_offline.sh
+wget -O math.sh https://registrationcenter-download.intel.com/akdlm/IRC_NAS/adb8a02c-4ee7-4882-97d6-a524150da358/l_onemkl_p_2023.2.0.49497_offline.sh
+
+# Install the Intel assets.
+sh cpp-compiler.sh -a --silent --eula accept 2>&1 | tee install.cpp-compiler.log
+sh fortran-compiler.sh -a --silent --eula accept | tee install.fortran-compiler.log
+sh tbb.sh -a --silent --eula accept | tee install.tbb.log
+sh mpi.sh -a --silent --eula accept | tee install.mpi.log
+sh math.sh -a --silent --eula accept | tee install.math.log
+
+popd
+exit
+```
+
+#### Install Intel Spack-Stack Environment
+
+```bash
+sudo su -
+
+source /opt/intel/oneapi/compiler/2023.2.3/env/vars.sh
+source /opt/intel/oneapi/mpi/2021.10.0/env/vars.sh
+source /opt/intel/oneapi/setvars.sh
+
+cd /opt/spack-stack
+source ./setup.sh
+
+spack stack create env --site linux.default --template unified-dev --name unified-env-intel --compiler intel
+cd envs/unified-env-intel
+spack env activate -p .
+
+export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site"
+
+spack external find --scope system --exclude bison --exclude openssl --exclude python --exclude gettext --exclude m4 --exclude cmake --exclude curl
+spack external find --scope system perl
+spack external find --scope system wget
+spack external find --scope system texlive
+spack external find --scope system mysql
+spack external find --scope system grep
+
+# No external find for pre-installed intel-oneapi-mpi,
+# and no way to add object entry to list using "spack config add".
+cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+ gcc:
+ buildable: false
+ externals:
+ - spec: gcc@11.4.0
+ prefix: /usr
+ gcc-runtime:
+ buildable: false
+ externals:
+ - spec: gcc-runtime@11.4.0
+ prefix: /usr
+ grep:
+ buildable: False
+ externals:
+ - spec: grep@3.11
+ prefix: /usr
+ intel-oneapi-mpi:
+ externals:
+ - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names
+ prefix: /opt/intel/oneapi
+EOF
+
+# Can't find qt5 because qtpluginfo is broken,
+# and no way to add object entry to list using "spack config add".
+cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+EOF
+
+spack compiler find --scope system
+
+unset SPACK_SYSTEM_CONFIG_PATH
+
+spack config add "packages:mpi:buildable:False"
+spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]"
+spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]"
+
+# edit envs/unified-env/site/compilers.yaml and replace the following line in the **Intel** compiler section:
+# environment: {}
+# -->
+# environment:
+# prepend_path:
+# LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin'
+
+spack concretize 2>&1 | tee log.concretize
+${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize
+spack install --fail-fast -j 12 2>&1 | tee log.install
+spack module lmod refresh
+spack stack setup-meta-modules
+
+cat << 'EOF' >> /etc/profile.d/z01_lmod.sh
+module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core
+EOF
+```
+
+
+
+## Test Installation
+
+
+GCC
+
+```bash
+# Example given for building jedi-bundle
+module use /opt/spack-stack/envs/unified-dev-gcc/install/modulefiles/Core
+module load stack-gcc/12.3.0
+module load stack-openmpi/5.0.5
+module load base-env
+module load jedi-mpas-env
+module load jedi-fv3-env
+module load ewok-env
+module load sp
+
+mkdir /opt/jedi
+cd /opt/jedi
+git clone https://github.com/JCSDA-internal/jedi-bundle.git
+cd jedi-bundle
+mkdir build && cd build
+ecbuild ../
+make update
+make -j10
+ctest
+```
+
+
+
+
+Intel
+
+```bash
+# Example given for building jedi-bundle
+module use /opt/spack-stack/envs/unified-env-intel/install/modulefiles/Core
+module load stack-intel/2021.10.0
+module load stack-intel-oneapi-mpi/2021.10.0
+module load base-env
+module load jedi-mpas-env
+module load jedi-fv3-env
+module load ewok-env
+module load sp
+
+# (Optional) Re-source the intel environment
+source /opt/intel/oneapi/setvars.sh
+
+mkdir /opt/jedi
+cd /opt/jedi
+git clone https://github.com/JCSDA-internal/jedi-bundle.git
+cd jedi-bundle
+mkdir build && cd build
+ecbuild ../
+make update
+make -j10
+ctest
+```
+
+
+
+The installation and configuration is now complete for the instance.
diff --git a/configs/sites/tier2/aws-ubuntu2404/compilers.yaml b/configs/sites/tier2/aws-ubuntu2404/compilers.yaml
new file mode 100644
index 000000000..7cfcf40d8
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/compilers.yaml
@@ -0,0 +1,30 @@
+compilers:
+- compiler:
+ spec: intel@=2021.10.0
+ paths:
+ cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc
+ cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc
+ f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
+ fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment:
+ prepend_path:
+ LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin'
+ extra_rpaths: []
+- compiler:
+ spec: gcc@11.4.0
+ paths:
+ cc: /usr/bin/gcc
+ cxx: /usr/bin/g++
+ f77: /usr/bin/gfortran
+ fc: /usr/bin/gfortran
+ flags: {}
+ operating_system: ubuntu24.04
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []
+
diff --git a/configs/sites/tier2/aws-ubuntu2404/config.yaml b/configs/sites/tier2/aws-ubuntu2404/config.yaml
new file mode 100644
index 000000000..cf54e6360
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/config.yaml
@@ -0,0 +1,9 @@
+config:
+ build_jobs: 8
+
+ # Overrides for spack build and staging areas to speed up builds
+ # by using a local directory instead of the EFS shared filesystem
+ build_stage: /tmp/spack-stack/cache/build_stage
+ test_stage: /tmp/spack-stack/cache/test_stage
+ source_cache: /tmp/spack-stack/cache/source_cache
+ misc_cache: /tmp/spack-stack/cache/misc_cache
diff --git a/configs/sites/tier2/aws-ubuntu2404/modules.yaml b/configs/sites/tier2/aws-ubuntu2404/modules.yaml
new file mode 100644
index 000000000..65801706f
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/modules.yaml
@@ -0,0 +1,7 @@
+modules:
+ default:
+ enable::
+ - lmod
+ lmod:
+ include:
+ - python
diff --git a/configs/sites/tier2/aws-ubuntu2404/packages.yaml b/configs/sites/tier2/aws-ubuntu2404/packages.yaml
new file mode 100644
index 000000000..6158d0119
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/packages.yaml
@@ -0,0 +1,94 @@
+packages:
+ mysql:
+ externals:
+ - spec: mysql@8.0.37
+ prefix: /usr
+ wget:
+ externals:
+ - spec: wget@1.21.2
+ prefix: /usr
+ binutils:
+ externals:
+ - spec: binutils@2.38
+ prefix: /usr
+ coreutils:
+ externals:
+ - spec: coreutils@8.32
+ prefix: /usr
+ diffutils:
+ externals:
+ - spec: diffutils@3.8
+ prefix: /usr
+ findutils:
+ externals:
+ - spec: findutils@4.8.0
+ prefix: /usr
+ gawk:
+ externals:
+ - spec: gawk@5.1.0
+ prefix: /usr
+ tar:
+ externals:
+ - spec: tar@1.34
+ prefix: /usr
+ autoconf:
+ externals:
+ - spec: autoconf@2.71
+ prefix: /usr
+ automake:
+ externals:
+ - spec: automake@1.16.5
+ prefix: /usr
+ flex:
+ externals:
+ - spec: flex@2.6.4+lex
+ prefix: /usr
+ gettext:
+ externals:
+ - spec: gettext@0.21
+ prefix: /usr
+ git:
+ externals:
+ - spec: git@2.34.1~tcltk
+ prefix: /usr
+ git-lfs:
+ externals:
+ - spec: git-lfs@3.0.2
+ prefix: /usr
+ gmake:
+ externals:
+ - spec: gmake@4.3
+ prefix: /usr
+ groff:
+ externals:
+ - spec: groff@1.22.4
+ prefix: /usr
+ libtool:
+ externals:
+ - spec: libtool@2.4.6
+ prefix: /usr
+ m4:
+ externals:
+ - spec: m4@1.4.18
+ prefix: /usr
+ perl:
+ externals:
+ - spec: perl@5.34.0~cpanm+opcode+open+shared+threads
+ prefix: /usr
+ pkg-config:
+ externals:
+ - spec: pkg-config@0.29.2
+ prefix: /usr
+ qt:
+ buildable: false
+ externals:
+ - spec: qt@5.15.3
+ prefix: /usr
+ sed:
+ externals:
+ - spec: sed@4.8
+ prefix: /usr
+ lcms:
+ externals:
+ - spec: lcms@2.13.1
+ prefix: /usr
diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml
new file mode 100644
index 000000000..4f9287676
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml
@@ -0,0 +1,5 @@
+packages:
+ all:
+ compiler:: [gcc@11.4.0]
+ providers:
+ mpi:: [openmpi@5.0.5]
diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml
new file mode 100644
index 000000000..66a9c4263
--- /dev/null
+++ b/configs/sites/tier2/aws-ubuntu2404/packages_intel.yaml
@@ -0,0 +1,30 @@
+packages:
+ all:
+ compiler:: [intel@2021.10.0,gcc@11.4.0]
+ providers:
+ mpi:: [intel-oneapi-mpi@2021.10.0]
+ # Remove the next three lines to switch to intel-oneapi-mkl
+ blas:: [openblas]
+ fftw-api:: [fftw]
+ lapack:: [openblas]
+ mpi:
+ buildable: False
+ intel-oneapi-mpi:
+ externals:
+ - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
+ prefix: /opt/intel/oneapi
+ intel-oneapi-mkl:
+ # Remove buildable: False and configure version+uncomment externals section below to use intel-oneapi-mkl
+ buildable: False
+ #externals:
+ #- spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0
+ # prefix: /opt/intel/oneapi
+ ectrans:
+ require::
+ - '@1.2.0 ~mkl +fftw'
+ gsibec:
+ require::
+ - '@1.2.1 ~mkl'
+ py-numpy:
+ require::
+ - '@:1.23 ^openblas'
diff --git a/configs/templates/ccpp-scm/spack.yaml b/configs/templates/ccpp-scm/spack.yaml
new file mode 100644
index 000000000..07ee4ff2d
--- /dev/null
+++ b/configs/templates/ccpp-scm/spack.yaml
@@ -0,0 +1,23 @@
+spack:
+ concretizer:
+ unify: when_possible
+
+ config:
+ install_tree:
+ root: $env/install
+ modules:
+ default:
+ roots:
+ lmod: $env/install/modulefiles
+ tcl: $env/install/modulefiles
+
+ view: false
+ include: []
+
+ specs:
+ - netcdf-fortran
+ - bacio
+ - sp
+ - w3emc
+ - py-f90nml
+
diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml
index ba3c9c1d6..104b6784b 100644
--- a/configs/templates/skylab-dev/spack.yaml
+++ b/configs/templates/skylab-dev/spack.yaml
@@ -8,7 +8,7 @@ spack:
definitions:
- compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi']
- packages:
- - ewok-env +ecflow +cylc
+ - ewok-env +ecflow ~cylc
- ai-env
- geos-gcm-env ^esmf@=8.6.1
- jedi-fv3-env
diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst
index 6d411af8a..6bb20905c 100644
--- a/doc/source/KnownIssues.rst
+++ b/doc/source/KnownIssues.rst
@@ -7,9 +7,9 @@ Known Issues
General
==============================
-1. ``gcc@13`` (``gcc``, ``g++``, ``gfortran``) and ``apple-clang@15`` (``clang``, ``clang++``) not yet supported
+1. ``python`` on Cray is non-functional: Could not find platform dependent libraries
- Our software stack doesn't build with ``gcc@13`` yet. This is also true when combining the LLVM or Apple ``clang`` compiler with ``gfortran@13``. We also don't support the latest release of ``apple-clang@15`` with Xcode 15.3 yet, and with Xcode 15.0 a workaround is described in :numref:`Section %s `.
+ This problem and the solution to unset the environment variable ``CONFIG_SITE`` before building spack-stack environments is described in https://github.com/spack/spack/issues/40110 and https://github.com/spack/spack/issues/42132.
2. Build errors for ``mapl@2.35.2`` with ``mpich@4.1.1``
@@ -19,23 +19,15 @@ General
On some systems, ``ecflow_server`` gets confused by multiple hostnames, e.g. ``localhost`` and ``MYORG-L-12345``. The ``ecflow_start.sh`` script reports the hostname it wants to use. This name (or both) must be in ``/etc/hosts`` in the correct address line, often the loopback address (``127.0.0.1``).
-4. Installation of duplicate packages ``ecbuild``, ``hdf5``
-
- One reason for this is an external ``cmake@3.20`` installation, which confuses the concretizer when building a complex environment such as the ``skylab-dev`` or ```unified-dev`` environment. For certain packages (and thus their dependencies), a newer version than ``cmake@3.20`` is required, for others ``cmake@3.20`` works, and spack then thinks that it needs to build two identical versions of the same package with different versions of ``cmake``. The solution is to remove any external ``cmake@3.20`` package (and best also earlier versions) in the site config and run the concretization step again. Another reason on Ubuntu 20 is the presence of external ``openssl`` packages, which should be removed before re-running the concretization step.
-
-5. Installation of duplicate package ``nco``
-
- We tracked this down to multiple versions of ``bison`` being used. The best solution is to remove external ``bison`` versions earlier than 3.8 from the site config (``packages.yaml``).
-
-6. Installing/using graphical applications after switching user using ``sudo su``
+4. Installing/using graphical applications after switching user using ``sudo su``
When using a role account to install spack-stack, it is sometimes necessary to run graphical applications such as the ``qt`` online installer. The following website describes in detail how this can be done: https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo/
-7. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh``
+5. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh``
This error usually indicates that the wrong module type is used in the ``spack module ... refresh`` command. For example, the system is configured for ``lmod``, but the command used is ``spack module tcl refresh``.
-8. Runtime segmentation faults for applications with ``intel@2021.10.0``: ``Relink `/opt/intel/oneapi/compiler/2024.0/lib/libirc.so' with `/lib/x86_64-linux-gnu/libc.so.6' for IFUNC symbol `memmove' - Segmentation fault (core dumped)``.
+6. Runtime segmentation faults for applications with ``intel@2021.10.0``: ``Relink `/opt/intel/oneapi/compiler/2024.0/lib/libirc.so' with `/lib/x86_64-linux-gnu/libc.so.6' for IFUNC symbol `memmove' - Segmentation fault (core dumped)``.
This problem is caused by a bad library in the Intel oneAPI installation. The solution is to fix the library using patchelf, which requires write access to the oneAPI installation: First, verify that ``ldd /opt/intel/oneapi/compiler/2024.0/lib/libirc.so`` says it is statically linked (it isn't). Then, run: ``patchelf --add-needed libc.so.6 /opt/intel/oneapi/compiler/2024.0/lib/libirc.so`` and your application should run rightaway (no need to recompile).
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index e5d900829..c11a2d782 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -532,6 +532,28 @@ Linux/macOS default configs
The Linux and macOS configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `.
+.. _Preconfigured_Sites_AWS_Ubuntu2404:
+
+----------------
+AWS Ubuntu 24.04
+----------------
+
+To build consult the `README.md` in the `sites/tier2/aws-ubuntu2404`.
+
+This image can contain GCC and Intel compilers. It is strongly suggested that if you are to use either environment, it is suggested to separate environments into their own terminal.
+
+
+.. _Preconfigured_Sites_AWS_Rocky8:
+
+----------
+AWS Rocky8
+----------
+
+To build consult the `README.md` in the `sites/tier2/aws-rocky8`.
+
+This image can contain GCC and Intel compilers. It is strongly suggested that if you are to use either environment, it is suggested to separate environments into their own terminal.
+
+
.. _Configurable_Sites_CreateEnv:
========================