Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port to Casper #496

Merged
merged 2 commits into from
Aug 2, 2024
Merged

Port to Casper #496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f9efd7
Port to casper
apcraig Jul 31, 2024
168d200
Update casper machine information
apcraig Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions configuration/scripts/icepack.batch.csh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,21 @@ cat >> ${jobfile} << EOFB
###PBS -m be
EOFB

else if (${ICE_MACHINE} =~ casper*) then
cat >> ${jobfile} << EOFB
#PBS -q ${ICE_MACHINE_QUEUE}
#PBS -l job_priority=regular
#PBS -N ${ICE_CASENAME}
#PBS -A ${acct}
#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
#PBS -l walltime=${ICE_RUNLENGTH}
#PBS -j oe
#PBS -W umask=022
#PBS -o ${ICE_CASEDIR}
###PBS -M [email protected]
###PBS -m be
EOFB

else if (${ICE_MACHINE} =~ hobart*) then
cat >> ${jobfile} << EOFB
#PBS -j oe
Expand Down
47 changes: 47 additions & 0 deletions configuration/scripts/machines/Macros.casper_gnu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#==============================================================================
# Makefile macros for NCAR derecho, gnu compiler
#==============================================================================

CPP := ftn -E
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c

FIXEDFLAGS := -ffixed-line-length-132
FREEFLAGS := -ffree-form
FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow --std f2008
# FFLAGS += -O0 -g -fcheck=all -finit-real=snan -fimplicit-none -ffpe-trap=invalid,zero,overflow
CFLAGS += -O0
endif

SCC := gcc
SFC := gfortran
CC := $(SCC)
FC := $(SFC)
LD := $(FC)

NETCDF_PATH := $(NETCDF)

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

49 changes: 49 additions & 0 deletions configuration/scripts/machines/Macros.casper_intel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#==============================================================================
# Makefile macros for NCAR derecho, intel compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -march=core-avx2

FIXEDFLAGS := -fixed -132
FREEFLAGS := -free
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
else
FFLAGS += -O2
endif

SCC := icx
SFC := ifort
CC := $(SCC)
FC := $(SFC)
LD := $(FC)

NETCDF_PATH := $(NETCDF)

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

51 changes: 51 additions & 0 deletions configuration/scripts/machines/Macros.casper_inteloneapi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#==============================================================================
# Makefile macros for NCAR derecho, inteloneapi compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -march=core-avx2

FIXEDFLAGS := -fixed -132
FREEFLAGS := -free
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
# LDFLAGS += -check uninit
else
FFLAGS += -O2
endif

SCC := icx
SFC := ifx
CC := $(SCC)
FC := $(SFC)
LD := $(FC)

NETCDF_PATH := $(NETCDF)

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

#PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

51 changes: 51 additions & 0 deletions configuration/scripts/machines/env.casper_gnu
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module --force purge
module load ncarenv/23.10
#module load craype
module load gcc/12.2.0
module load ncarcompilers
#module load cray-mpich/8.1.25
#module load hdf5/1.12.2
module load netcdf/4.9.2
#module load cray-libsci/23.02.1.1

# For perftools with mpiexec
# module load perftools-base
# module load perftools
#setenv PALS_TRANSFER FALSE

endif

limit coredumpsize unlimited
limit stacksize unlimited
setenv PALS_QUIET TRUE

# May be needed for OpenMP memory
setenv OMP_STACKSIZE 64M
# OMP runtime diagnostics
#setenv OMP_DISPLAY_ENV TRUE

setenv ICE_MACHINE_MACHNAME casper
setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
setenv ICE_MACHINE_ENVNAME gnu
setenv ICE_MACHINE_ENVINFO "gcc 12.2.0 20220819, netcdf4.9.2"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub"
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "casper"
setenv ICE_MACHINE_TPNODE 36
setenv ICE_MACHINE_BLDTHRDS 1
setenv ICE_MACHINE_QSTAT "qstat "
51 changes: 51 additions & 0 deletions configuration/scripts/machines/env.casper_intel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module --force purge
module load ncarenv/23.10
#module load craype
module load intel/2023.2.1
module load ncarcompilers
#module load cray-mpich/8.1.25
#module load hdf5/1.12.2
module load netcdf/4.9.2
#module load cray-libsci/23.02.1.1

# For perftools with mpiexec
# module load perftools-base
# module load perftools
#setenv PALS_TRANSFER FALSE

endif

limit coredumpsize unlimited
limit stacksize unlimited
setenv PALS_QUIET TRUE

# May be needed for OpenMP memory
setenv OMP_STACKSIZE 64M
# OMP runtime diagnostics
#setenv OMP_DISPLAY_ENV TRUE

setenv ICE_MACHINE_MACHNAME casper
setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
setenv ICE_MACHINE_ENVNAME intel
setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, netcdf4.9.2"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub"
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "casper"
setenv ICE_MACHINE_TPNODE 36
setenv ICE_MACHINE_BLDTHRDS 1
setenv ICE_MACHINE_QSTAT "qstat "
51 changes: 51 additions & 0 deletions configuration/scripts/machines/env.casper_inteloneapi
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module --force purge
module load ncarenv/23.10
#module load craype
module load intel-oneapi/2023.2.1
module load ncarcompilers
#module load cray-mpich/8.1.25
#module load hdf5/1.12.2
module load netcdf/4.9.2
#module load cray-libsci/23.02.1.1

# For perftools with mpiexec
# module load perftools-base
# module load perftools
#setenv PALS_TRANSFER FALSE

endif

limit coredumpsize unlimited
limit stacksize unlimited
setenv PALS_QUIET TRUE

# May be needed for OpenMP memory
setenv OMP_STACKSIZE 64M
# OMP runtime diagnostics
#setenv OMP_DISPLAY_ENV TRUE

setenv ICE_MACHINE_MACHNAME casper
setenv ICE_MACHINE_MACHINFO "NCAR cluster with Intel Cascade Lake 36 core nodes with Mellanox ConnectX-5"
setenv ICE_MACHINE_ENVNAME inteloneapi
setenv ICE_MACHINE_ENVINFO "oneAPI DPC++/C++/ifx 2023.2.0 20230721, netcdf4.9.2"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS
setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg
setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub"
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "casper"
setenv ICE_MACHINE_TPNODE 36
setenv ICE_MACHINE_BLDTHRDS 1
setenv ICE_MACHINE_QSTAT "qstat "
Loading