eb

#!/bin/bash
# remember NIXUSER_PROFILE before cleaning environment
USE_GENTOO="$EBROOTGENTOO"
USE_GENTOO_VERSION="$EBVERSIONGENTOO"
USE_NIX="$NIXUSER_PROFILE"
# clean environment incl. user variables
module --force purge
unset CPATH
unset LIBRARY_PATH
unset LD_LIBRARY_PATH
export PATH=/usr/bin
if [[ $HOSTNAME =~ archimedes.c3.ca$ ]]; then
	export PATH=$PATH:/opt/software/slurm/bin
	export EASYBUILD_JOB_BACKEND='Slurm'
	export SBATCH_MEM_PER_CPU='3500m'
	export EASYBUILD_TMP_LOGDIR="/shared_tmp/logs"
fi
EASYBUILD_ROOT=/cvmfs/soft.computecanada.ca/easybuild
if [ -z "$USE_NIX" ]; then
	module load gentoo/$USE_GENTOO_VERSION
	YEAR=$USE_GENTOO_VERSION
	export EASYBUILD_ALLOW_LOADED_MODULES=gentoo
else
	module load nixpkgs/16.09
	YEAR=2017
	unset USE_GENTOO
	unset USE_GENTOO_VERSION
	# use Gentoo Python 3.7.7 even here, only for EasyBuild itself
	export EB_PYTHON=/cvmfs/soft.computecanada.ca/gentoo/2020/usr/bin/python
fi
EASYBUILD_CONFIG_ROOT=$EASYBUILD_ROOT/easybuild-computecanada-config/$YEAR
export LOCAL_EASYBUILD_CONFIGFILES=$EASYBUILD_ROOT/config.cfg,$EASYBUILD_CONFIG_ROOT/config.cfg
export EASYBUILD_SUBDIR_MODULES=modules/$YEAR
export EASYBUILD_SUBDIR_SOFTWARE=software/$YEAR
export EASYBUILD_SUBDIR_USER_MODULES=.local/easybuild/$EASYBUILD_SUBDIR_MODULES
export PATH=${PATH%%:/usr/bin}
module use $EASYBUILD_ROOT/$EASYBUILD_SUBDIR_MODULES
export EASYBUILD_ROBOT_PATHS=$EASYBUILD_ROOT/ebfiles_repo/$YEAR
export EASYBUILD_REPOSITORY='GitRepository'
if [ "$RSNT_ARCH" == avx2 ]; then
	export EASYBUILD_REPOSITORYPATH=$EASYBUILD_ROOT/ebfiles_repo.git,$YEAR
else
	export EASYBUILD_REPOSITORYPATH=$EASYBUILD_ROOT/ebfiles_repo_$RSNT_ARCH.git,$YEAR
	export EASYBUILD_ROBOT_PATHS=$EASYBUILD_ROOT/ebfiles_repo_$RSNT_ARCH/$YEAR:$EASYBUILD_ROBOT_PATHS
	export EASYBUILD_USE_CCACHE=''
fi

if [ "$RSNT_ARCH" == avx2 ]; then
	if [ -n "$USE_GENTOO" ]; then
	    if [ "$YEAR" -ge 2023 ]; then
		export EASYBUILD_OPTARCH='NVHPC:march=x86-64-v3;Intel:march=core-avx2 -axCore-AVX512;GCC:march=x86-64-v3'
	    else
		export EASYBUILD_OPTARCH='NVHPC:tp=haswell;Intel:march=core-avx2 -axCore-AVX512;GCC:march=core-avx2'
	    fi
	else
	    export EASYBUILD_OPTARCH='PGI:tp=haswell;Intel:xCore-AVX2;GCC:march=core-avx2'
	fi
elif [ "$RSNT_ARCH" == avx512 ]; then
	if [ -n "$USE_GENTOO" ]; then
	    if [ "$YEAR" -ge 2023 ]; then
		export EASYBUILD_OPTARCH='NVHPC:march=x86-64-v4;Intel:march=skylake-avx512;GCC:march=x86-64-v4'
	    else
		export EASYBUILD_OPTARCH='NVHPC:tp=skylake;Intel:xCore-AVX512;GCC:march=skylake-avx512'
	    fi
	else
	    export EASYBUILD_OPTARCH='PGI:tp=skylake;Intel:xCore-AVX512;GCC:march=skylake-avx512'
	fi
elif [ "$RSNT_ARCH" == avx ]; then
	if [ -n "$USE_GENTOO" ]; then
	    export EASYBUILD_OPTARCH='NVHPC:tp=sandybridge;Intel:xAVX;GCC:march=corei7-avx'
	else
	    export EASYBUILD_OPTARCH='PGI:tp=sandybridge;Intel:xAVX;GCC:march=corei7-avx'
	fi
elif [ "$RSNT_ARCH" == sse3 ]; then
	if [ -n "$USE_GENTOO" ]; then
	    export EASYBUILD_OPTARCH='NVHPC:tp=px;Intel:msse3;GCC:march=nocona -mtune=generic'
	else
	    export EASYBUILD_OPTARCH='PGI:tp=px;Intel:msse3;GCC:march=nocona -mtune=generic'
	fi
else
	echo please set RSNT_ARCH to sse3, avx, or avx2
	exit
fi
if [ -z $USE_GENTOO ]; then
    export EASYBUILD_OPTARCH="$EASYBUILD_OPTARCH;GCCcore:GENERIC"
fi

CURRENT_USER="$(whoami)"
ORIG_USER="$(logname)"
if [ "$CURRENT_USER" != "ebuser" ] ; then
	export EASYBUILD_PREFIX=$HOME/.local/easybuild
	export EASYBUILD_SUBDIR_USER_MODULES=''
        export EASYBUILD_REPOSITORY='FileRepository'
	export EASYBUILD_REPOSITORYPATH=$EASYBUILD_PREFIX/ebfiles_repo,$YEAR
	export EASYBUILD_ROBOT_PATHS=$EASYBUILD_PREFIX/ebfiles_repo/$YEAR:$EASYBUILD_ROBOT_PATHS
	if [ "$RSNT_ARCH" != avx2 ]; then
	    export EASYBUILD_ROBOT_PATHS=$EASYBUILD_PREFIX/ebfiles_repo_$RSNT_ARCH/$YEAR:$EASYBUILD_ROBOT_PATHS
	fi
else
	if [[ "$ORIG_USER" != "ebuser" && ( -z "$GIT_AUTHOR_NAME" || $GIT_AUTHOR_NAME == "EasyBuild User" ) ]]; then
		echo "Failed to read \$HOME/.git_env_vars.sh. Please make sure they are readable and configured as described on https://wiki.computecanada.ca/staff/RSNT_Software_-_Setting_up#Configuring_your_build_node_account"
		exit 1
	fi
	if [ "$(id -ng)" != "ebuser" ] ; then
		# using ebuser with different group ID
		export EASYBUILD_INSTALLPATH_SOFTWARE=/cvmfs/restricted.computecanada.ca/easybuild/$EASYBUILD_SUBDIR_SOFTWARE
		export EASYBUILD_SOURCEPATH=/cvmfs/restricted.computecanada.ca/easybuild/sources
	fi
fi
export EASYBUILD_ROBOT_PATHS=$EASYBUILD_ROOT/site-packages/custom-easyconfigs/easybuild/easyconfigs:$EASYBUILD_ROBOT_PATHS:
if [ "$YEAR" -ge "2023" ]; then
	EB=$EBROOTGENTOO/bin/eb
else
	export PYTHONPATH=$EASYBUILD_ROOT/site-packages
	EB=$EASYBUILD_ROOT/framework/eb
fi
export EASYBUILD_JOB_EB_CMD="srun --mpi=none --ntasks=1 $EB"
export EASYBUILD_HOOKS=$EASYBUILD_CONFIG_ROOT/cc_hooks.py
export EASYBUILD_BUILDPATH=$EASYBUILD_BUILDPATH/$RSNT_ARCH
export LANG=en_US.UTF-8
export LC_ALL=en_US.UTF-8
if [ ! -d /cvmfs/local ]; then
	export EASYBUILD_USE_CCACHE=''
fi
export RSNT_EASYBUILD_MAGIC_COOKIE=263ca73bb634185aab1d1b41627fdbba

# argument validation
NEW_ARGS=()
for argument in "$@"; do
	if [[ $argument =~ ^--disable-enforce-checksums$ ]]; then
		# workaround for non-propagation of this argument
		export EASYBUILD_JOB_EB_CMD="$EASYBUILD_JOB_EB_CMD $argument"
	fi
	if [[ $argument =~ ^--job-cores=[0-9]*$ ]]; then
		kv=(${argument//=/ })
		echo "Job-cores specified to ${kv[1]}, setting EASYBUILD_PARALLEL=${kv[1]}"
		export EASYBUILD_PARALLEL=${kv[1]}
	fi
	if [[ $argument =~ ^--job$ ]]; then
		# so we can follow the job output with e.g. tail -f
		export PYTHONUNBUFFERED=1
		if [[ $HOSTNAME =~ archimedes.c3.ca$ ]]; then
			# Needed for --try-* --job and --from-pr --job so
			# on-the-fly easyconfigs are passed to the job from shared_tmp.
			# Setting TMPDIR (unlike EASYBUILD_TMPDIR) here the job itself
			# (unlike the submitter) will use a private TMPDIR
			export TMPDIR="/shared_tmp"
			export EASYBUILD_DISABLE_CLEANUP_TMPDIR=1
		fi
	fi
	if [[ "$CURRENT_USER" == "ebuser" ]]; then
		if [[ $argument =~ --inject-checksums* || $argument =~ --robot* || "$argument" == "-r" ]]; then
			echo "Please do not use $argument as $CURRENT_USER"
			exit 1
		fi
		if [[ $argument =~ ^--installpath-software=/cvmfs/data.rsnt.computecanada.ca/.* ]]; then
			if [[ "$argument" != "--installpath-software=/cvmfs/data.rsnt.computecanada.ca/content/easybuild/data/2020" ]]; then
				echo "When installing datasets, the install path should be /cvmfs/data.rsnt.computecanada.ca/content/easybuild/data/2020 instead of $argument"
				exit 1
			fi
		fi
		if [[ $argument =~ ^--job$ ]]; then
			echo "Output of job will be in /shared_tmp/logs/%x-%j.out"
			cd /shared_tmp/logs
		fi
		if [[ $argument =~ .*.eb$ ]]; then
			if [[ $argument =~ ^/cvmfs/soft.computecanada.ca/easybuild/software/.* ]]; then
				NEW_ARGS+=("$argument")
				# accept this path, it is used by the auto recompilation
				continue 
			elif [[ $argument =~ .*/.* ]]; then
				echo "Please do not use a path to specify the easyconfig."
				echo "Error on argument: $argument"
				exit 1
			else
				# We want to prevent eb from using $EASYBUILD_ROOT/easyconfigs for
				# dependencies so this is not in the robot path. To find the
				# main easyconfig there we switch to its directory.
				letter=${argument::1}
				letter=${letter,} # lowercase first letter
				name=${argument%%-*}
				# some names contain - (e.g. HOOMD-blue) so we need pathname expansion
				ec=$(find $EASYBUILD_ROOT/easyconfigs/$letter/$name* -name $argument 2>/dev/null)
				custom_ec=$(find $EASYBUILD_ROOT/site-packages/custom-easyconfigs/easybuild/easyconfigs/$letter/$name* -name $argument 2>/dev/null)
				if [[ -n "$ec" && -f $ec && -z "$custom_ec" ]]; then
					NEW_ARGS+=("$ec")
				else    # if no corresponding easyconfig was found in easyconfigs folder, we get it found normally
					NEW_ARGS+=("$argument")
				fi
			fi
		else
			NEW_ARGS+=("$argument")
		fi
	fi
done
if [[ -n "$SLURM_CPUS_ON_NODE" ]]; then
	export EASYBUILD_PARALLEL=$SLURM_CPUS_ON_NODE
fi

if [[ "$CURRENT_USER" == "ebuser" ]]; then
	exec $EB --configfiles=$LOCAL_EASYBUILD_CONFIGFILES "${NEW_ARGS[@]}"
else
	exec $EB --configfiles=$LOCAL_EASYBUILD_CONFIGFILES ${1+"$@"}
fi