From eb987a401413723950d7c66ed05f42b8f9d03ef1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 16 Feb 2023 14:57:27 -0700 Subject: [PATCH] Feature #2421 update base image (#2440) Co-authored-by: John Halley Gotway --- .github/jobs/build_docker_image.sh | 3 -- .github/jobs/set_job_controls.sh | 2 +- .github/jobs/test_env_vars.sh | 2 +- .../build_docker_and_trigger_metplus.yml | 2 +- internal/scripts/docker/build_met_docker.sh | 18 +++++--- .../scripts/environment/development.docker | 46 +++++++++---------- .../scripts/installation/compile_MET_all.sh | 35 ++++++++++---- 7 files changed, 62 insertions(+), 46 deletions(-) diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh index 4fabd88d2e..55bd37c644 100755 --- a/.github/jobs/build_docker_image.sh +++ b/.github/jobs/build_docker_image.sh @@ -15,8 +15,5 @@ time_command docker build -t ${DOCKERHUB_TAG} \ -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE} if [ $? != 0 ]; then cat ${GITHUB_WORKSPACE}/docker_build.log - # Append the full make_install.log file - echo "Appending make_install.log to docker_build.log. See the logs artifact for details." - cat ${GITHUB_WORKSPACE}/make_install.log >> ${GITHUB_WORKSPACE}/docker_build.log exit 1 fi diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index e897aaa912..76a123d18b 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -6,7 +6,7 @@ run_unit_tests=false run_diff=false run_update_truth=false met_base_repo=met-base -met_base_tag=v1.1 +met_base_tag=v2.0_debian10 input_data_version=develop truth_data_version=develop diff --git a/.github/jobs/test_env_vars.sh b/.github/jobs/test_env_vars.sh index 4607e58f07..a77a9ca0f8 100755 --- a/.github/jobs/test_env_vars.sh +++ b/.github/jobs/test_env_vars.sh @@ -10,4 +10,4 @@ export MET_TEST_TRUTH=/data/output/met_test_truth export MET_TEST_DIFF=/data/output/met_test_diff export MET_TEST_RSCRIPT=/usr/bin/Rscript -export MET_TEST_MET_PYTHON_EXE=/usr/bin/python3 +export MET_TEST_MET_PYTHON_EXE=/usr/local/bin/python3 diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index 15cd652caf..89e5e14f67 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -29,7 +29,7 @@ jobs: env: SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite MET_BASE_REPO: met-base - MET_BASE_TAG: v1.1 + MET_BASE_TAG: v2.0_debian10 - name: Push Docker Image run: .github/jobs/push_docker_image.sh diff --git a/internal/scripts/docker/build_met_docker.sh b/internal/scripts/docker/build_met_docker.sh index a7e2aa7a07..df02108390 100755 --- a/internal/scripts/docker/build_met_docker.sh +++ b/internal/scripts/docker/build_met_docker.sh @@ -2,33 +2,37 @@ echo "Running script to build MET in Docker" -LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log - source internal/scripts/environment/development.docker +mkdir -p /met/logs + +LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" ./bootstrap -./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python > ${LOG_FILE} +./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" > ${LOG_FILE} 2>&1 if [ $? != 0 ]; then - exit 1 + cat ${LOG_FILE} + exit 1 fi if [ ! -z "${MAKE_ARGS}" ]; then echo Adding make arguments: ${MAKE_ARGS} fi -LOG_FILE=/met/MET-${MET_GIT_NAME}/make_install.log +LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_install.log echo "Compiling MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" make ${MAKE_ARGS} install > ${LOG_FILE} if [ $? != 0 ]; then - exit 1 + cat ${LOG_FILE} + exit 1 fi LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_test.log echo "Testing MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" make ${MAKE_ARGS} test > ${LOG_FILE} 2>&1 if [ $? != 0 ]; then - exit 1 + cat ${LOG_FILE} + exit 1 fi if [[ $MET_GIT_NAME == "v"* ]]; then diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index c7be4a77fe..e7968889f6 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -1,5 +1,4 @@ -# Define the development environment for NCAR project machine seneca -# Based on settings in /usr/local/src/met/README.snat +# Define the development environment for Docker # Top-level MET project directory MET_PROJ_DIR=`ls -1d /met/MET*` @@ -7,33 +6,17 @@ MET_PROJ_DIR=`ls -1d /met/MET*` # Variables required to build MET export MET_DEVELOPMENT=true -export MET_NETCDF=/usr/local -export MET_HDF5INC=/usr/include -export MET_HDF5LIB=/usr/lib64 -export MET_HDF=/usr/local/hdf -export MET_HDFEOS=/usr/local/hdfeos -export MET_BUFR=/usr/local -export MET_GRIB2CINC=/usr/include -export MET_GRIB2CLIB=/usr/lib64 -export MET_GSLINC=/usr/include/gsl -export MET_GSLLIB=/usr/lib64 -export MET_CAIROINC=/usr/include/cairo -export MET_CAIROLIB=/usr/lib64 -export MET_FREETYPEINC=/usr/include/freetype2 -export MET_FREETYPELIB=/usr/lib64 -export MET_JASPERLIB=/usr/lib64 - -export MET_PYTHON_BIN_EXE=/usr/bin/python3 -export MET_PYTHON_CC="-I/usr/include/python3.6m" -export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm" - -# -D__64BIT__ is required because we've compiled libgrib2c.a with that flag -export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g" +export MET_PYTHON=/usr/local +export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3 +export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.10" +export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lm" + +export CFLAGS="-DUNDERSCORE -fPIC -g" export CXXFLAGS=${CFLAGS} # Set LDFLAGS to include -rpath settings when compiling MET export LDFLAGS="-Wl,--disable-new-dtags" -export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64:${MET_HDF}/lib:${MET_HDFEOS}/lib" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64" # Variables required to run MET export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test @@ -44,3 +27,16 @@ export MET_TEST_RSCRIPT=/usr/bin/Rscript # set make arguments to build using multiple jobs export MAKE_ARGS=-j + +export TEST_BASE=/met +export COMPILER=gnu +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=none +export USE_MODULES=FALSE + +export MET_INSTALL_DIR=/usr/local + +export COMPILE_HDF=1 +export COMPILE_HDFEOS=1 +export COMPILE_FREETYPE=1 +export COMPILE_CAIRO=1 diff --git a/internal/scripts/installation/compile_MET_all.sh b/internal/scripts/installation/compile_MET_all.sh index e0ff3290cc..9810a124f2 100755 --- a/internal/scripts/installation/compile_MET_all.sh +++ b/internal/scripts/installation/compile_MET_all.sh @@ -89,8 +89,23 @@ echo "MET_TARBALL = ${MET_TARBALL? "ERROR: MET_TARBALL must be set"}" echo "USE_MODULES = ${USE_MODULES? "ERROR: USE_MODULES must be set to TRUE if using modules or FALSE otherwise"}" echo ${MAKE_ARGS:+MAKE_ARGS = $MAKE_ARGS} -export LIB_DIR=${TEST_BASE}/external_libs +LIB_DIR=${TEST_BASE}/external_libs MET_DIR=${MET_SUBDIR} + +if [ -z "${BIN_DIR_PATH}" ]; then + if [ -z "${MET_INSTALL_DIR}" ]; then + BIN_DIR_PATH=${TEST_BASE}/bin + else + BIN_DIR_PATH=${MET_INSTALL_DIR}/bin + fi +fi + +if [ -z "${MET_INSTALL_DIR}" ]; then + MET_INSTALL_DIR=${MET_DIR} +else + LIB_DIR=${MET_INSTALL_DIR} +fi + TAR_DIR=${TEST_BASE}/tar_files MET_TARBALL=${TAR_DIR}/${MET_TARBALL} @@ -415,12 +430,16 @@ if [ $COMPILE_G2CLIB -eq 1 ]; then rm -rf ${LIB_DIR}/g2clib/g2clib* tar -xf ${TAR_DIR}/g2clib*.tar -C ${LIB_DIR}/g2clib cd ${LIB_DIR}/g2clib/g2clib* - cat makefile | \ - sed -r 's/INC=.*/INC=-I${LIB_DIR}\/include -I${LIB_DIR}\/include\/jasper/g' | \ - sed 's/CC=gcc/CC=${CC_COMPILER}/g' | \ - sed 's/-D__64BIT__//g' \ - > makefile_new - mv makefile_new makefile + sed -i 's|INC=.*|INC=-I${LIB_DIR}/include -I${LIB_DIR}/include/jasper|g' makefile + + # allow other compilers besides gcc + sed -i 's/CC=gcc/CC=${CC_COMPILER}/g' makefile + + # remove -D__64BIT__ flag because compiling with it has + # shown issues with GRIB/GRIB2 files that are over 2GB in size + # This flag was removed in g2clib 1.6.4 + # so this can be removed if the version is updated + sed -i 's/-D__64BIT__//g' makefile export CC_COMPILER=${CC} echo "cd `pwd`" # g2clib appears to compile but causes failure compiling MET if -j argument is used @@ -628,7 +647,7 @@ if [[ $COMPILER_FAMILY == "pgi" ]]; then export OPT_ARGS="${OPT_ARGS} FFLAGS=-lpgf90" fi -configure_cmd="./configure --prefix=${MET_DIR} --bindir=${BIN_DIR_PATH}" +configure_cmd="./configure --prefix=${MET_INSTALL_DIR} --bindir=${BIN_DIR_PATH}" configure_cmd="${configure_cmd} BUFRLIB_NAME=${BUFRLIB_NAME}" configure_cmd="${configure_cmd} GRIB2CLIB_NAME=${GRIB2CLIB_NAME} --enable-grib2" if [[ ! -z ${MET_FREETYPEINC} && ! -z ${MET_FREETYPELIB} && \