Feature #2421 update base image (#2440)

Co-authored-by: John Halley Gotway <[email protected]>

.github/jobs/build_docker_image.sh

@@ -15,8 +15,5 @@ time_command docker build -t ${DOCKERHUB_TAG} \
     -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE}
 if [ $? != 0 ]; then
   cat ${GITHUB_WORKSPACE}/docker_build.log
-  # Append the full make_install.log file
-  echo "Appending make_install.log to docker_build.log. See the logs artifact for details."
-  cat ${GITHUB_WORKSPACE}/make_install.log >> ${GITHUB_WORKSPACE}/docker_build.log
   exit 1
 fi

.github/jobs/set_job_controls.sh

@@ -6,7 +6,7 @@ run_unit_tests=false
 run_diff=false
 run_update_truth=false
 met_base_repo=met-base
-met_base_tag=v1.1
+met_base_tag=v2.0_debian10
 input_data_version=develop
 truth_data_version=develop
 

.github/jobs/test_env_vars.sh

@@ -10,4 +10,4 @@ export MET_TEST_TRUTH=/data/output/met_test_truth
 export MET_TEST_DIFF=/data/output/met_test_diff
 
 export MET_TEST_RSCRIPT=/usr/bin/Rscript
-export MET_TEST_MET_PYTHON_EXE=/usr/bin/python3
+export MET_TEST_MET_PYTHON_EXE=/usr/local/bin/python3

.github/workflows/build_docker_and_trigger_metplus.yml

@@ -29,7 +29,7 @@ jobs:
         env:
           SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
           MET_BASE_REPO: met-base
-          MET_BASE_TAG: v1.1
+          MET_BASE_TAG: v2.0_debian10
 
       - name: Push Docker Image
         run: .github/jobs/push_docker_image.sh

internal/scripts/docker/build_met_docker.sh

@@ -2,33 +2,37 @@
 
 echo "Running script to build MET in Docker"
 
-LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log
-
 source internal/scripts/environment/development.docker
 
+mkdir -p /met/logs
+
+LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log
 echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 ./bootstrap
-./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python > ${LOG_FILE}
+./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" > ${LOG_FILE} 2>&1
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 if [ ! -z "${MAKE_ARGS}" ]; then
   echo Adding make arguments: ${MAKE_ARGS}
 fi
 
-LOG_FILE=/met/MET-${MET_GIT_NAME}/make_install.log
+LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_install.log
 echo "Compiling MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 make ${MAKE_ARGS} install > ${LOG_FILE}
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_test.log
 echo "Testing MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 make ${MAKE_ARGS} test > ${LOG_FILE} 2>&1
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 if [[ $MET_GIT_NAME == "v"* ]]; then

internal/scripts/environment/development.docker

@@ -1,39 +1,22 @@
-# Define the development environment for NCAR project machine seneca
-# Based on settings in /usr/local/src/met/README.snat
+# Define the development environment for Docker
 
 # Top-level MET project directory
 MET_PROJ_DIR=`ls -1d /met/MET*`
 
 # Variables required to build MET
 export MET_DEVELOPMENT=true
 
-export MET_NETCDF=/usr/local
-export MET_HDF5INC=/usr/include
-export MET_HDF5LIB=/usr/lib64
-export MET_HDF=/usr/local/hdf
-export MET_HDFEOS=/usr/local/hdfeos
-export MET_BUFR=/usr/local
-export MET_GRIB2CINC=/usr/include
-export MET_GRIB2CLIB=/usr/lib64
-export MET_GSLINC=/usr/include/gsl
-export MET_GSLLIB=/usr/lib64
-export MET_CAIROINC=/usr/include/cairo
-export MET_CAIROLIB=/usr/lib64 
-export MET_FREETYPEINC=/usr/include/freetype2
-export MET_FREETYPELIB=/usr/lib64
-export MET_JASPERLIB=/usr/lib64
-
-export MET_PYTHON_BIN_EXE=/usr/bin/python3
-export MET_PYTHON_CC="-I/usr/include/python3.6m"
-export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm"
-
-# -D__64BIT__ is required because we've compiled libgrib2c.a with that flag
-export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g"
+export MET_PYTHON=/usr/local
+export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3
+export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.10"
+export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lm"
+
+export CFLAGS="-DUNDERSCORE -fPIC -g"
 export CXXFLAGS=${CFLAGS}
 
 # Set LDFLAGS to include -rpath settings when compiling MET
 export LDFLAGS="-Wl,--disable-new-dtags"
-export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64:${MET_HDF}/lib:${MET_HDFEOS}/lib"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64"
 
 # Variables required to run MET
 export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test
@@ -44,3 +27,16 @@ export MET_TEST_RSCRIPT=/usr/bin/Rscript
 
 # set make arguments to build using multiple jobs
 export MAKE_ARGS=-j
+
+export TEST_BASE=/met
+export COMPILER=gnu
+export MET_SUBDIR=${TEST_BASE}
+export MET_TARBALL=none
+export USE_MODULES=FALSE
+
+export MET_INSTALL_DIR=/usr/local
+
+export COMPILE_HDF=1
+export COMPILE_HDFEOS=1
+export COMPILE_FREETYPE=1
+export COMPILE_CAIRO=1

internal/scripts/installation/compile_MET_all.sh

@@ -89,8 +89,23 @@ echo "MET_TARBALL = ${MET_TARBALL? "ERROR: MET_TARBALL must be set"}"
 echo "USE_MODULES = ${USE_MODULES? "ERROR: USE_MODULES must be set to TRUE if using modules or FALSE otherwise"}"
 echo ${MAKE_ARGS:+MAKE_ARGS = $MAKE_ARGS}
 
-export LIB_DIR=${TEST_BASE}/external_libs
+LIB_DIR=${TEST_BASE}/external_libs
 MET_DIR=${MET_SUBDIR}
+
+if [ -z "${BIN_DIR_PATH}" ]; then
+  if [ -z "${MET_INSTALL_DIR}" ]; then
+    BIN_DIR_PATH=${TEST_BASE}/bin
+  else
+    BIN_DIR_PATH=${MET_INSTALL_DIR}/bin
+  fi
+fi
+
+if [ -z "${MET_INSTALL_DIR}" ]; then
+  MET_INSTALL_DIR=${MET_DIR}
+else
+  LIB_DIR=${MET_INSTALL_DIR}
+fi
+
 TAR_DIR=${TEST_BASE}/tar_files
 MET_TARBALL=${TAR_DIR}/${MET_TARBALL}
 
@@ -415,12 +430,16 @@ if [ $COMPILE_G2CLIB -eq 1 ]; then
   rm -rf ${LIB_DIR}/g2clib/g2clib*
   tar -xf ${TAR_DIR}/g2clib*.tar -C ${LIB_DIR}/g2clib
   cd ${LIB_DIR}/g2clib/g2clib*
-  cat makefile | \
-    sed -r 's/INC=.*/INC=-I${LIB_DIR}\/include -I${LIB_DIR}\/include\/jasper/g' | \
-    sed 's/CC=gcc/CC=${CC_COMPILER}/g' | \
-    sed 's/-D__64BIT__//g' \
-    > makefile_new
-  mv makefile_new makefile
+  sed -i 's|INC=.*|INC=-I${LIB_DIR}/include -I${LIB_DIR}/include/jasper|g' makefile
+
+  # allow other compilers besides gcc
+  sed -i 's/CC=gcc/CC=${CC_COMPILER}/g' makefile
+
+  # remove -D__64BIT__ flag because compiling with it has
+  # shown issues with GRIB/GRIB2 files that are over 2GB in size
+  # This flag was removed in g2clib 1.6.4
+  # so this can be removed if the version is updated
+  sed -i 's/-D__64BIT__//g' makefile
   export CC_COMPILER=${CC}
   echo "cd `pwd`"
   # g2clib appears to compile but causes failure compiling MET if -j argument is used
@@ -628,7 +647,7 @@ if [[ $COMPILER_FAMILY == "pgi" ]]; then
   export OPT_ARGS="${OPT_ARGS} FFLAGS=-lpgf90"
 fi
 
-configure_cmd="./configure --prefix=${MET_DIR} --bindir=${BIN_DIR_PATH}"
+configure_cmd="./configure --prefix=${MET_INSTALL_DIR} --bindir=${BIN_DIR_PATH}"
 configure_cmd="${configure_cmd} BUFRLIB_NAME=${BUFRLIB_NAME}"
 configure_cmd="${configure_cmd} GRIB2CLIB_NAME=${GRIB2CLIB_NAME} --enable-grib2"
 if [[ ! -z ${MET_FREETYPEINC} && ! -z ${MET_FREETYPELIB} && \