containerized MDTF

.github/workflows/docker-build-push-aws.yml

@@ -0,0 +1,53 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Build and push image to Amazon ECR.
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  AWS_REGION: us-east-1                  # set this to your preferred AWS region, e.g. us-west-1
+  ECR_REPOSITORY: mdtf/mdtf-test        # set this to your Amazon ECR repository name
+  ECS_SERVICE: custom-service                # MY_ECS_SERVICE   set this to your Amazon ECS service name
+  ECS_CLUSTER: mdtf-east1                 # MY_ECS_CLUSTER  set this to your Amazon ECS cluster name
+  ECS_TASK_DEFINITION: ecs-task-defn.json # set this to the path to your Amazon ECS task definition
+                                               # file, e.g. .aws/task-definition.json
+  CONTAINER_NAME: mdtf-test-a1r           # set this to the name of the container in the
+                                               # containerDefinitions section of your task definition MY_CONTAINER_NAME
+
+jobs:
+  deploy:
+    name: push-to-public-ecr-cache-test
+    runs-on: ubuntu-latest
+    environment: dev
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@13d241b293754004c80624b5567555c4a39ffbe3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        uses: aradhakrishnanGFDL/docker-build-with-cache-action@master #support-aws-ecr-public
+        env:
+         ECR_REGISTRY: public.ecr.aws
+         IMAGE_TAG: "latest"
+         ECS_IMAGE: $ECR_REGISTRY/ECR_REPOSITORY:$env.IMAGE_TAG
+        with:
+         registry: public.ecr.aws
+         username: "${{ secrets.AWS_ACCESS_KEY_ID }}"  
+         password: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"  
+         image_name: c6h4h4s8/mdtf/mdtf-test
+         push_image_and_stages: true
+         image_tag: latest

.github/workflows/docker-run-test.yml

@@ -0,0 +1,18 @@
+name: docker-run-minitest1
+on:
+  pull_request:
+    branches: [ main ]
+jobs:
+  container-test-job:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/c6h4h4s8/mdtf/mdtf-test:latest
+      env:
+        NODE_ENV: development
+      options: --cpus 1
+    steps:
+      - name: Check PATH
+        run: (echo $PATH) || (echo "No path")
+      - name: Activate environment
+        shell: bash 
+        run: (eval "$(micromamba shell hook -s bash -p ~/conda)"; micromamba activate; micromamba activate _MDTF_base;mdtf_framework.py --version)

Dockerfile

@@ -0,0 +1,36 @@
+# Base OS
+FROM ubuntu:latest
+FROM mambaorg/micromamba:latest as micromamba
+
+#micromamba set up
+
+USER root
+
+RUN micromamba info
+
+##
+# Container Metadata
+LABEL maintainer="mdtf-leads-AR"
+LABEL version="test-01"
+LABEL description="This is a docker image for the MDTF-diagnostics package"
+# Copy the MDTF-diagnostics package contents from local machine to image 
+ENV CODE_ROOT=/proj/MDTF-diagnostics
+COPY src ${CODE_ROOT}/src                          
+COPY data ${CODE_ROOT}/data
+#COPY diagnostics ${CODE_ROOT}/diagnostics
+COPY mdtf_framework.py ${CODE_ROOT}
+COPY shared ${CODE_ROOT}/shared
+COPY sites ${CODE_ROOT}/sites
+COPY tests ${CODE_ROOT}/tests
+# Install conda environments
+ENV CONDA_ROOT=/opt/conda/
+ENV CONDA_ENV_DIR=/opt/conda/envs
+
+#USER mambauser
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml
+
+ENV PATH="${PATH}:/proj/MDTF-diagnostics/"
+#RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base
+ENV PATH=$PATH:"/opt/conda/envs/_MDTF_NCL_base/bin/"
+

src/conda/micromamba_init.sh

@@ -161,5 +161,4 @@ else
         export PATH="${_CONDA_ROOT}/bin:$PATH"
     fi
 fi
-unset __conda_setup
-
+unset __conda_setup

src/environment_manager.py

@@ -256,7 +256,7 @@ def __init__(self, log=_log):
                 elif key == '_CONDA_ROOT':
                     self.conda_root = val
         except Exception as exc:
-            raise util.PodRuntimeError("Can't find conda.") from exc
+                raise util.PodRuntimeError("Can't find conda.") from exc
 
         # find where environments are installed
         if 'conda_env_root' in paths and paths.conda_env_root:
@@ -543,12 +543,14 @@ def setup(self):
             self.env_mgr.create_environment(env)
 
     def spawn_subprocess(self, p, env_vars_base):
+        print("SPAWN SUBPROCESS")
         run_cmds = p.validate_commands() + p.run_commands()
         if self.test_mode:
             run_cmds = ['echo "TEST MODE: call {}"'.format('; '.join(run_cmds))]
         commands = self.env_mgr.activate_env_commands(p.env) \
             + run_cmds \
             + self.env_mgr.deactivate_env_commands(p.env)
+        print(commands)
         if self.test_mode:
             for cmd in commands:
                 print('\tTEST MODE: call {}'.format(cmd))

tests/docker_test_set1.jsonc

@@ -0,0 +1,91 @@
+// Configuration for MDTF-diagnostics driver script self-test.
+// All text to the right of an unquoted "//" is a comment and ignored, as well
+// as blank lines (JSONC quasi-standard.)
+{
+    "case_list" : [
+      // The cases below correspond to the different sample model data sets. Note
+      // that the MDTF package does not currently support analyzing multiple
+      // models in a single invocation. Comment out or delete the first entry and
+      // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD,
+      // and likewise for the SM_ET_coupling POD.
+      {
+        "CASENAME" : "GFDL.Synthetic",
+        "convention" : "GFDL",
+        "FIRSTYR" : 1,
+        "LASTYR" : 10,
+        "pod_list": [
+            // Optional: PODs to run for this model only (defaults to all)
+          "MJO_prop_amp"
+        ]
+      }
+    ],
+    // PATHS ---------------------------------------------------------------------
+    // Location of supporting data downloaded when the framework was installed.
+
+    // If a relative path is given, it's resolved relative to the MDTF-diagnostics
+    // code directory. Environment variables (eg, $HOME) can be referenced with a
+    // "$" and will be expended to their current values when the framework runs.
+
+    // Parent directory containing observational data used by individual PODs.
+    //"OBS_DATA_ROOT": "../travis_test/obs_data",
+    "OBS_DATA_ROOT": "../inputdata/obs_data",
+
+    // Parent directory containing results from different models.
+    //"MODEL_DATA_ROOT": "../travis_test/model/",
+    "MODEL_DATA_ROOT": "/proj/inputdata/mdtf_test_data",
+
+    // Working directory. Defaults to working directory if blank.
+    "WORKING_DIR": "../wkdir",
+
+    // Directory to write output. The results of each run of the framework will be
+    // put in a subdirectory of this directory.
+    "OUTPUT_DIR": "../wkdir",
+
+    // Location of the Anaconda/miniconda installation to use for managing
+    // dependencies (path returned lsby running `conda info --base`.) If empty,
+    // framework will attempt to determine location of system's conda installation.
+    "conda_root": "/opt/conda",
+
+    // Directory containing the framework-specific conda environments. This should
+    // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left
+    // blank, the framework will look for its environments in the system default
+    // location.
+    "conda_env_root": "/opt/conda/envs/",
+
+    // SETTINGS ------------------------------------------------------------------
+    // Any command-line option recognized by the mdtf script (type `mdtf --help`)
+    // can be set here, in the form "flag name": "desired setting".
+
+    // Method used to fetch model data.
+    "data_manager": "Local_File",
+
+    // Method used to manage dependencies.
+    "environment_manager": "Conda",
+
+    // Settings affecting what output is generated:
+
+    // Set to true to have PODs save postscript figures in addition to bitmaps.
+    "save_ps": false,
+
+    // Set to true to have PODs save netCDF files of processed data.
+    "save_nc": false,
+
+    // Set to true to save HTML and bitmap plots in a .tar file.
+    "make_variab_tar": false,
+
+    // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved
+    // under a unique name.
+    "overwrite": false,
+
+    // Settings used in debugging:
+
+    // Log verbosity level.
+    "verbose": 1,
+
+    // Set to true for framework test. Data is fetched but PODs are not run.
+    "test_mode": false,
+
+    // Set to true for framework test. No external commands are run and no remote
+    // data is copied. Implies test_mode.
+    "dry_run": false
+  }