diff --git a/.github/workflows/docker-build-push-aws.yml b/.github/workflows/docker-build-push-aws.yml new file mode 100644 index 000000000..3f400a384 --- /dev/null +++ b/.github/workflows/docker-build-push-aws.yml @@ -0,0 +1,53 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build and push image to Amazon ECR. + +on: + pull_request: + branches: + - main + +env: + AWS_REGION: us-east-1 # set this to your preferred AWS region, e.g. us-west-1 + ECR_REPOSITORY: mdtf/mdtf-test # set this to your Amazon ECR repository name + ECS_SERVICE: custom-service # MY_ECS_SERVICE set this to your Amazon ECS service name + ECS_CLUSTER: mdtf-east1 # MY_ECS_CLUSTER set this to your Amazon ECS cluster name + ECS_TASK_DEFINITION: ecs-task-defn.json # set this to the path to your Amazon ECS task definition + # file, e.g. .aws/task-definition.json + CONTAINER_NAME: mdtf-test-a1r # set this to the name of the container in the + # containerDefinitions section of your task definition MY_CONTAINER_NAME + +jobs: + deploy: + name: push-to-public-ecr-cache-test + runs-on: ubuntu-latest + environment: dev + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@13d241b293754004c80624b5567555c4a39ffbe3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Build, tag, and push image to Amazon ECR + id: build-image + uses: aradhakrishnanGFDL/docker-build-with-cache-action@master #support-aws-ecr-public + env: + ECR_REGISTRY: public.ecr.aws + IMAGE_TAG: "latest" + ECS_IMAGE: $ECR_REGISTRY/ECR_REPOSITORY:$env.IMAGE_TAG + with: + registry: public.ecr.aws + username: "${{ secrets.AWS_ACCESS_KEY_ID }}" + password: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + image_name: c6h4h4s8/mdtf/mdtf-test + push_image_and_stages: true + image_tag: latest diff --git a/.github/workflows/docker-run-test.yml b/.github/workflows/docker-run-test.yml new file mode 100644 index 000000000..dfddcb24c --- /dev/null +++ b/.github/workflows/docker-run-test.yml @@ -0,0 +1,18 @@ +name: docker-run-minitest1 +on: + pull_request: + branches: [ main ] +jobs: + container-test-job: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/c6h4h4s8/mdtf/mdtf-test:latest + env: + NODE_ENV: development + options: --cpus 1 + steps: + - name: Check PATH + run: (echo $PATH) || (echo "No path") + - name: Activate environment + shell: bash + run: (eval "$(micromamba shell hook -s bash -p ~/conda)"; micromamba activate; micromamba activate _MDTF_base;mdtf_framework.py --version) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..8a97b11d5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Base OS +FROM ubuntu:latest +FROM mambaorg/micromamba:latest as micromamba + +#micromamba set up + +USER root + +RUN micromamba info + +## +# Container Metadata +LABEL maintainer="mdtf-leads-AR" +LABEL version="test-01" +LABEL description="This is a docker image for the MDTF-diagnostics package" +# Copy the MDTF-diagnostics package contents from local machine to image +ENV CODE_ROOT=/proj/MDTF-diagnostics +COPY src ${CODE_ROOT}/src +COPY data ${CODE_ROOT}/data +#COPY diagnostics ${CODE_ROOT}/diagnostics +COPY mdtf_framework.py ${CODE_ROOT} +COPY shared ${CODE_ROOT}/shared +COPY sites ${CODE_ROOT}/sites +COPY tests ${CODE_ROOT}/tests +# Install conda environments +ENV CONDA_ROOT=/opt/conda/ +ENV CONDA_ENV_DIR=/opt/conda/envs + +#USER mambauser +RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml +RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml + +ENV PATH="${PATH}:/proj/MDTF-diagnostics/" +#RUN eval "$(micromamba shell hook --shell bash )";micromamba activate _MDTF_base +ENV PATH=$PATH:"/opt/conda/envs/_MDTF_NCL_base/bin/" + diff --git a/src/conda/micromamba_init.sh b/src/conda/micromamba_init.sh index ddc8f8649..6dcb37cd2 100755 --- a/src/conda/micromamba_init.sh +++ b/src/conda/micromamba_init.sh @@ -161,5 +161,4 @@ else export PATH="${_CONDA_ROOT}/bin:$PATH" fi fi -unset __conda_setup - +unset __conda_setup \ No newline at end of file diff --git a/src/environment_manager.py b/src/environment_manager.py index 41232ecb3..2a1f8cfbf 100644 --- a/src/environment_manager.py +++ b/src/environment_manager.py @@ -256,7 +256,7 @@ def __init__(self, log=_log): elif key == '_CONDA_ROOT': self.conda_root = val except Exception as exc: - raise util.PodRuntimeError("Can't find conda.") from exc + raise util.PodRuntimeError("Can't find conda.") from exc # find where environments are installed if 'conda_env_root' in paths and paths.conda_env_root: @@ -543,12 +543,14 @@ def setup(self): self.env_mgr.create_environment(env) def spawn_subprocess(self, p, env_vars_base): + print("SPAWN SUBPROCESS") run_cmds = p.validate_commands() + p.run_commands() if self.test_mode: run_cmds = ['echo "TEST MODE: call {}"'.format('; '.join(run_cmds))] commands = self.env_mgr.activate_env_commands(p.env) \ + run_cmds \ + self.env_mgr.deactivate_env_commands(p.env) + print(commands) if self.test_mode: for cmd in commands: print('\tTEST MODE: call {}'.format(cmd)) diff --git a/tests/docker_test_set1.jsonc b/tests/docker_test_set1.jsonc new file mode 100644 index 000000000..6076d3b2c --- /dev/null +++ b/tests/docker_test_set1.jsonc @@ -0,0 +1,91 @@ +// Configuration for MDTF-diagnostics driver script self-test. +// All text to the right of an unquoted "//" is a comment and ignored, as well +// as blank lines (JSONC quasi-standard.) +{ + "case_list" : [ + // The cases below correspond to the different sample model data sets. Note + // that the MDTF package does not currently support analyzing multiple + // models in a single invocation. Comment out or delete the first entry and + // uncomment the second to run NOAA-GFDL-AM4 only for the MJO_prop_amp POD, + // and likewise for the SM_ET_coupling POD. + { + "CASENAME" : "GFDL.Synthetic", + "convention" : "GFDL", + "FIRSTYR" : 1, + "LASTYR" : 10, + "pod_list": [ + // Optional: PODs to run for this model only (defaults to all) + "MJO_prop_amp" + ] + } + ], + // PATHS --------------------------------------------------------------------- + // Location of supporting data downloaded when the framework was installed. + + // If a relative path is given, it's resolved relative to the MDTF-diagnostics + // code directory. Environment variables (eg, $HOME) can be referenced with a + // "$" and will be expended to their current values when the framework runs. + + // Parent directory containing observational data used by individual PODs. + //"OBS_DATA_ROOT": "../travis_test/obs_data", + "OBS_DATA_ROOT": "../inputdata/obs_data", + + // Parent directory containing results from different models. + //"MODEL_DATA_ROOT": "../travis_test/model/", + "MODEL_DATA_ROOT": "/proj/inputdata/mdtf_test_data", + + // Working directory. Defaults to working directory if blank. + "WORKING_DIR": "../wkdir", + + // Directory to write output. The results of each run of the framework will be + // put in a subdirectory of this directory. + "OUTPUT_DIR": "../wkdir", + + // Location of the Anaconda/miniconda installation to use for managing + // dependencies (path returned lsby running `conda info --base`.) If empty, + // framework will attempt to determine location of system's conda installation. + "conda_root": "/opt/conda", + + // Directory containing the framework-specific conda environments. This should + // be equal to the "--env_dir" flag passed to conda_env_setup.sh. If left + // blank, the framework will look for its environments in the system default + // location. + "conda_env_root": "/opt/conda/envs/", + + // SETTINGS ------------------------------------------------------------------ + // Any command-line option recognized by the mdtf script (type `mdtf --help`) + // can be set here, in the form "flag name": "desired setting". + + // Method used to fetch model data. + "data_manager": "Local_File", + + // Method used to manage dependencies. + "environment_manager": "Conda", + + // Settings affecting what output is generated: + + // Set to true to have PODs save postscript figures in addition to bitmaps. + "save_ps": false, + + // Set to true to have PODs save netCDF files of processed data. + "save_nc": false, + + // Set to true to save HTML and bitmap plots in a .tar file. + "make_variab_tar": false, + + // Set to true to overwrite results in OUTPUT_DIR; otherwise results saved + // under a unique name. + "overwrite": false, + + // Settings used in debugging: + + // Log verbosity level. + "verbose": 1, + + // Set to true for framework test. Data is fetched but PODs are not run. + "test_mode": false, + + // Set to true for framework test. No external commands are run and no remote + // data is copied. Implies test_mode. + "dry_run": false + }