From b1e1a8c88c8a4bfeae581f2fd1bc1165a3271e85 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:50:51 -0400 Subject: [PATCH 1/5] Update config.yaml --- configs/config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/config.yaml b/configs/config.yaml index 559e9c0..8548827 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -13,9 +13,9 @@ #with the ESM collection specification standards and the appropriate workflows. headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", - "frequency", "modeling_realm", "table_id", + "frequency", "realm", "table_id", "member_id", "grid_label", "variable_id", - "temporal_subset", "chunk_freq","platform","dimensions","cell_methods","path"] + "time_range", "chunk_freq","platform","dimensions","cell_methods","path"] #what kind of directory structure to expect? #For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp @@ -27,9 +27,9 @@ headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", #The fourth directory is am5f3b1r0 which does not map to an existing header value. So we simply NA in output_path_template #for the fourth value. -output_path_template: ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','modeling_realm','cell_methods','frequency','chunk_freq'] +output_path_template: ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','realm','cell_methods','frequency','chunk_freq'] -output_file_template: ['modeling_realm','temporal_subset','variable_id'] +output_file_template: ['realm','time_range','variable_id'] #OUTPUT FILE INFO is currently passed as command-line argument. #We will revisit adding a csvfile, jsonfile and logfile configuration to the builder configuration file in the future. From c085e7aa658daf38d6498023cca65a0dbf84472e Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:51:22 -0400 Subject: [PATCH 2/5] Update config-template.yaml --- configs/config-template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/config-template.yaml b/configs/config-template.yaml index 27e0696..4660ed6 100644 --- a/configs/config-template.yaml +++ b/configs/config-template.yaml @@ -1,8 +1,8 @@ #The headerlist contains expected column names in your catalog/csv file. This is usually determined by the users in conjuction #with the ESM collection specification standards and the appropriate workflows. -headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", "frequency", "modeling_realm", "table_id","member_id", - "grid_label", "variable_id", "temporal_subset", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"] +headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", "frequency", "realm", "table_id","member_id", + "grid_label", "variable_id", "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"] #what kind of directory structure to expect? @@ -16,7 +16,7 @@ headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", "fre output_path_template: ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','modeling_realm','cell_methods','frequency','chunk_freq'] -output_file_template: ['modeling_realm','temporal_subset','variable_id'] +output_file_template: ['realm','time_range','variable_id'] #ENTER INPUT PATH HERE (ex: /Users/ar46/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/) From 834b320adeba0d2220529733e4b30a41e2380aac Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:51:55 -0400 Subject: [PATCH 3/5] Update config-cmip.yaml --- configs/config-cmip.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/config-cmip.yaml b/configs/config-cmip.yaml index 3ad69b2..9f01d8d 100644 --- a/configs/config-cmip.yaml +++ b/configs/config-cmip.yaml @@ -13,9 +13,9 @@ #with the ESM collection specification standards and the appropriate workflows. headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", - "frequency", "modeling_realm", "table_id", + "frequency", "realm", "table_id", "member_id", "grid_label", "variable_id","version_id", - "temporal_subset", "chunk_freq","platform","dimensions","cell_methods","path"] + "time_range", "chunk_freq","platform","dimensions","cell_methods","path"] #what kind of directory structure to expect? #For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp @@ -30,7 +30,7 @@ headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", #/uda/CMIP6/CMIP/NOAA-GFDL/GFDL-ESM4/1pctCO2/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-ESM4_1pctCO2_r1i1p1f1_gr1_000101-010012.nc output_path_template: ['NA','activity_id','NA','institution_id','source_id','experiment_id','member_id','table_id','variable_id','grid_label','version_id'] -output_file_template: ['variable_id','table_id','source_id','experiment_id','member_id','grid_label','temporal_subset'] +output_file_template: ['variable_id','table_id','source_id','experiment_id','member_id','grid_label','time_range'] #OUTPUT FILE INFO is currently passed as command-line argument. #We will revisit adding a csvfile, jsonfile and logfile configuration to the builder configuration file in the future. From 17ef45898295b46135847dc29208f6cd628278b6 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 13:54:19 -0400 Subject: [PATCH 4/5] Update test_config.yaml --- tests/test_config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 803f1b8..9d9bbd1 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -13,9 +13,9 @@ #with the ESM collection specification standards and the appropriate workflows. headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", - "frequency", "modeling_realm", "table_id", + "frequency", "realm", "table_id", "member_id", "grid_label", "variable_id", - "temporal_subset", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"] + "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"] #what kind of directory structure to expect? #For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp @@ -27,9 +27,9 @@ headerlist: ["activity_id", "institution_id", "source_id", "experiment_id", #The fourth directory is am5f3b1r0 which does not map to an existing header value. So we simply NA in output_path_template #for the fourth value. -output_path_template: ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','modeling_realm','cell_methods','frequency','chunk_freq'] +output_path_template: ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','realm','cell_methods','frequency','chunk_freq'] -output_file_template: ['modeling_realm','temporal_subset','variable_id'] +output_file_template: ['realm','time_range','variable_id'] #OUTPUT FILE INFO is currently passed as command-line argument. #We will revisit adding a csvfile, jsonfile and logfile configuration to the builder configuration file in the future. From b25d0356d5bfdcb3b09cb23c47838d33f325ee45 Mon Sep 17 00:00:00 2001 From: Aparna Radhakrishnan Date: Tue, 23 Jul 2024 14:02:40 -0400 Subject: [PATCH 5/5] Delete .github/workflows/import-test.yml obsolete, and fails. --- .github/workflows/import-test.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/import-test.yml diff --git a/.github/workflows/import-test.yml b/.github/workflows/import-test.yml deleted file mode 100644 index fadafcc..0000000 --- a/.github/workflows/import-test.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Importing intakebuilder - -on: [push] - -jobs: - build-linux: - runs-on: ubuntu-latest - strategy: - max-parallel: 5 - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install dependencies - run: | - $CONDA/bin/conda --version - # $CONDA is an environment variable pointing to the root of the miniconda directory - $CONDA/bin/conda create -n catalogbuilder - $CONDA/bin/conda install -n catalogbuilder -c conda-forge pytest pandas xarray click intake-esm - - name: tests_prelim - run: | - $CONDA/envs/catalogbuilder/bin/pytest tests/default/