From c8633bae9d87f7f40e938416d82dfe3bac1e48be Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 10:17:52 -0700 Subject: [PATCH 01/14] feat(dataset): add ida-ndvi dataset config --- .../staging/dataset-config/ida-ndvi.json | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/ida-ndvi.json diff --git a/ingestion-data/staging/dataset-config/ida-ndvi.json b/ingestion-data/staging/dataset-config/ida-ndvi.json new file mode 100644 index 00000000..c3aa6548 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndvi.json @@ -0,0 +1,83 @@ +{ + "collection": "ida-ndvi", + "title": "NDVI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index before and after Hurricane Ida in southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "NDVI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylgn", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southenr Louisiana.)" + } + } + } \ No newline at end of file From 664cf0aa5a5098be38988a9fe244e2d96eac2f89 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 10:56:44 -0700 Subject: [PATCH 02/14] feat(datasets) add ida-ndvi-difference, ida-ndwi, and ida-ndwi-difference configs --- .../dataset-config/ida-ndvi-difference.json | 81 ++++++++++++++++++ .../staging/dataset-config/ida-ndvi.json | 2 +- .../dataset-config/ida-ndwi-difference.json | 81 ++++++++++++++++++ .../staging/dataset-config/ida-ndwi.json | 83 +++++++++++++++++++ 4 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 ingestion-data/staging/dataset-config/ida-ndvi-difference.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndwi-difference.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndwi.json diff --git a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json new file mode 100644 index 00000000..a4ce8217 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json @@ -0,0 +1,81 @@ +{ + "collection": "ida-ndvi-difference", + "title": "NDVI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index Difference of before and after Hurricane Ida in southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices/NDVI_Difference_Hurricane_cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDVI_Difference_Hurricane_cog.tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndvi.json b/ingestion-data/staging/dataset-config/ida-ndvi.json index c3aa6548..6c09e5fc 100644 --- a/ingestion-data/staging/dataset-config/ida-ndvi.json +++ b/ingestion-data/staging/dataset-config/ida-ndvi.json @@ -77,7 +77,7 @@ "thumbnail" ], "title": "Thumbnail", - "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southenr Louisiana.)" + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" } } } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json new file mode 100644 index 00000000..f03457b2 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json @@ -0,0 +1,81 @@ +{ + "collection": "ida-ndwi-difference", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_Hurricane_cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDWI_Difference_Hurricane_cog.tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndwi.json b/ingestion-data/staging/dataset-config/ida-ndwi.json new file mode 100644 index 00000000..fff1001b --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndwi.json @@ -0,0 +1,83 @@ +{ + "collection": "ida-ndwi", + "title": "NDWI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index before and after Hurricane Ida in southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "NDWI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylbu", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file From d4c2e2ae22fc8443549c856913a1338e52c6c4f5 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 11:48:16 -0700 Subject: [PATCH 03/14] test print --- .github/workflows/pr.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cb3a0e0d..0c8ca5fa 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -132,17 +132,18 @@ jobs: status_message='### Collection Publication Status ' - for file in "${ADDED_FILES[@]}"; do + for file in ${ADDED_FILES}; do echo $file if [ -f "$file" ]; then dataset_config=$(jq '.' "$file") collection_id=$(jq -r '.collection' "$file") - - response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -d "$dataset_config" - ) + echo $dataset_config + echo $collection_id + # response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \ + # -H "Content-Type: application/json" \ + # -H "Authorization: Bearer $AUTH_TOKEN" \ + # -d "$dataset_config" + # ) status_code=$(tail -n1 <<< "$response") From 3451ff7002a6b1435b5c1478d8e4667f65ab4ed6 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 11:49:40 -0700 Subject: [PATCH 04/14] test print --- .github/workflows/pr.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0c8ca5fa..cea5038b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -137,13 +137,13 @@ jobs: if [ -f "$file" ]; then dataset_config=$(jq '.' "$file") collection_id=$(jq -r '.collection' "$file") - echo $dataset_config - echo $collection_id - # response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \ - # -H "Content-Type: application/json" \ - # -H "Authorization: Bearer $AUTH_TOKEN" \ - # -d "$dataset_config" - # ) + + echo "Publishing $collection_id" + response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -d "$dataset_config" + ) status_code=$(tail -n1 <<< "$response") From 41a41f147f48480ba8fc334957c1737b1bfe7f56 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 11:52:26 -0700 Subject: [PATCH 05/14] fix discovery prefix --- ingestion-data/staging/dataset-config/ida-ndwi-difference.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json index f03457b2..6c4f9c5d 100644 --- a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json +++ b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json @@ -24,7 +24,7 @@ "cogify": false, "upload": false, "dry_run": false, - "prefix": "planet-indices/", + "prefix": "planet-indices-v2/", "bucket": "veda-data-store-staging", "filename_regex": "(.*)NDWI_Difference_Hurricane_cog.tif$" } From fda8649b2817356d48ec61377ce6112d9cf49b17 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 11:56:27 -0700 Subject: [PATCH 06/14] fix for loop in mdx generation step --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cea5038b..4d0cb90a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -222,7 +222,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES pip install -r ./scripts/requirements.txt - for file in "${PUBLISHED_COLLECTION_FILES[@]}" + for file in ${PUBLISHED_COLLECTION_FILES} do python3 ./scripts/generate-mdx.py "$file" done From b3adfae7668f25646fe5263a4b0a2169e5932ee5 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 12:57:33 -0700 Subject: [PATCH 07/14] fix for loop in mdx generation step --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4d0cb90a..c31db802 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -172,7 +172,7 @@ jobs: fi # Output only successful collections to be used in subsequent steps - echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT + # echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT # Update PR comment CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') From 204b9c43784ea13b436f2e55904dc4201af12c12 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 13:33:16 -0700 Subject: [PATCH 08/14] fix for loop in mdx generation step --- .github/workflows/pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c31db802..ed5d86bb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -172,7 +172,7 @@ jobs: fi # Output only successful collections to be used in subsequent steps - # echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT + echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT # Update PR comment CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') @@ -222,7 +222,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES pip install -r ./scripts/requirements.txt - for file in ${PUBLISHED_COLLECTION_FILES} + for file in "${PUBLISHED_COLLECTION_FILES[@]}"" do python3 ./scripts/generate-mdx.py "$file" done From 75fd67a260b9bf691108ea964d8450f09f8457a0 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 13:36:21 -0700 Subject: [PATCH 09/14] typo --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ed5d86bb..cea5038b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -222,7 +222,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES pip install -r ./scripts/requirements.txt - for file in "${PUBLISHED_COLLECTION_FILES[@]}"" + for file in "${PUBLISHED_COLLECTION_FILES[@]}" do python3 ./scripts/generate-mdx.py "$file" done From 71b5c306fd8426890fa9919b605e4c367c86767d Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 16:53:49 -0700 Subject: [PATCH 10/14] fix for loop in mdx generation step --- .github/workflows/pr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cea5038b..22f719a7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -128,7 +128,7 @@ jobs: # Track successful publications all_failed=true - success_collections=() + declare -a success_collections=() status_message='### Collection Publication Status ' @@ -172,7 +172,8 @@ jobs: fi # Output only successful collections to be used in subsequent steps - echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT + # echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT + echo "success_collections=${success_collections[*]}" >> $GITHUB_OUTPUT # Update PR comment CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') From 7064ae8b92e26acbc5d0a15e5cdf3b40f5dc0bd9 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 16:56:49 -0700 Subject: [PATCH 11/14] fix for loop in mdx generation step --- .github/workflows/pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 22f719a7..7e7ae29a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -223,8 +223,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES pip install -r ./scripts/requirements.txt - for file in "${PUBLISHED_COLLECTION_FILES[@]}" - do + for file in ${PUBLISHED_COLLECTION_FILES}; do python3 ./scripts/generate-mdx.py "$file" done From a11d3f5baa19d497e9088363e45b71c5a1551740 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 18 Dec 2024 17:15:08 -0700 Subject: [PATCH 12/14] Update .github/workflows/pr.yml --- .github/workflows/pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7e7ae29a..12d1142b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -172,7 +172,6 @@ jobs: fi # Output only successful collections to be used in subsequent steps - # echo "success_collections=$(IFS=','; echo "${success_collections[*]}")" >> $GITHUB_OUTPUT echo "success_collections=${success_collections[*]}" >> $GITHUB_OUTPUT # Update PR comment From c6b75b551bf8b3457ae3dae10dcd0a7573052472 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 8 Jan 2025 16:23:30 -0700 Subject: [PATCH 13/14] corrected ida-ndvi-difference config --- .../staging/dataset-config/ida-ndvi-difference.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json index a4ce8217..25238e1c 100644 --- a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json +++ b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json @@ -16,7 +16,7 @@ "enddate": "2021-09-09T23:59:59Z" }, "sample_files": [ - "s3://veda-data-store-staging/planet-indices/NDVI_Difference_Hurricane_cog.tif" + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Difference_2021-08-23_2021-09-09.tif" ], "discovery_items": [ { @@ -24,9 +24,9 @@ "cogify": false, "upload": false, "dry_run": false, - "prefix": "planet-indices/", + "prefix": "planet-indices-v2/", "bucket": "veda-data-store-staging", - "filename_regex": "(.*)NDVI_Difference_Hurricane_cog.tif$" + "filename_regex": "(.*)NDVI_Difference_(.*).tif$" } ], "data_type": "cog", From 5ead6b0cced6d13fea0a81ee9374fd7e045acb5d Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 10 Jan 2025 14:02:50 -0700 Subject: [PATCH 14/14] sm2a updates for ida-* collections --- .../staging/dataset-config/ida-ndvi-difference.json | 2 +- ingestion-data/staging/dataset-config/ida-ndvi.json | 4 ++-- .../staging/dataset-config/ida-ndwi-difference.json | 6 +++--- ingestion-data/staging/dataset-config/ida-ndwi.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json index 25238e1c..a46600b5 100644 --- a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json +++ b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json @@ -1,7 +1,7 @@ { "collection": "ida-ndvi-difference", "title": "NDVI Difference for Pre and Post-Hurricane Ida from PlanetScope", - "description": "Normalized Difference Vegetation Index Difference of before and after Hurricane Ida in southern Louisiana.", + "description": "Normalized Difference Vegetation Index Difference of before and after Hurricane Ida in Southern Louisiana.", "license": "CC0-1.0", "is_periodic": true, "time_density": "day", diff --git a/ingestion-data/staging/dataset-config/ida-ndvi.json b/ingestion-data/staging/dataset-config/ida-ndvi.json index 6c09e5fc..db49b360 100644 --- a/ingestion-data/staging/dataset-config/ida-ndvi.json +++ b/ingestion-data/staging/dataset-config/ida-ndvi.json @@ -1,7 +1,7 @@ { "collection": "ida-ndvi", "title": "NDVI for Pre and Post-Hurricane Ida from PlanetScope", - "description": "Normalized Difference Vegetation Index before and after Hurricane Ida in southern Louisiana.", + "description": "Normalized Difference Vegetation Index before and after Hurricane Ida in Southern Louisiana.", "license": "CC0-1.0", "is_periodic": true, "time_density": "day", @@ -27,7 +27,7 @@ "dry_run": false, "prefix": "planet-indices-v2/", "bucket": "veda-data-store-staging", - "filename_regex": "NDVI_(.*)_Hurricane_cog_(.*).tif$" + "filename_regex": "(.*)NDVI_(.*)_Hurricane_cog_(.*).tif$" } ], "data_type": "cog", diff --git a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json index 6c4f9c5d..416cb728 100644 --- a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json +++ b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json @@ -1,7 +1,7 @@ { "collection": "ida-ndwi-difference", "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope", - "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in southern Louisiana.", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", "license": "CC0-1.0", "is_periodic": true, "time_density": "day", @@ -16,7 +16,7 @@ "enddate": "2021-09-09T23:59:59Z" }, "sample_files": [ - "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_Hurricane_cog.tif" + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" ], "discovery_items": [ { @@ -26,7 +26,7 @@ "dry_run": false, "prefix": "planet-indices-v2/", "bucket": "veda-data-store-staging", - "filename_regex": "(.*)NDWI_Difference_Hurricane_cog.tif$" + "filename_regex": "(.*)NDWI_Difference_(.*).tif$" } ], "data_type": "cog", diff --git a/ingestion-data/staging/dataset-config/ida-ndwi.json b/ingestion-data/staging/dataset-config/ida-ndwi.json index fff1001b..a75ee1ca 100644 --- a/ingestion-data/staging/dataset-config/ida-ndwi.json +++ b/ingestion-data/staging/dataset-config/ida-ndwi.json @@ -1,7 +1,7 @@ { "collection": "ida-ndwi", "title": "NDWI for Pre and Post-Hurricane Ida from PlanetScope", - "description": "Normalized Difference Water Index before and after Hurricane Ida in southern Louisiana.", + "description": "Normalized Difference Water Index before and after Hurricane Ida in Southern Louisiana.", "license": "CC0-1.0", "is_periodic": true, "time_density": "day", @@ -27,7 +27,7 @@ "dry_run": false, "prefix": "planet-indices-v2/", "bucket": "veda-data-store-staging", - "filename_regex": "NDWI_(.*)_Hurricane_cog_(.*).tif$" + "filename_regex": "(.*)NDWI_(.*)_Hurricane_cog_(.*).tif$" } ], "data_type": "cog",