Data Hub: CSV and Web API: Configure default config

Depends on elifesciences/data-hub-core-airflow-dags#1449 The schedule for those pipelines will need to be configured via the yaml config instead.
elifesciences · Jun 18, 2024 · 9057556 · 9057556
1 parent 03954da
commit 9057556
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 0 deletions.
diff --git a/kustomizations/apps/data-hub-configs/s3-csv-data-pipeline.config.yaml b/kustomizations/apps/data-hub-configs/s3-csv-data-pipeline.config.yaml
@@ -3,6 +3,14 @@ importedTimestampFieldName: "imported_timestamp"
 stateFile:
   defaultBucketName: "{ENV}-elife-data-pipeline"
   defaultSystemGeneratedObjectPrefix: "airflow-config/{ENV}-s3-csv/state/s3-csv"
+
+defaultConfig:
+  airflow:
+    dagParameters:
+      schedule: '*/30 * * * *'  # At every 30th minute
+      tags:
+        - 'CSV'
+
 s3Csv:
   - bucketName: elife-ejp-ftp
     objectKeyPattern:

diff --git a/kustomizations/apps/data-hub-configs/web-api-data-pipeline.config.yaml b/kustomizations/apps/data-hub-configs/web-api-data-pipeline.config.yaml
@@ -1,5 +1,13 @@
 gcpProjectName: 'elife-data-pipeline'
 importedTimestampFieldName: 'imported_timestamp'
+
+defaultConfig:
+  airflow:
+    dagParameters:
+      schedule: '25 2 * * *'  # At 02:25
+      tags:
+        - 'Web API'
+
 webApi:
 
   # Observer API