Added cloudbuild, tx-id for appender

clingen-data-model · May 2, 2024 · 83e790b · 83e790b
1 parent bb16b8a
commit 83e790b
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 2 deletions.
diff --git a/.cloudbuild/cloudbuild-prod.yaml b/.cloudbuild/cloudbuild-prod.yaml
@@ -0,0 +1,26 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'build', '.', '-t', 'genegraph-gene-validity:$COMMIT_SHA']
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'tag', 'genegraph-gene-validity:$COMMIT_SHA',
+  'us-east1-docker.pkg.dev/clingen-dx/genegraph-prod/genegraph-gene-validity:$COMMIT_SHA']  
+- name: 'gcr.io/clingen-dx/git-image-updater'
+  secretEnv: ["GITHUB_TOKEN"]
+  args:
+    - '-c'
+    - |
+      git clone https://clingen-ci:[email protected]/clingen-data-model/architecture \
+      && cd architecture \
+      && /usr/bin/yq eval -i ".docker_image_tag = \"$COMMIT_SHA\"" ./helm/values/genegraph-gene-validity/values-prod.yaml \
+      && date "+%Y-%m-%dT%H%M" > /workspace/DATETIME.txt \
+      && git add -u \
+      && git -c user.name="Clingen CI Automation" -c user.email="[email protected]" commit -m "bumping docker image for prod genegraph-gene-validity" \
+      && git push origin master
+
+images:
+- 'us-east1-docker.pkg.dev/clingen-dx/genegraph-prod/genegraph-gene-validity:$COMMIT_SHA'
+
+availableSecrets:
+  secretManager:
+  - versionName: projects/clingen-dx/secrets/clingen-ci-github-token/versions/1
+    env: GITHUB_TOKEN
diff --git a/.cloudbuild/cloudbuild-stage.yaml b/.cloudbuild/cloudbuild-stage.yaml
@@ -0,0 +1,26 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'build', '.', '-t', 'genegraph-gene-validity:$COMMIT_SHA']
+- name: 'gcr.io/cloud-builders/docker'
+  args: [ 'tag', 'genegraph-gene-validity:$COMMIT_SHA',
+  'us-east1-docker.pkg.dev/clingen-stage/genegraph-stage/genegraph-gene-validity:$COMMIT_SHA']  
+- name: 'gcr.io/clingen-stage/git-image-updater'
+  secretEnv: ["GITHUB_TOKEN"]
+  args:
+    - '-c'
+    - |
+      git clone https://clingen-ci:[email protected]/clingen-data-model/architecture \
+      && cd architecture \
+      && /usr/bin/yq eval -i ".docker_image_tag = \"$COMMIT_SHA\"" ./helm/values/genegraph-gene-validity/values-stage.yaml \
+      && date "+%Y-%m-%dT%H%M" > /workspace/DATETIME.txt \
+      && git add -u \
+      && git -c user.name="Clingen CI Automation" -c user.email="[email protected]" commit -m "bumping docker image for stage genegraph-gene-validity" \
+      && git push origin master
+
+images:
+- 'us-east1-docker.pkg.dev/clingen-stage/genegraph-stage/genegraph-gene-validity:$COMMIT_SHA'
+
+availableSecrets:
+  secretManager:
+  - versionName: projects/clingen-stage/secrets/clingen-ci-github-token/versions/2
+    env: GITHUB_TOKEN
diff --git a/src/genegraph/gene_validity.clj b/src/genegraph/gene_validity.clj
@@ -86,7 +86,9 @@
                    :gene-validity-sepio-topic "gg-gvs-stage-1"
                    :base-data-topic "gg-base-stage-1"
                    :gv-transform-tx-id "gg-gv-transform-stage"
-                   :fetch-base-tx-id "gg-fetch-base-stage")
+                   :fetch-base-tx-id "gg-fetch-base-stage"
+                   :appender-legacy-tx-id "gg-legacy-appender-stage"
+                   :appender-raw-tx-id "gg-raw-appender-stage")
     "prod" (assoc (env/build-environment "974091131481" ["dataexchange-genegraph"])
                   :function (System/getenv "GENEGRAPH_FUNCTION")
                   :kafka-user "User:2592237"
@@ -103,7 +105,9 @@
                   :gene-validity-sepio-topic "gg-gvs-prod-1"
                   :base-data-topic "gg-base-prod-1"
                   :gv-transform-tx-id "gg-gv-transform-prod"
-                  :fetch-base-tx-id "gg-fetch-base-prod")
+                  :fetch-base-tx-id "gg-fetch-base-prod"
+                  :appender-legacy-tx-id "gg-legacy-appender-prod"
+                  :appender-raw-tx-id "gg-raw-appender-prod")
     {}))
 
 (def env
@@ -702,12 +706,14 @@
                  :type :processor
                  :subscribe :gene-validity-raw
                  :kafka-cluster :data-exchange
+                 :kafka-transactional-id (:appender-raw-tx-id env)
                  :interceptors [append-gene-validity-raw]}
                 :gene-validity-legacy-appender
                 {:name :gene-validity-legacy-appender
                  :type :processor
                  :subscribe :gene-validity-legacy
                  :kafka-cluster :data-exchange
+                 :kafka-transactional-id (:appender-legacy-tx-id env)
                  :interceptors [append-gene-validity-legacy]}}
    :http-servers gv-ready-server})