feat: git repo dbt (#70)

* feat: Adding custom dbt repo vars * feat: Adding custom dbt repo vars
bdg-tbd · Dec 22, 2023 · 4d79ffe · 4d79ffe
1 parent 3af9974
commit 4d79ffe
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/main.tf b/main.tf
@@ -12,6 +12,8 @@ locals {
   spark_blockmgr_port     = 30001
   dbt_version             = "1.7.3"
   dbt_spark_version       = "1.7.1"
+  dbt_git_repo            = "https://github.com/mwiewior/tbd-tpc-di.git"
+  dbt_git_repo_branch     = "main"
 }
 
 module "vpc" {
@@ -86,6 +88,8 @@ module "composer" {
     "AIRFLOW_VAR_BUCKET_NAME" : local.code_bucket_name
     "AIRFLOW_VAR_PHS_CLUSTER" : module.dataproc.dataproc_cluster_name,
     "AIRFLOW_VAR_WRK_NAMESPACE" : local.composer_work_namespace,
+    "AIRFLOW_VAR_DBT_GIT_REPO" : local.dbt_git_repo,
+    "AIRFLOW_VAR_DBT_GIT_REPO_BRANCH" : local.dbt_git_repo_branch
   }
 }
 

diff --git a/modules/data-pipeline/resources/dbt-dag.py b/modules/data-pipeline/resources/dbt-dag.py
@@ -51,7 +51,7 @@
         # entrypoint is used. The cmds parameter is templated.
         image_pull_policy="Always",
         cmds=["bash", "-c"],
-        arguments=["git clone https://github.com/mwiewior/tbd-tpc-di.git && cd tbd-tpc-di " 
+        arguments=["git clone {{ var.value.dbt_git_repo}} && cd tbd-tpc-di && git {{ var.value.dbt_git_repo_branch }}" 
                    "&& dbt deps && dbt run"],
         # The namespace to run within Kubernetes. In Composer 2 environments
         # after December 2022, the default namespace is