migrate apps to docker compose

apps/anyway/.env

@@ -0,0 +1,6 @@
+PIN_DB_IMAGE=ghcr.io/hasadna/anyway/db:sha-6dfd43b
+ANYWAY_IMAGE=ghcr.io/data-for-change/anyway/anyway:sha-e43df45
+AIRFLOW_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:v0.0.54
+ETL_NGINX_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:v0.0.54
+REPORTS_IMAGE=ghcr.io/data-for-change/anyway-reports/anyway-reports:sha-ff0aa3b
+NGINX_IMAGE=ghcr.io/data-for-change/anyway/nginx:sha-e43df45

apps/anyway/README.md

@@ -6,43 +6,41 @@ https://docs.google.com/presentation/d/1bXkcCgsXUr1FQA7hCZdb5_m7IXIiP1UixuOHuV88
 
 ![](image.png)
 
-## Initial Deployment
+## Install
 
-* Create secrets
-  * set env vars with the secret DB values
-    * `POSTGRES_PASSWORD=`
-    * `ANYWAY_PASSWORD=`
-    * `DBRESTORE_AWS_ACCESS_KEY_ID=`
-    * `DBRESTORE_AWS_SECRET_ACCESS_KEY=`
-    * `DBDUMP_AWS_ACCESS_KEY_ID=`
-    * `DBDUMP_AWS_SECRET_ACCESS_KEY=`
-  * create the DB secrets:
-    * `kubectl -n $NAMESPACE_NAME create secret generic anyway-db "--from-literal=DATABASE_URL=postgresql://anyway:${ANYWAY_PASSWORD}@db/anyway"`
-    * `kubectl -n $NAMESPACE_NAME create secret generic db "--from-literal=DBRESTORE_SET_ANYWAY_PASSWORD=${ANYWAY_PASSWORD}" "--from-literal=POSTGRES_PASSWORD=${POSTGRES_PASSWORD}" "--from-literal=DBRESTORE_AWS_ACCESS_KEY_ID=${DBRESTORE_AWS_ACCESS_KEY_ID}" "--from-literal=DBRESTORE_AWS_SECRET_ACCESS_KEY=${DBRESTORE_AWS_SECRET_ACCESS_KEY}"`
-    * `kubectl -n $NAMESPACE_NAME create secret generic db-backup "--from-literal=DBDUMP_AWS_ACCESS_KEY_ID=${DBDUMP_AWS_ACCESS_KEY_ID}" "--from-literal=DBDUMP_AWS_SECRET_ACCESS_KEY=${DBDUMP_AWS_SECRET_ACCESS_KEY}" "--from-literal=DBDUMP_PASSWORD=${POSTGRES_PASSWORD}"`
-  * Create the anyway secret (see the anyway production docker-compose for available values, or leave it empty just for basic testing)
-    * `kubectl -n $NAMESPACE_NAME create secret generic anyway`
+Set env vars for Vault access:
 
-## Deployment
-
-* For local deployment on Minikue - use Helm to deploy this chart with the values file `values-minikube.yaml`
-* For production deployment - Use ArgoCD, see [/docs/argocd.md](/docs/argocd.md) for details.
+```
+export VAULT_ADDR=
+export VAULT_TOKEN=
+```
 
-## Enabling the Airflow server
+Set secret values:
 
-Set the following values in `anyway` secret:
+```
+bin/render_env_template.py apps/anyway/secrets/anyway.env.template > apps/anyway/secrets/anyway.env
+bin/render_env_template.py apps/anyway/secrets/anyway-db.env.template > apps/anyway/secrets/anyway-db.env
+bin/render_env_template.py apps/anyway/secrets/db.env.template > apps/anyway/secrets/db.env
+bin/render_env_template.py apps/anyway/secrets/airflow-db.env.template > apps/anyway/secrets/airflow-db.env
+bin/render_env_template.py apps/anyway/secrets/airflow-scheduler.env.template > apps/anyway/secrets/airflow-scheduler.env
+bin/render_env_template.py apps/anyway/secrets/airflow-webserver.env.template > apps/anyway/secrets/airflow-webserver.env
+vault kv get -format=json kv/projects/anyway/prod/k8s-secret-anyway | jq -r '.data.data["GOOGLE_APPLICATION_CREDENTIALS_KEY.json"]' > apps/anyway/secrets/GOOGLE_APPLICATION_CREDENTIALS_KEY.json
+```
 
-* `AIRFLOW_DB_POSTGRES_PASSWORD`: Generate a password (`python3 -c 'import secrets; print(secrets.token_hex(16))'`)
-* `AIRFLOW_SQLALCHEMY_URL`: (replace AIRFLOW_DB_POSTGRES_PASSWORD with the password you generated) `postgresql://postgres:AIRFLOW_DB_POSTGRES_PASSWORD@airflow-db`
-* `AIRFLOW_ADMIN_PASSWORD`: Generate a password (`python3 -c 'import secrets; print(secrets.token_hex(16))'`)
+Run:
 
-Enable airflow by setting `enableAirflow: true` in the relevant environment's values
+```
+( cd apps/anyway && docker compose up -d )
+```
 
-Deploy
+### TODO: db-backup-cronjob
+### TODO: ingresses
+### TODO: airflow execute via kubectl exec - modify to execute in docker compose
+### TODO: check anyway nginx proxy and configurations - for new docker compose hostnames
 
 ## Enable DB Redash read-only user
 
-Start a shell on DB pod and run the following to start an sql session:
+Start a shell on DB container and run the following to start an sql session:
 
 ```
 su postgres

apps/anyway/compose.yaml

@@ -0,0 +1,146 @@
+x-anyway-command: &x-anyway-command
+  ["gunicorn", "-b", "0.0.0.0:5000", "-w", "4", "-t", "120", "anyway:app"]
+
+x-anyway-environment: &x-anyway-environment
+  PROXYFIX_X_FOR: "1"
+  PROXYFIX_X_PROTO: "1"
+  PROXYFIX_X_HOST: "1"
+  GOOGLE_APPLICATION_CREDENTIALS: "/secrets/GOOGLE_APPLICATION_CREDENTIALS_KEY.json"
+
+x-anyway: &x-anyway
+  image: ${ANYWAY_IMAGE:-ghcr.io/data-for-change/anyway/anyway:latest}
+  env_file:
+    - ./secrets/anyway.env
+    - ./secrets/anyway-db.env
+  volumes:
+    - ./secrets/GOOGLE_APPLICATION_CREDENTIALS_KEY.json:/secrets/GOOGLE_APPLICATION_CREDENTIALS_KEY.json:ro
+  restart: unless-stopped
+  networks: [dfc]
+
+services:
+  anyway-main:
+    <<: *x-anyway
+    command: *x-anyway-command
+    environment: *x-anyway-environment
+    depends_on:
+      - db
+
+  anyway-secondary:
+    <<: *x-anyway
+    entrypoint: *x-anyway-command
+    environment:
+      <<: *x-anyway-environment
+      ALLOW_ALEMBIC_UPGRADE: "no"
+    depends_on:
+      - anyway-main
+    networks: [dfc]
+    # we route all external traffic to the secondary container, to keep the main container free for airflow tasks
+    hostname: anyway
+
+  db:
+    hostname: anyway-db
+    image: ${PIN_DB_IMAGE:-ghcr.io/hasadna/anyway/db:latest}
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      # DBRESTORE_AWS_BUCKET: dfc-anyway-full-db-dumps
+      # DBRESTORE_FILE_NAME: 2024-01-24_anyway.pgdump
+    env_file:
+      - ./secrets/db.env
+    volumes:
+      - /data/anyway/db/dbdata:/var/lib/postgresql/data
+    tmpfs:
+      - /dev/shm:size=1024m
+    networks: [dfc]
+    ports:
+      - "9002:5432"
+
+  airflow-db:
+    image: postgres:13@sha256:6647385dd9ae11aa2216bf55c54d126b0a85637b3cf4039ef24e3234113588e3
+    restart: unless-stopped
+    env_file:
+      - ./secrets/airflow-db.env
+    volumes:
+      - /data/anyway/airflow-db/airflow_db:/var/lib/postgresql/data
+    networks: [dfc]
+
+  airflow-scheduler:
+    image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest}
+    restart: unless-stopped
+    environment:
+      ANYWAY_ETL_AIRFLOW_ROLE: "scheduler"
+      ANYWAY_ETL_AIRFLOW_PIP_INSTALL_DEPS: "yes"
+      ANYWAY_ETL_BRANCH: ""
+      ANYWAY_ETL_USE_LATEST_TAG: "yes"
+      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
+      AIRFLOW__WEBSERVER__BASE_URL: https://airflow.anyway.co.il
+    env_file:
+      - ./secrets/airflow-scheduler.env
+    volumes:
+      - /data/anyway/airflow-home-data:/var/airflow
+      - /data/anyway/airflow-etl-data:/var/anyway-etl-data
+    networks: [dfc]
+
+  airflow-nginx:
+    image: ${ETL_NGINX_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:latest}
+    restart: unless-stopped
+    volumes:
+      - /data/anyway/airflow-etl-data:/var/anyway-etl-data
+    networks: [dfc]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.airflow-nginx.loadbalancer.server.port=80"
+      - "traefik.http.routers.airflow-nginx.rule=Host(`airflow-data.anyway.co.il`)"
+      # - "traefik.http.routers.airflow-nginx.tls=true"
+      # - "traefik.http.routers.airflow-nginx.tls.certresolver=dfc"
+
+  airflow-webserver:
+    image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest}
+    restart: unless-stopped
+    environment:
+      ANYWAY_ETL_AIRFLOW_INITIALIZE: "yes"
+      ANYWAY_ETL_AIRFLOW_ROLE: "webserver"
+      AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "False"
+      AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth"
+    env_file:
+      - ./secrets/airflow-webserver.env
+    volumes:
+      - /data/anyway/airflow-home-data:/var/airflow
+    networks: [dfc]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.airflow-webserver.loadbalancer.server.port=8080"
+      - "traefik.http.routers.airflow-webserver.rule=Host(`airflow.anyway.co.il`)"
+      # - "traefik.http.routers.airflow-webserver.tls=true"
+      # - "traefik.http.routers.airflow-webserver.tls.certresolver=dfc"
+
+  reports:
+    hostname: anyway-reports
+    image: ${REPORTS_IMAGE:-ghcr.io/data-for-change/anyway-reports/anyway-reports:latest}
+    restart: unless-stopped
+    networks: [dfc]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.anyway-reports.loadbalancer.server.port=80"
+      - "traefik.http.routers.anyway-reports.rule=Host(`reports.anyway.co.il`)"
+      # - "traefik.http.routers.anyway-reports.tls=true"
+      # - "traefik.http.routers.anyway-reports.tls.certresolver=dfc"
+
+  nginx:
+    hostname: anyway-nginx
+    image: ${NGINX_IMAGE:-ghcr.io/data-for-change/anyway/nginx:latest}
+    restart: unless-stopped
+    volumes:
+      - ./nginx_anyway_proxy.conf:/etc/nginx/anyway_proxy.conf:ro
+    networks: [dfc]
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.anyway-nginx.loadbalancer.server.port=80"
+      - "traefik.http.routers.anyway-nginx.rule=Host(`www.anyway.co.il`)"
+      # - "traefik.http.routers.anyway-nginx.tls=true"
+      # - "traefik.http.routers.anyway-nginx.tls.certresolver=dfc"
+
+networks:
+  dfc:
+    external: true

apps/anyway/nginx_anyway_proxy.conf

@@ -0,0 +1,6 @@
+proxy_set_header Host $host;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto https;
+proxy_set_header X-Forwarded-Host www.anyway.co.il;
+proxy_pass http://anyway;
+proxy_redirect default;

apps/anyway/secrets/.gitignore

@@ -0,0 +1,2 @@
+*.env
+*.json

apps/anyway/secrets/airflow-db.env.template

@@ -0,0 +1 @@
+POSTGRES_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_DB_POSTGRES_PASSWORD~"

apps/anyway/secrets/airflow-scheduler.env.template

@@ -0,0 +1,11 @@
+AIRFLOW__CORE__SQL_ALCHEMY_CONN="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_SQLALCHEMY_URL~"
+SQLALCHEMY_URL="~vault:projects/anyway/prod/k8s-secret-anyway-db:DATABASE_URL~"
+IMAP_MAIL_USER="~vault:projects/anyway/prod/k8s-secret-anyway:MAILUSER~"
+IMAP_MAIL_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:MAILPASS~"
+AIRFLOW__EMAIL__EMAIL_BACKEND="airflow.utils.email.send_email_smtp"
+AIRFLOW__SMTP__SMTP_HOST="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_HOST~"
+AIRFLOW__SMTP__SMTP_PORT="2525"
+AIRFLOW__SMTP__SMTP_MAIL_FROM="Airflow <[email protected]>"
+AIRFLOW__SMTP__SMTP_USER="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_USER~"
+AIRFLOW__SMTP__SMTP_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_PASSWORD~"
+ANYWAY_ETL_ALERT_EMAILS="~vault:projects/anyway/prod/k8s-secret-anyway:ANYWAY_ETL_ALERT_EMAILS~"

apps/anyway/secrets/airflow-webserver.env.template

@@ -0,0 +1,2 @@
+AIRFLOW__CORE__SQL_ALCHEMY_CONN="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_SQLALCHEMY_URL~"
+ANYWAY_ETL_AIRFLOW_ADMIN_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_ADMIN_PASSWORD~"

apps/anyway/secrets/anyway-db.env.template

@@ -0,0 +1 @@
+DATABASE_URL="~vault:projects/anyway/prod/k8s-secret-anyway-db:DATABASE_URL~"

apps/anyway/secrets/anyway.env.template

@@ -0,0 +1,29 @@
+AIRFLOW_ADMIN_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_ADMIN_PASSWORD~"
+AIRFLOW_DB_POSTGRES_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_DB_POSTGRES_PASSWORD~"
+AIRFLOW_SQLALCHEMY_URL="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_SQLALCHEMY_URL~"
+AIRFLOW__SMTP__SMTP_HOST="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_HOST~"
+AIRFLOW__SMTP__SMTP_PORT="2525"
+AIRFLOW__SMTP__SMTP_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_PASSWORD~"
+AIRFLOW__SMTP__SMTP_USER="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW__SMTP__SMTP_USER~"
+ANYWAY_ETL_ALERT_EMAILS="~vault:projects/anyway/prod/k8s-secret-anyway:ANYWAY_ETL_ALERT_EMAILS~"
+APP_SECRET_KEY="~vault:projects/anyway/prod/k8s-secret-anyway:APP_SECRET_KEY~"
+AWS_ACCESS_KEY="~vault:projects/anyway/prod/aws_prod_app_user:access_key_id~"
+AWS_SECRET_KEY="~vault:projects/anyway/prod/aws_prod_app_user:secret_access_key~"
+FACEBOOK_KEY="~vault:projects/anyway/prod/k8s-secret-anyway:FACEBOOK_KEY~"
+FACEBOOK_SECRET="~vault:projects/anyway/prod/k8s-secret-anyway:FACEBOOK_SECRET~"
+FLASK_ENV="~vault:projects/anyway/prod/k8s-secret-anyway:FLASK_ENV~"
+GOOGLE_LOGIN_CLIENT_ID="~vault:projects/anyway/prod/k8s-secret-anyway:GOOGLE_LOGIN_CLIENT_ID~"
+GOOGLE_LOGIN_CLIENT_SECRET="~vault:projects/anyway/prod/k8s-secret-anyway:GOOGLE_LOGIN_CLIENT_SECRET~"
+GOOGLE_MAPS_KEY="~vault:projects/anyway/prod/k8s-secret-anyway:GOOGLE_MAPS_KEY~"
+MAILPASS="~vault:projects/anyway/prod/k8s-secret-anyway:MAILPASS~"
+MAILUSER="~vault:projects/anyway/prod/k8s-secret-anyway:MAILUSER~"
+SERVER_ENV="~vault:projects/anyway/prod/k8s-secret-anyway:SERVER_ENV~"
+SLACK_WEBHOOK_URL="~vault:projects/anyway/prod/k8s-secret-anyway:SLACK_WEBHOOK_URL~"
+TWITTER_ACCESS_KEY="~vault:projects/anyway/prod/k8s-secret-anyway:TWITTER_ACCESS_KEY~"
+TWITTER_ACCESS_SECRET="~vault:projects/anyway/prod/k8s-secret-anyway:TWITTER_ACCESS_SECRET~"
+TWITTER_CONSUMER_KEY="~vault:projects/anyway/prod/k8s-secret-anyway:TWITTER_CONSUMER_KEY~"
+TWITTER_CONSUMER_SECRET="~vault:projects/anyway/prod/k8s-secret-anyway:TWITTER_CONSUMER_SECRET~"
+SELENIUM_URL="~vault:projects/anyway/prod/k8s-secret-anyway:SELENIUM_URL~"
+BOT_TOKEN="~vault:projects/anyway/prod/k8s-secret-anyway:BOT_TOKEN~"
+AIRFLOW_USER="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_USER~"
+AIRFLOW_PASSWORD="~vault:projects/anyway/prod/k8s-secret-anyway:AIRFLOW_PASSWORD~"

apps/anyway/secrets/db.env.template

@@ -0,0 +1,4 @@
+DBRESTORE_AWS_ACCESS_KEY_ID="~vault:projects/anyway/prod/aws_db_dumps_reader_user:access_key_id~"
+DBRESTORE_AWS_SECRET_ACCESS_KEY="~vault:projects/anyway/prod/aws_db_dumps_reader_user:secret_access_key~"
+DBRESTORE_SET_ANYWAY_PASSWORD="~vault:projects/anyway/prod/k8s-secret-db:DBRESTORE_SET_ANYWAY_PASSWORD~"
+POSTGRES_PASSWORD="~vault:projects/anyway/prod/k8s-secret-db:POSTGRES_PASSWORD~"