[Issue 821] Create database ERD diagrams from SQLAlchemy models (#824)

[Issue 821] Create database ERD diagrams from SQLAlchemy models --------- Co-authored-by: nava-platform-bot <[email protected]>
HHS · Dec 13, 2023 · 735e75c · 735e75c
1 parent 32f6656
commit 735e75c
Show file tree

Hide file tree

Showing 9 changed files with 182 additions and 7 deletions.
diff --git a/.github/workflows/ci-erd-diagrams.yml b/.github/workflows/ci-erd-diagrams.yml
@@ -0,0 +1,43 @@
+# Update database ERD diagrams so that they remain up to date with the application
+name: Update Database ERD Diagrams
+
+on:
+  pull_request:
+    paths:
+      - api/src/db/models/**
+      - api/bin/create_erds.py
+      - Makefile
+      - .github/workflows/ci-erd-diagrams.yml
+
+defaults:
+  run:
+    working-directory: ./api
+
+# Only trigger one update of the ERD diagrams at a time on the branch.
+# If new commits are pushed to the branch, cancel in progress runs and start
+# a new one.
+concurrency:
+  group: ${{ github.head_ref }}
+  cancel-in-progress: true
+
+
+jobs:
+  update-openapi-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Checkout the feature branch associated with the pull request
+          ref: ${{ github.head_ref }}
+
+      - name: Update OpenAPI spec
+        run: make create-erds
+
+      - name: Push changes
+        run: |
+          git config user.name nava-platform-bot
+          git config user.email [email protected]
+          git add --all
+          # Commit changes (if no changes then no-op)
+          git diff-index --quiet HEAD || git commit -m "Update database ERD diagrams"
+          git push
diff --git a/api/Dockerfile b/api/Dockerfile
@@ -41,6 +41,13 @@ RUN : "${RUN_USER:?RUN_USER and RUN_UID need to be set and non-empty.}" && \
 
 FROM base AS dev
 ARG RUN_USER
+
+# In between ARG RUN_USER and USER ${RUN_USER}, the user is still root
+# If there is anything that needs to be ran as root, this is the spot
+
+# Install graphviz which is used to generate ERD diagrams
+RUN apt-get update && apt-get install --no-install-recommends --yes graphviz
+
 USER ${RUN_USER}
 WORKDIR /api
 

diff --git a/api/Makefile b/api/Makefile
@@ -164,6 +164,10 @@ db-migrate-heads: ## Show migrations marked as a head
 db-seed-local:
 	$(PY_RUN_CMD) db-seed-local
 
+create-erds: # Create ERD diagrams for our DB schema
+	$(PY_RUN_CMD) create-erds
+	mv bin/*.png ../documentation/api/database/erds
+
 ##################################################
 # Testing
 ##################################################
@@ -190,22 +194,22 @@ test-coverage-report: ## Open HTML test coverage report
 ##################################################
 
 format: ## Format files
-	$(PY_RUN_CMD) isort --atomic src tests
-	$(PY_RUN_CMD) black src tests
+	$(PY_RUN_CMD) isort --atomic src tests bin
+	$(PY_RUN_CMD) black src tests bin
 
 format-check: ## Check file formatting
-	$(PY_RUN_CMD) isort --atomic --check-only src tests
-	$(PY_RUN_CMD) black --check src tests
+	$(PY_RUN_CMD) isort --atomic --check-only src tests bin
+	$(PY_RUN_CMD) black --check src tests bin
 
 lint: lint-py ## Lint
 
 lint-py: lint-flake lint-mypy
 
 lint-flake:
-	$(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) src tests
+	$(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) src tests bin
 
 lint-mypy:
-	$(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) src $(MYPY_POSTPROC)
+	$(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) src bin $(MYPY_POSTPROC)
 
 lint-security: # https://bandit.readthedocs.io/en/latest/index.html
 	$(PY_RUN_CMD) bandit -c pyproject.toml -r . --number 3 --skip B101 -ll -x ./.venv

diff --git a/api/bin/__init__.py b/api/bin/__init__.py
diff --git a/api/bin/create_erds.py b/api/bin/create_erds.py
@@ -0,0 +1,59 @@
+# Generate database schema diagrams from our SQLAlchemy models
+import codecs
+import logging
+import os
+import pathlib
+from typing import Any
+
+import pydot
+import sadisplay
+
+import src.logging
+from src.db.models import opportunity_models
+
+logger = logging.getLogger(__name__)
+
+# Construct the path to the folder this file is within
+# This gets an absolute path so that where you run the script from won't matter
+# and should always resolve to the app/erds folder
+ERD_FOLDER = pathlib.Path(__file__).parent.resolve()
+
+# If we want to generate separate files for more specific groups, we can set that up here
+ALL_MODULES = [opportunity_models]
+
+
+def create_erds(modules: Any, file_name: str) -> None:
+    logger.info("Generating ERD diagrams for %s", file_name)
+
+    items = []
+    for module in modules:
+        items.extend([getattr(module, attr) for attr in dir(module)])
+
+    description = sadisplay.describe(
+        items,
+        show_methods=True,
+        show_properties=True,
+        show_indexes=True,
+    )
+
+    dot_file_name = ERD_FOLDER / f"{file_name}.dot"
+
+    # We create a temporary .dot file which we then convert to a png
+    with codecs.open(str(dot_file_name), "w", encoding="utf8") as f:
+        f.write(sadisplay.dot(description))
+
+    (graph,) = pydot.graph_from_dot_file(dot_file_name)
+
+    png_file_path = ERD_FOLDER / f"{file_name}.png"
+    logger.info("Creating ERD diagram at %s", png_file_path)
+    graph.write_png(png_file_path)
+
+    # remove the temporary .dot file
+    os.remove(dot_file_name)
+
+
+def main() -> None:
+    with src.logging.init(__package__):
+        logger.info("Generating ERD diagrams")
+
+        create_erds(ALL_MODULES, "full-schema")
diff --git a/api/poetry.lock b/api/poetry.lock
diff --git a/api/pyproject.toml b/api/pyproject.toml
@@ -40,6 +40,8 @@ pytest-watch = "^4.2.0"
 pytest-lazy-fixture = "^0.6.3"
 types-pyyaml = "^6.0.12.11"
 setuptools = "^68.2.2"
+pydot = "1.4.2"
+sadisplay = "0.4.9"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -50,6 +52,7 @@ db-migrate = "src.db.migrations.run:up"
 db-migrate-down = "src.db.migrations.run:down"
 db-migrate-down-all = "src.db.migrations.run:downall"
 db-seed-local = "tests.lib.seed_local_db:seed_local_db"
+create-erds = "bin.create_erds:main"
 
 [tool.black]
 line-length = 100

diff --git a/documentation/api/database/erds/README.md b/documentation/api/database/erds/README.md
@@ -0,0 +1,17 @@
+# Overview
+This folder contains ERD diagrams representing our database schema for both our postgres DB
+
+Diagrams can be manually generated by running `make create-erds` from the api folder.
+
+# Dependencies
+If running outside of Docker, you must install `graphviz` (`brew install graphviz`) for this to work, this should be automatically installed as part of the Dockerfile inside Docker.
+
+# Caveats
+The diagrams generated are based on our SQLAlchemy models, and not the database itself, so there are a few differences.
+
+* Fields that we name different in-code will have a different name
+* The table names use the class name
+* Property fields are SQLAlchemy only and generally represent relationships (ie. values fetched via a foreign key `join`)
+
+# Files
+![Postgres ERD](full-schema.png)
diff --git a/documentation/api/database/erds/full-schema.png b/documentation/api/database/erds/full-schema.png