Templatize via Cookiecutter for enhanced audience utility (#10)

* add cookiecutter json defaults

* add cookiecutter dependency

* utilize cookiecutter values within tf and docs

* remove comments and final comma

* move to template directory structure

* new default values

* update tests for template

* lint tf readme

* update template test to run git init for lint chks

* update template readme

* remove irrelevant pre-commit hooks for template

* Update readme.md

* Update readme.md

* test workflow on github

* init git name and email

* move from gh to dagger action for template test

* formatting

* add testing notes

* linting

* formatting for readability

* remove dev branch for gh actions testing

* specifics in action docs

* json linting additions + application

* add gh actions for pre-commit checks on template

* move to using root terraform.tfvars file

Co-Authored-By: Faisal Alquaddoomi <[email protected]>

* Revert "move to using root terraform.tfvars file"

This reverts commit 855b6a4.

* update versions of tf

* remove no longer applicable files

* update to use variables.tfvars

Co-Authored-By: Faisal Alquaddoomi <[email protected]>

* update readme content

* rename and add tfvars where appropriate

* update template tf readme files

* dagger testing updates for corrections

* remove readme for testing

* simplify and build distinct tests

* readd readme

* update tests to be inclusive as a group

* formatting for cue file

* lint cuefile

* ignore readmes under template terraform

* ignore template md's

* update to use appropriate names for state mgmt

* add mock gcp credentials for testing

* formatting and ignoring mock key for checks

* Updated references to variables.tfvars with the new filename, terraform.tfvars. Updated links to that file in the state mangement, infrastructure sections.

---------

Co-authored-by: Faisal Alquaddoomi <[email protected]>
Co-authored-by: Faisal Alquaddoomi <[email protected]>

.github/workflows/pre-commit-checks.yml

@@ -0,0 +1,19 @@
+---
+# used for running pre-commit checks
+name: pre-commit checks
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  run_pre_commit_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: pre-commit/[email protected]

.github/workflows/test.yml

@@ -1,4 +1,6 @@
 ---
+# note: this workflow definition will create a project from
+# the cookiecutter template to test the results through dagger
 name: run tests
 
 on:

.gitignore

@@ -213,3 +213,6 @@ hashicorp/
 
 # tf credential ignore
 utilities/data-provider/service-account.json
+
+# tests
+tests/

.pre-commit-config.yaml

@@ -8,10 +8,14 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+      - id: check-json
+      - id: pretty-format-json
+        args: [--autofix, --no-sort-keys]
       - id: check-yaml
       - id: check-added-large-files
       - id: check-toml
       - id: detect-private-key
+        exclude: "tests/data/gcp-mock-credentials.json"
   # checking yaml formatting
   - repo: https://github.com/adrienverge/yamllint
     rev: v1.32.0
@@ -31,21 +35,4 @@ repos:
     rev: 0.7.17
     hooks:
       - id: mdformat
-        # ignore terraform autogenerated docs
-        exclude: >
-              (?x)^(
-                  terraform/.* |
-                  terraform/.*/.*
-              )$
-  # linting for terraform
-  - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.3
-    hooks:
-      - id: terraform_docs
-        args:
-          - --hook-config=--path-to-file=README.md
-          - --hook-config=--add-to-existing-file=true
-          - --hook-config=--create-file-if-not-exist=true
-      - id: terraform_fmt
-      - id: terraform_tflint
-      - id: terraform_tfsec
+        exclude: "{{ cookiecutter.project_name }}/.*/.*"

cookiecutter.json

@@ -0,0 +1,8 @@
+{
+  "project_name": "lab-initiative-bucket",
+  "project_description": "Project for creating and managing a Cloud Storage bucket on Google Cloud.",
+  "project_author": "Username",
+  "project_license": "BSD-3-Clause License",
+  "project_gc_project": "cuhealthai-sandbox",
+  "project_gc_region": "us-central1"
+}

project.cue

@@ -57,7 +57,7 @@ import "universe.dagger.io/docker"
 	output: _tf_build.output
 
 	// tf build
-	_tf_pre_build: docker.#Build & {
+	_tf_build: docker.#Build & {
 		steps: [
 			docker.#Pull & {
 				source: "ghcr.io/antonbabenko/pre-commit-terraform:v1.83.3"
@@ -67,64 +67,39 @@ import "universe.dagger.io/docker"
 					workdir: "/lint"
 				}
 			},
-			// git init for pre-commit caching
-			bash.#Run & {
-				script: contents: """
-					    git init
-					"""
-			},
-			docker.#Copy & {
-				contents: filesystem
-				source:   "./.pre-commit-config.yaml"
-				dest:     "/lint/.pre-commit-config.yaml"
-			},
-			docker.#Run & {
-				command: {
-					name: "install-hooks"
-				}
-			},
-		]
-	}
 
-	// cue build for actions in this plan
-	_tf_build: docker.#Build & {
-		steps: [
 			docker.#Copy & {
-				input:    _tf_pre_build.output
 				contents: filesystem
 				source:   "./"
-				dest:     "/lint"
-				exclude: ["./.pre-commit-config.yaml"]
-			},
-		]
-	}
-
-}
-
-// Convenience terraform build for implementation
-#TerraformBuild: {
-	// client filesystem
-	filesystem: dagger.#FS
-
-	// output from the build
-	output: _tf_build.output
-
-	// tf build
-	_tf_build: docker.#Build & {
-		steps: [
-			docker.#Pull & {
-				source: "hashicorp/terraform:1.4.6"
+				dest:     "/workdir"
 			},
-			docker.#Run & {
-				command: {
-					name: "mkdir"
-					args: ["/workdir"]
-				}
-			},
-			docker.#Copy & {
-				contents: filesystem
-				source:   "./"
-				dest:     "/workdir/"
+			bash.#Run & {
+				script: contents: """
+					# cd into the workdir
+					cd /workdir
+
+					# remove already existing test content
+					rm -rf ./tests/lab-initiative-bucket
+
+					# install poetry and env
+					python3 -m pip install --no-cache-dir --upgrade poetry
+					poetry install --no-interaction --no-ansi
+
+					# run cookiecutter to create project from template
+					poetry run cookiecutter . --no-input --output-dir tests
+
+					# move project from template into lintable dir for container
+					cp -ra /workdir/tests/lab-initiative-bucket/. /lint
+
+					# reinit git for the cookiecutter project
+					rm -rf /lint/.git
+					cd /lint
+					git config --global user.email "[email protected]"
+					git config --global user.name "Your Name"
+					git init
+					git add .
+					git commit -m "example message"
+					"""
 			},
 		]
 	}
@@ -147,11 +122,6 @@ dagger.#Plan & {
 			filesystem: client.filesystem."./".read.contents
 		}
 
-		// an internal terraform build for use with this repo
-		_tf_build: #TerraformBuild & {
-			filesystem: client.filesystem."./".read.contents
-		}
-
 		// an internal terraform build for use with this repo
 		_tf_lint_build: #TFLintBuild & {
 			filesystem: client.filesystem."./".read.contents
@@ -173,16 +143,38 @@ dagger.#Plan & {
 			}
 		}
 
-		// various tests for this repo
 		test: {
 			// run pre-commit checks
-			test_pre_commit: docker.#Run & {
+			test_pre_commit: bash.#Run & {
 				input: _tf_lint_build.output
-				command: {
-					name: "run"
-					args: ["--all-files"]
-				}
+				script: contents: """
+						pre-commit run -a
+					"""
+			}
+
+			// run pre-commit checks
+			test_tfvars: bash.#Run & {
+				input: _tf_lint_build.output
+				script: contents: """
+						# change dir to where the cookiecutter created project lives
+						# to simulate the use of the directory after it's been used
+						cd /lint
+
+						# set terraform to use mock credentials for testing
+						export GOOGLE_APPLICATION_CREDENTIALS=/workdir/tests/data/gcp-mock-credentials.json
+
+						# initialize terraform for plan
+						terraform -chdir=terraform/state-management init
+
+						# run plan without explicit input from cli
+						# note: we expect variables to be inherited from terraform.tfvars or similar
+						# this command will fail when unable to read a related tfvars file
+						# see the following for more info:
+						# https://developer.hashicorp.com/terraform/language/values/variables#variable-definition-precedence
+						terraform -chdir=terraform/state-management plan -input=false
+					"""
 			}
 		}
 	}
+
 }

pyproject.toml

@@ -3,12 +3,13 @@ name = "gc-cloud-storage-bucket"
 version = "0.0.1"
 description = "Template for creating Cloud Storage bucket on Google Cloud."
 authors = ["d33bs <[email protected]>"]
-license = "BSD-3-Clause license"
+license = "BSD-3-Clause License"
 readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.10"
 pre-commit = "^3.0.0"
+cookiecutter = "^2.3.0"
 
 
 [build-system]

readme.md

@@ -0,0 +1,66 @@
+# Google Cloud - Cloud Storage Bucket - Cookiecutter Template
+
+```mermaid
+flowchart LR
+    subgraph upload
+      files[("file(s)")]
+      data-provider["👤 Data Provider"]
+
+    end
+    subgraph download
+      data-receiver["👤 Data Receiver"]
+      files2[("file(s)")]
+    end
+    bucket[\"Cloud Storage\n Bucket"/]
+    files --> |to| bucket
+    data-provider --> |uploads| files
+    bucket --> |provides\naccess to| files2
+    files2 --> |received by| data-receiver
+
+```
+
+This project is a [Cookiecutter](https://github.com/cookiecutter/cookiecutter) template for creating a [Cloud Storage](https://cloud.google.com/storage/) bucket on [Google Cloud](https://cloud.google.com/) with a service account and related key to enable data or file upload and use.
+
+The template uses [Terraform](https://developer.hashicorp.com/terraform/intro) to maintain cloud resources. See Terraform readme's under the `terraform` directory for documentation on Terraform elements.
+
+## ✍️ Using the template
+
+Install the latest version of Cookiecutter:
+
+```shell
+pip install cookiecutter
+```
+
+Generate a Python package project from the template in this repository:
+
+```shell
+python -m cookiecutter https://github.com/CU-DBMI/gc-cloud-storage-bucket.git
+```
+
+## 🧑‍💻 Development
+
+Development for this repository is assisted by the following technologies:
+
+- [Poetry](https://python-poetry.org/docs/): Used to help configure pre-commit for local development work.
+- [Pre-commit](https://pre-commit.com): Used for performing checks within local development environment and via Github Actions automated testing. The following sub-items are used as checks through [pre-commit-terraform](https://github.com/antonbabenko/pre-commit-terraform) and require local installation when testing outside of Dagger:
+  - [terraform_docs](https://github.com/terraform-docs/terraform-docs/): Used to automatically generate Terraform-specific documentation.
+  - [tflint](https://github.com/terraform-linters/tflint): Used to perform static analysis (linting) on Terraform content.
+  - [tfsec](https://github.com/aquasecurity/tfsec): Used to perform security-focused static analysis (security linting) on Terraform content.
+- [Dagger](https://docs.dagger.io/): Used to help orchestrate reproducible testing within local development environment and for automated testing.
+
+### Testing
+
+Tests for this project help ensure projects implemented from the Cookiecutter template function as expected.
+These tests automatically will use the content found under the directory `{{ cookiecutter.project_name }}` to create a project and check the content.
+
+Use [`dagger-cue`](https://docs.dagger.io/sdk/cue/) to run the tests.
+These tests are also used by a Github Actions workflow to perform the same checks.
+
+Example test command:
+
+```shell
+# update dagger project (only if needed)
+dagger-cue project update
+# run the tests
+dagger-cue do test
+```