From db316e448c6417c97da9e1a39187b3b6eb5e8cb3 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 2 Sep 2020 22:06:49 +0100 Subject: [PATCH 001/231] :bug: Search for PRs from the current token user Don't assume the default github actions token before checking if it is a user PAT --- .github/workflows/test-apply.yaml | 29 +++++++++++++++++++++++++++++ image/tools/github_pr_comment.py | 14 +++++++++++++- tests/terraform-cloud/main.tf | 2 +- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 34e74f37..aa97017a 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -343,3 +343,32 @@ jobs: echo "Apply did not fail correctly" exit 1 fi + + apply_user_token: + runs-on: ubuntu-latest + name: Apply using a personal access token + env: + GITHUB_TOKEN: ${{ secrets.USER_GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + label: User PAT + path: tests/apply/changes + + - name: Apply + uses: ./terraform-apply + id: output + with: + label: User PAT + path: tests/apply/changes + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index dbc8f3b5..4782d1a8 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -70,6 +70,18 @@ def find_pr() -> str: else: raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") +def current_user() -> str: + response = github.get('https://api.github.com/user') + if response.status_code != 403: + user = response.json() + debug('GITHUB_TOKEN user:') + debug(json.dumps(user)) + + return user['login'] + + # Assume this is the github actions app token + return 'github-actions[bot]' + class TerraformComment: """ The GitHub comment for this specific terraform plan @@ -90,7 +102,7 @@ def __init__(self, pr_url: str): debug('Looking for an existing comment:') for comment in response.json(): debug(json.dumps(comment)) - if comment['user']['login'] == 'github-actions[bot]': + if comment['user']['login'] == current_user(): match = re.match(rf'{re.escape(self._comment_identifier)}\n```(.*?)```(.*)', comment['body'], re.DOTALL) if not match: diff --git a/tests/terraform-cloud/main.tf b/tests/terraform-cloud/main.tf index b11cfa57..629aa747 100644 --- a/tests/terraform-cloud/main.tf +++ b/tests/terraform-cloud/main.tf @@ -6,7 +6,7 @@ terraform { prefix = "github-actions-" } } - required_version = "~> 0.13.0" + required_version = "0.13.0" } resource "random_id" "the_id" { From 739ac17fc5af8e6d1039c85da1e61bf794c4b033 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 2 Sep 2020 23:15:30 +0100 Subject: [PATCH 002/231] :bookmark: v1.4.2 --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b930c622..8fdaea48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.4.1` to use an exact release +- `@v1.4.2` to use an exact release - `@v1.4` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.4.2] - 2020-09-02 + +### Fixed +- Using a personal access token instead of the Actions provided token now works. + This can be used to customise the PR comment author + ## [1.4.1] - 2020-08-11 ### Fixed @@ -74,6 +80,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.4.2]: https://github.com/dflook/terraform-github-actions/compare/v1.4.1...v1.4.2 [1.4.1]: https://github.com/dflook/terraform-github-actions/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/dflook/terraform-github-actions/compare/v1.3.1...v1.4.0 [1.3.1]: https://github.com/dflook/terraform-github-actions/compare/v1.3.0...v1.3.1 From 1dd0dee57a89c0e49a3397caef582b78aba98988 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 17 Sep 2020 17:46:09 +0100 Subject: [PATCH 003/231] Add HCL highlighting to PR comment --- image/tools/github_pr_comment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 4782d1a8..a304866d 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -103,7 +103,7 @@ def __init__(self, pr_url: str): for comment in response.json(): debug(json.dumps(comment)) if comment['user']['login'] == current_user(): - match = re.match(rf'{re.escape(self._comment_identifier)}\n```(.*?)```(.*)', comment['body'], re.DOTALL) + match = re.match(rf'{re.escape(self._comment_identifier)}\n```(?:hcl)?(.*?)```(.*)', comment['body'], re.DOTALL) if not match: match = re.match(rf'{re.escape(self._old_comment_identifier)}\n```(.*?)```(.*)', comment['body'], re.DOTALL) @@ -244,7 +244,7 @@ def status(self, status: str) -> None: self._status = status.strip() def update_comment(self): - body = f'{self._comment_identifier}\n```\n{self.plan}\n```' + body = f'{self._comment_identifier}\n```hcl\n{self.plan}\n```' if self.status: body += '\n' + self.status From 08e1dbe0bc6fb4886b969ec7cec2a444162463b1 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 17 Sep 2020 18:02:15 +0100 Subject: [PATCH 004/231] :bookmark: v.1.5.0 --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fdaea48..99edbe9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.4.2` to use an exact release -- `@v1.4` to use the latest patch release for the specific minor version +- `@v1.5.0` to use an exact release +- `@v1.5` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.5.0] - 2020-09-18 + +### Added +- PR comments use HCL highlighting + ## [1.4.2] - 2020-09-02 ### Fixed @@ -80,6 +85,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.5.0]: https://github.com/dflook/terraform-github-actions/compare/v1.4.2...v1.5.0 [1.4.2]: https://github.com/dflook/terraform-github-actions/compare/v1.4.1...v1.4.2 [1.4.1]: https://github.com/dflook/terraform-github-actions/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/dflook/terraform-github-actions/compare/v1.3.1...v1.4.0 From b7eddb06fa7bbfb52de869124f0c338f954b75dd Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 27 Oct 2020 20:42:59 +0000 Subject: [PATCH 005/231] Publish image to github container registry --- .github/workflows/release.yaml | 11 ++++++----- image/Dockerfile | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index decf410f..f37c86eb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest name: Publish Docker Image env: - GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }} DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} steps: - name: Checkout @@ -17,7 +17,7 @@ jobs: - name: Registry login run: | - echo $GITHUB_TOKEN | docker login docker.pkg.github.com -u dflook --password-stdin + echo $GITHUB_TOKEN | docker login ghcr.io -u dflook --password-stdin echo $DOCKER_TOKEN | docker login --username danielflook --password-stdin - name: Action image @@ -28,14 +28,15 @@ jobs: docker build --tag dflook/terraform-github-actions \ --label org.opencontainers.image.created="$(date '+%Y-%m-%dT%H:%M:%S%z')" \ - --label org.opencontainers.image.source="github.com/${{ github.repository }}.git" \ + --label org.opencontainers.image.source="https://github.com/${{ github.repository }}" \ --label org.opencontainers.image.revision="${{ github.sha }}" \ + --label org.opencontainers.image.version="$RELEASE_TAG" \ --label vcs-ref="$RELEASE_TAG" \ --label build="$GITHUB_RUN_ID" \ image - docker tag dflook/terraform-github-actions docker.pkg.github.com/dflook/terraform-github-actions/action:$RELEASE_TAG - docker push docker.pkg.github.com/dflook/terraform-github-actions/action:$RELEASE_TAG + docker tag dflook/terraform-github-actions ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG + docker push ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG docker tag dflook/terraform-github-actions danielflook/terraform-github-actions:$RELEASE_TAG docker push danielflook/terraform-github-actions:$RELEASE_TAG diff --git a/image/Dockerfile b/image/Dockerfile index 528edb2c..1ca12f17 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -10,3 +10,5 @@ COPY tools/convert_output.py /usr/local/bin/convert_output COPY tools/plan_cmp.py /usr/local/bin/plan_cmp COPY tools/convert_version.py /usr/local/bin/convert_version COPY tools/workspace_exists.py /usr/local/bin/workspace_exists + +LABEL org.opencontainers.image.title="GitHub actions for terraform" From f2ef7c7d23907bd67aaa13b4b536d52367bbe81d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Dec 2020 21:14:54 +0000 Subject: [PATCH 006/231] Add plan PR comment tests for different terraform versions --- .github/workflows/test-plan.yaml | 98 +++++++++++++++++++++++++++++++- README.md | 2 +- tests/plan/plan_11/main.tf | 7 +++ tests/plan/plan_12/main.tf | 7 +++ tests/plan/plan_13/main.tf | 7 +++ tests/plan/plan_14/main.tf | 7 +++ 6 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 tests/plan/plan_11/main.tf create mode 100644 tests/plan/plan_12/main.tf create mode 100644 tests/plan/plan_13/main.tf create mode 100644 tests/plan/plan_14/main.tf diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 52871f65..0cd9f738 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -42,7 +42,103 @@ jobs: path: tests/plan/no_changes add_github_comment: false - plan_change_comment: + plan_change_comment_11: + runs-on: ubuntu-latest + name: Change + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_11 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + plan_change_comment_12: + runs-on: ubuntu-latest + name: Change + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_12 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + plan_change_comment_13: + runs-on: ubuntu-latest + name: Change + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_13 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + plan_change_comment_14: + runs-on: ubuntu-latest + name: Change + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_14 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + plan_change_comment_latest: runs-on: ubuntu-latest name: Change env: diff --git a/README.md b/README.md index 0083ac91..909a9aba 100644 --- a/README.md +++ b/README.md @@ -299,4 +299,4 @@ jobs: ## What if I don't use GitHub Actions? If you use CircleCI, check out OVO Energy's [`ovotech/terraform`](https://github.com/ovotech/circleci-orbs/tree/master/terraform) CircleCI orb. -If you use Jenkins, you have my sympathy. \ No newline at end of file +If you use Jenkins, you have my sympathy. diff --git a/tests/plan/plan_11/main.tf b/tests/plan/plan_11/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/plan/plan_11/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} diff --git a/tests/plan/plan_12/main.tf b/tests/plan/plan_12/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/plan/plan_12/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} diff --git a/tests/plan/plan_13/main.tf b/tests/plan/plan_13/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/plan/plan_13/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} diff --git a/tests/plan/plan_14/main.tf b/tests/plan/plan_14/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/plan/plan_14/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} From 7eb95ffd281d9a26f3529e3c51faffd6cf384a35 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Dec 2020 22:02:27 +0000 Subject: [PATCH 007/231] Fix version test --- .github/workflows/test-version.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-version.yaml b/.github/workflows/test-version.yaml index 7def7eab..2d6df343 100644 --- a/.github/workflows/test-version.yaml +++ b/.github/workflows/test-version.yaml @@ -113,8 +113,8 @@ jobs: - name: Check the version run: | - if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"0.13"* ]]; then - echo "::error:: Terraform version not set from required_version" + if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"0.14"* ]]; then + echo "::error:: Latest version was not used" exit 1 fi From f107591961c0c2f187a446403fdf228edff98947 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Dec 2020 22:03:29 +0000 Subject: [PATCH 008/231] Better compact_plan script sed wasn't cutting it with terraform 0.14 changes --- image/Dockerfile | 1 + image/entrypoints/apply.sh | 2 +- image/entrypoints/plan.sh | 2 +- image/tools/compact_plan.py | 25 +++++++++++++++++++++++++ tests/plan/plan_11/main.tf | 4 ++++ tests/plan/plan_12/main.tf | 4 ++++ tests/plan/plan_13/main.tf | 4 ++++ tests/plan/plan_14/main.tf | 4 ++++ 8 files changed, 44 insertions(+), 2 deletions(-) create mode 100755 image/tools/compact_plan.py diff --git a/image/Dockerfile b/image/Dockerfile index 1ca12f17..079176d4 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -10,5 +10,6 @@ COPY tools/convert_output.py /usr/local/bin/convert_output COPY tools/plan_cmp.py /usr/local/bin/plan_cmp COPY tools/convert_version.py /usr/local/bin/convert_version COPY tools/workspace_exists.py /usr/local/bin/workspace_exists +COPY tools/compact_plan.py /usr/local/bin/compact_plan LABEL org.opencontainers.image.title="GitHub actions for terraform" diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index a8c0a386..5a309719 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -38,7 +38,7 @@ function plan() { 2>"$PLAN_DIR/error.txt" \ | $TFMASK \ | tee /dev/fd/3 \ - | sed '1,/---/d' \ + | compact_plan \ >"$PLAN_DIR/plan.txt" PLAN_EXIT=${PIPESTATUS[0]} diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 2f85b6b4..a9f42e3d 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -19,7 +19,7 @@ set +e 2>"$PLAN_DIR/error.txt" \ | $TFMASK \ | tee /dev/fd/3 \ - | sed '1,/---/d' \ + | compact_plan \ >"$PLAN_DIR/plan.txt" readonly TF_EXIT=${PIPESTATUS[0]} diff --git a/image/tools/compact_plan.py b/image/tools/compact_plan.py new file mode 100755 index 00000000..a5a78a4f --- /dev/null +++ b/image/tools/compact_plan.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 + +import sys + +if __name__ == '__main__': + plan = False + buffer = [] + + for line in sys.stdin.readlines(): + + if not plan and ( + line.startswith('An execution plan has been generated and is shown below') or + line.startswith('No changes') or + line.startswith('Error') + ): + plan = True + + if plan: + sys.stdout.write(line) + else: + buffer.append(line) + + if not plan and buffer: + for line in buffer: + sys.stdout.write(line) diff --git a/tests/plan/plan_11/main.tf b/tests/plan/plan_11/main.tf index dee08246..a5a0f32f 100644 --- a/tests/plan/plan_11/main.tf +++ b/tests/plan/plan_11/main.tf @@ -5,3 +5,7 @@ resource "random_string" "my_string" { output "s" { value = "string" } + +terraform { + required_version = "~> 0.11.0" +} diff --git a/tests/plan/plan_12/main.tf b/tests/plan/plan_12/main.tf index dee08246..33afe2d1 100644 --- a/tests/plan/plan_12/main.tf +++ b/tests/plan/plan_12/main.tf @@ -5,3 +5,7 @@ resource "random_string" "my_string" { output "s" { value = "string" } + +terraform { + required_version = "~> 0.12.0" +} diff --git a/tests/plan/plan_13/main.tf b/tests/plan/plan_13/main.tf index dee08246..9b4048c5 100644 --- a/tests/plan/plan_13/main.tf +++ b/tests/plan/plan_13/main.tf @@ -5,3 +5,7 @@ resource "random_string" "my_string" { output "s" { value = "string" } + +terraform { + required_version = "~> 0.13.0" +} diff --git a/tests/plan/plan_14/main.tf b/tests/plan/plan_14/main.tf index dee08246..cbcbb864 100644 --- a/tests/plan/plan_14/main.tf +++ b/tests/plan/plan_14/main.tf @@ -5,3 +5,7 @@ resource "random_string" "my_string" { output "s" { value = "string" } + +terraform { + required_version = "~> 0.14.0" +} From 8917899beba77f564c74f7f59b7c734e74eb9aa3 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Dec 2020 11:45:26 +0000 Subject: [PATCH 009/231] :bookmark: v1.5.1 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99edbe9b..eb0d6bfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.5.0` to use an exact release +- `@v1.5.1` to use an exact release - `@v1.5` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.5.1] - 2020-012-05 + +### Fixed +- PR comments had an empty plan with Terraform 0.14 + ## [1.5.0] - 2020-09-18 ### Added @@ -85,6 +90,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.5.1]: https://github.com/dflook/terraform-github-actions/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/dflook/terraform-github-actions/compare/v1.4.2...v1.5.0 [1.4.2]: https://github.com/dflook/terraform-github-actions/compare/v1.4.1...v1.4.2 [1.4.1]: https://github.com/dflook/terraform-github-actions/compare/v1.4.0...v1.4.1 From f80fcc729c349d9b087b58ae22f033395d273e34 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Dec 2020 14:21:01 +0000 Subject: [PATCH 010/231] Add unit tests for compact_plan --- image/tools/compact_plan.py | 15 ++- tests/test_compact_plan.py | 258 ++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 tests/test_compact_plan.py diff --git a/image/tools/compact_plan.py b/image/tools/compact_plan.py index a5a78a4f..b7439f02 100755 --- a/image/tools/compact_plan.py +++ b/image/tools/compact_plan.py @@ -2,11 +2,12 @@ import sys -if __name__ == '__main__': + +def compact_plan(input): plan = False buffer = [] - for line in sys.stdin.readlines(): + for line in input: if not plan and ( line.startswith('An execution plan has been generated and is shown below') or @@ -16,10 +17,14 @@ plan = True if plan: - sys.stdout.write(line) + yield line else: buffer.append(line) if not plan and buffer: - for line in buffer: - sys.stdout.write(line) + yield from buffer + + +if __name__ == '__main__': + for line in compact_plan(sys.stdin.readlines()): + sys.stdout.write(line) diff --git a/tests/test_compact_plan.py b/tests/test_compact_plan.py new file mode 100644 index 00000000..3c76e1ea --- /dev/null +++ b/tests/test_compact_plan.py @@ -0,0 +1,258 @@ +from compact_plan import compact_plan + + +def test_plan_11(): + input = """Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + ++ random_string.my_string + id: + length: "11" + lower: "true" + min_lower: "0" + min_numeric: "0" + min_special: "0" + min_upper: "0" + number: "true" + result: + special: "true" + upper: "true" +Plan: 1 to add, 0 to change, 0 to destroy. +""" + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + ++ random_string.my_string + id: + length: "11" + lower: "true" + min_lower: "0" + min_numeric: "0" + min_special: "0" + min_upper: "0" + number: "true" + result: + special: "true" + upper: "true" +Plan: 1 to add, 0 to change, 0 to destroy.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_12(): + input = """Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. +""" + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_14(): + input = """ +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string" +""" + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string\"""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_error_11(): + input = """ +Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax + +""" + + expected_output = """Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax +""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_error_12(): + input = """ +Error: Incorrect attribute value type + + on main.tf line 2, in resource "random_string" "my_string": + 2: length = "ten" + +Inappropriate value for attribute "length": a number is required. +""" + + expected_output = """Error: Incorrect attribute value type + + on main.tf line 2, in resource "random_string" "my_string": + 2: length = "ten" + +Inappropriate value for attribute "length": a number is required.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_no_change_11(): + input = """Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_no_change_14(): + input = """ +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_no_output(): + input = """ +This is not anything like terraform output we know. We want this to be output unchanged. +This should protect against the output changing again. +""" + + expected_output = """ +This is not anything like terraform output we know. We want this to be output unchanged. +This should protect against the output changing again.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + From bc7033a24787375df39f480f488de2adc992e67f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 15 Jan 2021 22:25:11 +0000 Subject: [PATCH 011/231] Rename test jobs --- .github/workflows/test-apply.yaml | 4 ++-- .github/workflows/test-plan.yaml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index aa97017a..5bcf3ee1 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -140,7 +140,7 @@ jobs: backend_config_12: runs-on: ubuntu-latest - name: backend_config & backed_config_vars + name: backend_config terraform 12 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -191,7 +191,7 @@ jobs: backend_config_13: runs-on: ubuntu-latest - name: backend_config & backed_config_vars + name: backend_config terraform 13 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 0cd9f738..89b1b384 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -44,7 +44,7 @@ jobs: plan_change_comment_11: runs-on: ubuntu-latest - name: Change + name: Change terraform 11 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -68,7 +68,7 @@ jobs: plan_change_comment_12: runs-on: ubuntu-latest - name: Change + name: Change terraform 12 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -92,7 +92,7 @@ jobs: plan_change_comment_13: runs-on: ubuntu-latest - name: Change + name: Change terraform 13 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -116,7 +116,7 @@ jobs: plan_change_comment_14: runs-on: ubuntu-latest - name: Change + name: Change terraform 14 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -140,7 +140,7 @@ jobs: plan_change_comment_latest: runs-on: ubuntu-latest - name: Change + name: Change latest terraform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: From 33da839ff321df0af9081f0be99b0decfe2bc857 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 15 Jan 2021 22:27:49 +0000 Subject: [PATCH 012/231] Better handle GITHUB_TOKEN rate limiting --- image/tools/github_pr_comment.py | 39 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index a304866d..fecff3fe 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -4,6 +4,7 @@ import os import re import sys +import datetime from typing import Optional, Dict, Iterable import requests @@ -11,6 +12,32 @@ github = requests.Session() github.headers['authorization'] = f'Bearer {os.environ["GITHUB_TOKEN"]}' +def github_api_request(method, *args, **kw_args): + response = github.request(method, *args, **kw_args) + + if response.status_code >= 400 and response.status_code < 500: + debug(str(response.headers)) + + try: + message = response.json()['message'] + + if response.headers['X-RateLimit-Remaining'] == '0': + limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) + sys.stderr.write(message) + sys.stderr.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') + exit(1) + + if message != 'Resource not accessible by integration': + sys.stderr.write(message) + sys.stderr.write('\n') + debug(response.content.decode()) + + except Exception: + sys.stderr.write(response.content.decode()) + sys.stderr.write('\n') + raise + + return response def debug(msg: str) -> None: for line in msg.splitlines(): @@ -21,7 +48,7 @@ def prs(repo: str) -> Iterable[Dict]: url = f'https://api.github.com/repos/{repo}/pulls' while True: - response = github.get(url, params={'state': 'all'}) + response = github_api_request('get', url, params={'state': 'all'}) response.raise_for_status() for pr in response.json(): @@ -71,7 +98,7 @@ def find_pr() -> str: raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") def current_user() -> str: - response = github.get('https://api.github.com/user') + response = github_api_request('get', 'https://api.github.com/user') if response.status_code != 403: user = response.json() debug('GITHUB_TOKEN user:') @@ -91,11 +118,11 @@ def __init__(self, pr_url: str): self._plan = None self._status = None - response = github.get(pr_url) + response = github_api_request('get', pr_url) response.raise_for_status() self._issue_url = response.json()['_links']['issue']['href'] + '/comments' - response = github.get(self._issue_url) + response = github_api_request('get', self._issue_url) response.raise_for_status() self._comment_url = None @@ -254,11 +281,11 @@ def update_comment(self): if self._comment_url is None: # Create a new comment debug('Creating comment') - response = github.post(self._issue_url, json={'body': body}) + response = github_api_request('post', self._issue_url, json={'body': body}) else: # Update existing comment debug('Updating existing comment') - response = github.patch(self._comment_url, json={'body': body}) + response = github_api_request('patch', self._comment_url, json={'body': body}) debug(response.content.decode()) response.raise_for_status() From 5db7579fc50aceb74184563119d1d488cf37f4a6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 15 Jan 2021 22:28:55 +0000 Subject: [PATCH 013/231] Use terraform 0.14.4 as default version in image --- image/Dockerfile-base | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/image/Dockerfile-base b/image/Dockerfile-base index ddf351b6..b41a6eb3 100644 --- a/image/Dockerfile-base +++ b/image/Dockerfile-base @@ -5,7 +5,7 @@ RUN cd tfmask && make && make go/build FROM debian:buster-slim as base -ARG DEFAULT_TF_VERSION=0.12.28 +ARG DEFAULT_TF_VERSION=0.14.4 ARG TFSWITCH_VERSION=0.8.832 # Terraform environment variables @@ -33,11 +33,12 @@ RUN apt-get update && apt-get install -y \ RUN curl -fsL https://github.com/warrensbox/terraform-switcher/releases/download/${TFSWITCH_VERSION}/terraform-switcher_${TFSWITCH_VERSION}_linux_amd64.tar.gz -o tfswitch.tar.gz \ && tar -xvf tfswitch.tar.gz \ && mv tfswitch /usr/local/bin \ - && rm -rf tfswitch \ - && tfswitch $DEFAULT_TF_VERSION + && rm -rf README.md LICENSE terraform-switcher \ + && tfswitch $DEFAULT_TF_VERSION \ + && mv /root/.terraform.versions /root/.terraform.versions.default RUN mkdir -p $TF_PLUGIN_CACHE_DIR COPY --from=tfmask /go/tfmask/release/tfmask /usr/local/bin/tfmask ENV TFMASK_RESOURCES_REGEX="(?i)^(random_id|kubernetes_secret|acme_certificate).*$" -ENTRYPOINT ["/usr/local/bin/terraform"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/terraform"] From 145468bdefe29401837f15b17576661d47b4e8cf Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 15 Jan 2021 22:29:43 +0000 Subject: [PATCH 014/231] Reuse downloaded terraform binaries Store them in home directory --- image/actions.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index 4f32131e..4303f0d6 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -13,9 +13,10 @@ function debug_cmd() { } function debug() { + debug_cmd ls -la /root debug_cmd pwd debug_cmd ls -la - debug_cmd ls $HOME + debug_cmd ls -la $HOME debug_cmd printenv debug_cmd cat "$GITHUB_EVENT_PATH" echo @@ -24,6 +25,8 @@ function debug() { function detect-terraform-version() { local TF_SWITCH_OUTPUT + debug_cmd tfswitch --version + TF_SWITCH_OUTPUT=$(cd "$INPUT_PATH" && echo "" | tfswitch | grep -e Switched -e Reading | sed 's/^.*Switched/Switched/') if echo "$TF_SWITCH_OUTPUT" | grep Reading >/dev/null; then echo "$TF_SWITCH_OUTPUT" @@ -31,6 +34,8 @@ function detect-terraform-version() { echo "Reading latest terraform version" tfswitch "$(latest_terraform_version)" fi + + debug_cmd ls -la "$(which terraform)" } function job_markdown_ref() { @@ -47,10 +52,23 @@ function detect-tfmask() { } function setup() { + TERRAFORM_BIN_DIR="$HOME/.dflook-terraform-bin-dir" export TF_DATA_DIR="$HOME/.dflook-terraform-data-dir" export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" unset TF_WORKSPACE + # tfswitch guesses the wrong home directory... + if [[ ! -d $TERRAFORM_BIN_DIR ]]; then + debug_log "Initializing tfswitch with image default version" + cp --recursive /root/.terraform.versions.default $TERRAFORM_BIN_DIR + fi + + ln -s $TERRAFORM_BIN_DIR /root/.terraform.versions + + debug_cmd ls -lad /root/.terraform.versions + debug_cmd ls -lad $TERRAFORM_BIN_DIR + debug_cmd ls -la $TERRAFORM_BIN_DIR + mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" if [[ "$INPUT_PATH" == "" ]]; then @@ -64,6 +82,9 @@ function setup() { fi detect-terraform-version + + debug_cmd ls -la $TERRAFORM_BIN_DIR + detect-tfmask } From c92a85dee3331cdf4e331eff49f54688f6b8ca0d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 16 Jan 2021 10:28:48 +0000 Subject: [PATCH 015/231] :bookmark: v1.5.2 --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb0d6bfb..b6dc1925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.5.1` to use an exact release +- `@v1.5.2` to use an exact release - `@v1.5` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## [1.5.1] - 2020-012-05 +## [1.5.2] - 2021-01-16 + +### Fixed +- Multiple steps in the same job now only download the terraform binary once. + +## [1.5.1] - 2020-12-05 ### Fixed - PR comments had an empty plan with Terraform 0.14 From 4ad456ba906a3b45206dbb7448ad4ada011e1568 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 16 Jan 2021 10:29:09 +0000 Subject: [PATCH 016/231] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6dc1925..76396aa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.5.2]: https://github.com/dflook/terraform-github-actions/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/dflook/terraform-github-actions/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/dflook/terraform-github-actions/compare/v1.4.2...v1.5.0 [1.4.2]: https://github.com/dflook/terraform-github-actions/compare/v1.4.1...v1.4.2 From d8f958c2edd8a75b9ccb831835a6b0e008bcfa56 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 23 Feb 2021 00:34:22 +0000 Subject: [PATCH 017/231] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..85b14776 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [dflook] From da92199bf10ab6d392992cfb21eb7f893f624614 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 23 Feb 2021 23:31:32 +0000 Subject: [PATCH 018/231] Show a one line summary of terraform output The full output is in a collapsable pane. 'No changes' and plans longer than 10 lines are collapsed by default. --- .github/workflows/test.yaml | 4 +- .gitignore | 3 +- CHANGELOG.md | 13 +- image/tools/github_pr_comment.py | 67 +++++- terraform-plan/README.md | 15 ++ terraform-version/README.md | 2 +- tests/requirements.txt | 2 + tests/test_pr_comment.py | 397 +++++++++++++++++++++++++++++++ 8 files changed, 491 insertions(+), 12 deletions(-) create mode 100644 tests/requirements.txt create mode 100644 tests/test_pr_comment.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a5431452..bc5f16d6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,8 +18,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install -r tests/requirements.txt - name: Run tests run: | - PYTHONPATH=image/tools pytest tests + GITHUB_TOKEN=No PYTHONPATH=image/tools pytest tests diff --git a/.gitignore b/.gitignore index ab006c2f..623553e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .terraform/ /.idea/ .pytest_cache/ -/venv/ \ No newline at end of file +/venv/ +.terraform.lock.hcl diff --git a/CHANGELOG.md b/CHANGELOG.md index 76396aa1..61f731ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,18 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.5.2` to use an exact release -- `@v1.5` to use the latest patch release for the specific minor version +- `@v1.6.0` to use an exact release +- `@v1.6` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.6.0] - 2021-02-24 + +### Added +- PR comments use a one line summary of the terraform output, with the full output in a collapsable pane. + + If a plan is short the output is shown by default. This can be controlled with the `TF_PLAN_COLLAPSE_LENGTH` environment + variable for the [dflook/terraform-plan](terraform-plan) action. + ## [1.5.2] - 2021-01-16 ### Fixed @@ -95,6 +103,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.6.0]: https://github.com/dflook/terraform-github-actions/compare/v1.5.2...v1.6.0 [1.5.2]: https://github.com/dflook/terraform-github-actions/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/dflook/terraform-github-actions/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/dflook/terraform-github-actions/compare/v1.4.2...v1.5.0 diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index fecff3fe..c956236e 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -114,9 +114,13 @@ class TerraformComment: The GitHub comment for this specific terraform plan """ - def __init__(self, pr_url: str): + def __init__(self, pr_url: str=None): self._plan = None self._status = None + self._comment_url = None + + if pr_url is None: + return response = github_api_request('get', pr_url) response.raise_for_status() @@ -125,20 +129,18 @@ def __init__(self, pr_url: str): response = github_api_request('get', self._issue_url) response.raise_for_status() - self._comment_url = None debug('Looking for an existing comment:') for comment in response.json(): debug(json.dumps(comment)) if comment['user']['login'] == current_user(): - match = re.match(rf'{re.escape(self._comment_identifier)}\n```(?:hcl)?(.*?)```(.*)', comment['body'], re.DOTALL) + match = re.match(rf'{re.escape(self._comment_identifier)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) if not match: - match = re.match(rf'{re.escape(self._old_comment_identifier)}\n```(.*?)```(.*)', comment['body'], re.DOTALL) + match = re.match(rf'{re.escape(self._old_comment_identifier)}\n```(.*?)```.*', comment['body'], re.DOTALL) if match: self._comment_url = comment['url'] self._plan = match.group(1).strip() - self._status = match.group(2).strip() return @property @@ -270,12 +272,64 @@ def status(self) -> Optional[str]: def status(self, status: str) -> None: self._status = status.strip() - def update_comment(self): + def body(self) -> str: body = f'{self._comment_identifier}\n```hcl\n{self.plan}\n```' if self.status: body += '\n' + self.status + return body + + def collapsable_body(self) -> str: + + try: + collapse_threshold = int(os.environ['TF_PLAN_COLLAPSE_LENGTH']) + except (ValueError, KeyError): + collapse_threshold = 10 + + open = '' + highlighting = '' + + if self.plan.startswith('Error'): + open = ' open' + elif 'Plan:' in self.plan: + highlighting = 'hcl' + num_lines = len(self.plan.splitlines()) + if num_lines < collapse_threshold: + open = ' open' + + body = f'''{self._comment_identifier} + + {self.summary()} + +```{highlighting} +{self.plan} +``` + +''' + + if self.status: + body += '\n' + self.status + + return body + + def summary(self) -> str: + summary = None + + for line in self.plan.splitlines(): + if line.startswith('No changes') or line.startswith('Error'): + return line + + if line.startswith('Plan:'): + summary = line + + if line.startswith('Changes to Outputs'): + return summary + ' Changes to Outputs.' + + return summary + + def update_comment(self): + body = self.collapsable_body() debug(body) if self._comment_url is None: @@ -313,5 +367,6 @@ def update_comment(self): if tf_comment.plan is None: exit(1) print(tf_comment.plan) + exit(0) tf_comment.update_comment() diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 4e210b4b..2b29eb06 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -100,6 +100,21 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` + +* `TF_PLAN_COLLAPSE_LENGTH` + + When PR comments are enabled, the terraform output is included in a collapsable pane. + + If a terraform plan has fewer lines than this value, the pane is expanded + by default when the comment is displayed. + + ```yaml + env: + TF_PLAN_COLLAPSE_LENGTH: 30 + ``` + + - Optional + - Default: 10 ## Outputs diff --git a/terraform-version/README.md b/terraform-version/README.md index 52057810..533ae477 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -70,5 +70,5 @@ jobs: run: echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" - name: Print aws provider version - run: echo "The terraform version was ${{ steps.terraform-version.outputs.aws }}" + run: echo "The aws provider version was ${{ steps.terraform-version.outputs.aws }}" ``` diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..2001f3e2 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +requests +pytest diff --git a/tests/test_pr_comment.py b/tests/test_pr_comment.py new file mode 100644 index 00000000..a134fafc --- /dev/null +++ b/tests/test_pr_comment.py @@ -0,0 +1,397 @@ +import github_pr_comment +from github_pr_comment import TerraformComment + +import pytest + +def setup_comment(monkeypatch, *, + path ='/test/terraform', + workspace ='default', + backend_config ='', + backend_config_file ='', + var ='', + var_file ='', + label ='', + ): + + monkeypatch.setenv('INPUT_WORKSPACE', workspace) + monkeypatch.setenv('INPUT_PATH', path) + monkeypatch.setattr('github_pr_comment.current_user', lambda: 'github-actions[bot]') + monkeypatch.setenv('INPUT_BACKEND_CONFIG', backend_config) + monkeypatch.setenv('INPUT_BACKEND_CONFIG_FILE', backend_config_file) + monkeypatch.setenv('INPUT_VAR', var) + monkeypatch.setenv('INPUT_VAR_FILE', var_file) + monkeypatch.setenv('INPUT_LABEL', label) + monkeypatch.setenv('INIT_ARGS', '') + monkeypatch.setenv('PLAN_ARGS', '') + + +def test_path_only(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_nondefault_workspace(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + workspace='myworkspace' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ in the __myworkspace__ workspace +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_var(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + var='var1=value' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With vars: `var1=value` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_var_file(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + var_file='vars.tf' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With var files: `vars.tf` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_backend_config(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + backend_config='bucket=test,key=backend' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_backend_config_bad_words(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + backend_config='bucket=test,password=secret,key=backend,token=secret' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + + +def test_backend_config_file(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + backend_config_file='backend.tf' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With backend config files: `backend.tf` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_all(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + workspace='test', + var='myvar=hello', + var_file='vars.tf', + backend_config='bucket=mybucket,password=secret', + backend_config_file='backend.tf' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ in the __test__ workspace +With backend config: `bucket=mybucket` +With backend config files: `backend.tf` +With vars: `myvar=hello` +With var files: `vars.tf` +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + + +def test_label(monkeypatch): + + setup_comment(monkeypatch, + path='/test/terraform', + workspace='test', + var='myvar=hello', + var_file='vars.tf', + backend_config='bucket=mybucket,password=secret', + backend_config_file='backend.tf', + label='test_label' + ) + comment = TerraformComment() + comment.plan = 'Hello, this is my plan' + comment.status = 'Testing' + + expected = '''Terraform plan for __test_label__ +```hcl +Hello, this is my plan +``` +Testing''' + + assert comment.body() == expected + +def test_summary_plan_11(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + ++ random_string.my_string + id: + length: "11" + lower: "true" + min_lower: "0" + min_numeric: "0" + min_special: "0" + min_upper: "0" + number: "true" + result: + special: "true" + upper: "true" +Plan: 1 to add, 0 to change, 0 to destroy. +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' + + assert comment.summary() == expected + +def test_summary_plan_12(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' + + assert comment.summary() == expected + +def test_summary_plan_14(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string" +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy. Changes to Outputs.' + + assert comment.summary() == expected + +def test_summary_error_11(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """ +Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax + +""" + expected = "Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing \"ten\": invalid syntax" + + assert comment.summary() == expected + +def test_summary_error_12(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """ +Error: Incorrect attribute value type + + on main.tf line 2, in resource "random_string" "my_string": + 2: length = "ten" + +Inappropriate value for attribute "length": a number is required. +""" + + expected = "Error: Incorrect attribute value type" + assert comment.summary() == expected + + +def test_summary_no_change_11(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected = "No changes. Infrastructure is up-to-date." + assert comment.summary() == expected + +def test_summary_no_change_14(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected = "No changes. Infrastructure is up-to-date." + assert comment.summary() == expected + +def test_summary_output_only_change_14(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + +Terraform will perform the following actions: + +Plan: 0 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + hello = "world" + +""" + + expected = "Plan: 0 to add, 0 to change, 0 to destroy. Changes to Outputs." + assert comment.summary() == expected + +def test_summary_unknown(monkeypatch): + setup_comment(monkeypatch, + path='/test/terraform', + ) + comment = TerraformComment() + comment.plan = """ +This is not anything like terraform output we know. We don't want to generate a summary for this. +""" + + expected = None + assert comment.summary() == expected + From 166dcff4118dff3708272ea2e79adc963ce3af5c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 24 Feb 2021 21:37:55 +0000 Subject: [PATCH 019/231] Test TF_PLAN_COLLAPSE_LENGTH --- .github/workflows/test-plan.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 89b1b384..9198652b 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -118,6 +118,7 @@ jobs: runs-on: ubuntu-latest name: Change terraform 14 env: + TF_PLAN_COLLAPSE_LENGTH: 30 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout From 63b84db05b85c3aa8c412e5dbe39a2104701e1e2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 24 Feb 2021 22:08:30 +0000 Subject: [PATCH 020/231] Cache the token username This can reduce the number of github api requests --- CHANGELOG.md | 5 ++++- image/tools/github_pr_comment.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61f731ec..2d08370a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ When using an action you can specify the version as: - `@v1.6` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## [1.6.0] - 2021-02-24 +## [1.6.0] - 2021-02-25 ### Added - PR comments use a one line summary of the terraform output, with the full output in a collapsable pane. @@ -20,6 +20,9 @@ When using an action you can specify the version as: If a plan is short the output is shown by default. This can be controlled with the `TF_PLAN_COLLAPSE_LENGTH` environment variable for the [dflook/terraform-plan](terraform-plan) action. +### Fixed +- Now makes far fewer github api requests to avoid rate limiting. + ## [1.5.2] - 2021-01-16 ### Fixed diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index c956236e..0c4fd9b5 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -5,6 +5,7 @@ import re import sys import datetime +import hashlib from typing import Optional, Dict, Iterable import requests @@ -98,16 +99,37 @@ def find_pr() -> str: raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") def current_user() -> str: + + token_hash = hashlib.sha256(os.environ["GITHUB_TOKEN"].encode()).hexdigest() + + try: + with open(f'.dflook-terraform/token-cache/{token_hash}') as f: + username = f.read() + debug(f'GITHUB_TOKEN username: {username}') + return username + except Exception as e: + debug(str(e)) + response = github_api_request('get', 'https://api.github.com/user') if response.status_code != 403: user = response.json() debug('GITHUB_TOKEN user:') debug(json.dumps(user)) - return user['login'] + username = user['login'] + else: + # Assume this is the github actions app token + username = 'github-actions[bot]' + + try: + os.makedirs('.dflook-terraform/token-cache', exist_ok=True) + with open(f'.dflook-terraform/token-cache/{token_hash}', 'w') as f: + f.write(username) + except Exception as e: + debug(str(e)) - # Assume this is the github actions app token - return 'github-actions[bot]' + debug(f'GITHUB_TOKEN username: {username}') + return username class TerraformComment: """ @@ -129,10 +151,12 @@ def __init__(self, pr_url: str=None): response = github_api_request('get', self._issue_url) response.raise_for_status() + username = current_user() + debug('Looking for an existing comment:') for comment in response.json(): debug(json.dumps(comment)) - if comment['user']['login'] == current_user(): + if comment['user']['login'] == username: match = re.match(rf'{re.escape(self._comment_identifier)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) if not match: From 1f35217063c2b7284e02c7b7eb6a05fb5153d58e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 1 Apr 2021 13:08:51 +0100 Subject: [PATCH 021/231] Add support for pull_request_target events --- .github/workflows/pull_request_target.yaml | 33 ++++++++++++++++++++++ image/entrypoints/apply.sh | 2 +- image/entrypoints/plan.sh | 4 +-- image/tools/github_pr_comment.py | 2 +- 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/pull_request_target.yaml diff --git a/.github/workflows/pull_request_target.yaml b/.github/workflows/pull_request_target.yaml new file mode 100644 index 00000000..009db762 --- /dev/null +++ b/.github/workflows/pull_request_target.yaml @@ -0,0 +1,33 @@ +name: pull_request_target test + +on: [pull_request_target] + +jobs: + apply: + runs-on: ubuntu-latest + name: Apply approved changes on pull_request_target + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + label: pull_request_target + path: tests/apply/changes + + - name: Apply + uses: ./terraform-apply + id: output + with: + label: pull_request_target + path: tests/apply/changes + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 5a309719..7117405a 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -91,7 +91,7 @@ if [[ "$INPUT_AUTO_APPROVE" == "true" || $PLAN_EXIT -eq 0 ]]; then else - if [[ "$GITHUB_EVENT_NAME" != "push" && "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "issue_comment" && "$GITHUB_EVENT_NAME" != "pull_request_review_comment" ]]; then + if [[ "$GITHUB_EVENT_NAME" != "push" && "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "issue_comment" && "$GITHUB_EVENT_NAME" != "pull_request_review_comment" && "$GITHUB_EVENT_NAME" != "pull_request_target" ]]; then echo "Could not fetch plan from the PR - $GITHUB_EVENT_NAME event does not relate to a pull request. You can generate and apply a plan automatically by setting the auto_approve input to 'true'" exit 1 fi diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index a9f42e3d..e1b8d058 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -27,7 +27,7 @@ set -e cat "$PLAN_DIR/error.txt" -if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" ]]; then +if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" ]]; then if [[ -z "$GITHUB_TOKEN" ]]; then @@ -46,7 +46,7 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c fi else - debug_log "Not a pull_request, issue_comment or pull_request_review_comment event - not creating a PR comment" + debug_log "Not a pull_request, issue_comment, pull_request_target or pull_request_review_comment event - not creating a PR comment" fi if [[ $TF_EXIT -eq 1 ]]; then diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 0c4fd9b5..967875ee 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -75,7 +75,7 @@ def find_pr() -> str: event_type = os.environ['GITHUB_EVENT_NAME'] - if event_type in ['pull_request', 'pull_request_review_comment']: + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target']: return event['pull_request']['url'] elif event_type == 'issue_comment': From 46025a9ec9500975068a597496dbfb7d6f43cca5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 1 Apr 2021 21:51:07 +0100 Subject: [PATCH 022/231] Update changelog Trigger testing of pull_request_target --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d08370a..3a5d8ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ When using an action you can specify the version as: - `@v1.6` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Added +- Support for the [`pull_request_target`](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request_target) event + ## [1.6.0] - 2021-02-25 ### Added From 5e809ec7e66bb015d8d04b4e4b6801f883ca4737 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 27 Feb 2021 11:25:28 +0000 Subject: [PATCH 023/231] Add more logging --- image/tools/workspace_exists.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/image/tools/workspace_exists.py b/image/tools/workspace_exists.py index 35adb342..e3b7c150 100755 --- a/image/tools/workspace_exists.py +++ b/image/tools/workspace_exists.py @@ -2,11 +2,18 @@ import sys +def debug(msg: str) -> None: + for line in msg.splitlines(): + sys.stderr.write(f'::debug::{line}\n') + def workspace_exists(stdin, workspace: str) -> bool: for line in stdin: + debug(line) if line.strip().strip('*').strip() == workspace.strip(): + debug('workspace exists') return True + debug('workspace doesn\'t exist') return False if __name__ == '__main__': From a689dc5e52065642f2875139ba01f0d37244e259 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 27 Feb 2021 11:50:58 +0000 Subject: [PATCH 024/231] Add terraform 0.14 workspace tests --- .github/workflows/test-new-workspace.yaml | 84 +++++++++++++++++++++++ tests/new-workspace/remote_14/main.tf | 21 ++++++ 2 files changed, 105 insertions(+) create mode 100644 tests/new-workspace/remote_14/main.tf diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index f9ca7c30..601eb89e 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -170,3 +170,87 @@ jobs: path: tests/new-workspace/remote_13 workspace: ${{ github.head_ref }} var: my_string=world + + create_workspace_14: + runs-on: ubuntu-latest + name: Workspace tests 0.14 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create first workspace + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_14 + workspace: my-first-workspace + + - name: Create first workspace again + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_14 + workspace: my-first-workspace + + - name: Apply in first workspace + uses: ./terraform-apply + with: + path: tests/new-workspace/remote_14 + workspace: my-first-workspace + var: my_string=hello + auto_approve: true + + - name: Create a second workspace + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_14 + workspace: ${{ github.head_ref }} + + - name: Apply in second workspace + uses: ./terraform-apply + with: + path: tests/new-workspace/remote_14 + workspace: ${{ github.head_ref }} + var: my_string=world + auto_approve: true + + - name: Get first workspace outputs + uses: ./terraform-output + id: first_14 + with: + path: tests/new-workspace/remote_14 + workspace: my-first-workspace + + - name: Get second workspace outputs + uses: ./terraform-output + id: second_14 + with: + path: tests/new-workspace/remote_14 + workspace: ${{ github.head_ref }} + + - name: Verify outputs + run: | + if [[ "${{ steps.first_14.outputs.my_string }}" != "hello" ]]; then + echo "::error:: output my_string not set correctly for first workspace" + exit 1 + fi + + if [[ "${{ steps.second_14.outputs.my_string }}" != "world" ]]; then + echo "::error:: output my_string not set correctly for second workspace" + exit 1 + fi + + - name: Destroy first workspace + uses: ./terraform-destroy-workspace + with: + path: tests/new-workspace/remote_14 + workspace: my-first-workspace + var: my_string=hello + + - name: Destroy second workspace + uses: ./terraform-destroy-workspace + with: + path: tests/new-workspace/remote_14 + workspace: ${{ github.head_ref }} + var: my_string=world diff --git a/tests/new-workspace/remote_14/main.tf b/tests/new-workspace/remote_14/main.tf new file mode 100644 index 00000000..9c782c1b --- /dev/null +++ b/tests/new-workspace/remote_14/main.tf @@ -0,0 +1,21 @@ +resource "random_string" "my_string" { + length = 5 +} + +variable "my_string" { + type = string +} + +output "my_string" { + value = var.my_string +} + +terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "terraform-new-workspace-14" + region = "eu-west-2" + } + + required_version = "~> 0.14.0" +} From b2dc7804eb1188921d466c3a4402d8e6718247a4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 7 Mar 2021 13:24:03 +0000 Subject: [PATCH 025/231] Make new-workspace more robust --- .github/workflows/test-new-workspace.yaml | 10 ++--- image/entrypoints/new-workspace.sh | 51 ++++++++++++++++++++++- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index 601eb89e..e8bdf766 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -185,19 +185,19 @@ jobs: uses: ./terraform-new-workspace with: path: tests/new-workspace/remote_14 - workspace: my-first-workspace + workspace: test-workspace - name: Create first workspace again uses: ./terraform-new-workspace with: path: tests/new-workspace/remote_14 - workspace: my-first-workspace + workspace: test-workspace - name: Apply in first workspace uses: ./terraform-apply with: path: tests/new-workspace/remote_14 - workspace: my-first-workspace + workspace: test-workspace var: my_string=hello auto_approve: true @@ -220,7 +220,7 @@ jobs: id: first_14 with: path: tests/new-workspace/remote_14 - workspace: my-first-workspace + workspace: test-workspace - name: Get second workspace outputs uses: ./terraform-output @@ -245,7 +245,7 @@ jobs: uses: ./terraform-destroy-workspace with: path: tests/new-workspace/remote_14 - workspace: my-first-workspace + workspace: test-workspace var: my_string=hello - name: Destroy second workspace diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index 338376f6..b7ee4fc5 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -6,9 +6,56 @@ debug setup init-backend -if (cd "$INPUT_PATH" && terraform workspace list -no-color | workspace_exists "$INPUT_WORKSPACE"); then +WS_TMP_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) +rm -rf "$WS_TMP_DIR" +mkdir -p "$WS_TMP_DIR" + +set +e +(cd $INPUT_PATH && terraform workspace list -no-color) \ + 2>"$WS_TMP_DIR/list_err.txt" \ + >"$WS_TMP_DIR/list_out.txt" + +readonly TF_WS_LIST_EXIT=${PIPESTATUS[0]} +set -e + +debug_log "terraform workspace list: ${TF_WS_LIST_EXIT}" +debug_cmd cat "$WS_TMP_DIR/list_err.txt" +debug_cmd cat "$WS_TMP_DIR/list_out.txt" + +if [[ $TF_WS_LIST_EXIT -ne 0 ]]; then + echo "Error: Failed to list workspaces" + exit 1 +fi + +if workspace_exists "$INPUT_WORKSPACE" <"$WS_TMP_DIR/list_out.txt"; then echo "Workspace appears to exist, selecting it" (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") else - (cd "$INPUT_PATH" && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") + echo "Workspace does not appear to exist, attempting to create it" + + set +e + (cd $INPUT_PATH && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") \ + 2>"$WS_TMP_DIR/new_err.txt" \ + >"$WS_TMP_DIR/new_out.txt" + + readonly TF_WS_NEW_EXIT=${PIPESTATUS[0]} + set -e + + debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" + debug_cmd cat "$WS_TMP_DIR/new_err.txt" + debug_cmd cat "$WS_TMP_DIR/new_out.txt" + + if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then + + if grep -Fq "already exists" "$WS_TMP_DIR/new_err.txt"; then + echo "Workspace does exist, selecting it" + (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") + else + exit 1 + fi + else + cat "$WS_TMP_DIR/new_err.txt" + cat "$WS_TMP_DIR/new_out.txt" + fi + fi From 5e5b123232b3d4065666ba9f0a52343a7a9ee307 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 1 Apr 2021 22:05:02 +0100 Subject: [PATCH 026/231] Add support for pull_request_review events --- .github/workflows/pull_request_review.yaml | 33 +++++++++++++++++++ .../pull_request_review_trigger.yaml | 26 +++++++++++++++ CHANGELOG.md | 2 ++ image/entrypoints/apply.sh | 2 +- image/entrypoints/plan.sh | 4 +-- image/tools/github_pr_comment.py | 6 ++-- 6 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/pull_request_review.yaml create mode 100644 .github/workflows/pull_request_review_trigger.yaml diff --git a/.github/workflows/pull_request_review.yaml b/.github/workflows/pull_request_review.yaml new file mode 100644 index 00000000..d69893ee --- /dev/null +++ b/.github/workflows/pull_request_review.yaml @@ -0,0 +1,33 @@ +name: pull_request_review test + +on: [pull_request_review] + +jobs: + apply: + runs-on: ubuntu-latest + name: Apply approved changes on pull_request_review + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + label: pull_request_review + path: tests/apply/changes + + - name: Apply + uses: ./terraform-apply + id: output + with: + label: pull_request_review + path: tests/apply/changes + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi diff --git a/.github/workflows/pull_request_review_trigger.yaml b/.github/workflows/pull_request_review_trigger.yaml new file mode 100644 index 00000000..d8689818 --- /dev/null +++ b/.github/workflows/pull_request_review_trigger.yaml @@ -0,0 +1,26 @@ +name: Trigger pull_request_review + +on: [pull_request] + +jobs: + required_version: + runs-on: ubuntu-latest + name: pull_request_review + steps: + - name: Trigger pull_request_review event + run: | + cat >review.json < str: event_type = os.environ['GITHUB_EVENT_NAME'] - if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target']: + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: return event['pull_request']['url'] elif event_type == 'issue_comment': From d3f680483f28de9f36c471043597d6da3976872d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 2 Apr 2021 19:42:44 +0100 Subject: [PATCH 027/231] Remove redundant init -lock-timeout for tf 0.15 compat --- image/actions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index 4303f0d6..577db2f6 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -122,7 +122,7 @@ function init-backend() { rm -rf "$TF_DATA_DIR" set +e - (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false -lock-timeout=300s $INIT_ARGS \ + (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false $INIT_ARGS \ 2>"$PLAN_DIR/init_error.txt") local INIT_EXIT=$? From c1216dc504130c4ce7c404963aea4c3f69bf9dc4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 2 Apr 2021 19:42:44 +0100 Subject: [PATCH 028/231] Remove redundant init -lock-timeout for tf 0.15 compat --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09e3a94f..9c6cdce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ When using an action you can specify the version as: - Support for the [`pull_request_target`](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request_target) event - Support for the [`pull_request_review`](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request_review) event +### Fixed +- Terraform 0.15 compatibility ## [1.6.0] - 2021-02-25 From af0dd9be0e965fb2c1a9dab27cc479bccf45722d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 2 Apr 2021 19:58:17 +0100 Subject: [PATCH 029/231] :bookmark: v1.7.0 --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c6cdce7..330adef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.6.0` to use an exact release -- `@v1.6` to use the latest patch release for the specific minor version +- `@v1.7.0` to use an exact release +- `@v1.7` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.7.0] - 2021-04-02 ### Added - Support for the [`pull_request_target`](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#pull_request_target) event @@ -115,6 +115,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.7.0]: https://github.com/dflook/terraform-github-actions/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/dflook/terraform-github-actions/compare/v1.5.2...v1.6.0 [1.5.2]: https://github.com/dflook/terraform-github-actions/compare/v1.5.1...v1.5.2 [1.5.1]: https://github.com/dflook/terraform-github-actions/compare/v1.5.0...v1.5.1 From 9e66303250582ac3bd74c7a726a2439af7355c5b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 4 Apr 2021 10:21:50 +0100 Subject: [PATCH 030/231] Add support for specifying terraform cloud credentials --- .github/workflows/test_registry.yaml | 94 +++++++++++++++++++++++++++ CHANGELOG.md | 6 ++ image/Dockerfile | 1 + image/actions.sh | 8 +++ image/tools/format_tf_credentials.py | 29 +++++++++ terraform-apply/README.md | 41 +++++++++--- terraform-check/README.md | 24 +++++++ terraform-destroy-workspace/README.md | 24 +++++++ terraform-destroy/README.md | 24 +++++++ terraform-new-workspace/README.md | 24 +++++++ terraform-output/README.md | 24 +++++++ terraform-plan/README.md | 28 +++++++- terraform-remote-state/README.md | 24 +++++++ terraform-validate/README.md | 24 +++++++ terraform-version/README.md | 24 +++++++ tests/registry/main.tf | 8 +++ tests/registry/test-module/README.md | 1 + tests/registry/test-module/main.tf | 3 + tests/test_write_credentials.py | 49 ++++++++++++++ 19 files changed, 451 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test_registry.yaml create mode 100755 image/tools/format_tf_credentials.py create mode 100644 tests/registry/main.tf create mode 100644 tests/registry/test-module/README.md create mode 100644 tests/registry/test-module/main.tf create mode 100644 tests/test_write_credentials.py diff --git a/.github/workflows/test_registry.yaml b/.github/workflows/test_registry.yaml new file mode 100644 index 00000000..985e5af6 --- /dev/null +++ b/.github/workflows/test_registry.yaml @@ -0,0 +1,94 @@ +name: Test registry + +on: [pull_request] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + registry_module: + runs-on: ubuntu-latest + name: Use registry module + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} + with: + path: tests/registry + label: Single registry + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/registry + label: Single registry + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.word }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + multiple_registry_module: + runs-on: ubuntu-latest + name: Multiple registries + env: + TERRAFORM_CLOUD_TOKENS: | + + terraform.example.com = Registry doesn't exist + + app.terraform.io = ${{ secrets.TF_API_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + path: tests/registry + label: Multiple registries + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/registry + label: Multiple registries + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.word }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + nonsense_credentials: + runs-on: ubuntu-latest + name: Nonsense cloud credentials + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + continue-on-error: true + env: + TERRAFORM_CLOUD_TOKENS: No thanks + with: + path: tests/registry + + - name: Check failed + run: | + if [[ "${{ steps.plan.outcome }}" != "failure" ]]; then + echo "did not fail correctly with nonsense credentials" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 330adef7..469f5520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ When using an action you can specify the version as: - `@v1.7` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Added +- `TERRAFORM_CLOUD_TOKENS` environment variable may be set to configure tokens to use for Terraform Cloud/Enterprise etc + when using `remote` backend or module registry. + ## [1.7.0] - 2021-04-02 ### Added diff --git a/image/Dockerfile b/image/Dockerfile index 079176d4..c1a48c08 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -11,5 +11,6 @@ COPY tools/plan_cmp.py /usr/local/bin/plan_cmp COPY tools/convert_version.py /usr/local/bin/convert_version COPY tools/workspace_exists.py /usr/local/bin/workspace_exists COPY tools/compact_plan.py /usr/local/bin/compact_plan +COPY tools/format_tf_credentials.py /usr/local/bin/format_tf_credentials LABEL org.opencontainers.image.title="GitHub actions for terraform" diff --git a/image/actions.sh b/image/actions.sh index 577db2f6..75ee9628 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -98,11 +98,15 @@ function relative_to() { } function init() { + write_credentials + rm -rf "$TF_DATA_DIR" (cd "$INPUT_PATH" && terraform init -input=false -backend=false) } function init-backend() { + write_credentials + INIT_ARGS="" if [[ -n "$INPUT_BACKEND_CONFIG_FILE" ]]; then @@ -184,3 +188,7 @@ function update_status() { function random_string() { python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" } + +function write_credentials() { + format_tf_credentials >> $HOME/.terraformrc +} diff --git a/image/tools/format_tf_credentials.py b/image/tools/format_tf_credentials.py new file mode 100755 index 00000000..78021b69 --- /dev/null +++ b/image/tools/format_tf_credentials.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 + +import os +import sys +import re + + +def format_credentials(input): + for line in input.splitlines(): + if line.strip() == '': + continue + + match = re.search(r'(?P.+?)\s*=\s*(?P.+)', line.strip()) + + if match: + yield f'''credentials "{match.group('host')}" {{ + token = "{match.group('token')}" +}} +''' + else: + raise ValueError('terraform_cloud_credentials input should be "="') + +if __name__ == '__main__': + try: + for line in format_credentials(os.environ.get('TERRAFORM_CLOUD_TOKENS', '')): + sys.stdout.write(line) + except ValueError as e: + sys.stderr.write(str(e)) + exit(1) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 551ab006..0f796b28 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -115,16 +115,41 @@ the master branch: ## Environment Variables -### `GITHUB_TOKEN` +* `GITHUB_TOKEN` -The GitHub authorization token to use to fetch an approved plan from a PR. -The token provided by GitHub Actions can be used - it can be passed by -using the `${{ secrets.GITHUB_TOKEN }}` expression, e.g. + The GitHub authorization token to use to fetch an approved plan from a PR. + The token provided by GitHub Actions can be used - it can be passed by + using the `${{ secrets.GITHUB_TOKEN }}` expression, e.g. -```yaml -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -``` + ```yaml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ``` + + - Type: string + - Optional + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional ## Outputs diff --git a/terraform-check/README.md b/terraform-check/README.md index 2b6729b1..623be22c 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -61,6 +61,30 @@ This is intended to run on a schedule to notify if manual changes to your infras - Optional - Default: 10 +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs every morning and will fail if there has been diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 01038176..6ac41bff 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -58,6 +58,30 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Optional - Default: 10 +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example deletes the workspace named after the git branch when the associated PR is closed. diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 1d1f4391..d3950b06 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -60,6 +60,30 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Optional - Default: 10 +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example destroys the resources in a workspace named after the git branch when the associated PR is closed. diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index b87a72b6..d3cee712 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -35,6 +35,30 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit - Type: string - Optional +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example creates a workspace named after the git branch when the diff --git a/terraform-output/README.md b/terraform-output/README.md index 006b891d..3faa305a 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -36,6 +36,30 @@ Retrieve the root-level outputs from a terraform configuration. - Type: string - Optional +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 2b29eb06..9091072b 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -100,7 +100,32 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` - + + - Type: string + - Optional + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + * `TF_PLAN_COLLAPSE_LENGTH` When PR comments are enabled, the terraform output is included in a collapsable pane. @@ -113,6 +138,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ TF_PLAN_COLLAPSE_LENGTH: 30 ``` + - Type: integer - Optional - Default: 10 diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index b70dd01c..2dabce8d 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -36,6 +36,30 @@ Retrieves the root-level outputs from a terraform remote state. - Type: string - Optional +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Outputs An output will be created for each root-level output in the terraform remote state. diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 816bc6db..c52dc7eb 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -22,6 +22,30 @@ If the terraform configuration is not valid, the build is failed. - Type: string - Required +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs on every push and fails if the terraform diff --git a/terraform-version/README.md b/terraform-version/README.md index 533ae477..ae87baa7 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -27,6 +27,30 @@ outputs yourself. - Type: string - Required +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_CREDENTIALS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Outputs * `terraform` diff --git a/tests/registry/main.tf b/tests/registry/main.tf new file mode 100644 index 00000000..2b331600 --- /dev/null +++ b/tests/registry/main.tf @@ -0,0 +1,8 @@ +module "hello" { + source = "app.terraform.io/flooktech/test/aws" + version = "0.0.1" +} + +output "word" { + value = module.hello.my-output +} diff --git a/tests/registry/test-module/README.md b/tests/registry/test-module/README.md new file mode 100644 index 00000000..f8038413 --- /dev/null +++ b/tests/registry/test-module/README.md @@ -0,0 +1 @@ +This module is hosted in a registry that requires a token to access diff --git a/tests/registry/test-module/main.tf b/tests/registry/test-module/main.tf new file mode 100644 index 00000000..4cecf271 --- /dev/null +++ b/tests/registry/test-module/main.tf @@ -0,0 +1,3 @@ +output "my-output" { + value = "hello" +} diff --git a/tests/test_write_credentials.py b/tests/test_write_credentials.py new file mode 100644 index 00000000..4f1ef02d --- /dev/null +++ b/tests/test_write_credentials.py @@ -0,0 +1,49 @@ +from format_tf_credentials import format_credentials + + +def test_single_cred(): + input = """app.terraform.io=xxxxxx.atlasv1.zzzzzzzzzzzzz""" + + expected_output = """credentials "app.terraform.io" { + token = "xxxxxx.atlasv1.zzzzzzzzzzzzz" +} +""" + + output = ''.join(format_credentials(input)) + assert output == expected_output + +def test_multiple_creds(): + input = """ + + app.terraform.io=xxxxxx.atlasv1.zzzzzzzzzzzzz + +terraform.example.com=abcdefg + +""" + + expected_output = """credentials "app.terraform.io" { + token = "xxxxxx.atlasv1.zzzzzzzzzzzzz" +} +credentials "terraform.example.com" { + token = "abcdefg" +} +""" + + output = ''.join(format_credentials(input)) + assert output == expected_output + +def test_unrecognised_lines(): + input = """ + + app.terraform.io=xxxxxx.atlasv1.zzzzzzzzzzzzz + + This doesn't look anything like a credential + + """ + + try: + output = ''.join(format_credentials(input)) + except ValueError as e: + pass + else: + assert False, 'Should have raised an exception' From 801cac63cb5b17f7c872be847e4cab4211373215 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 4 Apr 2021 10:21:50 +0100 Subject: [PATCH 031/231] Add TERRAFORM_SSH_KEY environment variable This enables git module sources --- .github/workflows/test-ssh.yaml | 61 +++++++++++++++++++++++++++ CHANGELOG.md | 2 + image/Dockerfile | 4 ++ image/actions.sh | 4 ++ terraform-apply/README.md | 15 +++++++ terraform-check/README.md | 16 +++++++ terraform-destroy-workspace/README.md | 15 +++++++ terraform-destroy/README.md | 15 +++++++ terraform-new-workspace/README.md | 15 +++++++ terraform-output/README.md | 15 +++++++ terraform-plan/README.md | 15 +++++++ terraform-remote-state/README.md | 2 +- terraform-validate/README.md | 17 +++++++- terraform-version/README.md | 15 +++++++ tests/ssh-module/main.tf | 7 +++ 15 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-ssh.yaml create mode 100644 tests/ssh-module/main.tf diff --git a/.github/workflows/test-ssh.yaml b/.github/workflows/test-ssh.yaml new file mode 100644 index 00000000..8f083151 --- /dev/null +++ b/.github/workflows/test-ssh.yaml @@ -0,0 +1,61 @@ +name: Test SSH Keys + +on: [pull_request] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + ssh_key: + runs-on: ubuntu-latest + name: Git module source + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + with: + path: tests/ssh-module + label: SSH Module + + - name: Apply + uses: ./terraform-apply + id: output + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + with: + path: tests/ssh-module + label: SSH Module + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.word }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + no_ssh_key: + runs-on: ubuntu-latest + name: Git module source with no key + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + continue-on-error: true + id: plan + with: + path: tests/ssh-module + label: SSH Module + add_github_comment: false + + - name: Check failed + run: | + if [[ "${{ steps.plan.outcome }}" != "failure" ]]; then + echo "did not fail correctly with no SSH key" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 469f5520..7af9fdb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ When using an action you can specify the version as: ### Added - `TERRAFORM_CLOUD_TOKENS` environment variable may be set to configure tokens to use for Terraform Cloud/Enterprise etc when using `remote` backend or module registry. +- `TERRAFORM_SSH_KEY` environment variable may be set to configure an SSH private key to use for + [Git Repository](https://www.terraform.io/docs/language/modules/sources.html#generic-git-repository) module sources. ## [1.7.0] - 2021-04-02 diff --git a/image/Dockerfile b/image/Dockerfile index c1a48c08..a4f3f73d 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -13,4 +13,8 @@ COPY tools/workspace_exists.py /usr/local/bin/workspace_exists COPY tools/compact_plan.py /usr/local/bin/compact_plan COPY tools/format_tf_credentials.py /usr/local/bin/format_tf_credentials +RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \ + && echo "IdentityFile /.ssh/id_rsa" >> /etc/ssh/ssh_config \ + && mkdir -p /.ssh + LABEL org.opencontainers.image.title="GitHub actions for terraform" diff --git a/image/actions.sh b/image/actions.sh index 75ee9628..d24f7f9e 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -191,4 +191,8 @@ function random_string() { function write_credentials() { format_tf_credentials >> $HOME/.terraformrc + + echo "$TERRAFORM_SSH_KEY" >> /.ssh/id_rsa + chmod 600 /.ssh/id_rsa + chmod 700 /.ssh } diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 0f796b28..34849934 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -151,6 +151,21 @@ the master branch: - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-check/README.md b/terraform-check/README.md index 623be22c..da13d4f3 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -85,6 +85,22 @@ This is intended to run on a schedule to notify if manual changes to your infras - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + + ## Example usage This example workflow runs every morning and will fail if there has been diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 6ac41bff..967be708 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -82,6 +82,21 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Example usage This example deletes the workspace named after the git branch when the associated PR is closed. diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index d3950b06..7ee84b9a 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -84,6 +84,21 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Example usage This example destroys the resources in a workspace named after the git branch when the associated PR is closed. diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index d3cee712..7c3aaf12 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -59,6 +59,21 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Example usage This example creates a workspace named after the git branch when the diff --git a/terraform-output/README.md b/terraform-output/README.md index 3faa305a..67220ac9 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -60,6 +60,21 @@ Retrieve the root-level outputs from a terraform configuration. - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 9091072b..10dd5cd6 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -126,6 +126,21 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + * `TF_PLAN_COLLAPSE_LENGTH` When PR comments are enabled, the terraform output is included in a collapsable pane. diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index 2dabce8d..d2107c6f 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -41,7 +41,7 @@ Retrieves the root-level outputs from a terraform remote state. * `TERRAFORM_CLOUD_TOKENS` API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. - These tokens may be used with the `remote` backend and for fetching required modules from the registry. + These tokens may be used with the `remote` backend. e.g for terraform cloud: ```yaml diff --git a/terraform-validate/README.md b/terraform-validate/README.md index c52dc7eb..26025a3d 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -27,7 +27,7 @@ If the terraform configuration is not valid, the build is failed. * `TERRAFORM_CLOUD_TOKENS` API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. - These tokens may be used with the `remote` backend and for fetching required modules from the registry. + These tokens may be used for fetching required modules from the registry. e.g for terraform cloud: ```yaml @@ -46,6 +46,21 @@ If the terraform configuration is not valid, the build is failed. - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs on every push and fails if the terraform diff --git a/terraform-version/README.md b/terraform-version/README.md index ae87baa7..ada12d8f 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -51,6 +51,21 @@ outputs yourself. - Type: string - Optional +* `TERRAFORM_SSH_KEY` + + A SSH private key that terraform will use to fetch git module sources. + + This should be in PEM format. + + For example: + ```yaml + env: + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + ``` + + - Type: string + - Optional + ## Outputs * `terraform` diff --git a/tests/ssh-module/main.tf b/tests/ssh-module/main.tf new file mode 100644 index 00000000..edd08393 --- /dev/null +++ b/tests/ssh-module/main.tf @@ -0,0 +1,7 @@ +module "hello" { + source = "git::ssh://git@github.com/dflook/terraform-github-actions//tests/registry/test-module" +} + +output "word" { + value = module.hello.my-output +} From 5b09a73145f800f785aff6a162bd86b747dadc3b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 5 Apr 2021 08:45:32 +0100 Subject: [PATCH 032/231] Rename to TERRAFORM_CLOUD_TOKENS in docs --- image/tools/format_tf_credentials.py | 2 +- terraform-apply/README.md | 4 ++-- terraform-check/README.md | 4 ++-- terraform-destroy-workspace/README.md | 4 ++-- terraform-destroy/README.md | 4 ++-- terraform-new-workspace/README.md | 4 ++-- terraform-output/README.md | 4 ++-- terraform-plan/README.md | 4 ++-- terraform-remote-state/README.md | 4 ++-- terraform-validate/README.md | 4 ++-- terraform-version/README.md | 4 ++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/image/tools/format_tf_credentials.py b/image/tools/format_tf_credentials.py index 78021b69..1a055eee 100755 --- a/image/tools/format_tf_credentials.py +++ b/image/tools/format_tf_credentials.py @@ -18,7 +18,7 @@ def format_credentials(input): }} ''' else: - raise ValueError('terraform_cloud_credentials input should be "="') + raise ValueError('TERRAFORM_CLOUD_TOKENS environment variable should be "="') if __name__ == '__main__': try: diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 34849934..ae375309 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -137,13 +137,13 @@ the master branch: e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-check/README.md b/terraform-check/README.md index da13d4f3..45afdbeb 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -71,13 +71,13 @@ This is intended to run on a schedule to notify if manual changes to your infras e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 967be708..fa4ac7c1 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -68,13 +68,13 @@ This action uses the `terraform destroy` command to destroy all resources in a t e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 7ee84b9a..f40a2aa4 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -70,13 +70,13 @@ This action uses the `terraform destroy` command to destroy all resources in a t e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 7c3aaf12..8a6e2636 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -45,13 +45,13 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-output/README.md b/terraform-output/README.md index 67220ac9..13486318 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -46,13 +46,13 @@ Retrieve the root-level outputs from a terraform configuration. e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 10dd5cd6..bafa61fd 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -112,13 +112,13 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index d2107c6f..33adb01f 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -46,13 +46,13 @@ Retrieves the root-level outputs from a terraform remote state. e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 26025a3d..018ccfa9 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -32,13 +32,13 @@ If the terraform configuration is not valid, the build is failed. e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` diff --git a/terraform-version/README.md b/terraform-version/README.md index ada12d8f..3d6dfb24 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -37,13 +37,13 @@ outputs yourself. e.g for terraform cloud: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} ``` With Terraform Enterprise or other registries: ```yaml env: - TERRAFORM_CLOUD_CREDENTIALS: | + TERRAFORM_CLOUD_TOKENS: | app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} ``` From 578788ee0826986b4a4cf16a39aacabe85fae581 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 5 Apr 2021 09:00:16 +0100 Subject: [PATCH 033/231] :bookmark: v.1.8.0 --- CHANGELOG.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af9fdb5..f4ccaacb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,18 +8,21 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.7.0` to use an exact release -- `@v1.7` to use the latest patch release for the specific minor version +- `@v1.8.0` to use an exact release +- `@v1.8` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.8.0] - 2021-04-05 ### Added -- `TERRAFORM_CLOUD_TOKENS` environment variable may be set to configure tokens to use for Terraform Cloud/Enterprise etc - when using `remote` backend or module registry. -- `TERRAFORM_SSH_KEY` environment variable may be set to configure an SSH private key to use for +- `TERRAFORM_CLOUD_TOKENS` environment variable for use with Terraform Cloud/Enterprise etc + when using module registries or a `remote` backend. + +- `TERRAFORM_SSH_KEY` environment variable to configure an SSH private key to use for [Git Repository](https://www.terraform.io/docs/language/modules/sources.html#generic-git-repository) module sources. +See individual actions for details, e.g. [terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate#environment-variables). + ## [1.7.0] - 2021-04-02 ### Added @@ -123,6 +126,8 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) + +[1.8.0]: https://github.com/dflook/terraform-github-actions/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/dflook/terraform-github-actions/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/dflook/terraform-github-actions/compare/v1.5.2...v1.6.0 [1.5.2]: https://github.com/dflook/terraform-github-actions/compare/v1.5.1...v1.5.2 From 16b1370ff3749e5568b3e7bac682813a31df2d4d Mon Sep 17 00:00:00 2001 From: Vlad Emelianov Date: Wed, 7 Apr 2021 06:29:36 +0300 Subject: [PATCH 034/231] Fix example workflow errors in triggers --- example_workflows/create_plan.yaml | 2 +- example_workflows/validate.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example_workflows/create_plan.yaml b/example_workflows/create_plan.yaml index b9831b7a..0e41d28d 100644 --- a/example_workflows/create_plan.yaml +++ b/example_workflows/create_plan.yaml @@ -3,7 +3,7 @@ name: Create terraform plan on: pull_request: branches: - - !master + - "!master" jobs: plan: diff --git a/example_workflows/validate.yaml b/example_workflows/validate.yaml index c2d17555..8d36c1a9 100644 --- a/example_workflows/validate.yaml +++ b/example_workflows/validate.yaml @@ -3,7 +3,7 @@ name: Validate changes on: push: branches: - - !master + - "!master" jobs: fmt-check: From faabccc0e7b5b364abc875b5a5414d5882060d97 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 7 Apr 2021 10:08:04 +0100 Subject: [PATCH 035/231] Fix example lint.yaml --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 909a9aba..aa3fa8d2 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ name: Lint on: push: branches: - - !master + - '!master' jobs: validate: From 631fe459941f6f0f446e31d8cb301e246559755a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 10 Apr 2021 11:49:28 +0100 Subject: [PATCH 036/231] Add variables input and deprecate vars --- .github/workflows/test-apply.yaml | 82 ++++++++++++++++++++-- .github/workflows/test-new-workspace.yaml | 24 +++---- .github/workflows/test-plan.yaml | 17 +++++ CHANGELOG.md | 17 +++++ example_workflows/create_plan.yaml | 5 +- example_workflows/validate.yaml | 2 +- image/actions.sh | 5 ++ image/tools/github_pr_comment.py | 17 +++++ terraform-apply/README.md | 47 ++++++++++++- terraform-apply/action.yaml | 4 ++ terraform-check/README.md | 23 ++++++- terraform-check/action.yaml | 4 ++ terraform-destroy-workspace/README.md | 23 ++++++- terraform-destroy-workspace/action.yaml | 4 ++ terraform-destroy/README.md | 23 ++++++- terraform-destroy/action.yaml | 4 ++ terraform-output/README.md | 2 +- terraform-plan/README.md | 84 ++++++++++++++++++++++- terraform-plan/action.yaml | 4 ++ tests/apply/test.tfvars | 3 +- tests/apply/vars/main.tf | 21 ++++++ 21 files changed, 383 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 5bcf3ee1..c78ac7f6 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -97,9 +97,9 @@ jobs: exit 1 fi - apply_vars: + apply_variables: runs-on: ubuntu-latest - name: Apply approved changes with a variable + name: Apply approved changes with variables env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -110,7 +110,20 @@ jobs: uses: ./terraform-plan with: path: tests/apply/vars - var: my_var=hello + variables: | + my_var="hello" + complex_input=[ + { + internal = 2000 + external = 3000 + protocol = "udp" + }, + { + internal = 4000 + external = 5000 + protocol = "tcp" + }, + ] var_file: tests/apply/test.tfvars - name: Apply @@ -118,7 +131,20 @@ jobs: id: output with: path: tests/apply/vars - var: my_var=hello + variables: | + my_var="hello" + complex_input=[ + { + internal = 2000 + external = 3000 + protocol = "udp" + }, + { + internal = 4000 + external = 5000 + protocol = "tcp" + }, + ] var_file: tests/apply/test.tfvars - name: Verify outputs @@ -138,6 +164,11 @@ jobs: exit 1 fi + if [[ "${{ steps.output.outputs.complex_output }}" != "2000:3000:udp,4000:5000:tcp" ]]; then + echo "::error:: output complex_output not set correctly" + exit 1 + fi + backend_config_12: runs-on: ubuntu-latest name: backend_config terraform 12 @@ -254,7 +285,7 @@ jobs: with: path: tests/apply/vars label: TestLabel - var: my_var=world + variables: my_var="world" var_file: tests/apply/test.tfvars - name: Apply @@ -263,7 +294,7 @@ jobs: with: path: tests/apply/vars label: TestLabel - var: my_var=world + variables: my_var="world" var_file: tests/apply/test.tfvars - name: Verify outputs @@ -372,3 +403,42 @@ jobs: echo "::error:: output s not set correctly" exit 1 fi + + apply_vars: + runs-on: ubuntu-latest + name: Apply approved changes with deprecated vars + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + path: tests/apply/vars + var: my_var=hello + var_file: tests/apply/test.tfvars + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/apply/vars + var: my_var=hello + var_file: tests/apply/test.tfvars + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi + if [[ "${{ steps.output.outputs.from_var }}" != "this should be overridden" ]]; then + echo "::error:: output from_var not set correctly" + exit 1 + fi + if [[ "${{ steps.output.outputs.from_varfile }}" != "monkey" ]]; then + echo "::error:: output from_varfile not set correctly" + exit 1 + fi diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index e8bdf766..43634562 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -30,7 +30,7 @@ jobs: with: path: tests/new-workspace/remote_12 workspace: my-first-workspace - var: my_string=hello + variables: my_string="hello" auto_approve: true - name: Create a second workspace @@ -44,7 +44,7 @@ jobs: with: path: tests/new-workspace/remote_12 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" auto_approve: true - name: Get first workspace outputs @@ -78,14 +78,14 @@ jobs: with: path: tests/new-workspace/remote_12 workspace: my-first-workspace - var: my_string=hello + variables: my_string="hello" - name: Destroy second workspace uses: ./terraform-destroy-workspace with: path: tests/new-workspace/remote_12 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" create_workspace_13: runs-on: ubuntu-latest @@ -114,7 +114,7 @@ jobs: with: path: tests/new-workspace/remote_13 workspace: my-first-workspace - var: my_string=hello + variables: my_string="hello" auto_approve: true - name: Create a second workspace @@ -128,7 +128,7 @@ jobs: with: path: tests/new-workspace/remote_13 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" auto_approve: true - name: Get first workspace outputs @@ -162,14 +162,14 @@ jobs: with: path: tests/new-workspace/remote_13 workspace: my-first-workspace - var: my_string=hello + variables: my_string="hello" - name: Destroy second workspace uses: ./terraform-destroy-workspace with: path: tests/new-workspace/remote_13 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" create_workspace_14: runs-on: ubuntu-latest @@ -198,7 +198,7 @@ jobs: with: path: tests/new-workspace/remote_14 workspace: test-workspace - var: my_string=hello + variables: my_string="hello" auto_approve: true - name: Create a second workspace @@ -212,7 +212,7 @@ jobs: with: path: tests/new-workspace/remote_14 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" auto_approve: true - name: Get first workspace outputs @@ -246,11 +246,11 @@ jobs: with: path: tests/new-workspace/remote_14 workspace: test-workspace - var: my_string=hello + variables: my_string="hello" - name: Destroy second workspace uses: ./terraform-destroy-workspace with: path: tests/new-workspace/remote_14 workspace: ${{ github.head_ref }} - var: my_string=world + variables: my_string="world" diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 9198652b..e2ccea74 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -245,3 +245,20 @@ jobs: echo "Plan did not fail correctly" exit 1 fi + + plan_single_variable: + runs-on: ubuntu-latest + name: Plan single variable + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + path: tests/apply/vars + variables: | + my_var="single" + var_file: tests/apply/test.tfvars diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ccaacb..eeff3872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,23 @@ When using an action you can specify the version as: - `@v1.8` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Added +- `variables` input for actions that use terraform input variables. + + This value should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + Variable values set in `variables` override any given in var_files. + See action documentation for details, e.g. [terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#inputs). + +### Deprecated +- The `var` input has been deprecated due to the following limitations: + - Only primitive types can be set with `var` - number, bool and string. + - String values may not contain a comma. + - Values set with `var` will be overridden by values contained in `var_file`s + + `variables` is the preferred way to set input variables. + ## [1.8.0] - 2021-04-05 ### Added diff --git a/example_workflows/create_plan.yaml b/example_workflows/create_plan.yaml index 0e41d28d..87e51c49 100644 --- a/example_workflows/create_plan.yaml +++ b/example_workflows/create_plan.yaml @@ -1,9 +1,6 @@ name: Create terraform plan -on: - pull_request: - branches: - - "!master" +on: [pull_request] jobs: plan: diff --git a/example_workflows/validate.yaml b/example_workflows/validate.yaml index 8d36c1a9..25401e0b 100644 --- a/example_workflows/validate.yaml +++ b/example_workflows/validate.yaml @@ -3,7 +3,7 @@ name: Validate changes on: push: branches: - - "!master" + - '!master' jobs: fmt-check: diff --git a/image/actions.sh b/image/actions.sh index d24f7f9e..57e14fc6 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -169,6 +169,11 @@ function set-plan-args() { done fi + if [[ -n "$INPUT_VARIABLES" ]]; then + echo "$INPUT_VARIABLES" > /.terraform-variables.tfvars + PLAN_ARGS="$PLAN_ARGS -var-file=/.terraform-variables.tfvars" + fi + export PLAN_ARGS } diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 52ba372a..3b19c934 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -191,6 +191,19 @@ def _comment_identifier(self): if self.var_files: label += f'\nWith var files: `{self.var_files}`' + if self.variables: + stripped_vars = self.variables.strip() + if '\n' in stripped_vars: + label += f'''
With variables + +```hcl +{stripped_vars} +``` +
+''' + else: + label += f'\nWith variables: `{stripped_vars}`' + return label @property @@ -254,6 +267,10 @@ def has_bad_word(s: str) -> bool: def backend_config_files(self) -> str: return os.environ.get('INPUT_BACKEND_CONFIG_FILE') + @property + def variables(self) -> str: + return os.environ.get('INPUT_VARIABLES') + @property def vars(self) -> str: return os.environ.get('INPUT_VAR') diff --git a/terraform-apply/README.md b/terraform-apply/README.md index ae375309..bc728e10 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -31,6 +31,8 @@ the master branch: ## Inputs +These input values must be the same as any `terraform-plan` for the same configuration. (unless auto_approve: true) + * `path` Path to the terraform configuration to apply @@ -56,9 +58,50 @@ the master branch: - Type: string - Optional -* `var` +* `variables` + + Variables to set for the terraform plan. This should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + + ```yaml + with: + variables: | + image_id = "${{ secrets.AMI_ID }}" + availability_zone_names = [ + "us-east-1a", + "us-west-1c", + ] + ``` + + Variables set here override any given in `var_file`s. + + - Type: string + - Optional + +* ~~`var`~~ - Comma separated list of terraform vars to set + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set. + + This is deprecated due to the following limitations: + - Only primitive types can be set with `var` - number, bool and string. + - String values may not contain a comma. + - Values set with `var` will be overridden by values contained in `var_file`s + + You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. + + For example: + ```yaml + with: + var: instance_type=m5.xlarge,nat_type=instance + ``` + Becomes: + ```yaml + with: + variables: | + instance_type="m5.xlarge" + nat_type="instance" + ``` - Type: string - Optional diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index 1fe7f86a..1469986d 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -18,10 +18,14 @@ inputs: description: Path to a backend config file required: false default: "" + variables: + description: Variable definitions + required: false var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false default: "" + deprecationMessage: Use the variables input instead. var_file: description: Comma separated list of var file paths required: false diff --git a/terraform-check/README.md b/terraform-check/README.md index 45afdbeb..d3741d76 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -23,7 +23,28 @@ This is intended to run on a schedule to notify if manual changes to your infras - Optional - Default: `default` -* `var` +* `variables` + + Variables to set for the terraform plan. This should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + + ```yaml + with: + variables: | + image_id = ${{ secrets.AMI_ID }} + availability_zone_names = [ + "us-east-1a", + "us-west-1c", + ] + ``` + + Variables set here override any given in variable_files. + + - Type: string + - Optional + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. Comma separated list of terraform vars to set diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index d8f950d2..3f3d9404 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -16,9 +16,13 @@ inputs: backend_config_file: description: Path to a backend config file" required: false + variables: + description: Variable definitions + required: false var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false + deprecationMessage: Use the variables input instead. var_file: description: Comma separated list of var file paths required: false diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index fa4ac7c1..1ffe1815 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -20,7 +20,28 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Required -* `var` +* `variables` + + Variables to set for the terraform plan. This should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + + ```yaml + with: + variables: | + image_id = ${{ secrets.AMI_ID }} + availability_zone_names = [ + "us-east-1a", + "us-west-1c", + ] + ``` + + Variables set here override any given in variable_files. + + - Type: string + - Optional + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. Comma separated list of terraform vars to set diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index 90d8c606..b9678f67 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -15,9 +15,13 @@ inputs: backend_config_file: description: Path to a backend config file required: false + variables: + description: Variable definitions + required: false var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false + deprecationMessage: Use the variables input instead. var_file: description: Comma separated list of var file paths required: false diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index f40a2aa4..99e7b5c2 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -22,7 +22,28 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Optional - Default: `default` -* `var` +* `variables` + + Variables to set for the terraform destroy. This should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + + ```yaml + with: + variables: | + image_id = ${{ secrets.AMI_ID }} + availability_zone_names = [ + "us-east-1a", + "us-west-1c", + ] + ``` + + Variables set here override any given in variable_files. + + - Type: string + - Optional + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. Comma separated list of terraform vars to set diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 18148b91..8e15823f 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -16,6 +16,10 @@ inputs: backend_config_file: description: Path to a backend config file required: false + variables: + description: Variable definitions + required: false + deprecationMessage: Use the variables input instead. var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false diff --git a/terraform-output/README.md b/terraform-output/README.md index 13486318..5e645b0c 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -164,5 +164,5 @@ jobs: Which will print to the workflow log: ``` The vpc-id is vpc-01463b6b84e1454ce -The subnet-ids are is subnet-053008016a2c1768c,subnet-07d4ce437c43eba2f,subnet-0a5f8c3a20023b8c0 +The subnet-ids are subnet-053008016a2c1768c,subnet-07d4ce437c43eba2f,subnet-0a5f8c3a20023b8c0 ``` diff --git a/terraform-plan/README.md b/terraform-plan/README.md index bafa61fd..cd360cea 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -41,9 +41,50 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Type: string - Optional -* `var` +* `variables` - Comma separated list of terraform vars to set + Variables to set for the terraform plan. This should be valid terraform syntax - like a [variable definition file](https://www.terraform.io/docs/language/values/variables.html#variable-definitions-tfvars-files). + + ```yaml + with: + variables: | + image_id = ${{ secrets.AMI_ID }} + availability_zone_names = [ + "us-east-1a", + "us-west-1c", + ] + ``` + + Variables set here override any given in variable_files. + + - Type: string + - Optional + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set. + + This is deprecated due to the following limitations: + - Only primitive types can be set with `var` - number, bool and string. + - String values may not contain a comma. + - Values set with `var` will be overridden by values contained in `var_file`s + + You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. + + For example: + ```yaml + with: + var: instance_type=m5.xlarge,nat_type=instance + ``` + Becomes: + ```yaml + with: + variables: | + instance_type="m5.xlarge" + nat_type="instance" + ``` - Type: string - Optional @@ -195,6 +236,45 @@ jobs: path: my-terraform-config ``` +### A full example of inputs + +This example workflow demonstrates most of the available inputs: +- The environment variables are set at the workflow level. +- The PR comment will be labelled `production`, and the plan will use the `prod` workspace. +- Variables are read from `env/prod.tfvars`, with `turbo_mode` overridden to `true`. +- The backend config is taken from `env/prod.backend`, and the token is set from a secret. + +```yaml +name: PR Plan + +on: [pull_request] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TERRAFORM_CLOUD_TOKENS: terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} + +jobs: + plan: + runs-on: ubuntu-latest + name: Create terraform plan + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform plan + uses: dflook/terraform-plan@v1 + with: + path: my-terraform-config + label: production + workspace: prod + var_file: env/prod.tfvars + variables: + turbo_mode=true + backend_config_file: env/prod.backend + backend_config: token=${{ secrets.BACKEND_TOKEN }} +``` + ### Generating a plan using a comment This workflow generates a plan on demand, triggered by someone diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 3710c0fe..7249a815 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -16,9 +16,13 @@ inputs: backend_config_file: description: Path to a backend config file required: false + variables: + description: Variable definitions + required: false var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false + deprecationMessage: Use the variables input instead. var_file: description: Comma separated list of var file paths required: false diff --git a/tests/apply/test.tfvars b/tests/apply/test.tfvars index 7d2b1f3a..368d66db 100644 --- a/tests/apply/test.tfvars +++ b/tests/apply/test.tfvars @@ -1 +1,2 @@ -my_var_from_file="monkey" \ No newline at end of file +my_var_from_file="monkey" +my_var="this should be overridden" diff --git a/tests/apply/vars/main.tf b/tests/apply/vars/main.tf index 63c74f93..2e928fee 100644 --- a/tests/apply/vars/main.tf +++ b/tests/apply/vars/main.tf @@ -8,10 +8,27 @@ output "output_string" { variable "my_var" { type = string + default = "my_var_default" } variable "my_var_from_file" { type = string + default = "my_var_from_file_default" +} + +variable "complex_input" { + type = list(object({ + internal = number + external = number + protocol = string + })) + default = [ + { + internal = 8300 + external = 8300 + protocol = "tcp" + } + ] } output "from_var" { @@ -21,3 +38,7 @@ output "from_var" { output "from_varfile" { value = var.my_var_from_file } + +output "complex_output" { + value = join(",", [for input in var.complex_input : "${input.internal}:${input.external}:${input.protocol}"]) +} From 13bc2fa364a9f70ffef86bd3e95fb928fee6df47 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 10 Apr 2021 16:24:05 +0100 Subject: [PATCH 037/231] :bookmark: v1.9.0 --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeff3872..8c2aeab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.8.0` to use an exact release -- `@v1.8` to use the latest patch release for the specific minor version +- `@v1.9.0` to use an exact release +- `@v1.9` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.9.0] - 2021-04-10 ### Added - `variables` input for actions that use terraform input variables. @@ -144,6 +144,7 @@ First release of the GitHub Actions: - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.9.0]: https://github.com/dflook/terraform-github-actions/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/dflook/terraform-github-actions/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/dflook/terraform-github-actions/compare/v1.6.0...v1.7.0 [1.6.0]: https://github.com/dflook/terraform-github-actions/compare/v1.5.2...v1.6.0 From dbe33a8553714b247a741ff55372e68d7e9d6e3c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 10 Apr 2021 18:13:16 +0100 Subject: [PATCH 038/231] :pencil: Fix typo --- terraform-check/README.md | 2 +- terraform-destroy-workspace/README.md | 2 +- terraform-destroy/README.md | 2 +- terraform-plan/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform-check/README.md b/terraform-check/README.md index d3741d76..6f29f6b9 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -37,7 +37,7 @@ This is intended to run on a schedule to notify if manual changes to your infras ] ``` - Variables set here override any given in variable_files. + Variables set here override any given in `var_file`s. - Type: string - Optional diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 1ffe1815..3315cfb8 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -34,7 +34,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t ] ``` - Variables set here override any given in variable_files. + Variables set here override any given in `var_file`s. - Type: string - Optional diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 99e7b5c2..236370be 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -36,7 +36,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t ] ``` - Variables set here override any given in variable_files. + Variables set here override any given in `var_file`s. - Type: string - Optional diff --git a/terraform-plan/README.md b/terraform-plan/README.md index cd360cea..47d54199 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -55,7 +55,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ ] ``` - Variables set here override any given in variable_files. + Variables set here override any given in `var_file`s. - Type: string - Optional From 2aa0dc68c72d6d5b843590a38c50b8104fe02ca9 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 14 Apr 2021 18:38:45 +0100 Subject: [PATCH 039/231] Add job runs badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa3fa8d2..7882eded 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Terraform GitHub Actions ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/dflook/terraform-github-actions) +# Terraform GitHub Actions ![release](https://img.shields.io/github/v/release/dflook/terraform-github-actions)![job runs](https://img.shields.io/docker/pulls/danielflook/terraform-github-actions?label=job%20runs) This is a suite of terraform related GitHub Actions that can be used together to build effective Infrastructure as Code workflows. From 91c71184060d0fac7c69256377be5c83c32ea40f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 14 Apr 2021 18:48:20 +0100 Subject: [PATCH 040/231] Update test for terraform 0.15.0 release --- .github/workflows/test-version.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-version.yaml b/.github/workflows/test-version.yaml index 2d6df343..7a715d81 100644 --- a/.github/workflows/test-version.yaml +++ b/.github/workflows/test-version.yaml @@ -113,7 +113,7 @@ jobs: - name: Check the version run: | - if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"0.14"* ]]; then + if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"0.15"* ]]; then echo "::error:: Latest version was not used" exit 1 fi From df075fcb0161047f340cf2e8b33d14390c0038b3 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 19 Apr 2021 19:17:16 +0100 Subject: [PATCH 041/231] Fix missing quotes in examples --- terraform-check/README.md | 2 +- terraform-destroy-workspace/README.md | 2 +- terraform-destroy/README.md | 2 +- terraform-plan/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform-check/README.md b/terraform-check/README.md index 6f29f6b9..e0ec4f2a 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -30,7 +30,7 @@ This is intended to run on a schedule to notify if manual changes to your infras ```yaml with: variables: | - image_id = ${{ secrets.AMI_ID }} + image_id = "${{ secrets.AMI_ID }}" availability_zone_names = [ "us-east-1a", "us-west-1c", diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 3315cfb8..37b41a4a 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -27,7 +27,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t ```yaml with: variables: | - image_id = ${{ secrets.AMI_ID }} + image_id = "${{ secrets.AMI_ID }}" availability_zone_names = [ "us-east-1a", "us-west-1c", diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 236370be..2f58a08f 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -29,7 +29,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t ```yaml with: variables: | - image_id = ${{ secrets.AMI_ID }} + image_id = "${{ secrets.AMI_ID }}" availability_zone_names = [ "us-east-1a", "us-west-1c", diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 47d54199..dab5265c 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -48,7 +48,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ ```yaml with: variables: | - image_id = ${{ secrets.AMI_ID }} + image_id = "${{ secrets.AMI_ID }}" availability_zone_names = [ "us-east-1a", "us-west-1c", From d4aaaf5e87c60d01943380ccacb2dca5dfa1e7cf Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 19 Apr 2021 19:47:29 +0100 Subject: [PATCH 042/231] Add warning about unmasked variables --- terraform-plan/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform-plan/README.md b/terraform-plan/README.md index dab5265c..ab5d5a7c 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -57,6 +57,8 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ Variables set here override any given in `var_file`s. + > :warning: Secret values are not masked in the PR comment. Set a `label` to avoid revealing the variables in the PR. + - Type: string - Optional From 7219a09a55d8cddd8874ee9cb946141d250364cc Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 21 Apr 2021 09:09:56 +0100 Subject: [PATCH 043/231] Fix extracting terraform 0.15 plan --- .github/workflows/test-apply.yaml | 38 +++++++++++++++++++++++++++++++ .github/workflows/test-plan.yaml | 25 ++++++++++++++++++++ image/tools/compact_plan.py | 1 + tests/apply/refresh_15/main.tf | 13 +++++++++++ tests/plan/plan_15/main.tf | 11 +++++++++ 5 files changed, 88 insertions(+) create mode 100644 tests/apply/refresh_15/main.tf create mode 100644 tests/plan/plan_15/main.tf diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index c78ac7f6..1cdbaf9f 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -442,3 +442,41 @@ jobs: echo "::error:: output from_varfile not set correctly" exit 1 fi + + apply_refresh: + runs-on: ubuntu-latest + name: Apply changes are refresh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan 1 + uses: ./terraform-plan + with: + label: Refresh 1 + path: tests/apply/refresh_15 + variables: len=10 + + - name: Apply 1 + uses: ./terraform-apply + with: + label: Refresh 1 + path: tests/apply/refresh_15 + variables: len=10 + + - name: Plan 2 + uses: ./terraform-plan + with: + label: Refresh 2 + path: tests/apply/refresh_15 + variables: len=20 + + - name: Apply 2 + uses: ./terraform-apply + id: output + with: + label: Refresh 2 + path: tests/apply/refresh_15 + variables: len=20 diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index e2ccea74..0a373452 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -139,6 +139,31 @@ jobs: exit 1 fi + plan_change_comment_15: + runs-on: ubuntu-latest + name: Change terraform 15 + env: + TF_PLAN_COLLAPSE_LENGTH: 30 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_15 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + plan_change_comment_latest: runs-on: ubuntu-latest name: Change latest terraform diff --git a/image/tools/compact_plan.py b/image/tools/compact_plan.py index b7439f02..215bf2db 100755 --- a/image/tools/compact_plan.py +++ b/image/tools/compact_plan.py @@ -10,6 +10,7 @@ def compact_plan(input): for line in input: if not plan and ( + line.startswith('Terraform used the selected providers') or line.startswith('An execution plan has been generated and is shown below') or line.startswith('No changes') or line.startswith('Error') diff --git a/tests/apply/refresh_15/main.tf b/tests/apply/refresh_15/main.tf new file mode 100644 index 00000000..b874b220 --- /dev/null +++ b/tests/apply/refresh_15/main.tf @@ -0,0 +1,13 @@ +resource "random_string" "my_string" { + length = var.len +} + +output "s" { + value = "${random_string.my_string}" +} + +terraform { + required_version = "~> 0.15.0" +} + +variable "len" {} diff --git a/tests/plan/plan_15/main.tf b/tests/plan/plan_15/main.tf new file mode 100644 index 00000000..20c8aa84 --- /dev/null +++ b/tests/plan/plan_15/main.tf @@ -0,0 +1,11 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} + +terraform { + required_version = "~> 0.15.0" +} From 46179c727b42746cd9dbe88466815583ef9f2aa0 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 21 Apr 2021 09:48:27 +0100 Subject: [PATCH 044/231] :bookmark: v1.9.1 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c2aeab3..8ed402e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.9.0` to use an exact release +- `@v1.9.1` to use an exact release - `@v1.9` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.9.1] - 2021-04-21 + +### Fixed +- Terraform 0.15 plans were not being extracted correctly, causing failures to apply. + ## [1.9.0] - 2021-04-10 ### Added @@ -144,6 +149,7 @@ First release of the GitHub Actions: - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.9.1]: https://github.com/dflook/terraform-github-actions/compare/v1.9.0...v1.9.1 [1.9.0]: https://github.com/dflook/terraform-github-actions/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/dflook/terraform-github-actions/compare/v1.7.0...v1.8.0 [1.7.0]: https://github.com/dflook/terraform-github-actions/compare/v1.6.0...v1.7.0 From a51f3950eb994e23b711ed18c292e73538f72ea7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 5 May 2021 09:37:30 +0100 Subject: [PATCH 045/231] Don't include state lock status in plan These line may or may not appear depending on if the use is waiting. They should really be supressed by TF_IN_AUTOMATION.... --- image/tools/compact_plan.py | 4 +- tests/test_compact_plan.py | 124 ++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/image/tools/compact_plan.py b/image/tools/compact_plan.py index 215bf2db..feada2e7 100755 --- a/image/tools/compact_plan.py +++ b/image/tools/compact_plan.py @@ -18,7 +18,9 @@ def compact_plan(input): plan = True if plan: - yield line + if not (line.startswith('Releasing state lock. This may take a few moments...') + or line.startswith('Acquiring state lock. This may take a few moments...')): + yield line else: buffer.append(line) diff --git a/tests/test_compact_plan.py b/tests/test_compact_plan.py index 3c76e1ea..599df6bd 100644 --- a/tests/test_compact_plan.py +++ b/tests/test_compact_plan.py @@ -169,6 +169,64 @@ def test_plan_14(): output = '\n'.join(compact_plan(input.splitlines())) assert output == expected_output +def test_plan_15(): + input = """ + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string" +""" + + expected_output = """Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string\"""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + + def test_error_11(): input = """ Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax @@ -256,3 +314,69 @@ def test_no_output(): output = '\n'.join(compact_plan(input.splitlines())) assert output == expected_output +def test_state_lock_12(): + input = """Acquiring state lock. This may take a few moments... +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +Acquiring state lock. This may take a few moments... +Releasing state lock. This may take a few moments... + +------------------------------------------------------------------------ +Acquiring state lock. This may take a few moments... + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create +Acquiring state lock. This may take a few moments... + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true +Acquiring state lock. This may take a few moments... + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true +Releasing state lock. This may take a few moments... + } + +Plan: 1 to add, 0 to change, 0 to destroy. +Releasing state lock. This may take a few moments... +Releasing state lock. This may take a few moments... +""" + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output From 13df1d1a14ffa2a0e48ae32f2b9ada090d49d74b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 5 May 2021 09:54:22 +0100 Subject: [PATCH 046/231] :bookmark: v1.9.1 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed402e4..5732212e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.9.1` to use an exact release +- `@v1.9.2` to use an exact release - `@v1.9` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.9.2] - 2021-05-05 + +### Fixed +- Slow state locking messages were being considered part of the plan, which could cause applys to be aborted. + ## [1.9.1] - 2021-04-21 ### Fixed @@ -149,6 +154,7 @@ First release of the GitHub Actions: - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.9.2]: https://github.com/dflook/terraform-github-actions/compare/v1.9.1...v1.9.2 [1.9.1]: https://github.com/dflook/terraform-github-actions/compare/v1.9.0...v1.9.1 [1.9.0]: https://github.com/dflook/terraform-github-actions/compare/v1.8.0...v1.9.0 [1.8.0]: https://github.com/dflook/terraform-github-actions/compare/v1.7.0...v1.8.0 From 51d488afdd2767f1d65b49bbdd9fa9d2b6537948 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 5 May 2021 09:55:40 +0100 Subject: [PATCH 047/231] :bookmark: v1.9.2 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5732212e..19a192c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ When using an action you can specify the version as: ## [1.9.2] - 2021-05-05 ### Fixed -- Slow state locking messages were being considered part of the plan, which could cause applys to be aborted. +- Slow state locking messages were being considered part of the plan, which could cause apply actions to be aborted. ## [1.9.1] - 2021-04-21 From adb1eeb31024eb59ba6355fcf28d4d7193e1815a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 5 May 2021 10:33:15 +0100 Subject: [PATCH 048/231] Update base image every week Don't rebuild base when releasing --- .github/workflows/base-image.yaml | 2 ++ .github/workflows/release.yaml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index ee4c423f..54ebd718 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -6,6 +6,8 @@ on: - master paths: - image/Dockerfile-base + schedule: + - cron: "0 1 * * 1" jobs: push_image: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f37c86eb..9fa779e8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,8 +22,6 @@ jobs: - name: Action image run: | - docker build --tag dflook/terraform-github-actions-base -f image/Dockerfile-base image - RELEASE_TAG=$(cat $GITHUB_EVENT_PATH | jq -r '.release.tag_name' $GITHUB_EVENT_PATH) docker build --tag dflook/terraform-github-actions \ From 80da91d2ea4571c452f7ce8e6b4102baf48a5c2d Mon Sep 17 00:00:00 2001 From: "Mikhail S. Pabalavets" Date: Thu, 6 May 2021 12:01:53 +0300 Subject: [PATCH 049/231] Fix typo --- terraform-apply/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index bc728e10..49e9d1d2 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -50,7 +50,7 @@ These input values must be the same as any `terraform-plan` for the same configu * `label` - An friendly name for the environment the terraform configuration is for. + A friendly name for the environment the terraform configuration is for. This will be used in the PR comment for easy identification. It must be the same as the `label` used in the corresponding `terraform-plan` command. From 54fca1a8827fb874cab955b7ebac64507d5eb0f4 Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 10:29:03 +0100 Subject: [PATCH 050/231] allow shell commands to be executed before terraform is ran --- image/actions.sh | 9 +++++++++ terraform-apply/action.yaml | 3 +++ terraform-check/action.yaml | 3 +++ terraform-destroy-workspace/action.yaml | 3 +++ terraform-destroy/action.yaml | 3 +++ terraform-new-workspace/action.yaml | 3 +++ terraform-plan/action.yaml | 3 +++ 7 files changed, 27 insertions(+) diff --git a/image/actions.sh b/image/actions.sh index 57e14fc6..d4b6bd1f 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -51,6 +51,13 @@ function detect-tfmask() { export TFMASK } +function execute_run_commands() { + if [[ -z $INPUT_RUN ]]; then + echo "Executing init commands specified in 'run' parameter" + $INPUT_RUN + fi +} + function setup() { TERRAFORM_BIN_DIR="$HOME/.dflook-terraform-bin-dir" export TF_DATA_DIR="$HOME/.dflook-terraform-data-dir" @@ -86,6 +93,8 @@ function setup() { debug_cmd ls -la $TERRAFORM_BIN_DIR detect-tfmask + + execute_run_commands } function relative_to() { diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index 1469986d..08bb444e 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -45,6 +45,9 @@ inputs: description: "Comma separated list of targets to apply against, e.g. 'kubernetes_secret.tls_cert_public,kubernetes_secret.tls_cert_private' NOTE: this argument only takes effect if auto_approve is also set." required: false default: "" + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index 3f3d9404..3a80c8d5 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -30,6 +30,9 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index b9678f67..2011576a 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -29,6 +29,9 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 8e15823f..08dd0f40 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -30,6 +30,9 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker diff --git a/terraform-new-workspace/action.yaml b/terraform-new-workspace/action.yaml index 65672b4c..945d9154 100644 --- a/terraform-new-workspace/action.yaml +++ b/terraform-new-workspace/action.yaml @@ -15,6 +15,9 @@ inputs: backend_config_file: description: Path to a backend config file" required: false + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 7249a815..8b98c3e0 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -38,6 +38,9 @@ inputs: description: Add the plan to a GitHub PR required: false default: true + run: + description: Executes these shell commands before running terraform + required: false outputs: changes: From d6fee9662bb357c8182d828035f5834003f1e875 Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 10:34:03 +0100 Subject: [PATCH 051/231] missed one --- terraform-remote-state/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform-remote-state/action.yaml b/terraform-remote-state/action.yaml index ec532068..d4f025ed 100644 --- a/terraform-remote-state/action.yaml +++ b/terraform-remote-state/action.yaml @@ -16,6 +16,9 @@ inputs: backend_config_file: description: Path to a backend config file required: false + run: + description: Executes these shell commands before running terraform + required: false runs: using: docker From 72fe884cf596671f4ea62e17a34614b465a05914 Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 11:37:39 +0100 Subject: [PATCH 052/231] test new 'run' parameter --- .github/workflows/test-plan.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 0a373452..44c4e370 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -287,3 +287,21 @@ jobs: variables: | my_var="single" var_file: tests/apply/test.tfvars + + plan_change_run_commands: + runs-on: ubuntu-latest + name: Change with shell init commands + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + path: tests/plan/plan + run: | + echo testing command 1 + echo testing command 2 + From 6fb18ea3d7cae20b8046749067ad9f214195f95e Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 12:16:20 +0100 Subject: [PATCH 053/231] testing --- image/actions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/image/actions.sh b/image/actions.sh index d4b6bd1f..13f98746 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -52,6 +52,8 @@ function detect-tfmask() { } function execute_run_commands() { + echo "execute_run_commands" + echo "$INPUT_RUN" if [[ -z $INPUT_RUN ]]; then echo "Executing init commands specified in 'run' parameter" $INPUT_RUN From 3ff4fd16bf1cdc587b42a9361229b73738377a12 Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 12:18:04 +0100 Subject: [PATCH 054/231] remove testing --- image/actions.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 13f98746..d4b6bd1f 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -52,8 +52,6 @@ function detect-tfmask() { } function execute_run_commands() { - echo "execute_run_commands" - echo "$INPUT_RUN" if [[ -z $INPUT_RUN ]]; then echo "Executing init commands specified in 'run' parameter" $INPUT_RUN From 5fd423f41bdd1b597213847091fa97d13995c001 Mon Sep 17 00:00:00 2001 From: Alec Date: Wed, 12 May 2021 12:51:35 +0100 Subject: [PATCH 055/231] woops, corrected operator and also execute command with `eval` --- image/actions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index d4b6bd1f..cc7d08ca 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -52,9 +52,9 @@ function detect-tfmask() { } function execute_run_commands() { - if [[ -z $INPUT_RUN ]]; then + if [[ -n $INPUT_RUN ]]; then echo "Executing init commands specified in 'run' parameter" - $INPUT_RUN + eval "$INPUT_RUN" fi } From 230d58215e09ffcc87302762efaa8fef45f785c8 Mon Sep 17 00:00:00 2001 From: Alec Date: Mon, 17 May 2021 08:34:06 +0100 Subject: [PATCH 056/231] remove un-needed parameter from remote state and new-workspace actions --- terraform-new-workspace/action.yaml | 3 --- terraform-remote-state/action.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/terraform-new-workspace/action.yaml b/terraform-new-workspace/action.yaml index 945d9154..65672b4c 100644 --- a/terraform-new-workspace/action.yaml +++ b/terraform-new-workspace/action.yaml @@ -15,9 +15,6 @@ inputs: backend_config_file: description: Path to a backend config file" required: false - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker diff --git a/terraform-remote-state/action.yaml b/terraform-remote-state/action.yaml index d4f025ed..ec532068 100644 --- a/terraform-remote-state/action.yaml +++ b/terraform-remote-state/action.yaml @@ -16,9 +16,6 @@ inputs: backend_config_file: description: Path to a backend config file required: false - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker From ded64cb9e2349949f47cbc7935d05afd34784d69 Mon Sep 17 00:00:00 2001 From: Alec Date: Mon, 17 May 2021 08:34:30 +0100 Subject: [PATCH 057/231] add option to use `TERRAFORM_PRE_RUN` env variable --- image/actions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/image/actions.sh b/image/actions.sh index cc7d08ca..ee4a42d1 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -56,6 +56,10 @@ function execute_run_commands() { echo "Executing init commands specified in 'run' parameter" eval "$INPUT_RUN" fi + if [[ -n $TERRAFORM_PRE_RUN ]] + echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" + eval "$TERRAFORM_PRE_RUN" + fi } function setup() { From 4fa4ed426d1aa04ccc680c07afc834e83c5f5b3b Mon Sep 17 00:00:00 2001 From: Alec Date: Mon, 17 May 2021 08:42:09 +0100 Subject: [PATCH 058/231] add `TERRAFORM_PRE_RUN` to docs --- terraform-apply/README.md | 18 ++++++++++++++++++ terraform-check/README.md | 17 +++++++++++++++++ terraform-destroy-workspace/README.md | 18 ++++++++++++++++++ terraform-destroy/README.md | 18 ++++++++++++++++++ terraform-new-workspace/README.md | 18 ++++++++++++++++++ terraform-output/README.md | 18 ++++++++++++++++++ terraform-plan/README.md | 18 ++++++++++++++++++ terraform-validate/README.md | 18 ++++++++++++++++++ terraform-version/README.md | 18 ++++++++++++++++++ 9 files changed, 161 insertions(+) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index bc728e10..0f3dee9d 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -209,6 +209,24 @@ These input values must be the same as any `terraform-plan` for the same configu - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-check/README.md b/terraform-check/README.md index e0ec4f2a..16784bed 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -121,6 +121,23 @@ This is intended to run on a schedule to notify if manual changes to your infras - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional ## Example usage diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 37b41a4a..0e66fbc8 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -118,6 +118,24 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Example usage This example deletes the workspace named after the git branch when the associated PR is closed. diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 2f58a08f..7ecbf4c4 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -120,6 +120,24 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Example usage This example destroys the resources in a workspace named after the git branch when the associated PR is closed. diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 8a6e2636..4e15c677 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -74,6 +74,24 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Example usage This example creates a workspace named after the git branch when the diff --git a/terraform-output/README.md b/terraform-output/README.md index 5e645b0c..75c10ffa 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -75,6 +75,24 @@ Retrieve the root-level outputs from a terraform configuration. - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-plan/README.md b/terraform-plan/README.md index ab5d5a7c..312d48ed 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -200,6 +200,24 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Optional - Default: 10 +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Outputs * `changes` diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 018ccfa9..2145c85d 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -61,6 +61,24 @@ If the terraform configuration is not valid, the build is failed. - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs on every push and fails if the terraform diff --git a/terraform-version/README.md b/terraform-version/README.md index 3d6dfb24..4cad6698 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -66,6 +66,24 @@ outputs yourself. - Type: string - Optional +* `TERRAFORM_PRE_RUN` + + A set of commands that will be ran prior to `terraform init`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + ## Outputs * `terraform` From 539bfa7c9882d9c09534edd8a4e26a8cbd39c5a2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 29 May 2021 18:43:01 +0100 Subject: [PATCH 059/231] Fix 0.15.4 plan with only changes to outputs --- .github/github_sucks.md | 1 - .github/workflows/test-plan.yaml | 24 ++++++++++++++++++++++++ image/tools/github_pr_comment.py | 5 ++++- tests/plan/plan_15/main.tf | 2 +- tests/plan/plan_15_4/main.tf | 11 +++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/plan/plan_15_4/main.tf diff --git a/.github/github_sucks.md b/.github/github_sucks.md index cee65b7f..471620e2 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,3 +1,2 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. - diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 0a373452..13b03069 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -164,6 +164,30 @@ jobs: exit 1 fi + plan_change_comment_15_4: + runs-on: ubuntu-latest + name: Change terraform 15.4 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan_15_4 + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + plan_change_comment_latest: runs-on: ubuntu-latest name: Change latest terraform diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 3b19c934..dad111e2 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -367,7 +367,10 @@ def summary(self) -> str: summary = line if line.startswith('Changes to Outputs'): - return summary + ' Changes to Outputs.' + if summary: + return summary + ' Changes to Outputs.' + else: + return 'Changes to Outputs' return summary diff --git a/tests/plan/plan_15/main.tf b/tests/plan/plan_15/main.tf index 20c8aa84..f3fcb3bc 100644 --- a/tests/plan/plan_15/main.tf +++ b/tests/plan/plan_15/main.tf @@ -7,5 +7,5 @@ output "s" { } terraform { - required_version = "~> 0.15.0" + required_version = "0.15.3" } diff --git a/tests/plan/plan_15_4/main.tf b/tests/plan/plan_15_4/main.tf new file mode 100644 index 00000000..36a07973 --- /dev/null +++ b/tests/plan/plan_15_4/main.tf @@ -0,0 +1,11 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} + +terraform { + required_version = "~>0.15.4" +} From 052c63cec933eaf8d7adda211d831461dbc07852 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 29 May 2021 19:15:28 +0100 Subject: [PATCH 060/231] :bookmark: v1.9.3 --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19a192c2..5bb71582 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.9.2` to use an exact release +- `@v1.9.3` to use an exact release - `@v1.9` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.9.3] - 2021-05-29 + +### Fixed +- With terraform 0.15.4, terraform-plan jobs that only had changes to outputs would fail when creating a PR comment. + ## [1.9.2] - 2021-05-05 ### Fixed From f4b426c7c7ea1ad8c27c12274c1035c0ba01c989 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 29 May 2021 19:22:14 +0100 Subject: [PATCH 061/231] :bookmark: v1.9.3 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb71582..9e237d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,6 +159,7 @@ First release of the GitHub Actions: - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.9.3]: https://github.com/dflook/terraform-github-actions/compare/v1.9.2...v1.9.3 [1.9.2]: https://github.com/dflook/terraform-github-actions/compare/v1.9.1...v1.9.2 [1.9.1]: https://github.com/dflook/terraform-github-actions/compare/v1.9.0...v1.9.1 [1.9.0]: https://github.com/dflook/terraform-github-actions/compare/v1.8.0...v1.9.0 From ec5d6f6764cdb1b301f5a1639cc779b5cd343156 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 29 May 2021 11:19:36 +0100 Subject: [PATCH 062/231] Add TERRAFORM_HTTP_CREDENTIALS --- .github/workflows/test-http.yaml | 153 ++++++++++++++++++ CHANGELOG.md | 11 +- image/Dockerfile | 5 + image/actions.sh | 3 +- image/tools/http_credential_actions_helper.py | 108 +++++++++++++ terraform-apply/README.md | 25 +++ terraform-check/README.md | 24 +++ terraform-destroy-workspace/README.md | 25 +++ terraform-destroy/README.md | 25 +++ terraform-new-workspace/README.md | 25 +++ terraform-output/README.md | 25 +++ terraform-plan/README.md | 27 +++- terraform-validate/README.md | 25 +++ terraform-version/README.md | 25 +++ tests/git-http-module/main.tf | 7 + tests/http-module/main.tf | 7 + tests/infra/http-auth.py | 34 ++++ tests/test_git_credential_actions.py | 152 +++++++++++++++++ 18 files changed, 702 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-http.yaml create mode 100755 image/tools/http_credential_actions_helper.py create mode 100644 tests/git-http-module/main.tf create mode 100644 tests/http-module/main.tf create mode 100644 tests/infra/http-auth.py create mode 100644 tests/test_git_credential_actions.py diff --git a/.github/workflows/test-http.yaml b/.github/workflows/test-http.yaml new file mode 100644 index 00000000..0fef73bc --- /dev/null +++ b/.github/workflows/test-http.yaml @@ -0,0 +1,153 @@ +name: Test HTTP Credentials + +on: [pull_request] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + git_http_full_path_credentials: + runs-on: ubuntu-latest + name: git+http module source + env: + TERRAFORM_HTTP_CREDENTIALS: | + github.com/dflook/hello=dflook:notapassword + github.com/hello=dflook:stillnotapassword + github.com/dflook/terraform-github-actions-dev.git=dflook:${{ secrets.USER_GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/git-http-module + auto_approve: true + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.git_https }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + git_http_partial_path_credentials: + runs-on: ubuntu-latest + name: git+http module source + env: + TERRAFORM_HTTP_CREDENTIALS: | + github.com/dflook/hello=dflook:notapassword + github.com/hello=dflook:stillnotapassword + github.com/dflook=dflook:${{ secrets.USER_GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/git-http-module + auto_approve: true + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.git_https }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + git_http_no_path_credentials: + runs-on: ubuntu-latest + name: git+http module source + env: + TERRAFORM_HTTP_CREDENTIALS: | + github.com/dflook/hello=dflook:notapassword + github.com/hello=dflook:stillnotapassword + github.com=dflook:${{ secrets.USER_GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/git-http-module + auto_approve: true + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.git_https }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + git_no_credentials: + runs-on: ubuntu-latest + name: git_http module source with no key + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + continue-on-error: true + id: apply + with: + path: tests/git-http-module + auto_approve: true + + - name: Check failed + run: | + if [[ "${{ steps.apply.outcome }}" != "failure" ]]; then + echo "did not fail correctly with no http credentials" + exit 1 + fi + + http_credentials: + runs-on: ubuntu-latest + name: http module source + env: + TERRAFORM_HTTP_CREDENTIALS: | + 5qcb7mjppk.execute-api.eu-west-2.amazonaws.com=dflook:hello + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/http-module + auto_approve: true + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.https }}" != "hello" ]]; then + echo "::error:: output not set correctly" + exit 1 + fi + + http_no_credentials: + runs-on: ubuntu-latest + name: http module source with no credentials + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply + uses: ./terraform-apply + continue-on-error: true + id: apply + with: + path: tests/http-module + auto_approve: true + + - name: Check failed + run: | + if [[ "${{ steps.apply.outcome }}" != "failure" ]]; then + echo "did not fail correctly with no http credentials" + exit 1 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e237d97..26e23a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,17 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.9.3` to use an exact release -- `@v1.9` to use the latest patch release for the specific minor version +- `@v1.10.0` to use an exact release +- `@v1.10` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.10.0] - Unreleased + +### Added + +- `TERRAFORM_HTTP_CREDENTIALS` environment variable for configuring the username and password to use for + http module sources + ## [1.9.3] - 2021-05-29 ### Fixed diff --git a/image/Dockerfile b/image/Dockerfile index a4f3f73d..cabe1b3d 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -17,4 +17,9 @@ RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \ && echo "IdentityFile /.ssh/id_rsa" >> /etc/ssh/ssh_config \ && mkdir -p /.ssh +COPY tools/http_credential_actions_helper.py /usr/bin/git-credential-actions +RUN git config --system credential.helper /usr/bin/git-credential-actions \ + && git config --system credential.useHttpPath true \ + && ln -s /usr/bin/git-credential-actions /usr/bin/netrc-credential-actions + LABEL org.opencontainers.image.title="GitHub actions for terraform" diff --git a/image/actions.sh b/image/actions.sh index 57e14fc6..ed563c8b 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -196,8 +196,9 @@ function random_string() { function write_credentials() { format_tf_credentials >> $HOME/.terraformrc - + netrc-credential-actions >> $HOME/.netrc echo "$TERRAFORM_SSH_KEY" >> /.ssh/id_rsa chmod 600 /.ssh/id_rsa chmod 700 /.ssh + debug_cmd git config --list } diff --git a/image/tools/http_credential_actions_helper.py b/image/tools/http_credential_actions_helper.py new file mode 100755 index 00000000..1bfaf82e --- /dev/null +++ b/image/tools/http_credential_actions_helper.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +import sys +import os +import re +from typing import Dict, List, Iterable, Optional +from dataclasses import dataclass + + +@dataclass +class Credential: + hostname: str + path: List[str] + username: str + password: str + + +def git_credential(operation: str, attributes: Dict[str, str], credentials: List[Credential]): + att = attributes.copy() + sys.stderr.write(repr(att) + '\n') + + if operation != 'get': + return att + + if att.get('protocol') not in ['http', 'https']: + return att + + for cred in credentials: + if att.get('host') != cred.hostname: + continue + + if 'username' in att and att['username'] != cred.username: + continue + + if cred.path != split_path(att.get('path', ''))[:len(cred.path)]: + continue + + # Path matches + att['username'] = cred.username + att['password'] = cred.password + + path = '/'.join(cred.path) + sys.stderr.write(f'Using TERRAFORM_HTTP_CREDENTIALS for {cred.hostname}{f"/{path}" if path else ""}={cred.username}\n') + break + else: + path = att.get('path', '') + sys.stderr.write(f'No matching credentials found in TERRAFORM_HTTP_CREDENTIALS for {att.get("host")}{f"/{path}" if path else ""}\n') + + return att + +def split_path(path: Optional[str]) -> List[str]: + if path is None: + return [] + return [segment for segment in path.split('/') if segment] + + +def read_attributes(att_string: str) -> Dict[str, str]: + attributes = {} + for line in att_string.splitlines(): + match = re.match(r'^(.+?)=(.+)$', line) + if match: + attributes[match.group(1)] = match.group(2) + + return attributes + + +def write_attributes(attributes: Dict[str, str]) -> str: + return '\n'.join(f'{k}={v}' for k, v in attributes.items()) + + +def read_credentials(creds: str) -> Iterable[Credential]: + for line in creds.splitlines(): + match = re.match(r'(.*?)(/.*?)?=(.*?):(.*)', line.strip()) + if match: + yield Credential( + hostname=match.group(1).strip(), + path=split_path(match.group(2)), + username=match.group(3).strip(), + password=match.group(4).strip() + ) + +def netrc(credentials: List[Credential]) -> str: + s = '' + for cred in credentials: + s += f'machine {cred.hostname}\n' + s += f'login {cred.username}\n' + s += f'password {cred.password}\n' + return s + +def main(): + credentials = list(read_credentials(os.environ.get('TERRAFORM_HTTP_CREDENTIALS', ''))) + + if sys.argv[0] == '/usr/bin/netrc-credential-actions': + sys.stdout.write(netrc(credentials)) + else: + if len(sys.argv) != 2: + sys.stderr.write('This must be configured as a git credential helper\n') + exit(1) + + op = sys.argv[1] + + in_attributes = read_attributes(sys.stdin.read()) + out_attributes = git_credential(op, in_attributes, credentials) + sys.stdout.write(write_attributes(out_attributes)) + + +if __name__ == '__main__': + main() diff --git a/terraform-apply/README.md b/terraform-apply/README.md index bc728e10..09532e50 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -209,6 +209,31 @@ These input values must be the same as any `terraform-plan` for the same configu - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-check/README.md b/terraform-check/README.md index e0ec4f2a..50134245 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -121,6 +121,30 @@ This is intended to run on a schedule to notify if manual changes to your infras - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional ## Example usage diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 37b41a4a..1c00e694 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -118,6 +118,31 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Example usage This example deletes the workspace named after the git branch when the associated PR is closed. diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 2f58a08f..9b5613da 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -120,6 +120,31 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Example usage This example destroys the resources in a workspace named after the git branch when the associated PR is closed. diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 8a6e2636..33bb983a 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -74,6 +74,31 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Example usage This example creates a workspace named after the git branch when the diff --git a/terraform-output/README.md b/terraform-output/README.md index 5e645b0c..911dc17e 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -75,6 +75,31 @@ Retrieve the root-level outputs from a terraform configuration. - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-plan/README.md b/terraform-plan/README.md index ab5d5a7c..563fde06 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -171,7 +171,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `TERRAFORM_SSH_KEY` - A SSH private key that terraform will use to fetch git module sources. + A SSH private key that terraform will use to fetch git/mercurial module sources. This should be in PEM format. @@ -184,6 +184,31 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + * `TF_PLAN_COLLAPSE_LENGTH` When PR comments are enabled, the terraform output is included in a collapsable pane. diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 018ccfa9..f6f00f63 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -61,6 +61,31 @@ If the terraform configuration is not valid, the build is failed. - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs on every push and fails if the terraform diff --git a/terraform-version/README.md b/terraform-version/README.md index 3d6dfb24..1885b130 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -66,6 +66,31 @@ outputs yourself. - Type: string - Optional +* `TERRAFORM_HTTP_CREDENTIALS` + + Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. + + Credentials have the format `=:`. Multiple credentials may be specified, one per line. + + Each credential is evaluated in order, and the first matching credentials are used. + + Credentials that are used by git (`git::http://`, `git::https://`) allow a path after the hostname. + Paths are ignored by `http://` & `https://` schemes. + For git module sources, a credential matches if each mentioned path segment is an exact match. + + For example: + ```yaml + env: + TERRAFORM_HTTP_CREDENTIALS: | + example.com=dflook:${{ secrets.HTTPS_PASSWORD }} + github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} + github.com/dflook=dflook:${{ secrets.DFLOOK_PAT }} + github.com=graham:${{ secrets.GITHUB_PAT }} + ``` + + - Type: string + - Optional + ## Outputs * `terraform` diff --git a/tests/git-http-module/main.tf b/tests/git-http-module/main.tf new file mode 100644 index 00000000..9b252388 --- /dev/null +++ b/tests/git-http-module/main.tf @@ -0,0 +1,7 @@ +module "git_https_source" { + source = "git::https://github.com/dflook/terraform-github-actions-dev.git//tests/registry/test-module" +} + +output "git_https" { + value = module.git_https_source.my-output +} diff --git a/tests/http-module/main.tf b/tests/http-module/main.tf new file mode 100644 index 00000000..b2dfc173 --- /dev/null +++ b/tests/http-module/main.tf @@ -0,0 +1,7 @@ +module "https_source" { + source = "https://5qcb7mjppk.execute-api.eu-west-2.amazonaws.com/my_module" +} + +output "https" { + value = module.https_source.my-output +} diff --git a/tests/infra/http-auth.py b/tests/infra/http-auth.py new file mode 100644 index 00000000..28b077f6 --- /dev/null +++ b/tests/infra/http-auth.py @@ -0,0 +1,34 @@ +import base64 +import os + +unauth = { + 'statusCode': 401, + 'headers': { + 'www-authenticate': 'Basic realm="terraform-module"' + }, + 'body': 'Please authenticate' +} + + +def lambda_handler(event, context): + print(event) + + if 'authorization' not in event['headers']: + return unauth + + encoded = event['headers']['authorization'][len('Basic '):] + decoded = base64.b64decode(encoded) + separated = decoded.decode().split(':', maxsplit=1) + username = separated[0] + password = separated[1] + + if username != os.environ['USERNAME'] or password != os.environ['PASSWORD']: + return unauth + + return { + 'statusCode': 200, + 'body': '', + 'headers': { + 'content-type': 'text/html' + } + } diff --git a/tests/test_git_credential_actions.py b/tests/test_git_credential_actions.py new file mode 100644 index 00000000..385a369f --- /dev/null +++ b/tests/test_git_credential_actions.py @@ -0,0 +1,152 @@ +from http_credential_actions_helper import ( + read_credentials, + Credential, + read_attributes, + write_attributes, + split_path, + git_credential +) + + +def test_read_credentials(): + input = ''' + +nonsense +example.com=dflook:mypassword + github.com/dflook/terraform-github-actions.git=dflook-actions:anotherpassword + +github.com/dflook=dflook:secretpassword +github.com=graham:abcd + +almost/= user : pass + +''' + + expected_credentials = [ + Credential('example.com', [], 'dflook', 'mypassword'), + Credential('github.com', ['dflook', 'terraform-github-actions.git'], 'dflook-actions', 'anotherpassword'), + Credential('github.com', ['dflook'], 'dflook', 'secretpassword'), + Credential('github.com', [], 'graham', 'abcd'), + Credential('almost', [], 'user', 'pass') + ] + + actual_credentials = list(read_credentials(input)) + assert actual_credentials == expected_credentials + + assert [] == list(read_credentials('')) + +def test_read_attributes(): + input = ''' +protocol=https +host=example.com +path=abcd.git +username=bob +password=secr3t + ''' + + expected = { + 'protocol': 'https', + 'host': 'example.com', + 'path': 'abcd.git', + 'username': 'bob', + 'password': 'secr3t' + } + + actual = read_attributes(input) + assert actual == expected + + assert {} == read_attributes('') + +def test_write_attributes(): + input = { + 'protocol': 'https', + 'host': 'example.com', + 'path': 'abcd.git', + 'username': 'bob', + 'somethingelse': 'hello', + 'password': 'secr3t' + } + + expected = ''' +protocol=https +host=example.com +path=abcd.git +username=bob +somethingelse=hello +password=secr3t + '''.strip() + + actual = write_attributes(input) + assert actual == expected + + assert '' == write_attributes({}) + +def test_split_path(): + assert [] == split_path(None) + assert [] == split_path('/') + assert ['hello'] == split_path('/hello') + assert ['dflook', 'terraform-github-actions.git'] == split_path('/dflook/terraform-github-actions.git') + +def test_get(): + + credentials = [ + Credential('example.com', [], 'dflook', 'mypassword'), + Credential('github.com', ['dflook', 'terraform-github-actions.git'], 'dflook-actions', 'anotherpassword'), + Credential('github.com', ['dflook'], 'dflook-org', 'secretpassword'), + Credential('github.com', [], 'graham', 'abcd'), + Credential('almost', [], 'user', 'pass') + ] + + def merge(attributes, **kwargs): + return {**attributes, **kwargs} + + # No path, no username + attributes = dict(protocol='https', host='example.com') + assert git_credential('get', attributes, credentials) == merge(attributes, username='dflook', password='mypassword') + + # No path, required username match + attributes = dict(protocol='https', host='example.com', username='dflook') + assert git_credential('get', attributes, credentials) == merge(attributes, password='mypassword') + + # No path, required username no match + attributes = dict(protocol='https', host='example.com', username='sandra') + assert git_credential('get', attributes, credentials) == attributes + + # partial path, required username no match + attributes = dict(protocol='https', host='github.com', path='dflook', username='keith') + assert git_credential('get', attributes, credentials) == attributes + + # full path + attributes = dict(protocol='https', host='github.com', path='dflook/terraform-github-actions.git') + assert git_credential('get', attributes, credentials) == merge(attributes, username='dflook-actions', password='anotherpassword') + + # partial path multiple segments + attributes = dict(protocol='https', host='github.com', path='dflook/terraform-github-actions.git/additional-segment') + assert git_credential('get', attributes, credentials) == merge(attributes, username='dflook-actions', password='anotherpassword') + + # partial path single segment + attributes = dict(protocol='https', host='github.com', path='dflook') + assert git_credential('get', attributes, credentials) == merge(attributes, username='dflook-org', password='secretpassword') + + # no path match + attributes = dict(protocol='https', host='github.com', path='sausage') + assert git_credential('get', attributes, credentials) == merge(attributes, username='graham', password='abcd') + + # Cases we don't handle - return attributes unchanged + attributes = dict(protocol='https', host='example.com', username='dflook', password='mypassword') + assert git_credential('store', attributes, credentials) == attributes + + attributes = dict(protocol='https', host='example.com') + assert git_credential('erase', attributes, credentials) == attributes + + attributes = dict(protocol='https', host='example.com') + assert git_credential('nonsense', attributes, credentials) == attributes + + attributes = dict(protocol='git', host='example.com') + assert git_credential('get', attributes, credentials) == attributes + + attributes = dict(host='example.com') + assert git_credential('get', attributes, credentials) == attributes + + attributes = dict(protocol='http') + assert git_credential('get', attributes, credentials) == attributes From 29f8b2331c5f14c879e85730b1bc512f645672e5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 30 May 2021 12:23:41 +0100 Subject: [PATCH 063/231] :bookmark: v1.10.0 --- CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e23a86..5c89585c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,15 @@ When using an action you can specify the version as: - `@v1.10` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## [1.10.0] - Unreleased +## [1.10.0] - 2021-05-30 ### Added - `TERRAFORM_HTTP_CREDENTIALS` environment variable for configuring the username and password to use for - http module sources - + `git::https://` & `https://` module sources. + + See action documentation for details, e.g. [terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#environment-variables) + ## [1.9.3] - 2021-05-29 ### Fixed @@ -165,7 +167,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) - +[1.10.0]: https://github.com/dflook/terraform-github-actions/compare/v1.9.3...v1.10.0 [1.9.3]: https://github.com/dflook/terraform-github-actions/compare/v1.9.2...v1.9.3 [1.9.2]: https://github.com/dflook/terraform-github-actions/compare/v1.9.1...v1.9.2 [1.9.1]: https://github.com/dflook/terraform-github-actions/compare/v1.9.0...v1.9.1 From 1f4073a416044e741781aa073796420d095789e7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 30 May 2021 21:29:44 +0100 Subject: [PATCH 064/231] changes-only --- .github/workflows/test-changes-only.yaml | 171 +++++++++++++++++++++++ image/entrypoints/apply.sh | 4 +- image/entrypoints/plan.sh | 14 +- image/tools/github_pr_comment.py | 12 +- terraform-plan/README.md | 5 +- tests/plan/changes-only/main.tf | 12 ++ 6 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-changes-only.yaml create mode 100644 tests/plan/changes-only/main.tf diff --git a/.github/workflows/test-changes-only.yaml b/.github/workflows/test-changes-only.yaml new file mode 100644 index 00000000..add5ee9e --- /dev/null +++ b/.github/workflows/test-changes-only.yaml @@ -0,0 +1,171 @@ +name: Test changes-only PR comment + +on: [pull_request] + +jobs: + no_changes: + runs-on: ubuntu-latest + name: changes-only should not create a comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan without changes + uses: ./terraform-plan + id: plan + with: + label: no_changes + path: tests/plan/changes-only + add_github_comment: changes-only + + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "false" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + - name: Apply without changes + uses: ./terraform-apply + with: + label: no_changes + path: tests/plan/changes-only + + change_then_no_changes: + runs-on: ubuntu-latest + name: changes-only should still replace a change comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan changes + uses: ./terraform-plan + id: changes-plan + with: + label: change_then_no_changes + path: tests/plan/changes-only + variables: | + cause-changes=true + add_github_comment: changes-only + + - name: Verify changes + run: | + echo "changes=${{ steps.changes-plan.outputs.changes }}" + + if [[ "${{ steps.changes-plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + - name: Plan no changes + uses: ./terraform-plan + id: plan + with: + label: change_then_no_changes + path: tests/plan/changes-only + variables: | + cause-changes=false + add_github_comment: changes-only + + - name: Verify no changes + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "false" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + - name: Apply no changes + uses: ./terraform-apply + with: + label: change_then_no_changes + path: tests/plan/changes-only + variables: | + cause-changes=false + + no_changes_then_changes: + runs-on: ubuntu-latest + name: Apply with changes should fail after a changes-only plan with no changes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan no changes + uses: ./terraform-plan + id: plan + with: + path: tests/plan/changes-only + label: no_changes_then_changes + variables: | + cause-changes=false + add_github_comment: changes-only + + - name: Verify no changes + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "false" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + - name: Apply with changes + uses: ./terraform-apply + id: apply + continue-on-error: true + with: + path: tests/plan/changes-only + label: no_changes_then_changes + variables: | + cause-changes=true + + - name: Check failed to apply + run: | + if [[ "${{ steps.apply.outcome }}" != "failure" ]]; then + echo "Apply did not fail correctly" + exit 1 + fi + + apply_when_plan_has_changed: + runs-on: ubuntu-latest + name: Apply should fail if the approved plan has changed + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan Changes + uses: ./terraform-plan + with: + path: tests/plan/changes-only + label: apply_when_plan_has_changed + variables: | + cause-changes=true + + - name: Apply different changes + uses: ./terraform-apply + id: apply + continue-on-error: true + with: + path: tests/plan/changes-only + label: apply_when_plan_has_changed + variables: | + cause-changes=true + len=4 + + - name: Check failed to apply + run: | + if [[ "${{ steps.apply.outcome }}" != "failure" ]]; then + echo "Apply did not fail correctly" + exit 1 + fi diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 86fd7bd5..91bee7db 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -104,7 +104,9 @@ else fi if ! github_pr_comment get >"$PLAN_DIR/approved-plan.txt"; then - echo "Approved plan not found" + echo "Plan not found on PR" + echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" + echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" exit 1 fi diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 47088854..6e2354a1 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -28,7 +28,7 @@ set -e cat "$PLAN_DIR/error.txt" if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then - if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" ]]; then + if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then if [[ -z "$GITHUB_TOKEN" ]]; then echo "GITHUB_TOKEN environment variable must be set to add GitHub PR comments" @@ -40,7 +40,14 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c if [[ $TF_EXIT -eq 1 ]]; then STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/error.txt" else - STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/plan.txt" + + if [[ $TF_EXIT -eq 0 ]]; then + TF_CHANGES=false + else # [[ $TF_EXIT -eq 2 ]] + TF_CHANGES=true + fi + + TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/plan.txt" fi fi @@ -52,9 +59,8 @@ fi if [[ $TF_EXIT -eq 1 ]]; then debug_log "Error running terraform" exit 1 -fi -if [[ $TF_EXIT -eq 0 ]]; then +elif [[ $TF_EXIT -eq 0 ]]; then debug_log "No Changes to apply" echo "::set-output name=changes::false" diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index dad111e2..bd8d657e 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -374,11 +374,14 @@ def summary(self) -> str: return summary - def update_comment(self): + def update_comment(self, only_if_exists=False): body = self.collapsable_body() debug(body) if self._comment_url is None: + if only_if_exists: + debug('Comment doesn\'t already exist - not creating it') + return # Create a new comment debug('Creating comment') response = github_api_request('post', self._issue_url, json={'body': body}) @@ -400,10 +403,15 @@ def update_comment(self): {sys.argv[0]} get >plan.txt''') tf_comment = TerraformComment(find_pr()) + only_if_exists = False if sys.argv[1] == 'plan': tf_comment.plan = sys.stdin.read().strip() tf_comment.status = os.environ['STATUS'] + + if os.environ['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', 'true') == 'false': + only_if_exists = True + elif sys.argv[1] == 'status': if tf_comment.plan is None: exit(1) @@ -415,4 +423,4 @@ def update_comment(self): print(tf_comment.plan) exit(0) - tf_comment.update_comment() + tf_comment.update_comment(only_if_exists) diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 563fde06..eab54e22 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -124,10 +124,11 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `add_github_comment` - The default is `true`, which adds a comment to the PR with the generated plan. + The default is `true`, which adds a comment to the PR with the results of the plan. + Set to `changes-only` to add a comment only when the plan indicates there are changes to apply. Set to `false` to disable the comment - the plan will still appear in the workflow log. - - Type: bool + - Type: string - Optional - Default: true diff --git a/tests/plan/changes-only/main.tf b/tests/plan/changes-only/main.tf new file mode 100644 index 00000000..e5192d4f --- /dev/null +++ b/tests/plan/changes-only/main.tf @@ -0,0 +1,12 @@ +variable "cause-changes" { + default = false +} + +variable "len" { + default = 5 +} + +resource "random_string" "the_string" { + count = var.cause-changes ? 1 : 0 + length = var.len +} From c42fabc0d3140c2a0b04a6a5edbcbdebcb57ea94 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Jun 2021 22:27:32 +0100 Subject: [PATCH 065/231] Improve changed plan messaging --- image/entrypoints/apply.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 91bee7db..3d77b9a7 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -113,8 +113,14 @@ else if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then apply else - debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" + echo "Not applying the plan - it has changed from the plan on the PR" + echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" + + echo "Plan changes:" + debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" + diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true + exit 1 fi fi From 8607b031ae540bc6d712f31a5c01925276983f8f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Jun 2021 23:33:55 +0100 Subject: [PATCH 066/231] Update docs to use newlines instead of commas for multi value input variables --- .github/workflows/test-apply.yaml | 20 +++++++++--- .github/workflows/test-remote-state.yaml | 5 ++- README.md | 4 ++- terraform-apply/README.md | 40 +++++++++++++++++++----- terraform-apply/action.yaml | 6 ++-- terraform-check/README.md | 26 +++++++++++---- terraform-check/action.yaml | 4 +-- terraform-destroy-workspace/README.md | 23 ++++++++++++-- terraform-destroy-workspace/action.yaml | 4 +-- terraform-destroy/README.md | 26 ++++++++++----- terraform-destroy/action.yaml | 4 +-- terraform-new-workspace/README.md | 14 +++++++-- terraform-new-workspace/action.yaml | 2 +- terraform-output/README.md | 14 +++++++-- terraform-output/action.yaml | 2 +- terraform-plan/README.md | 27 +++++++++++++--- terraform-plan/action.yaml | 4 +-- terraform-remote-state/README.md | 19 +++++++++-- terraform-remote-state/action.yaml | 2 +- 19 files changed, 191 insertions(+), 55 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 1cdbaf9f..515dbaa2 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -204,14 +204,20 @@ jobs: uses: ./terraform-plan with: path: tests/apply/backend_config_12 - backend_config: bucket=terraform-github-actions,key=backend_config,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=backend_config + region=eu-west-2 - name: Apply uses: ./terraform-apply id: backend_config_12 with: path: tests/apply/backend_config_12 - backend_config: bucket=terraform-github-actions,key=backend_config,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=backend_config + region=eu-west-2 - name: Verify outputs run: | @@ -255,14 +261,20 @@ jobs: uses: ./terraform-plan with: path: tests/apply/backend_config_13 - backend_config: bucket=terraform-github-actions,key=backend_config_13,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=backend_config_13 + region=eu-west-2 - name: Apply uses: ./terraform-apply id: backend_config_13 with: path: tests/apply/backend_config_13 - backend_config: bucket=terraform-github-actions,key=backend_config_13,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=backend_config_13 + region=eu-west-2 - name: Verify outputs run: | diff --git a/.github/workflows/test-remote-state.yaml b/.github/workflows/test-remote-state.yaml index 6aa02b71..b39aa6b6 100644 --- a/.github/workflows/test-remote-state.yaml +++ b/.github/workflows/test-remote-state.yaml @@ -18,7 +18,10 @@ jobs: id: terraform-output with: backend_type: s3 - backend_config: bucket=terraform-github-actions,key=terraform-remote-state,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=terraform-remote-state + region=eu-west-2 - name: Print the outputs run: | diff --git a/README.md b/README.md index 7882eded..e14abe1d 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,9 @@ jobs: with: path: my-terraform-config auto_approve: true - target: acme_certificate.certificate,kubernetes_secret.certificate + target: | + acme_certificate.certificate + kubernetes_secret.certificate ``` ### Automatically fixing formatting diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 09532e50..2350e6d2 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -108,24 +108,41 @@ These input values must be the same as any `terraform-plan` for the same configu * `var_file` - Comma separated list of tfvars files to use. + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + var_file: | + common.tfvars + prod.tfvars + ``` - Type: string - Optional * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional @@ -139,10 +156,17 @@ These input values must be the same as any `terraform-plan` for the same configu * `target` - Comma separated list of targets to apply against, e.g. kubernetes_secret.tls_cert_public,kubernetes_secret.tls_cert_private - + List of resources to apply, one per line. + The apply operation will be limited to these resources and their dependencies. This only takes effect if auto_approve is also set to `true`. + ```yaml + with: + target: | + kubernetes_secret.tls_cert_public + kubernetes_secret.tls_cert_private + ``` + - Type: string - Optional @@ -332,7 +356,9 @@ jobs: with: path: my-terraform-config auto_approve: true - target: kubernetes_secret.tls_cert_public,kubernetes_secret.tls_cert_private + target: | + kubernetes_secret.tls_cert_public + kubernetes_secret.tls_cert_private ``` ### Applying a plan using a comment @@ -349,7 +375,7 @@ on: [issue_comment] jobs: apply: - if: github.event.issue.pull_request && contains(github.event.comment.body, 'terraform apply') + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'terraform apply') }} runs-on: ubuntu-latest name: Apply terraform plan env: diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index 1469986d..a32b1b66 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false default: "" backend_config_file: @@ -27,7 +27,7 @@ inputs: default: "" deprecationMessage: Use the variables input instead. var_file: - description: Comma separated list of var file paths + description: List of var file paths, one per line required: false default: "" parallelism: @@ -42,7 +42,7 @@ inputs: description: Automatically approve and apply plan default: false target: - description: "Comma separated list of targets to apply against, e.g. 'kubernetes_secret.tls_cert_public,kubernetes_secret.tls_cert_private' NOTE: this argument only takes effect if auto_approve is also set." + description: List of targets to apply against, one per line required: false default: "" diff --git a/terraform-check/README.md b/terraform-check/README.md index 50134245..6fa08300 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -53,24 +53,38 @@ This is intended to run on a schedule to notify if manual changes to your infras * `var_file` - Comma separated list of tfvars files to use. + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace - - - Type: string - - Optional + + ```yaml + with: + var_file: | + common.tfvars + prod.tfvars + ``` * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index 3f3d9404..cae2bc0c 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false backend_config_file: description: Path to a backend config file" @@ -24,7 +24,7 @@ inputs: required: false deprecationMessage: Use the variables input instead. var_file: - description: Comma separated list of var file paths + description: List of var file paths, one per line required: false parallelism: description: Limit the number of concurrent operations diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 1c00e694..7a73c3d5 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -50,24 +50,41 @@ This action uses the `terraform destroy` command to destroy all resources in a t * `var_file` - Comma separated list of tfvars files to use. + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + var_file: | + common.tfvars + prod.tfvars + ``` - Type: string - Optional * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index b9678f67..47e1e2db 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -10,7 +10,7 @@ inputs: description: Name of the terraform workspace required: true backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false backend_config_file: description: Path to a backend config file @@ -23,7 +23,7 @@ inputs: required: false deprecationMessage: Use the variables input instead. var_file: - description: Comma separated list of var file paths + description: List of var file paths, one per line required: false parallelism: description: Limit the number of concurrent operations diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 9b5613da..5c0afa16 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -50,26 +50,38 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional -* `var_file` - - Comma separated list of tfvars files to use. + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace - - Type: string - - Optional + ```yaml + with: + var_file: | + common.tfvars + prod.tfvars + ``` * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 8e15823f..1a3121a2 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false backend_config_file: description: Path to a backend config file @@ -24,7 +24,7 @@ inputs: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false var_file: - description: Comma separated list of var file paths + description: List of var file paths, one per line required: false parallelism: description: Limit the number of concurrent operations diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 33bb983a..9e7974f9 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -22,16 +22,26 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional diff --git a/terraform-new-workspace/action.yaml b/terraform-new-workspace/action.yaml index 65672b4c..12c53d85 100644 --- a/terraform-new-workspace/action.yaml +++ b/terraform-new-workspace/action.yaml @@ -10,7 +10,7 @@ inputs: description: Name of the terraform workspace required: true backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false backend_config_file: description: Path to a backend config file" diff --git a/terraform-output/README.md b/terraform-output/README.md index 911dc17e..3380b691 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -23,16 +23,26 @@ Retrieve the root-level outputs from a terraform configuration. * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional diff --git a/terraform-output/action.yaml b/terraform-output/action.yaml index c91f57a6..a9312ff8 100644 --- a/terraform-output/action.yaml +++ b/terraform-output/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend configs to set, e.g. 'foo=bar' + description: List of backend configs to set, one per line required: false backend_config_file: description: Path to a backend config file diff --git a/terraform-plan/README.md b/terraform-plan/README.md index eab54e22..a929c89f 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -36,7 +36,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ An friendly name for the environment the terraform configuration is for. This will be used in the PR comment for easy identification. - It must be the same as the `label` used in the corresponding `terraform-apply` command. + If set, must be the same as the `label` used in the corresponding `terraform-apply` command. - Type: string - Optional @@ -93,24 +93,41 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `var_file` - Comma separated list of tfvars files to use. + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + var_file: | + common.tfvars + prod.tfvars + ``` - Type: string - Optional * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional @@ -316,7 +333,7 @@ on: [issue_comment] jobs: plan: - if: github.event.issue.pull_request && contains(github.event.comment.body, 'terraform plan') + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'terraform plan') }} runs-on: ubuntu-latest name: Create terraform plan env: diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 7249a815..77233581 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend config values to set, e.g. 'foo=bar' + description: List of backend config values to set, one per line required: false backend_config_file: description: Path to a backend config file @@ -24,7 +24,7 @@ inputs: required: false deprecationMessage: Use the variables input instead. var_file: - description: Comma separated list of var file paths + description: List of var file paths, one per line required: false parallelism: description: Limit the number of concurrent operations diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index 33adb01f..873e8248 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -23,16 +23,26 @@ Retrieves the root-level outputs from a terraform remote state. * `backend_config` - Comma separated list of terraform backend config values. + List of terraform backend config values, one per line. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` - Type: string - Optional * `backend_config_file` - Comma separated list of terraform backend config files to use. + List of terraform backend config files to use, one per line. Paths should be relative to the GitHub Actions workspace + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + - Type: string - Optional @@ -99,7 +109,10 @@ jobs: id: remote-state with: backend_type: s3 - backend_config: bucket=terraform-github-actions,key=terraform-remote-state,region=eu-west-2 + backend_config: | + bucket=terraform-github-actions + key=terraform-remote-state + region=eu-west-2 - name: Send request run: | diff --git a/terraform-remote-state/action.yaml b/terraform-remote-state/action.yaml index ec532068..fde921f7 100644 --- a/terraform-remote-state/action.yaml +++ b/terraform-remote-state/action.yaml @@ -11,7 +11,7 @@ inputs: required: false default: default backend_config: - description: Comma separated list of backend config values to set, e.g. 'foo=bar' + description: List of backend config values to set, one per line required: false backend_config_file: description: Path to a backend config file From ccfcdeda62156d754d09561c38a4857214e21ef4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Jun 2021 23:43:25 +0100 Subject: [PATCH 067/231] Update changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c89585c..c3f7b6bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,19 @@ When using an action you can specify the version as: - `@v1.10` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Added + +- The `add_github_comment` input for dflook/terraform-plan may now be set to `changes-only`. This will only add a PR comment + for plans that result in changes to apply - no comment will be added for plans with no changes. + +### Changed + +- Improved messaging in the workflow log when dflook/terraform-apply is aborted because the plan has changed +- Update documentation for `backend_config`, `backend_config_file`, `var_file` & `target` inputs to use separate lines for multiple values. + Multiple values may still be separated by commas if preferred. + ## [1.10.0] - 2021-05-30 ### Added From 8734bf63c5f62008c4ce2d108a484144acf7c1f7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Jun 2021 23:53:59 +0100 Subject: [PATCH 068/231] :bookmark: v1.11.0 --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3f7b6bf..b428dd4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.10.0` to use an exact release -- `@v1.10` to use the latest patch release for the specific minor version +- `@v1.11.0` to use an exact release +- `@v1.11` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.11.0] - 2021-06-05 ### Added @@ -180,6 +180,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.11.0]: https://github.com/dflook/terraform-github-actions/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/dflook/terraform-github-actions/compare/v1.9.3...v1.10.0 [1.9.3]: https://github.com/dflook/terraform-github-actions/compare/v1.9.2...v1.9.3 [1.9.2]: https://github.com/dflook/terraform-github-actions/compare/v1.9.1...v1.9.2 From 045db9656da31be23629f031d3428ff5ead0f971 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 00:10:11 +0100 Subject: [PATCH 069/231] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b428dd4e..e63846b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ When using an action you can specify the version as: ## [1.11.0] - 2021-06-05 +⚠️ GitHub broke while this was being released, so it's probably not available. +I'll try and release it again someday. + ### Added - The `add_github_comment` input for dflook/terraform-plan may now be set to `changes-only`. This will only add a PR comment From f2a4136a2010d0d499b8217d1ba5cb9fcb5ce98a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 09:14:46 +0100 Subject: [PATCH 070/231] Be more resilient to github failures --- .github/workflows/release.yaml | 54 +++++++++++++++------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9fa779e8..3a069531 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,19 +10,21 @@ jobs: name: Publish Docker Image env: GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }} - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} steps: - name: Checkout uses: actions/checkout@v2 - name: Registry login + env: + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} run: | echo $GITHUB_TOKEN | docker login ghcr.io -u dflook --password-stdin echo $DOCKER_TOKEN | docker login --username danielflook --password-stdin - - name: Action image + - name: Build action image + id: image_build run: | - RELEASE_TAG=$(cat $GITHUB_EVENT_PATH | jq -r '.release.tag_name' $GITHUB_EVENT_PATH) + RELEASE_TAG="${{ github.event.release.tag_name }}" docker build --tag dflook/terraform-github-actions \ --label org.opencontainers.image.created="$(date '+%Y-%m-%dT%H:%M:%S%z')" \ @@ -39,47 +41,39 @@ jobs: docker tag dflook/terraform-github-actions danielflook/terraform-github-actions:$RELEASE_TAG docker push danielflook/terraform-github-actions:$RELEASE_TAG - release_actions: - runs-on: ubuntu-latest - name: Release Actions - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }} - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - needs: image - steps: - - name: Checkout - uses: actions/checkout@v2 + echo "::set-output name=digest::$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions:$RELEASE_TAG")" - - name: Action image + - name: Release actions run: | - export RELEASE_TAG=$(cat $GITHUB_EVENT_PATH | jq -r '.release.tag_name' $GITHUB_EVENT_PATH) + export RELEASE_TAG="${{ github.event.release.tag_name }}" + export major=`echo $RELEASE_TAG | cut -d. -f1` + export minor=`echo $RELEASE_TAG | cut -d. -f2` git config --global user.name "Daniel Flook" git config --global user.email "daniel@flook.org" - for action in $(find . -name action.yaml -printf "%h\n" | sed 's/^.\///'); do + for action in $(find $GITHUB_WORKSPACE -name action.yaml -printf "%h\n" | sed 's/^.\///'); do if git clone https://dflook:$GITHUB_TOKEN@github.com/dflook/$action.git "$HOME/$action"; then echo "Releasing dflook/$action@$RELEASE_TAG" - rsync -r $action/ $HOME/$action + rsync -r $GITHUB_WORKSPACE/$action/ $HOME/$action + + sed -i "s|../image/Dockerfile|docker://danielflook/terraform-github-actions:${RELEASE_TAG}|" $HOME/$action/action.yaml - sed -i "s|../image/Dockerfile|docker://danielflook/terraform-github-actions:$RELEASE_TAG|" $HOME/$action/action.yaml + git -C "$HOME/$action" add -A - export major=`echo $RELEASE_TAG | cut -d. -f1` - export minor=`echo $RELEASE_TAG | cut -d. -f2` + if ! git -C "$HOME/$action" diff --quiet; then + git -C "$HOME/$action" commit -m "$RELEASE_TAG" + fi - (cd "$HOME/$action" \ - && git add -A \ - && git commit -m "$RELEASE_TAG" \ - && git tag --force -a -m"$RELEASE_TAG" "$RELEASE_TAG" \ - && git tag --force -a -m"$RELEASE_TAG" "$major" \ - && git tag --force -a -m"$RELEASE_TAG" "$major.$minor" \ - && git push --force \ - && git push --force --tags - ) + git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$RELEASE_TAG" + git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$major" + git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$major.$minor" + git -C "$HOME/$action" push --force + git -C "$HOME/$action" push --force --tags - cat .github/release_template.md \ + cat $GITHUB_WORKSPACE/.github/release_template.md \ | envsubst \ | jq --slurp --raw-input --arg RELEASE_TAG "$RELEASE_TAG" '{"tag_name": $RELEASE_TAG, "name": $RELEASE_TAG, "body": . }' \ | curl -X POST \ From cebad39d3000ccd61bafc0f294047f28258fcd1c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 09:28:28 +0100 Subject: [PATCH 071/231] Fix release --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3a069531..425a6ba1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -52,7 +52,7 @@ jobs: git config --global user.name "Daniel Flook" git config --global user.email "daniel@flook.org" - for action in $(find $GITHUB_WORKSPACE -name action.yaml -printf "%h\n" | sed 's/^.\///'); do + for action in $(cd $GITHUB_WORKSPACE && find . -name action.yaml -printf "%h\n" | sed 's/^.\///'); do if git clone https://dflook:$GITHUB_TOKEN@github.com/dflook/$action.git "$HOME/$action"; then echo "Releasing dflook/$action@$RELEASE_TAG" From a12e7166b894cd8acc7bec11290a5b0b4ccd63c4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 09:33:45 +0100 Subject: [PATCH 072/231] Fix release --- .github/workflows/release.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 425a6ba1..280a92a6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -36,10 +36,10 @@ jobs: image docker tag dflook/terraform-github-actions ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG - docker push ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG + docker push --quiet ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG docker tag dflook/terraform-github-actions danielflook/terraform-github-actions:$RELEASE_TAG - docker push danielflook/terraform-github-actions:$RELEASE_TAG + docker push --quiet danielflook/terraform-github-actions:$RELEASE_TAG echo "::set-output name=digest::$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions:$RELEASE_TAG")" @@ -62,11 +62,7 @@ jobs: sed -i "s|../image/Dockerfile|docker://danielflook/terraform-github-actions:${RELEASE_TAG}|" $HOME/$action/action.yaml git -C "$HOME/$action" add -A - - if ! git -C "$HOME/$action" diff --quiet; then - git -C "$HOME/$action" commit -m "$RELEASE_TAG" - fi - + git -C "$HOME/$action" commit -m "$RELEASE_TAG" git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$RELEASE_TAG" git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$major" git -C "$HOME/$action" tag --force -a -m"$RELEASE_TAG" "$major.$minor" From d3a097b14bb63b2a25c8635ec1e1e110368e68f2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 10:03:12 +0100 Subject: [PATCH 073/231] Use docker digest in released metadata --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 280a92a6..fde45187 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: image: runs-on: ubuntu-latest - name: Publish Docker Image + name: Release Actions env: GITHUB_TOKEN: ${{ secrets.RELEASE_GITHUB_TOKEN }} steps: @@ -59,7 +59,7 @@ jobs: rsync -r $GITHUB_WORKSPACE/$action/ $HOME/$action - sed -i "s|../image/Dockerfile|docker://danielflook/terraform-github-actions:${RELEASE_TAG}|" $HOME/$action/action.yaml + sed -i "s|../image/Dockerfile|docker://${{ steps.image_build.outputs.digest }}|" $HOME/$action/action.yaml git -C "$HOME/$action" add -A git -C "$HOME/$action" commit -m "$RELEASE_TAG" From 37f1267bfc10e1657923ee0eda937947534b5608 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 10:23:52 +0100 Subject: [PATCH 074/231] Update CHANGELOG.md --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e63846b6..b428dd4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,6 @@ When using an action you can specify the version as: ## [1.11.0] - 2021-06-05 -⚠️ GitHub broke while this was being released, so it's probably not available. -I'll try and release it again someday. - ### Added - The `add_github_comment` input for dflook/terraform-plan may now be set to `changes-only`. This will only add a PR comment From ddb7bf54132d3fe983a5c594b946110c5ffabcde Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Jun 2021 13:37:22 +0100 Subject: [PATCH 075/231] Fix example syntax error --- terraform-plan/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform-plan/README.md b/terraform-plan/README.md index a929c89f..7188bb31 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -314,7 +314,7 @@ jobs: label: production workspace: prod var_file: env/prod.tfvars - variables: + variables: | turbo_mode=true backend_config_file: env/prod.backend backend_config: token=${{ secrets.BACKEND_TOKEN }} From b56ea6efcb94dba89047550abe9454d67d9373fe Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 6 Jun 2021 09:05:40 +0100 Subject: [PATCH 076/231] Show fmt-check diff in workflow log --- image/entrypoints/fmt-check.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index f4e3a800..95c97808 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -6,9 +6,13 @@ debug setup EXIT_CODE=0 -for file in $(terraform fmt -recursive -no-color -check "$INPUT_PATH"); do - echo "::error file=$file::File is not in canonical format (terraform fmt)" - EXIT_CODE=1 +terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read -r line; do + echo "$line" + + if [[ -f "$line" ]]; then + echo "::error file=$line::File is not in canonical format (terraform fmt)" + EXIT_CODE=1 + fi done if [[ "$EXIT_CODE" -eq 0 ]]; then From 41e9dbab26e1647c5a78063786f949bde34e9a80 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 6 Jun 2021 10:35:15 +0100 Subject: [PATCH 077/231] Remove redundant exit code stuff --- image/entrypoints/fmt-check.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index 95c97808..1fd680f8 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -5,18 +5,14 @@ source /usr/local/actions.sh debug setup -EXIT_CODE=0 terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read -r line; do echo "$line" if [[ -f "$line" ]]; then echo "::error file=$line::File is not in canonical format (terraform fmt)" - EXIT_CODE=1 fi done -if [[ "$EXIT_CODE" -eq 0 ]]; then - echo "All terraform configuration files are formatted correctly." -fi +# terraform fmt has non zero exit code if there are non canonical files -exit $EXIT_CODE +echo "All terraform configuration files are formatted correctly." From a69b2771ce9874f73e565baa6e712a44c5b2f28e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 8 Jun 2021 09:01:13 +0100 Subject: [PATCH 078/231] Show fmt diff in workflow log --- CHANGELOG.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b428dd4e..22d00cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,20 +8,26 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.11.0` to use an exact release -- `@v1.11` to use the latest patch release for the specific minor version +- `@v1.12.0` to use an exact release +- `@v1.12` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.12.0] - 2021-06-08 + +### Changed + +- [terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) now shows a diff in the workflow log when it finds files in non-canonical format + ## [1.11.0] - 2021-06-05 ### Added -- The `add_github_comment` input for dflook/terraform-plan may now be set to `changes-only`. This will only add a PR comment +- The `add_github_comment` input for [terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) may now be set to `changes-only`. This will only add a PR comment for plans that result in changes to apply - no comment will be added for plans with no changes. ### Changed -- Improved messaging in the workflow log when dflook/terraform-apply is aborted because the plan has changed +- Improved messaging in the workflow log when [terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) is aborted because the plan has changed - Update documentation for `backend_config`, `backend_config_file`, `var_file` & `target` inputs to use separate lines for multiple values. Multiple values may still be separated by commas if preferred. @@ -180,6 +186,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.12.0]: https://github.com/dflook/terraform-github-actions/compare/v1.11.0...v1.12.0 [1.11.0]: https://github.com/dflook/terraform-github-actions/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/dflook/terraform-github-actions/compare/v1.9.3...v1.10.0 [1.9.3]: https://github.com/dflook/terraform-github-actions/compare/v1.9.2...v1.9.3 From e4248e7cc5e146731a03cc083b263b365ef28fe4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 8 Jun 2021 13:39:25 +0100 Subject: [PATCH 079/231] Update version test for terraform 1.0 --- .github/workflows/test-version.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-version.yaml b/.github/workflows/test-version.yaml index 7a715d81..5866a78e 100644 --- a/.github/workflows/test-version.yaml +++ b/.github/workflows/test-version.yaml @@ -113,7 +113,7 @@ jobs: - name: Check the version run: | - if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"0.15"* ]]; then + if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"1.0"* ]]; then echo "::error:: Latest version was not used" exit 1 fi From 0205ee7d91543c4cda8fc49afb8d3212a70c3338 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 8 Jun 2021 14:02:26 +0100 Subject: [PATCH 080/231] Show error if workspace fails to create --- image/entrypoints/new-workspace.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index b7ee4fc5..dfea9465 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -51,6 +51,8 @@ else echo "Workspace does exist, selecting it" (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") else + cat "$WS_TMP_DIR/new_err.txt" + cat "$WS_TMP_DIR/new_out.txt" exit 1 fi else From fbb0fa57ba2bf6fb501a21dcbf300ff87faf2f8f Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa Date: Thu, 22 Jul 2021 14:46:28 +0200 Subject: [PATCH 081/231] fixed implementation of terraform pre-run --- image/actions.sh | 9 +++------ terraform-apply/action.yaml | 3 --- terraform-check/action.yaml | 3 --- terraform-destroy/action.yaml | 3 --- terraform-plan/action.yaml | 3 --- 5 files changed, 3 insertions(+), 18 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index ee4a42d1..f7ac4b98 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -52,13 +52,10 @@ function detect-tfmask() { } function execute_run_commands() { - if [[ -n $INPUT_RUN ]]; then - echo "Executing init commands specified in 'run' parameter" - eval "$INPUT_RUN" - fi - if [[ -n $TERRAFORM_PRE_RUN ]] + if [[ -n $TERRAFORM_PRE_RUN ]]; then echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" - eval "$TERRAFORM_PRE_RUN" + printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh + bash -x /.prerun.sh fi } diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index 08bb444e..1469986d 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -45,9 +45,6 @@ inputs: description: "Comma separated list of targets to apply against, e.g. 'kubernetes_secret.tls_cert_public,kubernetes_secret.tls_cert_private' NOTE: this argument only takes effect if auto_approve is also set." required: false default: "" - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index 3a80c8d5..3f3d9404 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -30,9 +30,6 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 08dd0f40..8e15823f 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -30,9 +30,6 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 8b98c3e0..7249a815 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -38,9 +38,6 @@ inputs: description: Add the plan to a GitHub PR required: false default: true - run: - description: Executes these shell commands before running terraform - required: false outputs: changes: From 5aacad5acf1de6348817c34cc66bb5fe9866fc3e Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa Date: Thu, 22 Jul 2021 15:27:01 +0200 Subject: [PATCH 082/231] updated test plan --- .github/workflows/test-plan.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 611c678e..f2b7bddd 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -325,7 +325,7 @@ jobs: uses: ./terraform-plan with: path: tests/plan/plan - run: | - echo testing command 1 - echo testing command 2 - + env: + TERRAFORM_PRE_RUN: | + echo "testing command 1" + echo "testing command 2" From 545207eafbe2bd6fe28d15e18b2d619f9f4fa209 Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa <68604164+GiuseppeChiesa-TomTom@users.noreply.github.com> Date: Fri, 23 Jul 2021 08:47:53 +0200 Subject: [PATCH 083/231] Update terraform-destroy-workspace/action.yaml Co-authored-by: Daniel Flook --- terraform-destroy-workspace/action.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index e5758f7b..47e1e2db 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -29,9 +29,6 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 - run: - description: Executes these shell commands before running terraform - required: false runs: using: docker From f84e27a363e6f2068abcc4f729c43b6a8456f72f Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa <68604164+GiuseppeChiesa-TomTom@users.noreply.github.com> Date: Fri, 23 Jul 2021 08:47:59 +0200 Subject: [PATCH 084/231] Update terraform-plan/README.md Co-authored-by: Daniel Flook --- terraform-plan/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/terraform-plan/README.md b/terraform-plan/README.md index f8a4391d..680d56ce 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -245,7 +245,11 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` For example: ```yaml From 0cb74f8e24d8d4ebac38fb9f07d866c886ba6683 Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa Date: Fri, 23 Jul 2021 10:51:14 +0200 Subject: [PATCH 085/231] updated documentation added apply with pre-run test --- .github/workflows/test-apply.yaml | 30 +++++++++++++++++++++++++++ terraform-apply/README.md | 20 +++++++++++++++++- terraform-check/README.md | 20 +++++++++++++++++- terraform-destroy-workspace/README.md | 21 ++++++++++++++++++- terraform-destroy/README.md | 20 +++++++++++++++++- terraform-new-workspace/README.md | 21 ++++++++++++++++++- terraform-output/README.md | 21 ++++++++++++++++++- terraform-validate/README.md | 20 +++++++++++++++++- terraform-version/README.md | 20 +++++++++++++++++- 9 files changed, 185 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 515dbaa2..01af1f94 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -492,3 +492,33 @@ jobs: label: Refresh 2 path: tests/apply/refresh_15 variables: len=20 + + apply_with_pre_run: + runs-on: ubuntu-latest + name: Apply with pre-run script + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TERRAFORM_PRE_RUN: | + echo "testing command 1" + echo "testing command 2" + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + path: tests/apply/changes + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/apply/changes + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi \ No newline at end of file diff --git a/terraform-apply/README.md b/terraform-apply/README.md index e13c6afd..3960fd0e 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -235,7 +235,25 @@ These input values must be the same as any `terraform-plan` for the same configu * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional * `TERRAFORM_HTTP_CREDENTIALS` diff --git a/terraform-check/README.md b/terraform-check/README.md index b7fb895d..b2fa03d4 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -137,7 +137,25 @@ This is intended to run on a schedule to notify if manual changes to your infras * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional * `TERRAFORM_HTTP_CREDENTIALS` diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index b10b1062..dc761f10 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -137,7 +137,26 @@ This action uses the `terraform destroy` command to destroy all resources in a t * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + * `TERRAFORM_HTTP_CREDENTIALS` Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index f5758396..c607dbb9 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -134,7 +134,25 @@ This action uses the `terraform destroy` command to destroy all resources in a t * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional * `TERRAFORM_HTTP_CREDENTIALS` diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 9f5d4c51..59594be5 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -86,7 +86,26 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + * `TERRAFORM_HTTP_CREDENTIALS` Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. diff --git a/terraform-output/README.md b/terraform-output/README.md index be5ba519..1b38a55f 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -87,7 +87,26 @@ Retrieve the root-level outputs from a terraform configuration. * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional + * `TERRAFORM_HTTP_CREDENTIALS` Credentials that will be used for fetching modules sources with `git::http://`, `git::https://`, `http://` & `https://` schemes. diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 8c0ea10a..0b5a58e4 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -63,7 +63,25 @@ If the terraform configuration is not valid, the build is failed. * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional * `TERRAFORM_HTTP_CREDENTIALS` diff --git a/terraform-version/README.md b/terraform-version/README.md index a4923ee5..1d0e6d8c 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -68,7 +68,25 @@ outputs yourself. * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. + A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + + The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster` + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + + - Type: string + - Optional * `TERRAFORM_HTTP_CREDENTIALS` From ec54c6dd8e39b1c9f1e537f8f7c76e5e118f8242 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 24 Jul 2021 09:49:46 +0100 Subject: [PATCH 086/231] Add label to pre-run test comments --- .github/workflows/test-apply.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 01af1f94..59582957 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -508,17 +508,19 @@ jobs: - name: Plan uses: ./terraform-plan with: + label: pre-run path: tests/apply/changes - name: Apply uses: ./terraform-apply id: output with: + label: pre-run path: tests/apply/changes - name: Verify outputs run: | if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then - echo "::error:: output s not set correctly" + echo "::error:: output_string not set correctly" exit 1 - fi \ No newline at end of file + fi From b21bd926f056d0dd8288bd2bd7e1c3be257bb7d7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 24 Jul 2021 12:21:03 +0100 Subject: [PATCH 087/231] Add label to pre-run test comments --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index 471620e2..cee65b7f 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,2 +1,3 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From f2de5447c346c2e0fae7028a274e3c335151d6f5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 24 Jul 2021 12:30:32 +0100 Subject: [PATCH 088/231] Execute pre-run commands with -eo pipefail --- image/actions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index e33bdfac..037807e6 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -55,7 +55,7 @@ function execute_run_commands() { if [[ -n $TERRAFORM_PRE_RUN ]]; then echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh - bash -x /.prerun.sh + bash -xeo pipefail /.prerun.sh fi } From 283f2acae7d78cb17452af9f14a841dd4c2cd87a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 24 Jul 2021 13:08:54 +0100 Subject: [PATCH 089/231] :bookmark: v1.13.0 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++-- terraform-plan/README.md | 2 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d00cac..c4b38737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,34 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.12.0` to use an exact release -- `@v1.12` to use the latest patch release for the specific minor version +- `@v1.13.0` to use an exact release +- `@v1.13` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.13.0] - 2021-07-24 + +### Added + +- `TERRAFORM_PRE_RUN` environment variable for customising the environment before running terraform. + + It can be set to a command that will be run prior to `terraform init`. + + The runtime environment for these actions is subject to change in minor version releases. + If using this environment variable, specify the minor version of the action to use. + + The runtime image is currently based on `debian:buster`, with the command run using `bash -xeo pipefail`. + + For example: + ```yaml + env: + TERRAFORM_PRE_RUN: | + # Install latest Azure CLI + curl -skL https://aka.ms/InstallAzureCLIDeb | bash + + # Install postgres client + apt-get install -y --no-install-recommends postgresql-client + ``` + ## [1.12.0] - 2021-06-08 ### Changed @@ -186,6 +210,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.13.0]: https://github.com/dflook/terraform-github-actions/compare/v1.12.0...v1.13.0 [1.12.0]: https://github.com/dflook/terraform-github-actions/compare/v1.11.0...v1.12.0 [1.11.0]: https://github.com/dflook/terraform-github-actions/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/dflook/terraform-github-actions/compare/v1.9.3...v1.10.0 diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 680d56ce..ca20ad1c 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -249,7 +249,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:buster`, with the command run using `bash -xeo pipefail`. For example: ```yaml From 4635503fed1aac794482f39ba5e0b39a96a057b9 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 24 Jul 2021 13:32:58 +0100 Subject: [PATCH 090/231] Mention contributers --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b38737..2c67913b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ When using an action you can specify the version as: # Install postgres client apt-get install -y --no-install-recommends postgresql-client ``` + + Thanks to @alec-pinson and @GiuseppeChiesa-TomTom for working on this feature. ## [1.12.0] - 2021-06-08 From 4522f53699088d384178d62fad4996a28046e1e9 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 25 Jul 2021 10:00:50 +0100 Subject: [PATCH 091/231] Remove misplaced example --- terraform-apply/README.md | 6 ------ terraform-check/README.md | 6 ------ terraform-destroy-workspace/README.md | 6 ------ terraform-destroy/README.md | 6 ------ terraform-new-workspace/README.md | 6 ------ terraform-validate/README.md | 6 ------ terraform-version/README.md | 6 ------ 7 files changed, 42 deletions(-) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 3960fd0e..43f8be92 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -270,12 +270,6 @@ These input values must be the same as any `terraform-plan` for the same configu For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-check/README.md b/terraform-check/README.md index b2fa03d4..727c12c9 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -172,12 +172,6 @@ This is intended to run on a schedule to notify if manual changes to your infras For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index dc761f10..263b8622 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -172,12 +172,6 @@ This action uses the `terraform destroy` command to destroy all resources in a t For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index c607dbb9..b956d578 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -169,12 +169,6 @@ This action uses the `terraform destroy` command to destroy all resources in a t For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 59594be5..2dd53e43 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -121,12 +121,6 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 0b5a58e4..fb6b8f61 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -98,12 +98,6 @@ If the terraform configuration is not valid, the build is failed. For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} diff --git a/terraform-version/README.md b/terraform-version/README.md index 1d0e6d8c..50a65583 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -103,12 +103,6 @@ outputs yourself. For example: ```yaml env: - TERRAFORM_PRE_RUN: | - # Install latest Azure CLI - curl -skL https://aka.ms/InstallAzureCLIDeb | bash - - # Install postgres client - apt-get install -y --no-install-recommends postgresql-client TERRAFORM_HTTP_CREDENTIALS: | example.com=dflook:${{ secrets.HTTPS_PASSWORD }} github.com/dflook/terraform-github-actions.git=dflook-actions:${{ secrets.ACTIONS_PAT }} From 596cf4c88dde2363a8cd8ee81b29c53c8ca36d36 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 25 Jul 2021 12:59:53 +0100 Subject: [PATCH 092/231] Disable workflow commands where possible while grouping logs --- image/actions.sh | 94 +++++++++++++++++++++----- image/entrypoints/apply.sh | 13 ++-- image/entrypoints/check.sh | 3 +- image/entrypoints/destroy-workspace.sh | 2 + image/entrypoints/destroy.sh | 2 + image/entrypoints/fmt.sh | 2 + image/entrypoints/new-workspace.sh | 18 +++-- image/entrypoints/output.sh | 2 + image/entrypoints/plan.sh | 4 ++ image/entrypoints/validate.sh | 2 + image/entrypoints/version.sh | 2 + 11 files changed, 113 insertions(+), 31 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 037807e6..94922889 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -12,14 +12,45 @@ function debug_cmd() { "$@" | while IFS= read -r line; do echo "::debug::${CMD_NAME}:${line}"; done; } +function disable_workflow_commands() { + if [[ -n "$WORKFLOW_COMMAND_TOKEN" ]]; then # Token is already set (not null) + echo "Tried to disable workflow commands, but they are already disabled" + exit 1 + fi + + WORKFLOW_COMMAND_TOKEN=$(random_string) + echo "::stop-commands::${WORKFLOW_COMMAND_TOKEN}" +} + +function enable_workflow_commands() { + if [[ -z "$WORKFLOW_COMMAND_TOKEN" ]]; then # Token is NOT set (null) + echo "Tried to enable workflow commands, but they are already enabled" + exit 1 + fi + + echo "::${WORKFLOW_COMMAND_TOKEN}::" + unset WORKFLOW_COMMAND_TOKEN +} + +function start_group() { + echo "::group::$1" +} + +function end_group() { + echo "::endgroup::" +} + function debug() { - debug_cmd ls -la /root - debug_cmd pwd - debug_cmd ls -la - debug_cmd ls -la $HOME - debug_cmd printenv - debug_cmd cat "$GITHUB_EVENT_PATH" - echo + if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then + start_group "Environment (ACTIONS_STEP_DEBUG)" + debug_cmd ls -la /root + debug_cmd pwd + debug_cmd ls -la + debug_cmd ls -la $HOME + debug_cmd printenv + debug_cmd cat "$GITHUB_EVENT_PATH" + end_group + fi } function detect-terraform-version() { @@ -53,19 +84,34 @@ function detect-tfmask() { function execute_run_commands() { if [[ -n $TERRAFORM_PRE_RUN ]]; then + start_group "Executing TERRAFORM_PRE_RUN" + disable_workflow_commands echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh bash -xeo pipefail /.prerun.sh + enable_workflow_commands + end_group fi } function setup() { + if [[ "$INPUT_PATH" == "" ]]; then + echo "::error:: input 'path' not set" + exit 1 + fi + + if [[ ! -d "$INPUT_PATH" ]]; then + echo "::error:: Path does not exist: \"$INPUT_PATH\"" + exit 1 + fi + TERRAFORM_BIN_DIR="$HOME/.dflook-terraform-bin-dir" export TF_DATA_DIR="$HOME/.dflook-terraform-data-dir" export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" unset TF_WORKSPACE # tfswitch guesses the wrong home directory... + start_group "Installing terraform" if [[ ! -d $TERRAFORM_BIN_DIR ]]; then debug_log "Initializing tfswitch with image default version" cp --recursive /root/.terraform.versions.default $TERRAFORM_BIN_DIR @@ -79,19 +125,10 @@ function setup() { mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" - if [[ "$INPUT_PATH" == "" ]]; then - echo "::error:: input 'path' not set" - exit 1 - fi - - if [[ ! -d "$INPUT_PATH" ]]; then - echo "::error:: Path does not exist: \"$INPUT_PATH\"" - exit 1 - fi - detect-terraform-version debug_cmd ls -la $TERRAFORM_BIN_DIR + end_group detect-tfmask @@ -99,7 +136,7 @@ function setup() { } function relative_to() { - local abspath + local absbase local relpath absbase="$1" @@ -108,13 +145,22 @@ function relative_to() { } function init() { + start_group "terraform init" + disable_workflow_commands + write_credentials rm -rf "$TF_DATA_DIR" (cd "$INPUT_PATH" && terraform init -input=false -backend=false) + + enable_workflow_commands + end_group } function init-backend() { + start_group "terraform init" + disable_workflow_commands + write_credentials INIT_ARGS="" @@ -154,10 +200,17 @@ function init-backend() { exit $INIT_EXIT fi fi + + enable_workflow_commands + end_group } function select-workspace() { + start_group "Select workspace" + disable_workflow_commands (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") + enable_workflow_commands + end_group } function set-plan-args() { @@ -188,6 +241,11 @@ function set-plan-args() { } function output() { + if [[ -n "$WORKFLOW_COMMAND_TOKEN" ]]; then + echo "Workflow commands are disabled, and they should not be" + exit 1 + fi + (cd "$INPUT_PATH" && terraform output -json | convert_output) } diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 3d77b9a7..ab3fb588 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -3,7 +3,6 @@ source /usr/local/actions.sh debug - setup init-backend select-workspace @@ -61,7 +60,7 @@ function apply() { } ### Generate a plan - +disable_workflow_commands plan if [[ $PLAN_EXIT -eq 1 ]]; then @@ -79,6 +78,7 @@ fi if [[ $PLAN_EXIT -eq 1 ]]; then cat "$PLAN_DIR/error.txt" + enable_workflow_commands update_status "Error applying plan in $(job_markdown_ref)" exit 1 fi @@ -113,16 +113,17 @@ else if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then apply else + echo "Plan changes:" + diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true + + enable_workflow_commands echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" - echo "Plan changes:" - debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" - diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true - exit 1 fi fi +enable_workflow_commands output diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index b6274929..01dacafe 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -7,7 +7,8 @@ setup init-backend select-workspace set-plan-args -output + +disable_workflow_commands set +e (cd $INPUT_PATH && terraform plan -input=false -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index c251daa7..c17b5647 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -8,6 +8,8 @@ init-backend select-workspace set-plan-args +disable_workflow_commands + (cd "$INPUT_PATH" \ && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index c8c2605e..2f8518ef 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -8,4 +8,6 @@ init-backend select-workspace set-plan-args +disable_workflow_commands + (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) diff --git a/image/entrypoints/fmt.sh b/image/entrypoints/fmt.sh index 8703805a..24344d17 100755 --- a/image/entrypoints/fmt.sh +++ b/image/entrypoints/fmt.sh @@ -5,4 +5,6 @@ source /usr/local/actions.sh debug setup +disable_workflow_commands + terraform fmt -recursive -no-color "$INPUT_PATH" diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index dfea9465..6ad28fac 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -6,6 +6,8 @@ debug setup init-backend +disable_workflow_commands + WS_TMP_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) rm -rf "$WS_TMP_DIR" mkdir -p "$WS_TMP_DIR" @@ -18,9 +20,11 @@ set +e readonly TF_WS_LIST_EXIT=${PIPESTATUS[0]} set -e -debug_log "terraform workspace list: ${TF_WS_LIST_EXIT}" -debug_cmd cat "$WS_TMP_DIR/list_err.txt" -debug_cmd cat "$WS_TMP_DIR/list_out.txt" +if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then + echo "terraform workspace list: ${TF_WS_LIST_EXIT}" + cat "$WS_TMP_DIR/list_err.txt" + cat "$WS_TMP_DIR/list_out.txt" +fi if [[ $TF_WS_LIST_EXIT -ne 0 ]]; then echo "Error: Failed to list workspaces" @@ -41,9 +45,11 @@ else readonly TF_WS_NEW_EXIT=${PIPESTATUS[0]} set -e - debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" - debug_cmd cat "$WS_TMP_DIR/new_err.txt" - debug_cmd cat "$WS_TMP_DIR/new_out.txt" + if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then + echo "terraform workspace new: ${TF_WS_NEW_EXIT}" + cat "$WS_TMP_DIR/new_err.txt" + cat "$WS_TMP_DIR/new_out.txt" + fi if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then diff --git a/image/entrypoints/output.sh b/image/entrypoints/output.sh index 148e927e..aaba69ea 100755 --- a/image/entrypoints/output.sh +++ b/image/entrypoints/output.sh @@ -2,7 +2,9 @@ source /usr/local/actions.sh +debug setup init-backend select-workspace + output diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 6e2354a1..4665363a 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -8,6 +8,8 @@ init-backend select-workspace set-plan-args +disable_workflow_commands + PLAN_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) rm -rf "$PLAN_DIR" mkdir -p "$PLAN_DIR" @@ -27,6 +29,8 @@ set -e cat "$PLAN_DIR/error.txt" +enable_workflow_commands + if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 9b4cb631..fa9d8da9 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -7,7 +7,9 @@ setup init if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH" ); then + disable_workflow_commands (cd "$INPUT_PATH" && terraform validate) + enable_workflow_commands else echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" fi diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index a3ade5bc..ac746b99 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,4 +6,6 @@ debug setup init +disble_workflow_commands + (cd "$INPUT_PATH" && terraform version -no-color | tee | convert_version) From a68f20bf0f04df84732ea5bbcdd1ab6ab78266c6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 00:12:52 +0100 Subject: [PATCH 093/231] Disable workflow commands unless needed --- image/Dockerfile | 1 + image/actions.sh | 98 +++++++---------------- image/entrypoints/apply.sh | 12 +-- image/entrypoints/check.sh | 4 +- image/entrypoints/destroy-workspace.sh | 2 - image/entrypoints/destroy.sh | 2 - image/entrypoints/fmt-check.sh | 2 + image/entrypoints/fmt.sh | 2 - image/entrypoints/new-workspace.sh | 6 +- image/entrypoints/plan.sh | 14 ++-- image/entrypoints/validate.sh | 3 +- image/entrypoints/version.sh | 4 +- image/tools/github_pr_comment.py | 2 +- image/workflow_commands.sh | 105 +++++++++++++++++++++++++ 14 files changed, 159 insertions(+), 98 deletions(-) create mode 100644 image/workflow_commands.sh diff --git a/image/Dockerfile b/image/Dockerfile index cabe1b3d..1fc04d69 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -2,6 +2,7 @@ FROM danielflook/terraform-github-actions-base:latest COPY entrypoints/ /entrypoints/ COPY actions.sh /usr/local/actions.sh +COPY workflow_commands.sh /usr/local/workflow_commands.sh COPY tools/convert_validate_report.py /usr/local/bin/convert_validate_report COPY tools/github_pr_comment.py /usr/local/bin/github_pr_comment diff --git a/image/actions.sh b/image/actions.sh index 94922889..c78ad9fd 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -2,43 +2,7 @@ set -eo pipefail -function debug_log() { - echo "::debug::" "$@" -} - -function debug_cmd() { - local CMD_NAME - CMD_NAME=$(echo "$@") - "$@" | while IFS= read -r line; do echo "::debug::${CMD_NAME}:${line}"; done; -} - -function disable_workflow_commands() { - if [[ -n "$WORKFLOW_COMMAND_TOKEN" ]]; then # Token is already set (not null) - echo "Tried to disable workflow commands, but they are already disabled" - exit 1 - fi - - WORKFLOW_COMMAND_TOKEN=$(random_string) - echo "::stop-commands::${WORKFLOW_COMMAND_TOKEN}" -} - -function enable_workflow_commands() { - if [[ -z "$WORKFLOW_COMMAND_TOKEN" ]]; then # Token is NOT set (null) - echo "Tried to enable workflow commands, but they are already enabled" - exit 1 - fi - - echo "::${WORKFLOW_COMMAND_TOKEN}::" - unset WORKFLOW_COMMAND_TOKEN -} - -function start_group() { - echo "::group::$1" -} - -function end_group() { - echo "::endgroup::" -} +source /usr/local/workflow_commands.sh function debug() { if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then @@ -46,7 +10,7 @@ function debug() { debug_cmd ls -la /root debug_cmd pwd debug_cmd ls -la - debug_cmd ls -la $HOME + debug_cmd ls -la "$HOME" debug_cmd printenv debug_cmd cat "$GITHUB_EVENT_PATH" end_group @@ -85,23 +49,23 @@ function detect-tfmask() { function execute_run_commands() { if [[ -n $TERRAFORM_PRE_RUN ]]; then start_group "Executing TERRAFORM_PRE_RUN" - disable_workflow_commands + echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh bash -xeo pipefail /.prerun.sh - enable_workflow_commands + end_group fi } function setup() { if [[ "$INPUT_PATH" == "" ]]; then - echo "::error:: input 'path' not set" + error_log "input 'path' not set" exit 1 fi if [[ ! -d "$INPUT_PATH" ]]; then - echo "::error:: Path does not exist: \"$INPUT_PATH\"" + error_log "Path does not exist: \"$INPUT_PATH\"" exit 1 fi @@ -111,23 +75,23 @@ function setup() { unset TF_WORKSPACE # tfswitch guesses the wrong home directory... - start_group "Installing terraform" + start_group "Installing Terraform" if [[ ! -d $TERRAFORM_BIN_DIR ]]; then debug_log "Initializing tfswitch with image default version" - cp --recursive /root/.terraform.versions.default $TERRAFORM_BIN_DIR + cp --recursive /root/.terraform.versions.default "$TERRAFORM_BIN_DIR" fi - ln -s $TERRAFORM_BIN_DIR /root/.terraform.versions + ln -s "$TERRAFORM_BIN_DIR" /root/.terraform.versions debug_cmd ls -lad /root/.terraform.versions - debug_cmd ls -lad $TERRAFORM_BIN_DIR - debug_cmd ls -la $TERRAFORM_BIN_DIR + debug_cmd ls -lad "$TERRAFORM_BIN_DIR" + debug_cmd ls -la "$TERRAFORM_BIN_DIR" mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" detect-terraform-version - debug_cmd ls -la $TERRAFORM_BIN_DIR + debug_cmd ls -la "$TERRAFORM_BIN_DIR" end_group detect-tfmask @@ -145,21 +109,18 @@ function relative_to() { } function init() { - start_group "terraform init" - disable_workflow_commands + start_group "Initializing Terraform" write_credentials rm -rf "$TF_DATA_DIR" (cd "$INPUT_PATH" && terraform init -input=false -backend=false) - enable_workflow_commands end_group } function init-backend() { - start_group "terraform init" - disable_workflow_commands + start_group "Initializing Terraform" write_credentials @@ -201,15 +162,15 @@ function init-backend() { fi fi - enable_workflow_commands + end_group } function select-workspace() { start_group "Select workspace" - disable_workflow_commands + (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") - enable_workflow_commands + end_group } @@ -241,21 +202,22 @@ function set-plan-args() { } function output() { - if [[ -n "$WORKFLOW_COMMAND_TOKEN" ]]; then - echo "Workflow commands are disabled, and they should not be" - exit 1 - fi - + enable_workflow_commands (cd "$INPUT_PATH" && terraform output -json | convert_output) + disable_workflow_commands } function update_status() { - local status="$1" + local status="$1" - if ! STATUS="$status" github_pr_comment status 2>&1 | sed 's/^/::debug::/'; then - echo "$status" - echo "Unable to update status on PR" - fi + enable_workflow_commands + if ! STATUS="$status" github_pr_comment status 2>&1 | sed 's/^/::debug::/'; then + disable_workflow_commands + echo "$status" + echo "Unable to update status on PR" + else + disable_workflow_commands + fi } function random_string() { @@ -263,8 +225,8 @@ function random_string() { } function write_credentials() { - format_tf_credentials >> $HOME/.terraformrc - netrc-credential-actions >> $HOME/.netrc + format_tf_credentials >> "$HOME/.terraformrc" + netrc-credential-actions >> "$HOME/.netrc" echo "$TERRAFORM_SSH_KEY" >> /.ssh/id_rsa chmod 600 /.ssh/id_rsa chmod 700 /.ssh diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index ab3fb588..da6cb7b4 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -33,7 +33,7 @@ function plan() { fi set +e - (cd $INPUT_PATH && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ + (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ 2>"$PLAN_DIR/error.txt" \ | $TFMASK \ | tee /dev/fd/3 \ @@ -47,7 +47,7 @@ function plan() { function apply() { set +e - (cd $INPUT_PATH && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT) | $TFMASK + (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT) | $TFMASK local APPLY_EXIT=${PIPESTATUS[0]} set -e @@ -60,7 +60,7 @@ function apply() { } ### Generate a plan -disable_workflow_commands + plan if [[ $PLAN_EXIT -eq 1 ]]; then @@ -78,7 +78,7 @@ fi if [[ $PLAN_EXIT -eq 1 ]]; then cat "$PLAN_DIR/error.txt" - enable_workflow_commands + update_status "Error applying plan in $(job_markdown_ref)" exit 1 fi @@ -103,12 +103,14 @@ else exit 1 fi + enable_workflow_commands if ! github_pr_comment get >"$PLAN_DIR/approved-plan.txt"; then echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" exit 1 fi + disable_workflow_commands if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then apply @@ -116,7 +118,6 @@ else echo "Plan changes:" diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true - enable_workflow_commands echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" @@ -125,5 +126,4 @@ else fi fi -enable_workflow_commands output diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index 01dacafe..86b2935e 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -8,10 +8,8 @@ init-backend select-workspace set-plan-args -disable_workflow_commands - set +e -(cd $INPUT_PATH && terraform plan -input=false -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ +(cd "$INPUT_PATH" && terraform plan -input=false -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ | $TFMASK readonly TF_EXIT=${PIPESTATUS[0]} diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index c17b5647..c251daa7 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -8,8 +8,6 @@ init-backend select-workspace set-plan-args -disable_workflow_commands - (cd "$INPUT_PATH" \ && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index 2f8518ef..c8c2605e 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -8,6 +8,4 @@ init-backend select-workspace set-plan-args -disable_workflow_commands - (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index 1fd680f8..16f617ef 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -9,7 +9,9 @@ terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read echo "$line" if [[ -f "$line" ]]; then + enable_workflow_commands echo "::error file=$line::File is not in canonical format (terraform fmt)" + disable_workflow_commands fi done diff --git a/image/entrypoints/fmt.sh b/image/entrypoints/fmt.sh index 24344d17..8703805a 100755 --- a/image/entrypoints/fmt.sh +++ b/image/entrypoints/fmt.sh @@ -5,6 +5,4 @@ source /usr/local/actions.sh debug setup -disable_workflow_commands - terraform fmt -recursive -no-color "$INPUT_PATH" diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index 6ad28fac..bf22d602 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -6,14 +6,12 @@ debug setup init-backend -disable_workflow_commands - WS_TMP_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) rm -rf "$WS_TMP_DIR" mkdir -p "$WS_TMP_DIR" set +e -(cd $INPUT_PATH && terraform workspace list -no-color) \ +(cd "$INPUT_PATH" && terraform workspace list -no-color) \ 2>"$WS_TMP_DIR/list_err.txt" \ >"$WS_TMP_DIR/list_out.txt" @@ -38,7 +36,7 @@ else echo "Workspace does not appear to exist, attempting to create it" set +e - (cd $INPUT_PATH && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") \ + (cd "$INPUT_PATH" && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") \ 2>"$WS_TMP_DIR/new_err.txt" \ >"$WS_TMP_DIR/new_out.txt" diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 4665363a..cf89bb95 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -8,8 +8,6 @@ init-backend select-workspace set-plan-args -disable_workflow_commands - PLAN_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) rm -rf "$PLAN_DIR" mkdir -p "$PLAN_DIR" @@ -17,7 +15,7 @@ mkdir -p "$PLAN_DIR" exec 3>&1 set +e -(cd $INPUT_PATH && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ +(cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ 2>"$PLAN_DIR/error.txt" \ | $TFMASK \ | tee /dev/fd/3 \ @@ -29,8 +27,6 @@ set -e cat "$PLAN_DIR/error.txt" -enable_workflow_commands - if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then @@ -42,7 +38,9 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c fi if [[ $TF_EXIT -eq 1 ]]; then + enable_workflow_commands STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/error.txt" + disable_workflow_commands else if [[ $TF_EXIT -eq 0 ]]; then @@ -51,7 +49,9 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c TF_CHANGES=true fi + enable_workflow_commands TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/plan.txt" + disable_workflow_commands fi fi @@ -66,9 +66,9 @@ if [[ $TF_EXIT -eq 1 ]]; then elif [[ $TF_EXIT -eq 0 ]]; then debug_log "No Changes to apply" - echo "::set-output name=changes::false" + set_output changes false elif [[ $TF_EXIT -eq 2 ]]; then debug_log "Changes to apply" - echo "::set-output name=changes::true" + set_output changes true fi diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index fa9d8da9..786dd0dd 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -6,10 +6,11 @@ debug setup init +enable_workflow_commands if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH" ); then disable_workflow_commands (cd "$INPUT_PATH" && terraform validate) - enable_workflow_commands else + disable_workflow_commands echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" fi diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index ac746b99..8ff6b978 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,6 +6,6 @@ debug setup init -disble_workflow_commands - +enable_workflow_commands (cd "$INPUT_PATH" && terraform version -no-color | tee | convert_version) +disable_workflow_commands diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index bd8d657e..7d494e35 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -18,7 +18,7 @@ def github_api_request(method, *args, **kw_args): response = github.request(method, *args, **kw_args) - if response.status_code >= 400 and response.status_code < 500: + if 400 <= response.status_code < 500: debug(str(response.headers)) try: diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh new file mode 100644 index 00000000..e89320c4 --- /dev/null +++ b/image/workflow_commands.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +## +# GitHub Actions workflow commands +# +# The processing of workflow commands is disabled, with these functions becoming the only way to +# use them. Processing can be enabled again using enable_workflow_commands + +## +# Send a string to the debug log +# +# This will be visible in the workflow log if ACTIONS_STEP_DEBUG workflow secret is set. +function debug_log() { + enable_workflow_commands + echo "::debug::" "$@" + disable_workflow_commands +} + +## +# Send a string to the error log +# +function error_log() { + enable_workflow_commands + echo "::error::" "$@" + disable_workflow_commands +} + + +## +# Run a command and send the output to the debug log +# +# This will be visible in the workflow log if ACTIONS_STEP_DEBUG workflow secret is set. +function debug_cmd() { + local CMD_NAME + CMD_NAME=$(echo "$@") + enable_workflow_commands + "$@" | while IFS= read -r line; do echo "::debug::${CMD_NAME}:${line}"; done; + disable_workflow_commands +} + +## +# Set an output value +# +function set_output() { + local name + local value + + name="$1" + value="${*:2}" + + enable_workflow_commands + echo "::set-output name=${name}::${value}" + disable_workflow_commands +} + +## +# Start a log group +# +# All output between this and the next end_group will be collapsed into an expandable group +function start_group() { + enable_workflow_commands + echo "::group::$1" + disable_workflow_commands +} + +## +# End a log group +# +function end_group() { + enable_workflow_commands + echo "::endgroup::" + disable_workflow_commands +} + +## +# Enable to processing of workflow commands +# +function enable_workflow_commands() { + if [[ -z "$WORKFLOW_COMMAND_TOKEN" ]]; then + echo "Tried to enable workflow commands, but they are already enabled" + exit 1 + fi + + echo "::${WORKFLOW_COMMAND_TOKEN}::" + unset WORKFLOW_COMMAND_TOKEN +} + +## +# Disable the processing of workflow commands +# +function disable_workflow_commands() { + if [[ -n "$WORKFLOW_COMMAND_TOKEN" ]]; then + echo "Tried to disable workflow commands, but they are already disabled" + exit 1 + fi + + WORKFLOW_COMMAND_TOKEN=$(generate_command_token) + echo "::stop-commands::${WORKFLOW_COMMAND_TOKEN}" +} + +function generate_command_token() { + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" +} + +disable_workflow_commands From f62f92aac6c35820fbb7da4f33f3dfe4c751b346 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 00:28:36 +0100 Subject: [PATCH 094/231] ACTIONS_STEP_DEBUG may have any value --- image/actions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index c78ad9fd..766aa18f 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -5,7 +5,7 @@ set -eo pipefail source /usr/local/workflow_commands.sh function debug() { - if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then + if [[ -n "$ACTIONS_STEP_DEBUG" ]]; then start_group "Environment (ACTIONS_STEP_DEBUG)" debug_cmd ls -la /root debug_cmd pwd @@ -167,7 +167,7 @@ function init-backend() { } function select-workspace() { - start_group "Select workspace" + start_group "Selecting workspace" (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") @@ -221,7 +221,7 @@ function update_status() { } function random_string() { - python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(32)))" } function write_credentials() { From fa313e1eb070a0240bdff6fed5bbaf5064b69626 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 00:38:18 +0100 Subject: [PATCH 095/231] ACTIONS_STEP_DEBUG is not available unless explicitly passed in This is at least the third time i've forgotten this. --- image/actions.sh | 19 +++++++++---------- image/entrypoints/new-workspace.sh | 16 ++++++---------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 766aa18f..53e4df35 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -5,16 +5,15 @@ set -eo pipefail source /usr/local/workflow_commands.sh function debug() { - if [[ -n "$ACTIONS_STEP_DEBUG" ]]; then - start_group "Environment (ACTIONS_STEP_DEBUG)" - debug_cmd ls -la /root - debug_cmd pwd - debug_cmd ls -la - debug_cmd ls -la "$HOME" - debug_cmd printenv - debug_cmd cat "$GITHUB_EVENT_PATH" - end_group - fi + start_group "Environment" + echo "When debug logging is enabled, additional information is output here" + debug_cmd ls -la /root + debug_cmd pwd + debug_cmd ls -la + debug_cmd ls -la "$HOME" + debug_cmd printenv + debug_cmd cat "$GITHUB_EVENT_PATH" + end_group } function detect-terraform-version() { diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index bf22d602..0b419ab2 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -18,11 +18,9 @@ set +e readonly TF_WS_LIST_EXIT=${PIPESTATUS[0]} set -e -if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then - echo "terraform workspace list: ${TF_WS_LIST_EXIT}" - cat "$WS_TMP_DIR/list_err.txt" - cat "$WS_TMP_DIR/list_out.txt" -fi +debug_log "terraform workspace list: ${TF_WS_LIST_EXIT}" +debug_cmd cat "$WS_TMP_DIR/list_err.txt" +debug_cmd cat "$WS_TMP_DIR/list_out.txt" if [[ $TF_WS_LIST_EXIT -ne 0 ]]; then echo "Error: Failed to list workspaces" @@ -43,11 +41,9 @@ else readonly TF_WS_NEW_EXIT=${PIPESTATUS[0]} set -e - if [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then - echo "terraform workspace new: ${TF_WS_NEW_EXIT}" - cat "$WS_TMP_DIR/new_err.txt" - cat "$WS_TMP_DIR/new_out.txt" - fi + debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" + debug_cmd cat "$WS_TMP_DIR/new_err.txt" + debug_cmd cat "$WS_TMP_DIR/new_out.txt" if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then From 77dff0294eee0741f11111589c4df4adb0f84e7d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 00:48:12 +0100 Subject: [PATCH 096/231] Increase length of workflow command token --- image/actions.sh | 2 +- image/entrypoints/apply.sh | 7 ++++--- image/workflow_commands.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 53e4df35..18026768 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -220,7 +220,7 @@ function update_status() { } function random_string() { - python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(32)))" + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" } function write_credentials() { diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index da6cb7b4..629d4b9f 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -115,13 +115,14 @@ else if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then apply else - echo "Plan changes:" - diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true - echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" + echo "Plan changes:" + debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" + diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true + exit 1 fi fi diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index e89320c4..059a7b63 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -99,7 +99,7 @@ function disable_workflow_commands() { } function generate_command_token() { - python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(32)))" } disable_workflow_commands From 6d6525663eb699801cb11c3c4e03ad0275ac9a61 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 09:08:58 +0100 Subject: [PATCH 097/231] :point_right: --- .github/github_sucks.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index cee65b7f..471620e2 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,3 +1,2 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. - From 85e508e809df4e1e733862a3c1f0af69d11a8f81 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 19:02:09 +0100 Subject: [PATCH 098/231] Refactor debug logging --- .github/workflows/test-http.yaml | 8 +++--- ...{test_registry.yaml => test-registry.yaml} | 0 .github/workflows/test-workflow-commands.yaml | 28 +++++++++++++++++++ image/actions.sh | 21 ++++++-------- image/entrypoints/apply.sh | 5 ++-- image/entrypoints/new-workspace.sh | 8 +++--- image/entrypoints/plan.sh | 14 ++++++---- image/entrypoints/validate.sh | 7 +++-- image/entrypoints/version.sh | 4 ++- image/tools/github_pr_comment.py | 21 +++++++------- image/workflow_commands.sh | 13 ++++++++- 11 files changed, 85 insertions(+), 44 deletions(-) rename .github/workflows/{test_registry.yaml => test-registry.yaml} (100%) create mode 100644 .github/workflows/test-workflow-commands.yaml diff --git a/.github/workflows/test-http.yaml b/.github/workflows/test-http.yaml index 0fef73bc..9026eb78 100644 --- a/.github/workflows/test-http.yaml +++ b/.github/workflows/test-http.yaml @@ -8,7 +8,7 @@ env: jobs: git_http_full_path_credentials: runs-on: ubuntu-latest - name: git+http module source + name: git+http full path creds env: TERRAFORM_HTTP_CREDENTIALS: | github.com/dflook/hello=dflook:notapassword @@ -34,7 +34,7 @@ jobs: git_http_partial_path_credentials: runs-on: ubuntu-latest - name: git+http module source + name: git+http partial path creds env: TERRAFORM_HTTP_CREDENTIALS: | github.com/dflook/hello=dflook:notapassword @@ -60,7 +60,7 @@ jobs: git_http_no_path_credentials: runs-on: ubuntu-latest - name: git+http module source + name: git+http no path env: TERRAFORM_HTTP_CREDENTIALS: | github.com/dflook/hello=dflook:notapassword @@ -86,7 +86,7 @@ jobs: git_no_credentials: runs-on: ubuntu-latest - name: git_http module source with no key + name: git_http no creds steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/test_registry.yaml b/.github/workflows/test-registry.yaml similarity index 100% rename from .github/workflows/test_registry.yaml rename to .github/workflows/test-registry.yaml diff --git a/.github/workflows/test-workflow-commands.yaml b/.github/workflows/test-workflow-commands.yaml new file mode 100644 index 00000000..acb50eed --- /dev/null +++ b/.github/workflows/test-workflow-commands.yaml @@ -0,0 +1,28 @@ +name: Test workflow command supression + +on: [ pull_request ] + +jobs: + workflow_command_injection: + runs-on: ubuntu-latest + name: Plan with workflow command injection + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + id: plan + with: + path: tests/plan/plan + add_github_comment: false + env: + TERRAFORM_PRE_RUN: | + echo "::set-output name=output_string::strawberry" + + - name: Verify outputs + run: | + if [[ -n "${{ steps.plan.outputs.output_string }}" ]]; then + echo "::error:: output_string should not have been set" + exit 1 + fi diff --git a/image/actions.sh b/image/actions.sh index 18026768..23640312 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -12,7 +12,7 @@ function debug() { debug_cmd ls -la debug_cmd ls -la "$HOME" debug_cmd printenv - debug_cmd cat "$GITHUB_EVENT_PATH" + debug_file "$GITHUB_EVENT_PATH" end_group } @@ -166,11 +166,13 @@ function init-backend() { } function select-workspace() { - start_group "Selecting workspace" + (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") >/tmp/select-workspace 2>&1 - (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") - - end_group + if [[ -s /tmp/select-workspace ]]; then + start_group "Selecting workspace" + cat /tmp/select-workspace + end_group + fi } function set-plan-args() { @@ -209,13 +211,8 @@ function output() { function update_status() { local status="$1" - enable_workflow_commands - if ! STATUS="$status" github_pr_comment status 2>&1 | sed 's/^/::debug::/'; then - disable_workflow_commands - echo "$status" - echo "Unable to update status on PR" - else - disable_workflow_commands + if ! STATUS="$status" github_pr_comment status 2>/tmp/github_pr_comment.error; then + debug_file /tmp/github_pr_comment.error fi } diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 629d4b9f..8f872065 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -103,14 +103,13 @@ else exit 1 fi - enable_workflow_commands - if ! github_pr_comment get >"$PLAN_DIR/approved-plan.txt"; then + if ! github_pr_comment get "$PLAN_DIR/approved-plan.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then + debug_file "$PLAN_DIR/github_pr_comment.error" echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" exit 1 fi - disable_workflow_commands if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then apply diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index 0b419ab2..31b965fe 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -19,8 +19,8 @@ readonly TF_WS_LIST_EXIT=${PIPESTATUS[0]} set -e debug_log "terraform workspace list: ${TF_WS_LIST_EXIT}" -debug_cmd cat "$WS_TMP_DIR/list_err.txt" -debug_cmd cat "$WS_TMP_DIR/list_out.txt" +debug_file "$WS_TMP_DIR/list_err.txt" +debug_file "$WS_TMP_DIR/list_out.txt" if [[ $TF_WS_LIST_EXIT -ne 0 ]]; then echo "Error: Failed to list workspaces" @@ -42,8 +42,8 @@ else set -e debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" - debug_cmd cat "$WS_TMP_DIR/new_err.txt" - debug_cmd cat "$WS_TMP_DIR/new_out.txt" + debug_file "$WS_TMP_DIR/new_err.txt" + debug_file "$WS_TMP_DIR/new_out.txt" if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index cf89bb95..87598458 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -38,9 +38,10 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c fi if [[ $TF_EXIT -eq 1 ]]; then - enable_workflow_commands - STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/error.txt" - disable_workflow_commands + if ! STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$PLAN_DIR/error.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then + debug_file "$PLAN_DIR/github_pr_comment.error" + exit 1 + fi else if [[ $TF_EXIT -eq 0 ]]; then @@ -49,9 +50,10 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c TF_CHANGES=true fi - enable_workflow_commands - TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"/$PLAN_DIR/plan.txt" - disable_workflow_commands + if ! TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$PLAN_DIR/plan.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then + debug_file "$PLAN_DIR/github_pr_comment.error" + exit 1 + fi fi fi diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 786dd0dd..366d6107 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -6,11 +6,12 @@ debug setup init -enable_workflow_commands -if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH" ); then +if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH") >/tmp/report.txt; then + enable_workflow_commands + cat /tmp/report.txt disable_workflow_commands + (cd "$INPUT_PATH" && terraform validate) else - disable_workflow_commands echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" fi diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index 8ff6b978..06166dfc 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,6 +6,8 @@ debug setup init +(cd "$INPUT_PATH" && terraform version -no-color | convert_version) >/tmp/version.txt + enable_workflow_commands -(cd "$INPUT_PATH" && terraform version -no-color | tee | convert_version) +cat /tmp/version.txt disable_workflow_commands diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 7d494e35..045ee9df 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -26,26 +26,25 @@ def github_api_request(method, *args, **kw_args): if response.headers['X-RateLimit-Remaining'] == '0': limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) - sys.stderr.write(message) - sys.stderr.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') + sys.stdout.write(message) + sys.stdout.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') exit(1) if message != 'Resource not accessible by integration': - sys.stderr.write(message) - sys.stderr.write('\n') + sys.stdout.write(message) + sys.stdout.write('\n') debug(response.content.decode()) except Exception: - sys.stderr.write(response.content.decode()) - sys.stderr.write('\n') + sys.stdout.write(response.content.decode()) + sys.stdout.write('\n') raise return response def debug(msg: str) -> None: - for line in msg.splitlines(): - sys.stderr.write(f'::debug::{line}\n') - + sys.stderr.write(msg) + sys.stderr.write('\n') def prs(repo: str) -> Iterable[Dict]: url = f'https://api.github.com/repos/{repo}/pulls' @@ -420,7 +419,9 @@ def update_comment(self, only_if_exists=False): elif sys.argv[1] == 'get': if tf_comment.plan is None: exit(1) - print(tf_comment.plan) + + with open(sys.argv[2], 'w') as f: + f.write(tf_comment.plan) exit(0) tf_comment.update_comment(only_if_exists) diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index 059a7b63..1305e3e9 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -25,7 +25,6 @@ function error_log() { disable_workflow_commands } - ## # Run a command and send the output to the debug log # @@ -38,6 +37,18 @@ function debug_cmd() { disable_workflow_commands } +## +# Print a file to the debug log +# +# This will be visible in the workflow log if ACTIONS_STEP_DEBUG workflow secret is set. +function debug_file() { + local FILE_PATH + FILE_PATH="$1" + enable_workflow_commands + sed "s|^|::debug $FILE_PATH:|" $FILE_PATH + disable_workflow_commands +} + ## # Set an output value # From 2d83ab2f99c590694cb77a3834272ce578c93ca0 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 19:13:12 +0100 Subject: [PATCH 099/231] Fix debug_file --- image/entrypoints/validate.sh | 5 ++--- image/entrypoints/version.sh | 4 +--- image/workflow_commands.sh | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 366d6107..2c32d01e 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -6,9 +6,8 @@ debug setup init -if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH") >/tmp/report.txt; then - enable_workflow_commands - cat /tmp/report.txt +enable_workflow_commands +if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH"); then disable_workflow_commands (cd "$INPUT_PATH" && terraform validate) diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index 06166dfc..259f6332 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,8 +6,6 @@ debug setup init -(cd "$INPUT_PATH" && terraform version -no-color | convert_version) >/tmp/version.txt - enable_workflow_commands -cat /tmp/version.txt +(cd "$INPUT_PATH" && terraform version -no-color | convert_version) disable_workflow_commands diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index 1305e3e9..e35ec9a5 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -45,7 +45,7 @@ function debug_file() { local FILE_PATH FILE_PATH="$1" enable_workflow_commands - sed "s|^|::debug $FILE_PATH:|" $FILE_PATH + sed "s|^|::debug::$FILE_PATH:|" $FILE_PATH disable_workflow_commands } From 3293e90208794c3080015e4602b68f338f22d062 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 22:40:25 +0100 Subject: [PATCH 100/231] Only stop workflow commands for pre-run stop-command is just too noisy --- image/actions.sh | 4 ++-- image/entrypoints/fmt-check.sh | 2 -- image/entrypoints/validate.sh | 3 --- image/entrypoints/version.sh | 2 -- image/workflow_commands.sh | 15 --------------- 5 files changed, 2 insertions(+), 24 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 23640312..48b48f3c 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -51,7 +51,9 @@ function execute_run_commands() { echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh + disable_workflow_commands bash -xeo pipefail /.prerun.sh + enable_workflow_commands end_group fi @@ -203,9 +205,7 @@ function set-plan-args() { } function output() { - enable_workflow_commands (cd "$INPUT_PATH" && terraform output -json | convert_output) - disable_workflow_commands } function update_status() { diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index 16f617ef..1fd680f8 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -9,9 +9,7 @@ terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read echo "$line" if [[ -f "$line" ]]; then - enable_workflow_commands echo "::error file=$line::File is not in canonical format (terraform fmt)" - disable_workflow_commands fi done diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 2c32d01e..1239622c 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -6,10 +6,7 @@ debug setup init -enable_workflow_commands if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH"); then - disable_workflow_commands - (cd "$INPUT_PATH" && terraform validate) else echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index 259f6332..49795586 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,6 +6,4 @@ debug setup init -enable_workflow_commands (cd "$INPUT_PATH" && terraform version -no-color | convert_version) -disable_workflow_commands diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index e35ec9a5..7daaecc5 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -11,18 +11,14 @@ # # This will be visible in the workflow log if ACTIONS_STEP_DEBUG workflow secret is set. function debug_log() { - enable_workflow_commands echo "::debug::" "$@" - disable_workflow_commands } ## # Send a string to the error log # function error_log() { - enable_workflow_commands echo "::error::" "$@" - disable_workflow_commands } ## @@ -32,9 +28,7 @@ function error_log() { function debug_cmd() { local CMD_NAME CMD_NAME=$(echo "$@") - enable_workflow_commands "$@" | while IFS= read -r line; do echo "::debug::${CMD_NAME}:${line}"; done; - disable_workflow_commands } ## @@ -44,9 +38,7 @@ function debug_cmd() { function debug_file() { local FILE_PATH FILE_PATH="$1" - enable_workflow_commands sed "s|^|::debug::$FILE_PATH:|" $FILE_PATH - disable_workflow_commands } ## @@ -59,9 +51,7 @@ function set_output() { name="$1" value="${*:2}" - enable_workflow_commands echo "::set-output name=${name}::${value}" - disable_workflow_commands } ## @@ -69,18 +59,14 @@ function set_output() { # # All output between this and the next end_group will be collapsed into an expandable group function start_group() { - enable_workflow_commands echo "::group::$1" - disable_workflow_commands } ## # End a log group # function end_group() { - enable_workflow_commands echo "::endgroup::" - disable_workflow_commands } ## @@ -113,4 +99,3 @@ function generate_command_token() { python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(32)))" } -disable_workflow_commands From 190ba9c38cd6bd977c551e99cc9e601fac93570e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 26 Jul 2021 23:17:59 +0100 Subject: [PATCH 101/231] Dont use Environment group --- image/actions.sh | 3 --- image/workflow_commands.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 48b48f3c..d6ef03ad 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -5,15 +5,12 @@ set -eo pipefail source /usr/local/workflow_commands.sh function debug() { - start_group "Environment" - echo "When debug logging is enabled, additional information is output here" debug_cmd ls -la /root debug_cmd pwd debug_cmd ls -la debug_cmd ls -la "$HOME" debug_cmd printenv debug_file "$GITHUB_EVENT_PATH" - end_group } function detect-terraform-version() { diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index 7daaecc5..75e9a92a 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -96,6 +96,6 @@ function disable_workflow_commands() { } function generate_command_token() { - python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(32)))" + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(64)))" } From 2fa7607e181a0b6f4c8f37a67c59dae50176f284 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 27 Jul 2021 23:16:32 +0100 Subject: [PATCH 102/231] . --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index 471620e2..cee65b7f 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,2 +1,3 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From b5eb548ebe39afe29860fe3bb955c2ab889a9512 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 27 Jul 2021 23:33:59 +0100 Subject: [PATCH 103/231] echo --- image/actions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/image/actions.sh b/image/actions.sh index d6ef03ad..ff887bff 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -11,6 +11,7 @@ function debug() { debug_cmd ls -la "$HOME" debug_cmd printenv debug_file "$GITHUB_EVENT_PATH" + echo } function detect-terraform-version() { From 77240d7edc87391f87869f8a524f34e3d94448eb Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 28 Jul 2021 09:33:55 +0100 Subject: [PATCH 104/231] Don't use a group for select-workspace it's more noise that it's worth --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index cee65b7f..9f9f9874 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,3 +1,4 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From 2041b174c7affc2ee350e83039562c0fc23cbafe Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 29 Jul 2021 19:23:49 +0100 Subject: [PATCH 105/231] Don't use a group for select-workspace it's more noise that it's worth --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index 9f9f9874..f941c83c 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -2,3 +2,4 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From 1cfa35b4ae66cd93c471d165ffb5aa8901027420 Mon Sep 17 00:00:00 2001 From: Brandon Doyle Date: Wed, 11 Aug 2021 14:31:03 -0400 Subject: [PATCH 106/231] Removed tfswitch tarball to decrease base image size slightly --- image/Dockerfile-base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/Dockerfile-base b/image/Dockerfile-base index b41a6eb3..c0383f5b 100644 --- a/image/Dockerfile-base +++ b/image/Dockerfile-base @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ RUN curl -fsL https://github.com/warrensbox/terraform-switcher/releases/download/${TFSWITCH_VERSION}/terraform-switcher_${TFSWITCH_VERSION}_linux_amd64.tar.gz -o tfswitch.tar.gz \ && tar -xvf tfswitch.tar.gz \ && mv tfswitch /usr/local/bin \ - && rm -rf README.md LICENSE terraform-switcher \ + && rm -rf README.md LICENSE terraform-switcher tfswitch.tar.gz \ && tfswitch $DEFAULT_TF_VERSION \ && mv /root/.terraform.versions /root/.terraform.versions.default RUN mkdir -p $TF_PLUGIN_CACHE_DIR From df9d383435cb24f72a14b9e9c290f30f8d9115f3 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 18 Aug 2021 11:27:02 -0400 Subject: [PATCH 107/231] Fix typo in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e14abe1d..eb6b29d7 100644 --- a/README.md +++ b/README.md @@ -181,9 +181,9 @@ on: - cron: "0 8 * * *" jobs: - check_drift: + rotate_certs: runs-on: ubuntu-latest - name: Check for drift of example terraform configuration + name: Rotate TLS certificates in example terraform configuration steps: - name: Checkout uses: actions/checkout@v2 From 8a35750099d3ffc8039a34d6917f08b4e8973d15 Mon Sep 17 00:00:00 2001 From: Nick Gray Date: Mon, 13 Sep 2021 19:59:31 -0400 Subject: [PATCH 108/231] Support any github instance --- image/actions.sh | 2 +- image/tools/github_pr_comment.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 037807e6..3d431fa3 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -39,7 +39,7 @@ function detect-terraform-version() { } function job_markdown_ref() { - echo "[${GITHUB_WORKFLOW} #${GITHUB_RUN_NUMBER}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" + echo "[${GITHUB_WORKFLOW} #${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" } function detect-tfmask() { diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index bd8d657e..81df5996 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -15,6 +15,9 @@ github.headers['user-agent'] = 'terraform-github-actions' github.headers['accept'] = 'application/vnd.github.v3+json' +github_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') +github_api_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com') + def github_api_request(method, *args, **kw_args): response = github.request(method, *args, **kw_args) @@ -48,7 +51,7 @@ def debug(msg: str) -> None: def prs(repo: str) -> Iterable[Dict]: - url = f'https://api.github.com/repos/{repo}/pulls' + url = f'{github_api_url}/repos/{repo}/pulls' while True: response = github_api_request('get', url, params={'state': 'all'}) @@ -112,7 +115,7 @@ def current_user() -> str: except Exception as e: debug(str(e)) - response = github_api_request('get', 'https://api.github.com/user') + response = github_api_request('get', f'{github_api_url}/user') if response.status_code != 403: user = response.json() debug('GITHUB_TOKEN user:') From 38a17a0a5dc577a670e786fb707eed2654308a5b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 15 Sep 2021 20:37:39 +0100 Subject: [PATCH 109/231] Use the action workspace by default --- .github/workflows/test-plan.yaml | 17 +++++++++++++++++ terraform-apply/README.md | 3 ++- terraform-apply/action.yaml | 3 ++- terraform-check/README.md | 3 ++- terraform-check/action.yaml | 3 ++- terraform-destroy-workspace/README.md | 3 ++- terraform-destroy-workspace/action.yaml | 3 ++- terraform-destroy/README.md | 3 ++- terraform-destroy/action.yaml | 3 ++- terraform-fmt-check/README.md | 3 ++- terraform-fmt-check/action.yaml | 3 ++- terraform-fmt/README.md | 3 ++- terraform-fmt/action.yaml | 3 ++- terraform-new-workspace/README.md | 3 ++- terraform-new-workspace/action.yaml | 3 ++- terraform-output/README.md | 3 ++- terraform-output/action.yaml | 3 ++- terraform-plan/README.md | 3 ++- terraform-plan/action.yaml | 3 ++- terraform-validate/README.md | 3 ++- terraform-validate/action.yaml | 3 ++- terraform-version/README.md | 3 ++- terraform-version/action.yaml | 3 ++- 23 files changed, 61 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index f2b7bddd..9ac531b3 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -329,3 +329,20 @@ jobs: TERRAFORM_PRE_RUN: | echo "testing command 1" echo "testing command 2" + + default_path: + runs-on: ubuntu-latest + name: Default path + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Copy + run: cp tests/plan/plan/main.tf ./main.tf + + - name: Plan + uses: ./terraform-plan + with: + label: Optional path diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 82aebfd2..57517a63 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -38,7 +38,8 @@ These input values must be the same as any `terraform-plan` for the same configu Path to the terraform configuration to apply - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index a32b1b66..b9551419 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: false diff --git a/terraform-check/README.md b/terraform-check/README.md index 727c12c9..4a664a53 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -13,7 +13,8 @@ This is intended to run on a schedule to notify if manual changes to your infras Path to the terraform configuration to check - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index cae2bc0c..37677cfc 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: false diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 263b8622..ea273c16 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -11,7 +11,8 @@ This action uses the `terraform destroy` command to destroy all resources in a t Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index 47e1e2db..ccb755c8 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: true diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index b956d578..0fd0e28b 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -12,7 +12,8 @@ This action uses the `terraform destroy` command to destroy all resources in a t Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 1a3121a2..9c625344 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: false diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index 58551493..4a1d8eb8 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -14,7 +14,8 @@ If any files are not correctly formatted a failing GitHub check will be added fo Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace ## Example usage diff --git a/terraform-fmt-check/action.yaml b/terraform-fmt-check/action.yaml index 78ded3fb..fda9a3ba 100644 --- a/terraform-fmt-check/action.yaml +++ b/terraform-fmt-check/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . runs: using: docker diff --git a/terraform-fmt/README.md b/terraform-fmt/README.md index 74d5aa2a..b002d0bf 100644 --- a/terraform-fmt/README.md +++ b/terraform-fmt/README.md @@ -11,7 +11,8 @@ This action uses the `terraform fmt` command to reformat files in a directory in Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace ## Example usage diff --git a/terraform-fmt/action.yaml b/terraform-fmt/action.yaml index 13ec7d66..8806d78e 100644 --- a/terraform-fmt/action.yaml +++ b/terraform-fmt/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . runs: using: docker diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 2dd53e43..b402ecf8 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -11,7 +11,8 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-new-workspace/action.yaml b/terraform-new-workspace/action.yaml index 12c53d85..374c8c9e 100644 --- a/terraform-new-workspace/action.yaml +++ b/terraform-new-workspace/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: true diff --git a/terraform-output/README.md b/terraform-output/README.md index 1b38a55f..60489d7f 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -11,7 +11,8 @@ Retrieve the root-level outputs from a terraform configuration. Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-output/action.yaml b/terraform-output/action.yaml index a9312ff8..63f18f11 100644 --- a/terraform-output/action.yaml +++ b/terraform-output/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: false diff --git a/terraform-plan/README.md b/terraform-plan/README.md index ca20ad1c..d958af77 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -21,7 +21,8 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace * `workspace` diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 77233581..56ea8f48 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . workspace: description: Name of the terraform workspace required: false diff --git a/terraform-validate/README.md b/terraform-validate/README.md index fb6b8f61..e633f978 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -20,7 +20,8 @@ If the terraform configuration is not valid, the build is failed. Path to the terraform configuration - Type: string - - Required + - Optional + - Default: The action workspace ## Environment Variables diff --git a/terraform-validate/action.yaml b/terraform-validate/action.yaml index 5f3c8305..1ab12e52 100644 --- a/terraform-validate/action.yaml +++ b/terraform-validate/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . runs: using: docker diff --git a/terraform-version/README.md b/terraform-version/README.md index 50a65583..a3d2b31c 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -25,7 +25,8 @@ outputs yourself. Path to the terraform configuration to apply - Type: string - - Required + - Optional + - Default: The action workspace ## Environment Variables diff --git a/terraform-version/action.yaml b/terraform-version/action.yaml index 50ef1c2d..04b1b1a5 100644 --- a/terraform-version/action.yaml +++ b/terraform-version/action.yaml @@ -5,7 +5,8 @@ author: Daniel Flook inputs: path: description: Path to the terraform configuration - required: true + required: false + default: . outputs: version: From 54099290c727b04fa5467f2e65157a6e290a9095 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 15 Sep 2021 21:39:01 +0100 Subject: [PATCH 110/231] Search comments over multiple api pages --- image/tools/github_pr_comment.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 9482b1ce..5c22a856 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -18,8 +18,8 @@ github_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') github_api_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com') -def github_api_request(method, *args, **kw_args): - response = github.request(method, *args, **kw_args) +def github_api_request(method, *args, **kwargs): + response = github.request(method, *args, **kwargs) if 400 <= response.status_code < 500: debug(str(response.headers)) @@ -49,21 +49,23 @@ def debug(msg: str) -> None: sys.stderr.write(msg) sys.stderr.write('\n') -def prs(repo: str) -> Iterable[Dict]: - url = f'{github_api_url}/repos/{repo}/pulls' +def paginate(url, *args, **kwargs) -> Iterable[Dict]: while True: - response = github_api_request('get', url, params={'state': 'all'}) + response = github_api_request('get', url, *args, **kwargs) response.raise_for_status() - for pr in response.json(): - yield pr + yield from response.json() if 'next' in response.links: url = response.links['next']['url'] else: return +def prs(repo: str) -> Iterable[Dict]: + url = f'{github_api_url}/repos/{repo}/pulls' + yield from paginate(url, params={'state': 'all'}) + def find_pr() -> str: """ @@ -152,13 +154,12 @@ def __init__(self, pr_url: str=None): response.raise_for_status() self._issue_url = response.json()['_links']['issue']['href'] + '/comments' - response = github_api_request('get', self._issue_url) - response.raise_for_status() username = current_user() debug('Looking for an existing comment:') - for comment in response.json(): + + for comment in paginate(self._issue_url): debug(json.dumps(comment)) if comment['user']['login'] == username: match = re.match(rf'{re.escape(self._comment_identifier)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) From 8e20e61cf0dc9b92776f798237d92d8ac8878467 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 15 Sep 2021 21:58:30 +0100 Subject: [PATCH 111/231] :bookmark: v1.14.0 --- CHANGELOG.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c67913b..64a43565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,14 +8,25 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.13.0` to use an exact release -- `@v1.13` to use the latest patch release for the specific minor version +- `@v1.14.0` to use an exact release +- `@v1.14` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## [1.13.0] - 2021-07-24 +## [1.14.0] - 2021-09-15 ### Added +- Support for self-hosted GitHub Enterprise deployments. Thanks [f0rkz](https://github.com/f0rkz)! + +### Changed +- The `path` input variable is now optional, defaulting to the Action workspace. +- Uninteresting workflow log output is now grouped and collapsed by default. + +### Fixed +- Applying PR approved plans where the plan comment is not within the first 30 comments. + +## [1.13.0] - 2021-07-24 +### Added - `TERRAFORM_PRE_RUN` environment variable for customising the environment before running terraform. It can be set to a command that will be run prior to `terraform init`. @@ -36,23 +47,20 @@ When using an action you can specify the version as: apt-get install -y --no-install-recommends postgresql-client ``` - Thanks to @alec-pinson and @GiuseppeChiesa-TomTom for working on this feature. + Thanks to [alec-pinson](https://github.com/alec-pinson) and [GiuseppeChiesa-TomTom](https://github.com/GiuseppeChiesa-TomTom) for working on this feature. ## [1.12.0] - 2021-06-08 ### Changed - - [terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) now shows a diff in the workflow log when it finds files in non-canonical format ## [1.11.0] - 2021-06-05 ### Added - - The `add_github_comment` input for [terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) may now be set to `changes-only`. This will only add a PR comment for plans that result in changes to apply - no comment will be added for plans with no changes. ### Changed - - Improved messaging in the workflow log when [terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) is aborted because the plan has changed - Update documentation for `backend_config`, `backend_config_file`, `var_file` & `target` inputs to use separate lines for multiple values. Multiple values may still be separated by commas if preferred. @@ -60,7 +68,6 @@ When using an action you can specify the version as: ## [1.10.0] - 2021-05-30 ### Added - - `TERRAFORM_HTTP_CREDENTIALS` environment variable for configuring the username and password to use for `git::https://` & `https://` module sources. @@ -212,6 +219,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.14.0]: https://github.com/dflook/terraform-github-actions/compare/v1.13.0...v1.14.0 [1.13.0]: https://github.com/dflook/terraform-github-actions/compare/v1.12.0...v1.13.0 [1.12.0]: https://github.com/dflook/terraform-github-actions/compare/v1.11.0...v1.12.0 [1.11.0]: https://github.com/dflook/terraform-github-actions/compare/v1.10.0...v1.11.0 From 455b995992363e80b1ccf1245daf98db3ae9c03f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 18 Sep 2021 12:27:08 +0100 Subject: [PATCH 112/231] Retain images --- .github/workflows/retain-images.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/retain-images.yaml diff --git a/.github/workflows/retain-images.yaml b/.github/workflows/retain-images.yaml new file mode 100644 index 00000000..ac029fe8 --- /dev/null +++ b/.github/workflows/retain-images.yaml @@ -0,0 +1,21 @@ +name: Retain images + +on: + schedule: + - cron: "0 0 1 * *" + +jobs: + pull_image: + runs-on: ubuntu-latest + name: Pull images + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: docker pull + run: | + for tag in $(git tag); do + docker pull --quiet danielflook/terraform-github-actions:$tag + done From 4a0228b4294829cc8dd2de0a1aaaa1ee107ebe63 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 18 Sep 2021 14:17:45 +0100 Subject: [PATCH 113/231] Add failure-reason to terraform-validate --- .github/workflows/test-validate.yaml | 13 ++ image/entrypoints/validate.sh | 7 +- image/tools/convert_validate_report.py | 14 +- terraform-validate/README.md | 32 +++ tests/validate/test_validate.py | 260 +++++++++++++++++-------- 5 files changed, 244 insertions(+), 82 deletions(-) diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 73f83973..38b47ff7 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -12,9 +12,17 @@ jobs: - name: validate uses: ./terraform-validate + id: validate with: path: tests/validate/valid + - name: Check valid + run: | + if [[ "${{ steps.validate.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + invalid: runs-on: ubuntu-latest name: Invalid terraform configuration @@ -35,3 +43,8 @@ jobs: echo "Validate did not fail correctly" exit 1 fi + + if [[ "${{ steps.validate.outputs.failure-reason }}" != "validate-failed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 1239622c..9fa9f04d 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -4,7 +4,12 @@ source /usr/local/actions.sh debug setup -init + +# You can't properly validate without initializing. +# You can't initialize without having valid terraform. +# How do you get a full validation report? You can't. + +init || true if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH"); then (cd "$INPUT_PATH" && terraform validate) diff --git a/image/tools/convert_validate_report.py b/image/tools/convert_validate_report.py index 13bf368d..546a3a36 100755 --- a/image/tools/convert_validate_report.py +++ b/image/tools/convert_validate_report.py @@ -19,9 +19,17 @@ def convert_to_github(report: Dict, base_path: str) -> Iterable[str]: if 'start' in diag['range']: params['line'] = diag['range']['start']['line'] params['col'] = diag['range']['start']['column'] + if 'end' in diag['range']: + params['endLine'] = diag['range']['end']['line'] + params['endColumn'] = diag['range']['end']['column'] summary = diag['summary'].split('\n')[0] params = ','.join(f'{k}={v}' for k, v in params.items()) + + if summary == 'Module not installed': + # This is most likely because other errors prevented init from running properly, and not an error in itself. + continue + yield f"::{diag['severity']} {params}::{summary}" @@ -39,4 +47,8 @@ def convert_to_github(report: Dict, base_path: str) -> Iterable[str]: for line in convert_to_github(report, sys.argv[1]): print(line) - exit(0 if report.get('valid', False) is True else 1) + if report.get('valid', False) is True: + exit(0) + else: + print('::set-output name=failure-reason::validate-failed') + exit(1) diff --git a/terraform-validate/README.md b/terraform-validate/README.md index e633f978..1cd60630 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -23,6 +23,14 @@ If the terraform configuration is not valid, the build is failed. - Optional - Default: The action workspace +## Outputs + +* `failure-reason` + + When the job outcome is `failure` because the validation failed, this will be set to 'validate-failed'. + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run a step when the validate fails. + ## Environment Variables * `TERRAFORM_CLOUD_TOKENS` @@ -130,3 +138,27 @@ jobs: with: path: my-terraform-config ``` + +This example executes a run step only if the validation failed. + +```yaml +on: [push] + +jobs: + validate: + runs-on: ubuntu-latest + name: Validate terraform + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform validate + uses: dflook/terraform-validate@v1 + id: validate + with: + path: my-terraform-config + + - name: Validate failed + if: ${{ failure() && steps.validate.outputs.failure-reason == 'validate-failed' }} + run: echo "terraform validate failed" +``` diff --git a/tests/validate/test_validate.py b/tests/validate/test_validate.py index 8e1625b9..64647135 100644 --- a/tests/validate/test_validate.py +++ b/tests/validate/test_validate.py @@ -1,32 +1,34 @@ from convert_validate_report import convert_to_github + def test_valid(): input = { - "valid": True, - "error_count": 0, - "warning_count": 0, - "diagnostics": [] + "valid": True, + "error_count": 0, + "warning_count": 0, + "diagnostics": [] } output = list(convert_to_github(input, 'terraform')) assert output == [] + def test_invalid(): input = { - "valid": False, - "error_count": 2, - "warning_count": 0, - "diagnostics": [ - { - "severity": "error", - "summary": "Could not satisfy plugin requirements", - "detail": "\nPlugin reinitialization required. Please run \"terraform init\".\n\nPlugins are external binaries that Terraform uses to access and manipulate\nresources. The configuration provided requires plugins which can't be located,\ndon't satisfy the version constraints, or are otherwise incompatible.\n\nTerraform automatically discovers providers requirements from your\nconfiguration, including plugin-cache used in child modules. To see the\nrequirements and constraints from each module, run \"terraform plugin-cache\".\n" - }, - { - "severity": "error", - "summary": "providers.null: no suitable version installed\n version requirements: \"(any version)\"\n versions installed: none" - } - ] + "valid": False, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Could not satisfy plugin requirements", + "detail": "\nPlugin reinitialization required. Please run \"terraform init\".\n\nPlugins are external binaries that Terraform uses to access and manipulate\nresources. The configuration provided requires plugins which can't be located,\ndon't satisfy the version constraints, or are otherwise incompatible.\n\nTerraform automatically discovers providers requirements from your\nconfiguration, including plugin-cache used in child modules. To see the\nrequirements and constraints from each module, run \"terraform plugin-cache\".\n" + }, + { + "severity": "error", + "summary": "providers.null: no suitable version installed\n version requirements: \"(any version)\"\n versions installed: none" + } + ] } expected_output = [ @@ -37,35 +39,36 @@ def test_invalid(): output = list(convert_to_github(input, 'terraform')) assert output == expected_output + def test_blah(): input = { - "valid": False, - "error_count": 1, - "warning_count": 0, - "diagnostics": [ - { - "severity": "error", - "summary": "Duplicate resource \"null_resource\" configuration", - "detail": "A null_resource resource named \"hello\" was already declared at main.tf:1,1-33. Resource names must be unique per type in each module.", - "range": { - "filename": "main.tf", - "start": { - "line": 2, - "column": 1, - "byte": 36 - }, - "end": { - "line": 2, - "column": 33, - "byte": 68 + "valid": False, + "error_count": 1, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate resource \"null_resource\" configuration", + "detail": "A null_resource resource named \"hello\" was already declared at main.tf:1,1-33. Resource names must be unique per type in each module.", + "range": { + "filename": "main.tf", + "start": { + "line": 2, + "column": 1, + "byte": 36 + }, + "end": { + "line": 2, + "column": 33, + "byte": 68 + } + } } - } - } - ] + ] } expected_output = [ - '::error file=terraform/main.tf,line=2,col=1::Duplicate resource "null_resource" configuration' + '::error file=terraform/main.tf,line=2,col=1,endLine=2,endColumn=33::Duplicate resource "null_resource" configuration' ] output = list(convert_to_github(input, 'terraform')) @@ -74,52 +77,149 @@ def test_blah(): def test_invalid_paths(): input = { - "valid": False, - "error_count": 2, - "warning_count": 0, - "diagnostics": [ - { - "severity": "error", - "summary": "Duplicate resource \"null_resource\" configuration", - "detail": "A null_resource resource named \"hello\" was already declared at main.tf:1,1-33. Resource names must be unique per type in each module.", - "range": { - "filename": "main.tf", - "start": { - "line": 2, - "column": 1, - "byte": 36 + "valid": False, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate resource \"null_resource\" configuration", + "detail": "A null_resource resource named \"hello\" was already declared at main.tf:1,1-33. Resource names must be unique per type in each module.", + "range": { + "filename": "main.tf", + "start": { + "line": 2, + "column": 1, + "byte": 36 + }, + "end": { + "line": 2, + "column": 33, + "byte": 68 + } + } }, - "end": { - "line": 2, - "column": 33, - "byte": 68 + { + "severity": "error", + "summary": "Duplicate resource \"null_resource\" configuration", + "detail": "A null_resource resource named \"goodbye\" was already declared at ../module/invalid.tf:1,1-33. Resource names must be unique per type in each module.", + "range": { + "filename": "../module/invalid.tf", + "start": { + "line": 2, + "column": 1, + "byte": 36 + }, + "end": { + "line": 5, + "column": 66, + "byte": 68 + } + } } - } - }, - { - "severity": "error", - "summary": "Duplicate resource \"null_resource\" configuration", - "detail": "A null_resource resource named \"goodbye\" was already declared at ../module/invalid.tf:1,1-33. Resource names must be unique per type in each module.", - "range": { - "filename": "../module/invalid.tf", - "start": { - "line": 2, - "column": 1, - "byte": 36 + ] + } + + expected_output = [ + '::error file=tests/validate/invalid/main.tf,line=2,col=1,endLine=2,endColumn=33::Duplicate resource "null_resource" configuration', + '::error file=tests/validate/module/invalid.tf,line=2,col=1,endLine=5,endColumn=66::Duplicate resource "null_resource" configuration' + ] + + output = list(convert_to_github(input, 'tests/validate/invalid')) + assert output == expected_output + + +def test_json_0_1(): + input = { + "format_version": "0.1", + "valid": False, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate resource \"null_resource\" configuration", + "detail": "A null_resource resource named \"hello\" was already declared at main.tf:1,1-33. Resource names must be unique per type in each module.", + "range": { + "filename": "main.tf", + "start": { + "line": 2, + "column": 1, + "byte": 36 + }, + "end": { + "line": 2, + "column": 33, + "byte": 68 + } + }, + "snippet": { + "context": None, + "code": "resource \"null_resource\" \"hello\" {}", + "start_line": 2, + "highlight_start_offset": 0, + "highlight_end_offset": 32, + "values": [] + } + }, + { + "severity": "error", + "summary": "Duplicate resource \"null_resource\" configuration", + "detail": "A null_resource resource named \"goodbye\" was already declared at main.tf:5,1-33. Resource names must be unique per type in each module.", + "range": { + "filename": "main.tf", + "start": { + "line": 6, + "column": 1, + "byte": 110 + }, + "end": { + "line": 6, + "column": 33, + "byte": 142 + } + }, + "snippet": { + "context": None, + "code": "resource \"null_resource\" goodbye {}", + "start_line": 6, + "highlight_start_offset": 0, + "highlight_end_offset": 32, + "values": [] + } }, - "end": { - "line": 2, - "column": 33, - "byte": 68 + { + "severity": "error", + "summary": "Module not installed", + "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + "range": { + "filename": "main.tf", + "start": { + "line": 11, + "column": 1, + "byte": 201 + }, + "end": { + "line": 11, + "column": 13, + "byte": 213 + } + }, + "snippet": { + "context": None, + "code": "module \"vpc\" {", + "start_line": 11, + "highlight_start_offset": 0, + "highlight_end_offset": 12, + "values": [] + } } - } - } - ] + ] } expected_output = [ - '::error file=tests/validate/invalid/main.tf,line=2,col=1::Duplicate resource "null_resource" configuration', - '::error file=tests/validate/module/invalid.tf,line=2,col=1::Duplicate resource "null_resource" configuration' + '::error file=tests/validate/invalid/main.tf,line=2,col=1,endLine=2,endColumn=33::Duplicate resource "null_resource" configuration', + '::error file=tests/validate/invalid/main.tf,line=6,col=1,endLine=6,endColumn=33::Duplicate resource "null_resource" configuration' ] output = list(convert_to_github(input, 'tests/validate/invalid')) From 934b1f47e36df3f4bff6b00d4efac838ae35ef02 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 19 Sep 2021 10:29:24 +0100 Subject: [PATCH 114/231] Add failure-reason to fmt-check --- .github/workflows/test-fmt-check.yaml | 13 ++++++++++ image/entrypoints/fmt-check.sh | 5 ++++ terraform-fmt-check/README.md | 36 ++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-fmt-check.yaml b/.github/workflows/test-fmt-check.yaml index 10e5cd84..b504c687 100644 --- a/.github/workflows/test-fmt-check.yaml +++ b/.github/workflows/test-fmt-check.yaml @@ -12,9 +12,17 @@ jobs: - name: fmt-check uses: ./terraform-fmt-check + id: fmt-check with: path: tests/fmt/canonical + - name: Check valid + run: | + if [[ "${{ steps.fmt-check.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + non_canonical_fmt: runs-on: ubuntu-latest name: Non canonical fmt @@ -36,3 +44,8 @@ jobs: echo "fmt-check did not fail correctly" exit 1 fi + + if [[ "${{ steps.fmt-check.outputs.failure-reason }}" != "check-failed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index 1fd680f8..3848e642 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -9,6 +9,11 @@ terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read echo "$line" if [[ -f "$line" ]]; then + if [[ -z "$FAILURE_REASON_SET" ]]; then + FAILURE_REASON_SET=yes + set_output failure-reason check-failed + fi + echo "::error file=$line::File is not in canonical format (terraform fmt)" fi done diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index 4a1d8eb8..bc1b9ae8 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -17,13 +17,21 @@ If any files are not correctly formatted a failing GitHub check will be added fo - Optional - Default: The action workspace +## Outputs + +* `failure-reason` + + When the job outcome is `failure` because the format check failed, this will be set to 'check-failed'. + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run a step when the format check fails. + ## Example usage This example workflow runs on every push and fails if any of the terraform files are not formatted correctly. ```yaml -name: Check terraform file format +name: Check terraform file formatting on: [push] @@ -40,3 +48,29 @@ jobs: with: path: my-terraform-config ``` + +This example executes a run step only if the format check failed. + +```yaml +name: Check terraform file formatting + +on: [push] + +jobs: + check_format: + runs-on: ubuntu-latest + name: Check terraform file are formatted correctly + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform fmt + uses: dflook/terraform-fmt-check@v1 + id: fmt-check + with: + path: my-terraform-config + + - name: Validate failed + if: ${{ failure() && steps.fmt-check.outputs.failure-reason == 'check-failed' }} + run: echo "terraform formatting check failed" +``` From feed5e3d8a0e0ac241d0abacc466afe72b48f6b9 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 19 Sep 2021 10:43:07 +0100 Subject: [PATCH 115/231] Add failure-reason to terraform-check --- .github/workflows/test-check.yaml | 15 ++++++++++++- image/entrypoints/check.sh | 1 + terraform-check/README.md | 36 +++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index b3e4339c..04758d82 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -12,9 +12,17 @@ jobs: - name: Check uses: ./terraform-check + id: check with: path: tests/plan/no_changes + - name: Check failure-reason + run: | + if [[ "${{ steps.check.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + plan_change_comment: runs-on: ubuntu-latest name: Changes @@ -29,9 +37,14 @@ jobs: with: path: tests/plan/plan - - name: Check invalid + - name: Check failure-reason run: | if [[ "${{ steps.check.outcome }}" != "failure" ]]; then echo "Check did not fail correctly" exit 1 fi + + if [[ "${{ steps.check.outputs.failure-reason }}" != "changes-to-apply" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index 86b2935e..029382c4 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -22,6 +22,7 @@ if [[ $TF_EXIT -eq 1 ]]; then elif [[ $TF_EXIT -eq 2 ]]; then echo "Changes detected!" + set_output failure-reason changes-to-apply exit 1 fi diff --git a/terraform-check/README.md b/terraform-check/README.md index 4a664a53..467f8822 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -97,6 +97,14 @@ This is intended to run on a schedule to notify if manual changes to your infras - Optional - Default: 10 +## Outputs + +* `failure-reason` + + When the job outcome is `failure` because the there are outstanding changes to apply, this will be set to 'changes-to-apply'. + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run a step when there are changes to apply. + ## Environment Variables * `TERRAFORM_CLOUD_TOKENS` @@ -208,3 +216,31 @@ jobs: with: path: my-terraform-configuration ``` + +This example executes a run step only if there are changes to apply. + +```yaml +name: Check for infrastructure drift + +on: + schedule: + - cron: "0 8 * * *" + +jobs: + check_drift: + runs-on: ubuntu-latest + name: Check for drift of terraform configuration + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Check + uses: dflook/terraform-check@v1 + id: check + with: + path: my-terraform-configuration + + - name: Changes detected + if: ${{ failure() && steps.check.outputs.failure-reason == 'changes-to-apply' }} + run: echo "There are outstanding terraform changes to apply" +``` From 8947f43dbd2bcceb445d73f540110cb43124c60c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 19 Sep 2021 11:03:07 +0100 Subject: [PATCH 116/231] Add apply-failed failure-reason for terraform-apply --- .github/workflows/test-apply.yaml | 47 +++++++++++++++++++++++++++++++ image/entrypoints/apply.sh | 1 + tests/apply/apply-error/main.tf | 10 +++++++ 3 files changed, 58 insertions(+) create mode 100644 tests/apply/apply-error/main.tf diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 59582957..217b167e 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -49,6 +49,48 @@ jobs: exit 1 fi + if [[ "${{ steps.apply.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + + apply_apply_error: + runs-on: ubuntu-latest + name: Auto Approve apply phase error + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan + uses: ./terraform-plan + with: + label: Apply Error + path: tests/apply/apply-error + + - name: Apply + uses: ./terraform-apply + id: apply + continue-on-error: true + with: + label: Apply Error + path: tests/apply/apply-error + + - name: Check failed to apply + run: | + if [[ "${{ steps.apply.outcome }}" != "failure" ]]; then + echo "Apply did not fail correctly" + exit 1 + fi + + if [[ "${{ steps.apply.outputs.failure-reason }}" != "apply-failed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + apply_no_token: runs-on: ubuntu-latest name: Apply without token @@ -70,6 +112,11 @@ jobs: exit 1 fi + if [[ "${{ steps.apply.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + apply: runs-on: ubuntu-latest name: Apply approved changes diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 8f872065..facae46f 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -54,6 +54,7 @@ function apply() { if [[ $APPLY_EXIT -eq 0 ]]; then update_status "Plan applied in $(job_markdown_ref)" else + set_output failure-reason apply-failed update_status "Error applying plan in $(job_markdown_ref)" exit 1 fi diff --git a/tests/apply/apply-error/main.tf b/tests/apply/apply-error/main.tf new file mode 100644 index 00000000..dd1b9b25 --- /dev/null +++ b/tests/apply/apply-error/main.tf @@ -0,0 +1,10 @@ +# This should be valid, and generate a valid plan +# But we expect it to fail when applied, as the bucket surely already exists + +resource "aws_s3_bucket" "my_bucket" { + bucket = "hello" +} + +provider aws { + region = "eu-west-2" +} From 2203979a23a5fdc5f85fe985e1922e599e530dc7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 19 Sep 2021 20:13:52 +0100 Subject: [PATCH 117/231] Add plan-changed failure-reason for terraform-apply --- .github/workflows/test-changes-only.yaml | 26 ++++++++ image/entrypoints/apply.sh | 3 + terraform-apply/README.md | 75 +++++++++++++++++++----- 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-changes-only.yaml b/.github/workflows/test-changes-only.yaml index add5ee9e..a8a2e5f3 100644 --- a/.github/workflows/test-changes-only.yaml +++ b/.github/workflows/test-changes-only.yaml @@ -31,10 +31,18 @@ jobs: - name: Apply without changes uses: ./terraform-apply + id: apply with: label: no_changes path: tests/plan/changes-only + - name: Check failure-reason + run: | + if [[ "${{ steps.apply.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + change_then_no_changes: runs-on: ubuntu-latest name: changes-only should still replace a change comment @@ -84,12 +92,20 @@ jobs: - name: Apply no changes uses: ./terraform-apply + id: apply with: label: change_then_no_changes path: tests/plan/changes-only variables: | cause-changes=false + - name: Check failure-reason + run: | + if [[ "${{ steps.apply.outputs.failure-reason }}" != "" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + no_changes_then_changes: runs-on: ubuntu-latest name: Apply with changes should fail after a changes-only plan with no changes @@ -135,6 +151,11 @@ jobs: exit 1 fi + if [[ "${{ steps.apply.outputs.failure-reason }}" != "plan-changed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi + apply_when_plan_has_changed: runs-on: ubuntu-latest name: Apply should fail if the approved plan has changed @@ -169,3 +190,8 @@ jobs: echo "Apply did not fail correctly" exit 1 fi + + if [[ "${{ steps.apply.outputs.failure-reason }}" != "plan-changed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index facae46f..1583fff4 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -109,6 +109,8 @@ else echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" + + set_output failure-reason plan-changed exit 1 fi @@ -123,6 +125,7 @@ else debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true + set_output failure-reason plan-changed exit 1 fi fi diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 57517a63..146edff9 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -181,6 +181,32 @@ These input values must be the same as any `terraform-plan` for the same configu - Optional - Default: false +## Outputs + +* Terraform Outputs + + An action output will be created for each output of the terraform configuration. + + For example, with the terraform config: + ```hcl + output "service_hostname" { + value = "example.com" + } + ``` + + Running this action will produce a `service_hostname` output with the same value. + See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/master/terraform-output) for details. + +* `failure-reason` + + When the job outcome is `failure`, this output may be set. The value may be one of: + + - `apply-failed` - The terraform apply operation failed. + - `plan-changed` - The approved plan is no longer accurate, so the apply will not be attempted. + + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run steps. + ## Environment Variables * `GITHUB_TOKEN` @@ -281,20 +307,6 @@ These input values must be the same as any `terraform-plan` for the same configu - Type: string - Optional -## Outputs - -An action output will be created for each output of the terraform configuration. - -For example, with the terraform config: -```hcl -output "service_hostname" { - value = "example.com" -} -``` - -Running this action will produce a `service_hostname` output with the same value. -See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/master/terraform-output) for details. - ## Example usage ### Apply PR approved plans @@ -414,3 +426,38 @@ jobs: with: path: my-terraform-config ``` + +This example retries the terraform apply operation if it fails. + +```yaml +name: Apply plan + +on: + push: + branches: + - master + +jobs: + plan: + runs-on: ubuntu-latest + name: Apply terraform plan + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform apply + uses: dflook/terraform-apply@v1 + continue-on-error: true + id: first_try + with: + path: terraform + + - name: Retry failed apply + uses: dflook/terraform-apply@1 + if: ${{ steps.first_try.outputs.failure-reason == 'apply-failed' }} + with: + path: terraform + auto_approve: true +``` From f39b4ee2ce14b83297cba1c376647184cfa4e32d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 19 Sep 2021 20:45:14 +0100 Subject: [PATCH 118/231] Add destroy-failed failure-reason to terraform-destroy & terraform-destroy-workspace --- image/entrypoints/destroy-workspace.sh | 9 +++--- image/entrypoints/destroy.sh | 5 +++- terraform-destroy-workspace/README.md | 41 ++++++++++++++++++++++++++ terraform-destroy/README.md | 41 ++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index c251daa7..1ba37451 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -8,13 +8,14 @@ init-backend select-workspace set-plan-args -(cd "$INPUT_PATH" \ - && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) +if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then + set_output failure-reason destroy-failed + exit 1 +fi # We can't delete an active workspace, so re-initialize with a 'default' workspace (which may not exist) workspace=$INPUT_WORKSPACE INPUT_WORKSPACE=default init-backend -(cd "$INPUT_PATH" \ - && terraform workspace delete -no-color -lock-timeout=300s "$workspace") +(cd "$INPUT_PATH" && terraform workspace delete -no-color -lock-timeout=300s "$workspace") diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index c8c2605e..63c9e151 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -8,4 +8,7 @@ init-backend select-workspace set-plan-args -(cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS) +if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then + set_output failure-reason destroy-failed + exit 1 +fi diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index ea273c16..241b7ccd 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -97,6 +97,14 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Optional - Default: 10 +## Outputs + +* `failure-reason` + + When the job outcome is `failure` because the terraform destroy operation failed, this is set to `destroy-failed`. + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run a step when the destroy fails. + ## Environment Variables * `TERRAFORM_CLOUD_TOKENS` @@ -208,3 +216,36 @@ jobs: path: terraform workspace: ${{ github.head_ref }} ``` + +This example retries the terraform destroy operation if it fails. + +```yaml +name: Cleanup + +on: + pull_request: + types: [closed] + +jobs: + destroy_workspace: + runs-on: ubuntu-latest + name: Destroy terraform workspace + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform destroy + uses: dflook/terraform-destroy-workspace@v1 + id: first_try + continue-on-error: true + with: + path: my-terraform-config + workspace: ${{ github.head_ref }} + + - name: Retry failed destroy + uses: dflook/terraform-destroy-workspace@v1 + if: ${{ steps.first_try.outputs.failure-reason == 'destroy-failed' }} + with: + path: my-terraform-config + workspace: ${{ github.head_ref }} +``` diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 0fd0e28b..9741a4b9 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -94,6 +94,14 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Optional - Default: 10 +## Outputs + +* `failure-reason` + + When the job outcome is `failure` because the terraform destroy operation failed, this is set to `destroy-failed`. + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run a step when the destroy fails. + ## Environment Variables * `TERRAFORM_CLOUD_TOKENS` @@ -205,3 +213,36 @@ jobs: path: my-terraform-config workspace: ${{ github.head_ref }} ``` + +This example retries the terraform destroy operation if it fails. + +```yaml +name: Cleanup + +on: + pull_request: + types: [closed] + +jobs: + destroy_workspace: + runs-on: ubuntu-latest + name: Destroy terraform workspace + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: terraform destroy + uses: dflook/terraform-destroy@v1 + id: first_try + continue-on-error: true + with: + path: my-terraform-config + workspace: ${{ github.head_ref }} + + - name: Retry failed destroy + uses: dflook/terraform-destroy@v1 + if: ${{ steps.first_try.outputs.failure-reason == 'destroy-failed' }} + with: + path: my-terraform-config + workspace: ${{ github.head_ref }} +``` From 4a7a263b8ca792389828f064c4c9ac1788cb3966 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 08:49:17 +0100 Subject: [PATCH 119/231] Add failure-reason to changelog --- CHANGELOG.md | 15 ++++++++++++++- terraform-apply/README.md | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64a43565..be6ba8e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,23 @@ When using an action you can specify the version as: - `@v1.14` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Added +- Actions that intentionally cause a build failure now set a `failure-reason` output to enable safely responding to those failures. + + Possible failure reasons are: + - [dflook/terraform-validate](terraform-validate): validate-failed + - [dflook/terraform-fmt-check](terraform-fmt-check): check-failed + - [dflook/terraform-check](terraform-check): changes-to-apply + - [dflook/terraform-apply](terraform-apply): apply-failed, plan-changed + - [dflook/terraform-destroy](terraform-destroy): destroy-failed + - [dflook/terraform-destroy-workspace](terraform-destroy-workspace): destroy-failed + ## [1.14.0] - 2021-09-15 ### Added -- Support for self-hosted GitHub Enterprise deployments. Thanks [f0rkz](https://github.com/f0rkz)! +- Support for self-hosted GitHub Enterprise deployments. Thanks [f0rkz](https://github.com/f0rkz)! ### Changed - The `path` input variable is now optional, defaulting to the Action workspace. diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 146edff9..da8374f6 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -455,7 +455,7 @@ jobs: path: terraform - name: Retry failed apply - uses: dflook/terraform-apply@1 + uses: dflook/terraform-apply@v1 if: ${{ steps.first_try.outputs.failure-reason == 'apply-failed' }} with: path: terraform From 5f0956bf16871afdb66ecbe30a1a1ecd423698c5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 09:47:34 +0100 Subject: [PATCH 120/231] Update base image to bullseye --- image/Dockerfile-base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/Dockerfile-base b/image/Dockerfile-base index c0383f5b..1f7e7bb3 100644 --- a/image/Dockerfile-base +++ b/image/Dockerfile-base @@ -3,7 +3,7 @@ FROM golang:1.12.6 AS tfmask RUN git clone https://github.com/cloudposse/tfmask.git RUN cd tfmask && make && make go/build -FROM debian:buster-slim as base +FROM debian:bullseye-slim as base ARG DEFAULT_TF_VERSION=0.14.4 ARG TFSWITCH_VERSION=0.8.832 From 65d0fcfec938901464218d080936ab520418a93e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 09:55:37 +0100 Subject: [PATCH 121/231] Reword --- terraform-fmt-check/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index bc1b9ae8..95cc2565 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -70,7 +70,7 @@ jobs: with: path: my-terraform-config - - name: Validate failed + - name: Wrong formatting found if: ${{ failure() && steps.fmt-check.outputs.failure-reason == 'check-failed' }} run: echo "terraform formatting check failed" ``` From 2e109c2bbad55ff2cbb9150679df07c1e29ed813 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 13:37:13 +0100 Subject: [PATCH 122/231] :bookmark: v1.15.0 --- CHANGELOG.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be6ba8e3..87637556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,22 +8,26 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.14.0` to use an exact release -- `@v1.14` to use the latest patch release for the specific minor version +- `@v1.15.0` to use an exact release +- `@v1.15` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.15.0] - 2021-09-20 ### Added - Actions that intentionally cause a build failure now set a `failure-reason` output to enable safely responding to those failures. Possible failure reasons are: - - [dflook/terraform-validate](terraform-validate): validate-failed - - [dflook/terraform-fmt-check](terraform-fmt-check): check-failed - - [dflook/terraform-check](terraform-check): changes-to-apply - - [dflook/terraform-apply](terraform-apply): apply-failed, plan-changed - - [dflook/terraform-destroy](terraform-destroy): destroy-failed - - [dflook/terraform-destroy-workspace](terraform-destroy-workspace): destroy-failed + - [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate#outputs): validate-failed + - [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check#outputs): check-failed + - [dflook/terraform-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-check#outputs): changes-to-apply + - [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply#outputs): apply-failed, plan-changed + - [dflook/terraform-destroy](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy#outputs): destroy-failed + - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace#outputs): destroy-failed + +### Fixed +- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) was + sometimes unable to create detailed check failures. ## [1.14.0] - 2021-09-15 @@ -232,6 +236,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.15.0]: https://github.com/dflook/terraform-github-actions/compare/v1.14.0...v1.15.0 [1.14.0]: https://github.com/dflook/terraform-github-actions/compare/v1.13.0...v1.14.0 [1.13.0]: https://github.com/dflook/terraform-github-actions/compare/v1.12.0...v1.13.0 [1.12.0]: https://github.com/dflook/terraform-github-actions/compare/v1.11.0...v1.12.0 From 04afb34bd7c103b8a55b26780350a6a5473876ed Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 20:57:08 +0100 Subject: [PATCH 123/231] :books: Add docs for GITHUB_TOKEN permissions --- terraform-apply/README.md | 7 +++++++ terraform-plan/README.md | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index da8374f6..cd1706d7 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -220,6 +220,13 @@ These input values must be the same as any `terraform-plan` for the same configu GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` + The token provided by GitHub Actions will work with the default permissions. + The minimum permissions are `pull-requests: write`. + It will also likely need `contents: read` so the job can checkout the repo. + + You can also use a Personal Access Token which has the `repo` scope. + This must belong to the same user as the token used by the terraform-plan action + - Type: string - Optional diff --git a/terraform-plan/README.md b/terraform-plan/README.md index d958af77..d62bb8de 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -163,6 +163,13 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` + The token provided by GitHub Actions will work with the default permissions. + The minimum permissions are `pull-requests: write`. + It will also likely need `contents: read` so the job can checkout the repo. + + You can also use a Personal Access Token which has the `repo` scope. + The GitHub user that owns the PAT will be the PR comment author. + - Type: string - Optional From 4a0c143a9b3d0ce2c4eec7b35afa97551269fad6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 20 Sep 2021 21:07:29 +0100 Subject: [PATCH 124/231] :books: Update base image to bullseye --- terraform-apply/README.md | 2 +- terraform-check/README.md | 2 +- terraform-destroy-workspace/README.md | 2 +- terraform-destroy/README.md | 4 ++-- terraform-new-workspace/README.md | 2 +- terraform-output/README.md | 2 +- terraform-plan/README.md | 2 +- terraform-validate/README.md | 2 +- terraform-version/README.md | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index cd1706d7..3694f71b 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -273,7 +273,7 @@ These input values must be the same as any `terraform-plan` for the same configu The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-check/README.md b/terraform-check/README.md index 467f8822..a49feee0 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -150,7 +150,7 @@ This is intended to run on a schedule to notify if manual changes to your infras The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 241b7ccd..6e3f57df 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -150,7 +150,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index 9741a4b9..ff39c77d 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -146,8 +146,8 @@ This action uses the `terraform destroy` command to destroy all resources in a t A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - - The runtime image is currently based on `debian:buster` + + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index b402ecf8..235dbc4f 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -91,7 +91,7 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-output/README.md b/terraform-output/README.md index 60489d7f..3c0df711 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -92,7 +92,7 @@ Retrieve the root-level outputs from a terraform configuration. The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-plan/README.md b/terraform-plan/README.md index d62bb8de..b8bce65a 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -257,7 +257,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster`, with the command run using `bash -xeo pipefail`. + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 1cd60630..904258ab 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -76,7 +76,7 @@ If the terraform configuration is not valid, the build is failed. The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml diff --git a/terraform-version/README.md b/terraform-version/README.md index a3d2b31c..235831eb 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -73,7 +73,7 @@ outputs yourself. The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. - The runtime image is currently based on `debian:buster` + The runtime image is currently based on `debian:bullseye`, with the command run using `bash -xeo pipefail`. For example: ```yaml From 0fdaa41cf4b516e12fd9cfb764de0a2117544f56 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 25 Sep 2021 14:26:04 +0100 Subject: [PATCH 125/231] Prefer the json output from terraform version --- image/actions.sh | 7 ++++++ image/entrypoints/version.sh | 2 +- image/tools/convert_version.py | 43 ++++++++++++++++++++++++++++++---- tests/version/test_version.py | 25 +++++++++++++++++++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index e07d3153..be0fc280 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -28,6 +28,13 @@ function detect-terraform-version() { fi debug_cmd ls -la "$(which terraform)" + + local TF_VERSION + TF_VERSION=$(terraform --version | grep 'Terraform v' | sed 's/Terraform v//') + + TERRAFORM_VER_MAJOR=`echo $TF_VERSION | cut -d. -f1` + TERRAFORM_VER_MINOR=`echo $TF_VERSION | cut -d. -f2` + TERRAFORM_VER_PATCH=`echo $TF_VERSION | cut -d. -f3` } function job_markdown_ref() { diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index 49795586..e14023f5 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,4 +6,4 @@ debug setup init -(cd "$INPUT_PATH" && terraform version -no-color | convert_version) +(cd "$INPUT_PATH" && terraform version -no-color -json | convert_version) diff --git a/image/tools/convert_version.py b/image/tools/convert_version.py index 763175e0..f59d29e8 100755 --- a/image/tools/convert_version.py +++ b/image/tools/convert_version.py @@ -1,8 +1,9 @@ #!/usr/bin/python3 +import json import re import sys -from typing import Iterable +from typing import Iterable, Dict def convert_version(tf_output: str) -> Iterable[str]: @@ -37,10 +38,44 @@ def convert_version(tf_output: str) -> Iterable[str]: yield f'::set-output name={provider_name.strip()}::{provider_version.strip()}' +def convert_version_from_json(tf_output: Dict) -> Iterable[str]: + """ + Convert terraform version JSON output human readable output and GitHub actions output commands + + >>> tf_output = { + "terraform_version": "0.13.7", + "terraform_revision": "", + "provider_selections": { + "registry.terraform.io/hashicorp/random": "2.2.0" + }, + "terraform_outdated": true + } + >>> list(convert_version(tf_output)) + ['Terraform v0.13.7', + '::set-output name=terraform::0.13.7', + '+ provider registry.terraform.io/hashicorp/random v2.2.0', + '::set-output name=random::2.2.0'] + """ + + yield f'Terraform v{tf_output["terraform_version"]}' + yield f'::set-output name=terraform::{tf_output["terraform_version"]}' + + for path, version in tf_output['provider_selections'].items(): + name_match = re.match(r'(.*?)/(.*?)/(.*)', path) + name = name_match.group(3) if name_match else path + + yield f'+ provider {path} v{version}' + yield f'::set-output name={name}::{version}' + + if __name__ == '__main__': tf_output = sys.stdin.read() - print(tf_output) + try: + for line in convert_version_from_json(json.loads(tf_output)): + print(line) + except: + print(tf_output) - for line in convert_version(tf_output): - print(line) + for line in convert_version(tf_output): + print(line) diff --git a/tests/version/test_version.py b/tests/version/test_version.py index e41a7e54..4d657571 100644 --- a/tests/version/test_version.py +++ b/tests/version/test_version.py @@ -1,4 +1,5 @@ -from convert_version import convert_version +from convert_version import convert_version, convert_version_from_json + def test_convert_version(): tf_version_output = 'Terraform v0.12.28' @@ -54,3 +55,25 @@ def test_convert_0_13_providers(): ] assert list(convert_version(tf_version_output)) == expected + +def test_convert_0_13_json_providers(): + tf_version_output = { + "terraform_version": "0.13.0", + "terraform_revision": "", + "provider_selections": { + "registry.terraform.io/hashicorp/random": "2.2.0", + "registry.terraform.io/terraform-providers/acme": "2.5.3" + }, + "terraform_outdated": True + } + + expected = [ + 'Terraform v0.13.0', + '::set-output name=terraform::0.13.0', + '+ provider registry.terraform.io/hashicorp/random v2.2.0', + '::set-output name=random::2.2.0', + '+ provider registry.terraform.io/terraform-providers/acme v2.5.3', + '::set-output name=acme::2.5.3' + ] + + assert list(convert_version_from_json(tf_version_output)) == expected From 78d27cc1aee4c40d9bcc15dfa084138dd93688d7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 26 Sep 2021 00:49:28 +0100 Subject: [PATCH 126/231] Prefer the json output from terraform version --- image/actions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/image/actions.sh b/image/actions.sh index be0fc280..32baf35f 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -35,6 +35,8 @@ function detect-terraform-version() { TERRAFORM_VER_MAJOR=`echo $TF_VERSION | cut -d. -f1` TERRAFORM_VER_MINOR=`echo $TF_VERSION | cut -d. -f2` TERRAFORM_VER_PATCH=`echo $TF_VERSION | cut -d. -f3` + + debug_log "Terraform version major $TERRAFORM_VER_MAJOR minor $TERRAFORM_VER_MINOR patch $TERRAFORM_VER_PATCH" } function job_markdown_ref() { From 2bb3cef8d19cbab50610f544b3981e414d71dae8 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 27 Sep 2021 20:48:43 +0100 Subject: [PATCH 127/231] Prefer the json output from terraform version --- image/actions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index 32baf35f..1c35f4f1 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -30,7 +30,7 @@ function detect-terraform-version() { debug_cmd ls -la "$(which terraform)" local TF_VERSION - TF_VERSION=$(terraform --version | grep 'Terraform v' | sed 's/Terraform v//') + TF_VERSION=$(terraform version -json | jq -r '.terraform_version' 2>/dev/null || terraform version | grep 'Terraform v' | sed 's/Terraform v//') TERRAFORM_VER_MAJOR=`echo $TF_VERSION | cut -d. -f1` TERRAFORM_VER_MINOR=`echo $TF_VERSION | cut -d. -f2` From e02641d7b4dc37fda2de8deb5bd4b2c533649601 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 28 Sep 2021 08:42:38 +0100 Subject: [PATCH 128/231] Remove -no-color for version It doesn't do anything. --- CHANGELOG.md | 3 +-- image/entrypoints/version.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87637556..aa91b895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,7 @@ When using an action you can specify the version as: - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace#outputs): destroy-failed ### Fixed -- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) was - sometimes unable to create detailed check failures. +- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) was sometimes unable to create detailed check failures. ## [1.14.0] - 2021-09-15 diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index e14023f5..af7efbe7 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -6,4 +6,4 @@ debug setup init -(cd "$INPUT_PATH" && terraform version -no-color -json | convert_version) +(cd "$INPUT_PATH" && terraform version -json | convert_version) From 71d74c7b8f180827572482df4d73f1c852c89cb2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 3 Oct 2021 00:19:17 +0100 Subject: [PATCH 129/231] Add json_plan_path output --- .github/github_sucks.md | 1 + .github/workflows/test-plan.yaml | 151 ++++++++++++ .github/workflows/test-remote.yaml | 13 + image/actions.sh | 314 +++++++++++++------------ image/entrypoints/apply.sh | 50 ++-- image/entrypoints/check.sh | 2 + image/entrypoints/destroy-workspace.sh | 2 + image/entrypoints/destroy.sh | 2 + image/entrypoints/fmt-check.sh | 3 +- image/entrypoints/fmt.sh | 1 + image/entrypoints/new-workspace.sh | 75 +++--- image/entrypoints/output.sh | 1 + image/entrypoints/plan.sh | 115 +++++---- image/entrypoints/remote-state.sh | 5 +- image/entrypoints/validate.sh | 5 +- image/entrypoints/version.sh | 1 + image/workflow_commands.sh | 57 +++-- terraform-plan/README.md | 13 + terraform-plan/action.yaml | 4 +- 19 files changed, 524 insertions(+), 291 deletions(-) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index f941c83c..325766d6 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -3,3 +3,4 @@ This is usually because GitHub Actions has broken in some way. + diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 9ac531b3..6fed2d48 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -27,6 +27,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .format_version "${{ steps.plan.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + no_changes_no_comment: runs-on: ubuntu-latest name: No changes without comment @@ -38,10 +49,24 @@ jobs: - name: Plan uses: ./terraform-plan + id: plan with: path: tests/plan/no_changes add_github_comment: false + - name: Verify outputs + run: | + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .format_version "${{ steps.plan.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_11: runs-on: ubuntu-latest name: Change terraform 11 @@ -66,6 +91,16 @@ jobs: exit 1 fi + if [[ -n "${{ steps.plan.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set - not available with terraform 11" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_12: runs-on: ubuntu-latest name: Change terraform 12 @@ -90,6 +125,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_13: runs-on: ubuntu-latest name: Change terraform 13 @@ -114,6 +160,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_14: runs-on: ubuntu-latest name: Change terraform 14 @@ -139,6 +196,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_15: runs-on: ubuntu-latest name: Change terraform 15 @@ -164,6 +232,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_15_4: runs-on: ubuntu-latest name: Change terraform 15.4 @@ -188,6 +267,17 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + plan_change_comment_latest: runs-on: ubuntu-latest name: Change latest terraform @@ -212,6 +302,23 @@ jobs: exit 1 fi + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + + - name: Checkov + run: | + cat '${{ steps.plan.outputs.json_plan_path }}' + pip3 install checkov + checkov -f '${{ steps.plan.outputs.json_plan_path }}' + plan_change_no_comment: runs-on: ubuntu-latest name: Change without github comment @@ -223,10 +330,24 @@ jobs: - name: Plan uses: ./terraform-plan + id: plan with: path: tests/plan/plan add_github_comment: false + - name: Verify outputs + run: | + cat '${{ steps.plan.outputs.json_plan_path }}' + if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + error: runs-on: ubuntu-latest name: Error @@ -250,6 +371,16 @@ jobs: exit 1 fi + if [[ -n "${{ steps.plan.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi + + if [[ -n "${{ steps.plan.outputs.text_plan_path }}" ]]; then + echo "::error:: text_plan_path should not be set" + exit 1 + fi + error_no_comment: runs-on: ubuntu-latest name: Error without comment @@ -274,6 +405,16 @@ jobs: exit 1 fi + if [[ -n "${{ steps.plan.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi + + if [[ -n "${{ steps.plan.outputs.text_plan_path }}" ]]; then + echo "::error:: text_plan_path should not be set" + exit 1 + fi + plan_without_token: runs-on: ubuntu-latest name: Add comment without token @@ -295,6 +436,16 @@ jobs: exit 1 fi + if [[ -n "${{ steps.plan.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi + + if [[ -n "${{ steps.plan.outputs.text_plan_path }}" ]]; then + echo "::error:: text_plan_path should not be set" + exit 1 + fi + plan_single_variable: runs-on: ubuntu-latest name: Plan single variable diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 9424b4fc..2fdb3884 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -48,6 +48,7 @@ jobs: - name: Plan workspace uses: ./terraform-plan + id: plan env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -55,6 +56,18 @@ jobs: workspace: ${{ github.head_ref }}-2 backend_config: "token=${{ secrets.TF_API_TOKEN }}" + - name: Verify plan outputs + run: | + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.plan.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + - name: Apply workspace uses: ./terraform-apply env: diff --git a/image/actions.sh b/image/actions.sh index 1c35f4f1..357c2318 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -1,237 +1,249 @@ #!/bin/bash -set -eo pipefail +set -euo pipefail +# shellcheck source=../workflow_commands.sh source /usr/local/workflow_commands.sh function debug() { - debug_cmd ls -la /root - debug_cmd pwd - debug_cmd ls -la - debug_cmd ls -la "$HOME" - debug_cmd printenv - debug_file "$GITHUB_EVENT_PATH" - echo + debug_cmd ls -la /root + debug_cmd pwd + debug_cmd ls -la + debug_cmd ls -la "$HOME" + debug_cmd printenv + debug_file "$GITHUB_EVENT_PATH" + echo } function detect-terraform-version() { - local TF_SWITCH_OUTPUT + local TF_SWITCH_OUTPUT - debug_cmd tfswitch --version + debug_cmd tfswitch --version - TF_SWITCH_OUTPUT=$(cd "$INPUT_PATH" && echo "" | tfswitch | grep -e Switched -e Reading | sed 's/^.*Switched/Switched/') - if echo "$TF_SWITCH_OUTPUT" | grep Reading >/dev/null; then - echo "$TF_SWITCH_OUTPUT" - else - echo "Reading latest terraform version" - tfswitch "$(latest_terraform_version)" - fi + TF_SWITCH_OUTPUT=$(cd "$INPUT_PATH" && echo "" | tfswitch | grep -e Switched -e Reading | sed 's/^.*Switched/Switched/') + if echo "$TF_SWITCH_OUTPUT" | grep Reading >/dev/null; then + echo "$TF_SWITCH_OUTPUT" + else + echo "Reading latest terraform version" + tfswitch "$(latest_terraform_version)" + fi - debug_cmd ls -la "$(which terraform)" + debug_cmd ls -la "$(which terraform)" - local TF_VERSION - TF_VERSION=$(terraform version -json | jq -r '.terraform_version' 2>/dev/null || terraform version | grep 'Terraform v' | sed 's/Terraform v//') + local TF_VERSION + TF_VERSION=$(terraform version -json | jq -r '.terraform_version' 2>/dev/null || terraform version | grep 'Terraform v' | sed 's/Terraform v//') - TERRAFORM_VER_MAJOR=`echo $TF_VERSION | cut -d. -f1` - TERRAFORM_VER_MINOR=`echo $TF_VERSION | cut -d. -f2` - TERRAFORM_VER_PATCH=`echo $TF_VERSION | cut -d. -f3` + TERRAFORM_VER_MAJOR=$(echo "$TF_VERSION" | cut -d. -f1) + TERRAFORM_VER_MINOR=$(echo "$TF_VERSION" | cut -d. -f2) + TERRAFORM_VER_PATCH=$(echo "$TF_VERSION" | cut -d. -f3) - debug_log "Terraform version major $TERRAFORM_VER_MAJOR minor $TERRAFORM_VER_MINOR patch $TERRAFORM_VER_PATCH" + debug_log "Terraform version major $TERRAFORM_VER_MAJOR minor $TERRAFORM_VER_MINOR patch $TERRAFORM_VER_PATCH" } function job_markdown_ref() { - echo "[${GITHUB_WORKFLOW} #${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" + echo "[${GITHUB_WORKFLOW} #${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" } function detect-tfmask() { - TFMASK="tfmask" - if ! hash tfmask 2>/dev/null; then - TFMASK="cat" - fi + TFMASK="tfmask" + if ! hash tfmask 2>/dev/null; then + TFMASK="cat" + fi - export TFMASK + export TFMASK } function execute_run_commands() { - if [[ -n $TERRAFORM_PRE_RUN ]]; then - start_group "Executing TERRAFORM_PRE_RUN" + if [[ -v TERRAFORM_PRE_RUN ]]; then + start_group "Executing TERRAFORM_PRE_RUN" - echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" - printf "%s" "$TERRAFORM_PRE_RUN" > /.prerun.sh - disable_workflow_commands - bash -xeo pipefail /.prerun.sh - enable_workflow_commands + echo "Executing init commands specified in 'TERRAFORM_PRE_RUN' environment variable" + printf "%s" "$TERRAFORM_PRE_RUN" >"$STEP_TMP_DIR/TERRAFORM_PRE_RUN.sh" + disable_workflow_commands + bash -xeo pipefail "$STEP_TMP_DIR/TERRAFORM_PRE_RUN.sh" + enable_workflow_commands - end_group - fi + end_group + fi } function setup() { - if [[ "$INPUT_PATH" == "" ]]; then - error_log "input 'path' not set" - exit 1 - fi + if [[ "$INPUT_PATH" == "" ]]; then + error_log "input 'path' not set" + exit 1 + fi - if [[ ! -d "$INPUT_PATH" ]]; then - error_log "Path does not exist: \"$INPUT_PATH\"" - exit 1 - fi + if [[ ! -d "$INPUT_PATH" ]]; then + error_log "Path does not exist: \"$INPUT_PATH\"" + exit 1 + fi - TERRAFORM_BIN_DIR="$HOME/.dflook-terraform-bin-dir" - export TF_DATA_DIR="$HOME/.dflook-terraform-data-dir" - export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" - unset TF_WORKSPACE + local TERRAFORM_BIN_DIR + TERRAFORM_BIN_DIR="$JOB_TMP_DIR/terraform-bin-dir" + # tfswitch guesses the wrong home directory... + start_group "Installing Terraform" + if [[ ! -d $TERRAFORM_BIN_DIR ]]; then + debug_log "Initializing tfswitch with image default version" + mkdir -p "$TERRAFORM_BIN_DIR" + cp --recursive /root/.terraform.versions.default "$TERRAFORM_BIN_DIR" + fi - # tfswitch guesses the wrong home directory... - start_group "Installing Terraform" - if [[ ! -d $TERRAFORM_BIN_DIR ]]; then - debug_log "Initializing tfswitch with image default version" - cp --recursive /root/.terraform.versions.default "$TERRAFORM_BIN_DIR" - fi + ln -s "$TERRAFORM_BIN_DIR" /root/.terraform.versions - ln -s "$TERRAFORM_BIN_DIR" /root/.terraform.versions + debug_cmd ls -lad /root/.terraform.versions + debug_cmd ls -lad "$TERRAFORM_BIN_DIR" + debug_cmd ls -la "$TERRAFORM_BIN_DIR" - debug_cmd ls -lad /root/.terraform.versions - debug_cmd ls -lad "$TERRAFORM_BIN_DIR" - debug_cmd ls -la "$TERRAFORM_BIN_DIR" + export TF_DATA_DIR="$STEP_TMP_DIR/terraform-data-dir" + export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" + mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" - mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" + unset TF_WORKSPACE - detect-terraform-version + detect-terraform-version - debug_cmd ls -la "$TERRAFORM_BIN_DIR" - end_group + debug_cmd ls -la "$TERRAFORM_BIN_DIR" + end_group - detect-tfmask + detect-tfmask - execute_run_commands + execute_run_commands } function relative_to() { - local absbase - local relpath + local absbase + local relpath - absbase="$1" - relpath="$2" - realpath --no-symlinks --canonicalize-missing --relative-to="$absbase" "$relpath" + absbase="$1" + relpath="$2" + realpath --no-symlinks --canonicalize-missing --relative-to="$absbase" "$relpath" } function init() { - start_group "Initializing Terraform" + start_group "Initializing Terraform" - write_credentials + write_credentials - rm -rf "$TF_DATA_DIR" - (cd "$INPUT_PATH" && terraform init -input=false -backend=false) + rm -rf "$TF_DATA_DIR" + (cd "$INPUT_PATH" && terraform init -input=false -backend=false) - end_group + end_group } function init-backend() { - start_group "Initializing Terraform" + start_group "Initializing Terraform" - write_credentials + write_credentials - INIT_ARGS="" + INIT_ARGS="" - if [[ -n "$INPUT_BACKEND_CONFIG_FILE" ]]; then - for file in $(echo "$INPUT_BACKEND_CONFIG_FILE" | tr ',' '\n'); do - INIT_ARGS="$INIT_ARGS -backend-config=$(relative_to "$INPUT_PATH" "$file")" - done - fi + if [[ -n "$INPUT_BACKEND_CONFIG_FILE" ]]; then + for file in $(echo "$INPUT_BACKEND_CONFIG_FILE" | tr ',' '\n'); do + INIT_ARGS="$INIT_ARGS -backend-config=$(relative_to "$INPUT_PATH" "$file")" + done + fi - if [[ -n "$INPUT_BACKEND_CONFIG" ]]; then - for config in $(echo "$INPUT_BACKEND_CONFIG" | tr ',' '\n'); do - INIT_ARGS="$INIT_ARGS -backend-config=$config" - done - fi + if [[ -n "$INPUT_BACKEND_CONFIG" ]]; then + for config in $(echo "$INPUT_BACKEND_CONFIG" | tr ',' '\n'); do + INIT_ARGS="$INIT_ARGS -backend-config=$config" + done + fi - export INIT_ARGS + export INIT_ARGS - rm -rf "$TF_DATA_DIR" + rm -rf "$TF_DATA_DIR" - set +e - (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false $INIT_ARGS \ - 2>"$PLAN_DIR/init_error.txt") + set +e + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false $INIT_ARGS \ + 2>"$STEP_TMP_DIR/terraform_init.stderr") - local INIT_EXIT=$? - set -e + local INIT_EXIT=$? + set -e - if [[ $INIT_EXIT -eq 0 ]]; then - cat "$PLAN_DIR/init_error.txt" >&2 - else - if grep -q "No existing workspaces." "$PLAN_DIR/init_error.txt" || grep -q "Failed to select workspace" "$PLAN_DIR/init_error.txt"; then - # Couldn't select workspace, but we don't really care. - # select-workspace will give a better error if the workspace is required to exist - : + if [[ $INIT_EXIT -eq 0 ]]; then + cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 else - cat "$PLAN_DIR/init_error.txt" >&2 - exit $INIT_EXIT + if grep -q "No existing workspaces." "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Failed to select workspace" "$STEP_TMP_DIR/terraform_init.stderr"; then + # Couldn't select workspace, but we don't really care. + # select-workspace will give a better error if the workspace is required to exist + : + else + cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 + exit $INIT_EXIT + fi fi - fi - - end_group + end_group } function select-workspace() { - (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") >/tmp/select-workspace 2>&1 + (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") >"$STEP_TMP_DIR/workspace_select" 2>&1 - if [[ -s /tmp/select-workspace ]]; then - start_group "Selecting workspace" - cat /tmp/select-workspace - end_group - fi + if [[ -s "$STEP_TMP_DIR/workspace_select" ]]; then + start_group "Selecting workspace" + cat "$STEP_TMP_DIR/workspace_select" + end_group + fi } function set-plan-args() { - PLAN_ARGS="" - - if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then - PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" - fi - - if [[ -n "$INPUT_VAR" ]]; then - for var in $(echo "$INPUT_VAR" | tr ',' '\n'); do - PLAN_ARGS="$PLAN_ARGS -var $var" - done - fi - - if [[ -n "$INPUT_VAR_FILE" ]]; then - for file in $(echo "$INPUT_VAR_FILE" | tr ',' '\n'); do - PLAN_ARGS="$PLAN_ARGS -var-file=$(relative_to "$INPUT_PATH" "$file")" - done - fi - - if [[ -n "$INPUT_VARIABLES" ]]; then - echo "$INPUT_VARIABLES" > /.terraform-variables.tfvars - PLAN_ARGS="$PLAN_ARGS -var-file=/.terraform-variables.tfvars" - fi - - export PLAN_ARGS + PLAN_ARGS="" + + if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then + PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" + fi + + if [[ -n "$INPUT_VAR" ]]; then + for var in $(echo "$INPUT_VAR" | tr ',' '\n'); do + PLAN_ARGS="$PLAN_ARGS -var $var" + done + fi + + if [[ -n "$INPUT_VAR_FILE" ]]; then + for file in $(echo "$INPUT_VAR_FILE" | tr ',' '\n'); do + PLAN_ARGS="$PLAN_ARGS -var-file=$(relative_to "$INPUT_PATH" "$file")" + done + fi + + if [[ -n "$INPUT_VARIABLES" ]]; then + echo "$INPUT_VARIABLES" >"$STEP_TMP_DIR/variables.tfvars" + PLAN_ARGS="$PLAN_ARGS -var-file=$STEP_TMP_DIR/variables.tfvars" + fi + + export PLAN_ARGS } function output() { - (cd "$INPUT_PATH" && terraform output -json | convert_output) + (cd "$INPUT_PATH" && terraform output -json | convert_output) } function update_status() { - local status="$1" + local status="$1" - if ! STATUS="$status" github_pr_comment status 2>/tmp/github_pr_comment.error; then - debug_file /tmp/github_pr_comment.error - fi + if ! STATUS="$status" github_pr_comment status 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + fi } function random_string() { - python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" + python3 -c "import random; import string; print(''.join(random.choice(string.ascii_lowercase) for i in range(8)))" } function write_credentials() { - format_tf_credentials >> "$HOME/.terraformrc" - netrc-credential-actions >> "$HOME/.netrc" - echo "$TERRAFORM_SSH_KEY" >> /.ssh/id_rsa - chmod 600 /.ssh/id_rsa - chmod 700 /.ssh - debug_cmd git config --list + format_tf_credentials >>"$HOME/.terraformrc" + netrc-credential-actions >>"$HOME/.netrc" + + chmod 700 /.ssh + if [[ -v TERRAFORM_SSH_KEY ]]; then + echo "$TERRAFORM_SSH_KEY" >>/.ssh/id_rsa + chmod 600 /.ssh/id_rsa + fi + + debug_cmd git config --list } + +# Every file written to disk should use one of these directories +readonly STEP_TMP_DIR="/tmp" +readonly JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" +readonly WORKSPACE_TMP_DIR=".dflook-terraform-github-actions/$(random_string)" diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 1583fff4..83dc7267 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -8,10 +9,7 @@ init-backend select-workspace set-plan-args -PLAN_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) -rm -rf "$PLAN_DIR" -mkdir -p "$PLAN_DIR" -PLAN_OUT="$PLAN_DIR/plan.out" +PLAN_OUT="$STEP_TMP_DIR/plan.out" if [[ "$INPUT_AUTO_APPROVE" == "true" && -n "$INPUT_TARGET" ]]; then for target in $(echo "$INPUT_TARGET" | tr ',' '\n'); do @@ -19,8 +17,8 @@ if [[ "$INPUT_AUTO_APPROVE" == "true" && -n "$INPUT_TARGET" ]]; then done fi -if [[ -n "$GITHUB_TOKEN" ]]; then - update_status "Applying plan in $(job_markdown_ref)" +if [[ -v GITHUB_TOKEN ]]; then + update_status "Applying plan in $(job_markdown_ref)" fi exec 3>&1 @@ -29,16 +27,19 @@ function plan() { local PLAN_OUT_ARG if [[ -n "$PLAN_OUT" ]]; then - PLAN_OUT_ARG=-out="$PLAN_OUT" + PLAN_OUT_ARG="-out=$PLAN_OUT" + else + PLAN_OUT_ARG="" fi set +e + # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ - 2>"$PLAN_DIR/error.txt" \ + 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ | $TFMASK \ | tee /dev/fd/3 \ | compact_plan \ - >"$PLAN_DIR/plan.txt" + >"$STEP_TMP_DIR/plan.txt" PLAN_EXIT=${PIPESTATUS[0]} set -e @@ -47,6 +48,7 @@ function plan() { function apply() { set +e + # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT) | $TFMASK local APPLY_EXIT=${PIPESTATUS[0]} set -e @@ -65,20 +67,20 @@ function apply() { plan if [[ $PLAN_EXIT -eq 1 ]]; then - if grep -q "Saving a generated plan is currently not supported" "$PLAN_DIR/error.txt"; then + if grep -q "Saving a generated plan is currently not supported" "$STEP_TMP_DIR/terraform_plan.stderr"; then PLAN_OUT="" if [[ "$INPUT_AUTO_APPROVE" == "true" ]]; then - # The apply will have to generate the plan, so skip doing it now - PLAN_EXIT=2 + # The apply will have to generate the plan, so skip doing it now + PLAN_EXIT=2 else - plan + plan fi fi fi if [[ $PLAN_EXIT -eq 1 ]]; then - cat "$PLAN_DIR/error.txt" + cat "$STEP_TMP_DIR/terraform_plan.stderr" update_status "Error applying plan in $(job_markdown_ref)" exit 1 @@ -97,15 +99,15 @@ else exit 1 fi - if [[ -z "$GITHUB_TOKEN" ]]; then - echo "GITHUB_TOKEN environment variable must be set to get plan approval from a PR" - echo "Either set the GITHUB_TOKEN environment variable or automatically approve by setting the auto_approve input to 'true'" - echo "See https://github.com/dflook/terraform-github-actions/ for details." - exit 1 + if [[ ! -v GITHUB_TOKEN ]]; then + echo "GITHUB_TOKEN environment variable must be set to get plan approval from a PR" + echo "Either set the GITHUB_TOKEN environment variable or automatically approve by setting the auto_approve input to 'true'" + echo "See https://github.com/dflook/terraform-github-actions/ for details." + exit 1 fi - if ! github_pr_comment get "$PLAN_DIR/approved-plan.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then - debug_file "$PLAN_DIR/github_pr_comment.error" + if ! github_pr_comment get "$STEP_TMP_DIR/approved-plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" @@ -114,7 +116,7 @@ else exit 1 fi - if plan_cmp "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt"; then + if plan_cmp "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt"; then apply else echo "Not applying the plan - it has changed from the plan on the PR" @@ -122,8 +124,8 @@ else update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" echo "Plan changes:" - debug_log diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" - diff "$PLAN_DIR/plan.txt" "$PLAN_DIR/approved-plan.txt" || true + debug_log diff "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt" + diff "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt" || true set_output failure-reason plan-changed exit 1 diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index 029382c4..ef201e1c 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -9,6 +10,7 @@ select-workspace set-plan-args set +e +# shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform plan -input=false -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ | $TFMASK diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index 1ba37451..594eb612 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -8,6 +9,7 @@ init-backend select-workspace set-plan-args +# shellcheck disable=SC2086 if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then set_output failure-reason destroy-failed exit 1 diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index 63c9e151..57d921db 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -8,6 +9,7 @@ init-backend select-workspace set-plan-args +# shellcheck disable=SC2086 if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then set_output failure-reason destroy-failed exit 1 diff --git a/image/entrypoints/fmt-check.sh b/image/entrypoints/fmt-check.sh index 3848e642..e5ce1fdb 100755 --- a/image/entrypoints/fmt-check.sh +++ b/image/entrypoints/fmt-check.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -9,7 +10,7 @@ terraform fmt -recursive -no-color -check -diff "$INPUT_PATH" | while IFS= read echo "$line" if [[ -f "$line" ]]; then - if [[ -z "$FAILURE_REASON_SET" ]]; then + if [[ ! -v FAILURE_REASON_SET ]]; then FAILURE_REASON_SET=yes set_output failure-reason check-failed fi diff --git a/image/entrypoints/fmt.sh b/image/entrypoints/fmt.sh index 8703805a..7566b7d7 100755 --- a/image/entrypoints/fmt.sh +++ b/image/entrypoints/fmt.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index 31b965fe..13af1ec5 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -1,63 +1,60 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug setup init-backend -WS_TMP_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) -rm -rf "$WS_TMP_DIR" -mkdir -p "$WS_TMP_DIR" - set +e (cd "$INPUT_PATH" && terraform workspace list -no-color) \ - 2>"$WS_TMP_DIR/list_err.txt" \ - >"$WS_TMP_DIR/list_out.txt" + 2>"$STEP_TMP_DIR/terraform_workspace_list.stderr" \ + >"$STEP_TMP_DIR/terraform_workspace_list.stdout" readonly TF_WS_LIST_EXIT=${PIPESTATUS[0]} set -e debug_log "terraform workspace list: ${TF_WS_LIST_EXIT}" -debug_file "$WS_TMP_DIR/list_err.txt" -debug_file "$WS_TMP_DIR/list_out.txt" +debug_file "$STEP_TMP_DIR/terraform_workspace_list.stderr" +debug_file "$STEP_TMP_DIR/terraform_workspace_list.stdout" if [[ $TF_WS_LIST_EXIT -ne 0 ]]; then - echo "Error: Failed to list workspaces" - exit 1 + echo "Error: Failed to list workspaces" + exit 1 fi -if workspace_exists "$INPUT_WORKSPACE" <"$WS_TMP_DIR/list_out.txt"; then - echo "Workspace appears to exist, selecting it" - (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") +if workspace_exists "$INPUT_WORKSPACE" <"$STEP_TMP_DIR/terraform_workspace_list.stdout"; then + echo "Workspace appears to exist, selecting it" + (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") else - echo "Workspace does not appear to exist, attempting to create it" - - set +e - (cd "$INPUT_PATH" && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") \ - 2>"$WS_TMP_DIR/new_err.txt" \ - >"$WS_TMP_DIR/new_out.txt" - - readonly TF_WS_NEW_EXIT=${PIPESTATUS[0]} - set -e - - debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" - debug_file "$WS_TMP_DIR/new_err.txt" - debug_file "$WS_TMP_DIR/new_out.txt" - - if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then - - if grep -Fq "already exists" "$WS_TMP_DIR/new_err.txt"; then - echo "Workspace does exist, selecting it" - (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") + echo "Workspace does not appear to exist, attempting to create it" + + set +e + (cd "$INPUT_PATH" && terraform workspace new -no-color -lock-timeout=300s "$INPUT_WORKSPACE") \ + 2>"$STEP_TMP_DIR/terraform_workspace_new.stderr" \ + >"$STEP_TMP_DIR/terraform_workspace_new.stdout" + + readonly TF_WS_NEW_EXIT=${PIPESTATUS[0]} + set -e + + debug_log "terraform workspace new: ${TF_WS_NEW_EXIT}" + debug_file "$STEP_TMP_DIR/terraform_workspace_new.stderr" + debug_file "$STEP_TMP_DIR/terraform_workspace_new.stdout" + + if [[ $TF_WS_NEW_EXIT -ne 0 ]]; then + + if grep -Fq "already exists" "$STEP_TMP_DIR/terraform_workspace_new.stderr"; then + echo "Workspace does exist, selecting it" + (cd "$INPUT_PATH" && terraform workspace select -no-color "$INPUT_WORKSPACE") + else + cat "$STEP_TMP_DIR/terraform_workspace_new.stderr" + cat "$STEP_TMP_DIR/terraform_workspace_new.stdout" + exit 1 + fi else - cat "$WS_TMP_DIR/new_err.txt" - cat "$WS_TMP_DIR/new_out.txt" - exit 1 + cat "$STEP_TMP_DIR/terraform_workspace_new.stderr" + cat "$STEP_TMP_DIR/terraform_workspace_new.stdout" fi - else - cat "$WS_TMP_DIR/new_err.txt" - cat "$WS_TMP_DIR/new_out.txt" - fi fi diff --git a/image/entrypoints/output.sh b/image/entrypoints/output.sh index aaba69ea..287b5ff6 100755 --- a/image/entrypoints/output.sh +++ b/image/entrypoints/output.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 87598458..98ed3613 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -1,5 +1,6 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug @@ -8,69 +9,99 @@ init-backend select-workspace set-plan-args -PLAN_DIR=$HOME/$GITHUB_RUN_ID-$(random_string) -rm -rf "$PLAN_DIR" -mkdir -p "$PLAN_DIR" +PLAN_OUT="$STEP_TMP_DIR/plan.out" exec 3>&1 -set +e -(cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ - 2>"$PLAN_DIR/error.txt" \ - | $TFMASK \ - | tee /dev/fd/3 \ - | compact_plan \ - >"$PLAN_DIR/plan.txt" +function plan() { -readonly TF_EXIT=${PIPESTATUS[0]} -set -e - -cat "$PLAN_DIR/error.txt" + local PLAN_OUT_ARG + if [[ -n "$PLAN_OUT" ]]; then + PLAN_OUT_ARG="-out=$PLAN_OUT" + else + PLAN_OUT_ARG="" + fi -if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then - if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then + set +e + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ + 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ + | $TFMASK \ + | tee /dev/fd/3 \ + | compact_plan \ + >"$STEP_TMP_DIR/plan.txt" - if [[ -z "$GITHUB_TOKEN" ]]; then - echo "GITHUB_TOKEN environment variable must be set to add GitHub PR comments" - echo "Either set the GITHUB_TOKEN environment variable, or disable by setting the add_github_comment input to 'false'" - echo "See https://github.com/dflook/terraform-github-actions/ for details." - exit 1 - fi + PLAN_EXIT=${PIPESTATUS[0]} + set -e +} - if [[ $TF_EXIT -eq 1 ]]; then - if ! STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$PLAN_DIR/error.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then - debug_file "$PLAN_DIR/github_pr_comment.error" - exit 1 - fi - else +### Generate a plan - if [[ $TF_EXIT -eq 0 ]]; then - TF_CHANGES=false - else # [[ $TF_EXIT -eq 2 ]] - TF_CHANGES=true - fi +plan - if ! TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$PLAN_DIR/plan.txt" 2>"$PLAN_DIR/github_pr_comment.error"; then - debug_file "$PLAN_DIR/github_pr_comment.error" - exit 1 - fi +if [[ $PLAN_EXIT -eq 1 ]]; then + if grep -q "Saving a generated plan is currently not supported" "$STEP_TMP_DIR/terraform_plan.stderr"; then + PLAN_OUT="" + plan fi +fi - fi +if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then + if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then + + if [[ ! -v GITHUB_TOKEN ]]; then + echo "GITHUB_TOKEN environment variable must be set to add GitHub PR comments" + echo "Either set the GITHUB_TOKEN environment variable, or disable by setting the add_github_comment input to 'false'" + echo "See https://github.com/dflook/terraform-github-actions/ for details." + exit 1 + fi + + if [[ $PLAN_EXIT -eq 1 ]]; then + if ! STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + exit 1 + fi + else + + if [[ $PLAN_EXIT -eq 0 ]]; then + TF_CHANGES=false + else # [[ $PLAN_EXIT -eq 2 ]] + TF_CHANGES=true + fi + + if ! TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + exit 1 + fi + fi + + fi else - debug_log "Not a pull_request, issue_comment, pull_request_target, pull_request_review or pull_request_review_comment event - not creating a PR comment" + debug_log "Not a pull_request, issue_comment, pull_request_target, pull_request_review or pull_request_review_comment event - not creating a PR comment" fi -if [[ $TF_EXIT -eq 1 ]]; then +if [[ $PLAN_EXIT -eq 1 ]]; then debug_log "Error running terraform" exit 1 -elif [[ $TF_EXIT -eq 0 ]]; then +elif [[ $PLAN_EXIT -eq 0 ]]; then debug_log "No Changes to apply" set_output changes false -elif [[ $TF_EXIT -eq 2 ]]; then +elif [[ $PLAN_EXIT -eq 2 ]]; then debug_log "Changes to apply" set_output changes true fi + +mkdir -p "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR" +cp "$STEP_TMP_DIR/plan.txt" "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR/plan.txt" +set_output text_plan_path "$WORKSPACE_TMP_DIR/plan.txt" + +if [[ -n "$PLAN_OUT" ]]; then + if (cd "$INPUT_PATH" && terraform show -json "$PLAN_OUT") >"$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR/plan.json" 2>"$STEP_TMP_DIR/terraform_show.stderr"; then + set_output json_plan_path "$WORKSPACE_TMP_DIR/plan.json" + else + debug_file "$STEP_TMP_DIR/terraform_show.stderr" + fi +fi diff --git a/image/entrypoints/remote-state.sh b/image/entrypoints/remote-state.sh index 7cdb19e1..fa315220 100755 --- a/image/entrypoints/remote-state.sh +++ b/image/entrypoints/remote-state.sh @@ -1,14 +1,15 @@ #!/bin/bash +# shellcheck source=../actions.sh source /usr/local/actions.sh debug -INPUT_PATH="$HOME/.dflook-terraform-remote-state" +INPUT_PATH="$STEP_TMP_DIR/remote-state" rm -rf "$INPUT_PATH" mkdir -p "$INPUT_PATH" -cat > "$INPUT_PATH/backend.tf" <"$INPUT_PATH/backend.tf" <=0.13 this is correctly set to 'true' whenever an apply needs to be run. +* `json_plan_path` + + This is the path to the generated plan in [JSON Output Format](https://www.terraform.io/docs/internals/json-format.html) + The path is relative to the Actions workspace. + + This is not available when using terraform 0.11 or earlier. + This also won't be set if the backend type is `remote` - Terraform does not support saving remote plans. + +* `text_plan_path` + + This is the path to the generated plan in a human readable format. + The path is relative to the Actions workspace. + ## Example usage ### Automatically generating a plan diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 56ea8f48..eec03141 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -42,7 +42,9 @@ inputs: outputs: changes: - description: If the plan changes any resources - 'true' or 'false' + description: If the generated plan would update any resources or outputs this is set to `true`, otherwise it's set to `false`. + json_plan_path: + description: Path to the generated plan in JSON format. This won't be set if the backend type is `remote`. runs: using: docker From 18e881af6a12e4d511e910a8c9eb618f04c9979e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 12:42:34 +0100 Subject: [PATCH 130/231] Add output doc for text_plan_path --- terraform-plan/action.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index eec03141..99a56c7b 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -43,8 +43,10 @@ inputs: outputs: changes: description: If the generated plan would update any resources or outputs this is set to `true`, otherwise it's set to `false`. + text_plan_path: + description: Path to a file in the workspace containing the generated plan in human readble format. json_plan_path: - description: Path to the generated plan in JSON format. This won't be set if the backend type is `remote`. + description: Path to a file in the workspace containing the generated plan in JSON format. This won't be set if the backend type is `remote`. runs: using: docker From 474faad022dbe5bda3838347a1552fe0a7aea2b4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 12:51:16 +0100 Subject: [PATCH 131/231] Remove test --- .github/workflows/test-plan.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 6fed2d48..c85066b5 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -313,12 +313,6 @@ jobs: exit 1 fi - - name: Checkov - run: | - cat '${{ steps.plan.outputs.json_plan_path }}' - pip3 install checkov - checkov -f '${{ steps.plan.outputs.json_plan_path }}' - plan_change_no_comment: runs-on: ubuntu-latest name: Change without github comment From a9cf2c50666e4f6f44d5f897b58dcc420502c0c6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 12:53:21 +0100 Subject: [PATCH 132/231] :bookmark: v1.16.0 --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa91b895..cd934078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,19 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.15.0` to use an exact release -- `@v1.15` to use the latest patch release for the specific minor version +- `@v1.16.0` to use an exact release +- `@v1.16` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.16.0] - 2021-10-04 + +### Added +- [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) has gained two new outputs: + - `json_plan_path` is a path to the generated plan in a JSON format file + - `text_plan_path` is a path to the generated plan in a human readable text file + + These paths are relative to the GitHub Actions workspace and can be read by other steps in the same job. + ## [1.15.0] - 2021-09-20 ### Added From fdbde0ff9cb7dbd5e819364e68e110b8073aecf4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 13:25:42 +0100 Subject: [PATCH 133/231] Add 1.16.0 changes link --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd934078..6cfdccf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,6 +244,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.16.0]: https://github.com/dflook/terraform-github-actions/compare/v1.15.0...v1.16.0 [1.15.0]: https://github.com/dflook/terraform-github-actions/compare/v1.14.0...v1.15.0 [1.14.0]: https://github.com/dflook/terraform-github-actions/compare/v1.13.0...v1.14.0 [1.13.0]: https://github.com/dflook/terraform-github-actions/compare/v1.12.0...v1.13.0 From 11ddadaa8ec9aad06c6f0930a2d3e0ab696a2f2a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 16:35:23 +0100 Subject: [PATCH 134/231] Add variables and var_files support for remote operations --- .github/workflows/test-remote.yaml | 48 ++++++++++++++++++++++++ image/actions.sh | 24 ++++++++++++ image/entrypoints/apply.sh | 10 +++++ image/entrypoints/plan.sh | 3 ++ tests/terraform-cloud/main.tf | 24 ++++++++++++ tests/terraform-cloud/my_variable.tfvars | 2 + 6 files changed, 111 insertions(+) create mode 100644 tests/terraform-cloud/my_variable.tfvars diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 2fdb3884..7f413d0a 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -33,11 +33,33 @@ jobs: - name: Auto apply workspace uses: ./terraform-apply + id: auto_apply with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 backend_config: "token=${{ secrets.TF_API_TOKEN }}" auto_approve: true + var_file: | + tests/terraform-cloud/my_variable.tfvars + variables: | + from_variables="from_variables" + + - name: Verify auto_apply terraform outputs + run: | + if [[ "${{ steps.auto_apply.outputs.default }}" != "default" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + + if [[ "${{ steps.auto_apply.outputs.from_tfvars }}" != "from_tfvars" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + + if [[ "${{ steps.auto_apply.outputs.from_variables }}" != "from_variables" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi - name: Destroy workspace uses: ./terraform-destroy-workspace @@ -55,6 +77,10 @@ jobs: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 backend_config: "token=${{ secrets.TF_API_TOKEN }}" + var_file: | + tests/terraform-cloud/my_variable.tfvars + variables: | + from_variables="from_variables" - name: Verify plan outputs run: | @@ -70,12 +96,34 @@ jobs: - name: Apply workspace uses: ./terraform-apply + id: apply env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 backend_config: "token=${{ secrets.TF_API_TOKEN }}" + var_file: | + tests/terraform-cloud/my_variable.tfvars + variables: | + from_variables="from_variables" + + - name: Verify apply terraform outputs + run: | + if [[ "${{ steps.apply.outputs.default }}" != "default" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + + if [[ "${{ steps.apply.outputs.from_tfvars }}" != "from_tfvars" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + + if [[ "${{ steps.apply.outputs.from_variables }}" != "from_variables" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi - name: Destroy the last workspace uses: ./terraform-destroy-workspace diff --git a/image/actions.sh b/image/actions.sh index 357c2318..9623c3cc 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -214,6 +214,30 @@ function set-plan-args() { export PLAN_ARGS } +function set-remote-plan-args() { + PLAN_ARGS="" + + if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then + PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" + fi + + local AUTO_TFVARS_COUNTER=0 + + if [[ -n "$INPUT_VAR_FILE" ]]; then + for file in $(echo "$INPUT_VAR_FILE" | tr ',' '\n'); do + cp "$file" "$INPUT_PATH/zzzz-dflook-terraform-github-actions-$AUTO_TFVARS_COUNTER.auto.tfvars" + AUTO_TFVARS_COUNTER=$(( AUTO_TFVARS_COUNTER + 1 )) + done + fi + + if [[ -n "$INPUT_VARIABLES" ]]; then + echo "$INPUT_VARIABLES" >"$STEP_TMP_DIR/variables.tfvars" + cp "$file" "$INPUT_PATH/zzzz-dflook-terraform-github-actions-$AUTO_TFVARS_COUNTER.auto.tfvars" + fi + + export PLAN_ARGS +} + function output() { (cd "$INPUT_PATH" && terraform output -json | convert_output) } diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 83dc7267..971a7b48 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -53,6 +53,8 @@ function apply() { local APPLY_EXIT=${PIPESTATUS[0]} set -e + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete + if [[ $APPLY_EXIT -eq 0 ]]; then update_status "Plan applied in $(job_markdown_ref)" else @@ -68,6 +70,7 @@ plan if [[ $PLAN_EXIT -eq 1 ]]; then if grep -q "Saving a generated plan is currently not supported" "$STEP_TMP_DIR/terraform_plan.stderr"; then + set-remote-plan-args PLAN_OUT="" if [[ "$INPUT_AUTO_APPROVE" == "true" ]]; then @@ -80,6 +83,7 @@ if [[ $PLAN_EXIT -eq 1 ]]; then fi if [[ $PLAN_EXIT -eq 1 ]]; then + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete cat "$STEP_TMP_DIR/terraform_plan.stderr" update_status "Error applying plan in $(job_markdown_ref)" @@ -95,11 +99,13 @@ if [[ "$INPUT_AUTO_APPROVE" == "true" || $PLAN_EXIT -eq 0 ]]; then else if [[ "$GITHUB_EVENT_NAME" != "push" && "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "issue_comment" && "$GITHUB_EVENT_NAME" != "pull_request_review_comment" && "$GITHUB_EVENT_NAME" != "pull_request_target" && "$GITHUB_EVENT_NAME" != "pull_request_review" ]]; then + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete echo "Could not fetch plan from the PR - $GITHUB_EVENT_NAME event does not relate to a pull request. You can generate and apply a plan automatically by setting the auto_approve input to 'true'" exit 1 fi if [[ ! -v GITHUB_TOKEN ]]; then + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete echo "GITHUB_TOKEN environment variable must be set to get plan approval from a PR" echo "Either set the GITHUB_TOKEN environment variable or automatically approve by setting the auto_approve input to 'true'" echo "See https://github.com/dflook/terraform-github-actions/ for details." @@ -107,6 +113,8 @@ else fi if ! github_pr_comment get "$STEP_TMP_DIR/approved-plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" @@ -119,6 +127,8 @@ else if plan_cmp "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt"; then apply else + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete + echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 98ed3613..f51e94f7 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -41,8 +41,11 @@ plan if [[ $PLAN_EXIT -eq 1 ]]; then if grep -q "Saving a generated plan is currently not supported" "$STEP_TMP_DIR/terraform_plan.stderr"; then + # This terraform module is using the remote backend, which is deficient. + set-remote-plan-args PLAN_OUT="" plan + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete fi fi diff --git a/tests/terraform-cloud/main.tf b/tests/terraform-cloud/main.tf index 629aa747..f0072073 100644 --- a/tests/terraform-cloud/main.tf +++ b/tests/terraform-cloud/main.tf @@ -12,3 +12,27 @@ terraform { resource "random_id" "the_id" { byte_length = 5 } + +variable "default" { + default = "default" +} + +output "default" { + value = var.default +} + +variable "from_tfvars" { + default = "default" +} + +output "from_tfvars" { + value = var.from_tfvars +} + +variable "from_variables" { + default = "default" +} + +output "from_variables" { + value = var.from_variables +} diff --git a/tests/terraform-cloud/my_variable.tfvars b/tests/terraform-cloud/my_variable.tfvars new file mode 100644 index 00000000..8fa611eb --- /dev/null +++ b/tests/terraform-cloud/my_variable.tfvars @@ -0,0 +1,2 @@ +from_tfvars="from_tfvars" +from_variables="from_tfvars" From e0300f881fbcf90a6bffa788270d39fe776408cb Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 16:49:52 +0100 Subject: [PATCH 135/231] fixup --- image/actions.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index 9623c3cc..c024a314 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -221,6 +221,8 @@ function set-remote-plan-args() { PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" fi + set -x + local AUTO_TFVARS_COUNTER=0 if [[ -n "$INPUT_VAR_FILE" ]]; then @@ -232,9 +234,11 @@ function set-remote-plan-args() { if [[ -n "$INPUT_VARIABLES" ]]; then echo "$INPUT_VARIABLES" >"$STEP_TMP_DIR/variables.tfvars" - cp "$file" "$INPUT_PATH/zzzz-dflook-terraform-github-actions-$AUTO_TFVARS_COUNTER.auto.tfvars" + cp "$STEP_TMP_DIR/variables.tfvars" "$INPUT_PATH/zzzz-dflook-terraform-github-actions-$AUTO_TFVARS_COUNTER.auto.tfvars" fi + debug_cmd ls -la "$INPUT_PATH" + export PLAN_ARGS } From 65b027c5a4800ed06a13068ae7cc7d2fea44a03e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 17:45:42 +0100 Subject: [PATCH 136/231] Add variables and var_files support for remote operations in terraform-check --- .github/workflows/test-remote.yaml | 36 +++++++++++++++++ image/actions.sh | 24 +++++++++++- image/entrypoints/apply.sh | 22 ----------- image/entrypoints/check.sh | 24 ++++++++---- image/entrypoints/plan.sh | 22 ----------- terraform-apply/README.md | 62 ++++++++++++++++-------------- terraform-check/README.md | 24 +++++++----- terraform-plan/README.md | 62 ++++++++++++++++-------------- 8 files changed, 155 insertions(+), 121 deletions(-) diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 7f413d0a..71faa417 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -61,6 +61,42 @@ jobs: exit 1 fi + - name: Check no changes + uses: ./terraform-check + with: + path: tests/terraform-cloud + workspace: ${{ github.head_ref }}-1 + backend_config: "token=${{ secrets.TF_API_TOKEN }}" + var_file: | + tests/terraform-cloud/my_variable.tfvars + variables: | + from_variables="from_variables" + + - name: Check changes + uses: ./terraform-check + id: check + continue-on-error: true + with: + path: tests/terraform-cloud + workspace: ${{ github.head_ref }}-1 + backend_config: "token=${{ secrets.TF_API_TOKEN }}" + var_file: | + tests/terraform-cloud/my_variable.tfvars + variables: | + from_variables="Changed!" + + - name: Verify changes detected + run: | + if [[ "${{ steps.check.outcome }}" != "failure" ]]; then + echo "Check didn't fail correctly" + exit 1 + fi + + if [[ "${{ steps.check.outputs.failure-reason }}" != "changes-to-apply" ]]; then + echo "failure-reason not set correctly" + exit 1 + fi + - name: Destroy workspace uses: ./terraform-destroy-workspace with: diff --git a/image/actions.sh b/image/actions.sh index c024a314..80137abc 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -221,8 +221,6 @@ function set-remote-plan-args() { PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" fi - set -x - local AUTO_TFVARS_COUNTER=0 if [[ -n "$INPUT_VAR_FILE" ]]; then @@ -271,6 +269,28 @@ function write_credentials() { debug_cmd git config --list } +function plan() { + + local PLAN_OUT_ARG + if [[ -n "$PLAN_OUT" ]]; then + PLAN_OUT_ARG="-out=$PLAN_OUT" + else + PLAN_OUT_ARG="" + fi + + set +e + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ + 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ + | $TFMASK \ + | tee /dev/fd/3 \ + | compact_plan \ + >"$STEP_TMP_DIR/plan.txt" + + PLAN_EXIT=${PIPESTATUS[0]} + set -e +} + # Every file written to disk should use one of these directories readonly STEP_TMP_DIR="/tmp" readonly JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 971a7b48..4069c566 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -23,28 +23,6 @@ fi exec 3>&1 -function plan() { - - local PLAN_OUT_ARG - if [[ -n "$PLAN_OUT" ]]; then - PLAN_OUT_ARG="-out=$PLAN_OUT" - else - PLAN_OUT_ARG="" - fi - - set +e - # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ - 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ - | $TFMASK \ - | tee /dev/fd/3 \ - | compact_plan \ - >"$STEP_TMP_DIR/plan.txt" - - PLAN_EXIT=${PIPESTATUS[0]} - set -e -} - function apply() { set +e diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index ef201e1c..d4d812d6 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -9,19 +9,27 @@ init-backend select-workspace set-plan-args -set +e -# shellcheck disable=SC2086 -(cd "$INPUT_PATH" && terraform plan -input=false -detailed-exitcode -lock-timeout=300s $PLAN_ARGS) \ - | $TFMASK +PLAN_OUT="$STEP_TMP_DIR/plan.out" -readonly TF_EXIT=${PIPESTATUS[0]} -set -e +exec 3>&1 -if [[ $TF_EXIT -eq 1 ]]; then +plan + +if [[ $PLAN_EXIT -eq 1 ]]; then + if grep -q "Saving a generated plan is currently not supported" "$STEP_TMP_DIR/terraform_plan.stderr"; then + # This terraform module is using the remote backend, which is deficient. + set-remote-plan-args + PLAN_OUT="" + plan + find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete + fi +fi + +if [[ $PLAN_EXIT -eq 1 ]]; then echo "Error running terraform" exit 1 -elif [[ $TF_EXIT -eq 2 ]]; then +elif [[ $PLAN_EXIT -eq 2 ]]; then echo "Changes detected!" set_output failure-reason changes-to-apply diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index f51e94f7..cacd8d15 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -13,28 +13,6 @@ PLAN_OUT="$STEP_TMP_DIR/plan.out" exec 3>&1 -function plan() { - - local PLAN_OUT_ARG - if [[ -n "$PLAN_OUT" ]]; then - PLAN_OUT_ARG="-out=$PLAN_OUT" - else - PLAN_OUT_ARG="" - fi - - set +e - # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ - 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ - | $TFMASK \ - | tee /dev/fd/3 \ - | compact_plan \ - >"$STEP_TMP_DIR/plan.txt" - - PLAN_EXIT=${PIPESTATUS[0]} - set -e -} - ### Generate a plan plan diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 3694f71b..32be18f4 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -74,35 +74,7 @@ These input values must be the same as any `terraform-plan` for the same configu ``` Variables set here override any given in `var_file`s. - - - Type: string - - Optional - -* ~~`var`~~ - - > :warning: **Deprecated**: Use the `variables` input instead. - - Comma separated list of terraform vars to set. - - This is deprecated due to the following limitations: - - Only primitive types can be set with `var` - number, bool and string. - - String values may not contain a comma. - - Values set with `var` will be overridden by values contained in `var_file`s - - You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. - - For example: - ```yaml - with: - var: instance_type=m5.xlarge,nat_type=instance - ``` - Becomes: - ```yaml - with: - variables: | - instance_type="m5.xlarge" - nat_type="instance" - ``` + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. - Type: string - Optional @@ -119,6 +91,8 @@ These input values must be the same as any `terraform-plan` for the same configu prod.tfvars ``` + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. + - Type: string - Optional @@ -181,6 +155,36 @@ These input values must be the same as any `terraform-plan` for the same configu - Optional - Default: false +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set. + + This is deprecated due to the following limitations: + - Only primitive types can be set with `var` - number, bool and string. + - String values may not contain a comma. + - Values set with `var` will be overridden by values contained in `var_file`s + - Does not work with the `remote` backend + + You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. + + For example: + ```yaml + with: + var: instance_type=m5.xlarge,nat_type=instance + ``` + Becomes: + ```yaml + with: + variables: | + instance_type="m5.xlarge" + nat_type="instance" + ``` + + - Type: string + - Optional + ## Outputs * Terraform Outputs diff --git a/terraform-check/README.md b/terraform-check/README.md index a49feee0..16ccc352 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -39,15 +39,7 @@ This is intended to run on a schedule to notify if manual changes to your infras ``` Variables set here override any given in `var_file`s. - - - Type: string - - Optional - -* ~~`var`~~ - - > :warning: **Deprecated**: Use the `variables` input instead. - - Comma separated list of terraform vars to set + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. - Type: string - Optional @@ -64,6 +56,11 @@ This is intended to run on a schedule to notify if manual changes to your infras prod.tfvars ``` + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. + + - Type: string + - Optional + * `backend_config` List of terraform backend config values, one per line. @@ -97,6 +94,15 @@ This is intended to run on a schedule to notify if manual changes to your infras - Optional - Default: 10 +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set + + - Type: string + - Optional + ## Outputs * `failure-reason` diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 4241c87c..e739be60 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -57,41 +57,13 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ ``` Variables set here override any given in `var_file`s. + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. > :warning: Secret values are not masked in the PR comment. Set a `label` to avoid revealing the variables in the PR. - Type: string - Optional -* ~~`var`~~ - - > :warning: **Deprecated**: Use the `variables` input instead. - - Comma separated list of terraform vars to set. - - This is deprecated due to the following limitations: - - Only primitive types can be set with `var` - number, bool and string. - - String values may not contain a comma. - - Values set with `var` will be overridden by values contained in `var_file`s - - You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. - - For example: - ```yaml - with: - var: instance_type=m5.xlarge,nat_type=instance - ``` - Becomes: - ```yaml - with: - variables: | - instance_type="m5.xlarge" - nat_type="instance" - ``` - - - Type: string - - Optional - * `var_file` List of tfvars files to use, one per line. @@ -104,6 +76,8 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ prod.tfvars ``` + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. + - Type: string - Optional @@ -150,6 +124,36 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Optional - Default: true +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set. + + This is deprecated due to the following limitations: + - Only primitive types can be set with `var` - number, bool and string. + - String values may not contain a comma. + - Values set with `var` will be overridden by values contained in `var_file`s + - Does not work with the `remote` backend + + You can change from `var` to `variables` by putting each variable on a separate line and ensuring each string value is quoted. + + For example: + ```yaml + with: + var: instance_type=m5.xlarge,nat_type=instance + ``` + Becomes: + ```yaml + with: + variables: | + instance_type="m5.xlarge" + nat_type="instance" + ``` + + - Type: string + - Optional + ## Environment Variables * `GITHUB_TOKEN` From 6c25198bec74fc5af85913a3554dff2fa0232681 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 4 Oct 2021 18:13:10 +0100 Subject: [PATCH 137/231] :bookmark: v1.17.0 --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfdccf8..341b33d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,18 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.16.0` to use an exact release -- `@v1.16` to use the latest patch release for the specific minor version +- `@v1.17.0` to use an exact release +- `@v1.17` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.17.0] - 2021-10-04 + +### Added +- `variables` and `var_file` support for remote operations in Terraform Cloud/Enterprise. + + The Terraform CLI & Terraform Cloud/Enterprise do not support using variables or variable files with remote plans or applies. + We can do better. `variables` and `var_file` input variables for the plan, apply & check actions now work, with the expected behavior. + ## [1.16.0] - 2021-10-04 ### Added @@ -244,6 +252,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.17.0]: https://github.com/dflook/terraform-github-actions/compare/v1.16.0...v1.17.0 [1.16.0]: https://github.com/dflook/terraform-github-actions/compare/v1.15.0...v1.16.0 [1.15.0]: https://github.com/dflook/terraform-github-actions/compare/v1.14.0...v1.15.0 [1.14.0]: https://github.com/dflook/terraform-github-actions/compare/v1.13.0...v1.14.0 From 2eea59cfa8ca96521e575033af14719d1e659f09 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 5 Oct 2021 20:48:29 +0100 Subject: [PATCH 138/231] Fix ownership of any files that are created in HOME or GITHUB_WORKSPACE --- image/actions.sh | 29 +++++++++++++++++++++++++++++ image/entrypoints/apply.sh | 11 ++--------- image/entrypoints/check.sh | 1 - image/entrypoints/plan.sh | 6 +++++- image/tools/github_pr_comment.py | 8 +++++--- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 80137abc..4aaf12b3 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -249,6 +249,8 @@ function update_status() { if ! STATUS="$status" github_pr_comment status 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + else + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi } @@ -258,7 +260,9 @@ function random_string() { function write_credentials() { format_tf_credentials >>"$HOME/.terraformrc" + chown -R --reference "$HOME" "$HOME/.terraformrc" netrc-credential-actions >>"$HOME/.netrc" + chown -R --reference "$HOME" "$HOME/.netrc" chmod 700 /.ssh if [[ -v TERRAFORM_SSH_KEY ]]; then @@ -295,3 +299,28 @@ function plan() { readonly STEP_TMP_DIR="/tmp" readonly JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" readonly WORKSPACE_TMP_DIR=".dflook-terraform-github-actions/$(random_string)" +export STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR + +function fix_owners() { + debug_cmd ls -la "$GITHUB_WORKSPACE" + if [[ -d "$GITHUB_WORKSPACE/.dflook-terraform-github-actions" ]]; then + chown -R --reference "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE/.dflook-terraform-github-actions" || true + debug_cmd ls -la "$GITHUB_WORKSPACE/.dflook-terraform-github-actions" + fi + + debug_cmd ls -la "$HOME" + if [[ -d "$HOME/.dflook-terraform-github-actions" ]]; then + chown -R --reference "$HOME" "$HOME/.dflook-terraform-github-actions" || true + debug_cmd ls -la "$HOME/.dflook-terraform-github-actions" + fi + if [[ -d "$HOME/.terraform.d" ]]; then + chown -R --reference "$HOME" "$HOME/.terraform.d" || true + debug_cmd ls -la "$HOME/.terraform.d" + fi + + if [[ -d "$INPUT_PATH" ]]; then + debug_cmd find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -print -delete || true + fi +} + +trap fix_owners EXIT diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 4069c566..d5a6e01d 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -31,8 +31,6 @@ function apply() { local APPLY_EXIT=${PIPESTATUS[0]} set -e - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete - if [[ $APPLY_EXIT -eq 0 ]]; then update_status "Plan applied in $(job_markdown_ref)" else @@ -61,7 +59,6 @@ if [[ $PLAN_EXIT -eq 1 ]]; then fi if [[ $PLAN_EXIT -eq 1 ]]; then - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete cat "$STEP_TMP_DIR/terraform_plan.stderr" update_status "Error applying plan in $(job_markdown_ref)" @@ -77,13 +74,11 @@ if [[ "$INPUT_AUTO_APPROVE" == "true" || $PLAN_EXIT -eq 0 ]]; then else if [[ "$GITHUB_EVENT_NAME" != "push" && "$GITHUB_EVENT_NAME" != "pull_request" && "$GITHUB_EVENT_NAME" != "issue_comment" && "$GITHUB_EVENT_NAME" != "pull_request_review_comment" && "$GITHUB_EVENT_NAME" != "pull_request_target" && "$GITHUB_EVENT_NAME" != "pull_request_review" ]]; then - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete echo "Could not fetch plan from the PR - $GITHUB_EVENT_NAME event does not relate to a pull request. You can generate and apply a plan automatically by setting the auto_approve input to 'true'" exit 1 fi if [[ ! -v GITHUB_TOKEN ]]; then - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete echo "GITHUB_TOKEN environment variable must be set to get plan approval from a PR" echo "Either set the GITHUB_TOKEN environment variable or automatically approve by setting the auto_approve input to 'true'" echo "See https://github.com/dflook/terraform-github-actions/ for details." @@ -91,8 +86,6 @@ else fi if ! github_pr_comment get "$STEP_TMP_DIR/approved-plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" @@ -100,13 +93,13 @@ else set_output failure-reason plan-changed exit 1 + else + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi if plan_cmp "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt"; then apply else - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete - echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index d4d812d6..07b0aa6c 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -21,7 +21,6 @@ if [[ $PLAN_EXIT -eq 1 ]]; then set-remote-plan-args PLAN_OUT="" plan - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete fi fi diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index cacd8d15..36130b09 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -23,7 +23,6 @@ if [[ $PLAN_EXIT -eq 1 ]]; then set-remote-plan-args PLAN_OUT="" plan - find "$INPUT_PATH" -regex '.*/zzzz-dflook-terraform-github-actions-[0-9]+\.auto\.tfvars' -delete fi fi @@ -41,7 +40,10 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c if ! STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" exit 1 + else + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi + else if [[ $PLAN_EXIT -eq 0 ]]; then @@ -53,6 +55,8 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c if ! TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" exit 1 + else + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi fi diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 5c22a856..3f955435 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -18,6 +18,8 @@ github_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') github_api_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com') +job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') + def github_api_request(method, *args, **kwargs): response = github.request(method, *args, **kwargs) @@ -109,7 +111,7 @@ def current_user() -> str: token_hash = hashlib.sha256(os.environ["GITHUB_TOKEN"].encode()).hexdigest() try: - with open(f'.dflook-terraform/token-cache/{token_hash}') as f: + with open(os.path.join(job_tmp_dir, 'token-cache', token_hash)) as f: username = f.read() debug(f'GITHUB_TOKEN username: {username}') return username @@ -128,8 +130,8 @@ def current_user() -> str: username = 'github-actions[bot]' try: - os.makedirs('.dflook-terraform/token-cache', exist_ok=True) - with open(f'.dflook-terraform/token-cache/{token_hash}', 'w') as f: + os.makedirs(os.path.join(job_tmp_dir, 'token-cache'), exist_ok=True) + with open(os.path.join(job_tmp_dir, 'token-cache', token_hash), 'w') as f: f.write(username) except Exception as e: debug(str(e)) From 1b9f8ca17cdb893028c4aba34ed01c9df157ea70 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 6 Oct 2021 16:33:31 +0100 Subject: [PATCH 139/231] :bookmark: 1.17.1 --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 341b33d4..2e51eea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,18 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.17.0` to use an exact release +- `@v1.17.1` to use an exact release - `@v1.17` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.17.1] - 2021-10-06 + +### Fixed +- Fix ownership of files created in runner mounted directories + + As the container is run as root, it can cause issues when root owned files are leftover that the runner can't cleanup. + This would only affect self-hosted, non-ephemeral, non-root runners. + ## [1.17.0] - 2021-10-04 ### Added @@ -252,6 +260,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.17.1]: https://github.com/dflook/terraform-github-actions/compare/v1.17.0...v1.17.1 [1.17.0]: https://github.com/dflook/terraform-github-actions/compare/v1.16.0...v1.17.0 [1.16.0]: https://github.com/dflook/terraform-github-actions/compare/v1.15.0...v1.16.0 [1.15.0]: https://github.com/dflook/terraform-github-actions/compare/v1.14.0...v1.15.0 From 8c919711f7510a2fad2c1de82874178bbde2c8e9 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 13 Oct 2021 19:18:22 +0100 Subject: [PATCH 140/231] Add plan output to workflow log --- CHANGELOG.md | 5 +++++ image/actions.sh | 4 ++-- image/entrypoints/plan.sh | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e51eea9..c1ff12e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ When using an action you can specify the version as: - `@v1.17` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Fixed +- Add `terraform plan` output that was missing from the workflow log + ## [1.17.1] - 2021-10-06 ### Fixed diff --git a/image/actions.sh b/image/actions.sh index 4aaf12b3..ea71003b 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -260,9 +260,9 @@ function random_string() { function write_credentials() { format_tf_credentials >>"$HOME/.terraformrc" - chown -R --reference "$HOME" "$HOME/.terraformrc" + chown --reference "$HOME" "$HOME/.terraformrc" netrc-credential-actions >>"$HOME/.netrc" - chown -R --reference "$HOME" "$HOME/.netrc" + chown --reference "$HOME" "$HOME/.netrc" chmod 700 /.ssh if [[ -v TERRAFORM_SSH_KEY ]]; then diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 36130b09..20afb668 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -26,6 +26,9 @@ if [[ $PLAN_EXIT -eq 1 ]]; then fi fi +cat "$STEP_TMP_DIR/plan.txt" +cat "$STEP_TMP_DIR/terraform_plan.stderr" + if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then From dbe625d7554093223e2319a819bbb9e07ac0ca42 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 13 Oct 2021 19:30:07 +0100 Subject: [PATCH 141/231] :bookmark: 1.17.2 --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ff12e0..a0b2b7c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.17.1` to use an exact release +- `@v1.17.2` to use an exact release - `@v1.17` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.17.2] - 2021-10-13 ### Fixed - Add `terraform plan` output that was missing from the workflow log @@ -265,6 +265,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.17.2]: https://github.com/dflook/terraform-github-actions/compare/v1.17.1...v1.17.2 [1.17.1]: https://github.com/dflook/terraform-github-actions/compare/v1.17.0...v1.17.1 [1.17.0]: https://github.com/dflook/terraform-github-actions/compare/v1.16.0...v1.17.0 [1.16.0]: https://github.com/dflook/terraform-github-actions/compare/v1.15.0...v1.16.0 From 455d0fdee67a8b4608744b73fb5a53ffd8e9907b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 24 Oct 2021 22:32:49 +0100 Subject: [PATCH 142/231] Refactor github_pr_comment.py --- .github/workflows/test.yaml | 2 +- image/tools/github_pr_comment.py | 511 +++++++++++++++---------------- tests/test_pr_comment.py | 460 ++++++++++++++++------------ 3 files changed, 517 insertions(+), 456 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bc5f16d6..21b88f05 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies run: | diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 3f955435..93c2fd98 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -1,26 +1,141 @@ #!/usr/bin/python3 +import datetime +import hashlib import json import os import re import sys -import datetime -import hashlib -from typing import Optional, Dict, Iterable +from typing import Optional, Dict, Iterable, cast, NewType, TypedDict, Tuple, Any import requests -github = requests.Session() -github.headers['authorization'] = f'token {os.environ["GITHUB_TOKEN"]}' -github.headers['user-agent'] = 'terraform-github-actions' -github.headers['accept'] = 'application/vnd.github.v3+json' +GitHubUrl = NewType('GitHubUrl', str) +PrUrl = NewType('PrUrl', GitHubUrl) +IssueUrl = NewType('IssueUrl', GitHubUrl) +CommentUrl = NewType('CommentUrl', GitHubUrl) +Plan = NewType('Plan', str) +Status = NewType('Status', str) + + +class GitHubActionsEnv(TypedDict): + GITHUB_API_URL: str + GITHUB_TOKEN: str + GITHUB_EVENT_PATH: str + GITHUB_EVENT_NAME: str + GITHUB_REPOSITORY: str + GITHUB_SHA: str -github_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') -github_api_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com') job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') -def github_api_request(method, *args, **kwargs): +env = cast(GitHubActionsEnv, os.environ) + + +def github_session(github_env: GitHubActionsEnv) -> requests.Session: + session = requests.Session() + session.headers['authorization'] = f'token {github_env["GITHUB_TOKEN"]}' + session.headers['user-agent'] = 'terraform-github-actions' + session.headers['accept'] = 'application/vnd.github.v3+json' + return session + + +github = github_session(env) + + +class ActionInputs(TypedDict): + """ + Information used to identify a plan + """ + INPUT_BACKEND_CONFIG: str + INPUT_BACKEND_CONFIG_FILE: str + INPUT_VARIABLES: str + INPUT_VAR: str + INPUT_VAR_FILE: str + INPUT_PATH: str + INPUT_WORKSPACE: str + INPUT_LABEL: str + INPUT_ADD_GITHUB_COMMENT: str + + +def plan_identifier(action_inputs: ActionInputs) -> str: + def mask_backend_config() -> Optional[str]: + if action_inputs.get('INPUT_BACKEND_CONFIG') is None: + return None + + bad_words = [ + 'token', + 'password', + 'sas_token', + 'access_key', + 'secret_key', + 'client_secret', + 'access_token', + 'http_auth', + 'secret_id', + 'encryption_key', + 'key_material', + 'security_token', + 'conn_str', + 'sse_customer_key', + 'application_credential_secret' + ] + + def has_bad_word(s: str) -> bool: + for bad_word in bad_words: + if bad_word in s: + return True + return False + + clean = [] + + for field in action_inputs.get('INPUT_BACKEND_CONFIG', '').split(','): + if not field: + continue + + if not has_bad_word(field): + clean.append(field) + + return ','.join(clean) + + if action_inputs['INPUT_LABEL']: + return f'Terraform plan for __{action_inputs["INPUT_LABEL"]}__' + + label = f'Terraform plan in __{action_inputs["INPUT_PATH"]}__' + + if action_inputs["INPUT_WORKSPACE"] != 'default': + label += f' in the __{action_inputs["INPUT_WORKSPACE"]}__ workspace' + + backend_config = mask_backend_config() + if backend_config: + label += f'\nWith backend config: `{backend_config}`' + + if action_inputs["INPUT_BACKEND_CONFIG_FILE"]: + label += f'\nWith backend config files: `{action_inputs["INPUT_BACKEND_CONFIG_FILE"]}`' + + if action_inputs["INPUT_VAR"]: + label += f'\nWith vars: `{action_inputs["INPUT_VAR"]}`' + + if action_inputs["INPUT_VAR_FILE"]: + label += f'\nWith var files: `{action_inputs["INPUT_VAR_FILE"]}`' + + if action_inputs["INPUT_VARIABLES"]: + stripped_vars = action_inputs["INPUT_VARIABLES"].strip() + if '\n' in stripped_vars: + label += f'''
With variables + +```hcl +{stripped_vars} +``` +
+''' + else: + label += f'\nWith variables: `{stripped_vars}`' + + return label + + +def github_api_request(method: str, *args, **kwargs) -> requests.Response: response = github.request(method, *args, **kwargs) if 400 <= response.status_code < 500: @@ -47,12 +162,13 @@ def github_api_request(method, *args, **kwargs): return response + def debug(msg: str) -> None: sys.stderr.write(msg) sys.stderr.write('\n') -def paginate(url, *args, **kwargs) -> Iterable[Dict]: +def paginate(url: GitHubUrl, *args, **kwargs) -> Iterable[Dict[str, Any]]: while True: response = github_api_request('get', url, *args, **kwargs) response.raise_for_status() @@ -64,12 +180,8 @@ def paginate(url, *args, **kwargs) -> Iterable[Dict]: else: return -def prs(repo: str) -> Iterable[Dict]: - url = f'{github_api_url}/repos/{repo}/pulls' - yield from paginate(url, params={'state': 'all'}) - -def find_pr() -> str: +def find_pr(env: GitHubActionsEnv) -> PrUrl: """ Find the pull request this event is related to @@ -78,37 +190,41 @@ def find_pr() -> str: """ - with open(os.environ['GITHUB_EVENT_PATH']) as f: + with open(env['GITHUB_EVENT_PATH']) as f: event = json.load(f) - event_type = os.environ['GITHUB_EVENT_NAME'] + event_type = env['GITHUB_EVENT_NAME'] if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: - return event['pull_request']['url'] + return cast(PrUrl, event['pull_request']['url']) elif event_type == 'issue_comment': if 'pull_request' in event['issue']: - return event['issue']['pull_request']['url'] + return cast(PrUrl, event['issue']['pull_request']['url']) else: - raise Exception(f'This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') + raise Exception('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') elif event_type == 'push': - repo = os.environ['GITHUB_REPOSITORY'] - commit = os.environ['GITHUB_SHA'] + repo = env['GITHUB_REPOSITORY'] + commit = env['GITHUB_SHA'] + + def prs() -> Iterable[Dict[str, Any]]: + url = f'{env.get("GITHUB_API_URL", "https://api.github.com")}/repos/{repo}/pulls' + yield from paginate(cast(PrUrl, url), params={'state': 'all'}) - for pr in prs(repo): + for pr in prs(): if pr['merge_commit_sha'] == commit: - return pr['url'] + return cast(PrUrl, pr['url']) raise Exception(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') else: raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") -def current_user() -> str: - token_hash = hashlib.sha256(os.environ["GITHUB_TOKEN"].encode()).hexdigest() +def current_user(github_env: GitHubActionsEnv) -> str: + token_hash = hashlib.sha256(github_env['GITHUB_TOKEN'].encode()).hexdigest() try: with open(os.path.join(job_tmp_dir, 'token-cache', token_hash)) as f: @@ -118,7 +234,7 @@ def current_user() -> str: except Exception as e: debug(str(e)) - response = github_api_request('get', f'{github_api_url}/user') + response = github_api_request('get', f'{github_env["GITHUB_API_URL"]}/user') if response.status_code != 403: user = response.json() debug('GITHUB_TOKEN user:') @@ -139,295 +255,162 @@ def current_user() -> str: debug(f'GITHUB_TOKEN username: {username}') return username -class TerraformComment: - """ - The GitHub comment for this specific terraform plan - """ - - def __init__(self, pr_url: str=None): - self._plan = None - self._status = None - self._comment_url = None - - if pr_url is None: - return - response = github_api_request('get', pr_url) - response.raise_for_status() - - self._issue_url = response.json()['_links']['issue']['href'] + '/comments' - - username = current_user() +def create_summary(plan) -> Optional[str]: + summary = None - debug('Looking for an existing comment:') + for line in plan.splitlines(): + if line.startswith('No changes') or line.startswith('Error'): + return line - for comment in paginate(self._issue_url): - debug(json.dumps(comment)) - if comment['user']['login'] == username: - match = re.match(rf'{re.escape(self._comment_identifier)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) + if line.startswith('Plan:'): + summary = line - if not match: - match = re.match(rf'{re.escape(self._old_comment_identifier)}\n```(.*?)```.*', comment['body'], re.DOTALL) - - if match: - self._comment_url = comment['url'] - self._plan = match.group(1).strip() - return + if line.startswith('Changes to Outputs'): + if summary: + return summary + ' Changes to Outputs.' + else: + return 'Changes to Outputs' - @property - def _comment_identifier(self): - if self.label: - return f'Terraform plan for __{self.label}__' + return summary - label = f'Terraform plan in __{self.path}__' - if self.workspace != 'default': - label += f' in the __{self.workspace}__ workspace' +def format_body(action_inputs: ActionInputs, plan: Plan, status: Status, collapse_threshold: int) -> str: - if self.backend_config: - label += f'\nWith backend config: `{self.backend_config}`' + details_open = '' + highlighting = '' - if self.backend_config_files: - label += f'\nWith backend config files: `{self.backend_config_files}`' + summary_line = create_summary(plan) - if self.vars: - label += f'\nWith vars: `{self.vars}`' + if plan.startswith('Error'): + details_open = ' open' + elif 'Plan:' in plan: + highlighting = 'hcl' + num_lines = len(plan.splitlines()) + if num_lines < collapse_threshold: + details_open = ' open' - if self.var_files: - label += f'\nWith var files: `{self.var_files}`' + if summary_line is None: + details_open = ' open' - if self.variables: - stripped_vars = self.variables.strip() - if '\n' in stripped_vars: - label += f'''
With variables + body = f'''{plan_identifier(action_inputs)} + +{ f'{summary_line}' if summary_line is not None else '' } -```hcl -{stripped_vars} +```{highlighting} +{plan} ```
''' - else: - label += f'\nWith variables: `{stripped_vars}`' - return label + if status: + body += '\n' + status - @property - def _old_comment_identifier(self): - if self.label: - return f'Terraform plan for __{self.label}__' + return body - label = f'Terraform plan in __{self.path}__' - if self.workspace != 'default': - label += f' in the __{self.workspace}__ workspace' +def update_comment(issue_url: IssueUrl, + comment_url: Optional[CommentUrl], + body: str, + only_if_exists: bool = False) -> Optional[CommentUrl]: + """ + Update (or create) a comment - if self.init_args: - label += f'\nUsing init args: `{self.init_args}`' - if self.plan_args: - label += f'\nUsing plan args: `{self.plan_args}`' + :param issue_url: The url of the issue to create or update the comment in + :param comment_url: The url of the comment to update + :param body: The new comment body + :param only_if_exists: Only update an existing comment - don't create it + """ - return label + debug(body) - @property - def backend_config(self) -> Optional[str]: - if os.environ.get('INPUT_BACKEND_CONFIG') is None: + if comment_url is None: + if only_if_exists: + debug('Comment doesn\'t already exist - not creating it') return None + # Create a new comment + debug('Creating comment') + response = github_api_request('post', issue_url, json={'body': body}) + else: + # Update existing comment + debug('Updating existing comment') + response = github_api_request('patch', comment_url, json={'body': body}) - bad_words = [ - 'token', - 'password', - 'sas_token', - 'access_key', - 'secret_key', - 'client_secret', - 'access_token', - 'http_auth', - 'secret_id', - 'encryption_key', - 'key_material', - 'security_token', - 'conn_str', - 'sse_customer_key', - 'application_credential_secret' - ] - - def has_bad_word(s: str) -> bool: - for bad_word in bad_words: - if bad_word in s: - return True - return False - - clean = [] - - for field in os.environ.get('INPUT_BACKEND_CONFIG', '').split(','): - if not field: - continue - - if not has_bad_word(field): - clean.append(field) - - return ','.join(clean) - - @property - def backend_config_files(self) -> str: - return os.environ.get('INPUT_BACKEND_CONFIG_FILE') - - @property - def variables(self) -> str: - return os.environ.get('INPUT_VARIABLES') - - @property - def vars(self) -> str: - return os.environ.get('INPUT_VAR') - - @property - def var_files(self) -> str: - return os.environ.get('INPUT_VAR_FILE') - - @property - def path(self) -> str: - return os.environ['INPUT_PATH'] - - @property - def workspace(self) -> str: - return os.environ.get('INPUT_WORKSPACE') - - @property - def label(self) -> str: - return os.environ.get('INPUT_LABEL') - - @property - def init_args(self) -> str: - return os.environ['INIT_ARGS'] - - @property - def plan_args(self) -> str: - return os.environ['PLAN_ARGS'] - - @property - def plan(self) -> Optional[str]: - return self._plan - - @plan.setter - def plan(self, plan: str) -> None: - self._plan = plan.strip() - - @property - def status(self) -> Optional[str]: - return self._status - - @status.setter - def status(self, status: str) -> None: - self._status = status.strip() - - def body(self) -> str: - body = f'{self._comment_identifier}\n```hcl\n{self.plan}\n```' - - if self.status: - body += '\n' + self.status - - return body - - def collapsable_body(self) -> str: - - try: - collapse_threshold = int(os.environ['TF_PLAN_COLLAPSE_LENGTH']) - except (ValueError, KeyError): - collapse_threshold = 10 - - open = '' - highlighting = '' - - if self.plan.startswith('Error'): - open = ' open' - elif 'Plan:' in self.plan: - highlighting = 'hcl' - num_lines = len(self.plan.splitlines()) - if num_lines < collapse_threshold: - open = ' open' - - body = f'''{self._comment_identifier} - - {self.summary()} - -```{highlighting} -{self.plan} -``` - -''' - - if self.status: - body += '\n' + self.status + debug(response.content.decode()) + response.raise_for_status() + return cast(CommentUrl, response.json()['url']) - return body - def summary(self) -> str: - summary = None +def find_issue_url(pr_url: str) -> IssueUrl: + response = github_api_request('get', pr_url) + response.raise_for_status() - for line in self.plan.splitlines(): - if line.startswith('No changes') or line.startswith('Error'): - return line + return cast(IssueUrl, response.json()['_links']['issue']['href'] + '/comments') - if line.startswith('Plan:'): - summary = line - if line.startswith('Changes to Outputs'): - if summary: - return summary + ' Changes to Outputs.' - else: - return 'Changes to Outputs' +def find_comment(issue_url: IssueUrl, username: str, action_inputs: ActionInputs) -> Tuple[ + Optional[CommentUrl], Optional[Plan]]: + debug('Looking for an existing comment:') - return summary + plan_id = plan_identifier(action_inputs) - def update_comment(self, only_if_exists=False): - body = self.collapsable_body() - debug(body) + for comment in paginate(issue_url): + debug(json.dumps(comment)) + if comment['user']['login'] == username: + match = re.match(rf'{re.escape(plan_id)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) - if self._comment_url is None: - if only_if_exists: - debug('Comment doesn\'t already exist - not creating it') - return - # Create a new comment - debug('Creating comment') - response = github_api_request('post', self._issue_url, json={'body': body}) - else: - # Update existing comment - debug('Updating existing comment') - response = github_api_request('patch', self._comment_url, json={'body': body}) + if match: + return comment['url'], cast(Plan, match.group(1).strip()) - debug(response.content.decode()) - response.raise_for_status() - self._comment_url = response.json()['url'] + return None, None -if __name__ == '__main__': +def main() -> None: if len(sys.argv) < 2: print(f'''Usage: STATUS="" {sys.argv[0]} plan plan.txt''') - tf_comment = TerraformComment(find_pr()) + action_inputs = cast(ActionInputs, os.environ) + + try: + collapse_threshold = int(os.environ['TF_PLAN_COLLAPSE_LENGTH']) + except (ValueError, KeyError): + collapse_threshold = 10 + + pr_url = find_pr(env) + issue_url = find_issue_url(pr_url) + username = current_user(env) + comment_url, plan = find_comment(issue_url, username, action_inputs) + status = cast(Status, os.environ.get('STATUS', '')) + only_if_exists = False if sys.argv[1] == 'plan': - tf_comment.plan = sys.stdin.read().strip() - tf_comment.status = os.environ['STATUS'] + plan = cast(Plan, sys.stdin.read().strip()) - if os.environ['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', 'true') == 'false': + if action_inputs['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', + 'true') == 'false': only_if_exists = True + body = format_body(action_inputs, plan, status, collapse_threshold) + update_comment(issue_url, comment_url, body, only_if_exists) + elif sys.argv[1] == 'status': - if tf_comment.plan is None: + if plan is None: exit(1) else: - tf_comment.status = os.environ['STATUS'] + body = format_body(action_inputs, plan, status, collapse_threshold) + update_comment(issue_url, comment_url, body, only_if_exists) + elif sys.argv[1] == 'get': - if tf_comment.plan is None: + if plan is None: exit(1) with open(sys.argv[2], 'w') as f: - f.write(tf_comment.plan) - exit(0) + f.write(plan) - tf_comment.update_comment(only_if_exists) + +if __name__ == '__main__': + main() diff --git a/tests/test_pr_comment.py b/tests/test_pr_comment.py index a134fafc..215d3cb6 100644 --- a/tests/test_pr_comment.py +++ b/tests/test_pr_comment.py @@ -1,217 +1,323 @@ -import github_pr_comment -from github_pr_comment import TerraformComment - -import pytest - -def setup_comment(monkeypatch, *, - path ='/test/terraform', - workspace ='default', - backend_config ='', - backend_config_file ='', - var ='', - var_file ='', - label ='', - ): - - monkeypatch.setenv('INPUT_WORKSPACE', workspace) - monkeypatch.setenv('INPUT_PATH', path) - monkeypatch.setattr('github_pr_comment.current_user', lambda: 'github-actions[bot]') - monkeypatch.setenv('INPUT_BACKEND_CONFIG', backend_config) - monkeypatch.setenv('INPUT_BACKEND_CONFIG_FILE', backend_config_file) - monkeypatch.setenv('INPUT_VAR', var) - monkeypatch.setenv('INPUT_VAR_FILE', var_file) - monkeypatch.setenv('INPUT_LABEL', label) - monkeypatch.setenv('INIT_ARGS', '') - monkeypatch.setenv('PLAN_ARGS', '') - - -def test_path_only(monkeypatch): - - setup_comment(monkeypatch, +from github_pr_comment import format_body, ActionInputs, create_summary + +plan = '''An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy.''' + + +def action_inputs(*, path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' + workspace='default', + backend_config='', + backend_config_file='', + variables='', + var='', + var_file='', + label='', ) -> ActionInputs: + return ActionInputs( + INPUT_WORKSPACE=workspace, + INPUT_PATH=path, + INPUT_BACKEND_CONFIG=backend_config, + INPUT_BACKEND_CONFIG_FILE=backend_config_file, + INPUT_VARIABLES=variables, + INPUT_VAR=var, + INPUT_VAR_FILE=var_file, + INPUT_LABEL=label, + INPUT_ADD_GITHUB_COMMENT='true' + ) + + +def test_path_only(): + inputs = action_inputs( + path='/test/terraform' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_nondefault_workspace(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - workspace='myworkspace' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_nondefault_workspace(): + inputs = action_inputs( + path='/test/terraform', + workspace='myworkspace' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ in the __myworkspace__ workspace +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_var(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - var='var1=value' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_variables_single_line(): + inputs = action_inputs( + path='/test/terraform', + variables='var1="value"' + ) + + status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +With variables: `var1="value"` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() + + +def test_variables_multi_line(): + inputs = action_inputs( + path='/test/terraform', + variables='''var1="value" +var2="value2"''' + ) + + status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__
With variables + +```hcl +var1="value" +var2="value2" +``` +
+ +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() + + +def test_var(): + inputs = action_inputs( + path='/test/terraform', + var='var1=value' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ With vars: `var1=value` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_var_file(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - var_file='vars.tf' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_var_file(): + inputs = action_inputs( + path='/test/terraform', + var_file='vars.tf' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ With var files: `vars.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_backend_config(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - backend_config='bucket=test,key=backend' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_backend_config(): + inputs = action_inputs( + path='/test/terraform', + backend_config='bucket=test,key=backend' + ) + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ With backend config: `bucket=test,key=backend` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_backend_config_bad_words(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - backend_config='bucket=test,password=secret,key=backend,token=secret' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_backend_config_bad_words(): + inputs = action_inputs( + path='/test/terraform', + backend_config='bucket=test,password=secret,key=backend,token=secret' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ With backend config: `bucket=test,key=backend` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_backend_config_file(monkeypatch): +def test_backend_config_file(): + inputs = action_inputs( + path='/test/terraform', + backend_config_file='backend.tf' + ) - setup_comment(monkeypatch, - path='/test/terraform', - backend_config_file='backend.tf' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ With backend config files: `backend.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_all(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - workspace='test', - var='myvar=hello', - var_file='vars.tf', - backend_config='bucket=mybucket,password=secret', - backend_config_file='backend.tf' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' +def test_all(): + inputs = action_inputs( + path='/test/terraform', + workspace='test', + var='myvar=hello', + var_file='vars.tf', + backend_config='bucket=mybucket,password=secret', + backend_config_file='backend.tf' + ) + + status = 'Testing' expected = '''Terraform plan in __/test/terraform__ in the __test__ workspace With backend config: `bucket=mybucket` With backend config files: `backend.tf` With vars: `myvar=hello` With var files: `vars.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_label(monkeypatch): +def test_label(): + inputs = action_inputs( + path='/test/terraform', + workspace='test', + var='myvar=hello', + var_file='vars.tf', + backend_config='bucket=mybucket,password=secret', + backend_config_file='backend.tf', + label='test_label' + ) - setup_comment(monkeypatch, - path='/test/terraform', - workspace='test', - var='myvar=hello', - var_file='vars.tf', - backend_config='bucket=mybucket,password=secret', - backend_config_file='backend.tf', - label='test_label' - ) - comment = TerraformComment() - comment.plan = 'Hello, this is my plan' - comment.status = 'Testing' + status = 'Testing' expected = '''Terraform plan for __test_label__ +
+Plan: 1 to add, 0 to change, 0 to destroy. + ```hcl -Hello, this is my plan +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. ``` +
+ Testing''' - assert comment.body() == expected + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() -def test_summary_plan_11(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = '''An execution plan has been generated and is shown below. + +def test_summary_plan_11(): + plan = '''An execution plan has been generated and is shown below. Resource actions are indicated with the following symbols: + create @@ -233,14 +339,11 @@ def test_summary_plan_11(monkeypatch): ''' expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_plan_12(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = '''An execution plan has been generated and is shown below. + +def test_summary_plan_12(): + plan = '''An execution plan has been generated and is shown below. Resource actions are indicated with the following symbols: + create @@ -265,14 +368,11 @@ def test_summary_plan_12(monkeypatch): ''' expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_plan_14(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = '''An execution plan has been generated and is shown below. + +def test_summary_plan_14(): + plan = '''An execution plan has been generated and is shown below. Resource actions are indicated with the following symbols: + create @@ -300,27 +400,21 @@ def test_summary_plan_14(monkeypatch): ''' expected = 'Plan: 1 to add, 0 to change, 0 to destroy. Changes to Outputs.' - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_error_11(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """ + +def test_summary_error_11(): + plan = """ Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax """ expected = "Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing \"ten\": invalid syntax" - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_error_12(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """ + +def test_summary_error_12(): + plan = """ Error: Incorrect attribute value type on main.tf line 2, in resource "random_string" "my_string": @@ -330,15 +424,11 @@ def test_summary_error_12(monkeypatch): """ expected = "Error: Incorrect attribute value type" - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_no_change_11(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """No changes. Infrastructure is up-to-date. +def test_summary_no_change_11(): + plan = """No changes. Infrastructure is up-to-date. This means that Terraform did not detect any differences between your configuration and real physical resources that exist. As a result, no @@ -346,14 +436,11 @@ def test_summary_no_change_11(monkeypatch): """ expected = "No changes. Infrastructure is up-to-date." - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_no_change_14(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """No changes. Infrastructure is up-to-date. + +def test_summary_no_change_14(): + plan = """No changes. Infrastructure is up-to-date. This means that Terraform did not detect any differences between your configuration and real physical resources that exist. As a result, no @@ -361,14 +448,11 @@ def test_summary_no_change_14(monkeypatch): """ expected = "No changes. Infrastructure is up-to-date." - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_output_only_change_14(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """An execution plan has been generated and is shown below. + +def test_summary_output_only_change_14(): + plan = """An execution plan has been generated and is shown below. Resource actions are indicated with the following symbols: Terraform will perform the following actions: @@ -381,17 +465,11 @@ def test_summary_output_only_change_14(monkeypatch): """ expected = "Plan: 0 to add, 0 to change, 0 to destroy. Changes to Outputs." - assert comment.summary() == expected + assert create_summary(plan) == expected -def test_summary_unknown(monkeypatch): - setup_comment(monkeypatch, - path='/test/terraform', - ) - comment = TerraformComment() - comment.plan = """ + +def test_summary_unknown(): + plan = """ This is not anything like terraform output we know. We don't want to generate a summary for this. """ - - expected = None - assert comment.summary() == expected - + assert create_summary(plan) is None From 2ea412e3dcfbab24a2581d2ad1aedc2b7f3467d5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 25 Oct 2021 09:12:34 +0100 Subject: [PATCH 143/231] Cache PR comment info This terraform-apply makes far fewer github API requests --- image/entrypoints/plan.sh | 1 - image/tools/github_pr_comment.py | 118 ++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 28 deletions(-) diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 20afb668..17554735 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -26,7 +26,6 @@ if [[ $PLAN_EXIT -eq 1 ]]; then fi fi -cat "$STEP_TMP_DIR/plan.txt" cat "$STEP_TMP_DIR/terraform_plan.stderr" if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 93c2fd98..4bb508a4 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -19,6 +19,9 @@ class GitHubActionsEnv(TypedDict): + """ + Environment variables that are set by the actions runner + """ GITHUB_API_URL: str GITHUB_TOKEN: str GITHUB_EVENT_PATH: str @@ -28,11 +31,15 @@ class GitHubActionsEnv(TypedDict): job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') +step_tmp_dir = os.environ.get('STEP_TMP_DIR', '.') env = cast(GitHubActionsEnv, os.environ) def github_session(github_env: GitHubActionsEnv) -> requests.Session: + """ + A request session that is configured for the github API + """ session = requests.Session() session.headers['authorization'] = f'token {github_env["GITHUB_TOKEN"]}' session.headers['user-agent'] = 'terraform-github-actions' @@ -45,7 +52,7 @@ def github_session(github_env: GitHubActionsEnv) -> requests.Session: class ActionInputs(TypedDict): """ - Information used to identify a plan + Actions input environment variables that are set by the runner """ INPUT_BACKEND_CONFIG: str INPUT_BACKEND_CONFIG_FILE: str @@ -60,8 +67,6 @@ class ActionInputs(TypedDict): def plan_identifier(action_inputs: ActionInputs) -> str: def mask_backend_config() -> Optional[str]: - if action_inputs.get('INPUT_BACKEND_CONFIG') is None: - return None bad_words = [ 'token', @@ -181,7 +186,7 @@ def paginate(url: GitHubUrl, *args, **kwargs) -> Iterable[Dict[str, Any]]: return -def find_pr(env: GitHubActionsEnv) -> PrUrl: +def find_pr(actions_env: GitHubActionsEnv) -> PrUrl: """ Find the pull request this event is related to @@ -190,10 +195,10 @@ def find_pr(env: GitHubActionsEnv) -> PrUrl: """ - with open(env['GITHUB_EVENT_PATH']) as f: + with open(actions_env['GITHUB_EVENT_PATH']) as f: event = json.load(f) - event_type = env['GITHUB_EVENT_NAME'] + event_type = actions_env['GITHUB_EVENT_NAME'] if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: return cast(PrUrl, event['pull_request']['url']) @@ -206,11 +211,11 @@ def find_pr(env: GitHubActionsEnv) -> PrUrl: raise Exception('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') elif event_type == 'push': - repo = env['GITHUB_REPOSITORY'] - commit = env['GITHUB_SHA'] + repo = actions_env['GITHUB_REPOSITORY'] + commit = actions_env['GITHUB_SHA'] def prs() -> Iterable[Dict[str, Any]]: - url = f'{env.get("GITHUB_API_URL", "https://api.github.com")}/repos/{repo}/pulls' + url = f'{actions_env["GITHUB_API_URL"]}/repos/{repo}/pulls' yield from paginate(cast(PrUrl, url), params={'state': 'all'}) for pr in prs(): @@ -223,21 +228,20 @@ def prs() -> Iterable[Dict[str, Any]]: raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") -def current_user(github_env: GitHubActionsEnv) -> str: - token_hash = hashlib.sha256(github_env['GITHUB_TOKEN'].encode()).hexdigest() +def current_user(actions_env: GitHubActionsEnv) -> str: + token_hash = hashlib.sha256(actions_env['GITHUB_TOKEN'].encode()).hexdigest() try: with open(os.path.join(job_tmp_dir, 'token-cache', token_hash)) as f: username = f.read() - debug(f'GITHUB_TOKEN username: {username}') + debug(f'GITHUB_TOKEN username from token-cache: {username}') return username except Exception as e: debug(str(e)) - response = github_api_request('get', f'{github_env["GITHUB_API_URL"]}/user') + response = github_api_request('get', f'{actions_env["GITHUB_API_URL"]}/user') if response.status_code != 403: user = response.json() - debug('GITHUB_TOKEN user:') debug(json.dumps(user)) username = user['login'] @@ -252,7 +256,7 @@ def current_user(github_env: GitHubActionsEnv) -> str: except Exception as e: debug(str(e)) - debug(f'GITHUB_TOKEN username: {username}') + debug(f'discovered GITHUB_TOKEN username: {username}') return username @@ -322,8 +326,6 @@ def update_comment(issue_url: IssueUrl, :param only_if_exists: Only update an existing comment - don't create it """ - debug(body) - if comment_url is None: if only_if_exists: debug('Comment doesn\'t already exist - not creating it') @@ -336,20 +338,40 @@ def update_comment(issue_url: IssueUrl, debug('Updating existing comment') response = github_api_request('patch', comment_url, json={'body': body}) + debug(body) debug(response.content.decode()) response.raise_for_status() return cast(CommentUrl, response.json()['url']) def find_issue_url(pr_url: str) -> IssueUrl: + pr_hash = hashlib.sha256(pr_url.encode()).hexdigest() + + try: + with open(os.path.join(job_tmp_dir, 'issue-href-cache', pr_hash)) as f: + issue_url = f.read() + debug(f'issue_url from issue-href-cache: {issue_url}') + return cast(IssueUrl, issue_url) + except Exception as e: + debug(str(e)) + response = github_api_request('get', pr_url) response.raise_for_status() - return cast(IssueUrl, response.json()['_links']['issue']['href'] + '/comments') + issue_url = cast(IssueUrl, response.json()['_links']['issue']['href'] + '/comments') + + try: + os.makedirs(os.path.join(job_tmp_dir, 'issue-href-cache'), exist_ok=True) + with open(os.path.join(job_tmp_dir, 'issue-href-cache', pr_hash), 'w') as f: + f.write(issue_url) + except Exception as e: + debug(str(e)) + + debug(f'discovered issue_url: {issue_url}') + return cast(IssueUrl, issue_url) -def find_comment(issue_url: IssueUrl, username: str, action_inputs: ActionInputs) -> Tuple[ - Optional[CommentUrl], Optional[Plan]]: +def find_comment(issue_url: IssueUrl, username: str, action_inputs: ActionInputs) -> Tuple[Optional[CommentUrl], Optional[Plan]]: debug('Looking for an existing comment:') plan_id = plan_identifier(action_inputs) @@ -364,6 +386,22 @@ def find_comment(issue_url: IssueUrl, username: str, action_inputs: ActionInputs return None, None +def read_step_cache() -> Dict[str, str]: + try: + with open(os.path.join(step_tmp_dir, 'github_pr_comment.cache')) as f: + debug('step cache loaded') + return json.load(f) + except Exception as e: + debug(str(e)) + return {} + +def save_step_cache(**kwargs) -> None: + try: + with open(os.path.join(step_tmp_dir, 'github_pr_comment.cache'), 'w') as f: + json.dump(kwargs, f) + debug('step cache saved') + except Exception as e: + debug(str(e)) def main() -> None: if len(sys.argv) < 2: @@ -372,6 +410,8 @@ def main() -> None: STATUS="" {sys.argv[0]} status {sys.argv[0]} get >plan.txt''') + debug(repr(sys.argv)) + action_inputs = cast(ActionInputs, os.environ) try: @@ -379,10 +419,34 @@ def main() -> None: except (ValueError, KeyError): collapse_threshold = 10 - pr_url = find_pr(env) - issue_url = find_issue_url(pr_url) + step_cache = read_step_cache() + + if step_cache.get('pr_url') is not None: + pr_url = step_cache['pr_url'] + debug(f'pr_url from step cache: {pr_url}') + else: + pr_url = find_pr(env) + debug(f'discovered pr_url: {pr_url}') + + if step_cache.get('pr_url') == pr_url and step_cache.get('issue_url') is not None: + issue_url = step_cache['issue_url'] + debug(f'issue_url from step cache: {issue_url}') + else: + issue_url = find_issue_url(pr_url) + + # Username is cached in the job tmp dir username = current_user(env) - comment_url, plan = find_comment(issue_url, username, action_inputs) + + if step_cache.get('comment_url') is not None and step_cache.get('plan') is not None: + comment_url = step_cache['comment_url'] + plan = step_cache['plan'] + debug(f'comment_url from step cache: {comment_url}') + debug(f'plan from step cache: {plan}') + else: + comment_url, plan = find_comment(issue_url, username, action_inputs) + debug(f'discovered comment_url: {comment_url}') + debug(f'discovered plan: {plan}') + status = cast(Status, os.environ.get('STATUS', '')) only_if_exists = False @@ -390,19 +454,18 @@ def main() -> None: if sys.argv[1] == 'plan': plan = cast(Plan, sys.stdin.read().strip()) - if action_inputs['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', - 'true') == 'false': + if action_inputs['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', 'true') == 'false': only_if_exists = True body = format_body(action_inputs, plan, status, collapse_threshold) - update_comment(issue_url, comment_url, body, only_if_exists) + comment_url = update_comment(issue_url, comment_url, body, only_if_exists) elif sys.argv[1] == 'status': if plan is None: exit(1) else: body = format_body(action_inputs, plan, status, collapse_threshold) - update_comment(issue_url, comment_url, body, only_if_exists) + comment_url = update_comment(issue_url, comment_url, body, only_if_exists) elif sys.argv[1] == 'get': if plan is None: @@ -411,6 +474,7 @@ def main() -> None: with open(sys.argv[2], 'w') as f: f.write(plan) + save_step_cache(pr_url=pr_url, issue_url=issue_url, comment_url=comment_url, plan=plan) if __name__ == '__main__': main() From 4d25c4245014bb9b9d835f654810bfdcf846a8b2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 29 Oct 2021 00:28:53 +0100 Subject: [PATCH 144/231] Add tests for 1.0.10 workspace changes --- .github/workflows/test-new-workspace.yaml | 84 +++++++++++++++++++++++ tests/new-workspace/remote_1_0_10/main.tf | 21 ++++++ 2 files changed, 105 insertions(+) create mode 100644 tests/new-workspace/remote_1_0_10/main.tf diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index 43634562..48639709 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -254,3 +254,87 @@ jobs: path: tests/new-workspace/remote_14 workspace: ${{ github.head_ref }} variables: my_string="world" + + create_workspace_1_0_10: + runs-on: ubuntu-latest + name: Workspace tests 1.0.10 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create first workspace + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_1_0_10 + workspace: test-workspace + + - name: Create first workspace again + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_1_0_10 + workspace: test-workspace + + - name: Apply in first workspace + uses: ./terraform-apply + with: + path: tests/new-workspace/remote_1_0_10 + workspace: test-workspace + variables: my_string="hello" + auto_approve: true + + - name: Create a second workspace + uses: ./terraform-new-workspace + with: + path: tests/new-workspace/remote_1_0_10 + workspace: ${{ github.head_ref }} + + - name: Apply in second workspace + uses: ./terraform-apply + with: + path: tests/new-workspace/remote_1_0_10 + workspace: ${{ github.head_ref }} + variables: my_string="world" + auto_approve: true + + - name: Get first workspace outputs + uses: ./terraform-output + id: first_1_0_10 + with: + path: tests/new-workspace/remote_1_0_10 + workspace: test-workspace + + - name: Get second workspace outputs + uses: ./terraform-output + id: second_1_0_10 + with: + path: tests/new-workspace/remote_1_0_10 + workspace: ${{ github.head_ref }} + + - name: Verify outputs + run: | + if [[ "${{ steps.first_1_0_10.outputs.my_string }}" != "hello" ]]; then + echo "::error:: output my_string not set correctly for first workspace" + exit 1 + fi + + if [[ "${{ steps.second_1_0_10.outputs.my_string }}" != "world" ]]; then + echo "::error:: output my_string not set correctly for second workspace" + exit 1 + fi + + - name: Destroy first workspace + uses: ./terraform-destroy-workspace + with: + path: tests/new-workspace/remote_1_0_10 + workspace: test-workspace + variables: my_string="hello" + + - name: Destroy second workspace + uses: ./terraform-destroy-workspace + with: + path: tests/new-workspace/remote_1_0_10 + workspace: ${{ github.head_ref }} + variables: my_string="world" diff --git a/tests/new-workspace/remote_1_0_10/main.tf b/tests/new-workspace/remote_1_0_10/main.tf new file mode 100644 index 00000000..20d23a9f --- /dev/null +++ b/tests/new-workspace/remote_1_0_10/main.tf @@ -0,0 +1,21 @@ +resource "random_string" "my_string" { + length = 5 +} + +variable "my_string" { + type = string +} + +output "my_string" { + value = var.my_string +} + +terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "terraform-new-workspace-1-0-10" + region = "eu-west-2" + } + + required_version = "~> 1.0.10" +} From 9ac3689c5bb32b93cbc202e54c774a2b0e225c85 Mon Sep 17 00:00:00 2001 From: Kyle Lacy Date: Thu, 28 Oct 2021 15:52:32 -0700 Subject: [PATCH 145/231] Update Terraform "workspace does not exist" regex Terraform 1.0.10 changed the error message returned when called with the `-input=false` flag is passed. Now, it will return an error message like the following: ``` Error: Currently selected workspace "foo" does not exist ``` --- image/actions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index ea71003b..f2fe56af 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -164,7 +164,7 @@ function init-backend() { if [[ $INIT_EXIT -eq 0 ]]; then cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 else - if grep -q "No existing workspaces." "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Failed to select workspace" "$STEP_TMP_DIR/terraform_init.stderr"; then + if grep -q "No existing workspaces." "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Failed to select workspace" "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Currently selected workspace.*does not exist" "$STEP_TMP_DIR/terraform_init.stderr"; then # Couldn't select workspace, but we don't really care. # select-workspace will give a better error if the workspace is required to exist : From 86a7aa56cb89985e6e7eda8eac7e638412e2c82c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 29 Oct 2021 00:57:14 +0100 Subject: [PATCH 146/231] :bookmark: v1.17.3 --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b2b7c4..15d9f759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.17.2` to use an exact release +- `@v1.17.3` to use an exact release - `@v1.17` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.17.3] - 2021-10-29 + +### Fixed +- [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) compatability with Terraform v1.0.10 - Thanks [kylewlacy ](https://github.com/kylewlacy )! +- Now makes even fewer github api requests to avoid rate limiting. + ## [1.17.2] - 2021-10-13 ### Fixed @@ -265,6 +271,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.17.3]: https://github.com/dflook/terraform-github-actions/compare/v1.17.2...v1.17.3 [1.17.2]: https://github.com/dflook/terraform-github-actions/compare/v1.17.1...v1.17.2 [1.17.1]: https://github.com/dflook/terraform-github-actions/compare/v1.17.0...v1.17.1 [1.17.0]: https://github.com/dflook/terraform-github-actions/compare/v1.16.0...v1.17.0 From e78ad9ca9a4f8903d3197d64b749dbb6b7f512d7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 29 Oct 2021 01:09:31 +0100 Subject: [PATCH 147/231] :pencil: Reword --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d9f759..549e1b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ When using an action you can specify the version as: ## [1.17.3] - 2021-10-29 ### Fixed -- [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) compatability with Terraform v1.0.10 - Thanks [kylewlacy ](https://github.com/kylewlacy )! +- Compatability with Terraform v1.0.10 - Thanks [kylewlacy](https://github.com/kylewlacy)! - Now makes even fewer github api requests to avoid rate limiting. ## [1.17.2] - 2021-10-13 From 20cde72cdb85b72ce94049c6f78f4cc724451d0e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 18 Oct 2021 22:21:36 +0100 Subject: [PATCH 148/231] Add target to plan, add replace to plan & apply --- .github/workflows/test-target-replace.yaml | 215 +++++++++++++++++++++ image/actions.sh | 38 +++- image/entrypoints/apply.sh | 8 +- image/tools/github_pr_comment.py | 10 + terraform-apply/README.md | 17 +- terraform-apply/action.yaml | 6 +- terraform-plan/README.md | 32 ++- terraform-plan/action.yaml | 8 + tests/target/main.tf | 23 +++ tests/test_pr_comment.py | 66 ++++++- 10 files changed, 405 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/test-target-replace.yaml create mode 100644 tests/target/main.tf diff --git a/.github/workflows/test-target-replace.yaml b/.github/workflows/test-target-replace.yaml new file mode 100644 index 00000000..df6a8632 --- /dev/null +++ b/.github/workflows/test-target-replace.yaml @@ -0,0 +1,215 @@ +name: Test plan target and replace + +on: [pull_request] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + plan_targeting: + runs-on: ubuntu-latest + name: Plan targeting + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Plan with no changes in targets + uses: ./terraform-plan + id: plan + with: + label: No targeted changes + path: tests/target + target: | + random_string.notpresent + variables: | + length = 5 + + - name: Verify outputs + run: | + if [[ "${{ steps.plan.outputs.changes }}" != "false" ]]; then + echo "::error:: Should not be any changes with this targeted plan" + exit 1 + fi + + - name: Plan targeted change + uses: ./terraform-plan + id: plan-first-change + with: + path: tests/target + target: | + random_string.count[0] + variables: | + length = 5 + + - name: Verify outputs + run: | + if [[ "${{ steps.plan-first-change.outputs.changes }}" != "true" ]]; then + echo "::error:: Targeted plan should have changes" + exit 1 + fi + + - name: Apply targeted change + uses: ./terraform-apply + id: apply-first-change + with: + path: tests/target + target: | + random_string.count[0] + variables: | + length = 5 + + - name: Verify outputs + run: | + if [[ "${{ steps.apply-first-change.outputs.count }}" == "" ]]; then + echo "::error:: output count not set correctly" + exit 1 + fi + + - name: Plan targeted change + uses: ./terraform-plan + id: plan-second-change + with: + path: tests/target + target: | + random_string.foreach["hello"] + variables: | + length = 6 + + - name: Verify outputs + run: | + if [[ "${{ steps.plan-second-change.outputs.changes }}" != "true" ]]; then + echo "::error:: Targeted plan should have changes" + exit 1 + fi + + - name: Apply targeted change + uses: ./terraform-apply + id: apply-second-change + with: + path: tests/target + target: | + random_string.foreach["hello"] + variables: | + length = 6 + + - name: Verify outputs + run: | + if [[ "${{ steps.apply-second-change.outputs.foreach }}" == "" ]]; then + echo "::error:: output foreach not set correctly" + exit 1 + fi + + if [[ "${{ steps.apply-second-change.outputs.count }}" != "${{ steps.apply-first-change.outputs.count }}" ]]; then + echo "::error:: Targeted change has affected untargeted resources" + exit 1 + fi + + - name: Auto Apply targeted change + uses: ./terraform-apply + id: apply-third-change + with: + path: tests/target + target: | + random_string.count[0] + random_string.foreach["hello"] + variables: | + length = 10 + auto_approve: true + + - name: Verify outputs + run: | + if [[ "${{ steps.apply-third-change.outputs.count }}" == "${{ steps.apply-second-change.outputs.count }}" ]]; then + echo "::error:: Targeted change has not affected targeted resources" + exit 1 + fi + + if [[ "${{ steps.apply-third-change.outputs.foreach }}" == "${{ steps.apply-second-change.outputs.foreach }}" ]]; then + echo "::error:: Targeted change has not affected targeted resources" + exit 1 + fi + + - name: Plan targeted replacement + uses: ./terraform-plan + id: plan-targeted-replacement + with: + path: tests/target + target: | + random_string.foreach["hello"] + replace: | + random_string.foreach["hello"] + random_string.count[0] + variables: | + length = 10 + + - name: Verify outputs + run: | + if [[ "${{ steps.plan-targeted-replacement.outputs.changes }}" != "true" ]]; then + echo "::error:: Targeted replacement should have changes" + exit 1 + fi + + - name: Apply targeted replacement + uses: ./terraform-apply + id: apply-targeted-replacement + with: + path: tests/target + target: | + random_string.foreach["hello"] + replace: | + random_string.foreach["hello"] + random_string.count[0] + variables: | + length = 10 + + - name: Verify outputs + run: | + if [[ "${{ steps.apply-targeted-replacement.outputs.count }}" != "${{ steps.apply-third-change.outputs.count }}" ]]; then + echo "::error:: Targeted replacement has affected non targeted resources" + exit 1 + fi + + if [[ "${{ steps.apply-targeted-replacement.outputs.foreach }}" == "${{ steps.apply-third-change.outputs.foreach }}" ]]; then + echo "::error:: Targeted replacement has not affected targeted resources" + exit 1 + fi + + - name: Plan replacement + uses: ./terraform-plan + id: plan-replacement + with: + path: tests/target + replace: | + random_string.foreach["hello"] + random_string.count[0] + variables: | + length = 10 + + - name: Verify outputs + run: | + if [[ "${{ steps.plan-replacement.outputs.changes }}" != "true" ]]; then + echo "::error:: Replacement should have changes" + exit 1 + fi + + - name: Apply replacement + uses: ./terraform-apply + id: apply-replacement + with: + path: tests/target + replace: | + random_string.foreach["hello"] + random_string.count[0] + variables: | + length = 10 + + - name: Verify outputs + run: | + if [[ "${{ steps.apply-replacement.outputs.count }}" == "${{ steps.apply-targeted-replacement.outputs.count }}" ]]; then + echo "::error:: Replacement has not affected targeted resources" + exit 1 + fi + + if [[ "${{ steps.apply-replacement.outputs.foreach }}" == "${{ steps.apply-targeted-replacement.outputs.foreach }}" ]]; then + echo "::error:: Replacement has not affected targeted resources" + exit 1 + fi diff --git a/image/actions.sh b/image/actions.sh index f2fe56af..25e041dc 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -40,6 +40,13 @@ function detect-terraform-version() { debug_log "Terraform version major $TERRAFORM_VER_MAJOR minor $TERRAFORM_VER_MINOR patch $TERRAFORM_VER_PATCH" } +function test-terraform-version() { + local OP="$1" + local VER="$2" + + python3 -c "exit(0 if ($TERRAFORM_VER_MAJOR, $TERRAFORM_VER_MINOR, $TERRAFORM_VER_PATCH) $OP tuple(int(v) for v in '$VER'.split('.')) else 1)" +} + function job_markdown_ref() { echo "[${GITHUB_WORKFLOW} #${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" } @@ -187,13 +194,34 @@ function select-workspace() { fi } -function set-plan-args() { +function set-common-plan-args() { PLAN_ARGS="" if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" fi + if [[ -v INPUT_TARGET ]]; then + if [[ -n "$INPUT_TARGET" ]]; then + for target in $(echo "$INPUT_TARGET" | tr ',' '\n'); do + PLAN_ARGS="$PLAN_ARGS -target $target" + done + fi + fi + + if [[ -v INPUT_REPLACE ]]; then + if [[ -n "$INPUT_REPLACE" ]]; then + for target in $(echo "$INPUT_REPLACE" | tr ',' '\n'); do + PLAN_ARGS="$PLAN_ARGS -replace $target" + done + fi + fi +} + + +function set-plan-args() { + set-common-plan-args + if [[ -n "$INPUT_VAR" ]]; then for var in $(echo "$INPUT_VAR" | tr ',' '\n'); do PLAN_ARGS="$PLAN_ARGS -var $var" @@ -215,11 +243,7 @@ function set-plan-args() { } function set-remote-plan-args() { - PLAN_ARGS="" - - if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then - PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" - fi + set-common-plan-args local AUTO_TFVARS_COUNTER=0 @@ -282,6 +306,8 @@ function plan() { PLAN_OUT_ARG="" fi + debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS + set +e # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index d5a6e01d..64ee2c7e 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -11,12 +11,6 @@ set-plan-args PLAN_OUT="$STEP_TMP_DIR/plan.out" -if [[ "$INPUT_AUTO_APPROVE" == "true" && -n "$INPUT_TARGET" ]]; then - for target in $(echo "$INPUT_TARGET" | tr ',' '\n'); do - PLAN_ARGS="$PLAN_ARGS -target $target" - done -fi - if [[ -v GITHUB_TOKEN ]]; then update_status "Applying plan in $(job_markdown_ref)" fi @@ -25,6 +19,8 @@ exec 3>&1 function apply() { + debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT + set +e # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT) | $TFMASK diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 4bb508a4..80ae0cd5 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -63,6 +63,8 @@ class ActionInputs(TypedDict): INPUT_WORKSPACE: str INPUT_LABEL: str INPUT_ADD_GITHUB_COMMENT: str + INPUT_TARGET: str + INPUT_REPLACE: str def plan_identifier(action_inputs: ActionInputs) -> str: @@ -111,6 +113,14 @@ def has_bad_word(s: str) -> bool: if action_inputs["INPUT_WORKSPACE"] != 'default': label += f' in the __{action_inputs["INPUT_WORKSPACE"]}__ workspace' + if action_inputs["INPUT_TARGET"]: + label += '\nTargeting resources: ' + label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_TARGET'].splitlines()) + + if action_inputs["INPUT_REPLACE"]: + label += '\nReplacing resources: ' + label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_REPLACE'].splitlines()) + backend_config = mask_backend_config() if backend_config: label += f'\nWith backend config: `{backend_config}`' diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 32be18f4..86497785 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -129,11 +129,26 @@ These input values must be the same as any `terraform-plan` for the same configu - Optional - Default: 10 +* `replace` + + List of resources to replace if any update to them is required. + + Only available with supported terraform versions (v0.15.2 onwards). + + ```yaml + with: + replace: | + kubernetes_secret.tls_cert_public + kubernetes_secret.tls_cert_private + ``` + + - Type: string + - Optional + * `target` List of resources to apply, one per line. The apply operation will be limited to these resources and their dependencies. - This only takes effect if auto_approve is also set to `true`. ```yaml with: diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index b9551419..f2558a43 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -43,7 +43,11 @@ inputs: description: Automatically approve and apply plan default: false target: - description: List of targets to apply against, one per line + description: List of resources to target for the apply, one per line + required: false + default: "" + replace: + description: List of resources to replace if an update is required, one per line required: false default: "" diff --git a/terraform-plan/README.md b/terraform-plan/README.md index e739be60..c43d7d26 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -9,7 +9,7 @@ If the triggering event relates to a PR it will add a comment on the PR containi

-The `GITHUB_TOKEN` environment variable is must be set for the PR comment to be added. +The `GITHUB_TOKEN` environment variable must be set for the PR comment to be added. The action can be run on other events, which prints the plan to the workflow log. The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) action can be used to apply the generated plan. @@ -114,6 +114,36 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Optional - Default: 10 +* `replace` + + List of resources to replace if any update to them is required, one per line. + + Only available with terraform versions that support replace (v0.15.2 onwards). + + ```yaml + with: + replace: | + random_password.database + ``` + + - Type: string + - Optional + +* `target` + + List of resources to apply, one per line. + The plan will be limited to these resources and their dependencies. + + ```yaml + with: + target: | + kubernetes_secret.tls_cert_public + kubernetes_secret.tls_cert_private + ``` + + - Type: string + - Optional + * `add_github_comment` The default is `true`, which adds a comment to the PR with the results of the plan. diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 99a56c7b..c55da9c1 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -31,6 +31,14 @@ inputs: description: Limit the number of concurrent operations required: false default: 0 + target: + description: List of resources to target for the plan, one per line + required: false + default: "" + replace: + description: List of resources to replace if an update is required, one per line + required: false + default: "" label: description: A friendly name for this plan required: false diff --git a/tests/target/main.tf b/tests/target/main.tf new file mode 100644 index 00000000..98b2d21f --- /dev/null +++ b/tests/target/main.tf @@ -0,0 +1,23 @@ +resource "random_string" "count" { + count = 1 + + length = var.length +} + +resource "random_string" "foreach" { + for_each = toset(["hello"]) + + length = var.length +} + +variable "length" { + +} + +output "count" { + value = random_string.count[0].result +} + +output "foreach" { + value = random_string.foreach["hello"].result +} diff --git a/tests/test_pr_comment.py b/tests/test_pr_comment.py index 215d3cb6..f49ce1bd 100644 --- a/tests/test_pr_comment.py +++ b/tests/test_pr_comment.py @@ -13,7 +13,10 @@ def action_inputs(*, variables='', var='', var_file='', - label='', ) -> ActionInputs: + label='', + target='', + replace='' + ) -> ActionInputs: return ActionInputs( INPUT_WORKSPACE=workspace, INPUT_PATH=path, @@ -23,7 +26,9 @@ def action_inputs(*, INPUT_VAR=var, INPUT_VAR_FILE=var_file, INPUT_LABEL=label, - INPUT_ADD_GITHUB_COMMENT='true' + INPUT_ADD_GITHUB_COMMENT='true', + INPUT_TARGET=target, + INPUT_REPLACE=replace ) @@ -229,6 +234,55 @@ def test_backend_config_bad_words(): assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() +def test_target(): + inputs = action_inputs( + path='/test/terraform', + target='''kubernetes_secret.tls_cert_public[0] +kubernetes_secret.tls_cert_private''' + ) + + status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() + +def test_replace(): + inputs = action_inputs( + path='/test/terraform', + replace='''kubernetes_secret.tls_cert_public[0] +kubernetes_secret.tls_cert_private''' + ) + + status = 'Testing' + + expected = '''Terraform plan in __/test/terraform__ +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() def test_backend_config_file(): inputs = action_inputs( @@ -262,12 +316,18 @@ def test_all(): var='myvar=hello', var_file='vars.tf', backend_config='bucket=mybucket,password=secret', - backend_config_file='backend.tf' + backend_config_file='backend.tf', + target = '''kubernetes_secret.tls_cert_public[0] +kubernetes_secret.tls_cert_private''', + replace='''kubernetes_secret.tls_cert_public[0] +kubernetes_secret.tls_cert_private''' ) status = 'Testing' expected = '''Terraform plan in __/test/terraform__ in the __test__ workspace +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` With backend config: `bucket=mybucket` With backend config files: `backend.tf` With vars: `myvar=hello` From 99e73eee0807c23ff671596831b1f3cc4ff0f0fe Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 28 Oct 2021 23:48:15 +0100 Subject: [PATCH 149/231] Use plan args for remote applies The remote backend can't save plans, so if we are doing a remote apply we need to use the arguments we would normally give to the plan. Breaks out the parallel flag from plan args for use with all plan/apply/destroy commands --- image/actions.sh | 7 ++++--- image/entrypoints/apply.sh | 20 +++++++++++++++----- image/entrypoints/destroy-workspace.sh | 4 +++- image/entrypoints/destroy.sh | 4 +++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 25e041dc..ed7f2071 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -196,9 +196,10 @@ function select-workspace() { function set-common-plan-args() { PLAN_ARGS="" + PARALLEL_ARG="" if [[ "$INPUT_PARALLELISM" -ne 0 ]]; then - PLAN_ARGS="$PLAN_ARGS -parallelism=$INPUT_PARALLELISM" + PARALLEL_ARG="-parallelism=$INPUT_PARALLELISM" fi if [[ -v INPUT_TARGET ]]; then @@ -306,11 +307,11 @@ function plan() { PLAN_OUT_ARG="" fi - debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS + debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG $PLAN_ARGS set +e # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PLAN_OUT_ARG $PLAN_ARGS) \ + (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG $PLAN_ARGS) \ 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ | $TFMASK \ | tee /dev/fd/3 \ diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 64ee2c7e..01be9c25 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -18,13 +18,23 @@ fi exec 3>&1 function apply() { - - debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT + local APPLY_EXIT set +e - # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PLAN_OUT) | $TFMASK - local APPLY_EXIT=${PIPESTATUS[0]} + if [[ -n "$PLAN_OUT" ]]; then + debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT) | $TFMASK + APPLY_EXIT=${PIPESTATUS[0]} + else + # There is no plan file to apply, since the remote backend can't produce them. + # Instead we need to do an auto approved apply using the arguments we would normally use for the plan + + debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK + APPLY_EXIT=${PIPESTATUS[0]} + fi set -e if [[ $APPLY_EXIT -eq 0 ]]; then diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index 594eb612..d7d43e80 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -9,8 +9,10 @@ init-backend select-workspace set-plan-args +debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS + # shellcheck disable=SC2086 -if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then +if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS); then set_output failure-reason destroy-failed exit 1 fi diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index 57d921db..8878d059 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -9,8 +9,10 @@ init-backend select-workspace set-plan-args +debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS + # shellcheck disable=SC2086 -if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PLAN_ARGS); then +if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS); then set_output failure-reason destroy-failed exit 1 fi From 93f7a741fc072642c666bcd17c5445cf97a0af50 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 29 Oct 2021 17:33:15 +0100 Subject: [PATCH 150/231] Add remote targeting tests --- .github/workflows/test-target-replace.yaml | 258 +++++++++++++++++++++ 1 file changed, 258 insertions(+) diff --git a/.github/workflows/test-target-replace.yaml b/.github/workflows/test-target-replace.yaml index df6a8632..7a987fd4 100644 --- a/.github/workflows/test-target-replace.yaml +++ b/.github/workflows/test-target-replace.yaml @@ -213,3 +213,261 @@ jobs: echo "::error:: Replacement has not affected targeted resources" exit 1 fi + + remote_plan_targeting: + runs-on: ubuntu-latest + name: Remote Plan targeting + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup remote backend + run: | + cat >tests/target/backend.tf < Date: Fri, 29 Oct 2021 23:08:13 +0100 Subject: [PATCH 151/231] Support variables in remote destroy operations --- image/actions.sh | 13 +++++++++++++ image/entrypoints/destroy-workspace.sh | 14 +++++++++++--- image/entrypoints/destroy.sh | 14 +++++++++++--- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index ed7f2071..6e8f9258 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -322,6 +322,19 @@ function plan() { set -e } +function destroy() { + debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS + + set +e + (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \ + 2>"$STEP_TMP_DIR/terraform_destroy.stderr" \ + | tee /dev/fd/3 \ + >"$STEP_TMP_DIR/terraform_destroy.stdout" + + DESTROY_EXIT=${PIPESTATUS[0]} + set -e +} + # Every file written to disk should use one of these directories readonly STEP_TMP_DIR="/tmp" readonly JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index d7d43e80..687537a8 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -9,10 +9,18 @@ init-backend select-workspace set-plan-args -debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS +exec 3>&1 -# shellcheck disable=SC2086 -if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS); then +destroy + +if [[ $DESTROY_EXIT -eq 1 ]]; then + if grep -q "Run variables are currently not supported" "$STEP_TMP_DIR/terraform_destroy.stderr"; then + set-remote-plan-args + destroy + fi +fi + +if [[ $DESTROY_EXIT -eq 1 ]]; then set_output failure-reason destroy-failed exit 1 fi diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index 8878d059..f91be7ed 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -9,10 +9,18 @@ init-backend select-workspace set-plan-args -debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS +exec 3>&1 -# shellcheck disable=SC2086 -if ! (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS); then +destroy + +if [[ $DESTROY_EXIT -eq 1 ]]; then + if grep -q "Run variables are currently not supported" "$STEP_TMP_DIR/terraform_destroy.stderr"; then + set-remote-plan-args + destroy + fi +fi + +if [[ $DESTROY_EXIT -eq 1 ]]; then set_output failure-reason destroy-failed exit 1 fi From 9455c36409590b0b22cabd37848bda390d6bef02 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 29 Oct 2021 23:24:43 +0100 Subject: [PATCH 152/231] Send captured errors to stderr --- image/entrypoints/apply.sh | 2 +- image/entrypoints/check.sh | 1 + image/entrypoints/destroy-workspace.sh | 1 + image/entrypoints/destroy.sh | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 01be9c25..ef762c8d 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -65,7 +65,7 @@ if [[ $PLAN_EXIT -eq 1 ]]; then fi if [[ $PLAN_EXIT -eq 1 ]]; then - cat "$STEP_TMP_DIR/terraform_plan.stderr" + cat >&2 "$STEP_TMP_DIR/terraform_plan.stderr" update_status "Error applying plan in $(job_markdown_ref)" exit 1 diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index 07b0aa6c..e971612e 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -26,6 +26,7 @@ fi if [[ $PLAN_EXIT -eq 1 ]]; then echo "Error running terraform" + cat >&2 "$STEP_TMP_DIR/terraform_plan.stderr" exit 1 elif [[ $PLAN_EXIT -eq 2 ]]; then diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index 687537a8..c417073b 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -21,6 +21,7 @@ if [[ $DESTROY_EXIT -eq 1 ]]; then fi if [[ $DESTROY_EXIT -eq 1 ]]; then + cat >&2 "$STEP_TMP_DIR/terraform_destroy.stderr" set_output failure-reason destroy-failed exit 1 fi diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index f91be7ed..860bab64 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -21,6 +21,7 @@ if [[ $DESTROY_EXIT -eq 1 ]]; then fi if [[ $DESTROY_EXIT -eq 1 ]]; then + cat >&2 "$STEP_TMP_DIR/terraform_destroy.stderr" set_output failure-reason destroy-failed exit 1 fi From ab4a5b3fce8d2b3623c3eeaea7ce34cfaef03c55 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 11:14:22 +0100 Subject: [PATCH 153/231] shellcheck --- image/actions.sh | 12 +++++++++--- image/entrypoints/apply.sh | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 6e8f9258..c96de49e 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -307,6 +307,7 @@ function plan() { PLAN_OUT_ARG="" fi + # shellcheck disable=SC2086 debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG $PLAN_ARGS set +e @@ -318,27 +319,32 @@ function plan() { | compact_plan \ >"$STEP_TMP_DIR/plan.txt" + # shellcheck disable=SC2034 PLAN_EXIT=${PIPESTATUS[0]} set -e } function destroy() { + # shellcheck disable=SC2086 debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS set +e + # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \ 2>"$STEP_TMP_DIR/terraform_destroy.stderr" \ | tee /dev/fd/3 \ >"$STEP_TMP_DIR/terraform_destroy.stdout" + # shellcheck disable=SC2034 DESTROY_EXIT=${PIPESTATUS[0]} set -e } # Every file written to disk should use one of these directories -readonly STEP_TMP_DIR="/tmp" -readonly JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" -readonly WORKSPACE_TMP_DIR=".dflook-terraform-github-actions/$(random_string)" +STEP_TMP_DIR="/tmp" +JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions" +WORKSPACE_TMP_DIR=".dflook-terraform-github-actions/$(random_string)" +readonly STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR export STEP_TMP_DIR JOB_TMP_DIR WORKSPACE_TMP_DIR function fix_owners() { diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index ef762c8d..86ca32a1 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -22,6 +22,7 @@ function apply() { set +e if [[ -n "$PLAN_OUT" ]]; then + # shellcheck disable=SC2086 debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT) | $TFMASK @@ -30,6 +31,7 @@ function apply() { # There is no plan file to apply, since the remote backend can't produce them. # Instead we need to do an auto approved apply using the arguments we would normally use for the plan + # shellcheck disable=SC2086 debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK From ef4759bb867e89b39464ec14c02f33e047d23e16 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 11:19:13 +0100 Subject: [PATCH 154/231] shfmt --- image/actions.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index c96de49e..e68c31e4 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -211,15 +211,14 @@ function set-common-plan-args() { fi if [[ -v INPUT_REPLACE ]]; then - if [[ -n "$INPUT_REPLACE" ]]; then - for target in $(echo "$INPUT_REPLACE" | tr ',' '\n'); do - PLAN_ARGS="$PLAN_ARGS -replace $target" - done - fi + if [[ -n "$INPUT_REPLACE" ]]; then + for target in $(echo "$INPUT_REPLACE" | tr ',' '\n'); do + PLAN_ARGS="$PLAN_ARGS -replace $target" + done + fi fi } - function set-plan-args() { set-common-plan-args @@ -251,7 +250,7 @@ function set-remote-plan-args() { if [[ -n "$INPUT_VAR_FILE" ]]; then for file in $(echo "$INPUT_VAR_FILE" | tr ',' '\n'); do cp "$file" "$INPUT_PATH/zzzz-dflook-terraform-github-actions-$AUTO_TFVARS_COUNTER.auto.tfvars" - AUTO_TFVARS_COUNTER=$(( AUTO_TFVARS_COUNTER + 1 )) + AUTO_TFVARS_COUNTER=$((AUTO_TFVARS_COUNTER + 1)) done fi @@ -291,8 +290,8 @@ function write_credentials() { chmod 700 /.ssh if [[ -v TERRAFORM_SSH_KEY ]]; then - echo "$TERRAFORM_SSH_KEY" >>/.ssh/id_rsa - chmod 600 /.ssh/id_rsa + echo "$TERRAFORM_SSH_KEY" >>/.ssh/id_rsa + chmod 600 /.ssh/id_rsa fi debug_cmd git config --list @@ -331,9 +330,9 @@ function destroy() { set +e # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \ - 2>"$STEP_TMP_DIR/terraform_destroy.stderr" \ - | tee /dev/fd/3 \ - >"$STEP_TMP_DIR/terraform_destroy.stdout" + 2>"$STEP_TMP_DIR/terraform_destroy.stderr" \ + | tee /dev/fd/3 \ + >"$STEP_TMP_DIR/terraform_destroy.stdout" # shellcheck disable=SC2034 DESTROY_EXIT=${PIPESTATUS[0]} From ba4c541ecea849eef4723a223077c5b2fb311209 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 12:26:12 +0100 Subject: [PATCH 155/231] :pencil: tweak docs --- terraform-apply/README.md | 18 +++++++-------- terraform-check/README.md | 4 ++-- terraform-destroy-workspace/README.md | 22 +++++++++--------- terraform-destroy/README.md | 32 ++++++++++++++++++--------- terraform-fmt-check/README.md | 2 +- terraform-fmt/README.md | 2 +- terraform-new-workspace/README.md | 2 +- terraform-output/README.md | 2 +- terraform-plan/README.md | 20 ++++++++--------- terraform-validate/README.md | 2 +- 10 files changed, 58 insertions(+), 48 deletions(-) diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 86497785..e3bf8da2 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -35,7 +35,7 @@ These input values must be the same as any `terraform-plan` for the same configu * `path` - Path to the terraform configuration to apply + Path to the terraform root module to apply - Type: string - Optional @@ -121,14 +121,6 @@ These input values must be the same as any `terraform-plan` for the same configu - Type: string - Optional -* `parallelism` - - Limit the number of concurrent operations - - - Type: number - - Optional - - Default: 10 - * `replace` List of resources to replace if any update to them is required. @@ -170,6 +162,14 @@ These input values must be the same as any `terraform-plan` for the same configu - Optional - Default: false +* `parallelism` + + Limit the number of concurrent operations + + - Type: number + - Optional + - Default: The terraform default (10) + * ~~`var`~~ > :warning: **Deprecated**: Use the `variables` input instead. diff --git a/terraform-check/README.md b/terraform-check/README.md index 16ccc352..54ec5ca4 100644 --- a/terraform-check/README.md +++ b/terraform-check/README.md @@ -10,7 +10,7 @@ This is intended to run on a schedule to notify if manual changes to your infras * `path` - Path to the terraform configuration to check + Path to the terraform root module to check - Type: string - Optional @@ -92,7 +92,7 @@ This is intended to run on a schedule to notify if manual changes to your infras - Type: number - Optional - - Default: 10 + - Default: The terraform default (10) * ~~`var`~~ diff --git a/terraform-destroy-workspace/README.md b/terraform-destroy-workspace/README.md index 6e3f57df..871baf14 100644 --- a/terraform-destroy-workspace/README.md +++ b/terraform-destroy-workspace/README.md @@ -8,7 +8,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t * `path` - Path to the terraform configuration + Path to the terraform root module - Type: string - Optional @@ -40,15 +40,6 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: string - Optional -* ~~`var`~~ - - > :warning: **Deprecated**: Use the `variables` input instead. - - Comma separated list of terraform vars to set - - - Type: string - - Optional - * `var_file` List of tfvars files to use, one per line. @@ -95,7 +86,16 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: number - Optional - - Default: 10 + - Default: The terraform default (10) + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set + + - Type: string + - Optional ## Outputs diff --git a/terraform-destroy/README.md b/terraform-destroy/README.md index ff39c77d..f5b38381 100644 --- a/terraform-destroy/README.md +++ b/terraform-destroy/README.md @@ -9,7 +9,7 @@ This action uses the `terraform destroy` command to destroy all resources in a t * `path` - Path to the terraform configuration + Path to the terraform root module - Type: string - Optional @@ -38,22 +38,18 @@ This action uses the `terraform destroy` command to destroy all resources in a t ``` Variables set here override any given in `var_file`s. + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. - - Type: string - - Optional - -* ~~`var`~~ - - > :warning: **Deprecated**: Use the `variables` input instead. - - Comma separated list of terraform vars to set + > :warning: Secret values are not masked in the PR comment. Set a `label` to avoid revealing the variables in the PR. - Type: string - Optional +* `var_file` + List of tfvars files to use, one per line. Paths should be relative to the GitHub Actions workspace - + ```yaml with: var_file: | @@ -61,6 +57,11 @@ This action uses the `terraform destroy` command to destroy all resources in a t prod.tfvars ``` + This **can** be used with remote backends such as Terraform Cloud/Enterprise, with variables set in the remote workspace having precedence. + + - Type: string + - Optional + * `backend_config` List of terraform backend config values, one per line. @@ -92,7 +93,16 @@ This action uses the `terraform destroy` command to destroy all resources in a t - Type: number - Optional - - Default: 10 + - Default: The terraform default (10) + +* ~~`var`~~ + + > :warning: **Deprecated**: Use the `variables` input instead. + + Comma separated list of terraform vars to set + + - Type: string + - Optional ## Outputs diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index 95cc2565..975b1907 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -11,7 +11,7 @@ If any files are not correctly formatted a failing GitHub check will be added fo * `path` - Path to the terraform configuration + Path containing terraform files - Type: string - Optional diff --git a/terraform-fmt/README.md b/terraform-fmt/README.md index b002d0bf..f87bda18 100644 --- a/terraform-fmt/README.md +++ b/terraform-fmt/README.md @@ -8,7 +8,7 @@ This action uses the `terraform fmt` command to reformat files in a directory in * `path` - Path to the terraform configuration + Path containing terraform files - Type: string - Optional diff --git a/terraform-new-workspace/README.md b/terraform-new-workspace/README.md index 235dbc4f..b8831c1f 100644 --- a/terraform-new-workspace/README.md +++ b/terraform-new-workspace/README.md @@ -8,7 +8,7 @@ Creates a new terraform workspace. If the workspace already exists, succeeds wit * `path` - Path to the terraform configuration + Path to the terraform root module - Type: string - Optional diff --git a/terraform-output/README.md b/terraform-output/README.md index 3c0df711..6dc2770c 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -8,7 +8,7 @@ Retrieve the root-level outputs from a terraform configuration. * `path` - Path to the terraform configuration + Path to the terraform root module - Type: string - Optional diff --git a/terraform-plan/README.md b/terraform-plan/README.md index c43d7d26..3b56287c 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -18,7 +18,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `path` - Path to the terraform configuration + Path to the terraform root module to apply - Type: string - Optional @@ -34,7 +34,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `label` - An friendly name for the environment the terraform configuration is for. + A friendly name for the environment the terraform configuration is for. This will be used in the PR comment for easy identification. If set, must be the same as the `label` used in the corresponding `terraform-apply` command. @@ -106,14 +106,6 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Type: string - Optional -* `parallelism` - - Limit the number of concurrent operations - - - Type: number - - Optional - - Default: 10 - * `replace` List of resources to replace if any update to them is required, one per line. @@ -154,6 +146,14 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Optional - Default: true +* `parallelism` + + Limit the number of concurrent operations + + - Type: number + - Optional + - Default: The terraform default (10) + * ~~`var`~~ > :warning: **Deprecated**: Use the `variables` input instead. diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 904258ab..061879b0 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -17,7 +17,7 @@ If the terraform configuration is not valid, the build is failed. * `path` - Path to the terraform configuration + Path to the terraform root module - Type: string - Optional From 71821fa9d2c7ce66f3e91381b9196082049dc039 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 12:39:47 +0100 Subject: [PATCH 156/231] :bookmark: v1.18.0 --- CHANGELOG.md | 31 +++++++++++++++++++++++++------ terraform-apply/README.md | 4 ++-- terraform-plan/README.md | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 549e1b3e..99873f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,33 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.17.3` to use an exact release -- `@v1.17` to use the latest patch release for the specific minor version +- `@v1.18.0` to use an exact release +- `@v1.18` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## [1.17.3] - 2021-10-29 +## [1.18.0] - 2021-10-30 -### Fixed -- Compatability with Terraform v1.0.10 - Thanks [kylewlacy](https://github.com/kylewlacy)! -- Now makes even fewer github api requests to avoid rate limiting. +### Added +- A new `replace` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#inputs) and [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply#inputs) + + This instructs terraform to replace the specified resources, and is available with terraform versions that support replace (v0.15.2 onwards). + + ```yaml + with: + replace: | + random_password.database + ``` + +- A `target` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#inputs) to match [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply#inputs) + + `target` limits the plan to the specified resources and their dependencies. This change removes the restriction that `target` can only be used with `auto_approve`. + + ```yaml + with: + target: | + kubernetes_secret.tls_cert_public + kubernetes_secret.tls_cert_private + ``` ## [1.17.2] - 2021-10-13 @@ -271,6 +289,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.18.0]: https://github.com/dflook/terraform-github-actions/compare/v1.17.3...v1.18.0 [1.17.3]: https://github.com/dflook/terraform-github-actions/compare/v1.17.2...v1.17.3 [1.17.2]: https://github.com/dflook/terraform-github-actions/compare/v1.17.1...v1.17.2 [1.17.1]: https://github.com/dflook/terraform-github-actions/compare/v1.17.0...v1.17.1 diff --git a/terraform-apply/README.md b/terraform-apply/README.md index e3bf8da2..d5d0b59b 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -123,9 +123,9 @@ These input values must be the same as any `terraform-plan` for the same configu * `replace` - List of resources to replace if any update to them is required. + List of resources to replace, one per line. - Only available with supported terraform versions (v0.15.2 onwards). + Only available with terraform versions that support replace (v0.15.2 onwards). ```yaml with: diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 3b56287c..318e18fb 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -108,7 +108,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `replace` - List of resources to replace if any update to them is required, one per line. + List of resources to replace, one per line. Only available with terraform versions that support replace (v0.15.2 onwards). From 03d709308159a49e60b14473504bcebc95b02efd Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 14:23:13 +0100 Subject: [PATCH 157/231] Add missing deprecation message --- terraform-destroy/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 9c625344..5f65432a 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -24,6 +24,7 @@ inputs: var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false + deprecationMessage: Use the variables input instead. var_file: description: List of var file paths, one per line required: false From 505cffe75d4980e8e8f3f1cfac28e0147482e048 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 30 Oct 2021 14:23:43 +0100 Subject: [PATCH 158/231] Add emojis to status updates --- .github/workflows/test-react.yaml | 15 ++++ CHANGELOG.md | 6 ++ image/Dockerfile | 1 + image/actions.sh | 13 +++ image/entrypoints/apply.sh | 10 +-- image/entrypoints/destroy-workspace.sh | 1 + image/entrypoints/plan.sh | 4 +- image/tools/convert_output.py | 7 +- image/tools/github_comment_react.py | 109 +++++++++++++++++++++++++ tests/target/main.tf | 6 ++ 10 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-react.yaml create mode 100755 image/tools/github_comment_react.py diff --git a/.github/workflows/test-react.yaml b/.github/workflows/test-react.yaml new file mode 100644 index 00000000..00fb9300 --- /dev/null +++ b/.github/workflows/test-react.yaml @@ -0,0 +1,15 @@ +name: Test Reaction + +on: [issue_comment] + +jobs: + issue_comment: + if: ${{ github.event.issue.pull_request && contains(github.event.comment.user.login, 'dflook') }} + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Plan + uses: ./terraform-version + with: + path: tests/plan/no_changes diff --git a/CHANGELOG.md b/CHANGELOG.md index 99873f8b..aed4de38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ When using an action you can specify the version as: - `@v1.18` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## Unreleased + +### Changed +- When triggered by `issue_comment` or `pull_request_review_comment` events, the action will first add a :+1: reaction to the comment +- PR comment status messages lead with a single emoji that gives a progress update at a glance + ## [1.18.0] - 2021-10-30 ### Added diff --git a/image/Dockerfile b/image/Dockerfile index 1fc04d69..ddd9a4c3 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -13,6 +13,7 @@ COPY tools/convert_version.py /usr/local/bin/convert_version COPY tools/workspace_exists.py /usr/local/bin/workspace_exists COPY tools/compact_plan.py /usr/local/bin/compact_plan COPY tools/format_tf_credentials.py /usr/local/bin/format_tf_credentials +COPY tools/github_comment_react.py /usr/local/bin/github_comment_react RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \ && echo "IdentityFile /.ssh/id_rsa" >> /etc/ssh/ssh_config \ diff --git a/image/actions.sh b/image/actions.sh index e68c31e4..53e9ab0a 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -85,6 +85,10 @@ function setup() { exit 1 fi + if ! github_comment_react +1 2>"$STEP_TMP_DIR/github_comment_react.stderr"; then + debug_file "$STEP_TMP_DIR/github_comment_react.stderr" + fi + local TERRAFORM_BIN_DIR TERRAFORM_BIN_DIR="$JOB_TMP_DIR/terraform-bin-dir" # tfswitch guesses the wrong home directory... @@ -185,13 +189,22 @@ function init-backend() { } function select-workspace() { + local WORKSPACE_EXIT + + set +e (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") >"$STEP_TMP_DIR/workspace_select" 2>&1 + WORKSPACE_EXIT=$? + set -e if [[ -s "$STEP_TMP_DIR/workspace_select" ]]; then start_group "Selecting workspace" cat "$STEP_TMP_DIR/workspace_select" end_group fi + + if [[ $WORKSPACE_EXIT -ne 0 ]]; then + exit $WORKSPACE_EXIT + fi } function set-common-plan-args() { diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 86ca32a1..b1385973 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -12,7 +12,7 @@ set-plan-args PLAN_OUT="$STEP_TMP_DIR/plan.out" if [[ -v GITHUB_TOKEN ]]; then - update_status "Applying plan in $(job_markdown_ref)" + update_status ":orange_circle: Applying plan in $(job_markdown_ref)" fi exec 3>&1 @@ -40,10 +40,10 @@ function apply() { set -e if [[ $APPLY_EXIT -eq 0 ]]; then - update_status "Plan applied in $(job_markdown_ref)" + update_status ":white_check_mark: Plan applied in $(job_markdown_ref)" else set_output failure-reason apply-failed - update_status "Error applying plan in $(job_markdown_ref)" + update_status ":x: Error applying plan in $(job_markdown_ref)" exit 1 fi } @@ -69,7 +69,7 @@ fi if [[ $PLAN_EXIT -eq 1 ]]; then cat >&2 "$STEP_TMP_DIR/terraform_plan.stderr" - update_status "Error applying plan in $(job_markdown_ref)" + update_status ":x: Error applying plan in $(job_markdown_ref)" exit 1 fi @@ -110,7 +110,7 @@ else else echo "Not applying the plan - it has changed from the plan on the PR" echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" - update_status "Plan not applied in $(job_markdown_ref) (Plan has changed)" + update_status ":x: Plan not applied in $(job_markdown_ref) (Plan has changed)" echo "Plan changes:" debug_log diff "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt" diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index c417073b..84f8559e 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -31,4 +31,5 @@ workspace=$INPUT_WORKSPACE INPUT_WORKSPACE=default init-backend +debug_log terraform workspace delete -no-color -lock-timeout=300s "$workspace" (cd "$INPUT_PATH" && terraform workspace delete -no-color -lock-timeout=300s "$workspace") diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 17554735..fac425bf 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -39,7 +39,7 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c fi if [[ $PLAN_EXIT -eq 1 ]]; then - if ! STATUS="Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + if ! STATUS=":x: Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" exit 1 else @@ -54,7 +54,7 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c TF_CHANGES=true fi - if ! TF_CHANGES=$TF_CHANGES STATUS="Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + if ! TF_CHANGES=$TF_CHANGES STATUS=":memo: Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" exit 1 else diff --git a/image/tools/convert_output.py b/image/tools/convert_output.py index 862c5c2a..d258c6d5 100755 --- a/image/tools/convert_output.py +++ b/image/tools/convert_output.py @@ -30,12 +30,15 @@ def convert_to_github(outputs: Dict) -> Iterable[str]: yield f'::set-output name={name}::{value}' if __name__ == '__main__': + + input_string = sys.stdin.read() try: - outputs = json.load(sys.stdin) + outputs = json.loads(input_string) if not isinstance(outputs, dict): raise Exception('Unable to parse outputs') except: - exit(1) + sys.stderr.write(input_string) + raise for line in convert_to_github(outputs): print(line) diff --git a/image/tools/github_comment_react.py b/image/tools/github_comment_react.py new file mode 100755 index 00000000..9c5e89b3 --- /dev/null +++ b/image/tools/github_comment_react.py @@ -0,0 +1,109 @@ +#!/usr/bin/python3 + +import datetime +import json +import os +import sys +from typing import Optional, cast, NewType, TypedDict + +import requests + +GitHubUrl = NewType('GitHubUrl', str) +CommentReactionUrl = NewType('CommentReactionUrl', GitHubUrl) + + +class GitHubActionsEnv(TypedDict): + """ + Environment variables that are set by the actions runner + """ + GITHUB_API_URL: str + GITHUB_TOKEN: str + GITHUB_EVENT_PATH: str + GITHUB_EVENT_NAME: str + GITHUB_REPOSITORY: str + GITHUB_SHA: str + + +job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') +step_tmp_dir = os.environ.get('STEP_TMP_DIR', '.') + +env = cast(GitHubActionsEnv, os.environ) + + +def github_session(github_env: GitHubActionsEnv) -> requests.Session: + """ + A request session that is configured for the github API + """ + session = requests.Session() + session.headers['authorization'] = f'token {github_env["GITHUB_TOKEN"]}' + session.headers['user-agent'] = 'terraform-github-actions' + session.headers['accept'] = 'application/vnd.github.v3+json' + return session + + +def github_api_request(method: str, *args, **kwargs) -> requests.Response: + response = github.request(method, *args, **kwargs) + + if 400 <= response.status_code < 500: + debug(str(response.headers)) + + try: + message = response.json()['message'] + + if response.headers['X-RateLimit-Remaining'] == '0': + limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) + sys.stdout.write(message) + sys.stdout.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') + exit(1) + + if message != 'Resource not accessible by integration': + sys.stdout.write(message) + sys.stdout.write('\n') + debug(response.content.decode()) + + except Exception: + sys.stdout.write(response.content.decode()) + sys.stdout.write('\n') + raise + + return response + + +def debug(msg: str) -> None: + sys.stderr.write(msg) + sys.stderr.write('\n') + + +def find_reaction_url(actions_env: GitHubActionsEnv) -> Optional[CommentReactionUrl]: + event_type = actions_env['GITHUB_EVENT_NAME'] + + if event_type not in ['issue_comment', 'pull_request_review_comment']: + return None + + with open(actions_env['GITHUB_EVENT_PATH']) as f: + event = json.load(f) + + return event['comment']['reactions']['url'] + + +def react(comment_reaction_url: CommentReactionUrl, reaction_type: str) -> None: + github_api_request('post', comment_reaction_url, json={'content': reaction_type}) + + +def main() -> None: + if len(sys.argv) < 2: + print(f'''Usage: + {sys.argv[0]} ''') + + debug(repr(sys.argv)) + + reaction_url = find_reaction_url(env) + if reaction_url is not None: + react(reaction_url, sys.argv[1]) + + +if __name__ == '__main__': + if 'GITHUB_TOKEN' not in env: + exit(0) + github = github_session(env) + main() diff --git a/tests/target/main.tf b/tests/target/main.tf index 98b2d21f..16fd8bd3 100644 --- a/tests/target/main.tf +++ b/tests/target/main.tf @@ -2,12 +2,18 @@ resource "random_string" "count" { count = 1 length = var.length + + special = false + min_special = 0 } resource "random_string" "foreach" { for_each = toset(["hello"]) length = var.length + + special = false + min_special = 0 } variable "length" { From 0792c3fe114912fd9211534e1f91d3f98d34b0dc Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 31 Oct 2021 21:53:08 +0000 Subject: [PATCH 159/231] Don't hold a state lock during the plan action --- image/entrypoints/plan.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index fac425bf..5e3140ca 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -9,12 +9,11 @@ init-backend select-workspace set-plan-args -PLAN_OUT="$STEP_TMP_DIR/plan.out" - exec 3>&1 ### Generate a plan - +PLAN_OUT="$STEP_TMP_DIR/plan.out" +PLAN_ARGS="$PLAN_ARGS -lock=false" plan if [[ $PLAN_EXIT -eq 1 ]]; then @@ -22,6 +21,7 @@ if [[ $PLAN_EXIT -eq 1 ]]; then # This terraform module is using the remote backend, which is deficient. set-remote-plan-args PLAN_OUT="" + PLAN_ARGS="$PLAN_ARGS -lock=false" plan fi fi From a9aac78bf9688439079f5fac64f84e126282d55d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 31 Oct 2021 22:12:39 +0000 Subject: [PATCH 160/231] Don't hold a state lock during the check action --- image/entrypoints/check.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index e971612e..4f14979c 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -9,10 +9,10 @@ init-backend select-workspace set-plan-args -PLAN_OUT="$STEP_TMP_DIR/plan.out" - exec 3>&1 +PLAN_OUT="$STEP_TMP_DIR/plan.out" +PLAN_ARGS="$PLAN_ARGS -lock=false" plan if [[ $PLAN_EXIT -eq 1 ]]; then @@ -20,6 +20,7 @@ if [[ $PLAN_EXIT -eq 1 ]]; then # This terraform module is using the remote backend, which is deficient. set-remote-plan-args PLAN_OUT="" + PLAN_ARGS="$PLAN_ARGS -lock=false" plan fi fi From fcaa2f415cb3631e8a46885d9f7d941706b2f196 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 1 Nov 2021 08:35:41 +0000 Subject: [PATCH 161/231] :bookmark: v1.19.0 --- CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aed4de38..5384b91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.18.0` to use an exact release -- `@v1.18` to use the latest patch release for the specific minor version +- `@v1.19.0` to use an exact release +- `@v1.19` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version -## Unreleased +## [1.19.0] - 2021-11-01 ### Changed - When triggered by `issue_comment` or `pull_request_review_comment` events, the action will first add a :+1: reaction to the comment -- PR comment status messages lead with a single emoji that gives a progress update at a glance +- PR comment status messages include a single emoji that shows progress at a glance +- Actions that don't write to the terraform state no longer lock it. ## [1.18.0] - 2021-10-30 @@ -295,6 +296,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.19.0]: https://github.com/dflook/terraform-github-actions/compare/v1.18.0...v1.19.0 [1.18.0]: https://github.com/dflook/terraform-github-actions/compare/v1.17.3...v1.18.0 [1.17.3]: https://github.com/dflook/terraform-github-actions/compare/v1.17.2...v1.17.3 [1.17.2]: https://github.com/dflook/terraform-github-actions/compare/v1.17.1...v1.17.2 From c56e9964bc89635a7b6c6a7f39a8f36aad0d3a2d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 1 Nov 2021 08:48:41 +0000 Subject: [PATCH 162/231] Remove test-react --- .github/workflows/test-react.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .github/workflows/test-react.yaml diff --git a/.github/workflows/test-react.yaml b/.github/workflows/test-react.yaml deleted file mode 100644 index 00fb9300..00000000 --- a/.github/workflows/test-react.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Test Reaction - -on: [issue_comment] - -jobs: - issue_comment: - if: ${{ github.event.issue.pull_request && contains(github.event.comment.user.login, 'dflook') }} - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Plan - uses: ./terraform-version - with: - path: tests/plan/no_changes From 2f1b3e90086af154ce3f213f15353bb4f6c0e0c8 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 27 Nov 2021 18:52:17 +0000 Subject: [PATCH 163/231] Add issue templates --- .github/ISSUE_TEMPLATE/config.yml | 4 +++ .github/ISSUE_TEMPLATE/problem.yml | 45 +++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/suggestion.yml | 10 ++++++ 3 files changed, 59 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/problem.yml create mode 100644 .github/ISSUE_TEMPLATE/suggestion.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..c8695979 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,4 @@ +contact_links: + - name: Question + url: https://github.com/dflook/terraform-github-actions/discussions + about: Please ask questions as a new discussion if others would benefit from seeing the answer diff --git a/.github/ISSUE_TEMPLATE/problem.yml b/.github/ISSUE_TEMPLATE/problem.yml new file mode 100644 index 00000000..969fc124 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/problem.yml @@ -0,0 +1,45 @@ +name: Problem +description: "I'm having a problem using these actions" +labels: ["problem"] +body: + - type: markdown + attributes: + value: | + Before creating an issue, enable debug logging by setting the `ACTIONS_STEP_DEBUG` secret to `true` and run the job again. + + - type: textarea + id: description + attributes: + description: What is not working? What do you think should be happening instead? + label: Problem description + validations: + required: true + + - type: input + id: terraform-version + attributes: + label: Terraform version + description: What terraform version are you using? + placeholder: 1.0.5 + + - type: input + id: backend + attributes: + label: Backend + description: What terraform backend are you using? + placeholder: s3 + + - type: textarea + id: workflow + attributes: + label: Workflow YAML + description: Please copy and paste the relevant workflow yaml. This will be automatically formatted, so no need for backticks. + render: yaml + + - type: textarea + id: workflow-logs + attributes: + label: Workflow log + description: Please copy and paste the relevant workflow log output. If this is long consider putting in a [gist](https://gist.github.com/). This will be automatically formatted, so no need for backticks. + render: shell + diff --git a/.github/ISSUE_TEMPLATE/suggestion.yml b/.github/ISSUE_TEMPLATE/suggestion.yml new file mode 100644 index 00000000..0e11e0fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/suggestion.yml @@ -0,0 +1,10 @@ +name: Suggestion +description: I have a suggestion for an enhancement +labels: ["enhancement"] +body: + - type: textarea + id: suggestion + attributes: + label: Suggestion + validations: + required: true From 940a1765c7bdeea3660b3abf74b10cf721412b08 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 27 Nov 2021 19:37:57 +0000 Subject: [PATCH 164/231] Add labels as code --- .github/labels.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/labels.yaml | 20 ++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 .github/labels.yml create mode 100644 .github/workflows/labels.yaml diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 00000000..65cf3559 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,30 @@ +- name: "defect" + color: "d73a4a" + description: "Something isn't working" + +- name: "documentation" + color: "0075ca" + description: "Improvements or additions to documentation" + +- name: "duplicate" + color: "cfd8d7" + description: "This issue or pull request already exists" + +- name: "enhancement" + color: "a22eef" + description: "New feature or request" + +- name: "invalid" + color: "e4e669" + description: "This doesn't seem right" + +- name: "problem" + color: "BFD4F2" + +- name: "question" + color: "d876e3" + description: "Further information is requested" + +- name: "wontfix" + color: "ffffff" + description: "This will not be worked on" diff --git a/.github/workflows/labels.yaml b/.github/workflows/labels.yaml new file mode 100644 index 00000000..4a70005c --- /dev/null +++ b/.github/workflows/labels.yaml @@ -0,0 +1,20 @@ +name: Update labels + +on: + push: + branches: + - master + paths: + - '.github/labels.yml' + +jobs: + labeler: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Run Labeler + uses: crazy-max/ghaction-github-labeler@52525cb66833763f651fc34e244e4f73b6e07ff5 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} From 3404f394d254d1dedea4b63dee6afa9c770a13b4 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 27 Nov 2021 19:50:14 +0000 Subject: [PATCH 165/231] Fix colours --- .github/labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 65cf3559..72080d36 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -7,11 +7,11 @@ description: "Improvements or additions to documentation" - name: "duplicate" - color: "cfd8d7" + color: "cfd3d7" description: "This issue or pull request already exists" - name: "enhancement" - color: "a22eef" + color: "a2eeef" description: "New feature or request" - name: "invalid" From 066c1c30af663e315c7deaa5bbff7a35e56ad93e Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 3 Dec 2021 19:31:43 +0000 Subject: [PATCH 166/231] Add `json_plan_path` and `text_plan_path` for dflook/terraform-apply --- .github/workflows/test-apply.yaml | 174 +++++++++++++++++++++++++---- .github/workflows/test-plan.yaml | 14 +-- .github/workflows/test-remote.yaml | 20 ++++ image/entrypoints/apply.sh | 18 +++ terraform-apply/README.md | 34 ++++-- terraform-apply/action.yaml | 8 ++ terraform-plan/README.md | 2 +- 7 files changed, 230 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 217b167e..a2701e6e 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -26,6 +26,16 @@ jobs: echo "::error:: output my_string not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.my_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_error: runs-on: ubuntu-latest @@ -53,6 +63,17 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi + + if [[ -n "${{ steps.apply.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi + + if [[ -n "${{ steps.apply.outputs.text_plan_path }}" ]]; then + echo "::error:: text_plan_path should not be set" + exit 1 + fi + apply_apply_error: runs-on: ubuntu-latest @@ -90,6 +111,16 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi + + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.apply.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_no_token: runs-on: ubuntu-latest @@ -143,6 +174,39 @@ jobs: echo "::error:: output s not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + + - name: Apply + uses: ./terraform-apply + id: output + with: + path: tests/apply/changes + + - name: Verify outputs + run: | + if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + echo "::error:: output s not set correctly" + exit 1 + fi + + if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_variables: runs-on: ubuntu-latest @@ -215,6 +279,16 @@ jobs: echo "::error:: output complex_output not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi backend_config_12: runs-on: ubuntu-latest @@ -246,6 +320,16 @@ jobs: echo "::error:: output from backend_config file not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_file_12.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.backend_config_file_12.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi - name: Plan uses: ./terraform-plan @@ -272,6 +356,16 @@ jobs: echo "::error:: Output from backend_config not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_12.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.backend_config_file_12.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi backend_config_13: runs-on: ubuntu-latest @@ -303,6 +397,16 @@ jobs: echo "::error:: output from backend_config file not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_file_13.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.backend_config_file_13.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi - name: Plan uses: ./terraform-plan @@ -329,6 +433,16 @@ jobs: echo "::error:: Output from backend_config not set correctly" exit 1 fi + + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_13.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.backend_config_13.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_label: runs-on: ubuntu-latest @@ -362,31 +476,17 @@ jobs: echo "::error:: output s not set correctly" exit 1 fi - - apply_changes_already_applied: - runs-on: ubuntu-latest - name: Apply when changes are already applied - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: - - apply - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Apply - uses: ./terraform-apply - id: output - with: - path: tests/apply/changes - - - name: Verify outputs - run: | - if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then - echo "::error:: output s not set correctly" + + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" exit 1 fi + if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + apply_no_changes: runs-on: ubuntu-latest name: Apply when there are no planned changes @@ -410,6 +510,16 @@ jobs: echo "::error:: output my_string not set correctly" exit 1 fi + + if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_no_plan: runs-on: ubuntu-latest @@ -433,6 +543,16 @@ jobs: echo "Apply did not fail correctly" exit 1 fi + + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.apply.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi apply_user_token: runs-on: ubuntu-latest @@ -463,6 +583,16 @@ jobs: exit 1 fi + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + apply_vars: runs-on: ubuntu-latest name: Apply approved changes with deprecated vars diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index c85066b5..c71451bd 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -127,7 +127,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -162,7 +162,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -198,7 +198,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -234,7 +234,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -269,7 +269,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -304,7 +304,7 @@ jobs: cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -333,7 +333,7 @@ jobs: run: | cat '${{ steps.plan.outputs.json_plan_path }}' if [[ $(jq -r .output_changes.s.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then - echo "::error:: json not set correctly" + echo "::error:: json_plan_path not set correctly" exit 1 fi diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 71faa417..1bde0009 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -60,6 +60,16 @@ jobs: echo "::error:: Variables not set correctly" exit 1 fi + + if [[ -n "${{ steps.auto_apply.outputs.text_plan_path }}" ]]; then + echo "::error:: text_plan_path should not be set" + exit 1 + fi + + if [[ -n "${{ steps.auto_apply.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi - name: Check no changes uses: ./terraform-check @@ -160,6 +170,16 @@ jobs: echo "::error:: Variables not set correctly" exit 1 fi + + if ! grep -q "Terraform will perform the following actions" '${{ steps.apply.outputs.text_plan_path }}'; then + echo "::error:: text_plan_path not set correctly" + exit 1 + fi + + if [[ -n "${{ steps.apply.outputs.json_plan_path }}" ]]; then + echo "::error:: json_plan_path should not be set" + exit 1 + fi - name: Destroy the last workspace uses: ./terraform-destroy-workspace diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index b1385973..3bf8890e 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -73,6 +73,24 @@ if [[ $PLAN_EXIT -eq 1 ]]; then exit 1 fi +if [[ -z "$PLAN_OUT" && "$INPUT_AUTO_APPROVE" == "true" ]]; then + # Since we are doing an auto approved remote apply there is no point in planning beforehand + # No text_plan_path output for this run + : +else + mkdir -p "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR" + cp "$STEP_TMP_DIR/plan.txt" "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR/plan.txt" + set_output text_plan_path "$WORKSPACE_TMP_DIR/plan.txt" +fi + +if [[ -n "$PLAN_OUT" ]]; then + if (cd "$INPUT_PATH" && terraform show -json "$PLAN_OUT") >"$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR/plan.json" 2>"$STEP_TMP_DIR/terraform_show.stderr"; then + set_output json_plan_path "$WORKSPACE_TMP_DIR/plan.json" + else + debug_file "$STEP_TMP_DIR/terraform_show.stderr" + fi +fi + ### Apply the plan if [[ "$INPUT_AUTO_APPROVE" == "true" || $PLAN_EXIT -eq 0 ]]; then diff --git a/terraform-apply/README.md b/terraform-apply/README.md index d5d0b59b..1982fc20 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -202,6 +202,30 @@ These input values must be the same as any `terraform-plan` for the same configu ## Outputs +* `json_plan_path` + + This is the path to the generated plan in [JSON Output Format](https://www.terraform.io/docs/internals/json-format.html) + The path is relative to the Actions workspace. + + This is not available when using terraform 0.11 or earlier. + This also won't be set if the backend type is `remote` - Terraform does not support saving remote plans. + +* `text_plan_path` + + This is the path to the generated plan in a human-readable format. + The path is relative to the Actions workspace. + This won't be set if `auto_approve` is true while using a `remote` backend. + +* `failure-reason` + + When the job outcome is `failure`, this output may be set. The value may be one of: + + - `apply-failed` - The Terraform apply operation failed. + - `plan-changed` - The approved plan is no longer accurate, so the apply will not be attempted. + + If the job fails for any other reason this will not be set. + This can be used with the Actions expression syntax to conditionally run steps. + * Terraform Outputs An action output will be created for each output of the terraform configuration. @@ -216,16 +240,6 @@ These input values must be the same as any `terraform-plan` for the same configu Running this action will produce a `service_hostname` output with the same value. See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/master/terraform-output) for details. -* `failure-reason` - - When the job outcome is `failure`, this output may be set. The value may be one of: - - - `apply-failed` - The terraform apply operation failed. - - `plan-changed` - The approved plan is no longer accurate, so the apply will not be attempted. - - If the job fails for any other reason this will not be set. - This can be used with the Actions expression syntax to conditionally run steps. - ## Environment Variables * `GITHUB_TOKEN` diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index f2558a43..8e509920 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -51,6 +51,14 @@ inputs: required: false default: "" +outputs: + text_plan_path: + description: Path to a file in the workspace containing the generated plan in human readble format. This won't be set if the backend type is `remote` and `auto_approve` is `true` + json_plan_path: + description: Path to a file in the workspace containing the generated plan in JSON format. This won't be set if the backend type is `remote`. + failure-reason: + description: The reason for the build failure. May be `apply-failed` or `plan-changed`. + runs: using: docker image: ../image/Dockerfile diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 318e18fb..7bc49590 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -327,7 +327,7 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ * `text_plan_path` - This is the path to the generated plan in a human readable format. + This is the path to the generated plan in a human-readable format. The path is relative to the Actions workspace. ## Example usage From 01ee72f5ba369e78f911d683ac4e31232c6ea411 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 3 Dec 2021 23:49:30 +0000 Subject: [PATCH 167/231] :bookmark: v1.20.0 --- CHANGELOG.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5384b91e..a23bf541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,21 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.19.0` to use an exact release -- `@v1.19` to use the latest patch release for the specific minor version +- `@v1.20.0` to use an exact release +- `@v1.20` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.20.0] - 2021-12-03 + +### Added +- New `text_plan_path` and `json_plan_path` outputs for [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) + to match the outputs for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan). + + These are paths to the generated plan in human-readable and JSON formats. + + If the plan generated by [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) is different from the plan generated by [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) the apply step will fail with `failure-reason` set to `plan-changed`. + These new outputs make it easier to inspect the differences. + ## [1.19.0] - 2021-11-01 ### Changed @@ -69,7 +80,7 @@ When using an action you can specify the version as: ### Added - [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) has gained two new outputs: - `json_plan_path` is a path to the generated plan in a JSON format file - - `text_plan_path` is a path to the generated plan in a human readable text file + - `text_plan_path` is a path to the generated plan in a human-readable text file These paths are relative to the GitHub Actions workspace and can be read by other steps in the same job. @@ -296,6 +307,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.20.0]: https://github.com/dflook/terraform-github-actions/compare/v1.19.0...v1.20.0 [1.19.0]: https://github.com/dflook/terraform-github-actions/compare/v1.18.0...v1.19.0 [1.18.0]: https://github.com/dflook/terraform-github-actions/compare/v1.17.3...v1.18.0 [1.17.3]: https://github.com/dflook/terraform-github-actions/compare/v1.17.2...v1.17.3 From a39c99c56fc754f77b742cac829065cc1f091b0a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 15:11:18 +0000 Subject: [PATCH 168/231] Add test for terraform-output on remote workspaces --- .github/workflows/test-remote.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 1bde0009..e937105b 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -71,6 +71,21 @@ jobs: exit 1 fi + - name: Get outputs + uses: ./terraform-output + id: output + with: + path: tests/terraform-cloud + workspace: ${{ github.head_ref }}-1 + backend_config: "token=${{ secrets.TF_API_TOKEN }}" + + - name: Verify auto_apply terraform outputs + run: | + if [[ "${{ steps.output.outputs.default }}" != "default" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + - name: Check no changes uses: ./terraform-check with: From 972bb4c003239c6b560e74baf5334eeaa9230fda Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 16:03:04 +0000 Subject: [PATCH 169/231] Add test for terraform-output on remote workspaces with fixed name --- .github/workflows/test-remote.yaml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index e937105b..1d9c4101 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -79,13 +79,32 @@ jobs: workspace: ${{ github.head_ref }}-1 backend_config: "token=${{ secrets.TF_API_TOKEN }}" - - name: Verify auto_apply terraform outputs + - name: Verify auto_apply terraform outputs with workspace prefix run: | if [[ "${{ steps.output.outputs.default }}" != "default" ]]; then echo "::error:: Variables not set correctly" exit 1 fi + - name: Setup terraform with workspace name + run: | + mkdir fixed-workspace-name + sed -e 's/prefix.*/name = "github-actions-${{ github.head_ref }}-1"/' tests/terraform-cloud/main.tf > fixed-workspace-name/main.tf + + - name: Get outputs + uses: ./terraform-output + id: name-output + with: + path: fixed-workspace-name + backend_config: "token=${{ secrets.TF_API_TOKEN }}" + + - name: Verify auto_apply terraform outputs with workspace name + run: | + if [[ "${{ steps.name-output.outputs.default }}" != "default" ]]; then + echo "::error:: Variables not set correctly" + exit 1 + fi + - name: Check no changes uses: ./terraform-check with: From 5dac8386c87c2d85521641f6684fb1d773df9d77 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 16:09:55 +0000 Subject: [PATCH 170/231] Support remote backends using the full workspace name in the configuration --- image/actions.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/image/actions.sh b/image/actions.sh index 53e9ab0a..685dd03a 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -203,7 +203,11 @@ function select-workspace() { fi if [[ $WORKSPACE_EXIT -ne 0 ]]; then - exit $WORKSPACE_EXIT + if grep -q "workspaces not supported" "$STEP_TMP_DIR/workspace_select" && [[ $INPUT_WORKSPACE == "default" ]]; then + echo "The full name of a remote workspace is set by the terraform configuration, selecting a different one is not supported" + else + exit $WORKSPACE_EXIT + fi fi } From 33db26f33c0eb834115049a5f025300217499079 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 16:21:29 +0000 Subject: [PATCH 171/231] Support remote backends using the full workspace name in the configuration --- image/actions.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index 685dd03a..e5eb34e2 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -198,16 +198,19 @@ function select-workspace() { if [[ -s "$STEP_TMP_DIR/workspace_select" ]]; then start_group "Selecting workspace" - cat "$STEP_TMP_DIR/workspace_select" - end_group - fi - if [[ $WORKSPACE_EXIT -ne 0 ]]; then - if grep -q "workspaces not supported" "$STEP_TMP_DIR/workspace_select" && [[ $INPUT_WORKSPACE == "default" ]]; then + if [[ $WORKSPACE_EXIT -ne 0 ]] && grep -q "workspaces not supported" "$STEP_TMP_DIR/workspace_select" && [[ $INPUT_WORKSPACE == "default" ]]; then echo "The full name of a remote workspace is set by the terraform configuration, selecting a different one is not supported" + WORKSPACE_EXIT=0 else - exit $WORKSPACE_EXIT + cat "$STEP_TMP_DIR/workspace_select" fi + + end_group + fi + + if [[ $WORKSPACE_EXIT -ne 0 ]]; then + exit $WORKSPACE_EXIT fi } From 2305d4f394f2c245d1af524a6d309c2549f34971 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 17:32:07 +0000 Subject: [PATCH 172/231] :bookmark: v1.20.1 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a23bf541..9e5b21c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.20.0` to use an exact release +- `@v1.20.1` to use an exact release - `@v1.20` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.20.1] - 2021-12-04 + +### Fixed +- There was a problem selecting the workspace when using the `remote` backend with a full workspace `name` in the backend block. + ## [1.20.0] - 2021-12-03 ### Added @@ -307,6 +312,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.20.1]: https://github.com/dflook/terraform-github-actions/compare/v1.20.0...v1.20.1 [1.20.0]: https://github.com/dflook/terraform-github-actions/compare/v1.19.0...v1.20.0 [1.19.0]: https://github.com/dflook/terraform-github-actions/compare/v1.18.0...v1.19.0 [1.18.0]: https://github.com/dflook/terraform-github-actions/compare/v1.17.3...v1.18.0 From f54173c186d99c0367744a9274f8265b1c71c6c7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 18:06:49 +0000 Subject: [PATCH 173/231] Add workspace input for terraform-validate `terraform.workspace` may be evaluated during a validate, even though it won't be initialized. The workspace input allows setting it for the validate operation. --- .github/workflows/test-validate.yaml | 38 +++++++++++++++++++++++++++ image/entrypoints/validate.sh | 7 +++-- terraform-validate/README.md | 8 ++++++ terraform-validate/action.yaml | 4 +++ tests/validate/workspace_eval/main.tf | 36 +++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 tests/validate/workspace_eval/main.tf diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 38b47ff7..1aabb0f6 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -48,3 +48,41 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi + + validate_workspace: + runs-on: ubuntu-latest + name: Invalid terraform configuration + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: validate prod + uses: ./terraform-validate + with: + path: tests/validate/workspace_eval + workspace: prod + + - name: validate dev + uses: ./terraform-validate + with: + path: tests/validate/workspace_eval + workspace: dev + + - name: validate nonexistant workspace + uses: ./terraform-validate + id: validate + continue-on-error: true + with: + path: tests/validate/workspace_eval + + - name: Check invalid + run: | + if [[ "${{ steps.validate.outcome }}" != "failure" ]]; then + echo "Validate did not fail correctly" + exit 1 + fi + + if [[ "${{ steps.validate.outputs.failure-reason }}" != "validate-failed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 78ca9340..6098a203 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -10,10 +10,13 @@ setup # You can't initialize without having valid terraform. # How do you get a full validation report? You can't. +# terraform.workspace will be evaluated during a validate, but it is not initialized properly. +# Pass through the workspace input, even if it doesn't make sense for some backends. + init || true -if ! (cd "$INPUT_PATH" && terraform validate -json | convert_validate_report "$INPUT_PATH"); then - (cd "$INPUT_PATH" && terraform validate) +if ! (cd "$INPUT_PATH" && TF_WORKSPACE="$INPUT_WORKSPACE" terraform validate -json | convert_validate_report "$INPUT_PATH"); then + (cd "$INPUT_PATH" && TF_WORKSPACE="$INPUT_WORKSPACE" terraform validate) else echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" fi diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 061879b0..11fda4a6 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -23,6 +23,14 @@ If the terraform configuration is not valid, the build is failed. - Optional - Default: The action workspace +* `workspace` + + Terraform workspace to use for the `terraform.workspace` value while validating. + + - Type: string + - Optional + - Default: `default` + ## Outputs * `failure-reason` diff --git a/terraform-validate/action.yaml b/terraform-validate/action.yaml index 1ab12e52..a15e20b2 100644 --- a/terraform-validate/action.yaml +++ b/terraform-validate/action.yaml @@ -7,6 +7,10 @@ inputs: description: Path to the terraform configuration required: false default: . + workspace: + description: Name of the workspace to use for the `terraform.workspace` value while validating. + required: false + default: default runs: using: docker diff --git a/tests/validate/workspace_eval/main.tf b/tests/validate/workspace_eval/main.tf new file mode 100644 index 00000000..3fc73324 --- /dev/null +++ b/tests/validate/workspace_eval/main.tf @@ -0,0 +1,36 @@ +locals { + aws_provider_config = { + prod = { + region = "..." + account_id = "..." + profile = "..." + } + dev = { + region = "..." + account_id = "..." + profile = "..." + } + } +} + +provider "aws" { + region = local.aws_provider_config[terraform.workspace].region + profile = local.aws_provider_config[terraform.workspace].profile + allowed_account_ids = [local.aws_provider_config[terraform.workspace].account_id] +} + +resource "aws_s3_bucket" "bucket" { + bucket = "hello" +} + +terraform { + backend "remote" { + hostname = "app.terraform.io" + organization = "flooktech" + + workspaces { + name = "banana" + } + } +} + From 83b8fe928d38cff3841160ed6b7eed631dbcef12 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 4 Dec 2021 18:49:58 +0000 Subject: [PATCH 174/231] :bookmark: v1.21.0 --- CHANGELOG.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5b21c1..98800609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,19 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.20.1` to use an exact release -- `@v1.20` to use the latest patch release for the specific minor version +- `@v1.21.0` to use an exact release +- `@v1.21` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.21.0] - 2021-12-04 + +### Added +- A new `workspace` input for [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) + allows validating usage of `terraform.workspace` in the terraform code. + + Terraform doesn't initialize `terraform.workspace` based on the backend configuration when running a validate operation. + This new input allows setting the full name of the workspace to use while validating, even when you wouldn't normally do so for a plan/apply (e.g. when using the `remote` backend) + ## [1.20.1] - 2021-12-04 ### Fixed @@ -312,6 +321,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.21.0]: https://github.com/dflook/terraform-github-actions/compare/v1.20.1...v1.21.0 [1.20.1]: https://github.com/dflook/terraform-github-actions/compare/v1.20.0...v1.20.1 [1.20.0]: https://github.com/dflook/terraform-github-actions/compare/v1.19.0...v1.20.0 [1.19.0]: https://github.com/dflook/terraform-github-actions/compare/v1.18.0...v1.19.0 From f70a557239564f0a59eb95f0a39cbb3868c80650 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 6 Dec 2021 09:57:21 +0000 Subject: [PATCH 175/231] Lint yaml --- .github/FUNDING.yml | 3 +- .github/ISSUE_TEMPLATE/problem.yml | 7 ++- .github/ISSUE_TEMPLATE/suggestion.yml | 3 +- .github/labels.yml | 46 +++++++++---------- .github/workflows/base-image.yaml | 2 +- .github/workflows/labels.yaml | 2 +- .github/workflows/pull_request_review.yaml | 3 +- .../pull_request_review_trigger.yaml | 3 +- .github/workflows/pull_request_target.yaml | 3 +- .github/workflows/release.yaml | 3 +- .github/workflows/retain-images.yaml | 2 +- .github/workflows/test-apply.yaml | 44 +++++++++--------- .github/workflows/test-changes-only.yaml | 3 +- .github/workflows/test-check.yaml | 3 +- .github/workflows/test-fmt-check.yaml | 3 +- .github/workflows/test-fmt.yaml | 3 +- .github/workflows/test-http.yaml | 3 +- .github/workflows/test-new-workspace.yaml | 3 +- .github/workflows/test-output.yaml | 5 +- .github/workflows/test-plan.yaml | 3 +- .github/workflows/test-registry.yaml | 3 +- .github/workflows/test-remote-state.yaml | 3 +- .github/workflows/test-remote.yaml | 39 ++++++++-------- .github/workflows/test-ssh.yaml | 3 +- .github/workflows/test-target-replace.yaml | 27 +++++------ .github/workflows/test-validate.yaml | 3 +- .github/workflows/test-version.yaml | 3 +- .github/workflows/test-workflow-commands.yaml | 3 +- .github/workflows/test.yaml | 3 +- example_workflows/check_for_drift.yaml | 2 +- example_workflows/create_plan.yaml | 3 +- 31 files changed, 131 insertions(+), 108 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 85b14776..24411b1b 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,2 @@ -github: [dflook] +github: + - dflook diff --git a/.github/ISSUE_TEMPLATE/problem.yml b/.github/ISSUE_TEMPLATE/problem.yml index 969fc124..77d4108a 100644 --- a/.github/ISSUE_TEMPLATE/problem.yml +++ b/.github/ISSUE_TEMPLATE/problem.yml @@ -1,12 +1,12 @@ name: Problem -description: "I'm having a problem using these actions" -labels: ["problem"] +description: I'm having a problem using these actions +labels: + - problem body: - type: markdown attributes: value: | Before creating an issue, enable debug logging by setting the `ACTIONS_STEP_DEBUG` secret to `true` and run the job again. - - type: textarea id: description attributes: @@ -42,4 +42,3 @@ body: label: Workflow log description: Please copy and paste the relevant workflow log output. If this is long consider putting in a [gist](https://gist.github.com/). This will be automatically formatted, so no need for backticks. render: shell - diff --git a/.github/ISSUE_TEMPLATE/suggestion.yml b/.github/ISSUE_TEMPLATE/suggestion.yml index 0e11e0fe..9a846c1f 100644 --- a/.github/ISSUE_TEMPLATE/suggestion.yml +++ b/.github/ISSUE_TEMPLATE/suggestion.yml @@ -1,6 +1,7 @@ name: Suggestion description: I have a suggestion for an enhancement -labels: ["enhancement"] +labels: + - enhancement body: - type: textarea id: suggestion diff --git a/.github/labels.yml b/.github/labels.yml index 72080d36..7c07da04 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -1,30 +1,30 @@ -- name: "defect" - color: "d73a4a" - description: "Something isn't working" +- name: defect + color: d73a4a + description: Something isn't working -- name: "documentation" - color: "0075ca" - description: "Improvements or additions to documentation" +- name: documentation + color: 0075ca + description: Improvements or additions to documentation -- name: "duplicate" - color: "cfd3d7" - description: "This issue or pull request already exists" +- name: duplicate + color: cfd3d7 + description: This issue or pull request already exists -- name: "enhancement" - color: "a2eeef" - description: "New feature or request" +- name: enhancement + color: a2eeef + description: New feature or request -- name: "invalid" - color: "e4e669" - description: "This doesn't seem right" +- name: invalid + color: e4e669 + description: This doesn't seem right -- name: "problem" - color: "BFD4F2" +- name: problem + color: BFD4F2 -- name: "question" - color: "d876e3" - description: "Further information is requested" +- name: question + color: d876e3 + description: Further information is requested -- name: "wontfix" - color: "ffffff" - description: "This will not be worked on" +- name: wontfix + color: ffffff + description: This will not be worked on diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 54ebd718..7c85e81a 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -7,7 +7,7 @@ on: paths: - image/Dockerfile-base schedule: - - cron: "0 1 * * 1" + - cron: 0 1 * * 1 jobs: push_image: diff --git a/.github/workflows/labels.yaml b/.github/workflows/labels.yaml index 4a70005c..490573e1 100644 --- a/.github/workflows/labels.yaml +++ b/.github/workflows/labels.yaml @@ -5,7 +5,7 @@ on: branches: - master paths: - - '.github/labels.yml' + - .github/labels.yml jobs: labeler: diff --git a/.github/workflows/pull_request_review.yaml b/.github/workflows/pull_request_review.yaml index d69893ee..4ad1f859 100644 --- a/.github/workflows/pull_request_review.yaml +++ b/.github/workflows/pull_request_review.yaml @@ -1,6 +1,7 @@ name: pull_request_review test -on: [pull_request_review] +on: + - pull_request_review jobs: apply: diff --git a/.github/workflows/pull_request_review_trigger.yaml b/.github/workflows/pull_request_review_trigger.yaml index d8689818..067121a3 100644 --- a/.github/workflows/pull_request_review_trigger.yaml +++ b/.github/workflows/pull_request_review_trigger.yaml @@ -1,6 +1,7 @@ name: Trigger pull_request_review -on: [pull_request] +on: + - pull_request jobs: required_version: diff --git a/.github/workflows/pull_request_target.yaml b/.github/workflows/pull_request_target.yaml index 009db762..7267a409 100644 --- a/.github/workflows/pull_request_target.yaml +++ b/.github/workflows/pull_request_target.yaml @@ -1,6 +1,7 @@ name: pull_request_target test -on: [pull_request_target] +on: + - pull_request_target jobs: apply: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fde45187..7220c53e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -2,7 +2,8 @@ name: Release on: release: - types: [released] + types: + - released jobs: image: diff --git a/.github/workflows/retain-images.yaml b/.github/workflows/retain-images.yaml index ac029fe8..bd429027 100644 --- a/.github/workflows/retain-images.yaml +++ b/.github/workflows/retain-images.yaml @@ -2,7 +2,7 @@ name: Retain images on: schedule: - - cron: "0 0 1 * *" + - cron: 0 0 1 * * jobs: pull_image: diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index a2701e6e..2e78dc26 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -1,6 +1,7 @@ name: Test terraform-apply -on: [pull_request] +on: + - pull_request jobs: auto_approve: @@ -26,7 +27,7 @@ jobs: echo "::error:: output my_string not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.my_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -35,7 +36,7 @@ jobs: if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi apply_error: runs-on: ubuntu-latest @@ -63,7 +64,7 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi - + if [[ -n "${{ steps.apply.outputs.json_plan_path }}" ]]; then echo "::error:: json_plan_path should not be set" exit 1 @@ -72,8 +73,7 @@ jobs: if [[ -n "${{ steps.apply.outputs.text_plan_path }}" ]]; then echo "::error:: text_plan_path should not be set" exit 1 - fi - + fi apply_apply_error: runs-on: ubuntu-latest @@ -111,7 +111,7 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi - + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -174,7 +174,7 @@ jobs: echo "::error:: output s not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -183,7 +183,7 @@ jobs: if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi - name: Apply uses: ./terraform-apply @@ -279,7 +279,7 @@ jobs: echo "::error:: output complex_output not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -288,7 +288,7 @@ jobs: if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi backend_config_12: runs-on: ubuntu-latest @@ -320,7 +320,7 @@ jobs: echo "::error:: output from backend_config file not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_file_12.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -356,7 +356,7 @@ jobs: echo "::error:: Output from backend_config not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_12.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -397,7 +397,7 @@ jobs: echo "::error:: output from backend_config file not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_file_13.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -433,7 +433,7 @@ jobs: echo "::error:: Output from backend_config not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.test.actions[0] "${{ steps.backend_config_13.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -476,7 +476,7 @@ jobs: echo "::error:: output s not set correctly" exit 1 fi - + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -485,7 +485,7 @@ jobs: if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi apply_no_changes: runs-on: ubuntu-latest @@ -510,7 +510,7 @@ jobs: echo "::error:: output my_string not set correctly" exit 1 fi - + if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.2" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -519,7 +519,7 @@ jobs: if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi apply_no_plan: runs-on: ubuntu-latest @@ -543,7 +543,7 @@ jobs: echo "Apply did not fail correctly" exit 1 fi - + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 @@ -552,7 +552,7 @@ jobs: if ! grep -q "Terraform will perform the following actions" '${{ steps.apply.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi apply_user_token: runs-on: ubuntu-latest @@ -591,7 +591,7 @@ jobs: if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 - fi + fi apply_vars: runs-on: ubuntu-latest diff --git a/.github/workflows/test-changes-only.yaml b/.github/workflows/test-changes-only.yaml index a8a2e5f3..0a725d28 100644 --- a/.github/workflows/test-changes-only.yaml +++ b/.github/workflows/test-changes-only.yaml @@ -1,6 +1,7 @@ name: Test changes-only PR comment -on: [pull_request] +on: + - pull_request jobs: no_changes: diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index 04758d82..6afd9ff2 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -1,6 +1,7 @@ name: Test terraform-check -on: [pull_request] +on: + - pull_request jobs: no_changes: diff --git a/.github/workflows/test-fmt-check.yaml b/.github/workflows/test-fmt-check.yaml index b504c687..05186fee 100644 --- a/.github/workflows/test-fmt-check.yaml +++ b/.github/workflows/test-fmt-check.yaml @@ -1,6 +1,7 @@ name: Test terraform-fmt-check -on: [pull_request] +on: + - pull_request jobs: canonical_fmt: diff --git a/.github/workflows/test-fmt.yaml b/.github/workflows/test-fmt.yaml index ad7198cc..56c24800 100644 --- a/.github/workflows/test-fmt.yaml +++ b/.github/workflows/test-fmt.yaml @@ -1,6 +1,7 @@ name: Test terraform-fmt -on: [pull_request] +on: + - pull_request jobs: canonical_fmt: diff --git a/.github/workflows/test-http.yaml b/.github/workflows/test-http.yaml index 9026eb78..cafca1b7 100644 --- a/.github/workflows/test-http.yaml +++ b/.github/workflows/test-http.yaml @@ -1,6 +1,7 @@ name: Test HTTP Credentials -on: [pull_request] +on: + - pull_request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index 48639709..dbba26e3 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -1,6 +1,7 @@ name: Test terraform-new/destroy-workspace -on: [pull_request] +on: + - pull_request jobs: create_workspace_12: diff --git a/.github/workflows/test-output.yaml b/.github/workflows/test-output.yaml index 0ebadcd4..ce4c8fad 100644 --- a/.github/workflows/test-output.yaml +++ b/.github/workflows/test-output.yaml @@ -1,6 +1,7 @@ name: Test terraform-output -on: [pull_request] +on: + - pull_request env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -88,4 +89,4 @@ jobs: if [[ "$AWKWARD_OBJ" != "hello \"there\", here are some 'quotes'." ]]; then echo "::error:: fromJson(awkward_compound_output).nested.thevalue not set correctly" exit 1 - fi \ No newline at end of file + fi diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index c71451bd..34a0e044 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -1,6 +1,7 @@ name: Test terraform-plan -on: [pull_request] +on: + - pull_request jobs: no_changes: diff --git a/.github/workflows/test-registry.yaml b/.github/workflows/test-registry.yaml index 985e5af6..a8d88f23 100644 --- a/.github/workflows/test-registry.yaml +++ b/.github/workflows/test-registry.yaml @@ -1,6 +1,7 @@ name: Test registry -on: [pull_request] +on: + - pull_request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-remote-state.yaml b/.github/workflows/test-remote-state.yaml index b39aa6b6..a008a70d 100644 --- a/.github/workflows/test-remote-state.yaml +++ b/.github/workflows/test-remote-state.yaml @@ -1,6 +1,7 @@ name: Test terraform-remote-state -on: [pull_request] +on: + - pull_request env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-remote.yaml index 1d9c4101..37695928 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-remote.yaml @@ -1,6 +1,7 @@ name: Test remote backend -on: [pull_request] +on: + - pull_request jobs: workspaces: @@ -15,21 +16,21 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it doesn't exist uses: ./terraform-new-workspace with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it already exists uses: ./terraform-new-workspace with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Auto apply workspace uses: ./terraform-apply @@ -37,7 +38,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} auto_approve: true var_file: | tests/terraform-cloud/my_variable.tfvars @@ -60,12 +61,12 @@ jobs: echo "::error:: Variables not set correctly" exit 1 fi - + if [[ -n "${{ steps.auto_apply.outputs.text_plan_path }}" ]]; then echo "::error:: text_plan_path should not be set" exit 1 fi - + if [[ -n "${{ steps.auto_apply.outputs.json_plan_path }}" ]]; then echo "::error:: json_plan_path should not be set" exit 1 @@ -77,7 +78,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify auto_apply terraform outputs with workspace prefix run: | @@ -96,7 +97,7 @@ jobs: id: name-output with: path: fixed-workspace-name - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify auto_apply terraform outputs with workspace name run: | @@ -110,7 +111,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | tests/terraform-cloud/my_variable.tfvars variables: | @@ -123,7 +124,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | tests/terraform-cloud/my_variable.tfvars variables: | @@ -146,7 +147,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Plan workspace uses: ./terraform-plan @@ -156,7 +157,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | tests/terraform-cloud/my_variable.tfvars variables: | @@ -182,7 +183,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | tests/terraform-cloud/my_variable.tfvars variables: | @@ -204,23 +205,23 @@ jobs: echo "::error:: Variables not set correctly" exit 1 fi - + if ! grep -q "Terraform will perform the following actions" '${{ steps.apply.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 fi - + if [[ -n "${{ steps.apply.outputs.json_plan_path }}" ]]; then echo "::error:: json_plan_path should not be set" exit 1 - fi + fi - name: Destroy the last workspace uses: ./terraform-destroy-workspace with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-2 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Destroy non-existant workspace uses: ./terraform-destroy-workspace @@ -229,7 +230,7 @@ jobs: with: path: tests/terraform-cloud workspace: ${{ github.head_ref }}-1 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Check failed to destroy run: | if [[ "${{ steps.destroy-non-existant-workspace.outcome }}" != "failure" ]]; then diff --git a/.github/workflows/test-ssh.yaml b/.github/workflows/test-ssh.yaml index 8f083151..3f8bc0ad 100644 --- a/.github/workflows/test-ssh.yaml +++ b/.github/workflows/test-ssh.yaml @@ -1,6 +1,7 @@ name: Test SSH Keys -on: [pull_request] +on: + - pull_request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-target-replace.yaml b/.github/workflows/test-target-replace.yaml index 7a987fd4..338b20c6 100644 --- a/.github/workflows/test-target-replace.yaml +++ b/.github/workflows/test-target-replace.yaml @@ -1,6 +1,7 @@ name: Test plan target and replace -on: [pull_request] +on: + - pull_request env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -240,7 +241,7 @@ jobs: with: path: tests/target workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Plan with no changes in targets uses: ./terraform-plan @@ -253,7 +254,7 @@ jobs: variables: | length = 5 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -272,7 +273,7 @@ jobs: variables: | length = 5 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -291,7 +292,7 @@ jobs: variables: | length = 5 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -310,7 +311,7 @@ jobs: variables: | length = 6 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -329,7 +330,7 @@ jobs: variables: | length = 6 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -355,7 +356,7 @@ jobs: length = 10 auto_approve: true workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -382,7 +383,7 @@ jobs: variables: | length = 10 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -404,7 +405,7 @@ jobs: variables: | length = 10 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -429,7 +430,7 @@ jobs: variables: | length = 10 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -449,7 +450,7 @@ jobs: variables: | length = 10 workspace: ${{ github.head_ref }} - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Verify outputs run: | @@ -470,4 +471,4 @@ jobs: workspace: ${{ github.head_ref }} variables: | length = 10 - backend_config: "token=${{ secrets.TF_API_TOKEN }}" + backend_config: token=${{ secrets.TF_API_TOKEN }} diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 1aabb0f6..45dc3e5c 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -1,6 +1,7 @@ name: Test terraform-validate -on: [pull_request] +on: + - pull_request jobs: valid: diff --git a/.github/workflows/test-version.yaml b/.github/workflows/test-version.yaml index 5866a78e..77c7e3bf 100644 --- a/.github/workflows/test-version.yaml +++ b/.github/workflows/test-version.yaml @@ -1,6 +1,7 @@ name: Test terraform-version -on: [pull_request] +on: + - pull_request jobs: required_version: diff --git a/.github/workflows/test-workflow-commands.yaml b/.github/workflows/test-workflow-commands.yaml index acb50eed..87c752b4 100644 --- a/.github/workflows/test-workflow-commands.yaml +++ b/.github/workflows/test-workflow-commands.yaml @@ -1,6 +1,7 @@ name: Test workflow command supression -on: [ pull_request ] +on: + - pull_request jobs: workflow_command_injection: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 21b88f05..f64fea8b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,6 +1,7 @@ name: Unit test -on: [push] +on: + - push jobs: pytest: diff --git a/example_workflows/check_for_drift.yaml b/example_workflows/check_for_drift.yaml index 86d73a53..da0bce5b 100644 --- a/example_workflows/check_for_drift.yaml +++ b/example_workflows/check_for_drift.yaml @@ -2,7 +2,7 @@ name: Check for infrastructure drift on: schedule: - - cron: "0 8 * * *" + - cron: 0 8 * * * jobs: check_drift: diff --git a/example_workflows/create_plan.yaml b/example_workflows/create_plan.yaml index 87e51c49..73fba43a 100644 --- a/example_workflows/create_plan.yaml +++ b/example_workflows/create_plan.yaml @@ -1,6 +1,7 @@ name: Create terraform plan -on: [pull_request] +on: + - pull_request jobs: plan: From ffeb5ff8b7526e4df659af484d9a4428c4a193be Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 6 Dec 2021 18:52:48 +0000 Subject: [PATCH 176/231] Fix invalid action metadata --- terraform-apply/action.yaml | 5 +++-- terraform-check/action.yaml | 2 +- terraform-destroy-workspace/action.yaml | 2 +- terraform-destroy/action.yaml | 2 +- terraform-plan/action.yaml | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index 8e509920..ec0c4bd7 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -34,14 +34,15 @@ inputs: parallelism: description: Limit the number of concurrent operations required: false - default: 0 + default: "0" label: description: A friendly name for this plan required: false default: "" auto_approve: description: Automatically approve and apply plan - default: false + required: false + default: "false" target: description: List of resources to target for the apply, one per line required: false diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index 37677cfc..c64375c4 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -30,7 +30,7 @@ inputs: parallelism: description: Limit the number of concurrent operations required: false - default: 0 + default: "0" runs: using: docker diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index ccb755c8..ad16642d 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -29,7 +29,7 @@ inputs: parallelism: description: Limit the number of concurrent operations required: false - default: 0 + default: "0" runs: using: docker diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 5f65432a..1f8145dd 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -31,7 +31,7 @@ inputs: parallelism: description: Limit the number of concurrent operations required: false - default: 0 + default: "0" runs: using: docker diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index c55da9c1..9772ffc9 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -30,7 +30,7 @@ inputs: parallelism: description: Limit the number of concurrent operations required: false - default: 0 + default: "0" target: description: List of resources to target for the plan, one per line required: false @@ -46,7 +46,7 @@ inputs: add_github_comment: description: Add the plan to a GitHub PR required: false - default: true + default: "true" outputs: changes: From e1acdc8877486d08086237aecd8ff2d21d4e6e8f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Thu, 9 Dec 2021 09:31:20 +0000 Subject: [PATCH 177/231] Don't try to select workspace during init Behavior changed in terraform 1.1.0 so that if the workspace fails to be selected, the init fails. --- .github/workflows/test-apply.yaml | 23 +- .github/workflows/test-new-workspace.yaml | 296 ++---------------- .github/workflows/test-plan.yaml | 4 +- .github/workflows/test-remote.yaml | 46 +-- .github/workflows/test-target-replace.yaml | 1 + image/actions.sh | 4 +- tests/new-workspace/main.tf | 11 + tests/new-workspace/remote_12/main.tf | 21 -- tests/new-workspace/remote_13/main.tf | 21 -- tests/new-workspace/remote_14/main.tf | 21 -- tests/new-workspace/remote_1_0_10/main.tf | 21 -- tests/terraform-cloud/{ => 0.13}/main.tf | 2 +- .../{ => 0.13}/my_variable.tfvars | 0 tests/terraform-cloud/1.0/main.tf | 38 +++ tests/terraform-cloud/1.0/my_variable.tfvars | 2 + tests/terraform-cloud/1.1/main.tf | 38 +++ tests/terraform-cloud/1.1/my_variable.tfvars | 2 + 17 files changed, 167 insertions(+), 384 deletions(-) create mode 100644 tests/new-workspace/main.tf delete mode 100644 tests/new-workspace/remote_12/main.tf delete mode 100644 tests/new-workspace/remote_13/main.tf delete mode 100644 tests/new-workspace/remote_14/main.tf delete mode 100644 tests/new-workspace/remote_1_0_10/main.tf rename tests/terraform-cloud/{ => 0.13}/main.tf (92%) rename tests/terraform-cloud/{ => 0.13}/my_variable.tfvars (100%) create mode 100644 tests/terraform-cloud/1.0/main.tf create mode 100644 tests/terraform-cloud/1.0/my_variable.tfvars create mode 100644 tests/terraform-cloud/1.1/main.tf create mode 100644 tests/terraform-cloud/1.1/my_variable.tfvars diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 2e78dc26..7b2f13bf 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -112,7 +112,7 @@ jobs: exit 1 fi - if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "1.0" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -164,46 +164,46 @@ jobs: - name: Apply uses: ./terraform-apply - id: output + id: first-apply with: path: tests/apply/changes - name: Verify outputs run: | - if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + if [[ "${{ steps.first-apply.outputs.output_string }}" != "the_string" ]]; then echo "::error:: output s not set correctly" exit 1 fi - if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.output.outputs.json_plan_path }}") != "create" ]]; then + if [[ $(jq -r .output_changes.output_string.actions[0] "${{ steps.first-apply.outputs.json_plan_path }}") != "create" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 fi - if ! grep -q "Terraform will perform the following actions" '${{ steps.output.outputs.text_plan_path }}'; then + if ! grep -q "Terraform will perform the following actions" '${{ steps.first-apply.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 fi - name: Apply uses: ./terraform-apply - id: output + id: second-apply with: path: tests/apply/changes - name: Verify outputs run: | - if [[ "${{ steps.output.outputs.output_string }}" != "the_string" ]]; then + if [[ "${{ steps.second-apply.outputs.output_string }}" != "the_string" ]]; then echo "::error:: output s not set correctly" exit 1 fi - if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.2" ]]; then + if [[ $(jq -r .format_version "${{ steps.second-apply.outputs.json_plan_path }}") != "1.0" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 fi - if ! grep -q "No changes" '${{ steps.output.outputs.text_plan_path }}'; then + if ! grep -q "No changes" '${{ steps.second-apply.outputs.text_plan_path }}'; then echo "::error:: text_plan_path not set correctly" exit 1 fi @@ -511,7 +511,8 @@ jobs: exit 1 fi - if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.2" ]]; then + cat "${{ steps.output.outputs.json_plan_path }}" + if [[ $(jq -r .format_version "${{ steps.output.outputs.json_plan_path }}") != "0.1" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 fi @@ -544,7 +545,7 @@ jobs: exit 1 fi - if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "0.2" ]]; then + if [[ $(jq -r .format_version "${{ steps.apply.outputs.json_plan_path }}") != "1.0" ]]; then echo "::error:: json_plan_path not set correctly" exit 1 fi diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index dbba26e3..66375641 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -4,9 +4,13 @@ on: - pull_request jobs: - create_workspace_12: + workspace_management: runs-on: ubuntu-latest - name: Workspace tests 0.12 + name: Workspace management + strategy: + fail-fast: false + matrix: + tf_version: ['~> 0.12.0', '~> 0.13.0', '~> 0.14.0', '~> 1.0.0', '~> 1.1.0'] env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -14,274 +18,36 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Create first workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_12 - workspace: my-first-workspace - - - name: Create first workspace again - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_12 - workspace: my-first-workspace - - - name: Apply in first workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_12 - workspace: my-first-workspace - variables: my_string="hello" - auto_approve: true - - - name: Create a second workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_12 - workspace: ${{ github.head_ref }} - - - name: Apply in second workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_12 - workspace: ${{ github.head_ref }} - variables: my_string="world" - auto_approve: true - - - name: Get first workspace outputs - uses: ./terraform-output - id: first_12 - with: - path: tests/new-workspace/remote_12 - workspace: my-first-workspace - - - name: Get second workspace outputs - uses: ./terraform-output - id: second_12 - with: - path: tests/new-workspace/remote_12 - workspace: ${{ github.head_ref }} - - - name: Verify outputs + - name: Setup remote backend run: | - if [[ "${{ steps.first_12.outputs.my_string }}" != "hello" ]]; then - echo "::error:: output my_string not set correctly for first workspace" - exit 1 - fi - - if [[ "${{ steps.second_12.outputs.my_string }}" != "world" ]]; then - echo "::error:: output my_string not set correctly for second workspace" - exit 1 - fi - - - name: Destroy first workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_12 - workspace: my-first-workspace - variables: my_string="hello" - - - name: Destroy second workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_12 - workspace: ${{ github.head_ref }} - variables: my_string="world" - - create_workspace_13: - runs-on: ubuntu-latest - name: Workspace tests 0.13 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Create first workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_13 - workspace: my-first-workspace - - - name: Create first workspace again - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_13 - workspace: my-first-workspace - - - name: Apply in first workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_13 - workspace: my-first-workspace - variables: my_string="hello" - auto_approve: true - - - name: Create a second workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_13 - workspace: ${{ github.head_ref }} - - - name: Apply in second workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_13 - workspace: ${{ github.head_ref }} - variables: my_string="world" - auto_approve: true - - - name: Get first workspace outputs - uses: ./terraform-output - id: first_13 - with: - path: tests/new-workspace/remote_13 - workspace: my-first-workspace - - - name: Get second workspace outputs - uses: ./terraform-output - id: second_13 - with: - path: tests/new-workspace/remote_13 - workspace: ${{ github.head_ref }} - - - name: Verify outputs - run: | - if [[ "${{ steps.first_13.outputs.my_string }}" != "hello" ]]; then - echo "::error:: output my_string not set correctly for first workspace" - exit 1 - fi - - if [[ "${{ steps.second_13.outputs.my_string }}" != "world" ]]; then - echo "::error:: output my_string not set correctly for second workspace" - exit 1 - fi - - - name: Destroy first workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_13 - workspace: my-first-workspace - variables: my_string="hello" - - - name: Destroy second workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_13 - workspace: ${{ github.head_ref }} - variables: my_string="world" - - create_workspace_14: - runs-on: ubuntu-latest - name: Workspace tests 0.14 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Create first workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_14 - workspace: test-workspace - - - name: Create first workspace again - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_14 - workspace: test-workspace - - - name: Apply in first workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_14 - workspace: test-workspace - variables: my_string="hello" - auto_approve: true - - - name: Create a second workspace - uses: ./terraform-new-workspace - with: - path: tests/new-workspace/remote_14 - workspace: ${{ github.head_ref }} - - - name: Apply in second workspace - uses: ./terraform-apply - with: - path: tests/new-workspace/remote_14 - workspace: ${{ github.head_ref }} - variables: my_string="world" - auto_approve: true - - - name: Get first workspace outputs - uses: ./terraform-output - id: first_14 - with: - path: tests/new-workspace/remote_14 - workspace: test-workspace - - - name: Get second workspace outputs - uses: ./terraform-output - id: second_14 - with: - path: tests/new-workspace/remote_14 - workspace: ${{ github.head_ref }} - - - name: Verify outputs - run: | - if [[ "${{ steps.first_14.outputs.my_string }}" != "hello" ]]; then - echo "::error:: output my_string not set correctly for first workspace" - exit 1 - fi - - if [[ "${{ steps.second_14.outputs.my_string }}" != "world" ]]; then - echo "::error:: output my_string not set correctly for second workspace" - exit 1 - fi - - - name: Destroy first workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_14 - workspace: test-workspace - variables: my_string="hello" - - - name: Destroy second workspace - uses: ./terraform-destroy-workspace - with: - path: tests/new-workspace/remote_14 - workspace: ${{ github.head_ref }} - variables: my_string="world" - - create_workspace_1_0_10: - runs-on: ubuntu-latest - name: Workspace tests 1.0.10 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v2 + cat >tests/new-workspace/backend.tf < fixed-workspace-name/main.tf + if [[ "${{ matrix.tf_version }}" == "0.13" ]]; then + sed -e 's/prefix.*/name = "github-actions-0-13-${{ github.head_ref }}-1"/' tests/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + else + sed -e 's/prefix.*/name = "github-actions-1-1-${{ github.head_ref }}-1"/' tests/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + fi - name: Get outputs uses: ./terraform-output @@ -109,11 +117,11 @@ jobs: - name: Check no changes uses: ./terraform-check with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/my_variable.tfvars + tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -122,11 +130,11 @@ jobs: id: check continue-on-error: true with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/my_variable.tfvars + tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="Changed!" @@ -145,7 +153,7 @@ jobs: - name: Destroy workspace uses: ./terraform-destroy-workspace with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -155,11 +163,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/my_variable.tfvars + tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -181,11 +189,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/my_variable.tfvars + tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -219,7 +227,7 @@ jobs: - name: Destroy the last workspace uses: ./terraform-destroy-workspace with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -228,7 +236,7 @@ jobs: continue-on-error: true id: destroy-non-existant-workspace with: - path: tests/terraform-cloud + path: tests/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Check failed to destroy diff --git a/.github/workflows/test-target-replace.yaml b/.github/workflows/test-target-replace.yaml index 338b20c6..a43a6068 100644 --- a/.github/workflows/test-target-replace.yaml +++ b/.github/workflows/test-target-replace.yaml @@ -233,6 +233,7 @@ jobs: prefix = "github-actions-replace-" } } + required_version = "~> 1.0.4" } EOF diff --git a/image/actions.sh b/image/actions.sh index e5eb34e2..7d0b726d 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -166,7 +166,7 @@ function init-backend() { set +e # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false $INIT_ARGS \ + (cd "$INPUT_PATH" && terraform init -input=false $INIT_ARGS \ 2>"$STEP_TMP_DIR/terraform_init.stderr") local INIT_EXIT=$? @@ -178,7 +178,7 @@ function init-backend() { if grep -q "No existing workspaces." "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Failed to select workspace" "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Currently selected workspace.*does not exist" "$STEP_TMP_DIR/terraform_init.stderr"; then # Couldn't select workspace, but we don't really care. # select-workspace will give a better error if the workspace is required to exist - : + cat "$STEP_TMP_DIR/terraform_init.stderr" else cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 exit $INIT_EXIT diff --git a/tests/new-workspace/main.tf b/tests/new-workspace/main.tf new file mode 100644 index 00000000..899e2ca4 --- /dev/null +++ b/tests/new-workspace/main.tf @@ -0,0 +1,11 @@ +resource "random_string" "my_string" { + length = 5 +} + +variable "my_string" { + type = string +} + +output "my_string" { + value = var.my_string +} diff --git a/tests/new-workspace/remote_12/main.tf b/tests/new-workspace/remote_12/main.tf deleted file mode 100644 index 602e9a26..00000000 --- a/tests/new-workspace/remote_12/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "random_string" "my_string" { - length = 5 -} - -variable "my_string" { - type = string -} - -output "my_string" { - value = var.my_string -} - -terraform { - backend "s3" { - bucket = "terraform-github-actions" - key = "terraform-new-workspace" - region = "eu-west-2" - } - - required_version = "~> 0.12.0" -} \ No newline at end of file diff --git a/tests/new-workspace/remote_13/main.tf b/tests/new-workspace/remote_13/main.tf deleted file mode 100644 index df825df8..00000000 --- a/tests/new-workspace/remote_13/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "random_string" "my_string" { - length = 5 -} - -variable "my_string" { - type = string -} - -output "my_string" { - value = var.my_string -} - -terraform { - backend "s3" { - bucket = "terraform-github-actions" - key = "terraform-new-workspace-13" - region = "eu-west-2" - } - - required_version = "~> 0.13.0" -} \ No newline at end of file diff --git a/tests/new-workspace/remote_14/main.tf b/tests/new-workspace/remote_14/main.tf deleted file mode 100644 index 9c782c1b..00000000 --- a/tests/new-workspace/remote_14/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "random_string" "my_string" { - length = 5 -} - -variable "my_string" { - type = string -} - -output "my_string" { - value = var.my_string -} - -terraform { - backend "s3" { - bucket = "terraform-github-actions" - key = "terraform-new-workspace-14" - region = "eu-west-2" - } - - required_version = "~> 0.14.0" -} diff --git a/tests/new-workspace/remote_1_0_10/main.tf b/tests/new-workspace/remote_1_0_10/main.tf deleted file mode 100644 index 20d23a9f..00000000 --- a/tests/new-workspace/remote_1_0_10/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "random_string" "my_string" { - length = 5 -} - -variable "my_string" { - type = string -} - -output "my_string" { - value = var.my_string -} - -terraform { - backend "s3" { - bucket = "terraform-github-actions" - key = "terraform-new-workspace-1-0-10" - region = "eu-west-2" - } - - required_version = "~> 1.0.10" -} diff --git a/tests/terraform-cloud/main.tf b/tests/terraform-cloud/0.13/main.tf similarity index 92% rename from tests/terraform-cloud/main.tf rename to tests/terraform-cloud/0.13/main.tf index f0072073..f8c913cb 100644 --- a/tests/terraform-cloud/main.tf +++ b/tests/terraform-cloud/0.13/main.tf @@ -3,7 +3,7 @@ terraform { organization = "flooktech" workspaces { - prefix = "github-actions-" + prefix = "github-actions-0-13-" } } required_version = "0.13.0" diff --git a/tests/terraform-cloud/my_variable.tfvars b/tests/terraform-cloud/0.13/my_variable.tfvars similarity index 100% rename from tests/terraform-cloud/my_variable.tfvars rename to tests/terraform-cloud/0.13/my_variable.tfvars diff --git a/tests/terraform-cloud/1.0/main.tf b/tests/terraform-cloud/1.0/main.tf new file mode 100644 index 00000000..62e605cb --- /dev/null +++ b/tests/terraform-cloud/1.0/main.tf @@ -0,0 +1,38 @@ +terraform { + backend "remote" { + organization = "flooktech" + + workspaces { + prefix = "github-actions-1-1-" + } + } + required_version = "~> 1.0.0" +} + +resource "random_id" "the_id" { + byte_length = 5 +} + +variable "default" { + default = "default" +} + +output "default" { + value = var.default +} + +variable "from_tfvars" { + default = "default" +} + +output "from_tfvars" { + value = var.from_tfvars +} + +variable "from_variables" { + default = "default" +} + +output "from_variables" { + value = var.from_variables +} diff --git a/tests/terraform-cloud/1.0/my_variable.tfvars b/tests/terraform-cloud/1.0/my_variable.tfvars new file mode 100644 index 00000000..8fa611eb --- /dev/null +++ b/tests/terraform-cloud/1.0/my_variable.tfvars @@ -0,0 +1,2 @@ +from_tfvars="from_tfvars" +from_variables="from_tfvars" diff --git a/tests/terraform-cloud/1.1/main.tf b/tests/terraform-cloud/1.1/main.tf new file mode 100644 index 00000000..e830f003 --- /dev/null +++ b/tests/terraform-cloud/1.1/main.tf @@ -0,0 +1,38 @@ +terraform { + backend "remote" { + organization = "flooktech" + + workspaces { + prefix = "github-actions-1-1" + } + } + required_version = "1.1.0" +} + +resource "random_id" "the_id" { + byte_length = 5 +} + +variable "default" { + default = "default" +} + +output "default" { + value = var.default +} + +variable "from_tfvars" { + default = "default" +} + +output "from_tfvars" { + value = var.from_tfvars +} + +variable "from_variables" { + default = "default" +} + +output "from_variables" { + value = var.from_variables +} diff --git a/tests/terraform-cloud/1.1/my_variable.tfvars b/tests/terraform-cloud/1.1/my_variable.tfvars new file mode 100644 index 00000000..8fa611eb --- /dev/null +++ b/tests/terraform-cloud/1.1/my_variable.tfvars @@ -0,0 +1,2 @@ +from_tfvars="from_tfvars" +from_variables="from_tfvars" From ac954182ed48710a5c73027d3226c4cb540638ce Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 12 Dec 2021 22:10:13 +0000 Subject: [PATCH 178/231] :bookmark: v1.21.1 --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98800609..194fa8cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,20 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.21.0` to use an exact release +- `@v1.21.1` to use an exact release - `@v1.21` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.21.1] - 2021-12-12 + +### Fixed +- [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) support for Terraform v1.1.0. + + This stopped working after a change in the behaviour of terraform init. + + There is an outstanding [issue in Terraform v1.1.0](https://github.com/hashicorp/terraform/issues/30129) using the `remote` backend that prevents creating a new workspace when no workspaces currently exist. + If you are affected by this, you can pin to an earlier version of Terraform using one of methods listed in the [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) docs. + ## [1.21.0] - 2021-12-04 ### Added @@ -321,6 +331,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.21.1]: https://github.com/dflook/terraform-github-actions/compare/v1.21.0...v1.21.1 [1.21.0]: https://github.com/dflook/terraform-github-actions/compare/v1.20.1...v1.21.0 [1.20.1]: https://github.com/dflook/terraform-github-actions/compare/v1.20.0...v1.20.1 [1.20.0]: https://github.com/dflook/terraform-github-actions/compare/v1.19.0...v1.20.0 From 2f894ab1fc479b91e00a57d6f333dbaa24ede215 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 17 Dec 2021 17:19:40 +0000 Subject: [PATCH 179/231] Expand tf version detection --- .github/github_sucks.md | 4 - .github/workflows/test-validate.yaml | 2 +- .github/workflows/test-version.yaml | 286 +++- .github/workflows/test.yaml | 6 +- .gitignore | 1 + image/Dockerfile | 10 +- image/Dockerfile-base | 15 +- image/actions.sh | 111 +- image/entrypoints/apply.sh | 5 +- image/entrypoints/check.sh | 3 +- image/entrypoints/destroy-workspace.sh | 17 +- image/entrypoints/destroy.sh | 3 +- image/entrypoints/new-workspace.sh | 8 +- image/entrypoints/output.sh | 3 +- image/entrypoints/plan.sh | 3 +- image/entrypoints/remote-state.sh | 5 +- image/entrypoints/version.sh | 1 + image/setup.py | 20 + image/src/github_actions/__init__.py | 1 + image/src/github_actions/debug.py | 10 + image/src/github_actions/env.py | 19 + image/src/github_actions/inputs.py | 88 ++ image/src/terraform/__init__.py | 1 + image/src/terraform/cloud.py | 246 ++++ image/src/terraform/download.py | 104 ++ image/src/terraform/exec.py | 25 + image/src/terraform/module.py | 254 ++++ image/src/terraform/versions.py | 220 ++++ image/src/terraform_backend/__init__.py | 1 + image/src/terraform_backend/__main__.py | 21 + .../src/terraform_cloud_workspace/__init__.py | 1 + .../src/terraform_cloud_workspace/__main__.py | 95 ++ image/src/terraform_version/__init__.py | 1 + image/src/terraform_version/__main__.py | 120 ++ image/src/terraform_version/asdf.py | 43 + .../backend_constraints.json | 1166 +++++++++++++++++ image/src/terraform_version/env.py | 26 + image/src/terraform_version/local_state.py | 35 + image/src/terraform_version/remote_state.py | 230 ++++ .../src/terraform_version/remote_workspace.py | 47 + .../src/terraform_version/required_version.py | 26 + image/src/terraform_version/tfenv.py | 61 + image/src/terraform_version/tfswitch.py | 43 + image/tools/convert_output.py | 1 + image/tools/convert_validate_report.py | 3 +- image/tools/convert_version.py | 2 +- image/tools/format_tf_credentials.py | 2 +- image/tools/github_comment_react.py | 2 +- image/tools/github_pr_comment.py | 3 +- image/tools/http_credential_actions_helper.py | 4 +- image/tools/latest_terraform_version.py | 35 - image/tools/workspace_exists.py | 1 + terraform-apply/action.yaml | 2 +- terraform-check/action.yaml | 6 +- terraform-destroy-workspace/action.yaml | 4 +- terraform-destroy/action.yaml | 4 +- terraform-fmt-check/README.md | 60 + terraform-fmt-check/action.yaml | 12 + terraform-fmt/README.md | 60 + terraform-fmt/action.yaml | 12 + terraform-new-workspace/action.yaml | 6 +- terraform-output/action.yaml | 4 +- terraform-plan/action.yaml | 2 + terraform-remote-state/README.md | 2 +- terraform-validate/README.md | 36 +- terraform-validate/action.yaml | 8 + terraform-version/README.md | 60 +- terraform-version/action.yaml | 12 + tests/python/terraform_version/test_asdf.py | 46 + .../terraform_version/test_local_state.py | 151 +++ .../terraform_version/test_remote_state_s3.py | 194 +++ tests/python/terraform_version/test_state.py | 82 ++ .../test_terraform_version.py | 203 +++ tests/python/terraform_version/test_tfc.py | 0 tests/python/terraform_version/test_tfenv.py | 46 + .../python/terraform_version/test_tfswitch.py | 27 + tests/requirements.txt | 6 + tests/version/asdf/.tool-versions | 5 + tests/version/cloud/main.tf | 8 + tests/version/local/main.tf | 4 + tests/version/local/terraform.tfstate | 13 + tests/version/providers/0.11/main.tf | 4 +- tests/version/providers/0.12/main.tf | 4 +- tests/version/providers/0.13/versions.tf | 2 +- tests/version/state/main.tf | 13 + tests/version/terraform-cloud/main.tf | 9 + tests/version/test_version.py | 4 + .../tfenv/{.tfswitchrc => .terraform-version} | 0 88 files changed, 4403 insertions(+), 148 deletions(-) create mode 100644 image/setup.py create mode 100644 image/src/github_actions/__init__.py create mode 100644 image/src/github_actions/debug.py create mode 100644 image/src/github_actions/env.py create mode 100644 image/src/github_actions/inputs.py create mode 100644 image/src/terraform/__init__.py create mode 100644 image/src/terraform/cloud.py create mode 100644 image/src/terraform/download.py create mode 100644 image/src/terraform/exec.py create mode 100644 image/src/terraform/module.py create mode 100644 image/src/terraform/versions.py create mode 100644 image/src/terraform_backend/__init__.py create mode 100644 image/src/terraform_backend/__main__.py create mode 100644 image/src/terraform_cloud_workspace/__init__.py create mode 100644 image/src/terraform_cloud_workspace/__main__.py create mode 100644 image/src/terraform_version/__init__.py create mode 100644 image/src/terraform_version/__main__.py create mode 100644 image/src/terraform_version/asdf.py create mode 100644 image/src/terraform_version/backend_constraints.json create mode 100644 image/src/terraform_version/env.py create mode 100644 image/src/terraform_version/local_state.py create mode 100644 image/src/terraform_version/remote_state.py create mode 100644 image/src/terraform_version/remote_workspace.py create mode 100644 image/src/terraform_version/required_version.py create mode 100644 image/src/terraform_version/tfenv.py create mode 100644 image/src/terraform_version/tfswitch.py delete mode 100755 image/tools/latest_terraform_version.py create mode 100644 tests/python/terraform_version/test_asdf.py create mode 100644 tests/python/terraform_version/test_local_state.py create mode 100644 tests/python/terraform_version/test_remote_state_s3.py create mode 100644 tests/python/terraform_version/test_state.py create mode 100644 tests/python/terraform_version/test_terraform_version.py create mode 100644 tests/python/terraform_version/test_tfc.py create mode 100644 tests/python/terraform_version/test_tfenv.py create mode 100644 tests/python/terraform_version/test_tfswitch.py create mode 100644 tests/version/asdf/.tool-versions create mode 100644 tests/version/cloud/main.tf create mode 100644 tests/version/local/main.tf create mode 100644 tests/version/local/terraform.tfstate create mode 100644 tests/version/state/main.tf create mode 100644 tests/version/terraform-cloud/main.tf rename tests/version/tfenv/{.tfswitchrc => .terraform-version} (100%) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index 325766d6..471620e2 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,6 +1,2 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. - - - - diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 45dc3e5c..af9573b3 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -52,7 +52,7 @@ jobs: validate_workspace: runs-on: ubuntu-latest - name: Invalid terraform configuration + name: Use workspace name during validationg steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/test-version.yaml b/.github/workflows/test-version.yaml index 77c7e3bf..40d484a2 100644 --- a/.github/workflows/test-version.yaml +++ b/.github/workflows/test-version.yaml @@ -96,6 +96,290 @@ jobs: exit 1 fi + asdf: + runs-on: ubuntu-latest + name: asdf + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + with: + path: tests/version/asdf + + - name: Print the version + run: echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" + + - name: Check the version + run: | + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "0.12.11" ]]; then + echo "::error:: Terraform version not set from .terraform-version" + exit 1 + fi + + env: + runs-on: ubuntu-latest + name: TERRAFORM_VERSION range + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + env: + TERRAFORM_VERSION: ">=0.12.0,<=0.12.5" + with: + path: tests/version/empty + + - name: Print the version + run: echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" + + - name: Check the version + run: | + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "0.12.5" ]]; then + echo "::error:: Terraform version not set from required_version range" + exit 1 + fi + + tfc_workspace: + runs-on: ubuntu-latest + name: TFC Workspace + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create workspace + uses: ./terraform-new-workspace + env: + TERRAFORM_VERSION: 0.12.13 + with: + path: tests/version/terraform-cloud + workspace: test-1 + backend_config: token=${{ secrets.TF_API_TOKEN }} + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + with: + path: tests/version/terraform-cloud + workspace: test-1 + backend_config: token=${{ secrets.TF_API_TOKEN }} + + - name: Destroy workspace + uses: ./terraform-destroy-workspace + with: + path: tests/version/terraform-cloud + workspace: test-1 + backend_config: token=${{ secrets.TF_API_TOKEN }} + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" + + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "0.12.13" ]]; then + echo "::error:: Terraform version not set from remote workspace" + exit 1 + fi + + tfc_cloud_workspace: + runs-on: ubuntu-latest + name: TFC Cloud Configuration + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create workspace + uses: ./terraform-new-workspace + env: + TERRAFORM_VERSION: 1.1.2 + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} + with: + path: tests/version/cloud + workspace: tfc_cloud_workspace-1 + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} + with: + path: tests/version/cloud + workspace: tfc_cloud_workspace-1 + + - name: Destroy workspace + uses: ./terraform-destroy-workspace + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} + with: + path: tests/version/cloud + workspace: tfc_cloud_workspace-1 + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" + + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "1.1.2" ]]; then + echo "::error:: Terraform version not set from remote workspace" + exit 1 + fi + + local_state: + runs-on: ubuntu-latest + name: Local State file + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + with: + path: tests/version/local + + - name: Print the version + run: | + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "0.15.4" ]]; then + echo "::error:: Terraform version not set from state file" + exit 1 + fi + + remote_state: + runs-on: ubuntu-latest + name: Remote State file + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Apply default workspace + uses: ./terraform-apply + env: + TERRAFORM_VERSION: 0.12.9 + with: + variables: my_variable="hello" + path: tests/version/state + auto_approve: true + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version + with: + path: tests/version/state + + - name: Destroy default workspace + uses: ./terraform-destroy + with: + path: tests/version/state + variables: my_variable="goodbye" + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" + + if [[ "${{ steps.terraform-version.outputs.terraform }}" != "0.12.9" ]]; then + echo "::error:: Terraform version not set from state file" + exit 1 + fi + + - name: Create second workspace + uses: ./terraform-new-workspace + env: + TERRAFORM_VERSION: 1.1.0 + with: + path: tests/version/state + workspace: second + + - name: Apply second workspace + uses: ./terraform-apply + with: + variables: my_variable="goodbye" + path: tests/version/state + workspace: second + auto_approve: true + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version-second + with: + path: tests/version/state + workspace: second + + - name: Destroy second workspace + uses: ./terraform-destroy-workspace + with: + path: tests/version/state + workspace: second + variables: my_variable="goodbye" + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version-second.outputs.terraform }}" + + if [[ "${{ steps.terraform-version-second.outputs.terraform }}" != "1.1.0" ]]; then + echo "::error:: Terraform version not set from state file" + exit 1 + fi + + - name: Create third workspace + uses: ./terraform-new-workspace + env: + TERRAFORM_VERSION: 0.13.0 + with: + path: tests/version/state + workspace: third + + - name: Apply third workspace + uses: ./terraform-apply + with: + variables: my_variable="goodbye" + path: tests/version/state + workspace: third + auto_approve: true + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version-third + with: + path: tests/version/state + workspace: third + + - name: Destroy third workspace + uses: ./terraform-destroy-workspace + with: + path: tests/version/state + workspace: third + variables: my_variable="goodbye" + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version-third.outputs.terraform }}" + + if [[ "${{ steps.terraform-version-third.outputs.terraform }}" != "0.13.0" ]]; then + echo "::error:: Terraform version not set from state file" + exit 1 + fi + + - name: Test terraform-version + uses: ./terraform-version + id: terraform-version-fourth + with: + path: tests/version/state + workspace: fourth + + - name: Print the version + run: | + echo "The terraform version was ${{ steps.terraform-version-fourth.outputs.terraform }}" + + if [[ "${{ steps.terraform-version-fourth.outputs.terraform }}" != "1."* ]]; then + echo "::error:: Terraform version not set to latest when no existing state" + exit 1 + fi + empty_path: runs-on: ubuntu-latest name: latest @@ -114,7 +398,7 @@ jobs: - name: Check the version run: | - if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"1.0"* ]]; then + if [[ "${{ steps.terraform-version.outputs.terraform }}" != *"1.1"* ]]; then echo "::error:: Latest version was not used" exit 1 fi diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f64fea8b..93340ffd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,5 +22,9 @@ jobs: pip install -r tests/requirements.txt - name: Run tests + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: No run: | - GITHUB_TOKEN=No PYTHONPATH=image/tools pytest tests + PYTHONPATH=image/tools:image/src pytest tests diff --git a/.gitignore b/.gitignore index 623553e8..38965798 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .pytest_cache/ /venv/ .terraform.lock.hcl +.terraform-bin-dir/ diff --git a/image/Dockerfile b/image/Dockerfile index ddd9a4c3..eba65267 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -1,4 +1,11 @@ -FROM danielflook/terraform-github-actions-base:latest +FROM danielflook/terraform-github-actions-base:2022-01-22 + +COPY src/ /tmp/src/ +COPY setup.py /tmp +RUN pip install /tmp \ + && rm -rf /tmp/src /tmp/setup.py \ + && TERRAFORM_BIN_DIR="/usr/local/bin" terraform-version 0.9.0 \ + && TERRAFORM_BIN_DIR="/usr/local/bin" terraform-version 0.12.0 COPY entrypoints/ /entrypoints/ COPY actions.sh /usr/local/actions.sh @@ -6,7 +13,6 @@ COPY workflow_commands.sh /usr/local/workflow_commands.sh COPY tools/convert_validate_report.py /usr/local/bin/convert_validate_report COPY tools/github_pr_comment.py /usr/local/bin/github_pr_comment -COPY tools/latest_terraform_version.py /usr/local/bin/latest_terraform_version COPY tools/convert_output.py /usr/local/bin/convert_output COPY tools/plan_cmp.py /usr/local/bin/plan_cmp COPY tools/convert_version.py /usr/local/bin/convert_version diff --git a/image/Dockerfile-base b/image/Dockerfile-base index 1f7e7bb3..556e68fe 100644 --- a/image/Dockerfile-base +++ b/image/Dockerfile-base @@ -5,16 +5,14 @@ RUN cd tfmask && make && make go/build FROM debian:bullseye-slim as base -ARG DEFAULT_TF_VERSION=0.14.4 -ARG TFSWITCH_VERSION=0.8.832 - # Terraform environment variables ENV CHECKPOINT_DISABLE=true -ENV TF_IN_AUTOMATION=yep +ENV TF_IN_AUTOMATION=true ENV TF_INPUT=false ENV TF_PLUGIN_CACHE_DIR=/usr/local/share/terraform/plugin-cache -RUN apt-get update && apt-get install -y \ +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ git \ ssh \ tar \ @@ -23,19 +21,12 @@ RUN apt-get update && apt-get install -y \ curl \ unzip \ jq \ - python2 \ python3 \ python3-requests \ python3-pip \ wget \ && rm -rf /var/lib/apt/lists/* -RUN curl -fsL https://github.com/warrensbox/terraform-switcher/releases/download/${TFSWITCH_VERSION}/terraform-switcher_${TFSWITCH_VERSION}_linux_amd64.tar.gz -o tfswitch.tar.gz \ - && tar -xvf tfswitch.tar.gz \ - && mv tfswitch /usr/local/bin \ - && rm -rf README.md LICENSE terraform-switcher tfswitch.tar.gz \ - && tfswitch $DEFAULT_TF_VERSION \ - && mv /root/.terraform.versions /root/.terraform.versions.default RUN mkdir -p $TF_PLUGIN_CACHE_DIR COPY --from=tfmask /go/tfmask/release/tfmask /usr/local/bin/tfmask diff --git a/image/actions.sh b/image/actions.sh index 7d0b726d..c0dc32af 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -16,18 +16,9 @@ function debug() { } function detect-terraform-version() { - local TF_SWITCH_OUTPUT - - debug_cmd tfswitch --version - - TF_SWITCH_OUTPUT=$(cd "$INPUT_PATH" && echo "" | tfswitch | grep -e Switched -e Reading | sed 's/^.*Switched/Switched/') - if echo "$TF_SWITCH_OUTPUT" | grep Reading >/dev/null; then - echo "$TF_SWITCH_OUTPUT" - else - echo "Reading latest terraform version" - tfswitch "$(latest_terraform_version)" - fi - + debug_cmd ls -la "/usr/local/bin" + debug_cmd ls -la "$JOB_TMP_DIR/terraform-bin-dir" + TERRAFORM_BIN_DIR="/usr/local/bin:$JOB_TMP_DIR/terraform-bin-dir" terraform-version debug_cmd ls -la "$(which terraform)" local TF_VERSION @@ -89,31 +80,23 @@ function setup() { debug_file "$STEP_TMP_DIR/github_comment_react.stderr" fi - local TERRAFORM_BIN_DIR - TERRAFORM_BIN_DIR="$JOB_TMP_DIR/terraform-bin-dir" - # tfswitch guesses the wrong home directory... - start_group "Installing Terraform" - if [[ ! -d $TERRAFORM_BIN_DIR ]]; then - debug_log "Initializing tfswitch with image default version" - mkdir -p "$TERRAFORM_BIN_DIR" - cp --recursive /root/.terraform.versions.default "$TERRAFORM_BIN_DIR" - fi - - ln -s "$TERRAFORM_BIN_DIR" /root/.terraform.versions - - debug_cmd ls -lad /root/.terraform.versions - debug_cmd ls -lad "$TERRAFORM_BIN_DIR" - debug_cmd ls -la "$TERRAFORM_BIN_DIR" - export TF_DATA_DIR="$STEP_TMP_DIR/terraform-data-dir" export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" - mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" + mkdir -p "$TF_DATA_DIR" "$TF_PLUGIN_CACHE_DIR" "$JOB_TMP_DIR/terraform-bin-dir" unset TF_WORKSPACE + write_credentials + + start_group "Installing Terraform" + detect-terraform-version - debug_cmd ls -la "$TERRAFORM_BIN_DIR" + readonly TERRAFORM_BACKEND_TYPE=$(terraform-backend) + if [[ "$TERRAFORM_BACKEND_TYPE" != "" ]]; then + echo "Detected $TERRAFORM_BACKEND_TYPE backend" + fi + end_group detect-tfmask @@ -130,22 +113,21 @@ function relative_to() { realpath --no-symlinks --canonicalize-missing --relative-to="$absbase" "$relpath" } +## +# Initialize terraform without a backend +# +# This only validates and installs plugins function init() { start_group "Initializing Terraform" - write_credentials - rm -rf "$TF_DATA_DIR" + debug_log terraform init -input=false -backend=false (cd "$INPUT_PATH" && terraform init -input=false -backend=false) end_group } -function init-backend() { - start_group "Initializing Terraform" - - write_credentials - +function set-init-args() { INIT_ARGS="" if [[ -n "$INPUT_BACKEND_CONFIG_FILE" ]]; then @@ -161,9 +143,60 @@ function init-backend() { fi export INIT_ARGS +} + +## +# Initialize the backend for a specific workspace +# +# The workspace must already exist, or the job will be failed +function init-backend-workspace() { + start_group "Initializing Terraform" + + set-init-args + + rm -rf "$TF_DATA_DIR" + + debug_log TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false '$INIT_ARGS' # don't expand INIT_ARGS + + set +e + # shellcheck disable=SC2086 + (cd "$INPUT_PATH" && TF_WORKSPACE=$INPUT_WORKSPACE terraform init -input=false $INIT_ARGS \ + 2>"$STEP_TMP_DIR/terraform_init.stderr") + + local INIT_EXIT=$? + set -e + + if [[ $INIT_EXIT -eq 0 ]]; then + cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 + else + if grep -q "No existing workspaces." "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Failed to select workspace" "$STEP_TMP_DIR/terraform_init.stderr" || grep -q "Currently selected workspace.*does not exist" "$STEP_TMP_DIR/terraform_init.stderr"; then + # Couldn't select workspace, but we don't really care. + # select-workspace will give a better error if the workspace is required to exist + cat "$STEP_TMP_DIR/terraform_init.stderr" + else + cat "$STEP_TMP_DIR/terraform_init.stderr" >&2 + exit $INIT_EXIT + fi + fi + + end_group + + select-workspace +} + +## +# Initialize terraform to use the default workspace +# +# This can be used to initialize when you don't know if a given workspace exists +# This can NOT be used with remote backend, as they have no default workspace +function init-backend-default-workspace() { + start_group "Initializing Terraform" + + set-init-args rm -rf "$TF_DATA_DIR" + debug_log terraform init -input=false '$INIT_ARGS' # don't expand INIT_ARGS set +e # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform init -input=false $INIT_ARGS \ @@ -191,6 +224,7 @@ function init-backend() { function select-workspace() { local WORKSPACE_EXIT + debug_log terraform workspace select "$INPUT_WORKSPACE" set +e (cd "$INPUT_PATH" && terraform workspace select "$INPUT_WORKSPACE") >"$STEP_TMP_DIR/workspace_select" 2>&1 WORKSPACE_EXIT=$? @@ -285,6 +319,7 @@ function set-remote-plan-args() { } function output() { + debug_log terraform output -json (cd "$INPUT_PATH" && terraform output -json | convert_output) } @@ -327,7 +362,7 @@ function plan() { fi # shellcheck disable=SC2086 - debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG $PLAN_ARGS + debug_log terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG '$PLAN_ARGS' # don't expand PLAN_ARGS set +e # shellcheck disable=SC2086 diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 3bf8890e..88bf6fb8 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -5,8 +5,7 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace set-plan-args PLAN_OUT="$STEP_TMP_DIR/plan.out" @@ -32,7 +31,7 @@ function apply() { # Instead we need to do an auto approved apply using the arguments we would normally use for the plan # shellcheck disable=SC2086 - debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS + debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' # don't expand plan args # shellcheck disable=SC2086 (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK APPLY_EXIT=${PIPESTATUS[0]} diff --git a/image/entrypoints/check.sh b/image/entrypoints/check.sh index 4f14979c..91bf1d4b 100755 --- a/image/entrypoints/check.sh +++ b/image/entrypoints/check.sh @@ -5,8 +5,7 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace set-plan-args exec 3>&1 diff --git a/image/entrypoints/destroy-workspace.sh b/image/entrypoints/destroy-workspace.sh index 84f8559e..1563c3bf 100755 --- a/image/entrypoints/destroy-workspace.sh +++ b/image/entrypoints/destroy-workspace.sh @@ -5,8 +5,7 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace set-plan-args exec 3>&1 @@ -26,10 +25,12 @@ if [[ $DESTROY_EXIT -eq 1 ]]; then exit 1 fi -# We can't delete an active workspace, so re-initialize with a 'default' workspace (which may not exist) -workspace=$INPUT_WORKSPACE -INPUT_WORKSPACE=default -init-backend +if [[ "$TERRAFORM_BACKEND_TYPE" == "remote" ]]; then + terraform-cloud-workspace delete "$INPUT_WORKSPACE" +else + # We can't delete an active workspace, so re-initialize with a 'default' workspace (which may not exist) + init-backend-default-workspace -debug_log terraform workspace delete -no-color -lock-timeout=300s "$workspace" -(cd "$INPUT_PATH" && terraform workspace delete -no-color -lock-timeout=300s "$workspace") + debug_log terraform workspace delete -no-color -lock-timeout=300s "$INPUT_WORKSPACE" + (cd "$INPUT_PATH" && terraform workspace delete -no-color -lock-timeout=300s "$INPUT_WORKSPACE") +fi diff --git a/image/entrypoints/destroy.sh b/image/entrypoints/destroy.sh index 860bab64..134a3895 100755 --- a/image/entrypoints/destroy.sh +++ b/image/entrypoints/destroy.sh @@ -5,8 +5,7 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace set-plan-args exec 3>&1 diff --git a/image/entrypoints/new-workspace.sh b/image/entrypoints/new-workspace.sh index 13af1ec5..8ab4982e 100755 --- a/image/entrypoints/new-workspace.sh +++ b/image/entrypoints/new-workspace.sh @@ -5,7 +5,13 @@ source /usr/local/actions.sh debug setup -init-backend + +if [[ "$TERRAFORM_BACKEND_TYPE" == "remote" ]]; then + TERRAFORM_VERSION="$TERRAFORM_VER_MAJOR.$TERRAFORM_VER_MINOR.$TERRAFORM_VER_PATCH" terraform-cloud-workspace new "$INPUT_WORKSPACE" + exit 0 +fi + +init-backend-default-workspace set +e (cd "$INPUT_PATH" && terraform workspace list -no-color) \ diff --git a/image/entrypoints/output.sh b/image/entrypoints/output.sh index 287b5ff6..76502d44 100755 --- a/image/entrypoints/output.sh +++ b/image/entrypoints/output.sh @@ -5,7 +5,6 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace output diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 5e3140ca..a9a2b718 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -5,8 +5,7 @@ source /usr/local/actions.sh debug setup -init-backend -select-workspace +init-backend-workspace set-plan-args exec 3>&1 diff --git a/image/entrypoints/remote-state.sh b/image/entrypoints/remote-state.sh index fa315220..1637b2ca 100755 --- a/image/entrypoints/remote-state.sh +++ b/image/entrypoints/remote-state.sh @@ -6,6 +6,8 @@ source /usr/local/actions.sh debug INPUT_PATH="$STEP_TMP_DIR/remote-state" +export INPUT_PATH + rm -rf "$INPUT_PATH" mkdir -p "$INPUT_PATH" @@ -16,6 +18,5 @@ terraform { EOF setup -init-backend -select-workspace +init-backend-workspace output diff --git a/image/entrypoints/version.sh b/image/entrypoints/version.sh index 5bde7087..7f5245cd 100755 --- a/image/entrypoints/version.sh +++ b/image/entrypoints/version.sh @@ -7,4 +7,5 @@ debug setup init +debug_cmd terraform version -json (cd "$INPUT_PATH" && terraform version -json | convert_version) diff --git a/image/setup.py b/image/setup.py new file mode 100644 index 00000000..d8ae8272 --- /dev/null +++ b/image/setup.py @@ -0,0 +1,20 @@ +from setuptools import find_packages, setup + +setup( + name='terraform-github-actions', + version='1.0.0', + packages=find_packages('src'), + package_dir={'': 'src'}, + package_data={'terraform_version': ['backend_constraints.json']}, + entry_points={ + 'console_scripts': [ + 'terraform-backend=terraform_backend.__main__:main', + 'terraform-version=terraform_version.__main__:main', + 'terraform-cloud-workspace=terraform_cloud_workspace.__main__:main' + ] + }, + install_requires=[ + 'requests', + 'python-hcl2' + ] +) diff --git a/image/src/github_actions/__init__.py b/image/src/github_actions/__init__.py new file mode 100644 index 00000000..8ec8c7cf --- /dev/null +++ b/image/src/github_actions/__init__.py @@ -0,0 +1 @@ +"""Package for interfacing with GitHub actions""" diff --git a/image/src/github_actions/debug.py b/image/src/github_actions/debug.py new file mode 100644 index 00000000..cf7eedad --- /dev/null +++ b/image/src/github_actions/debug.py @@ -0,0 +1,10 @@ +"""Actions debug logging""" + +import sys + + +def debug(msg: str) -> None: + """Add a message to the actions debug log.""" + + for line in msg.splitlines(): + sys.stderr.write(f'::debug::{line}\n') diff --git a/image/src/github_actions/env.py b/image/src/github_actions/env.py new file mode 100644 index 00000000..a4d6cb13 --- /dev/null +++ b/image/src/github_actions/env.py @@ -0,0 +1,19 @@ +"""GitHub action environment variables.""" + +from __future__ import annotations + +from typing import TypedDict + + +class ActionsEnv(TypedDict): + """Environment variables expected by these actions.""" + TERRAFORM_CLOUD_TOKENS: str + TERRAFORM_SSH_KEY: str + TERRAFORM_PRE_RUN: str + TERRAFORM_HTTP_CREDENTIALS: str + TERRAFORM_VERSION: str + + +class GithubEnv(TypedDict): + """Environment variables set by github actions.""" + GITHUB_WORKSPACE: str diff --git a/image/src/github_actions/inputs.py b/image/src/github_actions/inputs.py new file mode 100644 index 00000000..ad78c0a9 --- /dev/null +++ b/image/src/github_actions/inputs.py @@ -0,0 +1,88 @@ +""" +Typed Action input classes +""" + +from __future__ import annotations + +from typing import TypedDict + + +class InitInputs(TypedDict): + """Common input variables for actions the need to initialize terraform""" + INPUT_PATH: str + INPUT_WORKSPACE: str + INPUT_BACKEND_CONFIG: str + INPUT_BACKEND_CONFIG_FILE: str + + +class PlanInputs(InitInputs): + """Common input variables for actions that generate a plan""" + INPUT_VARIABLES: str + INPUT_VAR: str + INPUT_VAR_FILE: str + INPUT_PARALLELISM: str + + +class Plan(PlanInputs): + """Input variables for the plan action""" + INPUT_LABEL: str + INPUT_TARGET: str + INPUT_REPLACE: str + INPUT_ADD_GITHUB_COMMENT: str + + +class Apply(InitInputs): + """Input variables for the terraform-apply action""" + INPUT_LABEL: str + INPUT_TARGET: str + INPUT_REPLACE: str + INPUT_AUTO_APPROVE: str + + +class Check(PlanInputs): + """Input variables for the terraform-check action""" + + +class Destroy(PlanInputs): + """Input variables for the terraform-destroy action""" + + +class DestroyWorkspace(PlanInputs): + """Input variables for the terraform-destroy-workspace action""" + + +class Fmt(TypedDict): + """Input variables for the terraform-fmt action""" + INPUT_PATH: str + + +class FmtCheck(TypedDict): + """Input variables for the terraform-fmt-check action""" + INPUT_PATH: str + + +class Version(TypedDict): + """Input variables for the terraform-version action""" + INPUT_PATH: str + + +class NewWorkspace(InitInputs): + """Input variables for the terraform-new-workspace action""" + + +class Output(InitInputs): + """Input variables for the terraform-output action""" + + +class RemoteState(TypedDict): + """Input variables for the terraform-remote-state action""" + INPUT_BACKEND_TYPE: str + INPUT_WORKSPACE: str + INPUT_BACKEND_CONFIG: str + INPUT_BACKEND_CONFIG_FILE: str + + +class Validate(TypedDict): + """Input variables for the terraform-validate action""" + INPUT_PATH: str + INPUT_WORKSPACE: str diff --git a/image/src/terraform/__init__.py b/image/src/terraform/__init__.py new file mode 100644 index 00000000..513bf03f --- /dev/null +++ b/image/src/terraform/__init__.py @@ -0,0 +1 @@ +"""Package for working with terraform.""" diff --git a/image/src/terraform/cloud.py b/image/src/terraform/cloud.py new file mode 100644 index 00000000..bafcd290 --- /dev/null +++ b/image/src/terraform/cloud.py @@ -0,0 +1,246 @@ +"""Module for interacting with Terraform Cloud/Enterprise.""" + +from __future__ import annotations + +import datetime +import os +from typing import Iterable, Optional, TypedDict, Any, cast + +import requests +from requests import Response + +from github_actions.debug import debug +from terraform.module import BackendConfig + +session = requests.Session() + + +class Workspace(TypedDict): + """A Terraform cloud workspace""" + id: str + attributes: dict[str, Any] + + +class CloudException(Exception): + """Raised when there is an error interacting with terraform cloud.""" + + def __init__(self, msg: str, response: Optional[Response]): + super().__init__(msg) + self.response = response + + +class TerraformCloudApi: + def __init__(self, host: str, token: str): + self._host = host + self._token = token + + def api_request(self, method: str, path: str, /, headers: Optional[dict[str, str]] = None, **kwargs: Any) -> Response: + if headers is None: + headers = {} + + headers['Authorization'] = f'Bearer {self._token}' + + response = session.request(method, f'https://{self._host}/api/v2/{path}', headers=headers, **kwargs) + + debug(f'terraform cloud request url={response.url}') + debug(f'terraform cloud {response.status_code=}') + + if response.status_code == 401: + debug(str(response.content)) + raise CloudException('Terraform cloud operation failed: Unauthorized', response) + elif response.status_code == 429: + debug(str(response.content)) + raise CloudException('Terraform cloud rate limit reached', response) + elif not response.ok: + raise CloudException(f'Terraform cloud unexpected response code {response.status_code}', response) + + return response + + def get(self, path: str, **kwargs: Any) -> Response: + return self.api_request('GET', path, **kwargs) + + def delete(self, path: str, **kwargs: Any) -> Response: + return self.api_request('DELETE', path, **kwargs) + + def post(self, path: str, body: dict[str, Any], **kwargs: Any) -> Response: + return self.api_request('POST', path, headers={'Content-Type': 'application/vnd.api+json'}, json=body, **kwargs) + + def paged_get(self, path: str, **kwargs: Any) -> Iterable[Any]: + + page_num = 1 + while page_num is not None: + response = self.api_request('GET', path, params={'page[size]': 100, 'page[number]': page_num}, **kwargs) + + body = response.json() + yield from body.get('data', {}) + + page_num = body['meta']['pagination']['next-page'] + +def get_full_workspace_name(backend_config: BackendConfig, workspace_name: str) -> str: + + if 'prefix' in backend_config['workspaces']: + return backend_config['workspaces']['prefix'] + workspace_name + + elif 'name' in backend_config['workspaces']: + if backend_config['workspaces']['name'] != workspace_name: + raise CloudException(f'Only the configured workspace name {backend_config["workspaces"]["name"]!r} can be used, not {workspace_name!r}', None) + return workspace_name + + else: + return workspace_name + +def get_workspaces(backend_config: BackendConfig) -> Iterable[Workspace]: + """ + Return the workspaces that match the specified backend config. + + :param: The backend config to get workspaces for. + :return: The remote workspaces that match the backend config. + """ + + terraform_cloud = TerraformCloudApi(backend_config["hostname"], backend_config['token']) + + for workspace in terraform_cloud.paged_get( + f'/organizations/{backend_config["organization"]}/workspaces', + ): + + if 'name' in backend_config['workspaces']: + if workspace['attributes']['name'] == backend_config['workspaces']['name']: + yield workspace + elif 'prefix' in backend_config['workspaces']: + if workspace['attributes']['name'].startswith(backend_config['workspaces']['prefix']): + yield workspace + elif 'tags' in backend_config['workspaces']: + if all(tag in workspace['attributes']['tag-names'] for tag in backend_config['workspaces']['tags']): + yield workspace + + +def new_workspace(backend_config: BackendConfig, workspace_name: str) -> None: + """ + Create a new terraform cloud workspace. + + :param backend_config: Configuration for the backend to create the workspace in. + :param workspace_name: The name of the workspace to create. + :return: The new workspace. + """ + + full_workspace_name = get_full_workspace_name(backend_config, workspace_name) + + attributes = { + "name": full_workspace_name, + "resource-count": 0, + "updated-at": datetime.datetime.utcnow().isoformat() + 'Z', + } + + if version := os.environ.get('TERRAFORM_VERSION'): + attributes['terraform-version'] = version + + terraform_cloud = TerraformCloudApi(backend_config["hostname"], backend_config['token']) + + body = { + 'data': { + 'attributes': attributes, + 'type': 'workspaces' + } + } + + try: + response = terraform_cloud.post(f'/organizations/{backend_config["organization"]}/workspaces', body) + except CloudException as cloud_exception: + if cloud_exception.response is None: + raise + + content = cloud_exception.response.json() + + for error in content.get('errors', []): + if error.get('detail') != 'Name has already been taken': + raise + + # A workspace with this name already exists + debug(f'A workspace named {workspace_name!r} already exists') + + if 'tags' not in backend_config['workspaces']: + # We are done, the workspace exists + return None + + # For a cloud workspace, check the tags match + if get_workspace(backend_config, workspace_name): + # It has the correct tags + return None + + raise CloudException( + f'A workspace with the name {workspace_name!r} already exists, but without the correct tags. You must manually migrate this workspace by adding the correct tags.', + cloud_exception.response + ) + + raise + + workspace: dict[str, Any] = response.json()['data'] + + if 'tags' in backend_config['workspaces']: + terraform_cloud.post( + f'/workspaces/{workspace["id"]}/relationships/tags', + body={ + "data": [{ + "attributes": { + "name": tag, + }, + "type": "tags" + } for tag in sorted(backend_config['workspaces']['tags'])] + } + ) + + +def delete_workspace(backend_config: BackendConfig, workspace_name: str) -> None: + """ + Delete a terraform cloud workspace. + + :param backend_config: Configuration for the backend that contains the workspace. + :param workspace_name: The name of the workspace to delete. + """ + + full_workspace_name = get_full_workspace_name(backend_config, workspace_name) + + if 'tags' in backend_config['workspaces']: + # Try to get the workspace to check that it has the correct tags + if get_workspace(backend_config, workspace_name) is None: + raise CloudException(f'No such workspace {workspace_name!r} that matches the backend configuration', None) + + terraform_cloud = TerraformCloudApi(backend_config["hostname"], backend_config['token']) + + try: + terraform_cloud.delete(f'/organizations/{backend_config["organization"]}/workspaces/{full_workspace_name}') + except CloudException as cloud_exception: + if cloud_exception.response is not None and cloud_exception.response.status_code == 404: + raise CloudException(f'No such workspace {workspace_name!r} that matches the backend configuration', cloud_exception.response) + raise + + +def get_workspace(backend_config: BackendConfig, workspace_name: str) -> Optional[Workspace]: + """ + Get a remote workspace. + + :param backend_config: Configuration for the backend that contains the workspace. + :param workspace_name: The name of the workspace to get. + :return: The workspace, or None if there is no such workspace + """ + + full_workspace_name = get_full_workspace_name(backend_config, workspace_name) + + terraform_cloud = TerraformCloudApi(backend_config["hostname"], backend_config['token']) + + try: + response = terraform_cloud.get( + f'/organizations/{backend_config["organization"]}/workspaces/{full_workspace_name}' + ) + except CloudException as cloud_exception: + if cloud_exception.response is not None and cloud_exception.response.status_code == 404: + return None + raise + + workspace = response.json()['data'] + + if 'tags' in backend_config['workspaces']: + if not all(tag in workspace['attributes']['tag-names'] for tag in backend_config['workspaces']['tags']): + return None + + return cast(Workspace, workspace) diff --git a/image/src/terraform/download.py b/image/src/terraform/download.py new file mode 100644 index 00000000..1ee33a5c --- /dev/null +++ b/image/src/terraform/download.py @@ -0,0 +1,104 @@ +"""Module for downloading terraform executables.""" + +from __future__ import annotations + +import os.path +import platform +import sys +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.request import urlretrieve +from zipfile import ZipFile + +from github_actions.debug import debug + +if TYPE_CHECKING: + from terraform.versions import Version + + +def get_platform() -> str: + """Return terraform's idea of the current platform name.""" + + p = sys.platform + if p.startswith('freebsd'): + return 'freebsd' + elif p.startswith('linux'): + return 'linux' + elif p.startswith('win32'): + return 'windows' + elif p.startswith('openbsd'): + return 'openbsd' + elif p.startswith('darwin'): + return 'darwin' + + raise Exception(f'Unknown platform {p}') + + +def get_arch() -> str: + """Return terraforms idea of the current architecture.""" + + a = platform.machine() + if a in ['x86_64', 'amd64']: + return 'amd64' + elif a in ['i386', 'i686', 'x86']: + return '386' + elif a.startswith('armv8') or a.startswith('aarch64'): + return 'arm64' + elif a.startswith('arm'): + return 'arm' + + raise Exception(f'Unknown arch {a}') + + +def download_version(version: Version, target_dir: Path) -> Path: + """ + Download the executable for the given version of terraform. + + The return value is the path to the executable + """ + + terraform_path = Path(target_dir, 'terraform') + + if os.path.exists(terraform_path): + return terraform_path + + debug(f'Downloading terraform {version}') + + local_filename, headers = urlretrieve( + f'https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{get_platform()}_{get_arch()}.zip', + f'/tmp/terraform_{version}_linux_amd64.zip' + ) + + with ZipFile(local_filename) as f: + f.extract('terraform', target_dir) + + os.chmod(terraform_path, 755) + + return Path(os.path.abspath(terraform_path)) + + +def get_executable(version: Version) -> Path: + """ + Get the path to the specified terraform executable. + + Executables may be in any of the directories in TERRAFORM_BIN_DIR. + If executable doesn't exist, download it to the last directory in TERRAFORM_BIN_DIR. + Cache dirs are specified in the TERRAFORM_BIN_DIR env var as ':' separated paths. + The default is .terraform-bin-dir in the current directory. + + The return value is the path to the executable + """ + + cache_dirs = os.environ.get('TERRAFORM_BIN_DIR', '.terraform-bin-dir').split(':') + + download_dir = None + + for tf_dir in cache_dirs: + download_dir = Path(tf_dir, f'terraform_{version}') + terraform_path = os.path.join(download_dir, 'terraform') + if os.path.isfile(terraform_path): + return Path(os.path.abspath(terraform_path)) + + assert download_dir is not None + + return download_version(version, download_dir) diff --git a/image/src/terraform/exec.py b/image/src/terraform/exec.py new file mode 100644 index 00000000..14b4adad --- /dev/null +++ b/image/src/terraform/exec.py @@ -0,0 +1,25 @@ +"""Functions for executing terraform.""" + +from __future__ import annotations + +import os + +from github_actions.inputs import InitInputs + + +def init_args(inputs: InitInputs) -> list[str]: + """ + Generate arguments for the `terraform init` command from inputs + """ + + args = [] + + for path in inputs.get('INPUT_BACKEND_CONFIG_FILE', '').replace(',', '\n').splitlines(): + if path.strip(): + args.append(f'-backend-config={os.path.relpath(path.strip(), start=inputs["INPUT_PATH"])}') + + for config in inputs.get('INPUT_BACKEND_CONFIG', '').replace(',', '\n').splitlines(): + if stripped := config.strip(): + args.append(f'-backend-config={stripped}') + + return args diff --git a/image/src/terraform/module.py b/image/src/terraform/module.py new file mode 100644 index 00000000..84754c3f --- /dev/null +++ b/image/src/terraform/module.py @@ -0,0 +1,254 @@ +"""Functions for handling terraform modules.""" + +from __future__ import annotations + +import os +from typing import Any, cast, NewType, Optional, TYPE_CHECKING, TypedDict + +import hcl2 # type: ignore + +from github_actions.debug import debug +from terraform.versions import Constraint + +if TYPE_CHECKING: + from pathlib import Path + +TerraformModule = NewType('TerraformModule', dict[str, list[dict[str, Any]]]) + + +class BackendConfigWorkspaces(TypedDict): + """A workspaces block from a terraform backend config.""" + name: str + prefix: str + tags: list[str] + + +class BackendConfig(TypedDict): + """The backend config for a terraform module.""" + hostname: str + organization: str + token: str + workspaces: BackendConfigWorkspaces + + +def merge(a: TerraformModule, b: TerraformModule) -> TerraformModule: + """Combine two terraform module objects into one.""" + + merged = cast(TerraformModule, {}) + + for key in set(a.keys() | b.keys()): + if isinstance(a.get(key, []), list) and isinstance(b.get(key, []), list): + if key not in merged: + merged[key] = [] + + merged[key].extend(a.get(key, [])) + merged[key].extend(b.get(key, [])) + else: + if key in a: + merged[key] = a[key] + if key in b: + merged[key] = b[key] + + return merged + + +def load_module(path: Path) -> TerraformModule: + """ + Load the terraform module. + + Every .tf file in the given directory is read and merged into one terraform module. + If any .tf file fails to parse, it is ignored. + """ + + module = cast(TerraformModule, {}) + + for file in os.listdir(path): + if not file.endswith('.tf'): + continue + + with open(os.path.join(path, file)) as f: + try: + module = merge(module, cast(TerraformModule, hcl2.load(f))) + except Exception as e: + # ignore tf files that don't parse + debug(f'Failed to parse {file}') + debug(str(e)) + + return module + + +def load_backend_config_file(path: Path) -> TerraformModule: + """Load a backend config file.""" + + with open(path) as f: + return cast(TerraformModule, hcl2.load(f)) + + +def read_cli_config(config: str) -> dict[str, str]: + """ + Read a CLI config file + + :param config: The CLI config file contents + """ + + hosts = {} + + config_hcl = hcl2.loads(config) + + for credential in config_hcl.get('credentials', {}): + for cred_hostname, cred_conf in credential.items(): + if 'token' in cred_conf: + hosts[cred_hostname] = str(cred_conf['token']) + + return hosts + + +def get_cli_credentials(path: Path, hostname: str) -> Optional[str]: + """Get the terraform cloud token for a hostname from a cli credentials file.""" + + try: + with open(os.path.expanduser(path)) as f: + config = f.read() + except Exception: + debug('Failed to parse CLI Config file') + return None + + credentials = read_cli_config(config) + return credentials.get(hostname) + + +def get_version_constraints(module: TerraformModule) -> Optional[list[Constraint]]: + """Get the Terraform version constraint from the given module.""" + + for block in module.get('terraform', []): + if 'required_version' not in block: + continue + + try: + return [Constraint(c) for c in str(block['required_version']).split(',')] + except Exception: + debug('required_version constraint is malformed') + + return None + + +def get_remote_backend_config( + module: TerraformModule, + backend_config_files: str, + backend_config_vars: str, + cli_config_path: Path +) -> Optional[BackendConfig]: + """ + A complete backend config + + :param module: The terraform module to get the backend config from. At least a partial backend config must be present. + :param backend_config_files: Files containing additional backend config. + :param backend_config_vars: Additional backend config variables. + :param cli_config_path: A Terraform CLI config file to use. + """ + + found = False + backend_config = cast(BackendConfig, { + 'hostname': 'app.terraform.io', + 'workspaces': {} + }) + + for terraform in module.get('terraform', []): + for backend in terraform.get('backend', []): + if 'remote' not in backend: + return None + + found = True + if 'hostname' in backend['remote']: + backend_config['hostname'] = str(backend['remote']['hostname']) + + backend_config['organization'] = backend['remote'].get('organization') + backend_config['token'] = backend['remote'].get('token') + + if backend['remote'].get('workspaces', []): + backend_config['workspaces'] = backend['remote']['workspaces'][0] + + if not found: + return None + + def read_backend_files() -> None: + """Read backend config files specified in env var""" + for file in backend_config_files.replace(',', '\n').splitlines(): + for key, value in load_backend_config_file(Path(file)).items(): + backend_config[key] = value[0] if isinstance(value, list) else value # type: ignore + + def read_backend_vars() -> None: + """Read backend config values specified in env var""" + for line in backend_config_vars.replace(',', '\n').splitlines(): + key, value = line.split('=', maxsplit=1) + backend_config[key] = value # type: ignore + + read_backend_files() + read_backend_vars() + + if backend_config.get('token') is None and cli_config_path: + if token := get_cli_credentials(cli_config_path, str(backend_config['hostname'])): + backend_config['token'] = token + else: + debug(f'No token found for {backend_config["hostname"]}') + return backend_config + + return backend_config + + +def get_cloud_config(module: TerraformModule, cli_config_path: Path) -> Optional[BackendConfig]: + """ + Get a complete backend config for a module using terraform cloud + + :param module: The terraform module to get the cloud config from. + :param cli_config_path: A Terraform CLI config file to use. + """ + + found = False + backend_config = cast(BackendConfig, { + 'hostname': 'app.terraform.io', + 'workspaces': {} + }) + + for terraform in module.get('terraform', []): + for cloud in terraform.get('cloud', []): + + found = True + + if 'hostname' in cloud: + backend_config['hostname'] = cloud['hostname'] + + backend_config['organization'] = cloud.get('organization') + backend_config['token'] = cloud.get('token') + + if cloud.get('workspaces', []): + backend_config['workspaces'] = cloud['workspaces'][0] + + if not found: + return None + + if backend_config.get('token') is None and cli_config_path: + if token := get_cli_credentials(cli_config_path, backend_config['hostname']): + backend_config['token'] = token + + return backend_config + + +def get_backend_type(module: TerraformModule) -> Optional[str]: + """ + Get the backend type used by the module. + + :param module: The terraform module to get the backend for + :return: The name of the backend used by the module + """ + + for terraform in module.get('terraform', []): + for backend in terraform.get('backend', []): + for backend_type in backend: + return str(backend_type) + + for terraform in module.get('terraform', []): + if 'cloud' in terraform: + return 'remote' + + return 'local' diff --git a/image/src/terraform/versions.py b/image/src/terraform/versions.py new file mode 100644 index 00000000..2807eee3 --- /dev/null +++ b/image/src/terraform/versions.py @@ -0,0 +1,220 @@ +"""Module for working with Terraform versions and version constraints.""" + +from __future__ import annotations + +import re +from functools import total_ordering +from typing import Any, cast, Iterable, Literal + +import requests + +session = requests.Session() + +ConstraintOperator = Literal['=', '!=', '>', '>=', '<', '<=', '~>'] + + +@total_ordering +class Version: + """ + A Terraform version. + + Versions are made up of major, minor & patch numbers, plus an optional pre_release string. + """ + + def __init__(self, version: str): + + match = re.match(r'(?P\d+)\.(?P\d+)\.(?P\d+)(?:-(?P[\d\w]+))?', version) + if not match: + raise ValueError(f'Not a valid version {version}') + + self.major = int(match.group(1)) + self.minor = int(match.group(2)) + self.patch = int(match.group(3)) + self.pre_release = match.group(4) or '' + + def __repr__(self) -> str: + s = f'{self.major}.{self.minor}.{self.patch}' + + if self.pre_release: + s += f'-{self.pre_release}' + + return s + + def __hash__(self) -> int: + return hash(self.__repr__()) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Version): + return NotImplemented + + return self.major == other.major and self.minor == other.minor and self.patch == other.patch and self.pre_release == other.pre_release + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, Version): + return NotImplemented + + if self.major != other.major: + return self.major < other.major + if self.minor != other.minor: + return self.minor < other.minor + if self.patch != other.patch: + return self.patch < other.patch + if self.pre_release != other.pre_release: + if self.pre_release == '': + return False + if other.pre_release == '': + return True + return self.pre_release < other.pre_release + + return False + + +class Constraint: + """A Terraform version constraint.""" + + def __init__(self, constraint: str): + if match := re.match(r'([=!<>~]*)(.*)', constraint.replace(' ', '')): + self.operator = cast(ConstraintOperator, match.group(1) or '=') + constraint = match.group(2) + else: + raise ValueError(f'Invalid version constraint {constraint}') + + if match := re.match(r'(?P\d+)(?:\.(?P\d+))?(?:\.(?P\d+))?(?:-(?P.*))?', constraint): + self.major = int(match.group('major')) + self.minor = int(match.group('minor')) if match.group('minor') else None + self.patch = int(match.group('patch')) if match.group('patch') else None + self.pre_release = match.group('pre_release') or '' + else: + raise ValueError(f'Invalid version constraint {constraint}') + + def __repr__(self) -> str: + s = f'{self.operator}{self.major}' + + if self.minor is not None: + s += f'.{self.minor}' + + if self.patch is not None: + s += f'.{self.patch}' + + if self.pre_release: + s += f'-{self.pre_release}' + + return s + + def __hash__(self) -> int: + return hash(self.__repr__()) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Constraint): + return NotImplemented + + return self.major == other.major and self.minor == other.minor and self.patch == other.patch and self.pre_release == other.pre_release and self.operator == other.operator + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, Constraint): + return NotImplemented + + if self.major != other.major: + return self.major < other.major + if self.minor != other.minor: + return (self.minor or 0) < (other.minor or 0) + if self.patch != other.patch: + return (self.patch or 0) < (other.patch or 0) + if self.pre_release != other.pre_release: + if self.pre_release == '': + return False + if other.pre_release == '': + return True + return self.pre_release < other.pre_release + + operator_order = ['<', '<=', '=', '~>', '>=', '>'] + return operator_order.index(self.operator) < operator_order.index(other.operator) + + def is_allowed(self, version: Version) -> bool: + """Is the given version allowed by this constraint.""" + + def compare() -> int: + """ + Compare this version with the specified other version. + + If this version < other version, the return value is < 0 + If this version > other version, the return value is > 0 + If the versions are the same, the return value is 0 + """ + + if version.major != self.major: + return version.major - self.major + if version.minor != (self.minor or 0): + return version.minor - (self.minor or 0) + if version.patch != (self.patch or 0): + return version.patch - (self.patch or 0) + + if version.pre_release < self.pre_release: + return -1 + if version.pre_release > self.pre_release: + return 1 + + return 0 + + if self.operator == '=': + return compare() == 0 + if self.operator == '!=': + return compare() != 0 and not version.pre_release + if self.operator == '>': + return compare() > 0 and not version.pre_release + if self.operator == '>=': + return compare() >= 0 and not version.pre_release + if self.operator == '<': + return compare() < 0 and not version.pre_release + if self.operator == '<=': + return compare() <= 0 and not version.pre_release + if self.operator == '~>': + if version.pre_release: + return False + + if self.minor is None: + # ~> x + return version.major >= self.major + + if self.patch is None: + # ~> x.x + return version.major == self.major and version.minor >= self.minor + + # ~> x.x.x + return version.major == self.major and version.minor == self.minor and version.patch >= self.patch + + +def latest_version(versions: Iterable[Version]) -> Version: + """Return the latest version of the given versions.""" + + return sorted(versions, reverse=True)[0] + + +def earliest_version(versions: Iterable[Version]) -> Version: + """Return the earliest version of the given versions.""" + + return sorted(versions)[0] + + +def get_terraform_versions() -> Iterable[Version]: + """Return the currently available terraform versions.""" + + response = session.get('https://releases.hashicorp.com/terraform/') + response.raise_for_status() + + version_regex = re.compile(br'/(\d+\.\d+\.\d+(-[\d\w]+)?)/') + + for version in version_regex.finditer(response.content): + yield Version(version.group(1).decode()) + + +def apply_constraints(versions: Iterable[Version], constraints: Iterable[Constraint]) -> Iterable[Version]: + """ + Apply the given version constraints. + + Returns the terraform versions that are allowed by all the given constraints + """ + + for version in versions: + if all(constraint.is_allowed(version) for constraint in constraints): + yield version diff --git a/image/src/terraform_backend/__init__.py b/image/src/terraform_backend/__init__.py new file mode 100644 index 00000000..106bacf2 --- /dev/null +++ b/image/src/terraform_backend/__init__.py @@ -0,0 +1 @@ +"""terraform-backend command""" diff --git a/image/src/terraform_backend/__main__.py b/image/src/terraform_backend/__main__.py new file mode 100644 index 00000000..e8cd6351 --- /dev/null +++ b/image/src/terraform_backend/__main__.py @@ -0,0 +1,21 @@ +""" +Output the backend type in use by the terraform module in the current path + +Usage: + terraform-backend +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from terraform.module import load_module, get_backend_type + + +def main() -> None: + """Entrypoint for terraform-backend""" + + module = load_module(Path(os.environ.get('INPUT_PATH', '.'))) + sys.stdout.write(f'{get_backend_type(module)}\n') diff --git a/image/src/terraform_cloud_workspace/__init__.py b/image/src/terraform_cloud_workspace/__init__.py new file mode 100644 index 00000000..59d12359 --- /dev/null +++ b/image/src/terraform_cloud_workspace/__init__.py @@ -0,0 +1 @@ +"""terraform-cloud-workspace command""" diff --git a/image/src/terraform_cloud_workspace/__main__.py b/image/src/terraform_cloud_workspace/__main__.py new file mode 100644 index 00000000..07886f8b --- /dev/null +++ b/image/src/terraform_cloud_workspace/__main__.py @@ -0,0 +1,95 @@ +""" +Manage Terraform Cloud/Enterprise workspaces + +Usage: + terraform-cloud-workspace list + terraform-cloud-workspace new + terraform-cloud-workspace delete + +For whatever reason, the terraform workspace command needs an initialized backend. +When using the remote backend there may be no workspaces to initialize, so we are a bit stuck. + +This directly uses the cloud API to manage workspaces instead. + +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from terraform.cloud import delete_workspace, get_workspaces, new_workspace, CloudException +from terraform.module import load_module, get_remote_backend_config, get_cloud_config + + +def main() -> None: + """Entrypoint for terraform-cloud-workspace.""" + + if len(sys.argv) <= 1: + sys.stdout.write(f'{__doc__}\n') + sys.exit(1) + + module = load_module(Path(os.environ.get('INPUT_PATH', '.'))) + + backend_config = get_remote_backend_config( + module, + backend_config_files=os.environ.get('INPUT_BACKEND_CONFIG_FILE', ''), + backend_config_vars=os.environ.get('INPUT_BACKEND_CONFIG', ''), + cli_config_path=Path('~/.terraformrc'), + ) + + if backend_config is None: + backend_config = get_cloud_config( + module, + cli_config_path=Path('~/.terraformrc'), + ) + + if backend_config is None: + sys.stdout.write('Current directory doesn\'t use terraform cloud\n') + sys.exit(1) + + if backend_config.get('token') is None: + sys.stdout.write(f'No token found for {backend_config["hostname"]}\n') + sys.exit(1) + + if not backend_config.get('workspaces'): + sys.stdout.write('No required workspaces option found in backend block\n') + sys.exit(1) + + if len([k for k in backend_config['workspaces'] if k in ['tags', 'prefix', 'name']]) != 1: + sys.stdout.write('name or prefix required for remote backend. cloud config requires tags.\n') + sys.exit(1) + + try: + if sys.argv[1] == 'list': + for workspace in get_workspaces(backend_config): + if 'prefix' in backend_config['workspaces']: + sys.stdout.write(workspace['attributes']['name'][len(backend_config['workspaces']['prefix']):]) + else: + sys.stdout.write(workspace['attributes']['name']) + sys.stdout.write('\n') + sys.exit(0) + + if len(sys.argv) <= 2 or not sys.argv[2]: + sys.stdout.write(f'{__doc__}\n') + sys.exit(1) + + workspace_name = sys.argv[2] + + if sys.argv[1] == 'new': + new_workspace(backend_config, workspace_name) + sys.stdout.write(f'Created remote workspace {workspace_name}\n') + + elif sys.argv[1] == 'delete': + delete_workspace(backend_config, sys.argv[2]) + sys.stdout.write(f'Delete remote workspace {workspace_name}\n') + + else: + sys.stdout.write(f'{__doc__}\n') + sys.exit(1) + + except CloudException as cloud_exception: + sys.stderr.write(str(cloud_exception)) + sys.stderr.write('\n') + sys.exit(1) diff --git a/image/src/terraform_version/__init__.py b/image/src/terraform_version/__init__.py new file mode 100644 index 00000000..0bc49141 --- /dev/null +++ b/image/src/terraform_version/__init__.py @@ -0,0 +1 @@ +"""terraform-version command.""" diff --git a/image/src/terraform_version/__main__.py b/image/src/terraform_version/__main__.py new file mode 100644 index 00000000..8602e6e2 --- /dev/null +++ b/image/src/terraform_version/__main__.py @@ -0,0 +1,120 @@ +"""Determine the version of terraform to use.""" + +from __future__ import annotations + +import os +import os.path +import sys +from pathlib import Path +from typing import Optional, cast + +from terraform_version.remote_state import get_backend_constraints, read_backend_config_vars, try_guess_state_version + +from github_actions.debug import debug +from github_actions.env import ActionsEnv, GithubEnv +from github_actions.inputs import InitInputs +from terraform.download import get_executable +from terraform.module import load_module, get_backend_type +from terraform.versions import apply_constraints, get_terraform_versions, latest_version, Version, Constraint +from terraform_version.asdf import try_read_asdf +from terraform_version.env import try_read_env +from terraform_version.local_state import try_read_local_state +from terraform_version.remote_workspace import try_get_remote_workspace_version +from terraform_version.required_version import try_get_required_version +from terraform_version.tfenv import try_read_tfenv +from terraform_version.tfswitch import try_read_tfswitch + + +def determine_version(inputs: InitInputs, cli_config_path: Path, actions_env: ActionsEnv, github_env: GithubEnv) -> Version: + """Determine the terraform version to use""" + + versions = list(get_terraform_versions()) + + module = load_module(Path(inputs.get('INPUT_PATH', '.'))) + + version: Optional[Version] + + if version := try_get_remote_workspace_version(inputs, module, cli_config_path, versions): + sys.stdout.write(f'Using remote workspace terraform version, which is set to {version!r}\n') + return version + + if version := try_get_required_version(module, versions): + sys.stdout.write(f'Using latest terraform version that matches the required_version constraints\n') + return version + + if version := try_read_tfswitch(inputs): + sys.stdout.write('Using terraform version specified in .tfswitchrc file\n') + return version + + if version := try_read_tfenv(inputs, versions): + sys.stdout.write('Using terraform version specified in .terraform-version file\n') + return version + + if version := try_read_asdf(inputs, github_env.get('GITHUB_WORKSPACE', '/'), versions): + sys.stdout.write('Using terraform version specified in .tool-versions file\n') + return version + + if version := try_read_env(actions_env, versions): + sys.stdout.write('Using latest terraform version that matches the TERRAFORM_VERSION constraints\n') + return version + + if inputs.get('INPUT_BACKEND_CONFIG', '').strip(): + # key=value form of backend config was introduced in 0.9.1 + versions = list(apply_constraints(versions, [Constraint('>=0.9.1')])) + + try: + backend_config = read_backend_config_vars(inputs) + versions = list(apply_constraints(versions, get_backend_constraints(module, backend_config))) + backend_type = get_backend_type(module) + except Exception as e: + debug('Failed to get backend config') + debug(str(e)) + return latest_version(versions) + + if backend_type not in ['remote', 'local']: + if version := try_guess_state_version(inputs, module, versions): + sys.stdout.write('Using the same terraform version that wrote the existing remote state file\n') + return version + + if backend_type == 'local': + if version := try_read_local_state(Path(inputs.get('INPUT_PATH', '.'))): + sys.stdout.write('Using the same terraform version that wrote the existing local terraform.tfstate\n') + return version + + sys.stdout.write('Terraform version not specified, using the latest version\n') + return latest_version(versions) + + +def switch(version: Version) -> None: + """ + Switch to the specified version of terraform. + + Updates the /usr/local/bin/terraform symlink to point to the specified version. + The version will be downloaded if it doesn't already exist. + """ + + sys.stdout.write(f'Switching to Terraform v{version}\n') + + target_path = get_executable(version) + + link_path = '/usr/local/bin/terraform' + if os.path.exists(link_path): + os.remove(link_path) + + os.symlink(target_path, link_path) + + +def main() -> None: + """Entrypoint for terraform-version.""" + + if len(sys.argv) > 1: + switch(Version(sys.argv[1])) + else: + version = determine_version( + cast(InitInputs, os.environ), + Path('~/.terraformrc'), + cast(ActionsEnv, os.environ), + cast(GithubEnv, os.environ) + ) + + switch(version) diff --git a/image/src/terraform_version/asdf.py b/image/src/terraform_version/asdf.py new file mode 100644 index 00000000..748c1060 --- /dev/null +++ b/image/src/terraform_version/asdf.py @@ -0,0 +1,43 @@ +"""asdf .tool-versions file support.""" + +from __future__ import annotations + +import os +import re +from typing import Iterable, Optional + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.versions import latest_version, Version + + +def parse_asdf(tool_versions: str, versions: Iterable[Version]) -> Version: + """Return the version specified in an asdf .tool-versions file.""" + + for line in tool_versions.splitlines(): + if match := re.match(r'^\s*terraform\s+([^\s#]+)', line.strip()): + if match.group(1) == 'latest': + return latest_version(v for v in versions if not v.pre_release) + return Version(match.group(1)) + + raise Exception('No version for terraform found in .tool-versions') + + +def try_read_asdf(inputs: InitInputs, workspace_path: str, versions: Iterable[Version]) -> Optional[Version]: + """Return the version from an asdf .tool-versions file if possible.""" + + module_path = os.path.abspath(inputs.get('INPUT_PATH', '.')) + + while module_path not in ['/', workspace_path]: + asdf_path = os.path.join(module_path, '.tool-versions') + + if os.path.isfile(asdf_path): + try: + with open(asdf_path) as f: + return parse_asdf(f.read(), versions) + except Exception as e: + debug(str(e)) + + module_path = os.path.dirname(module_path) + + return None diff --git a/image/src/terraform_version/backend_constraints.json b/image/src/terraform_version/backend_constraints.json new file mode 100644 index 00000000..41ed83fb --- /dev/null +++ b/image/src/terraform_version/backend_constraints.json @@ -0,0 +1,1166 @@ +{ + "artifactory": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "password": [ + ">=0.9.0" + ], + "repo": [ + ">=0.9.0" + ], + "subpath": [ + ">=0.9.0" + ], + "url": [ + ">=0.9.0" + ], + "username": [ + ">=0.9.0" + ] + }, + "environment_variables": { + "ARTIFACTORY_PASSWORD": [ + ">=0.9.0" + ], + "ARTIFACTORY_URL": [ + ">=0.9.0" + ], + "ARTIFACTORY_USERNAME": [ + ">=0.9.0" + ] + } + }, + "atlas": { + "terraform": [ + ">=0.9.0", + "<=0.14.11" + ], + "config_variables": { + "access_token": [ + ">=0.9.0", + "<=0.14.11" + ], + "address": [ + ">=0.9.0", + "<=0.14.11" + ], + "name": [ + ">=0.9.0", + "<=0.14.11" + ] + }, + "environment_variables": { + "ATLAS_ADDRESS": [ + ">=0.9.1", + "<=0.14.11" + ], + "ATLAS_TOKEN": [ + ">=0.9.0", + "<=0.14.11" + ] + } + }, + "azure": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "access_key": [ + ">=0.9.0" + ], + "arm_client_id": [ + ">=0.9.0", + "<=0.14.11" + ], + "arm_client_secret": [ + ">=0.9.0", + "<=0.14.11" + ], + "arm_subscription_id": [ + ">=0.9.0", + "<=0.14.11" + ], + "arm_tenant_id": [ + ">=0.9.0", + "<=0.14.11" + ], + "client_certificate_password": [ + ">=0.13.1" + ], + "client_certificate_path": [ + ">=0.13.1" + ], + "client_id": [ + ">=0.12.0" + ], + "client_secret": [ + ">=0.12.0" + ], + "container_name": [ + ">=0.9.0" + ], + "endpoint": [ + ">=0.12.0" + ], + "environment": [ + ">=0.9.0" + ], + "key": [ + ">=0.9.0" + ], + "lease_id": [ + ">=0.9.0", + "<=0.10.2" + ], + "metadata_host": [ + ">=0.13.1" + ], + "msi_endpoint": [ + ">=0.12.0" + ], + "resource_group_name": [ + ">=0.9.0" + ], + "sas_token": [ + ">=0.12.0" + ], + "snapshot": [ + ">=0.13.0" + ], + "storage_account_name": [ + ">=0.9.0" + ], + "subscription_id": [ + ">=0.12.0" + ], + "tenant_id": [ + ">=0.12.0" + ], + "use_azuread_auth": [ + ">=0.15.0" + ], + "use_microsoft_graph": [ + ">=1.1.0" + ], + "use_msi": [ + ">=0.12.0" + ] + }, + "environment_variables": { + "ARM_ACCESS_KEY": [ + ">=0.9.0" + ], + "ARM_CLIENT_CERTIFICATE_PASSWORD": [ + ">=0.13.1" + ], + "ARM_CLIENT_CERTIFICATE_PATH": [ + ">=0.13.1" + ], + "ARM_CLIENT_ID": [ + ">=0.9.0" + ], + "ARM_CLIENT_SECRET": [ + ">=0.9.0" + ], + "ARM_ENDPOINT": [ + ">=0.12.0" + ], + "ARM_ENVIRONMENT": [ + ">=0.9.0" + ], + "ARM_MSI_ENDPOINT": [ + ">=0.12.0" + ], + "ARM_SAS_TOKEN": [ + ">=0.12.0" + ], + "ARM_SNAPSHOT": [ + ">=0.13.0" + ], + "ARM_SUBSCRIPTION_ID": [ + ">=0.9.0" + ], + "ARM_TENANT_ID": [ + ">=0.9.0" + ], + "ARM_USE_AZUREAD": [ + ">=0.15.0" + ], + "ARM_USE_MSI": [ + ">=0.12.0" + ], + "ARM_LEASE_ID": [ + ">=0.9.0", + "<=0.10.2" + ] + } + }, + "consul": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "access_token": [ + ">=0.9.0" + ], + "address": [ + ">=0.9.0" + ], + "ca_file": [ + ">=0.10.3" + ], + "cert_file": [ + ">=0.10.3" + ], + "datacenter": [ + ">=0.9.0" + ], + "gzip": [ + ">=0.9.0" + ], + "http_auth": [ + ">=0.9.0" + ], + "key_file": [ + ">=0.10.3" + ], + "lock": [ + ">=0.9.0" + ], + "path": [ + ">=0.9.0" + ], + "scheme": [ + ">=0.9.0" + ] + }, + "environment_variables": { + "CONSUL_CACERT": [ + ">=0.10.3" + ], + "CONSUL_CLIENT_CERT": [ + ">=0.10.3" + ], + "CONSUL_CLIENT_KEY": [ + ">=0.10.3" + ], + "CONSUL_HTTP_ADDR": [ + ">=0.9.0" + ], + "CONSUL_HTTP_AUTH": [ + ">=0.9.0" + ], + "CONSUL_HTTP_SSL": [ + ">=0.9.0" + ], + "CONSUL_HTTP_TOKEN": [ + ">=0.9.0" + ] + } + }, + "cos": { + "terraform": [ + ">=0.12.21" + ], + "config_variables": { + "acl": [ + ">=0.12.21" + ], + "bucket": [ + ">=0.12.21" + ], + "encrypt": [ + ">=0.12.21" + ], + "key": [ + ">=0.12.21" + ], + "prefix": [ + ">=0.12.21" + ], + "region": [ + ">=0.12.21" + ], + "secret_id": [ + ">=0.12.21" + ], + "secret_key": [ + ">=0.12.21" + ] + }, + "environment_variables": { + "TENCENTCLOUD_REGION": [ + ">=0.12.21" + ], + "TENCENTCLOUD_SECRET_ID": [ + ">=0.12.21" + ], + "TENCENTCLOUD_SECRET_KEY": [ + ">=0.12.21" + ] + } + }, + "etcd": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "endpoints": [ + ">=0.9.0" + ], + "password": [ + ">=0.9.0" + ], + "path": [ + ">=0.9.0" + ], + "username": [ + ">=0.9.0" + ] + } + }, + "etcdv3": { + "terraform": [ + ">=0.10.8" + ], + "config_variables": { + "cacert_path": [ + ">=0.10.8" + ], + "cert_path": [ + ">=0.10.8" + ], + "endpoints": [ + ">=0.10.8" + ], + "key_path": [ + ">=0.10.8" + ], + "lock": [ + ">=0.10.8" + ], + "max_request_bytes": [ + ">=1.0.3" + ], + "password": [ + ">=0.10.8" + ], + "prefix": [ + ">=0.10.8" + ], + "username": [ + ">=0.10.8" + ] + }, + "environment_variables": { + "ETCDV3_PASSWORD": [ + ">=0.10.8" + ], + "ETCDV3_USERNAME": [ + ">=0.10.8" + ] + } + }, + "gcs": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "access_token": [ + ">=0.12.10" + ], + "bucket": [ + ">=0.9.0" + ], + "credentials": [ + ">=0.9.0" + ], + "encryption_key": [ + ">=0.11.2" + ], + "impersonate_service_account": [ + ">=0.14.0" + ], + "impersonate_service_account_delegates": [ + ">=0.14.0" + ], + "path": [ + ">=0.9.0", + "<=0.14.11" + ], + "prefix": [ + ">=0.11.0" + ], + "project": [ + ">=0.11.0", + "<=0.15.3" + ], + "region": [ + ">=0.11.0", + "<=0.15.3" + ] + }, + "environment_variables": { + "GOOGLE_BACKEND_CREDENTIALS": [ + ">=0.9.0" + ], + "GOOGLE_CREDENTIALS": [ + ">=0.9.0" + ], + "GOOGLE_ENCRYPTION_KEY": [ + ">=0.11.2" + ], + "GOOGLE_IMPERSONATE_SERVICE_ACCOUNT": [ + ">=0.14.0" + ] + } + }, + "http": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "address": [ + ">=0.9.0" + ], + "lock_address": [ + ">=0.10.3" + ], + "lock_method": [ + ">=0.10.3" + ], + "password": [ + ">=0.9.0" + ], + "retry_max": [ + ">=0.11.15", + "!=0.12.0", + "!=0.12.1" + ], + "retry_wait_max": [ + ">=0.11.15", + "!=0.12.0", + "!=0.12.1" + ], + "retry_wait_min": [ + ">=0.11.15", + "!=0.12.0", + "!=0.12.1" + ], + "skip_cert_verification": [ + ">=0.9.0" + ], + "unlock_address": [ + ">=0.10.3" + ], + "unlock_method": [ + ">=0.10.3" + ], + "update_method": [ + ">=0.10.3" + ], + "username": [ + ">=0.9.0" + ] + }, + "environment_variables": { + "TF_HTTP_ADDRESS": [ + ">=0.13.2" + ], + "TF_HTTP_LOCK_ADDRESS": [ + ">=0.13.2" + ], + "TF_HTTP_LOCK_METHOD": [ + ">=0.13.2" + ], + "TF_HTTP_PASSWORD": [ + ">=0.13.2" + ], + "TF_HTTP_RETRY_MAX": [ + ">=0.13.2" + ], + "TF_HTTP_RETRY_WAIT_MAX": [ + ">=0.13.2" + ], + "TF_HTTP_RETRY_WAIT_MIN": [ + ">=0.13.2" + ], + "TF_HTTP_UNLOCK_ADDRESS": [ + ">=0.13.2" + ], + "TF_HTTP_UNLOCK_METHOD": [ + ">=0.13.2" + ], + "TF_HTTP_UPDATE_METHOD": [ + ">=0.13.2" + ], + "TF_HTTP_USERNAME": [ + ">=0.13.2" + ] + } + }, + "kubernetes": { + "terraform": [ + ">=0.13.0" + ], + "config_variables": { + "client_certificate": [ + ">=0.13.0" + ], + "client_key": [ + ">=0.13.0" + ], + "cluster_ca_certificate": [ + ">=0.13.0" + ], + "config_context": [ + ">=0.13.0" + ], + "config_context_auth_info": [ + ">=0.13.0" + ], + "config_context_cluster": [ + ">=0.13.0" + ], + "config_path": [ + ">=0.13.0" + ], + "config_paths": [ + ">=1.1.0" + ], + "exec": [ + ">=0.13.0" + ], + "host": [ + ">=0.13.0" + ], + "in_cluster_config": [ + ">=0.13.0" + ], + "insecure": [ + ">=0.13.0" + ], + "labels": [ + ">=0.13.0" + ], + "load_config_file": [ + ">=0.13.0" + ], + "namespace": [ + ">=0.13.0" + ], + "password": [ + ">=0.13.0" + ], + "secret_suffix": [ + ">=0.13.0" + ], + "token": [ + ">=0.13.0" + ], + "username": [ + ">=0.13.0" + ] + }, + "environment_variables": { + "KUBE_CLIENT_CERT_DATA": [ + ">=0.13.0" + ], + "KUBE_CLIENT_KEY_DATA": [ + ">=0.13.0" + ], + "KUBE_CLUSTER_CA_CERT_DATA": [ + ">=0.13.0" + ], + "KUBE_CONFIG_PATH": [ + ">=1.1.0" + ], + "KUBE_CONFIG_PATHS": [ + ">=1.1.0" + ], + "KUBE_CTX": [ + ">=0.13.0" + ], + "KUBE_CTX_AUTH_INFO": [ + ">=0.13.0" + ], + "KUBE_CTX_CLUSTER": [ + ">=0.13.0" + ], + "KUBE_HOST": [ + ">=0.13.0" + ], + "KUBE_INSECURE": [ + ">=0.13.0" + ], + "KUBE_IN_CLUSTER_CONFIG": [ + ">=0.13.0" + ], + "KUBE_NAMESPACE": [ + ">=0.13.0" + ], + "KUBE_PASSWORD": [ + ">=0.13.0" + ], + "KUBE_TOKEN": [ + ">=0.13.0" + ], + "KUBE_USER": [ + ">=0.13.0" + ], + "KUBE_LOAD_CONFIG_FILE": [ + ">=0.13.0" + ] + } + }, + "manta": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "account": [ + ">=0.11.0" + ], + "insecure_skip_tls_verify": [ + ">=0.11.0" + ], + "key_id": [ + ">=0.11.0" + ], + "key_material": [ + ">=0.11.0" + ], + "objectName": [ + ">=0.9.0", + "<=0.11.15" + ], + "object_name": [ + ">=0.11.9" + ], + "path": [ + ">=0.9.0" + ], + "url": [ + ">=0.11.0" + ], + "user": [ + ">=0.11.2" + ] + }, + "environment_variables": { + "MANTA_URL": [ + ">=0.11.0" + ], + "SDC_ACCOUNT": [ + ">=0.11.0" + ], + "SDC_KEY_ID": [ + ">=0.11.0" + ], + "SDC_KEY_MATERIAL": [ + ">=0.11.0" + ], + "SDC_USER": [ + ">=0.11.2" + ], + "TRITON_ACCOUNT": [ + ">=0.11.0" + ], + "TRITON_KEY_ID": [ + ">=0.11.0" + ], + "TRITON_KEY_MATERIAL": [ + ">=0.11.0" + ], + "TRITON_USER": [ + ">=0.11.2" + ], + "TRITON_SKIP_TLS_VERIFY": [ + ">=0.11.0" + ] + } + }, + "oss": { + "terraform": [ + ">=0.12.2" + ], + "config_variables": { + "access_key": [ + ">=0.12.2" + ], + "acl": [ + ">=0.12.2" + ], + "assume_role": [ + ">=0.12.6" + ], + "assume_role_policy": [ + ">=1.1.0" + ], + "assume_role_role_arn": [ + ">=1.1.0" + ], + "assume_role_session_expiration": [ + ">=1.1.0" + ], + "assume_role_session_name": [ + ">=1.1.0" + ], + "bucket": [ + ">=0.12.2" + ], + "ecs_role_name": [ + ">=1.1.0" + ], + "encrypt": [ + ">=0.12.2" + ], + "endpoint": [ + ">=1.1.0" + ], + "key": [ + ">=0.12.2" + ], + "prefix": [ + ">=0.12.2" + ], + "profile": [ + ">=1.1.0" + ], + "region": [ + ">=0.12.2" + ], + "secret_key": [ + ">=0.12.2" + ], + "security_token": [ + ">=0.12.2" + ], + "shared_credentials_file": [ + ">=1.1.0" + ], + "sts_endpoint": [ + ">=1.1.0" + ], + "tablestore_endpoint": [ + ">=1.1.0" + ], + "tablestore_table": [ + ">=1.1.0" + ] + }, + "environment_variables": { + "ALICLOUD_ACCESS_KEY_ID": [ + ">=0.12.2" + ], + "ALICLOUD_ACCESS_KEY_SECRET": [ + ">=0.12.2" + ], + "ALICLOUD_ACCESS_KEY": [ + ">=0.12.2" + ], + "ALICLOUD_ASSUME_ROLE_ARN": [ + ">=1.1.0" + ], + "ALICLOUD_ASSUME_ROLE_SESSION_EXPIRATION": [ + ">=1.1.0" + ], + "ALICLOUD_ASSUME_ROLE_SESSION_NAME": [ + ">=1.1.0" + ], + "ALICLOUD_DEFAULT_REGION": [ + ">=0.12.2" + ], + "ALICLOUD_OSS_ENDPOINT": [ + ">=1.1.0" + ], + "ALICLOUD_PROFILE": [ + ">=1.1.0" + ], + "ALICLOUD_REGION": [ + ">=0.12.2" + ], + "ALICLOUD_SECRET_KEY": [ + ">=0.12.2" + ], + "ALICLOUD_SECURITY_TOKEN": [ + ">=0.12.2" + ], + "ALICLOUD_SHARED_CREDENTIALS_FILE": [ + ">=1.1.0" + ], + "ALICLOUD_STS_ENDPOINT": [ + ">=1.1.0" + ], + "ALICLOUD_TABLESTORE_ENDPOINT": [ + ">=1.1.0" + ], + "OSS_ENDPOINT": [ + ">=1.1.0" + ] + } + }, + "pg": { + "terraform": [ + ">=0.12.0" + ], + "config_variables": { + "conn_str": [ + ">=0.12.0" + ], + "schema_name": [ + ">=0.12.0" + ], + "skip_index_creation": [ + ">=0.14.0" + ], + "skip_schema_creation": [ + ">=0.12.8" + ], + "skip_table_creation": [ + ">=0.14.0" + ] + } + }, + "s3": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "access_key": [ + ">=0.9.0" + ], + "acl": [ + ">=0.9.0" + ], + "assume_role_duration_seconds": [ + ">=0.13.0" + ], + "assume_role_policy": [ + ">=0.9.3" + ], + "assume_role_policy_arns": [ + ">=0.13.0" + ], + "assume_role_tags": [ + ">=0.13.0" + ], + "assume_role_transitive_tag_keys": [ + ">=0.13.0" + ], + "bucket": [ + ">=0.9.0" + ], + "dynamodb_endpoint": [ + ">=0.11.14" + ], + "dynamodb_table": [ + ">=0.9.7" + ], + "encrypt": [ + ">=0.9.0" + ], + "endpoint": [ + ">=0.9.0" + ], + "external_id": [ + ">=0.9.3" + ], + "force_path_style": [ + ">=0.11.2" + ], + "iam_endpoint": [ + ">=0.11.14" + ], + "key": [ + ">=0.9.0" + ], + "kms_key_id": [ + ">=0.9.0" + ], + "lock_table": [ + ">=0.9.0", + "<=0.12.31" + ], + "max_retries": [ + ">=0.11.14" + ], + "profile": [ + ">=0.9.0" + ], + "region": [ + ">=0.9.0" + ], + "role_arn": [ + ">=0.9.0" + ], + "secret_key": [ + ">=0.9.0" + ], + "session_name": [ + ">=0.9.3" + ], + "shared_credentials_file": [ + ">=0.9.0" + ], + "skip_credentials_validation": [ + ">=0.10.8" + ], + "skip_get_ec2_platforms": [ + ">=0.10.8", + "<=0.12.31" + ], + "skip_metadata_api_check": [ + ">=0.10.8" + ], + "skip_region_validation": [ + ">=0.11.2" + ], + "skip_requesting_account_id": [ + ">=0.10.8", + "<=0.12.31" + ], + "sse_customer_key": [ + ">=0.12.8" + ], + "sts_endpoint": [ + ">=0.11.14" + ], + "token": [ + ">=0.9.0" + ], + "workspace_key_prefix": [ + ">=0.10.0" + ] + }, + "environment_variables": { + "AWS_ACCESS_KEY_ID": [ + ">=0.9.0" + ], + "AWS_DEFAULT_REGION": [ + ">=0.9.0" + ], + "AWS_IAM_ENDPOINT": [ + ">=0.11.14" + ], + "AWS_PROFILE": [ + ">=0.9.0" + ], + "AWS_REGION": [ + ">=0.9.0" + ], + "AWS_S3_ENDPOINT": [ + ">=0.9.0" + ], + "AWS_SECRET_ACCESS_KEY": [ + ">=0.9.0" + ], + "AWS_SESSION_TOKEN": [ + ">=0.9.0" + ], + "AWS_SSE_CUSTOMER_KEY": [ + ">=0.12.8" + ], + "AWS_STS_ENDPOINT": [ + ">=0.11.14" + ], + "AWS_DYNAMODB_ENDPOINT": [ + ">=0.11.14" + ] + } + }, + "swift": { + "terraform": [ + ">=0.9.0" + ], + "config_variables": { + "allow_reauth": [ + ">=0.13.0" + ], + "application_credential_id": [ + ">=0.12.2" + ], + "application_credential_name": [ + ">=0.12.2" + ], + "application_credential_secret": [ + ">=0.12.2" + ], + "archive_container": [ + ">=0.10.0" + ], + "archive_path": [ + ">=0.9.0" + ], + "auth_url": [ + ">=0.9.0" + ], + "cacert_file": [ + ">=0.9.0" + ], + "cert": [ + ">=0.9.0" + ], + "cloud": [ + ">=0.12.2" + ], + "container": [ + ">=0.10.0" + ], + "default_domain": [ + ">=0.12.2" + ], + "disable_no_cache_header": [ + ">=0.13.0" + ], + "domain_id": [ + ">=0.9.0" + ], + "domain_name": [ + ">=0.9.0" + ], + "endpoint_type": [ + ">=0.10.0" + ], + "expire_after": [ + ">=0.9.0" + ], + "insecure": [ + ">=0.9.0" + ], + "key": [ + ">=0.9.0" + ], + "lock": [ + ">=0.12.0" + ], + "max_retries": [ + ">=0.13.0" + ], + "password": [ + ">=0.9.0" + ], + "path": [ + ">=0.9.0" + ], + "project_domain_id": [ + ">=0.12.2" + ], + "project_domain_name": [ + ">=0.12.2" + ], + "region_name": [ + ">=0.9.0" + ], + "state_name": [ + ">=0.12.4" + ], + "swauth": [ + ">=0.13.0" + ], + "tenant_id": [ + ">=0.9.0" + ], + "tenant_name": [ + ">=0.9.0" + ], + "token": [ + ">=0.9.0" + ], + "user_domain_id": [ + ">=0.12.2" + ], + "user_domain_name": [ + ">=0.12.2" + ], + "user_id": [ + ">=0.9.0" + ], + "user_name": [ + ">=0.9.0" + ] + }, + "environment_variables": { + "OS_ALLOW_REAUTH": [ + ">=0.13.0" + ], + "DEFAULT_DOMAIN": [ + ">=0.9.0" + ], + "OS_AUTH_TOKEN": [ + ">=0.9.0" + ], + "OS_AUTH_URL": [ + ">=0.9.0" + ], + "OS_CACERT": [ + ">=0.9.0" + ], + "OS_CERT": [ + ">=0.9.0" + ], + "OS_DEFAULT_DOMAIN": [ + ">=0.10.0" + ], + "OS_DOMAIN_ID": [ + ">=0.9.0" + ], + "OS_DOMAIN_NAME": [ + ">=0.9.0" + ], + "OS_ENDPOINT_TYPE": [ + ">=0.10.0" + ], + "OS_INSECURE": [ + ">=0.9.0" + ], + "OS_KEY": [ + ">=0.9.0" + ], + "OS_PASSWORD": [ + ">=0.9.0" + ], + "OS_PROJECT_DOMAIN_ID": [ + ">=0.9.0" + ], + "OS_PROJECT_DOMAIN_NAME": [ + ">=0.9.0" + ], + "OS_PROJECT_ID": [ + ">=0.9.0" + ], + "OS_PROJECT_NAME": [ + ">=0.9.0" + ], + "OS_REGION_NAME": [ + ">=0.9.0" + ], + "OS_SWAUTH": [ + ">=0.13.0" + ], + "OS_TENANT_ID": [ + ">=0.9.0" + ], + "OS_TENANT_NAME": [ + ">=0.9.0" + ], + "OS_TOKEN": [ + ">=0.12.2" + ], + "OS_USERNAME": [ + ">=0.9.0" + ], + "OS_USER_DOMAIN_ID": [ + ">=0.9.0" + ], + "OS_USER_DOMAIN_NAME": [ + ">=0.9.0" + ], + "OS_USER_ID": [ + ">=0.9.0" + ] + } + } +} diff --git a/image/src/terraform_version/env.py b/image/src/terraform_version/env.py new file mode 100644 index 00000000..2c99abb4 --- /dev/null +++ b/image/src/terraform_version/env.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import sys +from typing import Iterable, Optional + +from github_actions.debug import debug +from github_actions.env import ActionsEnv +from terraform.versions import Version, Constraint, apply_constraints, latest_version + + +def try_read_env(actions_env: ActionsEnv, versions: Iterable[Version]) -> Optional[Version]: + if 'TERRAFORM_VERSION' not in actions_env: + return None + + constraint = actions_env['TERRAFORM_VERSION'] + + try: + valid_versions = list(apply_constraints(versions, [Constraint(c) for c in constraint.split(',')])) + if not valid_versions: + return None + return latest_version(valid_versions) + + except Exception as exception: + debug(str(exception)) + + return None diff --git a/image/src/terraform_version/local_state.py b/image/src/terraform_version/local_state.py new file mode 100644 index 00000000..33e8277f --- /dev/null +++ b/image/src/terraform_version/local_state.py @@ -0,0 +1,35 @@ +import json +import os +from pathlib import Path +from typing import Optional + +from github_actions.debug import debug +from terraform.versions import Version + + +def read_local_state(module_dir: Path) -> Optional[Version]: + """Return the terraform version that wrote a local terraform.tfstate file.""" + + state_path = os.path.join(module_dir, 'terraform.tfstate') + + if not os.path.isfile(state_path): + return None + + try: + with open(state_path) as f: + state = json.load(f) + if state.get('serial') > 0: + return Version(state.get('terraform_version')) + except Exception as e: + debug(str(e)) + + return None + + +def try_read_local_state(module_dir: Path) -> Optional[Version]: + try: + return read_local_state(module_dir) + except Exception as e: + debug(str(e)) + + return None diff --git a/image/src/terraform_version/remote_state.py b/image/src/terraform_version/remote_state.py new file mode 100644 index 00000000..3e51aef8 --- /dev/null +++ b/image/src/terraform_version/remote_state.py @@ -0,0 +1,230 @@ +"""Discover the terraform version that wrote an existing state file.""" + +from __future__ import annotations + +import importlib.resources +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Iterable, Optional, Tuple, Union + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.download import get_executable +from terraform.exec import init_args +from terraform.module import load_backend_config_file, TerraformModule +from terraform.versions import apply_constraints, Constraint, Version, earliest_version + + +def read_backend_config_vars(init_inputs: InitInputs) -> dict[str, str]: + """Read any backend config from input variables.""" + + config: dict[str, str] = {} + + for path in init_inputs.get('INPUT_BACKEND_CONFIG_FILE', '').replace(',', '\n').splitlines(): + try: + config |= load_backend_config_file(Path(path)) # type: ignore + except Exception as e: + debug(f'Failed to load backend config file {path}') + debug(str(e)) + + for backend_var in init_inputs.get('INPUT_BACKEND_CONFIG', '').replace(',', '\n').splitlines(): + if match := re.match(r'(.*)\s*=\s*(.*)', backend_var): + config[match.group(1)] = match.group(2) + + return config + + +def backend_config(module: TerraformModule) -> Tuple[str, dict[str, Any]]: + """Return the backend config specified in the terraform module.""" + + for terraform in module.get('terraform', []): + for backend in terraform.get('backend', []): + for backend_type, config in backend.items(): + return backend_type, config + + return 'local', {} + + +def get_backend_constraints(module: TerraformModule, backend_config_vars: dict[str, str]) -> list[Constraint]: + """ + Get any version constraints we can glean from the backend configuration variables + + This should be enough to get a version of terraform that can init the backend and pull the state + """ + + backend_type, config = backend_config(module) + backend_constraints = json.loads(importlib.resources.read_binary('terraform_version', 'backend_constraints.json')) + + if backend_type == 'azurerm': + backend_type = 'azure' + + if backend_type not in backend_constraints: + return [] + + constraints = [Constraint(constraint) for constraint in backend_constraints[backend_type]['terraform']] + + for config_var in config | backend_config_vars: + if config_var not in backend_constraints[backend_type]['config_variables']: + continue + + for constraint in backend_constraints[backend_type]['config_variables'][config_var]: + constraints.append(Constraint(constraint)) + + for env_var in os.environ: + if env_var not in backend_constraints[backend_type]['environment_variables']: + continue + + for constraint in backend_constraints[backend_type]['environment_variables'][env_var]: + constraints.append(Constraint(constraint)) + + return constraints + + +def dump_backend_hcl(module: TerraformModule) -> str: + """Return a string representation of the backend config for the given module.""" + + def hcl_value(value: str | bool | int | float) -> str: + """The value as represented in hcl.""" + if isinstance(value, str): + return f'"{value}"' + elif value is True: + return 'true' + elif value is False: + return 'false' + else: + return str(value) + + backend_type, config = backend_config(module) + debug(f'{backend_type=}') + if backend_type == 'local': + return '' + + tf = 'terraform {\n backend "' + backend_type + '" {\n' + + for k, v in config.items(): + if isinstance(v, list): + tf += f' {k} {{\n' + for block in v: + for k, v in block.items(): + tf += f' {k} = {hcl_value(v)}\n' + tf += ' }\n' + else: + tf += f' {k} = {hcl_value(v)}\n' + + tf += ' }\n' + tf += '}\n' + + return tf + + +def try_init(terraform: Version, init_args: list[str], workspace: str, backend_tf: str) -> Optional[Union[Version, Constraint]]: + """ + Try and initialize the specified backend using the specified terraform version. + + Returns the information discovered from doing the init. This could be: + - Version: the version of terraform used to write the state + - Constraint: a constraint to apply to the available versions, that further narrows down to the version used to write the state + - None: There is no remote state + """ + + terraform_path = get_executable(terraform) + module_dir = tempfile.mkdtemp() + + with open(os.path.join(module_dir, 'terraform.tf'), 'w') as f: + f.write(backend_tf) + + # Here we go + result = subprocess.run( + [str(terraform_path), 'init'] + init_args, + env=os.environ | {'TF_INPUT': 'false', 'TF_WORKSPACE': workspace}, + capture_output=True, + cwd=module_dir + ) + debug(f'{result.args[:2]=}') + debug(f'{result.returncode=}') + debug(result.stdout.decode()) + debug(result.stderr.decode()) + + if result.returncode != 0: + if match := re.search(rb'state snapshot was created by Terraform v(.*),', result.stderr): + return Version(match.group(1).decode()) + elif b'does not support state version 4' in result.stderr: + return Constraint('>=0.12.0') + elif b'Failed to select workspace' in result.stderr: + return None + else: + debug(str(result.stderr)) + return None + + result = subprocess.run( + [terraform_path, 'state', 'pull'], + env=os.environ | {'TF_INPUT': 'false', 'TF_WORKSPACE': workspace}, + capture_output=True, + cwd=module_dir + ) + debug(f'{result.args=}') + debug(f'{result.returncode=}') + debug(f'{result.stdout.decode()=}') + debug(f'{result.stderr.decode()=}') + + if result.returncode != 0: + if b'does not support state version 4' in result.stderr: + return Constraint('>=0.12.0') + raise Exception(result.stderr) + + try: + state = json.loads(result.stdout.decode()) + if state['version'] == 4 and state['serial'] == 0 and not state.get('outputs', {}): + return None # This workspace has no state + + if b'no state' in result.stderr: + return None + + if terraform < Version('0.12.0'): + # terraform_version is reported correctly in state output + return Version(state['terraform_version']) + + # terraform_version is made up + except Exception as e: + debug(str(e)) + + # There is some state + return terraform + + +def guess_state_version(inputs: InitInputs, module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: + """Try and guess the terraform version that wrote the remote state file of the specified module.""" + + args = init_args(inputs) + backend_tf = dump_backend_hcl(module) + + candidate_versions = list(versions) + + while candidate_versions: + result = try_init(earliest_version(candidate_versions), args, inputs.get('INPUT_WORKSPACE', 'default'), backend_tf) + if isinstance(result, Version): + return result + elif isinstance(result, Constraint): + candidate_versions = list(apply_constraints(candidate_versions, [result])) + elif result is None: + return None + else: + candidate_versions = list(apply_constraints(candidate_versions, [Constraint(f'!={earliest_version}')])) + + return None + + +def try_guess_state_version(inputs: InitInputs, module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: + """Try and guess the terraform version that wrote the remote state file of the specified module.""" + + try: + return guess_state_version(inputs, module, versions) + except Exception as e: + debug('Failed to find the terraform version from existing state') + debug(str(e)) + + return None diff --git a/image/src/terraform_version/remote_workspace.py b/image/src/terraform_version/remote_workspace.py new file mode 100644 index 00000000..3007ebcb --- /dev/null +++ b/image/src/terraform_version/remote_workspace.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Iterable, Optional + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.cloud import get_workspace +from terraform.module import TerraformModule, get_remote_backend_config, get_cloud_config +from terraform.versions import Version, latest_version + + +def get_remote_workspace_version(inputs: InitInputs, module: TerraformModule, cli_config_path: Path, versions: Iterable[Version]) -> Optional[Version]: + """Get the terraform version set in a terraform cloud/enterprise workspace.""" + + backend_config = get_remote_backend_config( + module, + backend_config_files=inputs.get('INPUT_BACKEND_CONFIG_FILE', ''), + backend_config_vars=inputs.get('INPUT_BACKEND_CONFIG', ''), + cli_config_path=cli_config_path + ) + + if backend_config is None: + backend_config = get_cloud_config( + module, + cli_config_path=cli_config_path + ) + + if backend_config is None: + return None + + if workspace_info := get_workspace(backend_config, inputs['INPUT_WORKSPACE']): + version = str(workspace_info['attributes']['terraform-version']) + if version == 'latest': + return latest_version(versions) + else: + return Version(version) + + return None + + +def try_get_remote_workspace_version(inputs: InitInputs, module: TerraformModule, cli_config_path: Path, versions: Iterable[Version]) -> Optional[Version]: + try: + return get_remote_workspace_version(inputs, module, cli_config_path, versions) + except Exception as exception: + debug('Failed to get terraform version from remote workspace') + debug(str(exception)) + + return None diff --git a/image/src/terraform_version/required_version.py b/image/src/terraform_version/required_version.py new file mode 100644 index 00000000..cf4aa6a1 --- /dev/null +++ b/image/src/terraform_version/required_version.py @@ -0,0 +1,26 @@ +from typing import Optional, Iterable + +from github_actions.debug import debug +from terraform.module import get_version_constraints, TerraformModule +from terraform.versions import Version, apply_constraints, latest_version + + +def get_required_version(module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: + constraints = get_version_constraints(module) + if constraints is None: + return None + + valid_versions = list(apply_constraints(versions, constraints)) + if not valid_versions: + raise RuntimeError(f'No versions of terraform match the required_version constraints {constraints}\n') + + return latest_version(valid_versions) + + +def try_get_required_version(module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: + try: + return get_required_version(module, versions) + except Exception as e: + debug('Failed to get terraform version from required_version constraint') + + return None diff --git a/image/src/terraform_version/tfenv.py b/image/src/terraform_version/tfenv.py new file mode 100644 index 00000000..1dd20ba0 --- /dev/null +++ b/image/src/terraform_version/tfenv.py @@ -0,0 +1,61 @@ +"""tfenv .terraform-version file support.""" + +from __future__ import annotations + +import os +import re +from typing import Iterable, Optional + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.versions import latest_version, Version + + +def parse_tfenv(terraform_version_file: str, versions: Iterable[Version]) -> Version: + """ + Return the version specified in the terraform version file + + :param terraform_version_file: The contents of a tfenv .terraform-version file. + :param versions: The available terraform versions + :return: The terraform version specified by the file + """ + + version = terraform_version_file.strip() + + if version == 'latest': + return latest_version(v for v in versions if not v.pre_release) + + if version.startswith('latest:'): + version_regex = version.split(':', maxsplit=1)[1] + + matched = [v for v in versions if re.search(version_regex, str(v))] + + if not matched: + raise Exception(f'No terraform versions match regex {version_regex}') + + return latest_version(matched) + + return Version(version) + + +def try_read_tfenv(inputs: InitInputs, versions: Iterable[Version]) -> Optional[Version]: + """ + Return the terraform version specified by any .terraform-version file. + + :param inputs: The action inputs + :param versions: The available terraform versions + :returns: The terraform version specified by any .terraform-version file, which may be None. + """ + + tfenv_path = os.path.join(inputs.get('INPUT_PATH', '.'), '.terraform-version') + + if not os.path.exists(tfenv_path): + return None + + try: + with open(tfenv_path) as f: + return parse_tfenv(f.read(), versions) + except Exception as e: + debug(str(e)) + + return None diff --git a/image/src/terraform_version/tfswitch.py b/image/src/terraform_version/tfswitch.py new file mode 100644 index 00000000..74dcf2a5 --- /dev/null +++ b/image/src/terraform_version/tfswitch.py @@ -0,0 +1,43 @@ +"""tfswitch .tfswitchrc file support.""" + +from __future__ import annotations + +import os +from typing import Optional + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.versions import Version + + +def parse_tfswitch(tfswitch: str) -> Version: + """ + Return the terraform version specified by a tfswitch file + + :param tfswitch: The contents of a .tfswitch file + :return: The terraform version specified by the file + """ + + return Version(tfswitch.strip()) + + +def try_read_tfswitch(inputs: InitInputs) -> Optional[Version]: + """ + Return the terraform version specified by any .tfswitchrc file. + + :param inputs: The action inputs + :returns: The terraform version specified by the file, which may be None. + """ + + tfswitch_path = os.path.join(inputs.get('INPUT_PATH', '.'), '.tfswitchrc') + + if not os.path.exists(tfswitch_path): + return None + + try: + with open(tfswitch_path) as f: + return parse_tfswitch(f.read()) + except Exception as e: + debug(str(e)) + + return None diff --git a/image/tools/convert_output.py b/image/tools/convert_output.py index d258c6d5..5fbff15f 100755 --- a/image/tools/convert_output.py +++ b/image/tools/convert_output.py @@ -4,6 +4,7 @@ import sys from typing import Dict, Iterable + def convert_to_github(outputs: Dict) -> Iterable[str]: for name, output in outputs.items(): diff --git a/image/tools/convert_validate_report.py b/image/tools/convert_validate_report.py index 546a3a36..9dc760f5 100755 --- a/image/tools/convert_validate_report.py +++ b/image/tools/convert_validate_report.py @@ -1,9 +1,10 @@ #!/usr/bin/python3 import json +import os.path import sys from typing import Dict, Iterable -import os.path + def relative_to_base(file_path: str, base_path: str): return os.path.normpath(os.path.join(base_path, file_path)) diff --git a/image/tools/convert_version.py b/image/tools/convert_version.py index f59d29e8..b10ce991 100755 --- a/image/tools/convert_version.py +++ b/image/tools/convert_version.py @@ -3,7 +3,7 @@ import json import re import sys -from typing import Iterable, Dict +from typing import Dict, Iterable def convert_version(tf_output: str) -> Iterable[str]: diff --git a/image/tools/format_tf_credentials.py b/image/tools/format_tf_credentials.py index 1a055eee..c8c3a31d 100755 --- a/image/tools/format_tf_credentials.py +++ b/image/tools/format_tf_credentials.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 import os -import sys import re +import sys def format_credentials(input): diff --git a/image/tools/github_comment_react.py b/image/tools/github_comment_react.py index 9c5e89b3..83e69180 100755 --- a/image/tools/github_comment_react.py +++ b/image/tools/github_comment_react.py @@ -4,7 +4,7 @@ import json import os import sys -from typing import Optional, cast, NewType, TypedDict +from typing import NewType, Optional, TypedDict, cast import requests diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 80ae0cd5..2af195d3 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -6,7 +6,8 @@ import os import re import sys -from typing import Optional, Dict, Iterable, cast, NewType, TypedDict, Tuple, Any +from typing import (Any, Dict, Iterable, NewType, Optional, Tuple, TypedDict, + cast) import requests diff --git a/image/tools/http_credential_actions_helper.py b/image/tools/http_credential_actions_helper.py index 1bfaf82e..58764db3 100755 --- a/image/tools/http_credential_actions_helper.py +++ b/image/tools/http_credential_actions_helper.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -import sys import os import re -from typing import Dict, List, Iterable, Optional +import sys from dataclasses import dataclass +from typing import Dict, Iterable, List, Optional @dataclass diff --git a/image/tools/latest_terraform_version.py b/image/tools/latest_terraform_version.py deleted file mode 100755 index 06e67ff8..00000000 --- a/image/tools/latest_terraform_version.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/python3 - -import re -from distutils.version import StrictVersion -from typing import List - -import requests - -version = re.compile(br'/(\d+\.\d+\.\d+)/') - - -def get_versions() -> List[StrictVersion]: - response = requests.get('https://releases.hashicorp.com/terraform/') - response.raise_for_status() - - versions = [StrictVersion(v.group(1).decode()) for v in version.finditer(response.content)] - return versions - - -def latest_version(versions: List[StrictVersion]): - latest = sorted(versions, reverse=True)[0] - return '.'.join([str(x) for x in latest.version]) - - -if __name__ == '__main__': - print(latest_version(get_versions())) - - -def test_version(): - versions = get_versions() - print(versions) - assert StrictVersion('0.12.28') in versions - assert StrictVersion('0.12.5') in versions - assert StrictVersion('0.11.14') in versions - assert StrictVersion('0.13.0') in versions diff --git a/image/tools/workspace_exists.py b/image/tools/workspace_exists.py index e3b7c150..35d8a4d0 100755 --- a/image/tools/workspace_exists.py +++ b/image/tools/workspace_exists.py @@ -2,6 +2,7 @@ import sys + def debug(msg: str) -> None: for line in msg.splitlines(): sys.stderr.write(f'::debug::{line}\n') diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index ec0c4bd7..abba7447 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -12,7 +12,7 @@ inputs: required: false default: default backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false default: "" backend_config_file: diff --git a/terraform-check/action.yaml b/terraform-check/action.yaml index c64375c4..f74d5ea7 100644 --- a/terraform-check/action.yaml +++ b/terraform-check/action.yaml @@ -12,11 +12,13 @@ inputs: required: false default: default backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false + default: "" backend_config_file: - description: Path to a backend config file" + description: Path to a backend config file required: false + default: "" variables: description: Variable definitions required: false diff --git a/terraform-destroy-workspace/action.yaml b/terraform-destroy-workspace/action.yaml index ad16642d..ec09aded 100644 --- a/terraform-destroy-workspace/action.yaml +++ b/terraform-destroy-workspace/action.yaml @@ -11,11 +11,13 @@ inputs: description: Name of the terraform workspace required: true backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false + default: "" backend_config_file: description: Path to a backend config file required: false + default: "" variables: description: Variable definitions required: false diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index 1f8145dd..b9767f37 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -12,11 +12,13 @@ inputs: required: false default: default backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false + default: "" backend_config_file: description: Path to a backend config file required: false + default: "" variables: description: Variable definitions required: false diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index 975b1907..57dc1a44 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -17,6 +17,41 @@ If any files are not correctly formatted a failing GitHub check will be added fo - Optional - Default: The action workspace +* `workspace` + + Terraform workspace to inspect when discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + + - Type: string + - Optional + +* `backend_config` + + List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` + + - Type: string + - Optional + +* `backend_config_file` + + List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + + - Type: string + - Optional + ## Outputs * `failure-reason` @@ -25,6 +60,31 @@ If any files are not correctly formatted a failing GitHub check will be added fo If the job fails for any other reason this will not be set. This can be used with the Actions expression syntax to conditionally run a step when the format check fails. +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + For the purpose of detecting the terraform version to use from a TFC/E backend. + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_TOKENS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example workflow runs on every push and fails if any of the diff --git a/terraform-fmt-check/action.yaml b/terraform-fmt-check/action.yaml index fda9a3ba..d94bc0d6 100644 --- a/terraform-fmt-check/action.yaml +++ b/terraform-fmt-check/action.yaml @@ -7,6 +7,18 @@ inputs: description: Path to the terraform configuration required: false default: . + workspace: + description: Name of the terraform workspace + required: false + default: default + backend_config: + description: List of backend config values to set, one per line + required: false + default: "" + backend_config_file: + description: Path to a backend config file + required: false + default: "" runs: using: docker diff --git a/terraform-fmt/README.md b/terraform-fmt/README.md index f87bda18..3444ae0b 100644 --- a/terraform-fmt/README.md +++ b/terraform-fmt/README.md @@ -14,6 +14,66 @@ This action uses the `terraform fmt` command to reformat files in a directory in - Optional - Default: The action workspace +* `workspace` + + Terraform workspace to inspect when discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + + - Type: string + - Optional + +* `backend_config` + + List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` + + - Type: string + - Optional + +* `backend_config_file` + + List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + + - Type: string + - Optional + +## Environment Variables + +* `TERRAFORM_CLOUD_TOKENS` + + For the purpose of detecting the terraform version to use from a TFC/E backend. + API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. + These tokens may be used with the `remote` backend and for fetching required modules from the registry. + + e.g for terraform cloud: + ```yaml + env: + TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + ``` + + With Terraform Enterprise or other registries: + ```yaml + env: + TERRAFORM_CLOUD_TOKENS: | + app.terraform.io=${{ secrets.TF_CLOUD_TOKEN }} + terraform.example.com=${{ secrets.TF_REGISTRY_TOKEN }} + ``` + + - Type: string + - Optional + ## Example usage This example automatically creates a pull request to fix any formatting diff --git a/terraform-fmt/action.yaml b/terraform-fmt/action.yaml index 8806d78e..001b0191 100644 --- a/terraform-fmt/action.yaml +++ b/terraform-fmt/action.yaml @@ -7,6 +7,18 @@ inputs: description: Path to the terraform configuration required: false default: . + workspace: + description: Name of the terraform workspace + required: false + default: default + backend_config: + description: List of backend config values to set, one per line + required: false + default: "" + backend_config_file: + description: Path to a backend config file + required: false + default: "" runs: using: docker diff --git a/terraform-new-workspace/action.yaml b/terraform-new-workspace/action.yaml index 374c8c9e..02fe0ef5 100644 --- a/terraform-new-workspace/action.yaml +++ b/terraform-new-workspace/action.yaml @@ -11,11 +11,13 @@ inputs: description: Name of the terraform workspace required: true backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false + default: "" backend_config_file: - description: Path to a backend config file" + description: Path to a backend config file required: false + default: "" runs: using: docker diff --git a/terraform-output/action.yaml b/terraform-output/action.yaml index 63f18f11..1d6e25e9 100644 --- a/terraform-output/action.yaml +++ b/terraform-output/action.yaml @@ -12,11 +12,13 @@ inputs: required: false default: default backend_config: - description: List of backend configs to set, one per line + description: List of backend config values to set, one per line required: false + default: "" backend_config_file: description: Path to a backend config file required: false + default: "" runs: using: docker diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 9772ffc9..544477f2 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -14,9 +14,11 @@ inputs: backend_config: description: List of backend config values to set, one per line required: false + default: "" backend_config_file: description: Path to a backend config file required: false + default: "" variables: description: Variable definitions required: false diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index 873e8248..caedde82 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -2,7 +2,7 @@ This is one of a suite of terraform related actions - find them at [dflook/terraform-github-actions](https://github.com/dflook/terraform-github-actions). -Retrieves the root-level outputs from a terraform remote state. +Retrieves the root-level outputs from a Terraform remote state. ## Inputs diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 11fda4a6..377c2670 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -25,12 +25,42 @@ If the terraform configuration is not valid, the build is failed. * `workspace` - Terraform workspace to use for the `terraform.workspace` value while validating. + Terraform workspace to use for the `terraform.workspace` value while validating. Note that for remote operations in Terraform Cloud/Enterprise, this is always `default`. + + Also used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. - Type: string - Optional - Default: `default` +* `backend_config` + + List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` + + - Type: string + - Optional + +* `backend_config_file` + + List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + Paths should be relative to the GitHub Actions workspace + + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + + - Type: string + - Optional + ## Outputs * `failure-reason` @@ -44,7 +74,7 @@ If the terraform configuration is not valid, the build is failed. * `TERRAFORM_CLOUD_TOKENS` API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. - These tokens may be used for fetching required modules from the registry. + These tokens may be used for fetching required modules from the registry, and discovering the terraform version to use from a TFC/E workspace. e.g for terraform cloud: ```yaml @@ -80,7 +110,7 @@ If the terraform configuration is not valid, the build is failed. * `TERRAFORM_PRE_RUN` - A set of commands that will be ran prior to `terraform init`. This can be used to customise the environment before running terraform. + A set of commands that will be run prior to `terraform init`. This can be used to customise the environment before running terraform. The runtime environment for these actions is subject to change in minor version releases. If using this environment variable, specify the minor version of the action to use. diff --git a/terraform-validate/action.yaml b/terraform-validate/action.yaml index a15e20b2..8c7c908d 100644 --- a/terraform-validate/action.yaml +++ b/terraform-validate/action.yaml @@ -11,6 +11,14 @@ inputs: description: Name of the workspace to use for the `terraform.workspace` value while validating. required: false default: default + backend_config: + description: List of backend configs to set, one per line + required: false + default: "" + backend_config_file: + description: Path to a backend config file + required: false + default: "" runs: using: docker diff --git a/terraform-version/README.md b/terraform-version/README.md index 235831eb..73c4d274 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -2,15 +2,20 @@ This is one of a suite of terraform related actions - find them at [dflook/terraform-github-actions](https://github.com/dflook/terraform-github-actions). -This action determines the terraform and provider versions to use for a terraform configuration directory. +This action determines the terraform and provider versions to use for a Terraform root module. + +The best way to specify the version is using a [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) constraint. The version to use is discovered from the first of: -1. A [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) - constraint in the terraform configuration. -2. A [tfswitch](https://warrensbox.github.io/terraform-switcher/) `.tfswitchrc` file -3. A [tfenv](https://github.com/tfutils/tfenv) `.terraform-version` file in path of the terraform - configuration. -4. The latest terraform version +1. The version set in the TFC/TFE workspace if the module uses a `remote` backend or `cloud` configuration, and the remote workspace exists. +2. A [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) + constraint in the terraform configuration. If the constraint is range, the latest matching version is used. +3. A [tfswitch](https://warrensbox.github.io/terraform-switcher/) `.tfswitchrc` file in the module path +4. A [tfenv](https://github.com/tfutils/tfenv) `.terraform-version` file in the module path +5. An [asdf](https://asdf-vm.com/) `.tool-versions` file in the module path or any parent path +6. A `TERRAFORM_VERSION` environment variable containing a [version constraint](https://www.terraform.io/language/expressions/version-constraints). If the constraint allows multiple versions, the latest matching version is used. +7. The Terraform version that created the current state file (best effort). +8. The latest terraform version The version of terraform and all required providers will be output to the workflow log. @@ -22,12 +27,51 @@ outputs yourself. * `path` - Path to the terraform configuration to apply + Path to the terraform root module - Type: string - Optional - Default: The action workspace +* `workspace` + + The workspace to determine the Terraform version for. + + - Type: string + - Optional + - Default: `default` + +* `backend_config` + + List of terraform backend config values, one per line. + + This will be used to fetch the Terraform version set in the TFC/TFE workspace if using the `remote` backend. + For other backend types, this is used to fetch the version that most recently wrote to the terraform state. + + ```yaml + with: + backend_config: token=${{ secrets.BACKEND_TOKEN }} + ``` + + - Type: string + - Optional + +* `backend_config_file` + + List of terraform backend config files to use, one per line. + Paths should be relative to the GitHub Actions workspace + + This will be used to fetch the Terraform version set in the TFC/TFE workspace if using the `remote` backend. + For other backend types, this is used to fetch the version that most recently wrote to the terraform state. + + ```yaml + with: + backend_config_file: prod.backend.tfvars + ``` + + - Type: string + - Optional + ## Environment Variables * `TERRAFORM_CLOUD_TOKENS` diff --git a/terraform-version/action.yaml b/terraform-version/action.yaml index 04b1b1a5..007b55d9 100644 --- a/terraform-version/action.yaml +++ b/terraform-version/action.yaml @@ -7,6 +7,18 @@ inputs: description: Path to the terraform configuration required: false default: . + workspace: + description: Name of the terraform workspace to get the version for + required: false + default: default + backend_config: + description: List of backend config values to set, one per line + required: false + default: "" + backend_config_file: + description: Path to a backend config file + required: false + default: "" outputs: version: diff --git a/tests/python/terraform_version/test_asdf.py b/tests/python/terraform_version/test_asdf.py new file mode 100644 index 00000000..3bd91256 --- /dev/null +++ b/tests/python/terraform_version/test_asdf.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from terraform.versions import Version +from terraform_version.asdf import parse_asdf + + +def test_parse_asdf(): + versions = [ + Version('0.13.6'), + Version('1.1.8'), + Version('1.1.9'), + Version('1.1.7'), + Version('1.1.0-alpha20210811'), + Version('1.2.0-alpha20225555') + ] + + assert parse_asdf('terraform 0.13.6', versions) == Version('0.13.6') + assert parse_asdf(''' + # comment + terraform 0.15.6 #basdasd + + ''', versions) == Version('0.15.6') + + assert parse_asdf('terraform 1.1.1-cool', versions) == Version('1.1.1-cool') + + try: + parse_asdf('', versions) + except Exception: + pass + else: + assert False + + try: + parse_asdf('blahblah', versions) + except Exception: + pass + else: + assert False + + try: + parse_asdf('terraform blasdasf', versions) + except Exception: + pass + else: + assert False + + assert parse_asdf('terraform latest', versions) == Version('1.1.9') diff --git a/tests/python/terraform_version/test_local_state.py b/tests/python/terraform_version/test_local_state.py new file mode 100644 index 00000000..5e7f344f --- /dev/null +++ b/tests/python/terraform_version/test_local_state.py @@ -0,0 +1,151 @@ +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from terraform.download import get_executable +from terraform.versions import Version +from terraform_version.local_state import read_local_state + +terraform_versions = [ + '1.1.2', + '1.1.1', + '1.1.0', + '1.0.11', + '1.0.10', + '1.0.9', + '1.0.8', + '1.0.7', + '1.0.6', + '1.0.5', + '1.0.4', + '1.0.3', + '1.0.2', + '1.0.1', + '1.0.0', + '0.15.5', + '0.15.4', + '0.15.3', + '0.15.2', + '0.15.1', + '0.15.0', + '0.14.11', + '0.14.10', + '0.14.9', + '0.14.8', + '0.14.7', + '0.14.6', + '0.14.5', + '0.14.4', + '0.14.3', + '0.14.2', + '0.14.1', + '0.14.0', + '0.13.7', + '0.13.6', + '0.13.5', + '0.13.4', + '0.13.3', + '0.13.2', + '0.13.1', + '0.13.0', + '0.12.31', + '0.12.30', + '0.12.29', + '0.12.28', + '0.12.27', + '0.12.26', + '0.12.25', + '0.12.24', + '0.12.23', + '0.12.21', + '0.12.20', + '0.12.19', + '0.12.18', + '0.12.17', + '0.12.16', + '0.12.15', + '0.12.14', + '0.12.13', + '0.12.12', + '0.12.11', + '0.12.10', + '0.12.9', + '0.12.8', + '0.12.7', + '0.12.6', + '0.12.5', + '0.12.4', + '0.12.3', + '0.12.2', + '0.12.1', + '0.12.0', + '0.11.15', + '0.11.14', + '0.11.13', + '0.11.12', + '0.11.11', + '0.11.10', + '0.11.9', + '0.11.8', + '0.11.7', + '0.11.6', + '0.11.5', + '0.11.4', + '0.11.3', + '0.11.2', + '0.11.1', + '0.11.0', + '0.10.8', + '0.10.7', + '0.10.6', + '0.10.5', + '0.10.4', + '0.10.3', + '0.10.2', + '0.10.1', + '0.10.0' +] + + +@pytest.fixture(scope='module', params=["0.11.8", "1.1.2"]) +def local_state_version(request): + terraform_version = Version(request.param) + terraform_path = get_executable(Version(request.param)) + + module_dir = Path(os.getcwd(), '.local_state_version', str(terraform_version)) + os.makedirs(module_dir, exist_ok=True) + + with open(os.path.join(module_dir, 'main.tf'), 'w') as f: + f.write(''' + output "hello" { value = "hello" } + ''') + + # Here we go + result = subprocess.run( + [terraform_path, 'init'], + env={'TF_INPUT': 'false'}, + capture_output=True, + cwd=module_dir + ) + assert result.returncode == 0 + result = subprocess.run( + [terraform_path, 'apply', '-auto-approve'], + env={'TF_INPUT': 'false'}, + capture_output=True, + cwd=module_dir + ) + assert result.returncode == 0 + + shutil.rmtree(os.path.join(module_dir, '.terraform'), ignore_errors=True) + + yield module_dir, terraform_version + + shutil.rmtree(module_dir, ignore_errors=True) + + +def test_state(local_state_version): + module_dir, terraform_version = local_state_version + assert read_local_state(module_dir) == terraform_version diff --git a/tests/python/terraform_version/test_remote_state_s3.py b/tests/python/terraform_version/test_remote_state_s3.py new file mode 100644 index 00000000..216a5814 --- /dev/null +++ b/tests/python/terraform_version/test_remote_state_s3.py @@ -0,0 +1,194 @@ +import os +import shutil +import subprocess + +import hcl2 +import pytest + +from terraform.download import download_version, get_executable +from terraform.module import load_module +from terraform.versions import Version, apply_constraints +from terraform_version.remote_state import try_guess_state_version, get_backend_constraints + +terraform_versions = [ + '1.1.3', + '1.1.2', + '1.1.1', + '1.1.0', + '1.0.11', + '1.0.10', + '1.0.9', + '1.0.8', + '1.0.7', + '1.0.6', + '1.0.5', + '1.0.4', + '1.0.3', + '1.0.2', + '1.0.1', + '1.0.0', + '0.15.5', + '0.15.4', + '0.15.3', + '0.15.2', + '0.15.1', + '0.15.0', + '0.14.11', + '0.14.10', + '0.14.9', + '0.14.8', + '0.14.7', + '0.14.6', + '0.14.5', + '0.14.4', + '0.14.3', + '0.14.2', + '0.14.1', + '0.14.0', + '0.13.7', + '0.13.6', + '0.13.5', + '0.13.4', + '0.13.3', + '0.13.2', + '0.13.1', + '0.13.0', + '0.12.31', + '0.12.30', + '0.12.29', + '0.12.28', + '0.12.27', + '0.12.26', + '0.12.25', + '0.12.24', + '0.12.23', + '0.12.21', + '0.12.20', + '0.12.19', + '0.12.18', + '0.12.17', + '0.12.16', + '0.12.15', + '0.12.14', + '0.12.13', + '0.12.12', + '0.12.11', + '0.12.10', + '0.12.9', + '0.12.8', + '0.12.7', + '0.12.6', + '0.12.5', + '0.12.4', + '0.12.3', + '0.12.2', + '0.12.1', + '0.12.0', + '0.11.15', + '0.11.14', + '0.11.13', + '0.11.12', + '0.11.11', + '0.11.10', + '0.11.9', + '0.11.8', + '0.11.7', + '0.11.6', + '0.11.5', + '0.11.4', + '0.11.3', + '0.11.2', + '0.11.1', + '0.11.0', + '0.10.8', + '0.10.7', + '0.10.6', + '0.10.5', + '0.10.4', + '0.10.3', + '0.10.2', + '0.10.1', + '0.10.0', + "0.9.11", + "0.9.10", + "0.9.9", + "0.9.8", + "0.9.7", +] + +@pytest.fixture(scope='module', params=["0.9.7", "0.11.8", "1.1.2"]) +def state_version(request): + terraform_version = Version(request.param) + terraform_path = get_executable(terraform_version) + + module_dir = os.path.join(os.getcwd(), '.terraform-state', str(terraform_version)) + os.makedirs(module_dir, exist_ok=True) + + with open(os.path.join(module_dir, 'main.tf'), 'w') as f: + backend_tf = ''' +terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "test_remote_state_s3_''' + str(terraform_version) + '''" + region = "eu-west-2" + dynamodb_table = "terraform-github-actions" + } +} + ''' + + f.write(backend_tf + ''' + +output "hello" { + value = "hello" +} + ''') + + # Here we go + result = subprocess.run( + [terraform_path, 'init'], + env=os.environ | {'TF_INPUT': 'false'}, + capture_output=True, + cwd=module_dir + ) + print(f'{result.args=}') + print(f'{result.returncode=}') + print(f'{result.stdout.decode()=}') + print(f'{result.stderr.decode()=}') + assert result.returncode == 0 + + result = subprocess.run( + [terraform_path, 'apply'] + (['-auto-approve'] if terraform_version >= Version('0.10.0') else []), + env=os.environ | {'TF_INPUT': 'false'}, + capture_output=True, + cwd=module_dir + ) + print(f'{result.args=}') + print(f'{result.returncode=}') + print(f'{result.stdout.decode()=}') + print(f'{result.stderr.decode()=}') + assert result.returncode == 0 + + shutil.rmtree(os.path.join(module_dir, '.terraform'), ignore_errors=True) + + yield terraform_version, backend_tf + + shutil.rmtree(module_dir, ignore_errors=True) + +def test_state(state_version): + + terraform_version, backend_tf = state_version + + module = hcl2.loads(backend_tf) + + assert try_guess_state_version( + { + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_BACKEND_CONFIG_FILE': '', + 'INPUT_WORKSPACE': 'default' + }, + module, + versions=apply_constraints( + sorted(Version(v) for v in terraform_versions), + get_backend_constraints(module, {}) + ) + ) == terraform_version diff --git a/tests/python/terraform_version/test_state.py b/tests/python/terraform_version/test_state.py new file mode 100644 index 00000000..a745d920 --- /dev/null +++ b/tests/python/terraform_version/test_state.py @@ -0,0 +1,82 @@ +import hcl2 + +from terraform.versions import Constraint +from terraform_version.remote_state import dump_backend_hcl, get_backend_constraints + + +def test_simple_backend(): + + expected_backend = hcl2.loads(''' + terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "blah" + region = "eu-west-2" + } + } +''') + + assert expected_backend == hcl2.loads(dump_backend_hcl(expected_backend)) + +def test_no_backend(): + expected_backend = hcl2.loads(''' + terraform { + required_version = "1.0.0" + } + ''') + + assert dump_backend_hcl(expected_backend).strip() == '' + +def test_oss_assume_role(): + expected_backend = hcl2.loads(''' + terraform { + backend "oss" { + access_key = "sausage" + assume_role { + role_arn = "asdasd" + session_name = "hello" + } + } + } + ''') + + assert expected_backend == hcl2.loads(dump_backend_hcl(expected_backend)) + +def test_backend_constraints(): + + module = hcl2.loads(''' + terraform { + backend "oss" { + access_key = "sausage" + mystery = true + assume_role { + role_arn = "asdasd" + session_name = "hello" + } + } + } + ''') + + assert get_backend_constraints(module, {}) == [Constraint('>=0.12.2'), Constraint('>=0.12.2'), Constraint('>=0.12.6')] + + module = hcl2.loads(''' + terraform { + backend "gcs" { + bucket = "sausage" + impersonate_service_account = true + region = "europe-west2" + unknown = "??" + path = "hello" + } + } + ''') + + assert get_backend_constraints(module, {}) == [ + Constraint('>=0.9.0'), + Constraint('>=0.9.0'), + Constraint('>=0.14.0'), + Constraint('>=0.11.0'), + Constraint('<=0.15.3'), + Constraint('>=0.9.0'), + Constraint('<=0.14.11') + ] diff --git a/tests/python/terraform_version/test_terraform_version.py b/tests/python/terraform_version/test_terraform_version.py new file mode 100644 index 00000000..d07b97ee --- /dev/null +++ b/tests/python/terraform_version/test_terraform_version.py @@ -0,0 +1,203 @@ +from terraform.versions import Version, Constraint +from terraform.exec import init_args + +def test_init_args(): + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': '', + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_PATH': '.' + }) == [] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': 'tests/hello/terraform.backendconfig', + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_PATH': '.' + }) == ['-backend-config=tests/hello/terraform.backendconfig'] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': 'tests/hello/terraform.backendconfig', + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_PATH': 'tests' + }) == ['-backend-config=hello/terraform.backendconfig'] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': 'tests/terraform.backendconfig', + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_PATH': 'tests/hello' + }) == ['-backend-config=../terraform.backendconfig'] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': ''' + tests/terraform.backendconfig + env/prod/terraform.backendconfig,env/common.backendconfig + ''', + 'INPUT_BACKEND_CONFIG': '', + 'INPUT_PATH': '.' + }) == ['-backend-config=tests/terraform.backendconfig', '-backend-config=env/prod/terraform.backendconfig', '-backend-config=env/common.backendconfig'] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': '', + 'INPUT_BACKEND_CONFIG': 'test=hello', + 'INPUT_PATH': '.' + }) == ['-backend-config=test=hello'] + + assert init_args({ + 'INPUT_BACKEND_CONFIG_FILE': '', + 'INPUT_BACKEND_CONFIG': ''' + + "test=hello" + + foo=bar,xyz=abc + + ''', + 'INPUT_PATH': '.' + }) == ['-backend-config="test=hello"', '-backend-config=foo=bar', '-backend-config=xyz=abc'] + +def test_version(): + v0_1_1 = Version('0.1.1') + assert v0_1_1.major == 0 and v0_1_1.minor == 1 and v0_1_1.patch == 1 and v0_1_1.pre_release == '' + assert str(v0_1_1) == '0.1.1' + + v1_0_11 = Version('1.0.11') + assert v1_0_11.major == 1 and v1_0_11.minor == 0 and v1_0_11.patch == 11 and v1_0_11.pre_release == '' + assert str(v1_0_11) == '1.0.11' + + v0_15_0_rc2 = Version('0.15.0-rc2') + assert v0_15_0_rc2.major == 0 and v0_15_0_rc2.minor == 15 and v0_15_0_rc2.patch == 0 and v0_15_0_rc2.pre_release == 'rc2' + assert str(v0_15_0_rc2) == '0.15.0-rc2' + + v0_15_0 = Version('0.15.0') + assert v0_15_0.major == 0 and v0_15_0.minor == 15 and v0_15_0.patch == 0 and v0_15_0.pre_release == '' + assert str(v0_15_0) == '0.15.0' + + assert v0_1_1 == v0_1_1 + assert v1_0_11 != v0_1_1 + assert v0_15_0_rc2 < v0_15_0 + assert v1_0_11 > v0_15_0 > v0_1_1 + +def test_constraint(): + constraint = Constraint('0.12.4-hello') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == 'hello' and constraint.operator == '=' + assert str(constraint) == '=0.12.4-hello' + assert constraint.is_allowed(Version('0.12.4-hello')) + assert not constraint.is_allowed(Version('0.12.4')) + + constraint = Constraint('0.12.4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '=' + assert str(constraint) == '=0.12.4' + + constraint = Constraint(' = 0 .1 2. 4-hello') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == 'hello' and constraint.operator == '=' + assert str(constraint) == '=0.12.4-hello' + + constraint = Constraint(' = 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '=' + assert str(constraint) == '=0.12.4' + + constraint = Constraint(' >= 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '>=' + assert str(constraint) == '>=0.12.4' + assert constraint.is_allowed(Version('0.12.4')) + assert constraint.is_allowed(Version('0.12.8')) + assert constraint.is_allowed(Version('0.13.0')) + assert constraint.is_allowed(Version('1.1.1')) + assert not constraint.is_allowed(Version('0.12.3')) + assert not constraint.is_allowed(Version('0.10.0')) + + constraint = Constraint(' > 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '>' + assert str(constraint) == '>0.12.4' + assert not constraint.is_allowed(Version('0.12.4')) + assert constraint.is_allowed(Version('0.12.8')) + assert constraint.is_allowed(Version('0.13.0')) + assert constraint.is_allowed(Version('1.1.1')) + assert not constraint.is_allowed(Version('0.12.3')) + assert not constraint.is_allowed(Version('0.10.0')) + + constraint = Constraint(' < 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '<' + assert str(constraint) == '<0.12.4' + assert not constraint.is_allowed(Version('0.12.4')) + assert not constraint.is_allowed(Version('0.12.8')) + assert not constraint.is_allowed(Version('0.13.0')) + assert not constraint.is_allowed(Version('1.1.1')) + assert constraint.is_allowed(Version('0.12.3')) + assert constraint.is_allowed(Version('0.10.0')) + + constraint = Constraint(' <= 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '<=' + assert str(constraint) == '<=0.12.4' + assert constraint.is_allowed(Version('0.12.4')) + assert not constraint.is_allowed(Version('0.12.8')) + assert not constraint.is_allowed(Version('0.13.0')) + assert not constraint.is_allowed(Version('1.1.1')) + assert constraint.is_allowed(Version('0.12.3')) + assert constraint.is_allowed(Version('0.10.0')) + + constraint = Constraint(' != 0 .1 2. 4') + assert constraint.major == 0 and constraint.minor == 12 and constraint.patch == 4 and constraint.pre_release == '' and constraint.operator == '!=' + assert str(constraint) == '!=0.12.4' + assert not constraint.is_allowed(Version('0.12.4')) + assert constraint.is_allowed(Version('0.12.8')) + assert constraint.is_allowed(Version('0.13.0')) + assert constraint.is_allowed(Version('1.1.1')) + assert constraint.is_allowed(Version('0.12.3')) + assert constraint.is_allowed(Version('0.10.0')) + + constraint = Constraint('1') + assert ( + constraint.major == 1 + and constraint.minor is None + and constraint.patch is None + and constraint.pre_release == '' + and constraint.operator == '=' + ) + + assert str(constraint) == '=1' + assert constraint.is_allowed(Version('1.0.0')) + assert not constraint.is_allowed(Version('1.0.1')) + assert not constraint.is_allowed(Version('0.0.9')) + assert not constraint.is_allowed(Version('1.0.0-wooo')) + + constraint = Constraint('1.2') + assert ( + constraint.major == 1 + and constraint.minor == 2 + and constraint.patch is None + and constraint.pre_release == '' + and constraint.operator == '=' + ) + + assert str(constraint) == '=1.2' + + constraint = Constraint('~>1.2.3') + assert constraint.major == 1 and constraint.minor == 2 and constraint.patch == 3 and constraint.pre_release == '' and constraint.operator == '~>' + assert str(constraint) == '~>1.2.3' + + constraint = Constraint('~>1.2') + assert ( + constraint.major == 1 + and constraint.minor == 2 + and constraint.patch is None + and constraint.pre_release == '' + and constraint.operator == '~>' + ) + + assert str(constraint) == '~>1.2' + + assert Constraint('0.12.0') < Constraint('0.12.1') + assert Constraint('0.12.0') == Constraint('0.12.0') + assert Constraint('0.12.0') != Constraint('0.15.0') + + test_ordering = [ + Constraint('0.11.0'), + Constraint('<0.12.0'), + Constraint('<=0.12.0'), + Constraint('0.12.0'), + Constraint('~>0.12.0'), + Constraint('>=0.12.0'), + Constraint('>0.12.0'), + Constraint('0.12.5'), + Constraint('0.13.0'), + ] + assert test_ordering == sorted(test_ordering) diff --git a/tests/python/terraform_version/test_tfc.py b/tests/python/terraform_version/test_tfc.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/python/terraform_version/test_tfenv.py b/tests/python/terraform_version/test_tfenv.py new file mode 100644 index 00000000..30cb5ccf --- /dev/null +++ b/tests/python/terraform_version/test_tfenv.py @@ -0,0 +1,46 @@ +from terraform.versions import Version +from terraform_version.tfenv import parse_tfenv + +def test_parse_tfenv(): + versions = [ + Version('0.13.6'), + Version('1.1.8'), + Version('1.1.9'), + Version('1.1.7'), + Version('1.1.0-alpha20210811'), + Version('1.2.0-alpha20225555') + ] + + assert parse_tfenv('0.13.6', versions) == Version('0.13.6') + assert parse_tfenv(''' + + 0.15.6 + + ''', versions) == Version('0.15.6') + + assert parse_tfenv('1.1.1-cool', versions) == Version('1.1.1-cool') + + try: + parse_tfenv('', versions) + except ValueError: + pass + else: + assert False + + try: + parse_tfenv('blahblah', versions) + except ValueError: + pass + else: + assert False + + assert parse_tfenv('latest', versions) == Version('1.1.9') + assert parse_tfenv('latest:^1.1', versions) >= Version('1.1.8') + assert parse_tfenv('latest:1.8', versions) >= Version('1.1.0-alpha20210811') + + try: + parse_tfenv('latest:^1.8', versions) + except Exception: + pass + else: + assert False diff --git a/tests/python/terraform_version/test_tfswitch.py b/tests/python/terraform_version/test_tfswitch.py new file mode 100644 index 00000000..80c66ccc --- /dev/null +++ b/tests/python/terraform_version/test_tfswitch.py @@ -0,0 +1,27 @@ +from terraform.versions import Version +from terraform_version.tfswitch import parse_tfswitch + + +def test_parse_tfswitch(): + assert parse_tfswitch('0.13.6') == Version('0.13.6') + assert parse_tfswitch(''' + + 0.15.6 + + ''') == Version('0.15.6') + + assert parse_tfswitch('1.1.1-cool') == Version('1.1.1-cool') + + try: + parse_tfswitch('') + except ValueError: + pass + else: + assert False + + try: + parse_tfswitch('blahblah') + except ValueError: + pass + else: + assert False diff --git a/tests/requirements.txt b/tests/requirements.txt index 2001f3e2..1e3dab6f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,8 @@ requests pytest +python-hcl2 + +types-requests + +mypy +flake8 diff --git a/tests/version/asdf/.tool-versions b/tests/version/asdf/.tool-versions new file mode 100644 index 00000000..f9eddc6a --- /dev/null +++ b/tests/version/asdf/.tool-versions @@ -0,0 +1,5 @@ + + nothing 1.0.0 + + terraform 0.12.11 # woo + diff --git a/tests/version/cloud/main.tf b/tests/version/cloud/main.tf new file mode 100644 index 00000000..8203bbe8 --- /dev/null +++ b/tests/version/cloud/main.tf @@ -0,0 +1,8 @@ +terraform { + cloud { + organization = "flooktech" + workspaces { + tags = ["terraformgithubactions", "version", "cloud"] + } + } +} diff --git a/tests/version/local/main.tf b/tests/version/local/main.tf new file mode 100644 index 00000000..56b62e82 --- /dev/null +++ b/tests/version/local/main.tf @@ -0,0 +1,4 @@ +variable "my_variable" {} +output "out" { + value = "${var.my_variable}" +} diff --git a/tests/version/local/terraform.tfstate b/tests/version/local/terraform.tfstate new file mode 100644 index 00000000..18fad48a --- /dev/null +++ b/tests/version/local/terraform.tfstate @@ -0,0 +1,13 @@ +{ + "version": 4, + "terraform_version": "0.15.4", + "serial": 1, + "lineage": "9020ad30-7aa7-e1ac-3f14-98e8220a1545", + "outputs": { + "out": { + "value": "hello", + "type": "string" + } + }, + "resources": [] +} diff --git a/tests/version/providers/0.11/main.tf b/tests/version/providers/0.11/main.tf index 5ab0b1b9..124ea087 100644 --- a/tests/version/providers/0.11/main.tf +++ b/tests/version/providers/0.11/main.tf @@ -6,5 +6,5 @@ provider "acme" { } terraform { - required_version = "~>0.11" -} \ No newline at end of file + required_version = "~>0.11.0" +} diff --git a/tests/version/providers/0.12/main.tf b/tests/version/providers/0.12/main.tf index c4a99566..a76fe997 100644 --- a/tests/version/providers/0.12/main.tf +++ b/tests/version/providers/0.12/main.tf @@ -6,5 +6,5 @@ provider "acme" { } terraform { - required_version = "~>0.12" -} \ No newline at end of file + required_version = "~>0.12.0" +} diff --git a/tests/version/providers/0.13/versions.tf b/tests/version/providers/0.13/versions.tf index e923b367..4c035daa 100644 --- a/tests/version/providers/0.13/versions.tf +++ b/tests/version/providers/0.13/versions.tf @@ -8,5 +8,5 @@ terraform { version = "2.2.0" } } - required_version = "~> 0.13" + required_version = "~> 0.13.0" } diff --git a/tests/version/state/main.tf b/tests/version/state/main.tf new file mode 100644 index 00000000..fc02fa43 --- /dev/null +++ b/tests/version/state/main.tf @@ -0,0 +1,13 @@ +variable "my_variable" {} +output "out" { + value = "${var.my_variable}" +} + +terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "terraform-version" + region = "eu-west-2" + dynamodb_table = "terraform-github-actions" + } +} diff --git a/tests/version/terraform-cloud/main.tf b/tests/version/terraform-cloud/main.tf new file mode 100644 index 00000000..03a4bc49 --- /dev/null +++ b/tests/version/terraform-cloud/main.tf @@ -0,0 +1,9 @@ +terraform { + backend "remote" { + organization = "flooktech" + + workspaces { + prefix = "github-actions-version-" + } + } +} diff --git a/tests/version/test_version.py b/tests/version/test_version.py index 4d657571..42ae9c3f 100644 --- a/tests/version/test_version.py +++ b/tests/version/test_version.py @@ -1,5 +1,9 @@ +import os + from convert_version import convert_version, convert_version_from_json +from terraform.cloud import get_workspaces, new_workspace, delete_workspace + def test_convert_version(): tf_version_output = 'Terraform v0.12.28' diff --git a/tests/version/tfenv/.tfswitchrc b/tests/version/tfenv/.terraform-version similarity index 100% rename from tests/version/tfenv/.tfswitchrc rename to tests/version/tfenv/.terraform-version From ab85478c2ca459cd35efa761669a3bd152ca8cfc Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 22 Jan 2022 19:02:16 +0000 Subject: [PATCH 180/231] Update action inputs for InitArgs --- image/src/github_actions/inputs.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/image/src/github_actions/inputs.py b/image/src/github_actions/inputs.py index ad78c0a9..627c63c1 100644 --- a/image/src/github_actions/inputs.py +++ b/image/src/github_actions/inputs.py @@ -51,19 +51,16 @@ class DestroyWorkspace(PlanInputs): """Input variables for the terraform-destroy-workspace action""" -class Fmt(TypedDict): +class Fmt(InitInputs): """Input variables for the terraform-fmt action""" - INPUT_PATH: str -class FmtCheck(TypedDict): +class FmtCheck(InitInputs): """Input variables for the terraform-fmt-check action""" - INPUT_PATH: str -class Version(TypedDict): +class Version(InitInputs): """Input variables for the terraform-version action""" - INPUT_PATH: str class NewWorkspace(InitInputs): @@ -82,7 +79,5 @@ class RemoteState(TypedDict): INPUT_BACKEND_CONFIG_FILE: str -class Validate(TypedDict): +class Validate(InitInputs): """Input variables for the terraform-validate action""" - INPUT_PATH: str - INPUT_WORKSPACE: str From a5fe0708beeb78a3a1c882eb7f3a6fa34d5844ed Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 12:10:30 +0000 Subject: [PATCH 181/231] Use latest base image --- image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/Dockerfile b/image/Dockerfile index eba65267..dd13ed0c 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -1,4 +1,4 @@ -FROM danielflook/terraform-github-actions-base:2022-01-22 +FROM danielflook/terraform-github-actions-base:latest COPY src/ /tmp/src/ COPY setup.py /tmp From 3cd78ec52fa026141be131c741bf20b58e711204 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 12:24:18 +0000 Subject: [PATCH 182/231] Add labels to the base image --- .github/workflows/base-image.yaml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 7c85e81a..4052a0f4 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -6,6 +6,7 @@ on: - master paths: - image/Dockerfile-base + - .github/workflows/base-image.yaml schedule: - cron: 0 1 * * 1 @@ -26,9 +27,23 @@ jobs: - name: Base image run: | - docker build --tag dflook/terraform-github-actions-base -f image/Dockerfile-base image + docker pull --quiet debian:bullseye-slim + BASE_DIGEST="$(docker image inspect --format="{{index .RepoDigests 0}}" "debian:bullseye-slim" | sed 's/.*@//')" + + docker build --tag dflook/terraform-github-actions-base -f image/Dockerfile-base \ + --label org.opencontainers.image.created="$(date '+%Y-%m-%dT%H:%M:%S%z')" \ + --label org.opencontainers.image.source="https://github.com/${{ github.repository }}" \ + --label org.opencontainers.image.revision="${{ github.sha }}" \ + --label org.opencontainers.image.base.name="docker.io/library/debian:bullseye-slim" \ + --label org.opencontainers.image.base.digest="$BASE_DIGEST" \ + --label build="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + image docker tag dflook/terraform-github-actions-base danielflook/terraform-github-actions-base:latest - docker push danielflook/terraform-github-actions-base:latest + docker push --quiet danielflook/terraform-github-actions-base:latest + + IMAGE_DIGEST=$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions:latest" | sed 's/.*@//')" + echo "::set-output name=digest::$(docker image inspect --format="{{index .RepoDigests 0}}" "$IMAGE_DIGEST")" + docker tag dflook/terraform-github-actions-base danielflook/terraform-github-actions-base:$GITHUB_RUN_ID - docker push danielflook/terraform-github-actions-base:$GITHUB_RUN_ID + docker push --quiet danielflook/terraform-github-actions-base:$GITHUB_RUN_ID From 47172dcb9bb505786e92bb855a8932eb3821f416 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 12:35:34 +0000 Subject: [PATCH 183/231] Remove bad space --- .github/workflows/base-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 4052a0f4..2ef1ec1c 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -34,7 +34,7 @@ jobs: --label org.opencontainers.image.created="$(date '+%Y-%m-%dT%H:%M:%S%z')" \ --label org.opencontainers.image.source="https://github.com/${{ github.repository }}" \ --label org.opencontainers.image.revision="${{ github.sha }}" \ - --label org.opencontainers.image.base.name="docker.io/library/debian:bullseye-slim" \ + --label org.opencontainers.image.base.name="docker.io/library/debian:bullseye-slim" \ --label org.opencontainers.image.base.digest="$BASE_DIGEST" \ --label build="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ image From 44ef0cdd63ccc865c0798983347249067c60b6e6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 12:39:52 +0000 Subject: [PATCH 184/231] Correctly log base image digest --- .github/workflows/base-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 2ef1ec1c..5afbc4c3 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -42,7 +42,7 @@ jobs: docker tag dflook/terraform-github-actions-base danielflook/terraform-github-actions-base:latest docker push --quiet danielflook/terraform-github-actions-base:latest - IMAGE_DIGEST=$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions:latest" | sed 's/.*@//')" + IMAGE_DIGEST="$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions-base:latest" | sed 's/.*@//')" echo "::set-output name=digest::$(docker image inspect --format="{{index .RepoDigests 0}}" "$IMAGE_DIGEST")" docker tag dflook/terraform-github-actions-base danielflook/terraform-github-actions-base:$GITHUB_RUN_ID From 418d2ae4053ce9045d96ed12e45d3252a269571d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 12:44:07 +0000 Subject: [PATCH 185/231] Correctly log base image digest --- .github/workflows/base-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 5afbc4c3..fc74a771 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -43,7 +43,7 @@ jobs: docker push --quiet danielflook/terraform-github-actions-base:latest IMAGE_DIGEST="$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions-base:latest" | sed 's/.*@//')" - echo "::set-output name=digest::$(docker image inspect --format="{{index .RepoDigests 0}}" "$IMAGE_DIGEST")" + echo "::set-output name=digest::$IMAGE_DIGEST" docker tag dflook/terraform-github-actions-base danielflook/terraform-github-actions-base:$GITHUB_RUN_ID docker push --quiet danielflook/terraform-github-actions-base:$GITHUB_RUN_ID From 9b8fff4d64d5859eff99453f69a7276988cc7113 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 13:03:30 +0000 Subject: [PATCH 186/231] Add the base image digest to the release image --- .github/workflows/release.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7220c53e..bb32e44e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,13 +27,16 @@ jobs: run: | RELEASE_TAG="${{ github.event.release.tag_name }}" + docker pull --quiet danielflook/terraform-github-actions-base:latest + BASE_DIGEST="$(docker image inspect --format="{{index .RepoDigests 0}}" "danielflook/terraform-github-actions-base:latest" | sed 's/.*@//')" + docker build --tag dflook/terraform-github-actions \ --label org.opencontainers.image.created="$(date '+%Y-%m-%dT%H:%M:%S%z')" \ --label org.opencontainers.image.source="https://github.com/${{ github.repository }}" \ --label org.opencontainers.image.revision="${{ github.sha }}" \ - --label org.opencontainers.image.version="$RELEASE_TAG" \ - --label vcs-ref="$RELEASE_TAG" \ - --label build="$GITHUB_RUN_ID" \ + --label org.opencontainers.image.base.name="docker.io/danielflook/terraform-github-actions-base:latest" \ + --label org.opencontainers.image.base.digest="$BASE_DIGEST" \ + --label build="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ image docker tag dflook/terraform-github-actions ghcr.io/dflook/terraform-github-actions:$RELEASE_TAG From 563057267871c2072e7ba997520e3d2f5a0486cc Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 13:40:22 +0000 Subject: [PATCH 187/231] :bookmark: v1.22.0 --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 194fa8cd..d9e22eab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,44 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.21.1` to use an exact release -- `@v1.21` to use the latest patch release for the specific minor version +- `@v1.22.0` to use an exact release +- `@v1.22` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.22.0] - 2022-01-23 + +### Added + +- Workspace management for Terraform Cloud/Enterprise has been reimplemented to avoid issues with the `terraform workspace` command when using the `remote` backend or a cloud config block: + - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) can now create the first workspace + - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace) can now delete the last remaining workspace + - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) and [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace) work with a `remote` backend that specifies a workspace by `name` + +- The terraform version to use will now be detected from additional places: + + - The terraform version set in the remote workspace when using Terraform Cloud/Enterprise as the backend + - An [asdf](https://asdf-vm.com/) `.tool-versions` file + - The terraform version that wrote an existing state file + - A `TERRAFORM_VERSION` environment variable + + The best way to specify the version is using a [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) constraint. + + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) docs for details. + +### Changed + +As a result of the above terraform version detection additions, note these changes: + +- Actions always use the terraform version set in the remote workspace when using TFC/E, if it exists. This mostly effects [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate). + +- If the terraform version is not specified anywhere then new workspaces will be created with the latest terraform version. Existing workspaces will use the terraform version that was last used for that workspace. + +- If you want to always use the latest terraform version, instead of not specifying a version you now need to set an open-ended version constraint (e.g. `>1.0.0`) + +- All actions now support the inputs and environment variables related to the backend, for discovering the terraform version from a TFC/E workspace or remote state. This add the inputs `workspace`, `backend_config`, `backend_config_file`, and the `TERRAFORM_CLOUD_TOKENS` environment variable to the [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) actions. + +- :warning: Some unused packages were removed from the container image, most notably Python 2. + ## [1.21.1] - 2021-12-12 ### Fixed @@ -331,6 +365,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.22.0]: https://github.com/dflook/terraform-github-actions/compare/v1.21.1...v1.22.0 [1.21.1]: https://github.com/dflook/terraform-github-actions/compare/v1.21.0...v1.21.1 [1.21.0]: https://github.com/dflook/terraform-github-actions/compare/v1.20.1...v1.21.0 [1.20.1]: https://github.com/dflook/terraform-github-actions/compare/v1.20.0...v1.20.1 From f6b6482cfef21735fb92bfb9492ad69eb8a5d439 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 15:29:32 +0000 Subject: [PATCH 188/231] :books: Expand initialism --- terraform-version/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform-version/README.md b/terraform-version/README.md index 73c4d274..a02332ee 100644 --- a/terraform-version/README.md +++ b/terraform-version/README.md @@ -7,7 +7,7 @@ This action determines the terraform and provider versions to use for a Terrafor The best way to specify the version is using a [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) constraint. The version to use is discovered from the first of: -1. The version set in the TFC/TFE workspace if the module uses a `remote` backend or `cloud` configuration, and the remote workspace exists. +1. The version set in the Terraform Cloud/Enterprise workspace if the module uses a `remote` backend or `cloud` configuration, and the remote workspace exists. 2. A [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) constraint in the terraform configuration. If the constraint is range, the latest matching version is used. 3. A [tfswitch](https://warrensbox.github.io/terraform-switcher/) `.tfswitchrc` file in the module path @@ -77,7 +77,7 @@ outputs yourself. * `TERRAFORM_CLOUD_TOKENS` API tokens for terraform cloud hosts, of the form `=`. Multiple tokens may be specified, one per line. - These tokens may be used for fetching required modules from the registry. + These tokens may be used for fetching required modules from the registry, and determining the terraform version set in the remote workspace. e.g for terraform cloud: ```yaml From 6282232b70d0f815fd75fc9514f4beb94c749481 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 23 Jan 2022 15:40:37 +0000 Subject: [PATCH 189/231] Don't error if debug file doesn't exist --- image/workflow_commands.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/image/workflow_commands.sh b/image/workflow_commands.sh index ce428bac..4ec5ead8 100644 --- a/image/workflow_commands.sh +++ b/image/workflow_commands.sh @@ -38,7 +38,16 @@ function debug_cmd() { function debug_file() { local FILE_PATH FILE_PATH="$1" - sed "s|^|::debug::$FILE_PATH:|" "$FILE_PATH" + + if [[ -s "$FILE_PATH" ]]; then + # File exists, and is not empty + sed "s|^|::debug::$FILE_PATH:|" "$FILE_PATH" + elif [[ -f "$FILE_PATH" ]]; then + # file exists but is empty + echo "::debug::$FILE_PATH is empty" + else + echo "::debug::$FILE_PATH does not exist" + fi } ## From 1aa2b80d1fa3c16dd405118777d4f084bda63331 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 24 Jan 2022 19:19:33 +0000 Subject: [PATCH 190/231] Fallback to environment variables for pr url if event payload not available --- image/tools/github_comment_react.py | 11 +++-- image/tools/github_pr_comment.py | 63 +++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/image/tools/github_comment_react.py b/image/tools/github_comment_react.py index 83e69180..29244bc9 100755 --- a/image/tools/github_comment_react.py +++ b/image/tools/github_comment_react.py @@ -80,10 +80,15 @@ def find_reaction_url(actions_env: GitHubActionsEnv) -> Optional[CommentReaction if event_type not in ['issue_comment', 'pull_request_review_comment']: return None - with open(actions_env['GITHUB_EVENT_PATH']) as f: - event = json.load(f) + try: + with open(actions_env['GITHUB_EVENT_PATH']) as f: + event = json.load(f) - return event['comment']['reactions']['url'] + return event['comment']['reactions']['url'] + except Exception as e: + debug(str(e)) + + return None def react(comment_reaction_url: CommentReactionUrl, reaction_type: str) -> None: diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py index 2af195d3..52906bbb 100755 --- a/image/tools/github_pr_comment.py +++ b/image/tools/github_pr_comment.py @@ -29,6 +29,8 @@ class GitHubActionsEnv(TypedDict): GITHUB_EVENT_NAME: str GITHUB_REPOSITORY: str GITHUB_SHA: str + GITHUB_REF_TYPE: str + GITHUB_REF: str job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') @@ -67,6 +69,8 @@ class ActionInputs(TypedDict): INPUT_TARGET: str INPUT_REPLACE: str +class WorkflowException(Exception): + """An exception that should result in an error in the workflow log""" def plan_identifier(action_inputs: ActionInputs) -> str: def mask_backend_config() -> Optional[str]: @@ -164,7 +168,7 @@ def github_api_request(method: str, *args, **kwargs) -> requests.Response: limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) sys.stdout.write(message) sys.stdout.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') - exit(1) + sys.exit(1) if message != 'Resource not accessible by integration': sys.stdout.write(message) @@ -206,20 +210,42 @@ def find_pr(actions_env: GitHubActionsEnv) -> PrUrl: """ - with open(actions_env['GITHUB_EVENT_PATH']) as f: - event = json.load(f) + event: Optional[Dict[str, Any]] + + if os.path.isfile(actions_env['GITHUB_EVENT_PATH']): + with open(actions_env['GITHUB_EVENT_PATH']) as f: + event = json.load(f) + else: + debug('Event payload is not available') + event = None event_type = actions_env['GITHUB_EVENT_NAME'] - if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: - return cast(PrUrl, event['pull_request']['url']) + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review', 'issue_comment']: + + if event is not None: + # Pull pr url from event payload + + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: + return cast(PrUrl, event['pull_request']['url']) - elif event_type == 'issue_comment': + if event_type == 'issue_comment': + + if 'pull_request' in event['issue']: + return cast(PrUrl, event['issue']['pull_request']['url']) + else: + raise WorkflowException('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') - if 'pull_request' in event['issue']: - return cast(PrUrl, event['issue']['pull_request']['url']) else: - raise Exception('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') + # Event payload is not available + + if actions_env.get('GITHUB_REF_TYPE') == 'branch': + if match := re.match(r'refs/pull/(\d+)/', actions_env.get('GITHUB_REF', '')): + return cast(PrUrl, f'{actions_env["GITHUB_API_URL"]}/repos/{actions_env["GITHUB_REPOSITORY"]}/pulls/{match.group(1)}') + + raise WorkflowException(f'Event payload is not available at the GITHUB_EVENT_PATH {actions_env["GITHUB_EVENT_PATH"]!r}. ' + + f'This is required when run by {event_type} events. The environment has not been setup properly by the actions runner. ' + + 'This can happen when the runner is running in a container') elif event_type == 'push': repo = actions_env['GITHUB_REPOSITORY'] @@ -233,10 +259,10 @@ def prs() -> Iterable[Dict[str, Any]]: if pr['merge_commit_sha'] == commit: return cast(PrUrl, pr['url']) - raise Exception(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') + raise WorkflowException(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') else: - raise Exception(f"The {event_type} event doesn\'t relate to a Pull Request.") + raise WorkflowException(f"The {event_type} event doesn\'t relate to a Pull Request.") def current_user(actions_env: GitHubActionsEnv) -> str: @@ -416,10 +442,11 @@ def save_step_cache(**kwargs) -> None: def main() -> None: if len(sys.argv) < 2: - print(f'''Usage: + sys.stdout.write(f'''Usage: STATUS="" {sys.argv[0]} plan plan.txt''') + {sys.argv[0]} get >plan.txt +''') debug(repr(sys.argv)) @@ -436,7 +463,11 @@ def main() -> None: pr_url = step_cache['pr_url'] debug(f'pr_url from step cache: {pr_url}') else: - pr_url = find_pr(env) + try: + pr_url = find_pr(env) + except WorkflowException as e: + sys.stdout.write('\n' + str(e) + '\n') + sys.exit(1) debug(f'discovered pr_url: {pr_url}') if step_cache.get('pr_url') == pr_url and step_cache.get('issue_url') is not None: @@ -473,14 +504,14 @@ def main() -> None: elif sys.argv[1] == 'status': if plan is None: - exit(1) + sys.exit(1) else: body = format_body(action_inputs, plan, status, collapse_threshold) comment_url = update_comment(issue_url, comment_url, body, only_if_exists) elif sys.argv[1] == 'get': if plan is None: - exit(1) + sys.exit(1) with open(sys.argv[2], 'w') as f: f.write(plan) From 75b306c625243a8a6d7d0ba83dca43602c259d16 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 24 Jan 2022 21:04:48 +0000 Subject: [PATCH 191/231] :bookmark: 1.22.1 --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9e22eab..c4ca9dda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,14 +8,18 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.22.0` to use an exact release +- `@v1.22.1` to use an exact release - `@v1.22` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.22.1] - 2022-01-24 + +### Fixed +- Better support for some self-hosted runners that run in containers and don't correctly pass the event payload. + ## [1.22.0] - 2022-01-23 ### Added - - Workspace management for Terraform Cloud/Enterprise has been reimplemented to avoid issues with the `terraform workspace` command when using the `remote` backend or a cloud config block: - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) can now create the first workspace - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace) can now delete the last remaining workspace @@ -33,7 +37,6 @@ When using an action you can specify the version as: See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) docs for details. ### Changed - As a result of the above terraform version detection additions, note these changes: - Actions always use the terraform version set in the remote workspace when using TFC/E, if it exists. This mostly effects [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate). @@ -365,6 +368,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.22.1]: https://github.com/dflook/terraform-github-actions/compare/v1.22.0...v1.22.1 [1.22.0]: https://github.com/dflook/terraform-github-actions/compare/v1.21.1...v1.22.0 [1.21.1]: https://github.com/dflook/terraform-github-actions/compare/v1.21.0...v1.21.1 [1.21.0]: https://github.com/dflook/terraform-github-actions/compare/v1.20.1...v1.21.0 From 2766d46122711a1a6ed670572445a060b02faeef Mon Sep 17 00:00:00 2001 From: Daniel Grenner <40354411+dgrenner@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:07:12 +0100 Subject: [PATCH 192/231] Remove circular deprecation warning The other actions don't have this, so seems to be copied from the var deprecationMessage. --- terraform-destroy/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform-destroy/action.yaml b/terraform-destroy/action.yaml index b9767f37..cdcdddb6 100644 --- a/terraform-destroy/action.yaml +++ b/terraform-destroy/action.yaml @@ -22,7 +22,6 @@ inputs: variables: description: Variable definitions required: false - deprecationMessage: Use the variables input instead. var: description: Comma separated list of vars to set, e.g. 'foo=bar' required: false From e7b3ff510759c468276f9e80ef11982734ab147a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 28 Feb 2022 20:44:30 +0000 Subject: [PATCH 193/231] Fix compacting plans with only changes to outputs in >=0.15.4 --- image/tools/compact_plan.py | 3 +- tests/test_compact_plan.py | 283 ++++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+), 1 deletion(-) diff --git a/image/tools/compact_plan.py b/image/tools/compact_plan.py index feada2e7..94b88e17 100755 --- a/image/tools/compact_plan.py +++ b/image/tools/compact_plan.py @@ -13,7 +13,8 @@ def compact_plan(input): line.startswith('Terraform used the selected providers') or line.startswith('An execution plan has been generated and is shown below') or line.startswith('No changes') or - line.startswith('Error') + line.startswith('Error') or + line.startswith('Changes to Outputs:') ): plan = True diff --git a/tests/test_compact_plan.py b/tests/test_compact_plan.py index 599df6bd..6c3154d5 100644 --- a/tests/test_compact_plan.py +++ b/tests/test_compact_plan.py @@ -226,6 +226,289 @@ def test_plan_15(): output = '\n'.join(compact_plan(input.splitlines())) assert output == expected_output +def test_plan_refresh_on_changes_11(): + input = """ +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +random_string.my_string: Refreshing state... (ID: Zl$lcns(v>) + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_no_changes_14(): + input = """ +random_string.my_string: Refreshing state... [id=&)+#Z$b@=b] + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_no_changes_15(): + input = """ +random_string.my_string: Refreshing state... [id=&)+#Z$b@=b] + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your configuration and the remote system(s). As a result, there are no actions to take. +""" + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your configuration and the remote system(s). As a result, there are no actions to take.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_no_changes_1(): + input = """ +random_string.my_string: Refreshing state... [id=&)+#Z$b@=b] + +No changes. Your infrastructure matches the configuration. + +Terraform has compared your real infrastructure against your configuration and found no differences, so no changes are needed. +""" + + expected_output = """No changes. Your infrastructure matches the configuration. + +Terraform has compared your real infrastructure against your configuration and found no differences, so no changes are needed.""" + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + + +def test_plan_no_resource_output_only_11(): + input = """ +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. + """ + + expected_output = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_no_resource_output_only_14(): + input = """ +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + +Terraform will perform the following actions: + +Plan: 0 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + t = "hello" + """ + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + +Terraform will perform the following actions: + +Plan: 0 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + t = "hello" + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_no_resource_output_only_15_4(): + input = """ +Changes to Outputs: + + t = "hello" + +You can apply this plan to save these new output values to the Terraform state, without changing any real infrastructure. + """ + + expected_output = """Changes to Outputs: + + t = "hello" + +You can apply this plan to save these new output values to the Terraform state, without changing any real infrastructure. + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_changes_11(): + input = """ +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +random_string.my_string: Refreshing state... (ID: <2jMa%O-E$) + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + +-/+ random_string.my_string (new resource required) + id: "<2jMa%O-E$" => (forces new resource) + length: "10" => "5" (forces new resource) + lower: "true" => "true" + min_lower: "0" => "0" + min_numeric: "0" => "0" + min_special: "0" => "0" + min_upper: "0" => "0" + number: "true" => "true" + result: "<2jMa%O-E$" => + special: "true" => "true" + upper: "true" => "true" +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + +-/+ random_string.my_string (new resource required) + id: "<2jMa%O-E$" => (forces new resource) + length: "10" => "5" (forces new resource) + lower: "true" => "true" + min_lower: "0" => "0" + min_numeric: "0" => "0" + min_special: "0" => "0" + min_upper: "0" => "0" + number: "true" => "true" + result: "<2jMa%O-E$" => + special: "true" => "true" + upper: "true" => "true" +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_changes_14(): + input = """ +random_string.my_string: Refreshing state... [id=Iyh3jLKc] + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # random_string.my_string must be replaced +-/+ resource "random_string" "my_string" { + ~ id = "Iyh3jLKc" -> (known after apply) + ~ length = 8 -> 4 # forces replacement + ~ result = "Iyh3jLKc" -> (known after apply) + # (8 unchanged attributes hidden) + } + +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + expected_output = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # random_string.my_string must be replaced +-/+ resource "random_string" "my_string" { + ~ id = "Iyh3jLKc" -> (known after apply) + ~ length = 8 -> 4 # forces replacement + ~ result = "Iyh3jLKc" -> (known after apply) + # (8 unchanged attributes hidden) + } + +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output + +def test_plan_refresh_changes_15(): + input = """ +random_string.my_string: Refreshing state... [id=Iyh3jLKc] + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # random_string.my_string must be replaced +-/+ resource "random_string" "my_string" { + ~ id = "Iyh3jLKc" -> (known after apply) + ~ length = 8 -> 4 # forces replacement + ~ result = "Iyh3jLKc" -> (known after apply) + # (8 unchanged attributes hidden) + } + +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + expected_output = """Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # random_string.my_string must be replaced +-/+ resource "random_string" "my_string" { + ~ id = "Iyh3jLKc" -> (known after apply) + ~ length = 8 -> 4 # forces replacement + ~ result = "Iyh3jLKc" -> (known after apply) + # (8 unchanged attributes hidden) + } + +Plan: 1 to add, 0 to change, 1 to destroy. + """ + + output = '\n'.join(compact_plan(input.splitlines())) + assert output == expected_output def test_error_11(): input = """ From eaf38cf4b378f2b015a1ad7e17570de0b788e69b Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 28 Feb 2022 21:00:57 +0000 Subject: [PATCH 194/231] :bookmark: v1.22.2 --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4ca9dda..deb1cda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.22.1` to use an exact release +- `@v1.22.2` to use an exact release - `@v1.22` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.22.2] - 2022-02-28 + +### Fixed +- The PR plan comment was incorrectly including resource refresh lines when there were changes to outputs but not resources, while using Terraform >=0.15.4. As well as being noisy, this could lead to failures to apply due to incorrectly detecting changes in the plan. +- Removed incorrect deprecation warning in [dflook/terraform-destroy](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy). Thanks [dgrenner](https://github.com/dgrenner)! + ## [1.22.1] - 2022-01-24 ### Fixed @@ -368,6 +374,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.22.2]: https://github.com/dflook/terraform-github-actions/compare/v1.22.1...v1.22.2 [1.22.1]: https://github.com/dflook/terraform-github-actions/compare/v1.22.0...v1.22.1 [1.22.0]: https://github.com/dflook/terraform-github-actions/compare/v1.21.1...v1.22.0 [1.21.1]: https://github.com/dflook/terraform-github-actions/compare/v1.21.0...v1.21.1 From 1842b14da5fa3f42aefbe2c13ce82418c78b01b0 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 2 Mar 2022 10:57:17 +0000 Subject: [PATCH 195/231] :pencil: Fix typo --- terraform-output/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform-output/README.md b/terraform-output/README.md index 6dc2770c..998ed586 100644 --- a/terraform-output/README.md +++ b/terraform-output/README.md @@ -186,7 +186,7 @@ jobs: path: my-terraform-config - name: Print the hostname - run: echo "The terraform version was ${{ steps.tf-outputs.outputs.hostname }}" + run: echo "The hostname is ${{ steps.tf-outputs.outputs.hostname }}" ``` ### Complex output From fc6c9b65f759dcecd72b50a91ba781eeaf412412 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 16 Feb 2022 22:03:21 +0000 Subject: [PATCH 196/231] Refactor github_pr_comment --- image/Dockerfile | 1 - image/actions.sh | 7 +- image/entrypoints/apply.sh | 5 +- image/entrypoints/plan.sh | 10 +- image/setup.py | 3 +- image/src/github_actions/api.py | 73 +++ image/src/github_actions/cache.py | 34 ++ image/src/github_actions/env.py | 10 +- image/src/github_actions/find_pr.py | 76 +++ image/src/github_actions/inputs.py | 13 +- image/src/github_pr_comment/__init__.py | 0 image/src/github_pr_comment/__main__.py | 247 ++++++++ image/src/github_pr_comment/comment.py | 253 +++++++++ image/tools/github_pr_comment.py | 522 ----------------- tests/github_pr_comment/test_comment.py | 140 +++++ .../github_pr_comment/test_legacy_comment.py | 457 +++++++++++++++ tests/github_pr_comment/test_summary.py | 160 ++++++ tests/test_pr_comment.py | 535 ------------------ 18 files changed, 1464 insertions(+), 1082 deletions(-) create mode 100644 image/src/github_actions/api.py create mode 100644 image/src/github_actions/cache.py create mode 100644 image/src/github_actions/find_pr.py create mode 100644 image/src/github_pr_comment/__init__.py create mode 100644 image/src/github_pr_comment/__main__.py create mode 100644 image/src/github_pr_comment/comment.py delete mode 100755 image/tools/github_pr_comment.py create mode 100644 tests/github_pr_comment/test_comment.py create mode 100644 tests/github_pr_comment/test_legacy_comment.py create mode 100644 tests/github_pr_comment/test_summary.py delete mode 100644 tests/test_pr_comment.py diff --git a/image/Dockerfile b/image/Dockerfile index dd13ed0c..d2a9996a 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -12,7 +12,6 @@ COPY actions.sh /usr/local/actions.sh COPY workflow_commands.sh /usr/local/workflow_commands.sh COPY tools/convert_validate_report.py /usr/local/bin/convert_validate_report -COPY tools/github_pr_comment.py /usr/local/bin/github_pr_comment COPY tools/convert_output.py /usr/local/bin/convert_output COPY tools/plan_cmp.py /usr/local/bin/plan_cmp COPY tools/convert_version.py /usr/local/bin/convert_version diff --git a/image/actions.sh b/image/actions.sh index c0dc32af..ea850d92 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -96,6 +96,7 @@ function setup() { if [[ "$TERRAFORM_BACKEND_TYPE" != "" ]]; then echo "Detected $TERRAFORM_BACKEND_TYPE backend" fi + export TERRAFORM_BACKEND_TYPE end_group @@ -326,10 +327,8 @@ function output() { function update_status() { local status="$1" - if ! STATUS="$status" github_pr_comment status 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" - else - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + if ! STATUS="$status" github_pr_comment status; then + echo fi } diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 88bf6fb8..e5f08704 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -110,16 +110,13 @@ else exit 1 fi - if ! github_pr_comment get "$STEP_TMP_DIR/approved-plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + if ! github_pr_comment get "$STEP_TMP_DIR/approved-plan.txt"; then echo "Plan not found on PR" echo "Generate the plan first using the dflook/terraform-plan action. Alternatively set the auto_approve input to 'true'" echo "If dflook/terraform-plan was used with add_github_comment set to changes-only, this may mean the plan has since changed to include changes" set_output failure-reason plan-changed exit 1 - else - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi if plan_cmp "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt"; then diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index a9a2b718..c35b6455 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -38,11 +38,8 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c fi if [[ $PLAN_EXIT -eq 1 ]]; then - if ! STATUS=":x: Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + if ! STATUS=":x: Failed to generate plan in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/terraform_plan.stderr"; then exit 1 - else - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi else @@ -53,11 +50,8 @@ if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_c TF_CHANGES=true fi - if ! TF_CHANGES=$TF_CHANGES STATUS=":memo: Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt" 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + if ! TF_CHANGES=$TF_CHANGES STATUS=":memo: Plan generated in $(job_markdown_ref)" github_pr_comment plan <"$STEP_TMP_DIR/plan.txt"; then exit 1 - else - debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi fi diff --git a/image/setup.py b/image/setup.py index d8ae8272..d810109c 100644 --- a/image/setup.py +++ b/image/setup.py @@ -10,7 +10,8 @@ 'console_scripts': [ 'terraform-backend=terraform_backend.__main__:main', 'terraform-version=terraform_version.__main__:main', - 'terraform-cloud-workspace=terraform_cloud_workspace.__main__:main' + 'terraform-cloud-workspace=terraform_cloud_workspace.__main__:main', + 'github_pr_comment=github_pr_comment.__main__:main' ] }, install_requires=[ diff --git a/image/src/github_actions/api.py b/image/src/github_actions/api.py new file mode 100644 index 00000000..3d99a735 --- /dev/null +++ b/image/src/github_actions/api.py @@ -0,0 +1,73 @@ +import datetime +import sys +from typing import NewType, Iterable, Any + +import requests +from requests import Response + +from github_actions.debug import debug + +GitHubUrl = NewType('GitHubUrl', str) +PrUrl = NewType('PrUrl', GitHubUrl) +IssueUrl = NewType('IssueUrl', GitHubUrl) +CommentUrl = NewType('CommentUrl', GitHubUrl) +CommentReactionUrl = NewType('CommentReactionUrl', GitHubUrl) + + +class GithubApi: + def __init__(self, host: str, token: str): + self._host = host + self._token = token + + self._session = requests.Session() + self._session.headers['authorization'] = f'token {token}' + self._session.headers['user-agent'] = 'terraform-github-actions' + self._session.headers['accept'] = 'application/vnd.github.v3+json' + + def api_request(self, method: str, *args, **kwargs) -> requests.Response: + response = self._session.request(method, *args, **kwargs) + + if 400 <= response.status_code < 500: + debug(str(response.headers)) + + try: + message = response.json()['message'] + + if response.headers['X-RateLimit-Remaining'] == '0': + limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) + sys.stdout.write(message) + sys.stdout.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') + sys.exit(1) + + if message != 'Resource not accessible by integration': + sys.stdout.write(message) + sys.stdout.write('\n') + debug(response.content.decode()) + + except Exception: + sys.stdout.write(response.content.decode()) + sys.stdout.write('\n') + raise + + return response + + def get(self, path: str, **kwargs: Any) -> Response: + return self.api_request('GET', path, **kwargs) + + def post(self, path: str, **kwargs: Any) -> Response: + return self.api_request('POST', path, **kwargs) + + def patch(self, path: str, **kwargs: Any) -> Response: + return self.api_request('PATCH', path, **kwargs) + + def paged_get(self, url: GitHubUrl, *args, **kwargs) -> Iterable[dict[str, Any]]: + while True: + response = self.api_request('GET', url, *args, **kwargs) + response.raise_for_status() + + yield from response.json() + + if 'next' in response.links: + url = response.links['next']['url'] + else: + return diff --git a/image/src/github_actions/cache.py b/image/src/github_actions/cache.py new file mode 100644 index 00000000..c27b6290 --- /dev/null +++ b/image/src/github_actions/cache.py @@ -0,0 +1,34 @@ +import os +from pathlib import Path + +from github_actions.debug import debug + + +class ActionsCache: + + def __init__(self, cache_dir: Path, label: str=None): + self._cache_dir = cache_dir + self._label = label or self._cache_dir + + def __setitem__(self, key, value): + if value is None: + debug(f'Cache value for {key} should not be set to {value}') + return + + path = os.path.join(self._cache_dir, key) + + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(os.path.join(self._cache_dir, key), 'w') as f: + f.write(value) + debug(f'Wrote {key} to {self._label}') + + def __getitem__(self, key): + if os.path.isfile(os.path.join(self._cache_dir, key)): + with open(os.path.join(self._cache_dir, key)) as f: + debug(f'Read {key} from {self._label}') + return f.read() + + raise IndexError(key) + + def __contains__(self, key): + return os.path.isfile(os.path.join(self._cache_dir, key)) diff --git a/image/src/github_actions/env.py b/image/src/github_actions/env.py index a4d6cb13..f1ec3bc9 100644 --- a/image/src/github_actions/env.py +++ b/image/src/github_actions/env.py @@ -15,5 +15,13 @@ class ActionsEnv(TypedDict): class GithubEnv(TypedDict): - """Environment variables set by github actions.""" + """Environment variables that are set by the actions runner.""" + GITHUB_API_URL: str + GITHUB_TOKEN: str + GITHUB_EVENT_PATH: str + GITHUB_EVENT_NAME: str + GITHUB_REPOSITORY: str + GITHUB_SHA: str + GITHUB_REF_TYPE: str + GITHUB_REF: str GITHUB_WORKSPACE: str diff --git a/image/src/github_actions/find_pr.py b/image/src/github_actions/find_pr.py new file mode 100644 index 00000000..0fefb4bc --- /dev/null +++ b/image/src/github_actions/find_pr.py @@ -0,0 +1,76 @@ +import json +import os +import re +from typing import Optional, Any, cast, Iterable + +from github_actions.api import PrUrl, GithubApi +from github_actions.debug import debug +from github_actions.env import GithubEnv + + +class WorkflowException(Exception): + """An exception that should result in an error in the workflow log""" + + +def find_pr(github: GithubApi, actions_env: GithubEnv) -> PrUrl: + """ + Find the pull request this event is related to + + >>> find_pr() + 'https://api.github.com/repos/dflook/terraform-github-actions/pulls/8' + + """ + + event: Optional[dict[str, Any]] + + if os.path.isfile(actions_env['GITHUB_EVENT_PATH']): + with open(actions_env['GITHUB_EVENT_PATH']) as f: + event = json.load(f) + else: + debug('Event payload is not available') + event = None + + event_type = actions_env['GITHUB_EVENT_NAME'] + + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review', 'issue_comment']: + + if event is not None: + # Pull pr url from event payload + + if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: + return cast(PrUrl, event['pull_request']['url']) + + if event_type == 'issue_comment': + + if 'pull_request' in event['issue']: + return cast(PrUrl, event['issue']['pull_request']['url']) + else: + raise WorkflowException('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') + + else: + # Event payload is not available + + if actions_env.get('GITHUB_REF_TYPE') == 'branch': + if match := re.match(r'refs/pull/(\d+)/', actions_env.get('GITHUB_REF', '')): + return cast(PrUrl, f'{actions_env["GITHUB_API_URL"]}/repos/{actions_env["GITHUB_REPOSITORY"]}/pulls/{match.group(1)}') + + raise WorkflowException(f'Event payload is not available at the GITHUB_EVENT_PATH {actions_env["GITHUB_EVENT_PATH"]!r}. ' + + f'This is required when run by {event_type} events. The environment has not been setup properly by the actions runner. ' + + 'This can happen when the runner is running in a container') + + elif event_type == 'push': + repo = actions_env['GITHUB_REPOSITORY'] + commit = actions_env['GITHUB_SHA'] + + def prs() -> Iterable[dict[str, Any]]: + url = cast(PrUrl, f'{actions_env["GITHUB_API_URL"]}/repos/{repo}/pulls') + yield from github.paged_get(url, params={'state': 'all'}) + + for pr in prs(): + if pr['merge_commit_sha'] == commit: + return cast(PrUrl, pr['url']) + + raise WorkflowException(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') + + else: + raise WorkflowException(f"The {event_type} event doesn\'t relate to a Pull Request.") diff --git a/image/src/github_actions/inputs.py b/image/src/github_actions/inputs.py index 627c63c1..231d4fbc 100644 --- a/image/src/github_actions/inputs.py +++ b/image/src/github_actions/inputs.py @@ -23,19 +23,20 @@ class PlanInputs(InitInputs): INPUT_PARALLELISM: str -class Plan(PlanInputs): - """Input variables for the plan action""" +class PlanPrInputs(PlanInputs): + """Common input variables for actions that use a PR comment""" INPUT_LABEL: str INPUT_TARGET: str INPUT_REPLACE: str + + +class Plan(PlanPrInputs): + """Input variables for the plan action""" INPUT_ADD_GITHUB_COMMENT: str -class Apply(InitInputs): +class Apply(PlanPrInputs): """Input variables for the terraform-apply action""" - INPUT_LABEL: str - INPUT_TARGET: str - INPUT_REPLACE: str INPUT_AUTO_APPROVE: str diff --git a/image/src/github_pr_comment/__init__.py b/image/src/github_pr_comment/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py new file mode 100644 index 00000000..74719391 --- /dev/null +++ b/image/src/github_pr_comment/__main__.py @@ -0,0 +1,247 @@ +import hashlib +import json +import os +import sys +from typing import (NewType, Optional, cast) + +from github_actions.api import GithubApi, IssueUrl, PrUrl +from github_actions.cache import ActionsCache +from github_actions.debug import debug +from github_actions.env import GithubEnv +from github_actions.find_pr import find_pr, WorkflowException +from github_actions.inputs import PlanPrInputs +from github_pr_comment.comment import find_comment, TerraformComment, update_comment + +Plan = NewType('Plan', str) +Status = NewType('Status', str) + +job_cache = ActionsCache(os.environ.get('JOB_TMP_DIR', '.'), 'job_cache') +step_cache = ActionsCache(os.environ.get('STEP_TMP_DIR', '.'), 'step_cache') + +env = cast(GithubEnv, os.environ) + +github = GithubApi(env.get('GITHUB_API_URL', 'https://api.github.com'), env.get('GITHUB_TOKEN')) + + +def _mask_backend_config(action_inputs: PlanPrInputs) -> Optional[str]: + bad_words = [ + 'token', + 'password', + 'sas_token', + 'access_key', + 'secret_key', + 'client_secret', + 'access_token', + 'http_auth', + 'secret_id', + 'encryption_key', + 'key_material', + 'security_token', + 'conn_str', + 'sse_customer_key', + 'application_credential_secret' + ] + + clean = [] + + for field in action_inputs.get('INPUT_BACKEND_CONFIG', '').split(','): + if not field: + continue + + if not any(bad_word in field for bad_word in bad_words): + clean.append(field) + + return ','.join(clean) + + +def format_classic_description(action_inputs: PlanPrInputs) -> str: + if action_inputs['INPUT_LABEL']: + return f'Terraform plan for __{action_inputs["INPUT_LABEL"]}__' + + label = f'Terraform plan in __{action_inputs["INPUT_PATH"]}__' + + if action_inputs["INPUT_WORKSPACE"] != 'default': + label += f' in the __{action_inputs["INPUT_WORKSPACE"]}__ workspace' + + if action_inputs["INPUT_TARGET"]: + label += '\nTargeting resources: ' + label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_TARGET'].splitlines()) + + if action_inputs["INPUT_REPLACE"]: + label += '\nReplacing resources: ' + label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_REPLACE'].splitlines()) + + if backend_config := _mask_backend_config(action_inputs): + label += f'\nWith backend config: `{backend_config}`' + + if action_inputs["INPUT_BACKEND_CONFIG_FILE"]: + label += f'\nWith backend config files: `{action_inputs["INPUT_BACKEND_CONFIG_FILE"]}`' + + if action_inputs["INPUT_VAR"]: + label += f'\nWith vars: `{action_inputs["INPUT_VAR"]}`' + + if action_inputs["INPUT_VAR_FILE"]: + label += f'\nWith var files: `{action_inputs["INPUT_VAR_FILE"]}`' + + if action_inputs["INPUT_VARIABLES"]: + stripped_vars = action_inputs["INPUT_VARIABLES"].strip() + if '\n' in stripped_vars: + label += f'''
With variables + +```hcl +{stripped_vars} +``` +
+''' + else: + label += f'\nWith variables: `{stripped_vars}`' + + return label + + +def create_summary(plan: Plan) -> Optional[str]: + summary = None + + for line in plan.splitlines(): + if line.startswith('No changes') or line.startswith('Error'): + return line + + if line.startswith('Plan:'): + summary = line + + if line.startswith('Changes to Outputs'): + if summary: + return summary + ' Changes to Outputs.' + else: + return 'Changes to Outputs' + + return summary + + +def current_user(actions_env: GithubEnv) -> str: + token_hash = hashlib.sha256(actions_env['GITHUB_TOKEN'].encode()).hexdigest() + cache_key = f'token-cache/{token_hash}' + + if cache_key in job_cache: + username = job_cache[cache_key] + else: + response = github.get(f'{actions_env["GITHUB_API_URL"]}/user') + if response.status_code != 403: + user = response.json() + debug(json.dumps(user)) + + username = user['login'] + else: + # Assume this is the github actions app token + username = 'github-actions[bot]' + + job_cache[cache_key] = username + + return username + + +def get_issue_url(pr_url: str) -> IssueUrl: + pr_hash = hashlib.sha256(pr_url.encode()).hexdigest() + cache_key = f'issue-href-cache/{pr_hash}' + + if cache_key in job_cache: + issue_url = job_cache[cache_key] + else: + response = github.get(pr_url) + response.raise_for_status() + issue_url = response.json()['_links']['issue']['href'] + '/comments' + + job_cache[cache_key] = issue_url + + return cast(IssueUrl, issue_url) + + +def get_pr() -> PrUrl: + if 'pr_url' in step_cache: + pr_url = step_cache['pr_url'] + else: + try: + pr_url = find_pr(github, env) + step_cache['pr_url'] = pr_url + except WorkflowException as e: + sys.stderr.write('\n' + str(e) + '\n') + sys.exit(1) + + return cast(PrUrl, pr_url) + + +def get_comment(action_inputs: PlanPrInputs) -> TerraformComment: + pr_url = get_pr() + issue_url = get_issue_url(pr_url) + username = current_user(env) + + legacy_description = format_classic_description(action_inputs) + + headers = { + 'workspace': os.environ.get('INPUT_WORKSPACE', 'default'), + 'backend': hashlib.sha256(legacy_description.encode()).hexdigest() + } + + if backend_type := os.environ.get('TERRAFORM_BACKEND_TYPE'): + headers['backend_type'] = backend_type + + if label := os.environ.get('INPUT_LABEL'): + headers['label'] = hashlib.sha256(label.encode()).hexdigest() + + return find_comment(github, issue_url, username, headers, legacy_description) + + +def main() -> int: + if len(sys.argv) < 2: + sys.stderr.write(f'''Usage: + STATUS="" {sys.argv[0]} plan Optional[CommentUrl]: + return self._comment_url + + @comment_url.setter + def comment_url(self, comment_url: CommentUrl) -> None: + if self._comment_url is not None: + raise Exception('Can only set url for comments that don\'t exist yet') + self._comment_url = comment_url + + @property + def issue_url(self) -> IssueUrl: + return self._issue_url + + @property + def headers(self) -> dict[str, str]: + return self._headers + + @property + def description(self) -> str: + return self._description + + @property + def summary(self) -> str: + return self._summary + + @property + def body(self) -> str: + return self._body + + @property + def status(self) -> str: + return self._status + + +def _format_comment_header(**kwargs) -> str: + return f'' + +def _parse_comment_header(comment_header: Optional[str]) -> dict[str, str]: + if comment_header is None: + return {} + + if header := re.match(r'^', comment_header): + try: + return json.loads(header['args']) + except JSONDecodeError: + return {} + + return {} + + +def _from_api_payload(comment: dict[str, Any]) -> Optional[TerraformComment]: + match = re.match(rf''' + (?P\n)? + (?P.*) + \s* + (?:(?P.*?)\s*)? + ```(?:hcl)? + (?P.*) + ```\s* + + (?P.*) + ''', + comment['body'], + re.VERBOSE | re.DOTALL + ) + + if not match: + return None + + return TerraformComment( + issue_url=comment['issue_url'], + comment_url=comment['url'], + headers=_parse_comment_header(match.group('headers')), + description=match.group('description').strip(), + summary=match.group('summary').strip(), + body=match.group('body').strip(), + status=match.group('status').strip() + ) + + +def _to_api_payload(comment: TerraformComment) -> str: + details_open = False + hcl_highlighting = False + + if comment.body.startswith('Error'): + details_open = True + elif 'Plan:' in comment.body: + hcl_highlighting = True + num_lines = len(comment.body.splitlines()) + if num_lines < collapse_threshold: + details_open = True + + if comment.summary is None: + details_open = True + + header = _format_comment_header(**comment.headers) + + body = f'''{header} +{comment.description} + +{f'{comment.summary}' if comment.summary is not None else ''} + +```{'hcl' if hcl_highlighting else ''} +{comment.body} +``` + +''' + + if comment.status: + body += '\n' + comment.status + + return body + + +def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: dict[str, str], legacy_description: str) -> TerraformComment: + """ + Find a github comment that matches the given headers + + If no comment is found with the specified headers, tries to find a comment that matches the specified description instead. + This is in case the comment was made with an earlier version, where comments were matched by description only. + + If not existing comment is found a new TerraformComment object is returned which represents a PR comment yet to be created. + + :param github: The github api object to make requests with + :param issue_url: The issue to find the comment in + :param username: The user who made the comment + :param headers: The headers that must be present on the comment + :param legacy_description: The description that must be present on the comment, if not headers are found. + """ + + backup_comment = None + + for comment_payload in github.paged_get(issue_url): + if comment_payload['user']['login'] != username: + continue + + debug(json.dumps(comment_payload)) + + if comment := _from_api_payload(comment_payload): + + if comment.headers == headers: + debug('Found comment that matches headers') + return comment + + debug(f"Didn't match comment with {comment.headers=}") + + if comment.description == legacy_description: + backup_comment = comment + + debug(f"Didn't match comment with {comment.description=}") + + if backup_comment is not None: + debug('Found comment matching legacy description') + return backup_comment + + debug('No matching comment exists') + return TerraformComment( + issue_url=issue_url, + comment_url=None, + headers=headers, + description='', + summary='', + body='', + status='' + ) + + +def update_comment( + github: GithubApi, + comment: TerraformComment, + *, + headers: dict[str, str] = None, + description: str = None, + summary: str = None, + body: str = None, + status: str = None +) -> TerraformComment: + + new_comment = TerraformComment( + issue_url=comment.issue_url, + comment_url=comment.comment_url, + headers=headers if headers is not None else comment.headers, + description=description if description is not None else comment.description, + summary=summary if summary is not None else comment.summary, + body=body if body is not None else comment.body, + status=status if status is not None else comment.status + ) + + if comment.comment_url is not None: + response = github.patch(comment.comment_url, json={'body': _to_api_payload(new_comment)}) + response.raise_for_status() + else: + response = github.post(comment.issue_url, json={'body': _to_api_payload(new_comment)}) + response.raise_for_status() + new_comment.url = response.json()['url'] + + return new_comment diff --git a/image/tools/github_pr_comment.py b/image/tools/github_pr_comment.py deleted file mode 100755 index 52906bbb..00000000 --- a/image/tools/github_pr_comment.py +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/python3 - -import datetime -import hashlib -import json -import os -import re -import sys -from typing import (Any, Dict, Iterable, NewType, Optional, Tuple, TypedDict, - cast) - -import requests - -GitHubUrl = NewType('GitHubUrl', str) -PrUrl = NewType('PrUrl', GitHubUrl) -IssueUrl = NewType('IssueUrl', GitHubUrl) -CommentUrl = NewType('CommentUrl', GitHubUrl) -Plan = NewType('Plan', str) -Status = NewType('Status', str) - - -class GitHubActionsEnv(TypedDict): - """ - Environment variables that are set by the actions runner - """ - GITHUB_API_URL: str - GITHUB_TOKEN: str - GITHUB_EVENT_PATH: str - GITHUB_EVENT_NAME: str - GITHUB_REPOSITORY: str - GITHUB_SHA: str - GITHUB_REF_TYPE: str - GITHUB_REF: str - - -job_tmp_dir = os.environ.get('JOB_TMP_DIR', '.') -step_tmp_dir = os.environ.get('STEP_TMP_DIR', '.') - -env = cast(GitHubActionsEnv, os.environ) - - -def github_session(github_env: GitHubActionsEnv) -> requests.Session: - """ - A request session that is configured for the github API - """ - session = requests.Session() - session.headers['authorization'] = f'token {github_env["GITHUB_TOKEN"]}' - session.headers['user-agent'] = 'terraform-github-actions' - session.headers['accept'] = 'application/vnd.github.v3+json' - return session - - -github = github_session(env) - - -class ActionInputs(TypedDict): - """ - Actions input environment variables that are set by the runner - """ - INPUT_BACKEND_CONFIG: str - INPUT_BACKEND_CONFIG_FILE: str - INPUT_VARIABLES: str - INPUT_VAR: str - INPUT_VAR_FILE: str - INPUT_PATH: str - INPUT_WORKSPACE: str - INPUT_LABEL: str - INPUT_ADD_GITHUB_COMMENT: str - INPUT_TARGET: str - INPUT_REPLACE: str - -class WorkflowException(Exception): - """An exception that should result in an error in the workflow log""" - -def plan_identifier(action_inputs: ActionInputs) -> str: - def mask_backend_config() -> Optional[str]: - - bad_words = [ - 'token', - 'password', - 'sas_token', - 'access_key', - 'secret_key', - 'client_secret', - 'access_token', - 'http_auth', - 'secret_id', - 'encryption_key', - 'key_material', - 'security_token', - 'conn_str', - 'sse_customer_key', - 'application_credential_secret' - ] - - def has_bad_word(s: str) -> bool: - for bad_word in bad_words: - if bad_word in s: - return True - return False - - clean = [] - - for field in action_inputs.get('INPUT_BACKEND_CONFIG', '').split(','): - if not field: - continue - - if not has_bad_word(field): - clean.append(field) - - return ','.join(clean) - - if action_inputs['INPUT_LABEL']: - return f'Terraform plan for __{action_inputs["INPUT_LABEL"]}__' - - label = f'Terraform plan in __{action_inputs["INPUT_PATH"]}__' - - if action_inputs["INPUT_WORKSPACE"] != 'default': - label += f' in the __{action_inputs["INPUT_WORKSPACE"]}__ workspace' - - if action_inputs["INPUT_TARGET"]: - label += '\nTargeting resources: ' - label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_TARGET'].splitlines()) - - if action_inputs["INPUT_REPLACE"]: - label += '\nReplacing resources: ' - label += ', '.join(f'`{res.strip()}`' for res in action_inputs['INPUT_REPLACE'].splitlines()) - - backend_config = mask_backend_config() - if backend_config: - label += f'\nWith backend config: `{backend_config}`' - - if action_inputs["INPUT_BACKEND_CONFIG_FILE"]: - label += f'\nWith backend config files: `{action_inputs["INPUT_BACKEND_CONFIG_FILE"]}`' - - if action_inputs["INPUT_VAR"]: - label += f'\nWith vars: `{action_inputs["INPUT_VAR"]}`' - - if action_inputs["INPUT_VAR_FILE"]: - label += f'\nWith var files: `{action_inputs["INPUT_VAR_FILE"]}`' - - if action_inputs["INPUT_VARIABLES"]: - stripped_vars = action_inputs["INPUT_VARIABLES"].strip() - if '\n' in stripped_vars: - label += f'''
With variables - -```hcl -{stripped_vars} -``` -
-''' - else: - label += f'\nWith variables: `{stripped_vars}`' - - return label - - -def github_api_request(method: str, *args, **kwargs) -> requests.Response: - response = github.request(method, *args, **kwargs) - - if 400 <= response.status_code < 500: - debug(str(response.headers)) - - try: - message = response.json()['message'] - - if response.headers['X-RateLimit-Remaining'] == '0': - limit_reset = datetime.datetime.fromtimestamp(int(response.headers['X-RateLimit-Reset'])) - sys.stdout.write(message) - sys.stdout.write(f' Try again when the rate limit resets at {limit_reset} UTC.\n') - sys.exit(1) - - if message != 'Resource not accessible by integration': - sys.stdout.write(message) - sys.stdout.write('\n') - debug(response.content.decode()) - - except Exception: - sys.stdout.write(response.content.decode()) - sys.stdout.write('\n') - raise - - return response - - -def debug(msg: str) -> None: - sys.stderr.write(msg) - sys.stderr.write('\n') - - -def paginate(url: GitHubUrl, *args, **kwargs) -> Iterable[Dict[str, Any]]: - while True: - response = github_api_request('get', url, *args, **kwargs) - response.raise_for_status() - - yield from response.json() - - if 'next' in response.links: - url = response.links['next']['url'] - else: - return - - -def find_pr(actions_env: GitHubActionsEnv) -> PrUrl: - """ - Find the pull request this event is related to - - >>> find_pr() - 'https://api.github.com/repos/dflook/terraform-github-actions/pulls/8' - - """ - - event: Optional[Dict[str, Any]] - - if os.path.isfile(actions_env['GITHUB_EVENT_PATH']): - with open(actions_env['GITHUB_EVENT_PATH']) as f: - event = json.load(f) - else: - debug('Event payload is not available') - event = None - - event_type = actions_env['GITHUB_EVENT_NAME'] - - if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review', 'issue_comment']: - - if event is not None: - # Pull pr url from event payload - - if event_type in ['pull_request', 'pull_request_review_comment', 'pull_request_target', 'pull_request_review']: - return cast(PrUrl, event['pull_request']['url']) - - if event_type == 'issue_comment': - - if 'pull_request' in event['issue']: - return cast(PrUrl, event['issue']['pull_request']['url']) - else: - raise WorkflowException('This comment is not for a PR. Add a filter of `if: github.event.issue.pull_request`') - - else: - # Event payload is not available - - if actions_env.get('GITHUB_REF_TYPE') == 'branch': - if match := re.match(r'refs/pull/(\d+)/', actions_env.get('GITHUB_REF', '')): - return cast(PrUrl, f'{actions_env["GITHUB_API_URL"]}/repos/{actions_env["GITHUB_REPOSITORY"]}/pulls/{match.group(1)}') - - raise WorkflowException(f'Event payload is not available at the GITHUB_EVENT_PATH {actions_env["GITHUB_EVENT_PATH"]!r}. ' + - f'This is required when run by {event_type} events. The environment has not been setup properly by the actions runner. ' + - 'This can happen when the runner is running in a container') - - elif event_type == 'push': - repo = actions_env['GITHUB_REPOSITORY'] - commit = actions_env['GITHUB_SHA'] - - def prs() -> Iterable[Dict[str, Any]]: - url = f'{actions_env["GITHUB_API_URL"]}/repos/{repo}/pulls' - yield from paginate(cast(PrUrl, url), params={'state': 'all'}) - - for pr in prs(): - if pr['merge_commit_sha'] == commit: - return cast(PrUrl, pr['url']) - - raise WorkflowException(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') - - else: - raise WorkflowException(f"The {event_type} event doesn\'t relate to a Pull Request.") - - -def current_user(actions_env: GitHubActionsEnv) -> str: - token_hash = hashlib.sha256(actions_env['GITHUB_TOKEN'].encode()).hexdigest() - - try: - with open(os.path.join(job_tmp_dir, 'token-cache', token_hash)) as f: - username = f.read() - debug(f'GITHUB_TOKEN username from token-cache: {username}') - return username - except Exception as e: - debug(str(e)) - - response = github_api_request('get', f'{actions_env["GITHUB_API_URL"]}/user') - if response.status_code != 403: - user = response.json() - debug(json.dumps(user)) - - username = user['login'] - else: - # Assume this is the github actions app token - username = 'github-actions[bot]' - - try: - os.makedirs(os.path.join(job_tmp_dir, 'token-cache'), exist_ok=True) - with open(os.path.join(job_tmp_dir, 'token-cache', token_hash), 'w') as f: - f.write(username) - except Exception as e: - debug(str(e)) - - debug(f'discovered GITHUB_TOKEN username: {username}') - return username - - -def create_summary(plan) -> Optional[str]: - summary = None - - for line in plan.splitlines(): - if line.startswith('No changes') or line.startswith('Error'): - return line - - if line.startswith('Plan:'): - summary = line - - if line.startswith('Changes to Outputs'): - if summary: - return summary + ' Changes to Outputs.' - else: - return 'Changes to Outputs' - - return summary - - -def format_body(action_inputs: ActionInputs, plan: Plan, status: Status, collapse_threshold: int) -> str: - - details_open = '' - highlighting = '' - - summary_line = create_summary(plan) - - if plan.startswith('Error'): - details_open = ' open' - elif 'Plan:' in plan: - highlighting = 'hcl' - num_lines = len(plan.splitlines()) - if num_lines < collapse_threshold: - details_open = ' open' - - if summary_line is None: - details_open = ' open' - - body = f'''{plan_identifier(action_inputs)} - -{ f'{summary_line}' if summary_line is not None else '' } - -```{highlighting} -{plan} -``` - -''' - - if status: - body += '\n' + status - - return body - - -def update_comment(issue_url: IssueUrl, - comment_url: Optional[CommentUrl], - body: str, - only_if_exists: bool = False) -> Optional[CommentUrl]: - """ - Update (or create) a comment - - :param issue_url: The url of the issue to create or update the comment in - :param comment_url: The url of the comment to update - :param body: The new comment body - :param only_if_exists: Only update an existing comment - don't create it - """ - - if comment_url is None: - if only_if_exists: - debug('Comment doesn\'t already exist - not creating it') - return None - # Create a new comment - debug('Creating comment') - response = github_api_request('post', issue_url, json={'body': body}) - else: - # Update existing comment - debug('Updating existing comment') - response = github_api_request('patch', comment_url, json={'body': body}) - - debug(body) - debug(response.content.decode()) - response.raise_for_status() - return cast(CommentUrl, response.json()['url']) - - -def find_issue_url(pr_url: str) -> IssueUrl: - pr_hash = hashlib.sha256(pr_url.encode()).hexdigest() - - try: - with open(os.path.join(job_tmp_dir, 'issue-href-cache', pr_hash)) as f: - issue_url = f.read() - debug(f'issue_url from issue-href-cache: {issue_url}') - return cast(IssueUrl, issue_url) - except Exception as e: - debug(str(e)) - - response = github_api_request('get', pr_url) - response.raise_for_status() - - issue_url = cast(IssueUrl, response.json()['_links']['issue']['href'] + '/comments') - - try: - os.makedirs(os.path.join(job_tmp_dir, 'issue-href-cache'), exist_ok=True) - with open(os.path.join(job_tmp_dir, 'issue-href-cache', pr_hash), 'w') as f: - f.write(issue_url) - except Exception as e: - debug(str(e)) - - debug(f'discovered issue_url: {issue_url}') - return cast(IssueUrl, issue_url) - - -def find_comment(issue_url: IssueUrl, username: str, action_inputs: ActionInputs) -> Tuple[Optional[CommentUrl], Optional[Plan]]: - debug('Looking for an existing comment:') - - plan_id = plan_identifier(action_inputs) - - for comment in paginate(issue_url): - debug(json.dumps(comment)) - if comment['user']['login'] == username: - match = re.match(rf'{re.escape(plan_id)}.*```(?:hcl)?(.*?)```.*', comment['body'], re.DOTALL) - - if match: - return comment['url'], cast(Plan, match.group(1).strip()) - - return None, None - -def read_step_cache() -> Dict[str, str]: - try: - with open(os.path.join(step_tmp_dir, 'github_pr_comment.cache')) as f: - debug('step cache loaded') - return json.load(f) - except Exception as e: - debug(str(e)) - return {} - -def save_step_cache(**kwargs) -> None: - try: - with open(os.path.join(step_tmp_dir, 'github_pr_comment.cache'), 'w') as f: - json.dump(kwargs, f) - debug('step cache saved') - except Exception as e: - debug(str(e)) - -def main() -> None: - if len(sys.argv) < 2: - sys.stdout.write(f'''Usage: - STATUS="" {sys.argv[0]} plan plan.txt -''') - - debug(repr(sys.argv)) - - action_inputs = cast(ActionInputs, os.environ) - - try: - collapse_threshold = int(os.environ['TF_PLAN_COLLAPSE_LENGTH']) - except (ValueError, KeyError): - collapse_threshold = 10 - - step_cache = read_step_cache() - - if step_cache.get('pr_url') is not None: - pr_url = step_cache['pr_url'] - debug(f'pr_url from step cache: {pr_url}') - else: - try: - pr_url = find_pr(env) - except WorkflowException as e: - sys.stdout.write('\n' + str(e) + '\n') - sys.exit(1) - debug(f'discovered pr_url: {pr_url}') - - if step_cache.get('pr_url') == pr_url and step_cache.get('issue_url') is not None: - issue_url = step_cache['issue_url'] - debug(f'issue_url from step cache: {issue_url}') - else: - issue_url = find_issue_url(pr_url) - - # Username is cached in the job tmp dir - username = current_user(env) - - if step_cache.get('comment_url') is not None and step_cache.get('plan') is not None: - comment_url = step_cache['comment_url'] - plan = step_cache['plan'] - debug(f'comment_url from step cache: {comment_url}') - debug(f'plan from step cache: {plan}') - else: - comment_url, plan = find_comment(issue_url, username, action_inputs) - debug(f'discovered comment_url: {comment_url}') - debug(f'discovered plan: {plan}') - - status = cast(Status, os.environ.get('STATUS', '')) - - only_if_exists = False - - if sys.argv[1] == 'plan': - plan = cast(Plan, sys.stdin.read().strip()) - - if action_inputs['INPUT_ADD_GITHUB_COMMENT'] == 'changes-only' and os.environ.get('TF_CHANGES', 'true') == 'false': - only_if_exists = True - - body = format_body(action_inputs, plan, status, collapse_threshold) - comment_url = update_comment(issue_url, comment_url, body, only_if_exists) - - elif sys.argv[1] == 'status': - if plan is None: - sys.exit(1) - else: - body = format_body(action_inputs, plan, status, collapse_threshold) - comment_url = update_comment(issue_url, comment_url, body, only_if_exists) - - elif sys.argv[1] == 'get': - if plan is None: - sys.exit(1) - - with open(sys.argv[2], 'w') as f: - f.write(plan) - - save_step_cache(pr_url=pr_url, issue_url=issue_url, comment_url=comment_url, plan=plan) - -if __name__ == '__main__': - main() diff --git a/tests/github_pr_comment/test_comment.py b/tests/github_pr_comment/test_comment.py new file mode 100644 index 00000000..61823233 --- /dev/null +++ b/tests/github_pr_comment/test_comment.py @@ -0,0 +1,140 @@ +import random +import string + +from github_pr_comment.comment import _format_comment_header, _parse_comment_header, TerraformComment, _to_api_payload, _from_api_payload + + +def test_comment_header(): + header_args = { + 'workspace_name': 'default', + 'backend_config': 'backend_config1' + } + + expected_header = '' + actual_header = _format_comment_header(**header_args) + assert actual_header == expected_header + + assert _parse_comment_header(expected_header) == header_args + + wonky_header = '' + assert _parse_comment_header(wonky_header) == header_args + + +def test_no_headers(): + issue_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + comment_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + status = 'Testing' + description = 'Hello, this is a description' + summary = 'Some changes' + body = '''An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status=status, + headers={}, + description=description, + summary=summary, + body=body + ) + + assert _from_api_payload({ + 'body': _to_api_payload(expected), + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_headers(): + issue_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + comment_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + status = 'Testing' + description = 'Hello, this is a description' + summary = 'Some changes' + body = '''An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +''' + headers = { + 'hello': 'first_header_value', + 'there': 'second_header_value' + } + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status=status, + headers=headers, + description=description, + summary=summary, + body=body + ) + + assert _from_api_payload({ + 'body': _to_api_payload(expected), + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_bad_description(): + issue_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + comment_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + status = 'Testing' + summary = 'Some changes' + body = '''blah blah body''' + description = 'crap -->\nqweqwesomething something
' + + headers = { + 'hello': 'first_header_value', + 'there': 'second_header_value' + } + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status=status, + headers=headers, + description=description, + summary=summary, + body=body + ) + + assert _from_api_payload({ + 'body': _to_api_payload(expected), + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_bad_body(): + issue_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + comment_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + status = 'Testing' + summary = 'Some changes' + description = '''blah blah description''' + body = 'qweqwe
something something ```' + + headers = { + 'hello': 'first_header_value', + 'there': 'second_header_value' + } + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status=status, + headers=headers, + description=description, + summary=summary, + body=body + ) + + assert _from_api_payload({ + 'body': _to_api_payload(expected), + 'url': comment_url, + 'issue_url': issue_url + }) == expected diff --git a/tests/github_pr_comment/test_legacy_comment.py b/tests/github_pr_comment/test_legacy_comment.py new file mode 100644 index 00000000..045b84ee --- /dev/null +++ b/tests/github_pr_comment/test_legacy_comment.py @@ -0,0 +1,457 @@ +""" +These test verify that _from_api_payload continues to correctly match pre-existing comments, without headers +""" + +import random +import string + +from github_pr_comment.comment import TerraformComment, _from_api_payload + +plan = '''An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy.''' + +issue_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) +comment_url = ''.join(random.choice(string.ascii_letters) for _ in range(10)) + +def test_path_only(): + payload = '''Terraform plan in __/test/terraform__ +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='Terraform plan in __/test/terraform__', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_nondefault_workspace(): + payload = '''Terraform plan in __/test/terraform__ in the __myworkspace__ workspace +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='Terraform plan in __/test/terraform__ in the __myworkspace__ workspace', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_variables_single_line(): + payload = '''Terraform plan in __/test/terraform__ +With variables: `var1="value"` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='Terraform plan in __/test/terraform__\nWith variables: `var1="value"`', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_variables_multi_line(): + payload = '''Terraform plan in __/test/terraform__
With variables + +```hcl +var1="value" +var2="value2" +``` +
+ +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__
With variables + +```hcl +var1="value" +var2="value2" +``` +
''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_var(): + payload = '''Terraform plan in __/test/terraform__ +With vars: `var1=value` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +With vars: `var1=value`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_var_file(): + payload = '''Terraform plan in __/test/terraform__ +With var files: `vars.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +With var files: `vars.tf`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_backend_config(): + + payload = '''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_backend_config_bad_words(): + payload = '''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +With backend config: `bucket=test,key=backend`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + +def test_target(): + payload = '''Terraform plan in __/test/terraform__ +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + +def test_replace(): + payload = '''Terraform plan in __/test/terraform__ +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_backend_config_file(): + payload = '''Terraform plan in __/test/terraform__ +With backend config files: `backend.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ +With backend config files: `backend.tf`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_all(): + payload = '''Terraform plan in __/test/terraform__ in the __test__ workspace +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +With backend config: `bucket=mybucket` +With backend config files: `backend.tf` +With vars: `myvar=hello` +With var files: `vars.tf` +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan in __/test/terraform__ in the __test__ workspace +Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` +With backend config: `bucket=mybucket` +With backend config files: `backend.tf` +With vars: `myvar=hello` +With var files: `vars.tf`''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected + + +def test_label(): + payload = '''Terraform plan for __test_label__ +
+Plan: 1 to add, 0 to change, 0 to destroy. + +```hcl +An execution plan has been generated and is shown below. +... +Plan: 1 to add, 0 to change, 0 to destroy. +``` +
+ +Testing''' + + expected = TerraformComment( + issue_url=issue_url, + comment_url=comment_url, + status='Testing', + headers={}, + description='''Terraform plan for __test_label__''', + summary='Plan: 1 to add, 0 to change, 0 to destroy.', + body=plan + ) + + assert _from_api_payload({ + 'body': payload, + 'url': comment_url, + 'issue_url': issue_url + }) == expected diff --git a/tests/github_pr_comment/test_summary.py b/tests/github_pr_comment/test_summary.py new file mode 100644 index 00000000..179a6898 --- /dev/null +++ b/tests/github_pr_comment/test_summary.py @@ -0,0 +1,160 @@ +from github_pr_comment.__main__ import create_summary + + +def test_summary_plan_11(): + plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + ++ random_string.my_string + id: + length: "11" + lower: "true" + min_lower: "0" + min_numeric: "0" + min_special: "0" + min_upper: "0" + number: "true" + result: + special: "true" + upper: "true" +Plan: 1 to add, 0 to change, 0 to destroy. +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' + + assert create_summary(plan) == expected + + +def test_summary_plan_12(): + plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' + + assert create_summary(plan) == expected + + +def test_summary_plan_14(): + plan = '''An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_string.my_string will be created + + resource "random_string" "my_string" { + + id = (known after apply) + + length = 11 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + result = (known after apply) + + special = true + + upper = true + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + s = "string" +''' + expected = 'Plan: 1 to add, 0 to change, 0 to destroy. Changes to Outputs.' + + assert create_summary(plan) == expected + + +def test_summary_error_11(): + plan = """ +Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax + +""" + expected = "Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing \"ten\": invalid syntax" + + assert create_summary(plan) == expected + + +def test_summary_error_12(): + plan = """ +Error: Incorrect attribute value type + + on main.tf line 2, in resource "random_string" "my_string": + 2: length = "ten" + +Inappropriate value for attribute "length": a number is required. +""" + + expected = "Error: Incorrect attribute value type" + assert create_summary(plan) == expected + + +def test_summary_no_change_11(): + plan = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected = "No changes. Infrastructure is up-to-date." + assert create_summary(plan) == expected + + +def test_summary_no_change_14(): + plan = """No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +""" + + expected = "No changes. Infrastructure is up-to-date." + assert create_summary(plan) == expected + + +def test_summary_output_only_change_14(): + plan = """An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + +Terraform will perform the following actions: + +Plan: 0 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + hello = "world" + +""" + + expected = "Plan: 0 to add, 0 to change, 0 to destroy. Changes to Outputs." + assert create_summary(plan) == expected + + +def test_summary_unknown(): + plan = """ +This is not anything like terraform output we know. We don't want to generate a summary for this. +""" + assert create_summary(plan) is None diff --git a/tests/test_pr_comment.py b/tests/test_pr_comment.py deleted file mode 100644 index f49ce1bd..00000000 --- a/tests/test_pr_comment.py +++ /dev/null @@ -1,535 +0,0 @@ -from github_pr_comment import format_body, ActionInputs, create_summary - -plan = '''An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy.''' - - -def action_inputs(*, - path='/test/terraform', - workspace='default', - backend_config='', - backend_config_file='', - variables='', - var='', - var_file='', - label='', - target='', - replace='' - ) -> ActionInputs: - return ActionInputs( - INPUT_WORKSPACE=workspace, - INPUT_PATH=path, - INPUT_BACKEND_CONFIG=backend_config, - INPUT_BACKEND_CONFIG_FILE=backend_config_file, - INPUT_VARIABLES=variables, - INPUT_VAR=var, - INPUT_VAR_FILE=var_file, - INPUT_LABEL=label, - INPUT_ADD_GITHUB_COMMENT='true', - INPUT_TARGET=target, - INPUT_REPLACE=replace - ) - - -def test_path_only(): - inputs = action_inputs( - path='/test/terraform' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_nondefault_workspace(): - inputs = action_inputs( - path='/test/terraform', - workspace='myworkspace' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ in the __myworkspace__ workspace -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_variables_single_line(): - inputs = action_inputs( - path='/test/terraform', - variables='var1="value"' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With variables: `var1="value"` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_variables_multi_line(): - inputs = action_inputs( - path='/test/terraform', - variables='''var1="value" -var2="value2"''' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__
With variables - -```hcl -var1="value" -var2="value2" -``` -
- -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_var(): - inputs = action_inputs( - path='/test/terraform', - var='var1=value' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With vars: `var1=value` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_var_file(): - inputs = action_inputs( - path='/test/terraform', - var_file='vars.tf' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With var files: `vars.tf` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_backend_config(): - inputs = action_inputs( - path='/test/terraform', - backend_config='bucket=test,key=backend' - ) - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With backend config: `bucket=test,key=backend` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_backend_config_bad_words(): - inputs = action_inputs( - path='/test/terraform', - backend_config='bucket=test,password=secret,key=backend,token=secret' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With backend config: `bucket=test,key=backend` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - -def test_target(): - inputs = action_inputs( - path='/test/terraform', - target='''kubernetes_secret.tls_cert_public[0] -kubernetes_secret.tls_cert_private''' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - -def test_replace(): - inputs = action_inputs( - path='/test/terraform', - replace='''kubernetes_secret.tls_cert_public[0] -kubernetes_secret.tls_cert_private''' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - -def test_backend_config_file(): - inputs = action_inputs( - path='/test/terraform', - backend_config_file='backend.tf' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ -With backend config files: `backend.tf` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_all(): - inputs = action_inputs( - path='/test/terraform', - workspace='test', - var='myvar=hello', - var_file='vars.tf', - backend_config='bucket=mybucket,password=secret', - backend_config_file='backend.tf', - target = '''kubernetes_secret.tls_cert_public[0] -kubernetes_secret.tls_cert_private''', - replace='''kubernetes_secret.tls_cert_public[0] -kubernetes_secret.tls_cert_private''' - ) - - status = 'Testing' - - expected = '''Terraform plan in __/test/terraform__ in the __test__ workspace -Targeting resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` -Replacing resources: `kubernetes_secret.tls_cert_public[0]`, `kubernetes_secret.tls_cert_private` -With backend config: `bucket=mybucket` -With backend config files: `backend.tf` -With vars: `myvar=hello` -With var files: `vars.tf` -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_label(): - inputs = action_inputs( - path='/test/terraform', - workspace='test', - var='myvar=hello', - var_file='vars.tf', - backend_config='bucket=mybucket,password=secret', - backend_config_file='backend.tf', - label='test_label' - ) - - status = 'Testing' - - expected = '''Terraform plan for __test_label__ -
-Plan: 1 to add, 0 to change, 0 to destroy. - -```hcl -An execution plan has been generated and is shown below. -... -Plan: 1 to add, 0 to change, 0 to destroy. -``` -
- -Testing''' - - assert format_body(inputs, plan, status, 10).splitlines() == expected.splitlines() - - -def test_summary_plan_11(): - plan = '''An execution plan has been generated and is shown below. -Resource actions are indicated with the following symbols: - + create - -Terraform will perform the following actions: - -+ random_string.my_string - id: - length: "11" - lower: "true" - min_lower: "0" - min_numeric: "0" - min_special: "0" - min_upper: "0" - number: "true" - result: - special: "true" - upper: "true" -Plan: 1 to add, 0 to change, 0 to destroy. -''' - expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' - - assert create_summary(plan) == expected - - -def test_summary_plan_12(): - plan = '''An execution plan has been generated and is shown below. -Resource actions are indicated with the following symbols: - + create - -Terraform will perform the following actions: - - # random_string.my_string will be created - + resource "random_string" "my_string" { - + id = (known after apply) - + length = 11 - + lower = true - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = true - + result = (known after apply) - + special = true - + upper = true - } - -Plan: 1 to add, 0 to change, 0 to destroy. -''' - expected = 'Plan: 1 to add, 0 to change, 0 to destroy.' - - assert create_summary(plan) == expected - - -def test_summary_plan_14(): - plan = '''An execution plan has been generated and is shown below. -Resource actions are indicated with the following symbols: - + create - -Terraform will perform the following actions: - - # random_string.my_string will be created - + resource "random_string" "my_string" { - + id = (known after apply) - + length = 11 - + lower = true - + min_lower = 0 - + min_numeric = 0 - + min_special = 0 - + min_upper = 0 - + number = true - + result = (known after apply) - + special = true - + upper = true - } - -Plan: 1 to add, 0 to change, 0 to destroy. - -Changes to Outputs: - + s = "string" -''' - expected = 'Plan: 1 to add, 0 to change, 0 to destroy. Changes to Outputs.' - - assert create_summary(plan) == expected - - -def test_summary_error_11(): - plan = """ -Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing "ten": invalid syntax - -""" - expected = "Error: random_string.my_string: length: cannot parse '' as int: strconv.ParseInt: parsing \"ten\": invalid syntax" - - assert create_summary(plan) == expected - - -def test_summary_error_12(): - plan = """ -Error: Incorrect attribute value type - - on main.tf line 2, in resource "random_string" "my_string": - 2: length = "ten" - -Inappropriate value for attribute "length": a number is required. -""" - - expected = "Error: Incorrect attribute value type" - assert create_summary(plan) == expected - - -def test_summary_no_change_11(): - plan = """No changes. Infrastructure is up-to-date. - -This means that Terraform did not detect any differences between your -configuration and real physical resources that exist. As a result, no -actions need to be performed. -""" - - expected = "No changes. Infrastructure is up-to-date." - assert create_summary(plan) == expected - - -def test_summary_no_change_14(): - plan = """No changes. Infrastructure is up-to-date. - -This means that Terraform did not detect any differences between your -configuration and real physical resources that exist. As a result, no -actions need to be performed. -""" - - expected = "No changes. Infrastructure is up-to-date." - assert create_summary(plan) == expected - - -def test_summary_output_only_change_14(): - plan = """An execution plan has been generated and is shown below. -Resource actions are indicated with the following symbols: - -Terraform will perform the following actions: - -Plan: 0 to add, 0 to change, 0 to destroy. - -Changes to Outputs: - + hello = "world" - -""" - - expected = "Plan: 0 to add, 0 to change, 0 to destroy. Changes to Outputs." - assert create_summary(plan) == expected - - -def test_summary_unknown(): - plan = """ -This is not anything like terraform output we know. We don't want to generate a summary for this. -""" - assert create_summary(plan) is None From 47a58e58053db41e1381c78febe6c743c02a4aea Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 4 Mar 2022 23:29:10 +0000 Subject: [PATCH 197/231] Restore comment step cache --- image/src/github_pr_comment/__main__.py | 13 +++++++++---- image/src/github_pr_comment/comment.py | 24 +++++++++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index 74719391..a874819a 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -10,7 +10,7 @@ from github_actions.env import GithubEnv from github_actions.find_pr import find_pr, WorkflowException from github_actions.inputs import PlanPrInputs -from github_pr_comment.comment import find_comment, TerraformComment, update_comment +from github_pr_comment.comment import find_comment, TerraformComment, update_comment, serialize, deserialize Plan = NewType('Plan', str) Status = NewType('Status', str) @@ -171,6 +171,9 @@ def get_pr() -> PrUrl: def get_comment(action_inputs: PlanPrInputs) -> TerraformComment: + if 'comment' in step_cache: + return deserialize(step_cache['comment']) + pr_url = get_pr() issue_url = get_issue_url(pr_url) username = current_user(env) @@ -190,7 +193,6 @@ def get_comment(action_inputs: PlanPrInputs) -> TerraformComment: return find_comment(github, issue_url, username, headers, legacy_description) - def main() -> int: if len(sys.argv) < 2: sys.stderr.write(f'''Usage: @@ -220,7 +222,7 @@ def main() -> int: debug('Comment doesn\'t already exist - not creating it') return 0 - update_comment( + comment = update_comment( github, comment, description=description, @@ -231,17 +233,20 @@ def main() -> int: elif sys.argv[1] == 'status': if comment.comment_url is None: + debug("Can't set status of comment that doesn't exist") return 1 else: - update_comment(github, comment, status=status) + comment = update_comment(github, comment, status=status) elif sys.argv[1] == 'get': if comment.comment_url is None: + debug("Can't get the plan from comment that doesn't exist") return 1 with open(sys.argv[2], 'w') as f: f.write(comment.body) + step_cache['comment'] = serialize(comment) if __name__ == '__main__': sys.exit(main()) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 57a130c3..d8d7bd7f 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -12,7 +12,6 @@ except (ValueError, KeyError): collapse_threshold = 10 - class TerraformComment: """ Represents a Terraform PR comment @@ -87,6 +86,29 @@ def body(self) -> str: def status(self) -> str: return self._status +def serialize(comment: TerraformComment) -> str: + return json.dumps({ + 'issue_url': comment.issue_url, + 'comment_url': comment.comment_url, + 'headers': comment.headers, + 'description': comment.description, + 'summary': comment.summary, + 'body': comment.body, + 'status': comment.status + }) + +def deserialize(s) -> TerraformComment: + j = json.loads(s) + + return TerraformComment( + issue_url=j['issue_url'], + comment_url=j['comment_url'], + headers=j['headers'], + description=j['description'], + summary=j['summary'], + body=j['body'], + status=j['status'] + ) def _format_comment_header(**kwargs) -> str: return f'' From 4d4b5f47360b3c283b6aba663047a923aa5e8897 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Mar 2022 08:26:37 +0000 Subject: [PATCH 198/231] Restore comment step cache --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index 471620e2..cee65b7f 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,2 +1,3 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From 28b6681ab0175da77179ec8318dfd63882c9cafe Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sat, 5 Mar 2022 09:18:33 +0000 Subject: [PATCH 199/231] Don't use comments with headers as backup candidates --- image/src/github_pr_comment/comment.py | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index d8d7bd7f..4361e81e 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -189,6 +189,21 @@ def _to_api_payload(comment: TerraformComment) -> str: return body +def matching_headers(comment: TerraformComment, headers: dict[str, str]) -> bool: + """ + Does a comment have all the specified headers + + Additional headers may be present in the comment, they are ignored if not specified in the headers argument. + """ + + for header, value in headers.items(): + if header not in comment.headers: + return False + + if comment.headers[header] != value: + return False + + return True def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: dict[str, str], legacy_description: str) -> TerraformComment: """ @@ -212,20 +227,27 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: if comment_payload['user']['login'] != username: continue - debug(json.dumps(comment_payload)) + #debug(json.dumps(comment_payload)) if comment := _from_api_payload(comment_payload): - if comment.headers == headers: - debug('Found comment that matches headers') - return comment + if comment.headers: + # Match by headers only + + if matching_headers(comment, headers): + debug('Found comment that matches headers') + return comment + + debug(f"Didn't match comment with {comment.headers=}") - debug(f"Didn't match comment with {comment.headers=}") + else: + # Match by description only - if comment.description == legacy_description: - backup_comment = comment + if comment.description == legacy_description and backup_comment is None: + debug('Found backup comment that matches legacy description') + backup_comment = comment - debug(f"Didn't match comment with {comment.description=}") + debug(f"Didn't match comment with {comment.description=}") if backup_comment is not None: debug('Found comment matching legacy description') From a86fa1d77b4dbe1ab933a6dafa9974cba3e14d04 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 18 Mar 2022 19:43:04 +0000 Subject: [PATCH 200/231] Add backend fingerprint comment header --- image/setup.py | 3 +- image/src/github_pr_comment/__main__.py | 22 +- image/src/github_pr_comment/backend_config.py | 51 +++++ .../github_pr_comment/backend_fingerprint.py | 195 ++++++++++++++++++ image/src/github_pr_comment/comment.py | 8 +- 5 files changed, 270 insertions(+), 9 deletions(-) create mode 100644 image/src/github_pr_comment/backend_config.py create mode 100644 image/src/github_pr_comment/backend_fingerprint.py diff --git a/image/setup.py b/image/setup.py index d810109c..92ed5122 100644 --- a/image/setup.py +++ b/image/setup.py @@ -16,6 +16,7 @@ }, install_requires=[ 'requests', - 'python-hcl2' + 'python-hcl2', + 'canonicaljson' ] ) diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index a874819a..41237114 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -2,6 +2,7 @@ import json import os import sys +from pathlib import Path from typing import (NewType, Optional, cast) from github_actions.api import GithubApi, IssueUrl, PrUrl @@ -10,7 +11,10 @@ from github_actions.env import GithubEnv from github_actions.find_pr import find_pr, WorkflowException from github_actions.inputs import PlanPrInputs +from github_pr_comment.backend_config import complete_config +from github_pr_comment.backend_fingerprint import fingerprint from github_pr_comment.comment import find_comment, TerraformComment, update_comment, serialize, deserialize +from terraform.module import load_module Plan = NewType('Plan', str) Status = NewType('Status', str) @@ -169,8 +173,12 @@ def get_pr() -> PrUrl: return cast(PrUrl, pr_url) +def comment_hash(value: str, salt: str) -> str: + h = hashlib.sha256(f'dflook/terraform-github-actions/{salt}') + h.update(value) + return h.hexdigest() -def get_comment(action_inputs: PlanPrInputs) -> TerraformComment: +def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: str) -> TerraformComment: if 'comment' in step_cache: return deserialize(step_cache['comment']) @@ -182,14 +190,14 @@ def get_comment(action_inputs: PlanPrInputs) -> TerraformComment: headers = { 'workspace': os.environ.get('INPUT_WORKSPACE', 'default'), - 'backend': hashlib.sha256(legacy_description.encode()).hexdigest() + 'backend': comment_hash(backend_fingerprint, pr_url) } if backend_type := os.environ.get('TERRAFORM_BACKEND_TYPE'): headers['backend_type'] = backend_type if label := os.environ.get('INPUT_LABEL'): - headers['label'] = hashlib.sha256(label.encode()).hexdigest() + headers['label'] = label return find_comment(github, issue_url, username, headers, legacy_description) @@ -206,7 +214,13 @@ def main() -> int: action_inputs = cast(PlanPrInputs, os.environ) - comment = get_comment(action_inputs) + module = load_module(Path(action_inputs.get('INPUT_PATH', '.'))) + + backend_type, backend_config = complete_config(action_inputs, module) + + backend_fingerprint = fingerprint(backend_type, backend_config, os.environ) + + comment = get_comment(action_inputs, backend_fingerprint) status = cast(Status, os.environ.get('STATUS', '')) diff --git a/image/src/github_pr_comment/backend_config.py b/image/src/github_pr_comment/backend_config.py new file mode 100644 index 00000000..a12a3bdf --- /dev/null +++ b/image/src/github_pr_comment/backend_config.py @@ -0,0 +1,51 @@ +import re +from typing import Tuple, Any + +from github_actions.debug import debug +from github_actions.inputs import InitInputs +from terraform.module import TerraformModule + +BackendConfig = dict[str, Any] +BackendType = str + + +def partial_backend_config(module: TerraformModule) -> Tuple[BackendType, BackendConfig]: + """Return the backend config specified in the terraform module.""" + + for terraform in module.get('terraform', []): + for backend in terraform.get('backend', []): + for backend_type, config in backend.items(): + return backend_type, config + + for cloud in terraform.get('cloud', []): + return 'cloud', cloud + + return 'local', {} + + +def read_backend_config_vars(init_inputs: InitInputs) -> BackendConfig: + """Read any backend config from input variables.""" + + config: BackendConfig = {} + + for path in init_inputs.get('INPUT_BACKEND_CONFIG_FILE', '').replace(',', '\n').splitlines(): + try: + config |= load_backend_config_file(Path(path)) # type: ignore + except Exception as e: + debug(f'Failed to load backend config file {path}') + debug(str(e)) + + for backend_var in init_inputs.get('INPUT_BACKEND_CONFIG', '').replace(',', '\n').splitlines(): + if match := re.match(r'(.*)\s*=\s*(.*)', backend_var): + config[match.group(1)] = match.group(2) + + return config + + +def complete_config(action_inputs: InitInputs, module: TerraformModule) -> Tuple[BackendType, BackendConfig]: + backend_type, config = partial_backend_config(module) + + for key, value in read_backend_config_vars(action_inputs): + config[key] = value + + return backend_type, config diff --git a/image/src/github_pr_comment/backend_fingerprint.py b/image/src/github_pr_comment/backend_fingerprint.py new file mode 100644 index 00000000..c775901c --- /dev/null +++ b/image/src/github_pr_comment/backend_fingerprint.py @@ -0,0 +1,195 @@ +""" +Backend fingerprinting + +Given a completed backend config and environment variables, compute a fingerprint that identifies that backend config. +This disregards any config related to *how* that backend is used. + +Combined with the backend type and workspace name, this should uniquely identify a remote state file. + +""" +import canonicaljson + +from github_pr_comment.backend_config import BackendConfig, BackendType + + +def fingerprint_remote(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'hostname': backend_config.get('hostname', ''), + 'organization': backend_config.get('organization', ''), + 'workspaces': backend_config.get('workspaces', '') + } + + +def fingerprint_cloud(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'hostname': backend_config.get('hostname', ''), + 'organization': backend_config.get('organization', ''), + 'workspaces': backend_config.get('workspaces', '') + } + + +def fingerprint_artifactory(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'url': backend_config.get('url') or env.get('ARTIFACTORY_URL', ''), + 'repo': backend_config.get('repo', ''), + 'subpath': backend_config.get('subpath', '') + } + + +def fingerprint_azurerm(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'storage_account_name': backend_config.get('storage_account_name', ''), + 'container_name': backend_config.get('container_name', ''), + 'key': backend_config.get('key', ''), + 'environment': backend_config.get('environment') or env.get('ARM_ENVIRONMENT', ''), + 'endpoint': backend_config.get('endpoint') or env.get('ARM_ENDPOINT', ''), + 'resource_group_name': backend_config.get('resource_group_name', ''), + 'msi_endpoint': backend_config.get('msi_endpoint') or env.get('ARM_MSI_ENDPOINT', ''), + 'subscription_id': backend_config.get('subscription_id') or env.get('ARM_SUBSCRIPTION_ID', ''), + 'tenant_id': backend_config.get('tenant_id') or env.get('ARM_TENANT_ID', ''), + } + + +def fingerprint_consul(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'path': backend_config.get('path', ''), + 'address': backend_config.get('address') or env.get('CONSUL_HTTP_ADDR', ''), + } + + +def fingerprint_cos(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'bucket': backend_config.get('bucket', ''), + 'prefix': backend_config.get('prefix', ''), + 'key': backend_config.get('key', ''), + 'region': backend_config.get('region', '') + } + + +def fingerprint_etcd(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'path': backend_config.get('path', ''), + 'endpoints': ' '.join(sorted(backend_config.get('endpoints', '').split(' '))) + } + + +def fingerprint_etcd3(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'prefix': backend_config.get('prefix', ''), + 'endpoints': ' '.join(sorted(backend_config.get('endpoints', []))) + } + + +def fingerprint_gcs(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'bucket': backend_config.get('bucket', ''), + 'prefix': backend_config.get('prefix', '') + } + + +def fingerprint_http(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'address': backend_config.get('address') or env.get('TF_HTTP_ADDRESS', ''), + 'lock_address': backend_config.get('lock_address') or env.get('TF_HTTP_LOCK_ADDRESS', ''), + 'unlock_address': backend_config.get('unlock_address') or env.get('TF_HTTP_UNLOCK_ADDRESS', ''), + } + + +def fingerprint_kubernetes(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'secret_suffix': backend_config.get('secret_suffix', ''), + 'namespace': backend_config.get('namespace') or env.get('KUBE_NAMESPACE', ''), + 'host': backend_config.get('host') or env.get('KUBE_HOST', ''), + 'config_path': backend_config.get('config_path') or env.get('KUBE_CONFIG_PATH', ''), + 'config_paths': backend_config.get('config_paths') or env.get('KUBE_CONFIG_PATHS', ''), + 'config_context': backend_config.get('context') or env.get('KUBE_CTX', ''), + 'config_context_cluster': backend_config.get('config_context_cluster') or env.get('KUBE_CTX_CLUSTER', '') + } + + +def fingerprint_manta(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'account': backend_config.get('account') or env.get('SDC_ACCOUNT') or env.get('TRITON_ACCOUNT', ''), + 'url': backend_config.get('url') or env.get('MANTA_URL', ''), + 'path': backend_config.get('path', ''), + 'object_name': backend_config.get('object_name', '') + } + + +def fingerprint_oss(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'region': backend_config.get('region') or env.get('ALICLOUD_REGION') or env.get('ALICLOUD_DEFAULT_REGION', ''), + 'endpoint': backend_config.get('endpoint') or env.get('ALICLOUD_OSS_ENDPOINT') or env.get('OSS_ENDPOINT', ''), + 'bucket': backend_config.get('bucket', ''), + 'prefix': backend_config.get('prefix', ''), + 'key': backend_config.get('key', ''), + } + + +def fingerprint_pg(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'conn_str': backend_config.get('conn_str', ''), + 'schema_name': backend_config.get('schema_name', '') + } + + +def fingerprint_s3(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'endpoint': backend_config.get('endpoint') or env.get('AWS_S3_ENDPOINT', ''), + 'bucket': backend_config.get('bucket', ''), + 'workspace_key_prefix': backend_config.get('workspace_key_prefix', ''), + 'key': backend_config.get('key', ''), + } + + +def fingerprint_swift(backend_config: BackendConfig, env) -> dict[str, str]: + return { + 'auth_url': backend_config.get('auth_url') or env.get('OS_AUTH_URL', ''), + 'cloud': backend_config.get('cloud') or env.get('OS_CLOUD', ''), + 'region_name': backend_config.get('region_name') or env.get('OS_REGION_NAME', ''), + 'container': backend_config.get('container', ''), + 'state_name': backend_config.get('state_name', ''), + 'path': backend_config.get('path', ''), + 'tenant_id': backend_config.get('tenant_id') or env.get('OS_TENANT_NAME') or env.get('OS_PROJECT_NAME', ''), + 'project_domain_name': backend_config.get('project_domain_name') or env.get('OS_PROJECT_DOMAIN_NAME', ''), + 'project_domain_id': backend_config.get('project_domain_id') or env.get('OS_PROJECT_DOMAIN_ID', ''), + 'domain_name': backend_config.get('domain_name') or env.get('OS_USER_DOMAIN_NAME') or env.get('OS_PROJECT_DOMAIN_NAME') or env.get('OS_DOMAIN_NAME') or env.get('DEFAULT_DOMAIN'), + 'domain_id': backend_config.get('domain_id') or env.get('OS_PROJECT_DOMAIN_ID', ''), + 'default_domain': backend_config.get('default_domain') or env.get('OS_DEFAULT_DOMAIN', '') + } + + +def fingerprint_local(backend_config: BackendConfig, env) -> dict[str, str]: + fingerprint_inputs = { + 'path': backend_config.get('path', env['INPUT_PATH']) + } + + if 'workspace_dir' in backend_config: + fingerprint_inputs['workspace_dir'] = backend_config['workspace_dir'] + + return fingerprint_inputs + + +def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) -> str: + backends = { + 'remote': fingerprint_remote, + 'artifactory': fingerprint_artifactory, + 'azurerm': fingerprint_azurerm, + 'consul': fingerprint_consul, + 'cloud': fingerprint_cloud, + 'cos': fingerprint_cos, + 'etcd': fingerprint_etcd, + 'etcd3': fingerprint_etcd3, + 'gcs': fingerprint_gcs, + 'http': fingerprint_http, + 'kubernetes': fingerprint_kubernetes, + 'manta': fingerprint_manta, + 'oss': fingerprint_oss, + 'pg': fingerprint_pg, + 's3': fingerprint_s3, + 'swift': fingerprint_swift, + 'local': fingerprint_local, + } + + fingerprint_inputs = backends.get(backend_type, lambda c: c)(backend_config, env) + return canonicaljson.encode_canonical_json(fingerprint_inputs) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 4361e81e..8c2cfacb 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -127,7 +127,7 @@ def _parse_comment_header(comment_header: Optional[str]) -> dict[str, str]: def _from_api_payload(comment: dict[str, Any]) -> Optional[TerraformComment]: - match = re.match(rf''' + match = re.match(r''' (?P\n)? (?P.*) \s* @@ -138,9 +138,9 @@ def _from_api_payload(comment: dict[str, Any]) -> Optional[TerraformComment]: (?P.*) ''', - comment['body'], - re.VERBOSE | re.DOTALL - ) + comment['body'], + re.VERBOSE | re.DOTALL + ) if not match: return None From 75c1c2daa2644b43a2f08c4be46412a982a5481f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 18 Mar 2022 19:45:24 +0000 Subject: [PATCH 201/231] Add canonicaljson --- tests/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 1e3dab6f..1282130b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,8 +1,7 @@ requests pytest python-hcl2 - +canonicaljson types-requests - mypy flake8 From fbcdb839ddbdaa0d03e1487361890044f51602cd Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 18 Mar 2022 20:02:30 +0000 Subject: [PATCH 202/231] Typecheck --- image/src/github_actions/find_pr.py | 3 +-- image/src/github_pr_comment/__main__.py | 11 ++++++----- image/src/github_pr_comment/backend_config.py | 2 +- image/src/github_pr_comment/backend_fingerprint.py | 2 +- image/src/github_pr_comment/comment.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/image/src/github_actions/find_pr.py b/image/src/github_actions/find_pr.py index 0fefb4bc..9970fcfd 100644 --- a/image/src/github_actions/find_pr.py +++ b/image/src/github_actions/find_pr.py @@ -72,5 +72,4 @@ def prs() -> Iterable[dict[str, Any]]: raise WorkflowException(f'No PR found in {repo} for commit {commit} (was it pushed directly to the target branch?)') - else: - raise WorkflowException(f"The {event_type} event doesn\'t relate to a Pull Request.") + raise WorkflowException(f"The {event_type} event doesn\'t relate to a Pull Request.") diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index 41237114..23a1904f 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -19,12 +19,12 @@ Plan = NewType('Plan', str) Status = NewType('Status', str) -job_cache = ActionsCache(os.environ.get('JOB_TMP_DIR', '.'), 'job_cache') -step_cache = ActionsCache(os.environ.get('STEP_TMP_DIR', '.'), 'step_cache') +job_cache = ActionsCache(Path(os.environ.get('JOB_TMP_DIR', '.')), 'job_cache') +step_cache = ActionsCache(Path(os.environ.get('STEP_TMP_DIR', '.')), 'step_cache') env = cast(GithubEnv, os.environ) -github = GithubApi(env.get('GITHUB_API_URL', 'https://api.github.com'), env.get('GITHUB_TOKEN')) +github = GithubApi(env.get('GITHUB_API_URL', 'https://api.github.com'), env['GITHUB_TOKEN']) def _mask_backend_config(action_inputs: PlanPrInputs) -> Optional[str]: @@ -174,8 +174,8 @@ def get_pr() -> PrUrl: return cast(PrUrl, pr_url) def comment_hash(value: str, salt: str) -> str: - h = hashlib.sha256(f'dflook/terraform-github-actions/{salt}') - h.update(value) + h = hashlib.sha256(f'dflook/terraform-github-actions/{salt}'.encode()) + h.update(value.encode()) return h.hexdigest() def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: str) -> TerraformComment: @@ -261,6 +261,7 @@ def main() -> int: f.write(comment.body) step_cache['comment'] = serialize(comment) + return 0 if __name__ == '__main__': sys.exit(main()) diff --git a/image/src/github_pr_comment/backend_config.py b/image/src/github_pr_comment/backend_config.py index a12a3bdf..fc8b7c4e 100644 --- a/image/src/github_pr_comment/backend_config.py +++ b/image/src/github_pr_comment/backend_config.py @@ -45,7 +45,7 @@ def read_backend_config_vars(init_inputs: InitInputs) -> BackendConfig: def complete_config(action_inputs: InitInputs, module: TerraformModule) -> Tuple[BackendType, BackendConfig]: backend_type, config = partial_backend_config(module) - for key, value in read_backend_config_vars(action_inputs): + for key, value in read_backend_config_vars(action_inputs).items(): config[key] = value return backend_type, config diff --git a/image/src/github_pr_comment/backend_fingerprint.py b/image/src/github_pr_comment/backend_fingerprint.py index c775901c..5f6fad3e 100644 --- a/image/src/github_pr_comment/backend_fingerprint.py +++ b/image/src/github_pr_comment/backend_fingerprint.py @@ -191,5 +191,5 @@ def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) - 'local': fingerprint_local, } - fingerprint_inputs = backends.get(backend_type, lambda c: c)(backend_config, env) + fingerprint_inputs = backends.get(backend_type, lambda c, e: c)(backend_config, env) return canonicaljson.encode_canonical_json(fingerprint_inputs) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 8c2cfacb..c67dea84 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -292,6 +292,6 @@ def update_comment( else: response = github.post(comment.issue_url, json={'body': _to_api_payload(new_comment)}) response.raise_for_status() - new_comment.url = response.json()['url'] + new_comment.comment_url = response.json()['url'] return new_comment From 6867f533e34c80ce1820f26e42d4b4dd19b5b379 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 18 Mar 2022 20:08:21 +0000 Subject: [PATCH 203/231] bytes --- image/src/github_pr_comment/__main__.py | 6 +++--- image/src/github_pr_comment/backend_fingerprint.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index 23a1904f..49710d81 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -173,12 +173,12 @@ def get_pr() -> PrUrl: return cast(PrUrl, pr_url) -def comment_hash(value: str, salt: str) -> str: +def comment_hash(value: bytes, salt: str) -> str: h = hashlib.sha256(f'dflook/terraform-github-actions/{salt}'.encode()) - h.update(value.encode()) + h.update(value) return h.hexdigest() -def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: str) -> TerraformComment: +def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> TerraformComment: if 'comment' in step_cache: return deserialize(step_cache['comment']) diff --git a/image/src/github_pr_comment/backend_fingerprint.py b/image/src/github_pr_comment/backend_fingerprint.py index 5f6fad3e..f541252c 100644 --- a/image/src/github_pr_comment/backend_fingerprint.py +++ b/image/src/github_pr_comment/backend_fingerprint.py @@ -170,7 +170,7 @@ def fingerprint_local(backend_config: BackendConfig, env) -> dict[str, str]: return fingerprint_inputs -def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) -> str: +def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) -> bytes: backends = { 'remote': fingerprint_remote, 'artifactory': fingerprint_artifactory, From 35e091eedaa00bd849e14f7d969e1912f098535d Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 18 Mar 2022 23:13:49 +0000 Subject: [PATCH 204/231] Include plan modifiers in headers --- image/src/github_pr_comment/__main__.py | 13 +++++++++++++ image/src/github_pr_comment/backend_fingerprint.py | 4 ++++ image/src/github_pr_comment/comment.py | 6 ++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index 49710d81..69bc5e69 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import (NewType, Optional, cast) +import canonicaljson + from github_actions.api import GithubApi, IssueUrl, PrUrl from github_actions.cache import ActionsCache from github_actions.debug import debug @@ -199,6 +201,17 @@ def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> Terr if label := os.environ.get('INPUT_LABEL'): headers['label'] = label + plan_modifier = {} + if target := os.environ.get('INPUT_TARGET'): + plan_modifier['target'] = sorted(t.strip() for t in target.replace(',', '\n', ).split('\n')) + + if replace := os.environ.get('INPUT_REPLACE'): + plan_modifier['replace'] = sorted(t.strip() for t in replace.replace(',', '\n', ).split('\n')) + + if plan_modifier: + debug(f'Plan modifier: {plan_modifier}') + headers['plan_modifier'] = hashlib.sha256(canonicaljson.encode_canonical_json(plan_modifier)) + return find_comment(github, issue_url, username, headers, legacy_description) def main() -> int: diff --git a/image/src/github_pr_comment/backend_fingerprint.py b/image/src/github_pr_comment/backend_fingerprint.py index f541252c..23941276 100644 --- a/image/src/github_pr_comment/backend_fingerprint.py +++ b/image/src/github_pr_comment/backend_fingerprint.py @@ -9,6 +9,7 @@ """ import canonicaljson +from github_actions.debug import debug from github_pr_comment.backend_config import BackendConfig, BackendType @@ -192,4 +193,7 @@ def fingerprint(backend_type: BackendType, backend_config: BackendConfig, env) - } fingerprint_inputs = backends.get(backend_type, lambda c, e: c)(backend_config, env) + + debug(f'Backend fingerprint includes {fingerprint_inputs.keys()}') + return canonicaljson.encode_canonical_json(fingerprint_inputs) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index c67dea84..280e5293 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -221,6 +221,8 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: :param legacy_description: The description that must be present on the comment, if not headers are found. """ + debug(f"Searching for comment with {headers=}") + backup_comment = None for comment_payload in github.paged_get(issue_url): @@ -235,7 +237,7 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: # Match by headers only if matching_headers(comment, headers): - debug('Found comment that matches headers') + debug(f'Found comment that matches headers {comment.headers=} ') return comment debug(f"Didn't match comment with {comment.headers=}") @@ -253,7 +255,7 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: debug('Found comment matching legacy description') return backup_comment - debug('No matching comment exists') + debug('No existing comment exists') return TerraformComment( issue_url=issue_url, comment_url=None, From 380f13511ce038da21ead15b9f9c22e9ab7d8d20 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 20:04:24 +0100 Subject: [PATCH 205/231] Isolate test module per workflow --- .github/workflows/pull_request_review.yaml | 4 +- .github/workflows/pull_request_target.yaml | 4 +- .github/workflows/test-apply.yaml | 113 +++++++++-------- .github/workflows/test-changes-only.yaml | 16 +-- .github/workflows/test-check.yaml | 4 +- .../{test-remote.yaml => test-cloud.yaml} | 40 +++--- .github/workflows/test-fmt-check.yaml | 4 +- .github/workflows/test-fmt.yaml | 4 +- .github/workflows/test-http.yaml | 12 +- .github/workflows/test-new-workspace.yaml | 20 +-- .github/workflows/test-output.yaml | 2 +- .github/workflows/test-plan.yaml | 31 ++--- .github/workflows/test-registry.yaml | 10 +- .github/workflows/test-ssh.yaml | 6 +- .github/workflows/test-target-replace.yaml | 46 +++---- .github/workflows/test-validate.yaml | 10 +- .github/workflows/test-version.yaml | 58 ++++----- .github/workflows/test-workflow-commands.yaml | 2 +- .../terraform_version/test_asdf.py | 0 .../terraform_version/test_local_state.py | 0 .../terraform_version/test_remote_state_s3.py | 0 .../terraform_version/test_state.py | 0 .../test_terraform_version.py | 0 .../terraform_version/test_tfc.py | 0 .../terraform_version/test_tfenv.py | 0 .../terraform_version/test_tfswitch.py | 0 tests/{validate => }/test_validate.py | 0 tests/{version => }/test_version.py | 0 .../pull_request_review}/main.tf | 0 tests/workflows/pull_request_target/main.tf | 7 ++ .../test-apply}/apply-error/main.tf | 0 .../backend_config_12/backend_config | 0 .../test-apply}/backend_config_12/main.tf | 0 .../backend_config_13/backend_config | 0 .../test-apply}/backend_config_13/main.tf | 0 tests/workflows/test-apply/changes/main.tf | 7 ++ .../test-apply/deprecated_var}/main.tf | 0 .../test-apply}/error/main.tf | 0 .../test-apply}/no_changes/main.tf | 0 .../test-apply}/no_plan/main.tf | 0 .../test-apply}/refresh_15/main.tf | 0 .../test-apply/remote}/main.tf | 0 .../test-apply}/test.tfvars | 0 tests/workflows/test-apply/vars/main.tf | 44 +++++++ .../test-changes-only}/main.tf | 0 .../test-check/changes}/main.tf | 0 .../test-check}/no_changes/main.tf | 0 .../test-cloud}/0.13/main.tf | 0 .../test-cloud}/0.13/my_variable.tfvars | 0 .../test-cloud}/1.0/main.tf | 0 .../test-cloud}/1.0/my_variable.tfvars | 0 .../test-cloud}/1.1/main.tf | 0 .../test-cloud}/1.1/my_variable.tfvars | 0 .../test-fmt-check}/canonical/main.tf | 0 .../test-fmt-check}/canonical/subdir/main.tf | 0 .../test-fmt-check}/non-canonical/main.tf | 0 .../non-canonical/subdir/main.tf | 0 tests/workflows/test-fmt/canonical/main.tf | 4 + .../test-fmt/canonical/subdir/main.tf | 4 + .../workflows/test-fmt/non-canonical/main.tf | 10 ++ .../test-fmt/non-canonical/subdir/main.tf | 4 + .../test-http}/http-module/main.tf | 0 .../test-http}/main.tf | 0 .../test-new-workspace}/main.tf | 0 tests/workflows/test-output/main.tf | 116 ++++++++++++++++++ .../workflows/test-plan/changes-only/main.tf | 12 ++ .../test-plan}/error/main.tf | 0 tests/workflows/test-plan/no_changes/main.tf | 3 + tests/workflows/test-plan/plan/main.tf | 7 ++ .../test-plan}/plan_11/main.tf | 0 .../test-plan}/plan_12/main.tf | 0 .../test-plan}/plan_13/main.tf | 0 .../test-plan}/plan_14/main.tf | 0 .../test-plan}/plan_15/main.tf | 0 .../test-plan}/plan_15_4/main.tf | 0 .../test-registry}/main.tf | 0 .../test-registry}/test-module/README.md | 0 .../test-registry}/test-module/main.tf | 0 .../test-ssh}/main.tf | 0 .../test-target-replace}/main.tf | 0 .../test-validate}/invalid/main.tf | 0 .../test-validate}/report/error.json | 0 .../test-validate}/report/file_location.json | 0 .../test-validate}/report/line_num.json | 0 .../test-validate}/report/non_json.txt | 0 .../report/test_convert_validate_report.sh | 0 .../test-validate}/report/valid.json | 0 .../test-validate}/valid/main.tf | 0 .../test-validate}/workspace_eval/main.tf | 0 .../test-version}/asdf/.tool-versions | 0 .../test-version}/cloud/main.tf | 0 .../test-version}/empty/README.md | 0 .../test-version}/local/main.tf | 0 .../test-version}/local/terraform.tfstate | 0 .../test-version}/providers/0.11/main.tf | 0 .../test-version}/providers/0.12/main.tf | 0 .../test-version}/providers/0.13/versions.tf | 0 .../test-version}/range/main.tf | 0 .../test-version}/required_version/main.tf | 0 .../test-version}/state/main.tf | 0 .../test-version}/terraform-cloud/main.tf | 0 .../test-version}/tfenv/.terraform-version | 0 .../test-version}/tfenv/main.tf | 0 .../test-version}/tfswitch/.tfswitchrc | 0 .../test-version}/tfswitch/main.tf | 0 .../workflows/test-workflow-commands/main.tf | 7 ++ 106 files changed, 425 insertions(+), 186 deletions(-) rename .github/workflows/{test-remote.yaml => test-cloud.yaml} (82%) rename tests/{python => }/terraform_version/test_asdf.py (100%) rename tests/{python => }/terraform_version/test_local_state.py (100%) rename tests/{python => }/terraform_version/test_remote_state_s3.py (100%) rename tests/{python => }/terraform_version/test_state.py (100%) rename tests/{python => }/terraform_version/test_terraform_version.py (100%) rename tests/{python => }/terraform_version/test_tfc.py (100%) rename tests/{python => }/terraform_version/test_tfenv.py (100%) rename tests/{python => }/terraform_version/test_tfswitch.py (100%) rename tests/{validate => }/test_validate.py (100%) rename tests/{version => }/test_version.py (100%) rename tests/{apply/changes => workflows/pull_request_review}/main.tf (100%) create mode 100644 tests/workflows/pull_request_target/main.tf rename tests/{apply => workflows/test-apply}/apply-error/main.tf (100%) rename tests/{apply => workflows/test-apply}/backend_config_12/backend_config (100%) rename tests/{apply => workflows/test-apply}/backend_config_12/main.tf (100%) rename tests/{apply => workflows/test-apply}/backend_config_13/backend_config (100%) rename tests/{apply => workflows/test-apply}/backend_config_13/main.tf (100%) create mode 100644 tests/workflows/test-apply/changes/main.tf rename tests/{apply/vars => workflows/test-apply/deprecated_var}/main.tf (100%) rename tests/{apply => workflows/test-apply}/error/main.tf (100%) rename tests/{apply => workflows/test-apply}/no_changes/main.tf (100%) rename tests/{apply => workflows/test-apply}/no_plan/main.tf (100%) rename tests/{apply => workflows/test-apply}/refresh_15/main.tf (100%) rename tests/{remote-state/test-bucket_12 => workflows/test-apply/remote}/main.tf (100%) rename tests/{apply => workflows/test-apply}/test.tfvars (100%) create mode 100644 tests/workflows/test-apply/vars/main.tf rename tests/{plan/changes-only => workflows/test-changes-only}/main.tf (100%) rename tests/{plan/plan => workflows/test-check/changes}/main.tf (100%) rename tests/{plan => workflows/test-check}/no_changes/main.tf (100%) rename tests/{terraform-cloud => workflows/test-cloud}/0.13/main.tf (100%) rename tests/{terraform-cloud => workflows/test-cloud}/0.13/my_variable.tfvars (100%) rename tests/{terraform-cloud => workflows/test-cloud}/1.0/main.tf (100%) rename tests/{terraform-cloud => workflows/test-cloud}/1.0/my_variable.tfvars (100%) rename tests/{terraform-cloud => workflows/test-cloud}/1.1/main.tf (100%) rename tests/{terraform-cloud => workflows/test-cloud}/1.1/my_variable.tfvars (100%) rename tests/{fmt => workflows/test-fmt-check}/canonical/main.tf (100%) rename tests/{fmt => workflows/test-fmt-check}/canonical/subdir/main.tf (100%) rename tests/{fmt => workflows/test-fmt-check}/non-canonical/main.tf (100%) rename tests/{fmt => workflows/test-fmt-check}/non-canonical/subdir/main.tf (100%) create mode 100644 tests/workflows/test-fmt/canonical/main.tf create mode 100644 tests/workflows/test-fmt/canonical/subdir/main.tf create mode 100644 tests/workflows/test-fmt/non-canonical/main.tf create mode 100644 tests/workflows/test-fmt/non-canonical/subdir/main.tf rename tests/{ => workflows/test-http}/http-module/main.tf (100%) rename tests/{git-http-module => workflows/test-http}/main.tf (100%) rename tests/{new-workspace => workflows/test-new-workspace}/main.tf (100%) create mode 100644 tests/workflows/test-output/main.tf create mode 100644 tests/workflows/test-plan/changes-only/main.tf rename tests/{plan => workflows/test-plan}/error/main.tf (100%) create mode 100644 tests/workflows/test-plan/no_changes/main.tf create mode 100644 tests/workflows/test-plan/plan/main.tf rename tests/{plan => workflows/test-plan}/plan_11/main.tf (100%) rename tests/{plan => workflows/test-plan}/plan_12/main.tf (100%) rename tests/{plan => workflows/test-plan}/plan_13/main.tf (100%) rename tests/{plan => workflows/test-plan}/plan_14/main.tf (100%) rename tests/{plan => workflows/test-plan}/plan_15/main.tf (100%) rename tests/{plan => workflows/test-plan}/plan_15_4/main.tf (100%) rename tests/{registry => workflows/test-registry}/main.tf (100%) rename tests/{registry => workflows/test-registry}/test-module/README.md (100%) rename tests/{registry => workflows/test-registry}/test-module/main.tf (100%) rename tests/{ssh-module => workflows/test-ssh}/main.tf (100%) rename tests/{target => workflows/test-target-replace}/main.tf (100%) rename tests/{validate => workflows/test-validate}/invalid/main.tf (100%) rename tests/{validate => workflows/test-validate}/report/error.json (100%) rename tests/{validate => workflows/test-validate}/report/file_location.json (100%) rename tests/{validate => workflows/test-validate}/report/line_num.json (100%) rename tests/{validate => workflows/test-validate}/report/non_json.txt (100%) rename tests/{validate => workflows/test-validate}/report/test_convert_validate_report.sh (100%) rename tests/{validate => workflows/test-validate}/report/valid.json (100%) rename tests/{validate => workflows/test-validate}/valid/main.tf (100%) rename tests/{validate => workflows/test-validate}/workspace_eval/main.tf (100%) rename tests/{version => workflows/test-version}/asdf/.tool-versions (100%) rename tests/{version => workflows/test-version}/cloud/main.tf (100%) rename tests/{version => workflows/test-version}/empty/README.md (100%) rename tests/{version => workflows/test-version}/local/main.tf (100%) rename tests/{version => workflows/test-version}/local/terraform.tfstate (100%) rename tests/{version => workflows/test-version}/providers/0.11/main.tf (100%) rename tests/{version => workflows/test-version}/providers/0.12/main.tf (100%) rename tests/{version => workflows/test-version}/providers/0.13/versions.tf (100%) rename tests/{version => workflows/test-version}/range/main.tf (100%) rename tests/{version => workflows/test-version}/required_version/main.tf (100%) rename tests/{version => workflows/test-version}/state/main.tf (100%) rename tests/{version => workflows/test-version}/terraform-cloud/main.tf (100%) rename tests/{version => workflows/test-version}/tfenv/.terraform-version (100%) rename tests/{version => workflows/test-version}/tfenv/main.tf (100%) rename tests/{version => workflows/test-version}/tfswitch/.tfswitchrc (100%) rename tests/{version => workflows/test-version}/tfswitch/main.tf (100%) create mode 100644 tests/workflows/test-workflow-commands/main.tf diff --git a/.github/workflows/pull_request_review.yaml b/.github/workflows/pull_request_review.yaml index 4ad1f859..2fa0f4f9 100644 --- a/.github/workflows/pull_request_review.yaml +++ b/.github/workflows/pull_request_review.yaml @@ -17,14 +17,14 @@ jobs: uses: ./terraform-plan with: label: pull_request_review - path: tests/apply/changes + path: tests/workflows/pull_request_review - name: Apply uses: ./terraform-apply id: output with: label: pull_request_review - path: tests/apply/changes + path: tests/workflows/pull_request_review - name: Verify outputs run: | diff --git a/.github/workflows/pull_request_target.yaml b/.github/workflows/pull_request_target.yaml index 7267a409..885938f0 100644 --- a/.github/workflows/pull_request_target.yaml +++ b/.github/workflows/pull_request_target.yaml @@ -17,14 +17,14 @@ jobs: uses: ./terraform-plan with: label: pull_request_target - path: tests/apply/changes + path: tests/workflows/pull_request_target - name: Apply uses: ./terraform-apply id: output with: label: pull_request_target - path: tests/apply/changes + path: tests/workflows/pull_request_target - name: Verify outputs run: | diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 7b2f13bf..7d17eda8 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -18,7 +18,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/remote-state/test-bucket_12 + path: tests/workflows/test-apply/remote auto_approve: true - name: Verify outputs @@ -50,7 +50,7 @@ jobs: id: apply continue-on-error: true with: - path: tests/remote-state/test-bucket_12 + path: tests/workflows/test-apply/remote auto_approve: true - name: Check failed to apply @@ -89,16 +89,16 @@ jobs: - name: Plan uses: ./terraform-plan with: - label: Apply Error - path: tests/apply/apply-error + label: test-apply apply_apply_error + path: tests/workflows/test-apply/apply-error - name: Apply uses: ./terraform-apply id: apply continue-on-error: true with: - label: Apply Error - path: tests/apply/apply-error + label: test-apply apply_apply_error + path: tests/workflows/test-apply/apply-error - name: Check failed to apply run: | @@ -134,7 +134,7 @@ jobs: id: apply continue-on-error: true with: - path: tests/apply/changes + path: tests/workflows/test-apply/changes - name: Check failed to apply run: | @@ -160,13 +160,15 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/changes + label: test-apply apply + path: tests/workflows/test-apply/changes - name: Apply uses: ./terraform-apply id: first-apply with: - path: tests/apply/changes + label: test-apply apply + path: tests/workflows/test-apply/changes - name: Verify outputs run: | @@ -189,7 +191,8 @@ jobs: uses: ./terraform-apply id: second-apply with: - path: tests/apply/changes + label: test-apply apply + path: tests/workflows/test-apply/changes - name: Verify outputs run: | @@ -220,7 +223,8 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/vars + label: test-apply apply_variables + path: tests/workflows/test-apply/vars variables: | my_var="hello" complex_input=[ @@ -235,13 +239,14 @@ jobs: protocol = "tcp" }, ] - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Apply uses: ./terraform-apply id: output with: - path: tests/apply/vars + label: test-apply apply_variables + path: tests/workflows/test-apply/vars variables: | my_var="hello" complex_input=[ @@ -256,7 +261,7 @@ jobs: protocol = "tcp" }, ] - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Verify outputs run: | @@ -304,15 +309,17 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/backend_config_12 - backend_config_file: tests/apply/backend_config_12/backend_config + label: test-apply backend_config_12 backend_config_file + path: tests/workflows/test-apply/backend_config_12 + backend_config_file: tests/workflows/test-apply/backend_config_12/backend_config - name: Apply uses: ./terraform-apply id: backend_config_file_12 with: - path: tests/apply/backend_config_12 - backend_config_file: tests/apply/backend_config_12/backend_config + label: test-apply backend_config_12 backend_config_file + path: tests/workflows/test-apply/backend_config_12 + backend_config_file: tests/workflows/test-apply/backend_config_12/backend_config - name: Verify outputs run: | @@ -334,7 +341,8 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/backend_config_12 + label: test-apply backend_config_12 backend_config + path: tests/workflows/test-apply/backend_config_12 backend_config: | bucket=terraform-github-actions key=backend_config @@ -344,7 +352,8 @@ jobs: uses: ./terraform-apply id: backend_config_12 with: - path: tests/apply/backend_config_12 + label: test-apply backend_config_12 backend_config + path: tests/workflows/test-apply/backend_config_12 backend_config: | bucket=terraform-github-actions key=backend_config @@ -381,15 +390,17 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/backend_config_13 - backend_config_file: tests/apply/backend_config_13/backend_config + label: test-apply backend_config_12 backend_config_file + path: tests/workflows/test-apply/backend_config_13 + backend_config_file: tests/workflows/test-apply/backend_config_13/backend_config - name: Apply uses: ./terraform-apply id: backend_config_file_13 with: - path: tests/apply/backend_config_13 - backend_config_file: tests/apply/backend_config_13/backend_config + label: test-apply backend_config_12 backend_config_file + path: tests/workflows/test-apply/backend_config_13 + backend_config_file: tests/workflows/test-apply/backend_config_13/backend_config - name: Verify outputs run: | @@ -411,7 +422,8 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/backend_config_13 + label: test-apply backend_config_12 backend_config + path: tests/workflows/test-apply/backend_config_13 backend_config: | bucket=terraform-github-actions key=backend_config_13 @@ -421,7 +433,8 @@ jobs: uses: ./terraform-apply id: backend_config_13 with: - path: tests/apply/backend_config_13 + label: test-apply backend_config_12 backend_config + path: tests/workflows/test-apply/backend_config_13 backend_config: | bucket=terraform-github-actions key=backend_config_13 @@ -456,19 +469,19 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/vars + path: tests/workflows/test-apply/vars label: TestLabel variables: my_var="world" - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Apply uses: ./terraform-apply id: output with: - path: tests/apply/vars + path: tests/workflows/test-apply/vars label: TestLabel variables: my_var="world" - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Verify outputs run: | @@ -502,7 +515,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/remote-state/test-bucket_12 + path: tests/workflows/test-apply/remote - name: Verify outputs run: | @@ -536,7 +549,7 @@ jobs: id: apply continue-on-error: true with: - path: tests/apply/no_plan + path: tests/workflows/test-apply/no_plan - name: Check failed to apply run: | @@ -568,14 +581,14 @@ jobs: uses: ./terraform-plan with: label: User PAT - path: tests/apply/changes + path: tests/workflows/test-apply/changes - name: Apply uses: ./terraform-apply id: output with: label: User PAT - path: tests/apply/changes + path: tests/workflows/test-apply/changes - name: Verify outputs run: | @@ -606,17 +619,17 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/vars + path: tests/workflows/test-apply/deprecated_var var: my_var=hello - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Apply uses: ./terraform-apply id: output with: - path: tests/apply/vars + path: tests/workflows/test-apply/deprecated_var var: my_var=hello - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-apply/test.tfvars - name: Verify outputs run: | @@ -645,30 +658,30 @@ jobs: - name: Plan 1 uses: ./terraform-plan with: - label: Refresh 1 - path: tests/apply/refresh_15 + label: test-apply apply_refresh 1 + path: tests/workflows/test-apply/refresh_15 variables: len=10 - name: Apply 1 uses: ./terraform-apply with: - label: Refresh 1 - path: tests/apply/refresh_15 + label: test-apply apply_refresh 1 + path: tests/workflows/test-apply/refresh_15 variables: len=10 - name: Plan 2 uses: ./terraform-plan with: - label: Refresh 2 - path: tests/apply/refresh_15 + label: test-apply apply_refresh 2 + path: tests/workflows/test-apply/refresh_15 variables: len=20 - name: Apply 2 uses: ./terraform-apply id: output with: - label: Refresh 2 - path: tests/apply/refresh_15 + label: test-apply apply_refresh 2 + path: tests/workflows/test-apply/refresh_15 variables: len=20 apply_with_pre_run: @@ -686,15 +699,15 @@ jobs: - name: Plan uses: ./terraform-plan with: - label: pre-run - path: tests/apply/changes + label: test-apply apply_with_pre_run + path: tests/workflows/test-apply/changes - name: Apply uses: ./terraform-apply id: output with: - label: pre-run - path: tests/apply/changes + label: test-apply apply_with_pre_run + path: tests/workflows/test-apply/changes - name: Verify outputs run: | diff --git a/.github/workflows/test-changes-only.yaml b/.github/workflows/test-changes-only.yaml index 0a725d28..151d7a4e 100644 --- a/.github/workflows/test-changes-only.yaml +++ b/.github/workflows/test-changes-only.yaml @@ -35,7 +35,7 @@ jobs: id: apply with: label: no_changes - path: tests/plan/changes-only + path: tests/workflows/test-changes-only - name: Check failure-reason run: | @@ -58,7 +58,7 @@ jobs: id: changes-plan with: label: change_then_no_changes - path: tests/plan/changes-only + path: tests/workflows/test-changes-only variables: | cause-changes=true add_github_comment: changes-only @@ -77,7 +77,7 @@ jobs: id: plan with: label: change_then_no_changes - path: tests/plan/changes-only + path: tests/workflows/test-changes-only variables: | cause-changes=false add_github_comment: changes-only @@ -96,7 +96,7 @@ jobs: id: apply with: label: change_then_no_changes - path: tests/plan/changes-only + path: tests/workflows/test-changes-only variables: | cause-changes=false @@ -120,7 +120,7 @@ jobs: uses: ./terraform-plan id: plan with: - path: tests/plan/changes-only + path: tests/workflows/test-changes-only label: no_changes_then_changes variables: | cause-changes=false @@ -140,7 +140,7 @@ jobs: id: apply continue-on-error: true with: - path: tests/plan/changes-only + path: tests/workflows/test-changes-only label: no_changes_then_changes variables: | cause-changes=true @@ -169,7 +169,7 @@ jobs: - name: Plan Changes uses: ./terraform-plan with: - path: tests/plan/changes-only + path: tests/workflows/test-changes-only label: apply_when_plan_has_changed variables: | cause-changes=true @@ -179,7 +179,7 @@ jobs: id: apply continue-on-error: true with: - path: tests/plan/changes-only + path: tests/workflows/test-changes-only label: apply_when_plan_has_changed variables: | cause-changes=true diff --git a/.github/workflows/test-check.yaml b/.github/workflows/test-check.yaml index 6afd9ff2..33ec7fd2 100644 --- a/.github/workflows/test-check.yaml +++ b/.github/workflows/test-check.yaml @@ -15,7 +15,7 @@ jobs: uses: ./terraform-check id: check with: - path: tests/plan/no_changes + path: tests/workflows/test-check/no_changes - name: Check failure-reason run: | @@ -36,7 +36,7 @@ jobs: continue-on-error: true id: check with: - path: tests/plan/plan + path: tests/workflows/test-check/changes - name: Check failure-reason run: | diff --git a/.github/workflows/test-remote.yaml b/.github/workflows/test-cloud.yaml similarity index 82% rename from .github/workflows/test-remote.yaml rename to .github/workflows/test-cloud.yaml index 6efff610..6427cbf3 100644 --- a/.github/workflows/test-remote.yaml +++ b/.github/workflows/test-cloud.yaml @@ -1,4 +1,4 @@ -name: Test remote backend +name: Test Terraform cloud on: - pull_request @@ -18,21 +18,21 @@ jobs: - name: Create a new workspace with no existing workspaces uses: ./terraform-new-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it doesn't exist uses: ./terraform-new-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it already exists uses: ./terraform-new-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -40,12 +40,12 @@ jobs: uses: ./terraform-apply id: auto_apply with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} auto_approve: true var_file: | - tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -80,7 +80,7 @@ jobs: uses: ./terraform-output id: output with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -95,9 +95,9 @@ jobs: run: | mkdir fixed-workspace-name if [[ "${{ matrix.tf_version }}" == "0.13" ]]; then - sed -e 's/prefix.*/name = "github-actions-0-13-${{ github.head_ref }}-1"/' tests/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + sed -e 's/prefix.*/name = "github-actions-0-13-${{ github.head_ref }}-1"/' tests/workflows/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf else - sed -e 's/prefix.*/name = "github-actions-1-1-${{ github.head_ref }}-1"/' tests/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + sed -e 's/prefix.*/name = "github-actions-1-1-${{ github.head_ref }}-1"/' tests/workflows/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf fi - name: Get outputs @@ -117,11 +117,11 @@ jobs: - name: Check no changes uses: ./terraform-check with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -130,11 +130,11 @@ jobs: id: check continue-on-error: true with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="Changed!" @@ -153,7 +153,7 @@ jobs: - name: Destroy workspace uses: ./terraform-destroy-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -163,11 +163,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -189,11 +189,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -227,7 +227,7 @@ jobs: - name: Destroy the last workspace uses: ./terraform-destroy-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -236,7 +236,7 @@ jobs: continue-on-error: true id: destroy-non-existant-workspace with: - path: tests/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Check failed to destroy diff --git a/.github/workflows/test-fmt-check.yaml b/.github/workflows/test-fmt-check.yaml index 05186fee..f316521d 100644 --- a/.github/workflows/test-fmt-check.yaml +++ b/.github/workflows/test-fmt-check.yaml @@ -15,7 +15,7 @@ jobs: uses: ./terraform-fmt-check id: fmt-check with: - path: tests/fmt/canonical + path: tests/workflows/test-fmt-check/canonical - name: Check valid run: | @@ -37,7 +37,7 @@ jobs: continue-on-error: true id: fmt-check with: - path: tests/fmt/non-canonical + path: tests/workflows/test-fmt-check/non-canonical - name: Check invalid run: | diff --git a/.github/workflows/test-fmt.yaml b/.github/workflows/test-fmt.yaml index 56c24800..760d210d 100644 --- a/.github/workflows/test-fmt.yaml +++ b/.github/workflows/test-fmt.yaml @@ -14,9 +14,9 @@ jobs: - name: terraform fmt uses: ./terraform-fmt with: - path: tests/fmt/non-canonical + path: tests/workflows/test-fmt/non-canonical - name: fmt-check uses: ./terraform-fmt-check with: - path: tests/fmt/non-canonical + path: tests/workflows/test-fmt/non-canonical diff --git a/.github/workflows/test-http.yaml b/.github/workflows/test-http.yaml index cafca1b7..8e231ce8 100644 --- a/.github/workflows/test-http.yaml +++ b/.github/workflows/test-http.yaml @@ -23,7 +23,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/git-http-module + path: tests/workflows/test-http auto_approve: true - name: Verify outputs @@ -49,7 +49,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/git-http-module + path: tests/workflows/test-http auto_approve: true - name: Verify outputs @@ -75,7 +75,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/git-http-module + path: tests/workflows/test-http auto_approve: true - name: Verify outputs @@ -97,7 +97,7 @@ jobs: continue-on-error: true id: apply with: - path: tests/git-http-module + path: tests/workflows/test-http auto_approve: true - name: Check failed @@ -121,7 +121,7 @@ jobs: uses: ./terraform-apply id: output with: - path: tests/http-module + path: tests/workflows/test-http/http-module auto_approve: true - name: Verify outputs @@ -143,7 +143,7 @@ jobs: continue-on-error: true id: apply with: - path: tests/http-module + path: tests/workflows/test-http/http-module auto_approve: true - name: Check failed diff --git a/.github/workflows/test-new-workspace.yaml b/.github/workflows/test-new-workspace.yaml index 66375641..64959cd2 100644 --- a/.github/workflows/test-new-workspace.yaml +++ b/.github/workflows/test-new-workspace.yaml @@ -20,7 +20,7 @@ jobs: - name: Setup remote backend run: | - cat >tests/new-workspace/backend.tf <tests/workflows/test-new-workspace/backend.tf <tests/target/backend.tf <tests/workflows/test-target-replace/backend.tf <=0.12.0,<=0.12.5" with: - path: tests/version/empty + path: tests/workflows/test-version/empty - name: Print the version run: echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" @@ -156,7 +156,7 @@ jobs: env: TERRAFORM_VERSION: 0.12.13 with: - path: tests/version/terraform-cloud + path: tests/workflows/test-version/terraform-cloud workspace: test-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -164,14 +164,14 @@ jobs: uses: ./terraform-version id: terraform-version with: - path: tests/version/terraform-cloud + path: tests/workflows/test-version/terraform-cloud workspace: test-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Destroy workspace uses: ./terraform-destroy-workspace with: - path: tests/version/terraform-cloud + path: tests/workflows/test-version/terraform-cloud workspace: test-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -197,7 +197,7 @@ jobs: TERRAFORM_VERSION: 1.1.2 TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} with: - path: tests/version/cloud + path: tests/workflows/test-version/cloud workspace: tfc_cloud_workspace-1 - name: Test terraform-version @@ -206,7 +206,7 @@ jobs: env: TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} with: - path: tests/version/cloud + path: tests/workflows/test-version/cloud workspace: tfc_cloud_workspace-1 - name: Destroy workspace @@ -214,7 +214,7 @@ jobs: env: TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} with: - path: tests/version/cloud + path: tests/workflows/test-version/cloud workspace: tfc_cloud_workspace-1 - name: Print the version @@ -237,7 +237,7 @@ jobs: uses: ./terraform-version id: terraform-version with: - path: tests/version/local + path: tests/workflows/test-version/local - name: Print the version run: | @@ -262,19 +262,19 @@ jobs: TERRAFORM_VERSION: 0.12.9 with: variables: my_variable="hello" - path: tests/version/state + path: tests/workflows/test-version/state auto_approve: true - name: Test terraform-version uses: ./terraform-version id: terraform-version with: - path: tests/version/state + path: tests/workflows/test-version/state - name: Destroy default workspace uses: ./terraform-destroy with: - path: tests/version/state + path: tests/workflows/test-version/state variables: my_variable="goodbye" - name: Print the version @@ -291,14 +291,14 @@ jobs: env: TERRAFORM_VERSION: 1.1.0 with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: second - name: Apply second workspace uses: ./terraform-apply with: variables: my_variable="goodbye" - path: tests/version/state + path: tests/workflows/test-version/state workspace: second auto_approve: true @@ -306,13 +306,13 @@ jobs: uses: ./terraform-version id: terraform-version-second with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: second - name: Destroy second workspace uses: ./terraform-destroy-workspace with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: second variables: my_variable="goodbye" @@ -330,14 +330,14 @@ jobs: env: TERRAFORM_VERSION: 0.13.0 with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: third - name: Apply third workspace uses: ./terraform-apply with: variables: my_variable="goodbye" - path: tests/version/state + path: tests/workflows/test-version/state workspace: third auto_approve: true @@ -345,13 +345,13 @@ jobs: uses: ./terraform-version id: terraform-version-third with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: third - name: Destroy third workspace uses: ./terraform-destroy-workspace with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: third variables: my_variable="goodbye" @@ -368,7 +368,7 @@ jobs: uses: ./terraform-version id: terraform-version-fourth with: - path: tests/version/state + path: tests/workflows/test-version/state workspace: fourth - name: Print the version @@ -391,7 +391,7 @@ jobs: uses: ./terraform-version id: terraform-version with: - path: tests/version/empty + path: tests/workflows/test-version/empty - name: Print the version run: echo "The terraform version was ${{ steps.terraform-version.outputs.terraform }}" @@ -414,7 +414,7 @@ jobs: uses: ./terraform-version id: terraform-version-12 with: - path: tests/version/providers/0.12 + path: tests/workflows/test-version/providers/0.12 - name: Print the version run: | @@ -438,7 +438,7 @@ jobs: uses: ./terraform-version id: terraform-version-13 with: - path: tests/version/providers/0.13 + path: tests/workflows/test-version/providers/0.13 - name: Print the version run: | @@ -462,7 +462,7 @@ jobs: uses: ./terraform-version id: terraform-version-11 with: - path: tests/version/providers/0.11 + path: tests/workflows/test-version/providers/0.11 - name: Print the version run: | diff --git a/.github/workflows/test-workflow-commands.yaml b/.github/workflows/test-workflow-commands.yaml index 87c752b4..67860efb 100644 --- a/.github/workflows/test-workflow-commands.yaml +++ b/.github/workflows/test-workflow-commands.yaml @@ -15,7 +15,7 @@ jobs: uses: ./terraform-plan id: plan with: - path: tests/plan/plan + path: tests/workflows/test-workflow-commands add_github_comment: false env: TERRAFORM_PRE_RUN: | diff --git a/tests/python/terraform_version/test_asdf.py b/tests/terraform_version/test_asdf.py similarity index 100% rename from tests/python/terraform_version/test_asdf.py rename to tests/terraform_version/test_asdf.py diff --git a/tests/python/terraform_version/test_local_state.py b/tests/terraform_version/test_local_state.py similarity index 100% rename from tests/python/terraform_version/test_local_state.py rename to tests/terraform_version/test_local_state.py diff --git a/tests/python/terraform_version/test_remote_state_s3.py b/tests/terraform_version/test_remote_state_s3.py similarity index 100% rename from tests/python/terraform_version/test_remote_state_s3.py rename to tests/terraform_version/test_remote_state_s3.py diff --git a/tests/python/terraform_version/test_state.py b/tests/terraform_version/test_state.py similarity index 100% rename from tests/python/terraform_version/test_state.py rename to tests/terraform_version/test_state.py diff --git a/tests/python/terraform_version/test_terraform_version.py b/tests/terraform_version/test_terraform_version.py similarity index 100% rename from tests/python/terraform_version/test_terraform_version.py rename to tests/terraform_version/test_terraform_version.py diff --git a/tests/python/terraform_version/test_tfc.py b/tests/terraform_version/test_tfc.py similarity index 100% rename from tests/python/terraform_version/test_tfc.py rename to tests/terraform_version/test_tfc.py diff --git a/tests/python/terraform_version/test_tfenv.py b/tests/terraform_version/test_tfenv.py similarity index 100% rename from tests/python/terraform_version/test_tfenv.py rename to tests/terraform_version/test_tfenv.py diff --git a/tests/python/terraform_version/test_tfswitch.py b/tests/terraform_version/test_tfswitch.py similarity index 100% rename from tests/python/terraform_version/test_tfswitch.py rename to tests/terraform_version/test_tfswitch.py diff --git a/tests/validate/test_validate.py b/tests/test_validate.py similarity index 100% rename from tests/validate/test_validate.py rename to tests/test_validate.py diff --git a/tests/version/test_version.py b/tests/test_version.py similarity index 100% rename from tests/version/test_version.py rename to tests/test_version.py diff --git a/tests/apply/changes/main.tf b/tests/workflows/pull_request_review/main.tf similarity index 100% rename from tests/apply/changes/main.tf rename to tests/workflows/pull_request_review/main.tf diff --git a/tests/workflows/pull_request_target/main.tf b/tests/workflows/pull_request_target/main.tf new file mode 100644 index 00000000..615bfe89 --- /dev/null +++ b/tests/workflows/pull_request_target/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "output_string" { + value = "the_string" +} diff --git a/tests/apply/apply-error/main.tf b/tests/workflows/test-apply/apply-error/main.tf similarity index 100% rename from tests/apply/apply-error/main.tf rename to tests/workflows/test-apply/apply-error/main.tf diff --git a/tests/apply/backend_config_12/backend_config b/tests/workflows/test-apply/backend_config_12/backend_config similarity index 100% rename from tests/apply/backend_config_12/backend_config rename to tests/workflows/test-apply/backend_config_12/backend_config diff --git a/tests/apply/backend_config_12/main.tf b/tests/workflows/test-apply/backend_config_12/main.tf similarity index 100% rename from tests/apply/backend_config_12/main.tf rename to tests/workflows/test-apply/backend_config_12/main.tf diff --git a/tests/apply/backend_config_13/backend_config b/tests/workflows/test-apply/backend_config_13/backend_config similarity index 100% rename from tests/apply/backend_config_13/backend_config rename to tests/workflows/test-apply/backend_config_13/backend_config diff --git a/tests/apply/backend_config_13/main.tf b/tests/workflows/test-apply/backend_config_13/main.tf similarity index 100% rename from tests/apply/backend_config_13/main.tf rename to tests/workflows/test-apply/backend_config_13/main.tf diff --git a/tests/workflows/test-apply/changes/main.tf b/tests/workflows/test-apply/changes/main.tf new file mode 100644 index 00000000..615bfe89 --- /dev/null +++ b/tests/workflows/test-apply/changes/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "output_string" { + value = "the_string" +} diff --git a/tests/apply/vars/main.tf b/tests/workflows/test-apply/deprecated_var/main.tf similarity index 100% rename from tests/apply/vars/main.tf rename to tests/workflows/test-apply/deprecated_var/main.tf diff --git a/tests/apply/error/main.tf b/tests/workflows/test-apply/error/main.tf similarity index 100% rename from tests/apply/error/main.tf rename to tests/workflows/test-apply/error/main.tf diff --git a/tests/apply/no_changes/main.tf b/tests/workflows/test-apply/no_changes/main.tf similarity index 100% rename from tests/apply/no_changes/main.tf rename to tests/workflows/test-apply/no_changes/main.tf diff --git a/tests/apply/no_plan/main.tf b/tests/workflows/test-apply/no_plan/main.tf similarity index 100% rename from tests/apply/no_plan/main.tf rename to tests/workflows/test-apply/no_plan/main.tf diff --git a/tests/apply/refresh_15/main.tf b/tests/workflows/test-apply/refresh_15/main.tf similarity index 100% rename from tests/apply/refresh_15/main.tf rename to tests/workflows/test-apply/refresh_15/main.tf diff --git a/tests/remote-state/test-bucket_12/main.tf b/tests/workflows/test-apply/remote/main.tf similarity index 100% rename from tests/remote-state/test-bucket_12/main.tf rename to tests/workflows/test-apply/remote/main.tf diff --git a/tests/apply/test.tfvars b/tests/workflows/test-apply/test.tfvars similarity index 100% rename from tests/apply/test.tfvars rename to tests/workflows/test-apply/test.tfvars diff --git a/tests/workflows/test-apply/vars/main.tf b/tests/workflows/test-apply/vars/main.tf new file mode 100644 index 00000000..2e928fee --- /dev/null +++ b/tests/workflows/test-apply/vars/main.tf @@ -0,0 +1,44 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "output_string" { + value = "the_string" +} + +variable "my_var" { + type = string + default = "my_var_default" +} + +variable "my_var_from_file" { + type = string + default = "my_var_from_file_default" +} + +variable "complex_input" { + type = list(object({ + internal = number + external = number + protocol = string + })) + default = [ + { + internal = 8300 + external = 8300 + protocol = "tcp" + } + ] +} + +output "from_var" { + value = var.my_var +} + +output "from_varfile" { + value = var.my_var_from_file +} + +output "complex_output" { + value = join(",", [for input in var.complex_input : "${input.internal}:${input.external}:${input.protocol}"]) +} diff --git a/tests/plan/changes-only/main.tf b/tests/workflows/test-changes-only/main.tf similarity index 100% rename from tests/plan/changes-only/main.tf rename to tests/workflows/test-changes-only/main.tf diff --git a/tests/plan/plan/main.tf b/tests/workflows/test-check/changes/main.tf similarity index 100% rename from tests/plan/plan/main.tf rename to tests/workflows/test-check/changes/main.tf diff --git a/tests/plan/no_changes/main.tf b/tests/workflows/test-check/no_changes/main.tf similarity index 100% rename from tests/plan/no_changes/main.tf rename to tests/workflows/test-check/no_changes/main.tf diff --git a/tests/terraform-cloud/0.13/main.tf b/tests/workflows/test-cloud/0.13/main.tf similarity index 100% rename from tests/terraform-cloud/0.13/main.tf rename to tests/workflows/test-cloud/0.13/main.tf diff --git a/tests/terraform-cloud/0.13/my_variable.tfvars b/tests/workflows/test-cloud/0.13/my_variable.tfvars similarity index 100% rename from tests/terraform-cloud/0.13/my_variable.tfvars rename to tests/workflows/test-cloud/0.13/my_variable.tfvars diff --git a/tests/terraform-cloud/1.0/main.tf b/tests/workflows/test-cloud/1.0/main.tf similarity index 100% rename from tests/terraform-cloud/1.0/main.tf rename to tests/workflows/test-cloud/1.0/main.tf diff --git a/tests/terraform-cloud/1.0/my_variable.tfvars b/tests/workflows/test-cloud/1.0/my_variable.tfvars similarity index 100% rename from tests/terraform-cloud/1.0/my_variable.tfvars rename to tests/workflows/test-cloud/1.0/my_variable.tfvars diff --git a/tests/terraform-cloud/1.1/main.tf b/tests/workflows/test-cloud/1.1/main.tf similarity index 100% rename from tests/terraform-cloud/1.1/main.tf rename to tests/workflows/test-cloud/1.1/main.tf diff --git a/tests/terraform-cloud/1.1/my_variable.tfvars b/tests/workflows/test-cloud/1.1/my_variable.tfvars similarity index 100% rename from tests/terraform-cloud/1.1/my_variable.tfvars rename to tests/workflows/test-cloud/1.1/my_variable.tfvars diff --git a/tests/fmt/canonical/main.tf b/tests/workflows/test-fmt-check/canonical/main.tf similarity index 100% rename from tests/fmt/canonical/main.tf rename to tests/workflows/test-fmt-check/canonical/main.tf diff --git a/tests/fmt/canonical/subdir/main.tf b/tests/workflows/test-fmt-check/canonical/subdir/main.tf similarity index 100% rename from tests/fmt/canonical/subdir/main.tf rename to tests/workflows/test-fmt-check/canonical/subdir/main.tf diff --git a/tests/fmt/non-canonical/main.tf b/tests/workflows/test-fmt-check/non-canonical/main.tf similarity index 100% rename from tests/fmt/non-canonical/main.tf rename to tests/workflows/test-fmt-check/non-canonical/main.tf diff --git a/tests/fmt/non-canonical/subdir/main.tf b/tests/workflows/test-fmt-check/non-canonical/subdir/main.tf similarity index 100% rename from tests/fmt/non-canonical/subdir/main.tf rename to tests/workflows/test-fmt-check/non-canonical/subdir/main.tf diff --git a/tests/workflows/test-fmt/canonical/main.tf b/tests/workflows/test-fmt/canonical/main.tf new file mode 100644 index 00000000..5cc55884 --- /dev/null +++ b/tests/workflows/test-fmt/canonical/main.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "hello" { + bucket = "asd" + bucket_prefix = "hgd" +} diff --git a/tests/workflows/test-fmt/canonical/subdir/main.tf b/tests/workflows/test-fmt/canonical/subdir/main.tf new file mode 100644 index 00000000..5cc55884 --- /dev/null +++ b/tests/workflows/test-fmt/canonical/subdir/main.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "hello" { + bucket = "asd" + bucket_prefix = "hgd" +} diff --git a/tests/workflows/test-fmt/non-canonical/main.tf b/tests/workflows/test-fmt/non-canonical/main.tf new file mode 100644 index 00000000..46d6e863 --- /dev/null +++ b/tests/workflows/test-fmt/non-canonical/main.tf @@ -0,0 +1,10 @@ +resource "aws_s3_bucket" "hello" { + bucket = "asd" + bucket_prefix = "hgd" +} + +variable "test-var" { + type = string + description = "A test variable that is formatted wrong" + +} \ No newline at end of file diff --git a/tests/workflows/test-fmt/non-canonical/subdir/main.tf b/tests/workflows/test-fmt/non-canonical/subdir/main.tf new file mode 100644 index 00000000..bca928f7 --- /dev/null +++ b/tests/workflows/test-fmt/non-canonical/subdir/main.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "hello" { + bucket = "asd" + bucket_prefix = "hgd" +} diff --git a/tests/http-module/main.tf b/tests/workflows/test-http/http-module/main.tf similarity index 100% rename from tests/http-module/main.tf rename to tests/workflows/test-http/http-module/main.tf diff --git a/tests/git-http-module/main.tf b/tests/workflows/test-http/main.tf similarity index 100% rename from tests/git-http-module/main.tf rename to tests/workflows/test-http/main.tf diff --git a/tests/new-workspace/main.tf b/tests/workflows/test-new-workspace/main.tf similarity index 100% rename from tests/new-workspace/main.tf rename to tests/workflows/test-new-workspace/main.tf diff --git a/tests/workflows/test-output/main.tf b/tests/workflows/test-output/main.tf new file mode 100644 index 00000000..0aa9bc90 --- /dev/null +++ b/tests/workflows/test-output/main.tf @@ -0,0 +1,116 @@ +terraform { + backend "s3" { + bucket = "terraform-github-actions" + key = "terraform-remote-state" + region = "eu-west-2" + } + required_version = "~> 0.12.0" +} + +output "my_number" { + value = 5 +} + +output "my_sensitive_number" { + value = 6 + sensitive = true +} + +output "my_string" { + value = "hello" +} + +output "my_sensitive_string" { + value = "password" + sensitive = true +} + +output "my_bool" { + value = true +} + +output "my_sensitive_bool" { + value = false + sensitive = true +} + +output "my_list" { + value = tolist(toset(["one", "two"])) +} + +output "my_sensitive_list" { + value = tolist(toset(["one", "two"])) + sensitive = true +} + +output "my_map" { + value = tomap({ + first = "one" + second = "two" + third = 3 + }) +} + +output "my_sensitive_map" { + value = tomap({ + first = "one" + second = "two" + third = 3 + }) + sensitive = true +} + +output "my_set" { + value = toset(["one", "two"]) +} + +output "my_sensitive_set" { + value = toset(["one", "two"]) + sensitive = true +} + +output "my_object" { + value = { + first = "one" + second = "two" + third = 3 + } +} + +output "my_sensitive_object" { + value = { + first = "one" + second = "two" + third = 3 + } + sensitive = true +} + +output "my_tuple" { + value = ["one", "two"] +} + +output "my_sensitive_tuple" { + value = ["one", "two"] + sensitive = true +} + +output "my_compound_output" { + value = { + first = tolist(toset(["one", "two"])) + second = toset(["one", "two"]) + third = 3 + } +} + +output "awkward_string" { + value = "hello \"there\", here are some 'quotes'." +} + +output "awkward_compound_output" { + value = { + nested = { + thevalue = ["hello \"there\", here are some 'quotes'."] + } + } +} diff --git a/tests/workflows/test-plan/changes-only/main.tf b/tests/workflows/test-plan/changes-only/main.tf new file mode 100644 index 00000000..e5192d4f --- /dev/null +++ b/tests/workflows/test-plan/changes-only/main.tf @@ -0,0 +1,12 @@ +variable "cause-changes" { + default = false +} + +variable "len" { + default = 5 +} + +resource "random_string" "the_string" { + count = var.cause-changes ? 1 : 0 + length = var.len +} diff --git a/tests/plan/error/main.tf b/tests/workflows/test-plan/error/main.tf similarity index 100% rename from tests/plan/error/main.tf rename to tests/workflows/test-plan/error/main.tf diff --git a/tests/workflows/test-plan/no_changes/main.tf b/tests/workflows/test-plan/no_changes/main.tf new file mode 100644 index 00000000..646825e0 --- /dev/null +++ b/tests/workflows/test-plan/no_changes/main.tf @@ -0,0 +1,3 @@ +locals { + hello = "world" +} \ No newline at end of file diff --git a/tests/workflows/test-plan/plan/main.tf b/tests/workflows/test-plan/plan/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/workflows/test-plan/plan/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} diff --git a/tests/plan/plan_11/main.tf b/tests/workflows/test-plan/plan_11/main.tf similarity index 100% rename from tests/plan/plan_11/main.tf rename to tests/workflows/test-plan/plan_11/main.tf diff --git a/tests/plan/plan_12/main.tf b/tests/workflows/test-plan/plan_12/main.tf similarity index 100% rename from tests/plan/plan_12/main.tf rename to tests/workflows/test-plan/plan_12/main.tf diff --git a/tests/plan/plan_13/main.tf b/tests/workflows/test-plan/plan_13/main.tf similarity index 100% rename from tests/plan/plan_13/main.tf rename to tests/workflows/test-plan/plan_13/main.tf diff --git a/tests/plan/plan_14/main.tf b/tests/workflows/test-plan/plan_14/main.tf similarity index 100% rename from tests/plan/plan_14/main.tf rename to tests/workflows/test-plan/plan_14/main.tf diff --git a/tests/plan/plan_15/main.tf b/tests/workflows/test-plan/plan_15/main.tf similarity index 100% rename from tests/plan/plan_15/main.tf rename to tests/workflows/test-plan/plan_15/main.tf diff --git a/tests/plan/plan_15_4/main.tf b/tests/workflows/test-plan/plan_15_4/main.tf similarity index 100% rename from tests/plan/plan_15_4/main.tf rename to tests/workflows/test-plan/plan_15_4/main.tf diff --git a/tests/registry/main.tf b/tests/workflows/test-registry/main.tf similarity index 100% rename from tests/registry/main.tf rename to tests/workflows/test-registry/main.tf diff --git a/tests/registry/test-module/README.md b/tests/workflows/test-registry/test-module/README.md similarity index 100% rename from tests/registry/test-module/README.md rename to tests/workflows/test-registry/test-module/README.md diff --git a/tests/registry/test-module/main.tf b/tests/workflows/test-registry/test-module/main.tf similarity index 100% rename from tests/registry/test-module/main.tf rename to tests/workflows/test-registry/test-module/main.tf diff --git a/tests/ssh-module/main.tf b/tests/workflows/test-ssh/main.tf similarity index 100% rename from tests/ssh-module/main.tf rename to tests/workflows/test-ssh/main.tf diff --git a/tests/target/main.tf b/tests/workflows/test-target-replace/main.tf similarity index 100% rename from tests/target/main.tf rename to tests/workflows/test-target-replace/main.tf diff --git a/tests/validate/invalid/main.tf b/tests/workflows/test-validate/invalid/main.tf similarity index 100% rename from tests/validate/invalid/main.tf rename to tests/workflows/test-validate/invalid/main.tf diff --git a/tests/validate/report/error.json b/tests/workflows/test-validate/report/error.json similarity index 100% rename from tests/validate/report/error.json rename to tests/workflows/test-validate/report/error.json diff --git a/tests/validate/report/file_location.json b/tests/workflows/test-validate/report/file_location.json similarity index 100% rename from tests/validate/report/file_location.json rename to tests/workflows/test-validate/report/file_location.json diff --git a/tests/validate/report/line_num.json b/tests/workflows/test-validate/report/line_num.json similarity index 100% rename from tests/validate/report/line_num.json rename to tests/workflows/test-validate/report/line_num.json diff --git a/tests/validate/report/non_json.txt b/tests/workflows/test-validate/report/non_json.txt similarity index 100% rename from tests/validate/report/non_json.txt rename to tests/workflows/test-validate/report/non_json.txt diff --git a/tests/validate/report/test_convert_validate_report.sh b/tests/workflows/test-validate/report/test_convert_validate_report.sh similarity index 100% rename from tests/validate/report/test_convert_validate_report.sh rename to tests/workflows/test-validate/report/test_convert_validate_report.sh diff --git a/tests/validate/report/valid.json b/tests/workflows/test-validate/report/valid.json similarity index 100% rename from tests/validate/report/valid.json rename to tests/workflows/test-validate/report/valid.json diff --git a/tests/validate/valid/main.tf b/tests/workflows/test-validate/valid/main.tf similarity index 100% rename from tests/validate/valid/main.tf rename to tests/workflows/test-validate/valid/main.tf diff --git a/tests/validate/workspace_eval/main.tf b/tests/workflows/test-validate/workspace_eval/main.tf similarity index 100% rename from tests/validate/workspace_eval/main.tf rename to tests/workflows/test-validate/workspace_eval/main.tf diff --git a/tests/version/asdf/.tool-versions b/tests/workflows/test-version/asdf/.tool-versions similarity index 100% rename from tests/version/asdf/.tool-versions rename to tests/workflows/test-version/asdf/.tool-versions diff --git a/tests/version/cloud/main.tf b/tests/workflows/test-version/cloud/main.tf similarity index 100% rename from tests/version/cloud/main.tf rename to tests/workflows/test-version/cloud/main.tf diff --git a/tests/version/empty/README.md b/tests/workflows/test-version/empty/README.md similarity index 100% rename from tests/version/empty/README.md rename to tests/workflows/test-version/empty/README.md diff --git a/tests/version/local/main.tf b/tests/workflows/test-version/local/main.tf similarity index 100% rename from tests/version/local/main.tf rename to tests/workflows/test-version/local/main.tf diff --git a/tests/version/local/terraform.tfstate b/tests/workflows/test-version/local/terraform.tfstate similarity index 100% rename from tests/version/local/terraform.tfstate rename to tests/workflows/test-version/local/terraform.tfstate diff --git a/tests/version/providers/0.11/main.tf b/tests/workflows/test-version/providers/0.11/main.tf similarity index 100% rename from tests/version/providers/0.11/main.tf rename to tests/workflows/test-version/providers/0.11/main.tf diff --git a/tests/version/providers/0.12/main.tf b/tests/workflows/test-version/providers/0.12/main.tf similarity index 100% rename from tests/version/providers/0.12/main.tf rename to tests/workflows/test-version/providers/0.12/main.tf diff --git a/tests/version/providers/0.13/versions.tf b/tests/workflows/test-version/providers/0.13/versions.tf similarity index 100% rename from tests/version/providers/0.13/versions.tf rename to tests/workflows/test-version/providers/0.13/versions.tf diff --git a/tests/version/range/main.tf b/tests/workflows/test-version/range/main.tf similarity index 100% rename from tests/version/range/main.tf rename to tests/workflows/test-version/range/main.tf diff --git a/tests/version/required_version/main.tf b/tests/workflows/test-version/required_version/main.tf similarity index 100% rename from tests/version/required_version/main.tf rename to tests/workflows/test-version/required_version/main.tf diff --git a/tests/version/state/main.tf b/tests/workflows/test-version/state/main.tf similarity index 100% rename from tests/version/state/main.tf rename to tests/workflows/test-version/state/main.tf diff --git a/tests/version/terraform-cloud/main.tf b/tests/workflows/test-version/terraform-cloud/main.tf similarity index 100% rename from tests/version/terraform-cloud/main.tf rename to tests/workflows/test-version/terraform-cloud/main.tf diff --git a/tests/version/tfenv/.terraform-version b/tests/workflows/test-version/tfenv/.terraform-version similarity index 100% rename from tests/version/tfenv/.terraform-version rename to tests/workflows/test-version/tfenv/.terraform-version diff --git a/tests/version/tfenv/main.tf b/tests/workflows/test-version/tfenv/main.tf similarity index 100% rename from tests/version/tfenv/main.tf rename to tests/workflows/test-version/tfenv/main.tf diff --git a/tests/version/tfswitch/.tfswitchrc b/tests/workflows/test-version/tfswitch/.tfswitchrc similarity index 100% rename from tests/version/tfswitch/.tfswitchrc rename to tests/workflows/test-version/tfswitch/.tfswitchrc diff --git a/tests/version/tfswitch/main.tf b/tests/workflows/test-version/tfswitch/main.tf similarity index 100% rename from tests/version/tfswitch/main.tf rename to tests/workflows/test-version/tfswitch/main.tf diff --git a/tests/workflows/test-workflow-commands/main.tf b/tests/workflows/test-workflow-commands/main.tf new file mode 100644 index 00000000..dee08246 --- /dev/null +++ b/tests/workflows/test-workflow-commands/main.tf @@ -0,0 +1,7 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "s" { + value = "string" +} From b50436b035b9981c7f321dea09cacec40b3b6a41 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 20:04:42 +0100 Subject: [PATCH 206/231] Fix header value bugs --- image/src/github_pr_comment/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index 69bc5e69..cddd02c4 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -203,14 +203,14 @@ def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> Terr plan_modifier = {} if target := os.environ.get('INPUT_TARGET'): - plan_modifier['target'] = sorted(t.strip() for t in target.replace(',', '\n', ).split('\n')) + plan_modifier['target'] = sorted(t.strip() for t in target.replace(',', '\n', ).split('\n') if t.strip()) if replace := os.environ.get('INPUT_REPLACE'): - plan_modifier['replace'] = sorted(t.strip() for t in replace.replace(',', '\n', ).split('\n')) + plan_modifier['replace'] = sorted(t.strip() for t in replace.replace(',', '\n', ).split('\n') if t.strip()) if plan_modifier: debug(f'Plan modifier: {plan_modifier}') - headers['plan_modifier'] = hashlib.sha256(canonicaljson.encode_canonical_json(plan_modifier)) + headers['plan_modifier'] = hashlib.sha256(canonicaljson.encode_canonical_json(plan_modifier)).hexdigest() return find_comment(github, issue_url, username, headers, legacy_description) From 4e1ee355e11c585c4d55d5a1d068914562e3009a Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 20:27:51 +0100 Subject: [PATCH 207/231] Don't select prerelease versions when using latest terraform --- image/src/terraform/versions.py | 14 +++++++++++++- image/src/terraform_version/__main__.py | 9 ++++----- image/src/terraform_version/asdf.py | 4 ++-- image/src/terraform_version/env.py | 5 ++--- image/src/terraform_version/remote_state.py | 6 +++--- .../src/terraform_version/remote_workspace.py | 4 ++-- .../src/terraform_version/required_version.py | 4 ++-- image/src/terraform_version/tfenv.py | 4 ++-- tests/terraform_version/test_latest.py | 19 +++++++++++++++++++ 9 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 tests/terraform_version/test_latest.py diff --git a/image/src/terraform/versions.py b/image/src/terraform/versions.py index 2807eee3..8d3ac7ee 100644 --- a/image/src/terraform/versions.py +++ b/image/src/terraform/versions.py @@ -4,7 +4,7 @@ import re from functools import total_ordering -from typing import Any, cast, Iterable, Literal +from typing import Any, cast, Iterable, Literal, Optional import requests @@ -183,12 +183,24 @@ def compare() -> int: # ~> x.x.x return version.major == self.major and version.minor == self.minor and version.patch >= self.patch +def latest_non_prerelease_version(versions: Iterable[Version]) -> Optional[Version]: + """Return the latest non prerelease version of the given versions.""" + + for v in sorted(versions, reverse=True): + if not v.pre_release: + return v def latest_version(versions: Iterable[Version]) -> Version: """Return the latest version of the given versions.""" return sorted(versions, reverse=True)[0] +def earliest_non_prerelease_version(versions: Iterable[Version]) -> Optional[Version]: + """Return the earliest non prerelease version of the given versions.""" + + for v in sorted(versions): + if not v.pre_release: + return v def earliest_version(versions: Iterable[Version]) -> Version: """Return the earliest version of the given versions.""" diff --git a/image/src/terraform_version/__main__.py b/image/src/terraform_version/__main__.py index 8602e6e2..77f2e86d 100644 --- a/image/src/terraform_version/__main__.py +++ b/image/src/terraform_version/__main__.py @@ -8,17 +8,16 @@ from pathlib import Path from typing import Optional, cast -from terraform_version.remote_state import get_backend_constraints, read_backend_config_vars, try_guess_state_version - from github_actions.debug import debug from github_actions.env import ActionsEnv, GithubEnv from github_actions.inputs import InitInputs from terraform.download import get_executable from terraform.module import load_module, get_backend_type -from terraform.versions import apply_constraints, get_terraform_versions, latest_version, Version, Constraint +from terraform.versions import apply_constraints, get_terraform_versions, Version, Constraint, latest_non_prerelease_version from terraform_version.asdf import try_read_asdf from terraform_version.env import try_read_env from terraform_version.local_state import try_read_local_state +from terraform_version.remote_state import get_backend_constraints, read_backend_config_vars, try_guess_state_version from terraform_version.remote_workspace import try_get_remote_workspace_version from terraform_version.required_version import try_get_required_version from terraform_version.tfenv import try_read_tfenv @@ -69,7 +68,7 @@ def determine_version(inputs: InitInputs, cli_config_path: Path, actions_env: Ac except Exception as e: debug('Failed to get backend config') debug(str(e)) - return latest_version(versions) + return latest_non_prerelease_version(versions) if backend_type not in ['remote', 'local']: if version := try_guess_state_version(inputs, module, versions): @@ -82,7 +81,7 @@ def determine_version(inputs: InitInputs, cli_config_path: Path, actions_env: Ac return version sys.stdout.write('Terraform version not specified, using the latest version\n') - return latest_version(versions) + return latest_non_prerelease_version(versions) def switch(version: Version) -> None: diff --git a/image/src/terraform_version/asdf.py b/image/src/terraform_version/asdf.py index 748c1060..558428f7 100644 --- a/image/src/terraform_version/asdf.py +++ b/image/src/terraform_version/asdf.py @@ -8,7 +8,7 @@ from github_actions.debug import debug from github_actions.inputs import InitInputs -from terraform.versions import latest_version, Version +from terraform.versions import Version, latest_non_prerelease_version def parse_asdf(tool_versions: str, versions: Iterable[Version]) -> Version: @@ -17,7 +17,7 @@ def parse_asdf(tool_versions: str, versions: Iterable[Version]) -> Version: for line in tool_versions.splitlines(): if match := re.match(r'^\s*terraform\s+([^\s#]+)', line.strip()): if match.group(1) == 'latest': - return latest_version(v for v in versions if not v.pre_release) + return latest_non_prerelease_version(v for v in versions if not v.pre_release) return Version(match.group(1)) raise Exception('No version for terraform found in .tool-versions') diff --git a/image/src/terraform_version/env.py b/image/src/terraform_version/env.py index 2c99abb4..3efb2171 100644 --- a/image/src/terraform_version/env.py +++ b/image/src/terraform_version/env.py @@ -1,11 +1,10 @@ from __future__ import annotations -import sys from typing import Iterable, Optional from github_actions.debug import debug from github_actions.env import ActionsEnv -from terraform.versions import Version, Constraint, apply_constraints, latest_version +from terraform.versions import Version, Constraint, apply_constraints, latest_non_prerelease_version def try_read_env(actions_env: ActionsEnv, versions: Iterable[Version]) -> Optional[Version]: @@ -18,7 +17,7 @@ def try_read_env(actions_env: ActionsEnv, versions: Iterable[Version]) -> Option valid_versions = list(apply_constraints(versions, [Constraint(c) for c in constraint.split(',')])) if not valid_versions: return None - return latest_version(valid_versions) + return latest_non_prerelease_version(valid_versions) except Exception as exception: debug(str(exception)) diff --git a/image/src/terraform_version/remote_state.py b/image/src/terraform_version/remote_state.py index 3e51aef8..1cde9886 100644 --- a/image/src/terraform_version/remote_state.py +++ b/image/src/terraform_version/remote_state.py @@ -16,7 +16,7 @@ from terraform.download import get_executable from terraform.exec import init_args from terraform.module import load_backend_config_file, TerraformModule -from terraform.versions import apply_constraints, Constraint, Version, earliest_version +from terraform.versions import apply_constraints, Constraint, Version, earliest_version, earliest_non_prerelease_version def read_backend_config_vars(init_inputs: InitInputs) -> dict[str, str]: @@ -205,7 +205,7 @@ def guess_state_version(inputs: InitInputs, module: TerraformModule, versions: I candidate_versions = list(versions) while candidate_versions: - result = try_init(earliest_version(candidate_versions), args, inputs.get('INPUT_WORKSPACE', 'default'), backend_tf) + result = try_init(earliest_non_prerelease_version(candidate_versions), args, inputs.get('INPUT_WORKSPACE', 'default'), backend_tf) if isinstance(result, Version): return result elif isinstance(result, Constraint): @@ -213,7 +213,7 @@ def guess_state_version(inputs: InitInputs, module: TerraformModule, versions: I elif result is None: return None else: - candidate_versions = list(apply_constraints(candidate_versions, [Constraint(f'!={earliest_version}')])) + candidate_versions = list(apply_constraints(candidate_versions, [Constraint(f'!={earliest_version(candidate_versions)}')])) return None diff --git a/image/src/terraform_version/remote_workspace.py b/image/src/terraform_version/remote_workspace.py index 3007ebcb..e772ad8a 100644 --- a/image/src/terraform_version/remote_workspace.py +++ b/image/src/terraform_version/remote_workspace.py @@ -5,7 +5,7 @@ from github_actions.inputs import InitInputs from terraform.cloud import get_workspace from terraform.module import TerraformModule, get_remote_backend_config, get_cloud_config -from terraform.versions import Version, latest_version +from terraform.versions import Version, latest_non_prerelease_version def get_remote_workspace_version(inputs: InitInputs, module: TerraformModule, cli_config_path: Path, versions: Iterable[Version]) -> Optional[Version]: @@ -30,7 +30,7 @@ def get_remote_workspace_version(inputs: InitInputs, module: TerraformModule, cl if workspace_info := get_workspace(backend_config, inputs['INPUT_WORKSPACE']): version = str(workspace_info['attributes']['terraform-version']) if version == 'latest': - return latest_version(versions) + return latest_non_prerelease_version(versions) else: return Version(version) diff --git a/image/src/terraform_version/required_version.py b/image/src/terraform_version/required_version.py index cf4aa6a1..6aecba4e 100644 --- a/image/src/terraform_version/required_version.py +++ b/image/src/terraform_version/required_version.py @@ -2,7 +2,7 @@ from github_actions.debug import debug from terraform.module import get_version_constraints, TerraformModule -from terraform.versions import Version, apply_constraints, latest_version +from terraform.versions import Version, apply_constraints, latest_non_prerelease_version def get_required_version(module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: @@ -14,7 +14,7 @@ def get_required_version(module: TerraformModule, versions: Iterable[Version]) - if not valid_versions: raise RuntimeError(f'No versions of terraform match the required_version constraints {constraints}\n') - return latest_version(valid_versions) + return latest_non_prerelease_version(valid_versions) def try_get_required_version(module: TerraformModule, versions: Iterable[Version]) -> Optional[Version]: diff --git a/image/src/terraform_version/tfenv.py b/image/src/terraform_version/tfenv.py index 1dd20ba0..3842faeb 100644 --- a/image/src/terraform_version/tfenv.py +++ b/image/src/terraform_version/tfenv.py @@ -8,7 +8,7 @@ from github_actions.debug import debug from github_actions.inputs import InitInputs -from terraform.versions import latest_version, Version +from terraform.versions import latest_version, Version, latest_non_prerelease_version def parse_tfenv(terraform_version_file: str, versions: Iterable[Version]) -> Version: @@ -23,7 +23,7 @@ def parse_tfenv(terraform_version_file: str, versions: Iterable[Version]) -> Ver version = terraform_version_file.strip() if version == 'latest': - return latest_version(v for v in versions if not v.pre_release) + return latest_non_prerelease_version(v for v in versions if not v.pre_release) if version.startswith('latest:'): version_regex = version.split(':', maxsplit=1)[1] diff --git a/tests/terraform_version/test_latest.py b/tests/terraform_version/test_latest.py new file mode 100644 index 00000000..6028e88c --- /dev/null +++ b/tests/terraform_version/test_latest.py @@ -0,0 +1,19 @@ +from __future__ import annotations +from terraform.versions import Version, earliest_version, latest_version, earliest_non_prerelease_version, latest_non_prerelease_version + + +def test_latest(): + versions = [ + Version('0.13.6-alpha-23'), + Version('0.13.6'), + Version('1.1.8'), + Version('1.1.9'), + Version('1.1.7'), + Version('1.1.0-alpha20210811'), + Version('1.2.0-alpha20225555') + ] + + assert earliest_version(versions) == Version('0.13.6-alpha-23') + assert latest_version(versions) == Version('1.2.0-alpha20225555') + assert earliest_non_prerelease_version(versions) == Version('0.13.6') + assert latest_non_prerelease_version(versions) == Version('1.1.9') From c2d303710bf35ac45bed3ba4feac1b5b5b687aa5 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 20:45:32 +0100 Subject: [PATCH 208/231] Fix paths --- .github/workflows/test-changes-only.yaml | 20 +++++------ .github/workflows/test-cloud.yaml | 38 ++++++++++---------- .github/workflows/test-output.yaml | 2 +- .github/workflows/test-plan.yaml | 4 +-- tests/workflows/test-plan/test.tfvars | 2 ++ tests/workflows/test-plan/vars/main.tf | 44 ++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 32 deletions(-) create mode 100644 tests/workflows/test-plan/test.tfvars create mode 100644 tests/workflows/test-plan/vars/main.tf diff --git a/.github/workflows/test-changes-only.yaml b/.github/workflows/test-changes-only.yaml index 151d7a4e..262cd318 100644 --- a/.github/workflows/test-changes-only.yaml +++ b/.github/workflows/test-changes-only.yaml @@ -17,8 +17,8 @@ jobs: uses: ./terraform-plan id: plan with: - label: no_changes - path: tests/plan/changes-only + label: test-changes-only change-only THIS SHOULD NOT BE A COMMENT + path: tests/workflows/test-changes-only add_github_comment: changes-only - name: Verify outputs @@ -34,7 +34,7 @@ jobs: uses: ./terraform-apply id: apply with: - label: no_changes + label: test-changes-only change-only THIS SHOULD NOT BE A COMMENT path: tests/workflows/test-changes-only - name: Check failure-reason @@ -57,7 +57,7 @@ jobs: uses: ./terraform-plan id: changes-plan with: - label: change_then_no_changes + label: test-changes-only change_then_no_changes path: tests/workflows/test-changes-only variables: | cause-changes=true @@ -76,7 +76,7 @@ jobs: uses: ./terraform-plan id: plan with: - label: change_then_no_changes + label: test-changes-only change_then_no_changes path: tests/workflows/test-changes-only variables: | cause-changes=false @@ -95,7 +95,7 @@ jobs: uses: ./terraform-apply id: apply with: - label: change_then_no_changes + label: test-changes-only change_then_no_changes path: tests/workflows/test-changes-only variables: | cause-changes=false @@ -121,7 +121,7 @@ jobs: id: plan with: path: tests/workflows/test-changes-only - label: no_changes_then_changes + label: test-changes-only no_changes_then_changes variables: | cause-changes=false add_github_comment: changes-only @@ -141,7 +141,7 @@ jobs: continue-on-error: true with: path: tests/workflows/test-changes-only - label: no_changes_then_changes + label: test-changes-only no_changes_then_changes variables: | cause-changes=true @@ -170,7 +170,7 @@ jobs: uses: ./terraform-plan with: path: tests/workflows/test-changes-only - label: apply_when_plan_has_changed + label: test-changes-only apply_when_plan_has_changed variables: | cause-changes=true @@ -180,7 +180,7 @@ jobs: continue-on-error: true with: path: tests/workflows/test-changes-only - label: apply_when_plan_has_changed + label: test-changes-only apply_when_plan_has_changed variables: | cause-changes=true len=4 diff --git a/.github/workflows/test-cloud.yaml b/.github/workflows/test-cloud.yaml index 6427cbf3..5caca69d 100644 --- a/.github/workflows/test-cloud.yaml +++ b/.github/workflows/test-cloud.yaml @@ -18,21 +18,21 @@ jobs: - name: Create a new workspace with no existing workspaces uses: ./terraform-new-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it doesn't exist uses: ./terraform-new-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Create a new workspace when it already exists uses: ./terraform-new-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -40,12 +40,12 @@ jobs: uses: ./terraform-apply id: auto_apply with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} auto_approve: true var_file: | - tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/test-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -80,7 +80,7 @@ jobs: uses: ./terraform-output id: output with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -95,9 +95,9 @@ jobs: run: | mkdir fixed-workspace-name if [[ "${{ matrix.tf_version }}" == "0.13" ]]; then - sed -e 's/prefix.*/name = "github-actions-0-13-${{ github.head_ref }}-1"/' tests/workflows/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + sed -e 's/prefix.*/name = "github-actions-0-13-${{ github.head_ref }}-1"/' tests/workflows/test-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf else - sed -e 's/prefix.*/name = "github-actions-1-1-${{ github.head_ref }}-1"/' tests/workflows/terraform-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf + sed -e 's/prefix.*/name = "github-actions-1-1-${{ github.head_ref }}-1"/' tests/workflows/test-cloud/${{ matrix.tf_version }}/main.tf > fixed-workspace-name/main.tf fi - name: Get outputs @@ -117,11 +117,11 @@ jobs: - name: Check no changes uses: ./terraform-check with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/test-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -130,11 +130,11 @@ jobs: id: check continue-on-error: true with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/test-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="Changed!" @@ -153,7 +153,7 @@ jobs: - name: Destroy workspace uses: ./terraform-destroy-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -163,11 +163,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/test-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -189,11 +189,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} var_file: | - tests/workflows/terraform-cloud/${{ matrix.tf_version }}/my_variable.tfvars + tests/workflows/test-cloud/${{ matrix.tf_version }}/my_variable.tfvars variables: | from_variables="from_variables" @@ -227,7 +227,7 @@ jobs: - name: Destroy the last workspace uses: ./terraform-destroy-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-2 backend_config: token=${{ secrets.TF_API_TOKEN }} @@ -236,7 +236,7 @@ jobs: continue-on-error: true id: destroy-non-existant-workspace with: - path: tests/workflows/terraform-cloud/${{ matrix.tf_version }} + path: tests/workflows/test-cloud/${{ matrix.tf_version }} workspace: ${{ github.head_ref }}-1 backend_config: token=${{ secrets.TF_API_TOKEN }} - name: Check failed to destroy diff --git a/.github/workflows/test-output.yaml b/.github/workflows/test-output.yaml index 8ddcba5b..d48775bc 100644 --- a/.github/workflows/test-output.yaml +++ b/.github/workflows/test-output.yaml @@ -19,7 +19,7 @@ jobs: uses: ./terraform-output id: terraform-output with: - path: tests/workflows/test-output/test-bucket_12 + path: tests/workflows/test-output - name: Print the outputs run: | diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 77984075..9fecc22e 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -453,10 +453,10 @@ jobs: - name: Plan uses: ./terraform-plan with: - path: tests/apply/vars + path: tests/workflows/test-plan/vars variables: | my_var="single" - var_file: tests/apply/test.tfvars + var_file: tests/workflows/test-plan/test.tfvars plan_change_run_commands: runs-on: ubuntu-latest diff --git a/tests/workflows/test-plan/test.tfvars b/tests/workflows/test-plan/test.tfvars new file mode 100644 index 00000000..368d66db --- /dev/null +++ b/tests/workflows/test-plan/test.tfvars @@ -0,0 +1,2 @@ +my_var_from_file="monkey" +my_var="this should be overridden" diff --git a/tests/workflows/test-plan/vars/main.tf b/tests/workflows/test-plan/vars/main.tf new file mode 100644 index 00000000..2e928fee --- /dev/null +++ b/tests/workflows/test-plan/vars/main.tf @@ -0,0 +1,44 @@ +resource "random_string" "my_string" { + length = 11 +} + +output "output_string" { + value = "the_string" +} + +variable "my_var" { + type = string + default = "my_var_default" +} + +variable "my_var_from_file" { + type = string + default = "my_var_from_file_default" +} + +variable "complex_input" { + type = list(object({ + internal = number + external = number + protocol = string + })) + default = [ + { + internal = 8300 + external = 8300 + protocol = "tcp" + } + ] +} + +output "from_var" { + value = var.my_var +} + +output "from_varfile" { + value = var.my_var_from_file +} + +output "complex_output" { + value = join(",", [for input in var.complex_input : "${input.internal}:${input.external}:${input.protocol}"]) +} From fda43c428d6a4d0aa572604848975763dfcde4f0 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 22:22:36 +0100 Subject: [PATCH 209/231] Use job name in label --- .github/workflows/test-apply.yaml | 8 ++++---- .github/workflows/test-plan.yaml | 2 +- .github/workflows/test-registry.yaml | 8 ++++---- .github/workflows/test-ssh.yaml | 4 ++-- .github/workflows/test-target-replace.yaml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 7d17eda8..046c557b 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -470,7 +470,7 @@ jobs: uses: ./terraform-plan with: path: tests/workflows/test-apply/vars - label: TestLabel + label: test-apply apply_label variables: my_var="world" var_file: tests/workflows/test-apply/test.tfvars @@ -479,7 +479,7 @@ jobs: id: output with: path: tests/workflows/test-apply/vars - label: TestLabel + label: test-apply apply_label variables: my_var="world" var_file: tests/workflows/test-apply/test.tfvars @@ -580,14 +580,14 @@ jobs: - name: Plan uses: ./terraform-plan with: - label: User PAT + label: test-apply apply_user_token path: tests/workflows/test-apply/changes - name: Apply uses: ./terraform-apply id: output with: - label: User PAT + label: test-apply apply_user_token path: tests/workflows/test-apply/changes - name: Verify outputs diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index 9fecc22e..d091503a 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -492,4 +492,4 @@ jobs: - name: Plan uses: ./terraform-plan with: - label: Optional path + label: test-plan default_path diff --git a/.github/workflows/test-registry.yaml b/.github/workflows/test-registry.yaml index 85d540fb..ce8b6bec 100644 --- a/.github/workflows/test-registry.yaml +++ b/.github/workflows/test-registry.yaml @@ -20,14 +20,14 @@ jobs: TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} with: path: tests/workflows/test-registry - label: Single registry + label: test-registry registry_module - name: Apply uses: ./terraform-apply id: output with: path: tests/workflows/test-registry - label: Single registry + label: test-registry registry_module env: TERRAFORM_CLOUD_TOKENS: app.terraform.io=${{ secrets.TF_API_TOKEN }} @@ -55,14 +55,14 @@ jobs: uses: ./terraform-plan with: path: tests/workflows/test-registry - label: Multiple registries + label: test-registry multiple_registry_module - name: Apply uses: ./terraform-apply id: output with: path: tests/workflows/test-registry - label: Multiple registries + label: test-registry multiple_registry_module - name: Verify outputs run: | diff --git a/.github/workflows/test-ssh.yaml b/.github/workflows/test-ssh.yaml index 56fa49d7..55870e80 100644 --- a/.github/workflows/test-ssh.yaml +++ b/.github/workflows/test-ssh.yaml @@ -29,7 +29,7 @@ jobs: TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} with: path: tests/workflows/test-ssh - label: SSH Module + label: test-ssh ssh_key - name: Verify outputs run: | @@ -51,7 +51,7 @@ jobs: id: plan with: path: tests/workflows/test-ssh - label: SSH Module + label: test-ssh no_ssh_key add_github_comment: false - name: Check failed diff --git a/.github/workflows/test-target-replace.yaml b/.github/workflows/test-target-replace.yaml index f4162e45..395f5225 100644 --- a/.github/workflows/test-target-replace.yaml +++ b/.github/workflows/test-target-replace.yaml @@ -18,7 +18,7 @@ jobs: uses: ./terraform-plan id: plan with: - label: No targeted changes + label: test-target-replace plan_targeting path: tests/workflows/test-target-replace target: | random_string.notpresent @@ -248,7 +248,7 @@ jobs: uses: ./terraform-plan id: plan with: - label: No targeted changes + label: test-target-replace remote_plan_targeting path: tests/workflows/test-target-replace target: | random_string.notpresent From 037fa185bd1cc0fdddbc61cd764ad6d6a06e43df Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Sun, 1 May 2022 22:52:30 +0100 Subject: [PATCH 210/231] Don't match comments with a label if no label --- .github/workflows/test-ssh.yaml | 2 +- image/src/github_pr_comment/__main__.py | 3 +-- image/src/github_pr_comment/comment.py | 6 +++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-ssh.yaml b/.github/workflows/test-ssh.yaml index 55870e80..be19cbc9 100644 --- a/.github/workflows/test-ssh.yaml +++ b/.github/workflows/test-ssh.yaml @@ -20,7 +20,7 @@ jobs: TERRAFORM_SSH_KEY: ${{ secrets.TERRAFORM_SSH_KEY }} with: path: tests/workflows/test-ssh - label: SSH Module + label: test-ssh ssh_key - name: Apply uses: ./terraform-apply diff --git a/image/src/github_pr_comment/__main__.py b/image/src/github_pr_comment/__main__.py index cddd02c4..73c54683 100644 --- a/image/src/github_pr_comment/__main__.py +++ b/image/src/github_pr_comment/__main__.py @@ -198,8 +198,7 @@ def get_comment(action_inputs: PlanPrInputs, backend_fingerprint: bytes) -> Terr if backend_type := os.environ.get('TERRAFORM_BACKEND_TYPE'): headers['backend_type'] = backend_type - if label := os.environ.get('INPUT_LABEL'): - headers['label'] = label + headers['label'] = os.environ.get('INPUT_LABEL') or None plan_modifier = {} if target := os.environ.get('INPUT_TARGET'): diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 280e5293..1367c693 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -194,10 +194,14 @@ def matching_headers(comment: TerraformComment, headers: dict[str, str]) -> bool Does a comment have all the specified headers Additional headers may be present in the comment, they are ignored if not specified in the headers argument. + If a header should NOT be present in the comment, specify a header with a value of None """ for header, value in headers.items(): - if header not in comment.headers: + if value is not None and header not in comment.headers: + return False + + if value is None and header in comment.headers: return False if comment.headers[header] != value: From a32bf192ddb082e179edbc137c8e47e8c4f59be2 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 09:00:01 +0100 Subject: [PATCH 211/231] Don't match comments with a label if no label --- image/src/github_pr_comment/comment.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 1367c693..61f788e0 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -198,13 +198,10 @@ def matching_headers(comment: TerraformComment, headers: dict[str, str]) -> bool """ for header, value in headers.items(): - if value is not None and header not in comment.headers: - return False - if value is None and header in comment.headers: return False - if comment.headers[header] != value: + if value is not None and comment.headers.get(header) != value: return False return True @@ -263,7 +260,7 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: return TerraformComment( issue_url=issue_url, comment_url=None, - headers=headers, + headers={k: v for k, v in headers.items() if v is not None}, description='', summary='', body='', From 68fa447af9aa13008ca08addfdb1b3a129bc7d95 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 10:04:58 +0100 Subject: [PATCH 212/231] Insert known headers into legacy comment --- image/src/github_pr_comment/comment.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/image/src/github_pr_comment/comment.py b/image/src/github_pr_comment/comment.py index 61f788e0..281d35c5 100644 --- a/image/src/github_pr_comment/comment.py +++ b/image/src/github_pr_comment/comment.py @@ -230,8 +230,6 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: if comment_payload['user']['login'] != username: continue - #debug(json.dumps(comment_payload)) - if comment := _from_api_payload(comment_payload): if comment.headers: @@ -247,14 +245,24 @@ def find_comment(github: GithubApi, issue_url: IssueUrl, username: str, headers: # Match by description only if comment.description == legacy_description and backup_comment is None: - debug('Found backup comment that matches legacy description') + debug(f'Found backup comment that matches legacy description {comment.description=}') backup_comment = comment debug(f"Didn't match comment with {comment.description=}") if backup_comment is not None: debug('Found comment matching legacy description') - return backup_comment + + # Insert known headers into legacy comment + return TerraformComment( + issue_url=backup_comment.issue_url, + comment_url=backup_comment.comment_url, + headers={k: v for k, v in headers.items() if v is not None}, + description=backup_comment.description, + summary=backup_comment.summary, + body=backup_comment.body, + status=backup_comment.status + ) debug('No existing comment exists') return TerraformComment( From 1857bf934b41eaf421b45c6fa7f98fc927300125 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 10:54:38 +0100 Subject: [PATCH 213/231] Don't print status update errors to workflow log --- image/actions.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/image/actions.sh b/image/actions.sh index ea850d92..e74e6c34 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -327,8 +327,10 @@ function output() { function update_status() { local status="$1" - if ! STATUS="$status" github_pr_comment status; then - echo + if ! STATUS="$status" github_pr_comment status 2>"$STEP_TMP_DIR/github_pr_comment.stderr"; then + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" + else + debug_file "$STEP_TMP_DIR/github_pr_comment.stderr" fi } From 153549edfcb7aa70df6d9ff3ed1a807ddbb06325 Mon Sep 17 00:00:00 2001 From: Callum Tait <15716903+toast-gear@users.noreply.github.com> Date: Fri, 22 Apr 2022 11:52:43 +0100 Subject: [PATCH 214/231] refactor: diff key to help debug --- image/entrypoints/apply.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index e5f08704..69cddc26 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -126,6 +126,9 @@ else echo "The plan on the PR must be up to date. Alternatively, set the auto_approve input to 'true' to apply outdated plans" update_status ":x: Plan not applied in $(job_markdown_ref) (Plan has changed)" + echo "Performing diff between the pull request plan and the plan generated at execution time ..." + echo "> are lines from the plan in the pull request" + echo "< are lines from the plan generated at execution" echo "Plan changes:" debug_log diff "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt" diff "$STEP_TMP_DIR/plan.txt" "$STEP_TMP_DIR/approved-plan.txt" || true From efb21fe30b2aa7789cbcf5178d357ee9b7d3aec8 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 11:17:56 +0100 Subject: [PATCH 215/231] Fix path to ssh test module --- tests/workflows/test-ssh/main.tf | 2 +- tests/workflows/test-ssh/test-module/README.md | 1 + tests/workflows/test-ssh/test-module/main.tf | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 tests/workflows/test-ssh/test-module/README.md create mode 100644 tests/workflows/test-ssh/test-module/main.tf diff --git a/tests/workflows/test-ssh/main.tf b/tests/workflows/test-ssh/main.tf index edd08393..c8dd0374 100644 --- a/tests/workflows/test-ssh/main.tf +++ b/tests/workflows/test-ssh/main.tf @@ -1,5 +1,5 @@ module "hello" { - source = "git::ssh://git@github.com/dflook/terraform-github-actions//tests/registry/test-module" + source = "git::ssh://git@github.com/dflook/terraform-github-actions//tests/workflows/test-ssh/test-module" } output "word" { diff --git a/tests/workflows/test-ssh/test-module/README.md b/tests/workflows/test-ssh/test-module/README.md new file mode 100644 index 00000000..6aa9053f --- /dev/null +++ b/tests/workflows/test-ssh/test-module/README.md @@ -0,0 +1 @@ +This module is a module stored in a git repo diff --git a/tests/workflows/test-ssh/test-module/main.tf b/tests/workflows/test-ssh/test-module/main.tf new file mode 100644 index 00000000..4cecf271 --- /dev/null +++ b/tests/workflows/test-ssh/test-module/main.tf @@ -0,0 +1,3 @@ +output "my-output" { + value = "hello" +} From 81c1dcedfeeb3f63c81998ab94b9cb6323257ee6 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 13:56:51 +0100 Subject: [PATCH 216/231] Fix path to http test module --- tests/infra/http-auth.py | 2 +- tests/workflows/test-http/main.tf | 2 +- tests/workflows/test-http/test-module/README.md | 1 + tests/workflows/test-http/test-module/main.tf | 3 +++ 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 tests/workflows/test-http/test-module/README.md create mode 100644 tests/workflows/test-http/test-module/main.tf diff --git a/tests/infra/http-auth.py b/tests/infra/http-auth.py index 28b077f6..85b65c5d 100644 --- a/tests/infra/http-auth.py +++ b/tests/infra/http-auth.py @@ -27,7 +27,7 @@ def lambda_handler(event, context): return { 'statusCode': 200, - 'body': '', + 'body': '', 'headers': { 'content-type': 'text/html' } diff --git a/tests/workflows/test-http/main.tf b/tests/workflows/test-http/main.tf index 9b252388..e7c52c01 100644 --- a/tests/workflows/test-http/main.tf +++ b/tests/workflows/test-http/main.tf @@ -1,5 +1,5 @@ module "git_https_source" { - source = "git::https://github.com/dflook/terraform-github-actions-dev.git//tests/registry/test-module" + source = "git::https://github.com/dflook/terraform-github-actions-dev.git//tests/workflows/test-http/test-module" } output "git_https" { diff --git a/tests/workflows/test-http/test-module/README.md b/tests/workflows/test-http/test-module/README.md new file mode 100644 index 00000000..aebfdf32 --- /dev/null +++ b/tests/workflows/test-http/test-module/README.md @@ -0,0 +1 @@ +This module is hosted in a git repo diff --git a/tests/workflows/test-http/test-module/main.tf b/tests/workflows/test-http/test-module/main.tf new file mode 100644 index 00000000..4cecf271 --- /dev/null +++ b/tests/workflows/test-http/test-module/main.tf @@ -0,0 +1,3 @@ +output "my-output" { + value = "hello" +} From 99535aeeeb39d43d12f4f33ff2091768e79d2630 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 15 Feb 2022 15:34:06 +0000 Subject: [PATCH 217/231] Use correct value for terraform.workspace when validating with remote backend Now we know the backend type, we can use the correct value for validation with remote backends --- image/entrypoints/validate.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/image/entrypoints/validate.sh b/image/entrypoints/validate.sh index 6098a203..96c54360 100755 --- a/image/entrypoints/validate.sh +++ b/image/entrypoints/validate.sh @@ -11,12 +11,18 @@ setup # How do you get a full validation report? You can't. # terraform.workspace will be evaluated during a validate, but it is not initialized properly. -# Pass through the workspace input, even if it doesn't make sense for some backends. +# Pass through the workspace input, except for remote backend where it should be 'default' + +if [[ "$TERRAFORM_BACKEND_TYPE" == "remote" ]]; then + TF_WORKSPACE="default" +else + TF_WORKSPACE="$INPUT_WORKSPACE" +fi init || true -if ! (cd "$INPUT_PATH" && TF_WORKSPACE="$INPUT_WORKSPACE" terraform validate -json | convert_validate_report "$INPUT_PATH"); then - (cd "$INPUT_PATH" && TF_WORKSPACE="$INPUT_WORKSPACE" terraform validate) +if ! (cd "$INPUT_PATH" && TF_WORKSPACE="$TF_WORKSPACE" terraform validate -json | convert_validate_report "$INPUT_PATH"); then + (cd "$INPUT_PATH" && TF_WORKSPACE="$TF_WORKSPACE" terraform validate) else echo -e "\033[1;32mSuccess!\033[0m The configuration is valid" fi From f973c8069ba1ff80e206133faeec44226a74b868 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 16:51:40 +0100 Subject: [PATCH 218/231] Add test for remote workspace name validation --- .github/workflows/test-validate.yaml | 15 ++++++++- tests/workflows/test-http/main.tf | 2 +- .../test-validate/workspace_eval/main.tf | 12 ------- .../workspace_eval_remote/main.tf | 31 +++++++++++++++++++ 4 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 tests/workflows/test-validate/workspace_eval_remote/main.tf diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 4c9ddc5f..b3058abb 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -52,7 +52,7 @@ jobs: validate_workspace: runs-on: ubuntu-latest - name: Use workspace name during validationg + name: Use workspace name during validation steps: - name: Checkout uses: actions/checkout@v2 @@ -87,3 +87,16 @@ jobs: echo "::error:: failure-reason not set correctly" exit 1 fi + + validate_remote_workspace: + runs-on: ubuntu-latest + name: Use workspace name during validation + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: validate prod + uses: ./terraform-validate + with: + path: tests/workflows/test-validate/workspace_eval_remote + workspace: prod diff --git a/tests/workflows/test-http/main.tf b/tests/workflows/test-http/main.tf index e7c52c01..8afc183b 100644 --- a/tests/workflows/test-http/main.tf +++ b/tests/workflows/test-http/main.tf @@ -1,5 +1,5 @@ module "git_https_source" { - source = "git::https://github.com/dflook/terraform-github-actions-dev.git//tests/workflows/test-http/test-module" + source = "git::https://github.com/dflook/terraform-github-actions.git//tests/workflows/test-http/test-module" } output "git_https" { diff --git a/tests/workflows/test-validate/workspace_eval/main.tf b/tests/workflows/test-validate/workspace_eval/main.tf index 3fc73324..461a726d 100644 --- a/tests/workflows/test-validate/workspace_eval/main.tf +++ b/tests/workflows/test-validate/workspace_eval/main.tf @@ -22,15 +22,3 @@ provider "aws" { resource "aws_s3_bucket" "bucket" { bucket = "hello" } - -terraform { - backend "remote" { - hostname = "app.terraform.io" - organization = "flooktech" - - workspaces { - name = "banana" - } - } -} - diff --git a/tests/workflows/test-validate/workspace_eval_remote/main.tf b/tests/workflows/test-validate/workspace_eval_remote/main.tf new file mode 100644 index 00000000..e49faf6f --- /dev/null +++ b/tests/workflows/test-validate/workspace_eval_remote/main.tf @@ -0,0 +1,31 @@ +locals { + aws_provider_config = { + default = { + region = "..." + account_id = "..." + profile = "..." + } + } +} + +provider "aws" { + region = local.aws_provider_config[terraform.workspace].region + profile = local.aws_provider_config[terraform.workspace].profile + allowed_account_ids = [local.aws_provider_config[terraform.workspace].account_id] +} + +resource "aws_s3_bucket" "bucket" { + bucket = "hello" +} + +terraform { + backend "remote" { + hostname = "app.terraform.io" + organization = "flooktech" + + workspaces { + name = "banana" + } + } +} + From 5a5a2ac295e28e94f25279775a3e6c8e6f6fd2c8 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 17:00:14 +0100 Subject: [PATCH 219/231] Fix path --- .github/workflows/test-http.yaml | 2 +- tests/workflows/test-http/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-http.yaml b/.github/workflows/test-http.yaml index 8e231ce8..b365dbfe 100644 --- a/.github/workflows/test-http.yaml +++ b/.github/workflows/test-http.yaml @@ -87,7 +87,7 @@ jobs: git_no_credentials: runs-on: ubuntu-latest - name: git_http no creds + name: git+http no creds steps: - name: Checkout uses: actions/checkout@v2 diff --git a/tests/workflows/test-http/main.tf b/tests/workflows/test-http/main.tf index 8afc183b..e7c52c01 100644 --- a/tests/workflows/test-http/main.tf +++ b/tests/workflows/test-http/main.tf @@ -1,5 +1,5 @@ module "git_https_source" { - source = "git::https://github.com/dflook/terraform-github-actions.git//tests/workflows/test-http/test-module" + source = "git::https://github.com/dflook/terraform-github-actions-dev.git//tests/workflows/test-http/test-module" } output "git_https" { From f87a90c6199a9391ed5defd4e4647e683f13b65f Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 18:01:51 +0100 Subject: [PATCH 220/231] Try loading hcl files in another process To mitigate hangs from unterminated strings --- .github/workflows/test-validate.yaml | 12 ++++ image/src/terraform/hcl.py | 60 +++++++++++++++++++ image/src/terraform/module.py | 21 ++++--- .../test-validate/unterminated-string/main.tf | 10 ++++ 4 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 image/src/terraform/hcl.py create mode 100644 tests/workflows/test-validate/unterminated-string/main.tf diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index b3058abb..91ec8f16 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -100,3 +100,15 @@ jobs: with: path: tests/workflows/test-validate/workspace_eval_remote workspace: prod + + validate_unterminated_string: + runs-on: ubuntu-latest + name: Validate with unterminated string + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: validate + uses: ./terraform-validate + with: + path: tests/workflows/test-validate/unterminated-string diff --git a/image/src/terraform/hcl.py b/image/src/terraform/hcl.py new file mode 100644 index 00000000..192d6ab7 --- /dev/null +++ b/image/src/terraform/hcl.py @@ -0,0 +1,60 @@ +""" +Wraps python-hcl +""" + +import hcl2 # type: ignore +import sys +import subprocess +from pathlib import Path + +from github_actions.debug import debug + + +def try_load(path: Path) -> dict: + try: + with open(path) as f: + return hcl2.load(f) + except: + return {} + + +def is_loadable(path: Path) -> bool: + try: + subprocess.run( + [sys.executable, '-m', 'terraform.hcl', path], + timeout=10 + ) + except subprocess.TimeoutExpired: + debug('TimeoutExpired') + # We found a file that won't parse :( + return False + except: + # If we get an exception, we can still try and load it. + return True + + return True + + +def load(path: Path) -> dict: + if is_loadable(path): + return try_load(path) + + debug(f'Unable to load {path}') + raise ValueError(f'Unable to load {path}') + + +def loads(hcl: str) -> dict: + tmp_path = Path('/tmp/load_test.hcl') + + with open(tmp_path, 'w') as f: + f.write(hcl) + + if is_loadable(tmp_path): + return hcl2.loads(hcl) + + debug(f'Unable to load hcl') + raise ValueError(f'Unable to load hcl') + + +if __name__ == '__main__': + try_load(Path(sys.argv[1])) diff --git a/image/src/terraform/module.py b/image/src/terraform/module.py index 84754c3f..3381c447 100644 --- a/image/src/terraform/module.py +++ b/image/src/terraform/module.py @@ -5,7 +5,7 @@ import os from typing import Any, cast, NewType, Optional, TYPE_CHECKING, TypedDict -import hcl2 # type: ignore +import terraform.hcl from github_actions.debug import debug from terraform.versions import Constraint @@ -66,13 +66,13 @@ def load_module(path: Path) -> TerraformModule: if not file.endswith('.tf'): continue - with open(os.path.join(path, file)) as f: - try: - module = merge(module, cast(TerraformModule, hcl2.load(f))) - except Exception as e: - # ignore tf files that don't parse - debug(f'Failed to parse {file}') - debug(str(e)) + try: + tf_file = cast(TerraformModule, terraform.hcl.load(os.path.join(path, file))) + module = merge(module, tf_file) + except Exception as e: + # ignore tf files that don't parse + debug(f'Failed to parse {file}') + debug(str(e)) return module @@ -80,8 +80,7 @@ def load_module(path: Path) -> TerraformModule: def load_backend_config_file(path: Path) -> TerraformModule: """Load a backend config file.""" - with open(path) as f: - return cast(TerraformModule, hcl2.load(f)) + return cast(TerraformModule, terraform.hcl.load(path)) def read_cli_config(config: str) -> dict[str, str]: @@ -93,7 +92,7 @@ def read_cli_config(config: str) -> dict[str, str]: hosts = {} - config_hcl = hcl2.loads(config) + config_hcl = terraform.hcl.loads(config) for credential in config_hcl.get('credentials', {}): for cred_hostname, cred_conf in credential.items(): diff --git a/tests/workflows/test-validate/unterminated-string/main.tf b/tests/workflows/test-validate/unterminated-string/main.tf new file mode 100644 index 00000000..84efaccf --- /dev/null +++ b/tests/workflows/test-validate/unterminated-string/main.tf @@ -0,0 +1,10 @@ +module "enforce_mfa" { + source = "terraform-module/enforce-mfa/aws" + version = "0.13.0” + policy_name = "managed-mfa-enforce" + account_id = data.aws_caller_identity.current.id + groups = [aws_iam_group.console_group.name] + manage_own_signing_certificates = true + manage_own_ssh_public_keys = true + manage_own_git_credentials = true + } From b0d2672f6bc26f4ee28bc5032749c3dd47c83b65 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 18:12:01 +0100 Subject: [PATCH 221/231] Try loading hcl files in another process To mitigate hangs from unterminated strings --- .github/workflows/test-validate.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/test-validate.yaml b/.github/workflows/test-validate.yaml index 91ec8f16..d2396fc6 100644 --- a/.github/workflows/test-validate.yaml +++ b/.github/workflows/test-validate.yaml @@ -112,3 +112,17 @@ jobs: uses: ./terraform-validate with: path: tests/workflows/test-validate/unterminated-string + id: validate + continue-on-error: true + + - name: Check invalid + run: | + if [[ "${{ steps.validate.outcome }}" != "failure" ]]; then + echo "Validate did not fail correctly" + exit 1 + fi + + if [[ "${{ steps.validate.outputs.failure-reason }}" != "validate-failed" ]]; then + echo "::error:: failure-reason not set correctly" + exit 1 + fi From 9683a53ccf8138cbd0a47be71b9124245d2bab37 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 19:40:59 +0100 Subject: [PATCH 222/231] :bookmark: v1.23.0 --- CHANGELOG.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deb1cda6..39723fad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,25 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.22.2` to use an exact release -- `@v1.22` to use the latest patch release for the specific minor version +- `@v1.23.0` to use an exact release +- `@v1.23` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.23.0] - 2022-05-02 + +### Changes +- Input variables no longer help identify the plan comment. Each PR comment is still identified by it's configured terraform backend state file. + This is a very subtle change but enables better reporting of why an apply operation is aborted, e.g. "plan has changed" vs "plan not found". + + This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same root module but with different variables, you should ensure they use different `label`s. + +- The workflow output when an apply has been aborted because of changes in the plan has been clarified - thanks [toast-gear](https://github.com/toast-gear)! + +### Fixed +- Pre-release terraform versions now won't be used when selecting the latest terraform version. +- Invalid terraform files that contained an unterminated string would take an extremely long time to parse before failing the job. +- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) now automatically sets `terraform.workspace` to `default` when validating a module that uses a `remote` or `cloud` backend. + ## [1.22.2] - 2022-02-28 ### Fixed @@ -374,6 +389,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.23.0]: https://github.com/dflook/terraform-github-actions/compare/v1.22.2...v1.23.0 [1.22.2]: https://github.com/dflook/terraform-github-actions/compare/v1.22.1...v1.22.2 [1.22.1]: https://github.com/dflook/terraform-github-actions/compare/v1.22.0...v1.22.1 [1.22.0]: https://github.com/dflook/terraform-github-actions/compare/v1.21.1...v1.22.0 From 0f1203a5c35026b982813de1ece5c2b1dd03a75c Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Mon, 2 May 2022 19:46:27 +0100 Subject: [PATCH 223/231] :pencil: typo --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39723fad..1a1ca8d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,11 +14,10 @@ When using an action you can specify the version as: ## [1.23.0] - 2022-05-02 -### Changes -- Input variables no longer help identify the plan comment. Each PR comment is still identified by it's configured terraform backend state file. - This is a very subtle change but enables better reporting of why an apply operation is aborted, e.g. "plan has changed" vs "plan not found". +### Changed +- Input variables no longer help identify the plan comment. Each PR comment is still identified by it's configured terraform backend state file. This is a very subtle change but enables better reporting of why an apply operation is aborted, e.g. "plan has changed" vs "plan not found". - This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same root module but with different variables, you should ensure they use different `label`s. + This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same root module & backend but with different variables, you should ensure they use different `label`s. - The workflow output when an apply has been aborted because of changes in the plan has been clarified - thanks [toast-gear](https://github.com/toast-gear)! From f8a7b4f04f59dccec348afb71c32392d2a18bac7 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 3 May 2022 22:35:32 +0100 Subject: [PATCH 224/231] Output the number if resources that would be changed by the plan --- .github/workflows/test-apply.yaml | 31 ++++++ .github/workflows/test-plan.yaml | 159 +++++++++++++++++++++++++++ image/entrypoints/plan.sh | 2 + image/setup.py | 3 +- image/src/github_actions/commands.py | 6 + image/src/plan_summary/__init__.py | 0 image/src/plan_summary/__main__.py | 31 ++++++ terraform-plan/README.md | 24 ++++ terraform-plan/action.yaml | 6 + 9 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 image/src/github_actions/commands.py create mode 100644 image/src/plan_summary/__init__.py create mode 100644 image/src/plan_summary/__main__.py diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index 046c557b..c68d7406 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -671,11 +671,42 @@ jobs: - name: Plan 2 uses: ./terraform-plan + id: plan with: label: test-apply apply_refresh 2 path: tests/workflows/test-apply/refresh_15 variables: len=20 + - name: Verify outputs + run: | + echo "changes=${{ steps.plan.outputs.changes }}" + + if [[ "${{ steps.plan.outputs.changes }}" != "true" ]]; then + echo "::error:: output changes not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 1 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 1 + run: | + echo "if expression should not have evaluated true" + exit 1 + - name: Apply 2 uses: ./terraform-apply id: output diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index d091503a..fec1052e 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -39,6 +39,12 @@ jobs: exit 1 fi + - name: Test output expressions + if: steps.plan.outputs.to_add != 0 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + no_changes_no_comment: runs-on: ubuntu-latest name: No changes without comment @@ -102,6 +108,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_comment_12: runs-on: ubuntu-latest name: Change terraform 12 @@ -136,6 +163,27 @@ jobs: echo "::error:: text_plan_path not set correctly" exit 1 fi + + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 plan_change_comment_13: runs-on: ubuntu-latest @@ -172,6 +220,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_comment_14: runs-on: ubuntu-latest name: Change terraform 14 @@ -208,6 +277,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_comment_15: runs-on: ubuntu-latest name: Change terraform 15 @@ -244,6 +334,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_comment_15_4: runs-on: ubuntu-latest name: Change terraform 15.4 @@ -279,6 +390,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_comment_latest: runs-on: ubuntu-latest name: Change latest terraform @@ -314,6 +446,27 @@ jobs: exit 1 fi + if [[ ${{ steps.plan.outputs.to_add }} -ne 1 ]]; then + echo "::error:: to_add not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_change }} -ne 0 ]]; then + echo "::error:: to_change not set correctly" + exit 1 + fi + + if [[ ${{ steps.plan.outputs.to_destroy }} -ne 0 ]]; then + echo "::error:: to_destroy not set correctly" + exit 1 + fi + + - name: Test output expressions + if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + plan_change_no_comment: runs-on: ubuntu-latest name: Change without github comment @@ -376,6 +529,12 @@ jobs: exit 1 fi + - name: Test output expressions + if: steps.plan.outputs.to_add != 0 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 + run: | + echo "if expression should not have evaluated true" + exit 1 + error_no_comment: runs-on: ubuntu-latest name: Error without comment diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index c35b6455..12eec4fc 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -72,6 +72,8 @@ elif [[ $PLAN_EXIT -eq 0 ]]; then elif [[ $PLAN_EXIT -eq 2 ]]; then debug_log "Changes to apply" set_output changes true + + plan_summary "$STEP_TMP_DIR/plan.txt" fi mkdir -p "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR" diff --git a/image/setup.py b/image/setup.py index 92ed5122..778cb3a2 100644 --- a/image/setup.py +++ b/image/setup.py @@ -11,7 +11,8 @@ 'terraform-backend=terraform_backend.__main__:main', 'terraform-version=terraform_version.__main__:main', 'terraform-cloud-workspace=terraform_cloud_workspace.__main__:main', - 'github_pr_comment=github_pr_comment.__main__:main' + 'github_pr_comment=github_pr_comment.__main__:main', + 'plan_summary=plan_summary.__main__:main' ] }, install_requires=[ diff --git a/image/src/github_actions/commands.py b/image/src/github_actions/commands.py new file mode 100644 index 00000000..bff567b6 --- /dev/null +++ b/image/src/github_actions/commands.py @@ -0,0 +1,6 @@ +import sys +from typing import Any + + +def output(name: str, value: Any) -> None: + sys.stdout.write(f'::set-output name={name}::{value}\n') diff --git a/image/src/plan_summary/__init__.py b/image/src/plan_summary/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/image/src/plan_summary/__main__.py b/image/src/plan_summary/__main__.py new file mode 100644 index 00000000..44c8f224 --- /dev/null +++ b/image/src/plan_summary/__main__.py @@ -0,0 +1,31 @@ +""" +Create plan summary actions outputs + +Creates the outputs: +- to_add +- to_change +- to_destroy + +Usage: + plan_summary +""" + +from __future__ import annotations + +import re +import sys +from github_actions.commands import output + +def main() -> None: + """Entrypoint for terraform-backend""" + + with open(sys.argv[1]) as f: + plan = f.read() + + if match := re.search(r'^Plan: (\d+) to add, (\d+) to change, (\d+) to destroy', plan, re.MULTILINE): + output('to_add', match[1]) + output('to_change', match[2]) + output('to_destroy', match[3]) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 7bc49590..c8c7bc8c 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -317,6 +317,8 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ resources would change. With terraform >=0.13 this is correctly set to 'true' whenever an apply needs to be run. + - Type: boolean + * `json_plan_path` This is the path to the generated plan in [JSON Output Format](https://www.terraform.io/docs/internals/json-format.html) @@ -325,11 +327,33 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ This is not available when using terraform 0.11 or earlier. This also won't be set if the backend type is `remote` - Terraform does not support saving remote plans. + - Type: string + * `text_plan_path` This is the path to the generated plan in a human-readable format. The path is relative to the Actions workspace. + - Type: string + +* `to_add` + + The number of resources that would be added by this plan. + + - Type: number + +* `to_change` + + The number of resources that would be changed by this plan. + + - Type: number + +* `to_destroy` + + The number of resources that would be destroyed by this plan. + + - Type: number + ## Example usage ### Automatically generating a plan diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 544477f2..9880f714 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -53,6 +53,12 @@ inputs: outputs: changes: description: If the generated plan would update any resources or outputs this is set to `true`, otherwise it's set to `false`. + to_add: + description: The number of resources that would be added by this plan + to_change: + description: The number of resources that would be changed by this plan + to_destroy: + description: The number of resources that would be destroyed by this plan text_plan_path: description: Path to a file in the workspace containing the generated plan in human readble format. json_plan_path: From 22daf28c1820ca16f30c92d20c3237c5ca71b089 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 3 May 2022 23:04:36 +0100 Subject: [PATCH 225/231] :bookmark: v1.24.0 --- CHANGELOG.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1ca8d3..88597eb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,16 +8,23 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.23.0` to use an exact release -- `@v1.23` to use the latest patch release for the specific minor version +- `@v1.24.0` to use an exact release +- `@v1.24` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.24.0] - 2022-05-03 + +### Added +- New `to_add`, `to_change` and `to_destroy` outputs for the [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action that contain the number of resources that would be added, changed or deleted by the plan. + + These can be used in an [if expression](https://docs.github.com/en/enterprise-server@3.2/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idif) in a workflow to conditionally run steps, e.g. when the plan would destroy something. + ## [1.23.0] - 2022-05-02 ### Changed - Input variables no longer help identify the plan comment. Each PR comment is still identified by it's configured terraform backend state file. This is a very subtle change but enables better reporting of why an apply operation is aborted, e.g. "plan has changed" vs "plan not found". - This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same root module & backend but with different variables, you should ensure they use different `label`s. + This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same `path` and backend but with different variables, you should ensure they use different `label`s. - The workflow output when an apply has been aborted because of changes in the plan has been clarified - thanks [toast-gear](https://github.com/toast-gear)! @@ -388,6 +395,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.24.0]: https://github.com/dflook/terraform-github-actions/compare/v1.23.0...v1.24.0 [1.23.0]: https://github.com/dflook/terraform-github-actions/compare/v1.22.2...v1.23.0 [1.22.2]: https://github.com/dflook/terraform-github-actions/compare/v1.22.1...v1.22.2 [1.22.1]: https://github.com/dflook/terraform-github-actions/compare/v1.22.0...v1.22.1 From 4bc7e8d6dbc5b13c8d370d22d23531328d670541 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Wed, 4 May 2022 08:58:37 +0100 Subject: [PATCH 226/231] Rename default branch --- .github/release_template.md | 2 +- .github/workflows/base-image.yaml | 2 +- CHANGELOG.md | 36 +++++++++++++-------------- README.md | 10 ++++---- example_workflows/apply_plan.yaml | 2 +- example_workflows/fix_formatting.yaml | 2 +- example_workflows/validate.yaml | 2 +- terraform-apply/README.md | 18 +++++++------- terraform-fmt-check/README.md | 6 ++--- terraform-fmt/README.md | 10 ++++---- terraform-plan/README.md | 2 +- terraform-remote-state/README.md | 4 +-- terraform-validate/README.md | 6 ++--- 13 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.github/release_template.md b/.github/release_template.md index b8255446..71bdbfe7 100644 --- a/.github/release_template.md +++ b/.github/release_template.md @@ -1,6 +1,6 @@ This is one of a suite of terraform related actions - find them at [dflook/terraform-github-actions](https://github.com/dflook/terraform-github-actions). -You can see the changes for this release in the [CHANGELOG](https://github.com/dflook/terraform-github-actions/blob/master/CHANGELOG.md) +You can see the changes for this release in the [CHANGELOG](https://github.com/dflook/terraform-github-actions/blob/main/CHANGELOG.md) You can specify the action version as: diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index fc74a771..7349ff1c 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -3,7 +3,7 @@ name: Update base image on: push: branches: - - master + - main paths: - image/Dockerfile-base - .github/workflows/base-image.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 88597eb5..1a779186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ When using an action you can specify the version as: ## [1.24.0] - 2022-05-03 ### Added -- New `to_add`, `to_change` and `to_destroy` outputs for the [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action that contain the number of resources that would be added, changed or deleted by the plan. +- New `to_add`, `to_change` and `to_destroy` outputs for the [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) action that contain the number of resources that would be added, changed or deleted by the plan. These can be used in an [if expression](https://docs.github.com/en/enterprise-server@3.2/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idif) in a workflow to conditionally run steps, e.g. when the plan would destroy something. @@ -24,20 +24,20 @@ When using an action you can specify the version as: ### Changed - Input variables no longer help identify the plan comment. Each PR comment is still identified by it's configured terraform backend state file. This is a very subtle change but enables better reporting of why an apply operation is aborted, e.g. "plan has changed" vs "plan not found". - This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) action for the same `path` and backend but with different variables, you should ensure they use different `label`s. + This means that if you have more than one [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) action for the same `path` and backend but with different variables, you should ensure they use different `label`s. - The workflow output when an apply has been aborted because of changes in the plan has been clarified - thanks [toast-gear](https://github.com/toast-gear)! ### Fixed - Pre-release terraform versions now won't be used when selecting the latest terraform version. - Invalid terraform files that contained an unterminated string would take an extremely long time to parse before failing the job. -- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) now automatically sets `terraform.workspace` to `default` when validating a module that uses a `remote` or `cloud` backend. +- [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/main/terraform-validate) now automatically sets `terraform.workspace` to `default` when validating a module that uses a `remote` or `cloud` backend. ## [1.22.2] - 2022-02-28 ### Fixed - The PR plan comment was incorrectly including resource refresh lines when there were changes to outputs but not resources, while using Terraform >=0.15.4. As well as being noisy, this could lead to failures to apply due to incorrectly detecting changes in the plan. -- Removed incorrect deprecation warning in [dflook/terraform-destroy](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy). Thanks [dgrenner](https://github.com/dgrenner)! +- Removed incorrect deprecation warning in [dflook/terraform-destroy](https://github.com/dflook/terraform-github-actions/tree/main/terraform-destroy). Thanks [dgrenner](https://github.com/dgrenner)! ## [1.22.1] - 2022-01-24 @@ -48,9 +48,9 @@ When using an action you can specify the version as: ### Added - Workspace management for Terraform Cloud/Enterprise has been reimplemented to avoid issues with the `terraform workspace` command when using the `remote` backend or a cloud config block: - - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) can now create the first workspace - - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace) can now delete the last remaining workspace - - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) and [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-destroy-workspace) work with a `remote` backend that specifies a workspace by `name` + - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/main/terraform-new-workspace) can now create the first workspace + - [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/main/terraform-destroy-workspace) can now delete the last remaining workspace + - [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/main/terraform-new-workspace) and [dflook/terraform-destroy-workspace](https://github.com/dflook/terraform-github-actions/tree/main/terraform-destroy-workspace) work with a `remote` backend that specifies a workspace by `name` - The terraform version to use will now be detected from additional places: @@ -61,35 +61,35 @@ When using an action you can specify the version as: The best way to specify the version is using a [`required_version`](https://www.terraform.io/docs/configuration/terraform.html#specifying-a-required-terraform-version) constraint. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) docs for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) docs for details. ### Changed As a result of the above terraform version detection additions, note these changes: -- Actions always use the terraform version set in the remote workspace when using TFC/E, if it exists. This mostly effects [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate). +- Actions always use the terraform version set in the remote workspace when using TFC/E, if it exists. This mostly effects [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/main/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/main/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/main/terraform-validate). - If the terraform version is not specified anywhere then new workspaces will be created with the latest terraform version. Existing workspaces will use the terraform version that was last used for that workspace. - If you want to always use the latest terraform version, instead of not specifying a version you now need to set an open-ended version constraint (e.g. `>1.0.0`) -- All actions now support the inputs and environment variables related to the backend, for discovering the terraform version from a TFC/E workspace or remote state. This add the inputs `workspace`, `backend_config`, `backend_config_file`, and the `TERRAFORM_CLOUD_TOKENS` environment variable to the [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/master/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) actions. +- All actions now support the inputs and environment variables related to the backend, for discovering the terraform version from a TFC/E workspace or remote state. This add the inputs `workspace`, `backend_config`, `backend_config_file`, and the `TERRAFORM_CLOUD_TOKENS` environment variable to the [dflook/terraform-fmt](https://github.com/dflook/terraform-github-actions/tree/main/terraform-fmt), [dflook/terraform-fmt-check](https://github.com/dflook/terraform-github-actions/tree/main/terraform-fmt-check) and [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/main/terraform-validate) actions. - :warning: Some unused packages were removed from the container image, most notably Python 2. ## [1.21.1] - 2021-12-12 ### Fixed -- [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/master/terraform-new-workspace) support for Terraform v1.1.0. +- [dflook/terraform-new-workspace](https://github.com/dflook/terraform-github-actions/tree/main/terraform-new-workspace) support for Terraform v1.1.0. This stopped working after a change in the behaviour of terraform init. There is an outstanding [issue in Terraform v1.1.0](https://github.com/hashicorp/terraform/issues/30129) using the `remote` backend that prevents creating a new workspace when no workspaces currently exist. - If you are affected by this, you can pin to an earlier version of Terraform using one of methods listed in the [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) docs. + If you are affected by this, you can pin to an earlier version of Terraform using one of methods listed in the [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) docs. ## [1.21.0] - 2021-12-04 ### Added -- A new `workspace` input for [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/master/terraform-validate) +- A new `workspace` input for [dflook/terraform-validate](https://github.com/dflook/terraform-github-actions/tree/main/terraform-validate) allows validating usage of `terraform.workspace` in the terraform code. Terraform doesn't initialize `terraform.workspace` based on the backend configuration when running a validate operation. @@ -103,12 +103,12 @@ As a result of the above terraform version detection additions, note these chang ## [1.20.0] - 2021-12-03 ### Added -- New `text_plan_path` and `json_plan_path` outputs for [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) - to match the outputs for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan). +- New `text_plan_path` and `json_plan_path` outputs for [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply) + to match the outputs for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan). These are paths to the generated plan in human-readable and JSON formats. - If the plan generated by [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) is different from the plan generated by [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) the apply step will fail with `failure-reason` set to `plan-changed`. + If the plan generated by [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) is different from the plan generated by [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply) the apply step will fail with `failure-reason` set to `plan-changed`. These new outputs make it easier to inspect the differences. ## [1.19.0] - 2021-11-01 @@ -121,7 +121,7 @@ As a result of the above terraform version detection additions, note these chang ## [1.18.0] - 2021-10-30 ### Added -- A new `replace` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#inputs) and [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply#inputs) +- A new `replace` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan#inputs) and [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply#inputs) This instructs terraform to replace the specified resources, and is available with terraform versions that support replace (v0.15.2 onwards). @@ -131,7 +131,7 @@ As a result of the above terraform version detection additions, note these chang random_password.database ``` -- A `target` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan#inputs) to match [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply#inputs) +- A `target` input for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan#inputs) to match [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply#inputs) `target` limits the plan to the specified resources and their dependencies. This change removes the restriction that `target` can only be used with `auto_approve`. diff --git a/README.md b/README.md index eb6b29d7..32d5b4e1 100644 --- a/README.md +++ b/README.md @@ -67,14 +67,14 @@ jobs: ``` #### apply.yaml -This workflow runs when the PR is merged into the master branch, and applies the planned changes. +This workflow runs when the PR is merged into the main branch, and applies the planned changes. ```yaml name: Apply terraform plan on: push: branches: - - master + - main jobs: apply: @@ -93,7 +93,7 @@ jobs: ``` ### Linting -This workflow runs on every push to non-master branches and checks the terraform configuration is valid. +This workflow runs on every push to non-main branches and checks the terraform configuration is valid. For extra strictness, we check the files are in the canonical format.

@@ -109,7 +109,7 @@ name: Lint on: push: branches: - - '!master' + - '!main' jobs: validate: @@ -208,7 +208,7 @@ name: Check terraform file formatting on: push: branches: - - master + - main jobs: format: diff --git a/example_workflows/apply_plan.yaml b/example_workflows/apply_plan.yaml index 9a1127c7..268d3441 100644 --- a/example_workflows/apply_plan.yaml +++ b/example_workflows/apply_plan.yaml @@ -3,7 +3,7 @@ name: Apply plan on: push: branches: - - master + - main jobs: plan: diff --git a/example_workflows/fix_formatting.yaml b/example_workflows/fix_formatting.yaml index faabd9c5..3a8b1753 100644 --- a/example_workflows/fix_formatting.yaml +++ b/example_workflows/fix_formatting.yaml @@ -3,7 +3,7 @@ name: Fix terraform formatting on: push: branches: - - master + - main jobs: fix_formatting: diff --git a/example_workflows/validate.yaml b/example_workflows/validate.yaml index 25401e0b..25b71e33 100644 --- a/example_workflows/validate.yaml +++ b/example_workflows/validate.yaml @@ -3,7 +3,7 @@ name: Validate changes on: push: branches: - - '!master' + - '!main' jobs: fmt-check: diff --git a/terraform-apply/README.md b/terraform-apply/README.md index 1982fc20..f0953b76 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -11,7 +11,7 @@ This is to ensure that the action only applies changes that have been reviewed b You can instead set `auto_approve: true` which will generate a plan and apply it immediately, without looking for a plan attached to a PR. ## Demo -This a demo of the process for apply a terraform change using the [`dflook/terraform-plan`](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) and [`dflook/terraform-apply`](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) actions. +This a demo of the process for apply a terraform change using the [`dflook/terraform-plan`](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) and [`dflook/terraform-apply`](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply) actions.

@@ -22,7 +22,7 @@ This a demo of the process for apply a terraform change using the [`dflook/terra To make best use of this action, require that the plan is always reviewed before merging the PR to approve. You can enforce this in github by going to the branch settings for the repo and enable protection for -the master branch: +the main branch: 1. Enable 'Require pull request reviews before merging' 2. Check 'Dismiss stale pull request approvals when new commits are pushed' @@ -238,7 +238,7 @@ These input values must be the same as any `terraform-plan` for the same configu ``` Running this action will produce a `service_hostname` output with the same value. - See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/master/terraform-output) for details. + See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/main/terraform-output) for details. ## Environment Variables @@ -351,7 +351,7 @@ These input values must be the same as any `terraform-plan` for the same configu ### Apply PR approved plans -This example workflow runs for every push to master. If the commit +This example workflow runs for every push to main. If the commit came from a PR that has been merged, applies the plan from the PR. ```yaml @@ -360,7 +360,7 @@ name: Apply on: push: branches: - - master + - main jobs: apply: @@ -380,7 +380,7 @@ jobs: ### Always apply changes -This example workflow runs for every push to master. +This example workflow runs for every push to main. Changes are planned and applied. ```yaml @@ -389,7 +389,7 @@ name: Apply on: push: branches: - - master + - main jobs: apply: @@ -440,7 +440,7 @@ jobs: This workflow applies a plan on demand, triggered by someone commenting `terraform apply` on the PR. The plan is taken -from an existing comment generated by the [`dflook/terraform-plan`](https://github.com/dflook/terraform-github-actions/tree/master/terraform-plan) +from an existing comment generated by the [`dflook/terraform-plan`](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) action. ```yaml @@ -475,7 +475,7 @@ name: Apply plan on: push: branches: - - master + - main jobs: plan: diff --git a/terraform-fmt-check/README.md b/terraform-fmt-check/README.md index 57dc1a44..368239e2 100644 --- a/terraform-fmt-check/README.md +++ b/terraform-fmt-check/README.md @@ -20,7 +20,7 @@ If any files are not correctly formatted a failing GitHub check will be added fo * `workspace` Terraform workspace to inspect when discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. - Type: string - Optional @@ -28,7 +28,7 @@ If any files are not correctly formatted a failing GitHub check will be added fo * `backend_config` List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. ```yaml with: @@ -41,7 +41,7 @@ If any files are not correctly formatted a failing GitHub check will be added fo * `backend_config_file` List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. Paths should be relative to the GitHub Actions workspace ```yaml diff --git a/terraform-fmt/README.md b/terraform-fmt/README.md index 3444ae0b..1b6a2a1e 100644 --- a/terraform-fmt/README.md +++ b/terraform-fmt/README.md @@ -17,7 +17,7 @@ This action uses the `terraform fmt` command to reformat files in a directory in * `workspace` Terraform workspace to inspect when discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. - Type: string - Optional @@ -25,7 +25,7 @@ This action uses the `terraform fmt` command to reformat files in a directory in * `backend_config` List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. ```yaml with: @@ -38,7 +38,7 @@ This action uses the `terraform fmt` command to reformat files in a directory in * `backend_config_file` List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. Paths should be relative to the GitHub Actions workspace ```yaml @@ -77,7 +77,7 @@ This action uses the `terraform fmt` command to reformat files in a directory in ## Example usage This example automatically creates a pull request to fix any formatting -problems that get merged into the master branch. +problems that get merged into the main branch. ```yaml name: Fix terraform file formatting @@ -85,7 +85,7 @@ name: Fix terraform file formatting on: push: branches: - - master + - main jobs: format: diff --git a/terraform-plan/README.md b/terraform-plan/README.md index c8c7bc8c..945a9182 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -12,7 +12,7 @@ If the triggering event relates to a PR it will add a comment on the PR containi The `GITHUB_TOKEN` environment variable must be set for the PR comment to be added. The action can be run on other events, which prints the plan to the workflow log. -The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/master/terraform-apply) action can be used to apply the generated plan. +The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply) action can be used to apply the generated plan. ## Inputs diff --git a/terraform-remote-state/README.md b/terraform-remote-state/README.md index caedde82..7a6dd846 100644 --- a/terraform-remote-state/README.md +++ b/terraform-remote-state/README.md @@ -81,7 +81,7 @@ output "service_hostname" { } ``` Running this action will produce a `service_hostname` output with the same value. -See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/master/terraform-output) for details. +See [terraform-output](https://github.com/dflook/terraform-github-actions/tree/main/terraform-output) for details. ## Example usage @@ -93,7 +93,7 @@ name: Send request on: push: branches: - - master + - main env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/terraform-validate/README.md b/terraform-validate/README.md index 377c2670..75ff19de 100644 --- a/terraform-validate/README.md +++ b/terraform-validate/README.md @@ -28,7 +28,7 @@ If the terraform configuration is not valid, the build is failed. Terraform workspace to use for the `terraform.workspace` value while validating. Note that for remote operations in Terraform Cloud/Enterprise, this is always `default`. Also used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. - Type: string - Optional @@ -37,7 +37,7 @@ If the terraform configuration is not valid, the build is failed. * `backend_config` List of terraform backend config values, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. ```yaml with: @@ -50,7 +50,7 @@ If the terraform configuration is not valid, the build is failed. * `backend_config_file` List of terraform backend config files to use, one per line. This is used for discovering the terraform version to use, if not otherwise specified. - See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/master/terraform-version#terraform-version-action) for details. + See [dflook/terraform-version](https://github.com/dflook/terraform-github-actions/tree/main/terraform-version#terraform-version-action) for details. Paths should be relative to the GitHub Actions workspace ```yaml From bf5714590bbc232961cca03ce4d2091ffdb332d1 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 6 May 2022 14:26:26 +0100 Subject: [PATCH 227/231] set json_plan_path with remote execution --- .github/workflows/test-apply.yaml | 80 +++++++++++++++++++++ .github/workflows/test-cloud.yaml | 25 +++++++ .github/workflows/test-plan.yaml | 65 +++++++++++++++++ image/actions.sh | 2 +- image/entrypoints/apply.sh | 10 ++- image/entrypoints/plan.sh | 20 ++++++ image/setup.py | 4 +- image/src/terraform_cloud_state/__init__.py | 0 image/src/terraform_cloud_state/__main__.py | 64 +++++++++++++++++ terraform-apply/README.md | 6 ++ terraform-apply/action.yaml | 2 + terraform-plan/README.md | 7 +- terraform-plan/action.yaml | 6 +- tests/test_cloud_state.py | 45 ++++++++++++ 14 files changed, 330 insertions(+), 6 deletions(-) create mode 100644 image/src/terraform_cloud_state/__init__.py create mode 100644 image/src/terraform_cloud_state/__main__.py create mode 100644 tests/test_cloud_state.py diff --git a/.github/workflows/test-apply.yaml b/.github/workflows/test-apply.yaml index c68d7406..2dc5d954 100644 --- a/.github/workflows/test-apply.yaml +++ b/.github/workflows/test-apply.yaml @@ -38,6 +38,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_error: runs-on: ubuntu-latest name: Auto Approve plan error @@ -75,6 +80,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_apply_error: runs-on: ubuntu-latest name: Auto Approve apply phase error @@ -122,6 +132,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_no_token: runs-on: ubuntu-latest name: Apply without token @@ -148,6 +163,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply: runs-on: ubuntu-latest name: Apply approved changes @@ -187,6 +207,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.first-apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Apply uses: ./terraform-apply id: second-apply @@ -211,6 +236,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.second-apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_variables: runs-on: ubuntu-latest name: Apply approved changes with variables @@ -295,6 +325,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + backend_config_12: runs-on: ubuntu-latest name: backend_config terraform 12 @@ -338,6 +373,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.backend_config_file_12.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Plan uses: ./terraform-plan with: @@ -376,6 +416,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.backend_config_12.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + backend_config_13: runs-on: ubuntu-latest name: backend_config terraform 13 @@ -419,6 +464,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.backend_config_file_13.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Plan uses: ./terraform-plan with: @@ -457,6 +507,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.backend_config_13.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_label: runs-on: ubuntu-latest name: Apply approved with a variable and label @@ -500,6 +555,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_no_changes: runs-on: ubuntu-latest name: Apply when there are no planned changes @@ -535,6 +595,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_no_plan: runs-on: ubuntu-latest name: Apply when there is no approved plan @@ -568,6 +633,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_user_token: runs-on: ubuntu-latest name: Apply using a personal access token @@ -607,6 +677,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_vars: runs-on: ubuntu-latest name: Apply approved changes with deprecated vars @@ -646,6 +721,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.output.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + apply_refresh: runs-on: ubuntu-latest name: Apply changes are refresh diff --git a/.github/workflows/test-cloud.yaml b/.github/workflows/test-cloud.yaml index 5caca69d..dd553e8e 100644 --- a/.github/workflows/test-cloud.yaml +++ b/.github/workflows/test-cloud.yaml @@ -76,6 +76,12 @@ jobs: exit 1 fi + echo '${{ steps.auto_apply.outputs.run_id }}' + if [[ "${{ steps.auto_apply.outputs.run_id }}" != "run-"* ]]; then + echo "::error:: output run_id not set correctly" + exit 1 + fi + - name: Get outputs uses: ./terraform-output id: output @@ -183,6 +189,19 @@ jobs: exit 1 fi + echo '${{ steps.plan.outputs.run_id }}' + if [[ "${{ steps.plan.outputs.run_id }}" != "run-"* ]]; then + echo "::error:: output run_id not set correctly" + exit 1 + fi + + echo '${{ steps.plan.outputs.json_plan_path }}' + jq .output_changes.from_variables.actions[0] "${{ steps.plan.outputs.json_plan_path }}" + if [[ $(jq -r .output_changes.from_variables.actions[0] "${{ steps.plan.outputs.json_plan_path }}") != "create" ]]; then + echo "::error:: json_plan_path not set correctly" + exit 1 + fi + - name: Apply workspace uses: ./terraform-apply id: apply @@ -224,6 +243,12 @@ jobs: exit 1 fi + echo '${{ steps.apply.outputs.run_id }}' + if [[ "${{ steps.apply.outputs.run_id }}" != "run-"* ]]; then + echo "::error:: output run_id not set correctly" + exit 1 + fi + - name: Destroy the last workspace uses: ./terraform-destroy-workspace with: diff --git a/.github/workflows/test-plan.yaml b/.github/workflows/test-plan.yaml index fec1052e..638e1101 100644 --- a/.github/workflows/test-plan.yaml +++ b/.github/workflows/test-plan.yaml @@ -39,6 +39,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 0 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -74,6 +79,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + plan_change_comment_11: runs-on: ubuntu-latest name: Change terraform 11 @@ -123,6 +133,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -179,6 +194,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -235,6 +255,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -292,6 +317,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -349,6 +379,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -405,6 +440,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -461,6 +501,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 1 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -496,6 +541,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + error: runs-on: ubuntu-latest name: Error @@ -529,6 +579,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + - name: Test output expressions if: steps.plan.outputs.to_add != 0 || steps.plan.outputs.to_change != 0 || steps.plan.outputs.to_destroy != 0 run: | @@ -569,6 +624,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + plan_without_token: runs-on: ubuntu-latest name: Add comment without token @@ -600,6 +660,11 @@ jobs: exit 1 fi + if [[ -n "${{ steps.apply.outputs.run_id }}" ]]; then + echo "::error:: run_id should not be set" + exit 1 + fi + plan_single_variable: runs-on: ubuntu-latest name: Plan single variable diff --git a/image/actions.sh b/image/actions.sh index e74e6c34..7cec9ac4 100644 --- a/image/actions.sh +++ b/image/actions.sh @@ -370,7 +370,7 @@ function plan() { (cd "$INPUT_PATH" && terraform plan -input=false -no-color -detailed-exitcode -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT_ARG $PLAN_ARGS) \ 2>"$STEP_TMP_DIR/terraform_plan.stderr" \ | $TFMASK \ - | tee /dev/fd/3 \ + | tee /dev/fd/3 "$STEP_TMP_DIR/terraform_plan.stdout" \ | compact_plan \ >"$STEP_TMP_DIR/plan.txt" diff --git a/image/entrypoints/apply.sh b/image/entrypoints/apply.sh index 69cddc26..ac0d43da 100755 --- a/image/entrypoints/apply.sh +++ b/image/entrypoints/apply.sh @@ -33,8 +33,16 @@ function apply() { # shellcheck disable=SC2086 debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' # don't expand plan args # shellcheck disable=SC2086 - (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK + (cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK | tee "$STEP_TMP_DIR/terraform_apply.stdout" APPLY_EXIT=${PIPESTATUS[0]} + + if remote-run-id "$STEP_TMP_DIR/terraform_apply.stdout" >"$STEP_TMP_DIR/remote-run-id.stdout" 2>"$STEP_TMP_DIR/remote-run-id.stderr"; then + RUN_ID="$(<"$STEP_TMP_DIR/remote-run-id.stdout")" + set_output run_id "$RUN_ID" + else + debug_log "Failed to get remote run-id" + debug_file "$STEP_TMP_DIR/remote-run-id.stderr" + fi fi set -e diff --git a/image/entrypoints/plan.sh b/image/entrypoints/plan.sh index 12eec4fc..fa53c0d7 100755 --- a/image/entrypoints/plan.sh +++ b/image/entrypoints/plan.sh @@ -27,6 +27,16 @@ fi cat "$STEP_TMP_DIR/terraform_plan.stderr" +if [[ -z "$PLAN_OUT" ]]; then + if remote-run-id "$STEP_TMP_DIR/terraform_plan.stdout" >"$STEP_TMP_DIR/remote-run-id.stdout" 2>"$STEP_TMP_DIR/remote-run-id.stderr"; then + RUN_ID="$(<"$STEP_TMP_DIR/remote-run-id.stdout")" + set_output run_id "$RUN_ID" + else + debug_log "Failed to get remote run-id" + debug_file "$STEP_TMP_DIR/remote-run-id.stderr" + fi +fi + if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "issue_comment" || "$GITHUB_EVENT_NAME" == "pull_request_review_comment" || "$GITHUB_EVENT_NAME" == "pull_request_target" || "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then if [[ "$INPUT_ADD_GITHUB_COMMENT" == "true" || "$INPUT_ADD_GITHUB_COMMENT" == "changes-only" ]]; then @@ -86,4 +96,14 @@ if [[ -n "$PLAN_OUT" ]]; then else debug_file "$STEP_TMP_DIR/terraform_show.stderr" fi +elif [[ -n "$RUN_ID" ]]; then + if terraform-cloud-state "$RUN_ID" >"$STEP_TMP_DIR/terraform_cloud_state.stdout" 2>"$STEP_TMP_DIR/terraform_cloud_state.stderr"; then + debug_log "Fetched JSON plan from TFC" + cp "$STEP_TMP_DIR/terraform_cloud_state.stdout" "$GITHUB_WORKSPACE/$WORKSPACE_TMP_DIR/plan.json" + set_output json_plan_path "$WORKSPACE_TMP_DIR/plan.json" + else + debug_log "Failed to fetch JSON plan from TFC" + debug_file "$STEP_TMP_DIR/terraform_cloud_state.stdout" + debug_file "$STEP_TMP_DIR/terraform_cloud_state.stderr" + fi fi diff --git a/image/setup.py b/image/setup.py index 778cb3a2..1029f61d 100644 --- a/image/setup.py +++ b/image/setup.py @@ -12,7 +12,9 @@ 'terraform-version=terraform_version.__main__:main', 'terraform-cloud-workspace=terraform_cloud_workspace.__main__:main', 'github_pr_comment=github_pr_comment.__main__:main', - 'plan_summary=plan_summary.__main__:main' + 'plan_summary=plan_summary.__main__:main', + 'terraform-cloud-state=terraform_cloud_state.__main__:main', + 'remote-run-id=terraform_cloud_state.__main__:remote_run_id' ] }, install_requires=[ diff --git a/image/src/terraform_cloud_state/__init__.py b/image/src/terraform_cloud_state/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/image/src/terraform_cloud_state/__main__.py b/image/src/terraform_cloud_state/__main__.py new file mode 100644 index 00000000..92024716 --- /dev/null +++ b/image/src/terraform_cloud_state/__main__.py @@ -0,0 +1,64 @@ +import os +import re +import sys +from pathlib import Path +from typing import Optional + +from github_actions.commands import output +from terraform.cloud import TerraformCloudApi +from terraform.module import BackendConfig +from terraform.module import load_module, get_remote_backend_config, get_cloud_config + + +def get_run_id(plan: str) -> Optional[str]: + if match := re.search(r'https://.*/(?P[^/]*)/runs/(?Prun-.*)$', plan, re.MULTILINE): + return match[2] + + +def get_cloud_json_plan(backend_config: BackendConfig, run_id: str) -> bytes: + terraform_cloud = TerraformCloudApi(backend_config["hostname"], backend_config['token']) + response = terraform_cloud.get(f'runs/{run_id}/plan/json-output') + response.raise_for_status() + return response.content + +def remote_run_id(): + if len(sys.argv) < 2: + sys.stderr.write('Usage: remote-run-id \n') + sys.exit(1) + + with open(sys.argv[1]) as f: + run_id = get_run_id(f.read()) + + if run_id is None: + sys.stderr.write('run_id not found in plan\n') + sys.exit(1) + + sys.stdout.write(run_id) + +def main(): + if len(sys.argv) < 2: + sys.stderr.write('Usage: terraform-cloud-state RUN_ID\n') + sys.exit(1) + + module = load_module(Path(os.environ.get('INPUT_PATH', '.'))) + + backend_config = get_remote_backend_config( + module, + backend_config_files=os.environ.get('INPUT_BACKEND_CONFIG_FILE', ''), + backend_config_vars=os.environ.get('INPUT_BACKEND_CONFIG', ''), + cli_config_path=Path('~/.terraformrc'), + ) + + if backend_config is None: + backend_config = get_cloud_config( + module, + cli_config_path=Path('~/.terraformrc'), + ) + + run_id = sys.argv[1] + + sys.stdout.write(get_cloud_json_plan(backend_config, run_id).decode()) + sys.stdout.write('\n') + +if __name__ == '__main__': + main() diff --git a/terraform-apply/README.md b/terraform-apply/README.md index f0953b76..29b81281 100644 --- a/terraform-apply/README.md +++ b/terraform-apply/README.md @@ -226,6 +226,12 @@ These input values must be the same as any `terraform-plan` for the same configu If the job fails for any other reason this will not be set. This can be used with the Actions expression syntax to conditionally run steps. +* `run_id` + + If the root module uses the `remote` or `cloud` backend in remote execution mode, this output will be set to the remote run id. + + - Type: string + * Terraform Outputs An action output will be created for each output of the terraform configuration. diff --git a/terraform-apply/action.yaml b/terraform-apply/action.yaml index abba7447..5554583b 100644 --- a/terraform-apply/action.yaml +++ b/terraform-apply/action.yaml @@ -59,6 +59,8 @@ outputs: description: Path to a file in the workspace containing the generated plan in JSON format. This won't be set if the backend type is `remote`. failure-reason: description: The reason for the build failure. May be `apply-failed` or `plan-changed`. + run_id: + description: If the root module uses the `remote` or `cloud` backend in remote execution mode, this output will be set to the remote run id. runs: using: docker diff --git a/terraform-plan/README.md b/terraform-plan/README.md index 945a9182..a1a83103 100644 --- a/terraform-plan/README.md +++ b/terraform-plan/README.md @@ -325,7 +325,6 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ The path is relative to the Actions workspace. This is not available when using terraform 0.11 or earlier. - This also won't be set if the backend type is `remote` - Terraform does not support saving remote plans. - Type: string @@ -354,6 +353,12 @@ The [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/ - Type: number +* `run_id` + + If the root module uses the `remote` or `cloud` backend in remote execution mode, this output will be set to the remote run id. + + - Type: string + ## Example usage ### Automatically generating a plan diff --git a/terraform-plan/action.yaml b/terraform-plan/action.yaml index 9880f714..dd662601 100644 --- a/terraform-plan/action.yaml +++ b/terraform-plan/action.yaml @@ -60,9 +60,11 @@ outputs: to_destroy: description: The number of resources that would be destroyed by this plan text_plan_path: - description: Path to a file in the workspace containing the generated plan in human readble format. + description: Path to a file in the workspace containing the generated plan in human readable format. json_plan_path: - description: Path to a file in the workspace containing the generated plan in JSON format. This won't be set if the backend type is `remote`. + description: Path to a file in the workspace containing the generated plan in JSON format. + run_id: + description: If the root module uses the `remote` or `cloud` backend in remote execution mode, this output will be set to the remote run id. runs: using: docker diff --git a/tests/test_cloud_state.py b/tests/test_cloud_state.py new file mode 100644 index 00000000..8514a843 --- /dev/null +++ b/tests/test_cloud_state.py @@ -0,0 +1,45 @@ +from terraform_cloud_state.__main__ import get_run_id, get_cloud_json_plan + + +def test_get_run_url(): + plan = """ +Running plan in the remote backend. Output will stream here. Pressing Ctrl-C +will stop streaming the logs, but will not stop the plan running remotely. + +Preparing the remote plan... + +To view this run in a browser, visit: +https://app.terraform.io/app/flooktech/github-actions-1-1temp/runs/run-6m9eAyLdeDSrPYqz + +Waiting for the plan to start... + +Terraform v1.1.6 +on linux_amd64 +Initializing plugins and modules... + +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # random_id.the_id will be created + + resource "random_id" "the_id" { + + b64_std = (known after apply) + + b64_url = (known after apply) + + byte_length = 5 + + dec = (known after apply) + + hex = (known after apply) + + id = (known after apply) + } + +Plan: 1 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + default = "default" + + from_tfvars = "default" + + from_variables = "default" + +""" + + assert get_run_id(plan) == 'run-6m9eAyLdeDSrPYqz' From 9b444defed66e9b0460ce76884bdec224be65d53 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Fri, 6 May 2022 17:58:09 +0100 Subject: [PATCH 228/231] :bookmark: v1.25.0 --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a779186..517aa99d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,16 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.24.0` to use an exact release -- `@v1.24` to use the latest patch release for the specific minor version +- `@v1.25.0` to use an exact release +- `@v1.25` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.25.0] - 2022-05-06 + +### Added +- New `run_id` output for [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) and [dflook/terraform-apply](https://github.com/dflook/terraform-github-actions/tree/main/terraform-apply) which are set when using Terraform Cloud/Enterprise. It is the remote run-id of the plan or apply operation. +- The `json_plan_path` output of [dflook/terraform-plan](https://github.com/dflook/terraform-github-actions/tree/main/terraform-plan) now works when using Terraform Cloud/Enterprise. + ## [1.24.0] - 2022-05-03 ### Added @@ -395,6 +401,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.25.0]: https://github.com/dflook/terraform-github-actions/compare/v1.24.0...v1.25.0 [1.24.0]: https://github.com/dflook/terraform-github-actions/compare/v1.23.0...v1.24.0 [1.23.0]: https://github.com/dflook/terraform-github-actions/compare/v1.22.2...v1.23.0 [1.22.2]: https://github.com/dflook/terraform-github-actions/compare/v1.22.1...v1.22.2 From fa9a9f194521a5101abe07f5b9dca926ab7cb536 Mon Sep 17 00:00:00 2001 From: Kyle Lacy Date: Mon, 9 May 2022 13:33:47 -0700 Subject: [PATCH 229/231] Fix version regex not matching version links --- image/src/terraform/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/src/terraform/versions.py b/image/src/terraform/versions.py index 8d3ac7ee..7f0ea901 100644 --- a/image/src/terraform/versions.py +++ b/image/src/terraform/versions.py @@ -214,7 +214,7 @@ def get_terraform_versions() -> Iterable[Version]: response = session.get('https://releases.hashicorp.com/terraform/') response.raise_for_status() - version_regex = re.compile(br'/(\d+\.\d+\.\d+(-[\d\w]+)?)/') + version_regex = re.compile(br'/(\d+\.\d+\.\d+(-[\d\w]+)?)') for version in version_regex.finditer(response.content): yield Version(version.group(1).decode()) From 3a263fe294400ada77cc2eeb1326a9e7b8e9facc Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 10 May 2022 01:11:20 +0100 Subject: [PATCH 230/231] Test terraform version scraping --- .github/github_sucks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/github_sucks.md b/.github/github_sucks.md index cee65b7f..9f9f9874 100644 --- a/.github/github_sucks.md +++ b/.github/github_sucks.md @@ -1,3 +1,4 @@ Everytime I need to generate a push or synchronise event I will touch this file. This is usually because GitHub Actions has broken in some way. + From 3972b2ebbe82e953d4aea4d17b5e241a351ae847 Mon Sep 17 00:00:00 2001 From: Daniel Flook Date: Tue, 10 May 2022 01:36:10 +0100 Subject: [PATCH 231/231] :bookmark: v1.25.1 --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 517aa99d..24de7bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,15 @@ The actions are versioned as a suite. Some actions may have no change in behavio When using an action you can specify the version as: -- `@v1.25.0` to use an exact release +- `@v1.25.1` to use an exact release - `@v1.25` to use the latest patch release for the specific minor version - `@v1` to use the latest patch release for the specific major version +## [1.25.1] - 2022-05-10 + +### Fixed +- Failure to install terraform after change in the download page - Thanks [kylewlacy](https://github.com/kylewlacy) + ## [1.25.0] - 2022-05-06 ### Added @@ -401,6 +406,7 @@ First release of the GitHub Actions: - [dflook/terraform-new-workspace](terraform-new-workspace) - [dflook/terraform-destroy-workspace](terraform-destroy-workspace) +[1.25.1]: https://github.com/dflook/terraform-github-actions/compare/v1.25.0...v1.25.1 [1.25.0]: https://github.com/dflook/terraform-github-actions/compare/v1.24.0...v1.25.0 [1.24.0]: https://github.com/dflook/terraform-github-actions/compare/v1.23.0...v1.24.0 [1.23.0]: https://github.com/dflook/terraform-github-actions/compare/v1.22.2...v1.23.0