From d4a47c502d8db9ccce143c930d8ae4d60c14cc4f Mon Sep 17 00:00:00 2001 From: tyler Date: Thu, 7 Dec 2023 23:57:54 +0000 Subject: [PATCH 01/14] Provision an ECS cluster --- terraform/ecs_cluster.tf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 terraform/ecs_cluster.tf diff --git a/terraform/ecs_cluster.tf b/terraform/ecs_cluster.tf new file mode 100644 index 00000000..4fc47500 --- /dev/null +++ b/terraform/ecs_cluster.tf @@ -0,0 +1,26 @@ +resource "aws_ecs_cluster" "default" { + name = var.namespace + + setting { + name = "containerInsights" + value = var.ecs_cluster_container_insights_enabled ? "enabled" : "disabled" + } + + configuration { + execute_command_configuration { + logging = "DEFAULT" + } + } +} + +resource "aws_ecs_cluster_capacity_providers" "default" { + count = length(aws_ecs_cluster.default.*) + + cluster_name = aws_ecs_cluster.default[count.index].name + capacity_providers = ["FARGATE"] +} + +resource "aws_cloudwatch_log_group" "ecs" { + name_prefix = "${var.namespace}-ecs-" + retention_in_days = var.log_retention_in_days +} From e7721d70a0d98d7f211fd53ca46375dd6cd15b79 Mon Sep 17 00:00:00 2001 From: tyler Date: Thu, 7 Dec 2023 23:58:22 +0000 Subject: [PATCH 02/14] Provision ECS task service for RedwoodJS console --- terraform/ecs_redwood_console.tf | 223 +++++++++++++++++++++++++++++++ terraform/variables.tf | 5 +- 2 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 terraform/ecs_redwood_console.tf diff --git a/terraform/ecs_redwood_console.tf b/terraform/ecs_redwood_console.tf new file mode 100644 index 00000000..52e89cf4 --- /dev/null +++ b/terraform/ecs_redwood_console.tf @@ -0,0 +1,223 @@ +module "ecs_console_container_definition" { + source = "cloudposse/ecs-container-definition/aws" + version = "0.61.1" + + container_name = "console" + container_image = var.console_container_image + essential = true + readonly_root_filesystem = "false" + + linux_parameters = { + capabilities = { + add = [] + drop = [] + } + devices = [] + initProcessEnabled = true + maxSwap = null + sharedMemorySize = null + swappiness = null + tmpfs = [] + } + + map_environment = { + DATABASE_URL = format( + "postgres://%s@%s:%s/%s?%s", + module.postgres.cluster_master_username, + module.postgres.cluster_endpoint, + module.postgres.cluster_port, + module.postgres.cluster_database_name, + join("&", [ + "sslmode=verify", + "sslcert=rds-combined-ca-bundle.pem" + ]) + ) + CI = "" + NODE_ENV = "development" + } + + map_secrets = { + PGPASSWORD = join("", aws_ssm_parameter.postgres_master_password.arn) + } + + log_configuration = { + logDriver = "awslogs" + options = { + awslogs-group = join("", aws_cloudwatch_log_group.ecs.name) + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "console" + } + } +} + +resource "aws_iam_role" "ecs_console_execution" { + name_prefix = "${var.namespace}-console-ECSTaskExecution-" + permissions_boundary = local.permissions_boundary_arn + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { + Service = "ecs-tasks.amazonaws.com" + } + }, + ] + }) +} + +data "aws_iam_policy_document" "ecs_console_execution" { + statement { + sid = "DecryptSSMSecrets" + effect = "Allow" + actions = [ + "kms:Decrypt", + "ssm:GetParameters", + ] + resources = [ + data.data.aws_kms_key.ssm.arn, + aws_ssm_parameter.postgres_master_password.arnm, + ] + } + statement { + sid = "WriteLogs" + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ] + resources = [ + aws_cloudwatch_log_group.ecs.arn, + "${aws_cloudwatch_log_group.ecs.arn}:log-stream:*", + ] + } +} + +resource "aws_iam_role_policy" "ecs_console_execution" { + role = aws_iam_role.ecs_console_execution.name + policy = data.aws_iam_policy_document.ecs_console_execution.json +} + +resource "aws_iam_role" "ecs_console_task" { + name_prefix = "${var.namespace}-ECSConsoleTask-" + permissions_boundary = local.permissions_boundary_arn + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { + Service = "ecs-tasks.amazonaws.com" + } + Condition = { + ArnLike = { + "aws:SourceArn" = "arn:aws:ecs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*", + }, + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + ] + }) +} + +data "aws_iam_policy_document" "ecs_console_task" { + statement { + sid = "PostgresIAMAuth" + effect = "Allow" + actions = ["rds-db:connect"] + resources = ["${local.postgres_rds_connect_resource_base_arn}/${module.postgres.cluster_master_username}"] + } + statement { + sid = "AllowECSExec" + effect = "Allow" + actions = [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ] + resources = ["*"] + } + statement { + sid = "InventoryLogGroupsForExec" + effect = "Allow" + actions = "logs:DescribeLogGroups" + resources = ["*"] + } + statement { + sid = "WriteExecLogs" + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ] + resources = ["${aws_cloudwatch_log_group.ecs.arn}:log-stream:*"] + } +} + +resource "aws_iam_role_policy" "ecs_console_task" { + role = aws_iam_role.ecs_console_task + policy = data.aws_iam_policy_document.ecs_console_task.json +} + +resource "aws_ecs_task_definition" "console" { + family = "${var.namespace}-console" + execution_role_arn = aws_iam_role.ecs_console_execution.arn + task_role_arn = aws_iam_role.ecs_console_task.arn + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + container_definitions = jsonencode([module.ecs_console_container_definition.json_map_object]) + + cpu = 256 + memory = 512 + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + + lifecycle { + create_before_destroy = true + } +} + +module "ecs_console_security_group" { + source = "cloudposse/security-group/aws" + version = "2.2.0" + context = module.this.context + + vpc_id = data.aws_ssm_parameter.vpc_id.value + attributes = ["ecs", "console"] + allow_all_egress = true + + create_before_destroy = true +} + +resource "aws_ecs_service" "console" { + name = "${var.namespace}-console" + cluster = aws_ecs_cluster.default.id + task_definition = aws_ecs_task_definition.console.arn + desired_count = 1 + launch_type = "FARGATE" + enable_execute_command = true + + network_configuration { + assign_public_ip = false + subnets = local.private_subnet_ids + security_groups = [ + module.module.ecs_console_security_group.id, + module.postgres.security_group_id, + ] + } + + depends_on = [ + aws_iam_role.ecs_console_execution, + aws_iam_role.ecs_console_task + ] +} diff --git a/terraform/variables.tf b/terraform/variables.tf index c307b008..b0904c42 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -191,10 +191,9 @@ variable "postgres_snapshot_before_destroy" { } // ECS (currently unused) -variable "api_container_image_tag" { - description = "Complete Docker image tag to pull for API tasks." +variable "console_container_image" { + description = "Complete Docker image tag to pull for RedwoodJS console ECS tasks." type = string - default = "" } // Lambda From 090608afb9677ca4a853bd97bd9fa7e128c1bb23 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 00:14:35 +0000 Subject: [PATCH 03/14] Update CI/CD workflows to build console images --- .github/workflows/build.yml | 205 +++++++++++++++++++++++---- .github/workflows/ci.yml | 19 +-- .github/workflows/deploy-staging.yml | 17 +-- .github/workflows/terraform-plan.yml | 6 +- 4 files changed, 198 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49f14459..7e36270e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,58 +18,66 @@ on: description: Whether to build zip artifacts for API functions. type: boolean default: true + build-console-image: + description: Whether to build a RedwoodJS console Docker image. + type: boolean + default: true build-web: description: Whether to build website distribution artifacts for the client package. type: boolean default: true - api-image-args-ref: + api-image-name: + description: Name of the docker image. Use caution when setting a non-default value. + type: string + default: ${{ github.repository }}-api + api-function-artifact-retention-days: + description: Number of days to retain API function build artifacts. + type: number + default: 90 + console-image-name: + description: Name of the docker image. Use caution when setting a non-default value. + type: string + default: ${{ github.repository }}-console + docker-image-args-ref: description: Ref name for the build commit, like refs/heads/my-feature-branch-1. type: string required: true - api-image-artifacts-retention-days: + docker-image-artifacts-retention-days: description: Number of days to store Docker attestation artifacts. type: number default: 90 - api-image-upload-attestations: - description: Whether to upload attestation files for the Docker build as artifacts. + docker-image-upload-attestations: + description: Whether to upload attestation files for Docker builds as artifacts. type: boolean default: false - api-image-push: + docker-image-push: description: Whether to push Docker images to the registry after building. type: boolean default: true - api-image-name: - description: Name of the docker image. Use caution when setting a non-default value. - type: string - default: ${{ github.repository }}-api - api-image-registry: + docker-image-registry: description: The Docker image registry. Use caution when setting a non-default value. type: string default: ghcr.io - api-image-tag-latest: + docker-image-tag-latest: description: Tags image builds with `latest`. type: boolean default: false - api-image-tag-production: + docker-image-tag-production: description: Tags image builds with `production`. type: boolean required: false - api-image-tag-pr: + docker-image-tag-pr: description: A PR number to add as a Docker image tag (as `pr-`) when building for a pull request. type: string required: false - api-image-tag-release: + docker-image-tag-release: description: A tag value that, if provided, signifies the release version associated with the Docker image. type: string required: false - api-image-version: + docker-image-version: description: Value to set for the `org.opencontainers.image.version label`. type: string default: "" - api-function-artifact-retention-days: - description: Number of days to retain API function build artifacts. - type: number - default: 90 web-artifact-retention-days: description: Number of days to retain website build artifacts. type: number @@ -83,6 +91,8 @@ on: value: ${{ jobs.api-docker-image.result }} build-api-functions-result: value: ${{ jobs.api-function-zips.result }} + build-console-image-result: + value: ${{ jobs.redwood-console-image.result }} build-web-result: value: ${{ jobs.web-bundle.result }} api-image-digest: @@ -97,6 +107,12 @@ on: value: ${{ jobs.api-function-zips.outputs.artifacts-path }} api-functions-checksums-sha256: value: ${{ jobs.api-function-zips.outputs.checksums-sha256 }} + console-image-digest: + value: ${{ jobs.redwood-console-image.outputs.digest }} + console-attestation-artifacts-key: + value: ${{ jobs.redwood-console-image.outputs.attestation-artifacts-key }} + console-attestation-artifacts-path: + value: ${{ jobs.redwood-console-image.outputs.attestation-artifacts-path }} web-artifacts-key: value: ${{ jobs.web-bundle.outputs.artifacts-key }} web-artifacts-path: @@ -151,15 +167,15 @@ jobs: id: meta uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 with: - images: ${{ inputs.api-image-registry }}/${{ inputs.api-image-name }} + images: ${{ inputs.docker-image-registry }}/${{ inputs.api-image-name }} tags: | type=raw,enable=true,priority=100,prefix=sha-,value=${{ steps.commit-sha.outputs.long }} - type=raw,enable=${{ inputs.api-image-tag-release != '' }},priority=200,value=${{ inputs.api-image-tag-release }} - type=raw,enable=${{ inputs.api-image-tag-latest }},priority=300,value=latest - type=raw,enable=${{ inputs.api-image-tag-pr != '' }},priority=600,prefix=pr-,value=${{ inputs.api-image-tag-pr }} + type=raw,enable=${{ inputs.docker-image-tag-release != '' }},priority=200,value=${{ inputs.docker-image-tag-release }} + type=raw,enable=${{ inputs.docker-image-tag-latest }},priority=300,value=latest + type=raw,enable=${{ inputs.docker-image-tag-pr != '' }},priority=600,prefix=pr-,value=${{ inputs.docker-image-tag-pr }} labels: | org.opencontainers.image.title=${{ inputs.api-image-name }} - org.opencontainers.image.version=${{ inputs.api-image-version }} + org.opencontainers.image.version=${{ inputs.docker-image-version }} org.opencontainers.image.revision=${{ steps.commit-sha.outputs.long }} com.datadoghq.tags.service=cpf-reporter com.datadoghq.tags.version=${{ steps.commit-sha.outputs.long }} @@ -178,7 +194,7 @@ jobs: with: context: . github-token: ${{ secrets.GITHUB_TOKEN }} - push: ${{ inputs.api-image-push }} + push: ${{ inputs.docker-image-push }} file: Dockerfile target: api_serve platforms: linux/amd64,linux/arm64 @@ -190,7 +206,138 @@ jobs: sbom: true build-args: | GIT_COMMIT=${{ inputs.ref }} - GIT_REF=${{ inputs.api-image-args-ref }} + GIT_REF=${{ inputs.docker-image-args-ref }} + TIMESTAMP=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} + - name: Publish build results + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Docker Build Summary + + **Image ID:** `${{ steps.build-push.outputs.imageid }}` + **Image Digest:** `${{ steps.build-push.outputs.digest }}` + +
+ Bake File + + ```json + ${{ steps.bakefile.outputs.result }} + ``` + +
+
+ Build Metadata + + ```json + ${{ steps.build-push.outputs.metadata }} + ``` + +
+ ENDOFREPORT + cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY + - name: Store attestations + id: store-attestations + if: inputs.docker-image-upload-attestations + run: | + ATTESTATIONS_DIR=$(mktemp -d) + echo "path=$ATTESTATIONS_DIR" >> $GITHUB_OUTPUT + docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .SBOM }}" > $ATTESTATIONS_DIR/sbom.sdpx.json + docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .Provenance }}" > $ATTESTATIONS_DIR/provenance.json + env: + INSPECT_NAME: ${{ inputs.docker-image-registry }}/${{ inputs.api-image-name }}@${{ steps.build-push.outputs.digest }} + - name: Upload attestations + if: steps.store-attestations.outcome == 'success' + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: ${{ env.ATTESTATION_ARTIFACTS_KEY }} + path: ${{ steps.store-attestations.outputs.path }} + if-no-files-found: error + retention-days: ${{ inputs.docker-image-artifacts-retention-days }} + + redwood-console-image: + name: Build RedwoodJS console Docker image + if: inputs.build-console-image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + ATTESTATION_ARTIFACTS_KEY: "console-image-attestations-${{ inputs.ref }}" + outputs: + commit-tag: ${{ inputs.ref }} + digest: ${{ steps.build-push.outputs.digest }} + attestation-artifacts-key: ${{ env.ATTESTATION_ARTIFACTS_KEY }} + attestation-artifacts-path: ${{ steps.store-attestations.outputs.path }} + steps: + - uses: step-security/harden-runner@1b05615854632b887b69ae1be8cbefe72d3ae423 # v2.6.0 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + id: checkout + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Set build info for the checked-out commit + id: commit-sha + run: echo "long=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + with: + platforms: linux/amd64,linux/arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + with: + platforms: linux/amd64,linux/arm64 + - name: Authenticate docker + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ inputs.docker-image-registry }}/${{ inputs.console-image-name }} + tags: | + type=raw,enable=true,priority=100,prefix=sha-,value=${{ steps.commit-sha.outputs.long }} + type=raw,enable=${{ inputs.docker-image-tag-release != '' }},priority=200,value=${{ inputs.docker-image-tag-release }} + type=raw,enable=${{ inputs.docker-image-tag-latest }},priority=300,value=latest + type=raw,enable=${{ inputs.docker-image-tag-pr != '' }},priority=600,prefix=pr-,value=${{ inputs.docker-image-tag-pr }} + labels: | + org.opencontainers.image.title=${{ inputs.console-image-name }} + org.opencontainers.image.version=${{ inputs.docker-image-version }} + org.opencontainers.image.revision=${{ steps.commit-sha.outputs.long }} + - name: Set bake file definition as step output + id: bakefile + run: | + BAKEFILE_CONTENTS="$(cat $BAKEFILE_PATH)" + echo "result<> $GITHUB_OUTPUT + echo "$BAKEFILE_CONTENTS" >> $GITHUB_OUTPUT + echo "ENDOFBAKEFILE" >> $GITHUB_OUTPUT + env: + BAKEFILE_PATH: ${{ steps.meta.outputs.bake-file }} + - name: Build and push Docker image + id: build-push + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + context: . + github-token: ${{ secrets.GITHUB_TOKEN }} + push: ${{ inputs.docker-image-push }} + file: Dockerfile + target: console + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true + build-args: | + GIT_COMMIT=${{ inputs.ref }} + GIT_REF=${{ inputs.docker-image-args-ref }} TIMESTAMP=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} - name: Publish build results run: | @@ -221,14 +368,14 @@ jobs: cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY - name: Store attestations id: store-attestations - if: inputs.api-image-upload-attestations + if: inputs.docker-image-upload-attestations run: | ATTESTATIONS_DIR=$(mktemp -d) echo "path=$ATTESTATIONS_DIR" >> $GITHUB_OUTPUT docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .SBOM }}" > $ATTESTATIONS_DIR/sbom.sdpx.json docker buildx imagetools inspect "$INSPECT_NAME" --format "{{ json .Provenance }}" > $ATTESTATIONS_DIR/provenance.json env: - INSPECT_NAME: ${{ inputs.api-image-registry }}/${{ inputs.api-image-name }}@${{ steps.build-push.outputs.digest }} + INSPECT_NAME: ${{ inputs.docker-image-registry }}/${{ inputs.console-image-name }}@${{ steps.build-push.outputs.digest }} - name: Upload attestations if: steps.store-attestations.outcome == 'success' uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 @@ -236,7 +383,7 @@ jobs: name: ${{ env.ATTESTATION_ARTIFACTS_KEY }} path: ${{ steps.store-attestations.outputs.path }} if-no-files-found: error - retention-days: ${{ inputs.api-image-artifacts-retention-days }} + retention-days: ${{ inputs.docker-image-artifacts-retention-days }} api-function-zips: name: Build API Lambda function zip artifacts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d22837fc..b7a566f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,14 +42,15 @@ jobs: uses: ./.github/workflows/build.yml with: ref: ${{ github.event.pull_request.head.sha }} + docker-image-push: true + docker-image-args-ref: ${{ github.ref }} + docker-image-tag-latest: false + docker-image-tag-pr: "${{ github.event.pull_request.number }}" + docker-image-upload-attestations: true + docker-image-artifacts-retention-days: 14 + docker-image-version: "rc-pr-${{ github.event.pull_request.number }}" build-api-image: true - api-image-push: true - api-image-args-ref: ${{ github.ref }} - api-image-tag-latest: false - api-image-tag-pr: "${{ github.event.pull_request.number }}" - api-image-version: "rc-pr-${{ github.event.pull_request.number }}" - api-image-upload-attestations: true - api-image-artifacts-retention-days: 14 + build-console-image: true build-api-functions: true api-function-artifact-retention-days: 14 build-web: true @@ -83,8 +84,8 @@ jobs: concurrency-group: run_terraform-staging api-functions-artifacts-key: ${{ needs.build.outputs.api-functions-artifacts-key }} api-functions-artifacts-path: ${{ needs.build.outputs.api-functions-artifacts-path }} - api-image-tag: ${{ github.event.pull_request.head.sha }} - api-image-digest: ${{ needs.build.outputs.api-image-digest }} + console-image-tag: ${{ github.event.pull_request.head.sha }} + console-image-digest: ${{ needs.build.outputs.console-image-digest }} web-artifacts-key: ${{ needs.build.outputs.web-artifacts-key }} web-artifacts-path: ${{ needs.build.outputs.web-artifacts-path }} aws-region: us-west-2 diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0583aa9e..e2f7e023 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -33,13 +33,14 @@ jobs: uses: ./.github/workflows/build.yml with: ref: ${{ github.sha }} + docker-image-push: true + docker-image-args-ref: ${{ github.ref }} + docker-image-tag-latest: true + docker-image-upload-attestations: true + docker-image-artifacts-retention-days: 14 + docker-image-version: "rc-${{ github.sha }}" build-api-image: true - api-image-push: true - api-image-args-ref: ${{ github.ref }} - api-image-tag-latest: true - api-image-version: "rc-${{ github.sha }}" - api-image-upload-attestations: true - api-image-artifacts-retention-days: 14 + build-console-image: true build-api-functions: true api-function-artifact-retention-days: 14 build-web: true @@ -75,8 +76,8 @@ jobs: concurrency-group: run_terraform-staging api-functions-artifacts-key: ${{ needs.build.outputs.api-functions-artifacts-key }} api-functions-artifacts-path: ${{ needs.build.outputs.api-functions-artifacts-path }} - api-image-tag: ${{ github.sha }} - api-image-digest: ${{ needs.build.outputs.api-image-digest }} + console-image-tag: ${{ github.sha }} + console-image-digest: ${{ needs.build.outputs.console-image-digest }} web-artifacts-key: ${{ needs.build.outputs.web-artifacts-key }} web-artifacts-path: ${{ needs.build.outputs.web-artifacts-path }} aws-region: us-west-2 diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml index b7dc48f1..b4e4d055 100644 --- a/.github/workflows/terraform-plan.yml +++ b/.github/workflows/terraform-plan.yml @@ -32,10 +32,10 @@ on: description: Name of the concurrency group (avoids simultaneous Terraform execution against the same environment) type: string default: run_terraform - api-image-tag: + console-image-tag: type: string required: true - api-image-digest: + console-image-digest: type: string required: true api-functions-artifacts-key: @@ -197,7 +197,7 @@ jobs: TF_VAR_git_commit_sha: ${{ inputs.ref }} TF_VAR_datadog_api_key: ${{ secrets.datadog-api-key }} TF_VAR_datadog_app_key: ${{ secrets.datadog-app-key }} - TF_VAR_api_container_image_tag: "${{ inputs.api-image-tag }}@sha256:${{ inputs.api-image-digest }}" + TF_VAR_console_container_image: "${{ inputs.console-image-tag }}@sha256:${{ inputs.console-image-digest }}" TF_VAR_lambda_artifacts_base_path: ${{ inputs.api-functions-artifacts-path }} TF_VAR_website_origin_artifacts_dist_path: "${{ inputs.web-artifacts-path }}" - name: Generate plaintext plan From b660e2a2fa2f462af4c7122d9c73867719e385ce Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 00:36:29 +0000 Subject: [PATCH 04/14] Add test workflow for building console --- .github/workflows/temp-build.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/temp-build.yml diff --git a/.github/workflows/temp-build.yml b/.github/workflows/temp-build.yml new file mode 100644 index 00000000..4ee5c100 --- /dev/null +++ b/.github/workflows/temp-build.yml @@ -0,0 +1,29 @@ +name: Test workflow + +on: + pull_request: {} + +permissions: + contents: read + packages: write + +jobs: + build-console: + permissions: + contents: read + packages: write + name: Build RedwoodJS console + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + docker-image-push: true + docker-image-args-ref: ${{ github.ref }} + docker-image-tag-latest: false + docker-image-tag-pr: "${{ github.event.pull_request.number }}" + docker-image-upload-attestations: true + docker-image-artifacts-retention-days: 14 + docker-image-version: "rc-pr-${{ github.event.pull_request.number }}" + build-api-image: false + build-console-image: true + build-api-functions: false + build-web: false From df3db0f767394fbdd628ae80cf469639fe73976a Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 00:41:50 +0000 Subject: [PATCH 05/14] Add missing tf var --- terraform/variables.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/variables.tf b/terraform/variables.tf index b0904c42..a3851cdb 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -191,6 +191,12 @@ variable "postgres_snapshot_before_destroy" { } // ECS (currently unused) +variable "ecs_cluster_container_insights_enabled" { + description = "Whether to enable ECS container insights." + type = bool + default = false +} + variable "console_container_image" { description = "Complete Docker image tag to pull for RedwoodJS console ECS tasks." type = string From 5e6fb65d8400a2f473c3fe48a7bf8ea5c578f13d Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 01:13:37 +0000 Subject: [PATCH 06/14] Test disabling attestations/provenance --- .github/workflows/build.yml | 2 +- .github/workflows/temp-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e36270e..64b401e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,7 +333,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - provenance: true + provenance: false sbom: true build-args: | GIT_COMMIT=${{ inputs.ref }} diff --git a/.github/workflows/temp-build.yml b/.github/workflows/temp-build.yml index 4ee5c100..924aee47 100644 --- a/.github/workflows/temp-build.yml +++ b/.github/workflows/temp-build.yml @@ -20,7 +20,7 @@ jobs: docker-image-args-ref: ${{ github.ref }} docker-image-tag-latest: false docker-image-tag-pr: "${{ github.event.pull_request.number }}" - docker-image-upload-attestations: true + docker-image-upload-attestations: false docker-image-artifacts-retention-days: 14 docker-image-version: "rc-pr-${{ github.event.pull_request.number }}" build-api-image: false From a787df94efc2a8303875085b5deb06afb0b2842d Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 01:16:41 +0000 Subject: [PATCH 07/14] Fix tf typos --- terraform/ecs_redwood_console.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/ecs_redwood_console.tf b/terraform/ecs_redwood_console.tf index 52e89cf4..0009f042 100644 --- a/terraform/ecs_redwood_console.tf +++ b/terraform/ecs_redwood_console.tf @@ -76,8 +76,8 @@ data "aws_iam_policy_document" "ecs_console_execution" { "ssm:GetParameters", ] resources = [ - data.data.aws_kms_key.ssm.arn, - aws_ssm_parameter.postgres_master_password.arnm, + data.aws_kms_key.ssm.arn, + aws_ssm_parameter.postgres_master_password.arn, ] } statement { @@ -146,7 +146,7 @@ data "aws_iam_policy_document" "ecs_console_task" { statement { sid = "InventoryLogGroupsForExec" effect = "Allow" - actions = "logs:DescribeLogGroups" + actions = ["logs:DescribeLogGroups"] resources = ["*"] } statement { @@ -211,7 +211,7 @@ resource "aws_ecs_service" "console" { assign_public_ip = false subnets = local.private_subnet_ids security_groups = [ - module.module.ecs_console_security_group.id, + module.ecs_console_security_group.id, module.postgres.security_group_id, ] } From ad72c107b5c014fbfc4475da2247c1e38e7a5655 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 01:50:51 +0000 Subject: [PATCH 08/14] Last test build --- .github/workflows/build.yml | 2 +- .github/workflows/temp-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 64b401e8..7e36270e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,7 +333,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - provenance: false + provenance: true sbom: true build-args: | GIT_COMMIT=${{ inputs.ref }} diff --git a/.github/workflows/temp-build.yml b/.github/workflows/temp-build.yml index 924aee47..4ee5c100 100644 --- a/.github/workflows/temp-build.yml +++ b/.github/workflows/temp-build.yml @@ -20,7 +20,7 @@ jobs: docker-image-args-ref: ${{ github.ref }} docker-image-tag-latest: false docker-image-tag-pr: "${{ github.event.pull_request.number }}" - docker-image-upload-attestations: false + docker-image-upload-attestations: true docker-image-artifacts-retention-days: 14 docker-image-version: "rc-pr-${{ github.event.pull_request.number }}" build-api-image: false From 1ea66e3c90ee425673a2f639a5b8d6ba268ff872 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 01:54:29 +0000 Subject: [PATCH 09/14] Build with pull: true --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e36270e..0b30bdd7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,6 +333,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + pull: true provenance: true sbom: true build-args: | From d2f6c5b80fc7ff3282b3745890d9502b00326305 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 02:00:25 +0000 Subject: [PATCH 10/14] Always pull latest base images --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b30bdd7..06c49fc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -202,6 +202,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + pull: true provenance: true sbom: true build-args: | From 8c18ea93ee36fc4d073dd72409bfe202082567ac Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 02:02:10 +0000 Subject: [PATCH 11/14] Remove test builder --- .github/workflows/temp-build.yml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/temp-build.yml diff --git a/.github/workflows/temp-build.yml b/.github/workflows/temp-build.yml deleted file mode 100644 index 4ee5c100..00000000 --- a/.github/workflows/temp-build.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Test workflow - -on: - pull_request: {} - -permissions: - contents: read - packages: write - -jobs: - build-console: - permissions: - contents: read - packages: write - name: Build RedwoodJS console - uses: ./.github/workflows/build.yml - with: - ref: ${{ github.event.pull_request.head.sha }} - docker-image-push: true - docker-image-args-ref: ${{ github.ref }} - docker-image-tag-latest: false - docker-image-tag-pr: "${{ github.event.pull_request.number }}" - docker-image-upload-attestations: true - docker-image-artifacts-retention-days: 14 - docker-image-version: "rc-pr-${{ github.event.pull_request.number }}" - build-api-image: false - build-console-image: true - build-api-functions: false - build-web: false From 881cc563291b912afa0d2bb7b97e4997f36663b4 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 03:07:04 +0000 Subject: [PATCH 12/14] Use abs path to RDS ssl cert --- terraform/ecs_redwood_console.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/ecs_redwood_console.tf b/terraform/ecs_redwood_console.tf index 0009f042..5f1c6e2c 100644 --- a/terraform/ecs_redwood_console.tf +++ b/terraform/ecs_redwood_console.tf @@ -21,6 +21,8 @@ module "ecs_console_container_definition" { } map_environment = { + AWS_REGION = data.aws_region.current.name + AWS_DEFAULT_REGION = data.aws_region.current.name DATABASE_URL = format( "postgres://%s@%s:%s/%s?%s", module.postgres.cluster_master_username, @@ -29,7 +31,7 @@ module "ecs_console_container_definition" { module.postgres.cluster_database_name, join("&", [ "sslmode=verify", - "sslcert=rds-combined-ca-bundle.pem" + "sslcert=/home/node/app/api/db/rds-combined-ca-bundle.pem" ]) ) CI = "" From f4385b34757a6918c7ed6adddb621b894c11e905 Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 03:08:12 +0000 Subject: [PATCH 13/14] Install awscli and postgresql-client-15 in console --- .github/CODEOWNERS | 3 +++ Dockerfile | 18 ++++++++++++++++++ aws-public-key.gpg | 29 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 aws-public-key.gpg diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a4addc22..3f4f2256 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,9 @@ # Require admin approval when Postgres root CA bundle is modified /api/db/rds-combined-ca-bundle.pem @usdigitalresponse/grants-admins +# Require admin approval when GPG keys are modified +aws-public-key.gpg @usdigitalresponse/grants-admins + # Require admin approval for special doc modifications README.md @usdigitalresponse/grants-admins LICENSE @usdigitalresponse/grants-admins diff --git a/Dockerfile b/Dockerfile index 4688b4cf..63a465bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -127,6 +127,24 @@ FROM base as console # # USER node # ``` +USER root + +RUN DEBIAN_FRONTEND="noninteractive" apt update && apt install -y \ + curl gnupg gnupg2 gnupg1 groff postgresql-client-15 unzip + +# Install awscli +RUN mkdir -p /tmp/awscli +COPY aws-public-key.gpg /tmp/awscli/key.gpg +RUN gpg --import /tmp/awscli/key.gpg +RUN sh -c 'curl -o "/tmp/awscli/awscliv2.sig" https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip.sig' +RUN sh -c 'curl -o "/tmp/awscli/awscliv2.zip" "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip"' +RUN gpg --verify /tmp/awscli/awscliv2.sig /tmp/awscli/awscliv2.zip +RUN unzip -u -d /tmp/awscli/ /tmp/awscli/awscliv2.zip +RUN /tmp/awscli/aws/install +RUN aws --version +RUN rm -rf /tmp/awscli + +USER node COPY --chown=node:node api api COPY --chown=node:node web web diff --git a/aws-public-key.gpg b/aws-public-key.gpg new file mode 100644 index 00000000..59518484 --- /dev/null +++ b/aws-public-key.gpg @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG +ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx +PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G +TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz +gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk +C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG +94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO +lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG +fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG +EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX +XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB +tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4WIQT7 +Xbd/1cEYuAURraimMQrMRnJHXAUCXYKvtQIbAwUJB4TOAAULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRCmMQrMRnJHXJIXEAChLUIkg80uPUkGjE3jejvQSA1aWuAM +yzy6fdpdlRUz6M6nmsUhOExjVIvibEJpzK5mhuSZ4lb0vJ2ZUPgCv4zs2nBd7BGJ +MxKiWgBReGvTdqZ0SzyYH4PYCJSE732x/Fw9hfnh1dMTXNcrQXzwOmmFNNegG0Ox +au+VnpcR5Kz3smiTrIwZbRudo1ijhCYPQ7t5CMp9kjC6bObvy1hSIg2xNbMAN/Do +ikebAl36uA6Y/Uczjj3GxZW4ZWeFirMidKbtqvUz2y0UFszobjiBSqZZHCreC34B +hw9bFNpuWC/0SrXgohdsc6vK50pDGdV5kM2qo9tMQ/izsAwTh/d/GzZv8H4lV9eO +tEis+EpR497PaxKKh9tJf0N6Q1YLRHof5xePZtOIlS3gfvsH5hXA3HJ9yIxb8T0H +QYmVr3aIUes20i6meI3fuV36VFupwfrTKaL7VXnsrK2fq5cRvyJLNzXucg0WAjPF +RrAGLzY7nP1xeg1a0aeP+pdsqjqlPJom8OCWc1+6DWbg0jsC74WoesAqgBItODMB +rsal1y/q+bPzpsnWjzHV8+1/EtZmSc8ZUGSJOPkfC7hObnfkl18h+1QtKTjZme4d +H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe +YLZATHZKTJyiqA== +=vYOk +-----END PGP PUBLIC KEY BLOCK----- From e96e627f4d2f4aecb27ab3db005a045d043db11e Mon Sep 17 00:00:00 2001 From: tyler Date: Fri, 8 Dec 2023 03:27:18 +0000 Subject: [PATCH 14/14] Remove unnecessary tf count --- terraform/ecs_cluster.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/ecs_cluster.tf b/terraform/ecs_cluster.tf index 4fc47500..53dab980 100644 --- a/terraform/ecs_cluster.tf +++ b/terraform/ecs_cluster.tf @@ -14,8 +14,6 @@ resource "aws_ecs_cluster" "default" { } resource "aws_ecs_cluster_capacity_providers" "default" { - count = length(aws_ecs_cluster.default.*) - cluster_name = aws_ecs_cluster.default[count.index].name capacity_providers = ["FARGATE"] }