From f1cf5c1186183b738f53a29121dab49fab795bd0 Mon Sep 17 00:00:00 2001 From: jacobgill1 <130056516+jacobgill1@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:27:49 +0100 Subject: [PATCH] Jagi5 sonar updates (#4) ## Description Merge in latest form repo template and update sonar properties to exclude examples ## Context ## Type of changes - [ ] Refactoring (non-breaking change) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would change existing functionality) - [ ] Bug fix (non-breaking change which fixes an issue) ## Checklist - [ ] I am familiar with the [contributing guidelines](../docs/CONTRIBUTING.md) - [ ] I have followed the code style of the project - [ ] I have added tests to cover my changes - [ ] I have updated the documentation accordingly - [ ] This PR is a result of pair or mob programming --- ## Sensitive Information Declaration To ensure the utmost confidentiality and protect your and others privacy, we kindly ask you to NOT including [PII (Personal Identifiable Information) / PID (Personal Identifiable Data)](https://digital.nhs.uk/data-and-information/keeping-data-safe-and-benefitting-the-public) or any other sensitive data in this PR (Pull Request) and the codebase changes. We will remove any PR that do contain any sensitive information. We really appreciate your cooperation in this matter. - [ ] I confirm that neither PII/PID nor sensitive data are included in this PR and the codebase changes. --------- Signed-off-by: regularfry Signed-off-by: dependabot[bot] Co-authored-by: Dan Stefaniuk Co-authored-by: Dan Stefaniuk <499338+stefaniuk@users.noreply.github.com> Co-authored-by: amaanibn-nasar1-nhs <94694678+amaanibn-nasar1-nhs@users.noreply.github.com> Co-authored-by: Tim Ireland Co-authored-by: Jonathan Pearce Co-authored-by: Tamara Goldschmidt Co-authored-by: Tim Rickwood Co-authored-by: Robert Ball Co-authored-by: robertball@outlook.com Co-authored-by: Robert Ball Co-authored-by: Alex Young Co-authored-by: amaanibn-nasar1-nhs Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Young --- .../actions/check-markdown-format/action.yaml | 2 +- .github/actions/scan-secrets/action.yaml | 4 +- .github/workflows/cicd-2-publish.yaml | 85 ++++++++++++++++ .tool-versions | 37 +++---- Makefile | 4 + .../assets/ADR-003/examples/bash/script.sh | 8 +- scripts/config/pre-commit.yaml | 59 +++++------ scripts/config/sonar-scanner.properties | 4 +- scripts/docker/docker.lib.sh | 15 +-- scripts/docker/docker.mk | 1 + scripts/docker/dockerfile-linter.sh | 3 +- .../examples/python/.tool-versions.example | 2 + .../assets/hello_world/requirements.txt | 14 ++- scripts/docker/tests/.tool-versions.test | 2 + scripts/docker/tests/docker.test.sh | 5 +- scripts/githooks/check-file-format.sh | 24 ++--- scripts/githooks/check-markdown-format.sh | 42 +++++--- scripts/githooks/check-terraform-format.sh | 4 +- scripts/githooks/scan-secrets.sh | 97 +++++++++++++------ scripts/init.mk | 20 ---- scripts/reports/cloc-repository.sh | 30 +++--- scripts/reports/generate-sbom.sh | 33 ++++--- scripts/shellscript-linter.sh | 2 +- scripts/terraform/terraform.mk | 1 + scripts/terraform/terraform.sh | 3 + 25 files changed, 321 insertions(+), 180 deletions(-) create mode 100644 .github/workflows/cicd-2-publish.yaml create mode 100644 scripts/docker/examples/python/.tool-versions.example create mode 100644 scripts/docker/tests/.tool-versions.test diff --git a/.github/actions/check-markdown-format/action.yaml b/.github/actions/check-markdown-format/action.yaml index 1e5d006..53a715b 100644 --- a/.github/actions/check-markdown-format/action.yaml +++ b/.github/actions/check-markdown-format/action.yaml @@ -7,4 +7,4 @@ runs: shell: bash run: | export BRANCH_NAME=origin/${{ github.event.repository.default_branch }} - ./scripts/githooks/check-markdown-format.sh + check=branch ./scripts/githooks/check-markdown-format.sh diff --git a/.github/actions/scan-secrets/action.yaml b/.github/actions/scan-secrets/action.yaml index fce857a..1ed8bac 100644 --- a/.github/actions/scan-secrets/action.yaml +++ b/.github/actions/scan-secrets/action.yaml @@ -6,5 +6,5 @@ runs: - name: "Scan secrets" shell: bash run: | - export ALL_FILES=true # Do not change this line, as new patterns may be added or history may be rewritten - ./scripts/githooks/scan-secrets.sh + # Please do not change this `check=whole-history` setting, as new patterns may be added or history may be rewritten. + check=whole-history ./scripts/githooks/scan-secrets.sh diff --git a/.github/workflows/cicd-2-publish.yaml b/.github/workflows/cicd-2-publish.yaml new file mode 100644 index 0000000..51b687d --- /dev/null +++ b/.github/workflows/cicd-2-publish.yaml @@ -0,0 +1,85 @@ +name: "CI/CD publish" + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + metadata: + name: "Set CI/CD metadata" + runs-on: ubuntu-latest + if: github.event.pull_request.merged == true + timeout-minutes: 1 + outputs: + build_datetime: ${{ steps.variables.outputs.build_datetime }} + build_timestamp: ${{ steps.variables.outputs.build_timestamp }} + build_epoch: ${{ steps.variables.outputs.build_epoch }} + nodejs_version: ${{ steps.variables.outputs.nodejs_version }} + python_version: ${{ steps.variables.outputs.python_version }} + terraform_version: ${{ steps.variables.outputs.terraform_version }} + version: ${{ steps.variables.outputs.version }} + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + - name: "Set CI/CD variables" + id: variables + run: | + datetime=$(date -u +'%Y-%m-%dT%H:%M:%S%z') + echo "build_datetime=$datetime" >> $GITHUB_OUTPUT + echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT + echo "nodejs_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + echo "python_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + echo "terraform_version=$(grep "^terraform" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + # TODO: Get the version, but it may not be the .version file as this should come from the CI/CD Pull Request Workflow + echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT + - name: "List variables" + run: | + export BUILD_DATETIME="${{ steps.variables.outputs.build_datetime }}" + export BUILD_TIMESTAMP="${{ steps.variables.outputs.build_timestamp }}" + export BUILD_EPOCH="${{ steps.variables.outputs.build_epoch }}" + export NODEJS_VERSION="${{ steps.variables.outputs.nodejs_version }}" + export PYTHON_VERSION="${{ steps.variables.outputs.python_version }}" + export TERRAFORM_VERSION="${{ steps.variables.outputs.terraform_version }}" + export VERSION="${{ steps.variables.outputs.version }}" + make list-variables + publish: + name: "Publish packages" + runs-on: ubuntu-latest + needs: [metadata] + if: github.event.pull_request.merged == true + timeout-minutes: 3 + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + - name: "Get the artefacts" + run: | + echo "Building artefact ..." + export release_name=terraform-aws-api-bootstrap-${{ needs.metadata.outputs.version }}.tar.gz + tar -czf $release_name ./lambda ./terraform + - name: "Create release" + uses: softprops/action-gh-release@v1 + with: + files: terraform-aws-api-bootstrap-${{ needs.metadata.outputs.version }}.tar.gz + name: Release ${{ needs.metadata.outputs.version }} + tag_name: ${{ needs.metadata.outputs.version }} + body: Release of ${{ needs.metadata.outputs.version }} + success: + name: "Success notification" + runs-on: ubuntu-latest + needs: [publish] + steps: + - name: "Check prerequisites for notification" + id: check + run: echo "secret_exist=${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL != '' }}" >> $GITHUB_OUTPUT + - name: "Notify on publishing packages" + if: steps.check.outputs.secret_exist == 'true' + uses: nhs-england-tools/notify-msteams-action@v0.0.4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + teams-webhook-url: ${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL }} + message-title: "Release ${{ needs.metadata.outputs.version }}" + message-text: "Release successfully published" + link: ${{ github.event.pull_request.html_url }} diff --git a/.tool-versions b/.tool-versions index dedc6cd..b22303c 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,30 +1,15 @@ -nodejs 18.17.1 # Always check AWS and Azure runtime support -python 3.11.4 # Always check AWS and Azure runtime support -poetry 1.6.1 -yarn 1.22.19 -terraform 1.5.6 -pre-commit 3.3.3 +# This file is for you! Please, updated to the versions agreed by your team. +terraform 1.5.7 +pre-commit 3.4.0 +python 3.9.17 # ============================================================================== # The section below is reserved for Docker image versions. -# alpine, SEE: https://hub.docker.com/_/alpine/tags -# docker/alpine 3.18.3@sha256:c5c5fda71656f28e49ac9c5416b3643eaa6a108a8093151d6d1afc9463be8e33 - -# nodejs, SEE: https://hub.docker.com/_/node/tags -# docker/node 18.17.1-alpine3.18@sha256:982b5b6f07cd9241c9ebb163829067deac8eaefc57cfa8f31927f4b18943d971 - -# python, SEE: https://hub.docker.com/_/python/tags -# docker/python 3.11.4-alpine3.18@sha256:0135ae6442d1269379860b361760ad2cf6ab7c403d21935a8015b48d5bf78a86 - -# terraform, SEE: https://hub.docker.com/r/hashicorp/terraform/tags -# docker/hashicorp/terraform 1.5.6@sha256:180a7efa983386a27b43657ed610e9deed9e6c3848d54f9ea9b6cb8a5c8c25f5 - -# shellcheck, SEE: https://hub.docker.com/r/koalaman/shellcheck/tags -# docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c - -# hadolint, SEE: https://hub.docker.com/r/hadolint/hadolint/tags -# docker/hadolint/hadolint 2.12.0-alpine@sha256:7dba9a9f1a0350f6d021fb2f6f88900998a4fb0aaf8e4330aa8c38544f04db42 - -# ghcr.io/nhs-england-tools/github-runner-image, SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image -# docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 +# TODO: Move this section - consider using a different file for the repository template dependencies. +# docker/ghcr.io/gitleaks/gitleaks v8.18.0@sha256:fd2b5cab12b563d2cc538b14631764a1c25577780e3b7dba71657d58da45d9d9 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks +# docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image +# docker/hadolint/hadolint 2.12.0-alpine@sha256:7dba9a9f1a0350f6d021fb2f6f88900998a4fb0aaf8e4330aa8c38544f04db42 # SEE: https://hub.docker.com/r/hadolint/hadolint/tags +# docker/hashicorp/terraform 1.5.6@sha256:180a7efa983386a27b43657ed610e9deed9e6c3848d54f9ea9b6cb8a5c8c25f5 # SEE: https://hub.docker.com/r/hashicorp/terraform/tags +# docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c # SEE: https://hub.docker.com/r/koalaman/shellcheck/tags +# docker/sonarsource/sonar-scanner-cli 5.0.1@sha256:494ecc3b5b1ee1625bd377b3905c4284e4f0cc155cff397805a244dee1c7d575 # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags diff --git a/Makefile b/Makefile index 1ce9764..1c8ed33 100644 --- a/Makefile +++ b/Makefile @@ -14,4 +14,8 @@ config:: # Configure development environment (main) @Configuration # ============================================================================== ${VERBOSE}.SILENT: \ + build \ + clean \ config \ + dependencies \ + deploy \ diff --git a/docs/adr/assets/ADR-003/examples/bash/script.sh b/docs/adr/assets/ADR-003/examples/bash/script.sh index 11c42fb..7dd8346 100755 --- a/docs/adr/assets/ADR-003/examples/bash/script.sh +++ b/docs/adr/assets/ADR-003/examples/bash/script.sh @@ -17,7 +17,7 @@ function main() { function get-jwt-token() { header=$(echo -n '{"alg":"RS256","typ":"JWT"}' | base64 | tr -d '=' | tr -d '\n=' | tr -- '+/' '-_') - payload=$(echo -n '{"iat":'$(date +%s)',"exp":'$(($(date +%s)+600))',"iss":"'$GITHUB_APP_ID'"}' | base64 | tr -d '\n=' | tr -- '+/' '-_') + payload=$(echo -n '{"iat":'"$(date +%s)"',"exp":'$(($(date +%s)+600))',"iss":"'"$GITHUB_APP_ID"'"}' | base64 | tr -d '\n=' | tr -- '+/' '-_') signature=$(echo -n "$header.$payload" | openssl dgst -binary -sha256 -sign "$GITHUB_APP_PK_FILE" | openssl base64 | tr -d '\n=' | tr -- '+/' '-_') echo "$header.$payload.$signature" @@ -30,7 +30,7 @@ function get-installation-id() { -H "Accept: application/vnd.github.v3+json" \ https://api.github.com/app/installations) - echo "$(echo $installations_response | jq '.[] | select(.account.login == "'"$GITHUB_ORG"'") .id')" + echo "$installations_response" | jq '.[] | select(.account.login == "'"$GITHUB_ORG"'") .id' } function get-access-token() { @@ -38,9 +38,9 @@ function get-access-token() { token_response=$(curl -sX POST \ -H "Authorization: Bearer $jwt_token" \ -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/app/installations/$installation_id/access_tokens) + "https://api.github.com/app/installations/$installation_id/access_tokens") - echo "$(echo $token_response | jq .token -r)" + echo "$token_response" | jq .token -r } main diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index c1ea76b..12f9eeb 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -1,29 +1,32 @@ repos: -- repo: local - hooks: - - id: scan-secrets - name: Scan Secrets - entry: ./scripts/githooks/scan-secrets.sh - language: script - pass_filenames: false -- repo: local - hooks: - - id: check-file-format - name: Check File Format - entry: check=staged-changes./scripts/githooks/check-file-format.sh - language: script - pass_filenames: false -- repo: local - hooks: - - id: check-markdown-format - name: Check Markdown Format - entry: ./scripts/githooks/check-markdown-format.sh - language: script - pass_filenames: false -- repo: local - hooks: - - id: check-terraform-format - name: Check Terraform Format - entry: ./scripts/githooks/check-terraform-format.sh - language: script - pass_filenames: false + - repo: local + hooks: + - id: scan-secrets + name: Scan Secrets + entry: ./scripts/githooks/scan-secrets.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false + - repo: local + hooks: + - id: check-file-format + name: Check File Format + entry: ./scripts/githooks/check-file-format.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false + - repo: local + hooks: + - id: check-markdown-format + name: Check Markdown Format + entry: ./scripts/githooks/check-markdown-format.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false + - repo: local + hooks: + - id: check-terraform-format + name: Check Terraform Format + entry: ./scripts/githooks/check-terraform-format.sh + language: script + pass_filenames: false diff --git a/scripts/config/sonar-scanner.properties b/scripts/config/sonar-scanner.properties index 2ae21c7..dee15b6 100644 --- a/scripts/config/sonar-scanner.properties +++ b/scripts/config/sonar-scanner.properties @@ -4,6 +4,4 @@ sonar.host.url=https://sonarcloud.io sonar.qualitygate.wait=true sonar.sourceEncoding=UTF-8 sonar.sources=. - -#sonar.python.coverage.reportPaths=.coverage/coverage.xml -#sonar.[javascript|typescript].lcov.reportPaths=.coverage/lcov.info +sonar.exclusions=**/examples/** diff --git a/scripts/docker/docker.lib.sh b/scripts/docker/docker.lib.sh index aecc15b..d0bcd1d 100644 --- a/scripts/docker/docker.lib.sh +++ b/scripts/docker/docker.lib.sh @@ -11,8 +11,9 @@ set -euo pipefail # $ source ./docker.lib.sh # # Arguments (provided as environment variables): -# DOCKER_IMAGE=ghcr.io/org/repo # Docker image name -# DOCKER_TITLE="My Docker image" # Docker image title +# DOCKER_IMAGE=ghcr.io/org/repo # Docker image name +# DOCKER_TITLE="My Docker image" # Docker image title +# TOOL_VERSIONS=$project_dir/.tool-versions # Path to the tool versions file # ============================================================================== # Functions to be used with custom images. @@ -164,10 +165,10 @@ function docker-get-image-version-and-pull() { # Get the image full version from the '.tool-versions' file, # match it by name and version regex, if given. - local versions_file="$(git rev-parse --show-toplevel)/.tool-versions" + local versions_file="${TOOL_VERSIONS:=$(git rev-parse --show-toplevel)/.tool-versions}" local version="latest" if [ -f "$versions_file" ]; then - line=$(grep "docker/${name} " "$versions_file" | sed "s/^#\s*//; s/\s*#.*$//" | grep "${match_version:-'.*'}") + line=$(grep "docker/${name} " "$versions_file" | sed "s/^#\s*//; s/\s*#.*$//" | grep "${match_version:-".*"}") [ -n "$line" ] && version=$(echo "$line" | awk '{print $2}') fi @@ -176,7 +177,7 @@ function docker-get-image-version-and-pull() { local digest="$(echo "$version" | sed 's/^.*@//')" # Check if the image exists locally already - if ! docker images | awk '{ print $1 ":" $2 }' | grep "^${name}:${tag}$"; then + if ! docker images | awk '{ print $1 ":" $2 }' | grep -q "^${name}:${tag}$"; then if [ "$digest" != "latest" ]; then # Pull image by the digest sha256 and tag it docker pull \ @@ -217,13 +218,13 @@ function _create-effective-dockerfile() { function _replace-image-latest-by-specific-version() { local dir=${dir:-$PWD} - local versions_file=$(git rev-parse --show-toplevel)/.tool-versions + local versions_file="${TOOL_VERSIONS:=$(git rev-parse --show-toplevel)/.tool-versions}" local dockerfile="${dir}/Dockerfile.effective" local build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} if [ -f "$versions_file" ]; then # First, list the entries specific for Docker to take precedence, then the rest but exclude comments - content=$(grep " docker/" "$versions_file"; grep -v " docker/" "$versions_file" | grep -v "^#") + content=$(grep " docker/" "$versions_file"; grep -v " docker/" "$versions_file" ||: | grep -v "^#") echo "$content" | while IFS= read -r line; do [ -z "$line" ] && continue line=$(echo "$line" | sed "s/^#\s*//; s/\s*#.*$//" | sed "s;docker/;;") diff --git a/scripts/docker/docker.mk b/scripts/docker/docker.mk index 83e26b4..49fc456 100644 --- a/scripts/docker/docker.mk +++ b/scripts/docker/docker.mk @@ -45,6 +45,7 @@ docker-example-build: # Build Docker example @ExamplesAndTests cd scripts/docker/examples/python DOCKER_IMAGE=repository-template/docker-example-python DOCKER_TITLE="Repository Template Docker Python Example" + TOOL_VERSIONS="$(shell git rev-parse --show-toplevel)/scripts/docker/examples/python/.tool-versions.example" docker-build docker-example-lint: # Lint Docker example @ExamplesAndTests diff --git a/scripts/docker/dockerfile-linter.sh b/scripts/docker/dockerfile-linter.sh index a81e5da..6cb2623 100755 --- a/scripts/docker/dockerfile-linter.sh +++ b/scripts/docker/dockerfile-linter.sh @@ -46,7 +46,8 @@ function docker-run-hadolint() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh - image=$(name=hadolint/hadolint docker-get-image-version-and-pull) + # shellcheck disable=SC2155 + local image=$(name=hadolint/hadolint docker-get-image-version-and-pull) # shellcheck disable=SC2001 docker run --rm --platform linux/amd64 \ --volume "$PWD:/workdir" \ diff --git a/scripts/docker/examples/python/.tool-versions.example b/scripts/docker/examples/python/.tool-versions.example new file mode 100644 index 0000000..9209311 --- /dev/null +++ b/scripts/docker/examples/python/.tool-versions.example @@ -0,0 +1,2 @@ +# python, SEE: https://hub.docker.com/_/python/tags +# docker/python 3.11.4-alpine3.18@sha256:0135ae6442d1269379860b361760ad2cf6ab7c403d21935a8015b48d5bf78a86 diff --git a/scripts/docker/examples/python/assets/hello_world/requirements.txt b/scripts/docker/examples/python/assets/hello_world/requirements.txt index 09909f8..ae33eb7 100644 --- a/scripts/docker/examples/python/assets/hello_world/requirements.txt +++ b/scripts/docker/examples/python/assets/hello_world/requirements.txt @@ -1,2 +1,12 @@ -flask==2.3.2 -Flask-WTF==1.1.1 +blinker==1.6.2 +click==8.1.7 +Flask-WTF==1.2.0 +Flask==2.3.3 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.3 +pip==23.1.2 +setuptools==65.5.1 +Werkzeug==3.0.0 +wheel==0.41.1 +WTForms==3.0.1 diff --git a/scripts/docker/tests/.tool-versions.test b/scripts/docker/tests/.tool-versions.test new file mode 100644 index 0000000..9209311 --- /dev/null +++ b/scripts/docker/tests/.tool-versions.test @@ -0,0 +1,2 @@ +# python, SEE: https://hub.docker.com/_/python/tags +# docker/python 3.11.4-alpine3.18@sha256:0135ae6442d1269379860b361760ad2cf6ab7c403d21935a8015b48d5bf78a86 diff --git a/scripts/docker/tests/docker.test.sh b/scripts/docker/tests/docker.test.sh index 4f66887..2f5b764 100755 --- a/scripts/docker/tests/docker.test.sh +++ b/scripts/docker/tests/docker.test.sh @@ -74,6 +74,7 @@ function test-docker-build() { function test-docker-image-from-signature() { # Arrange + TOOL_VERSIONS="$(git rev-parse --show-toplevel)/scripts/docker/tests/.tool-versions.test" cp Dockerfile Dockerfile.effective # Act _replace-image-latest-by-specific-version @@ -143,7 +144,7 @@ function test-docker-get-image-version-and-pull() { # ============================================================================== -function is_arg_true() { +function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then return 0 @@ -154,7 +155,7 @@ function is_arg_true() { # ============================================================================== -is_arg_true "${VERBOSE:-false}" && set -x +is-arg-true "${VERBOSE:-false}" && set -x main "$@" diff --git a/scripts/githooks/check-file-format.sh b/scripts/githooks/check-file-format.sh index b15d145..965f90c 100755 --- a/scripts/githooks/check-file-format.sh +++ b/scripts/githooks/check-file-format.sh @@ -1,6 +1,8 @@ #!/bin/bash -set +e +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail # Pre-commit git hook to check the EditorConfig rules compliance over changed # files. It ensures all non-binary files across the codebase are formatted @@ -17,7 +19,6 @@ set +e # 0 - All files are formatted correctly # 1 - Files are not formatted correctly # -# # The `check` parameter controls which files are checked, so you can # limit the scope of the check according to what is appropriate at the # point the check is being applied. @@ -45,9 +46,10 @@ image_version=2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2 function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" - is-arg-true "$dry_run" && dry_run_opt="--dry-run" + # shellcheck disable=SC2154 + is-arg-true "${dry_run:-false}" && dry_run_opt="--dry-run" check=${check:-working-tree-changes} case $check in @@ -68,16 +70,17 @@ function main() { ;; esac - # We use /dev/null here as a backstop in case there are no files in the state - # we choose. If the filter comes back empty, adding `/dev/null` onto it has + # we choose. If the filter comes back empty, adding `/dev/null` onto it has # the effect of preventing `ec` from treating "no files" as "all the files". docker run --rm --platform linux/amd64 \ - --volume=$PWD:/check \ + --volume "$PWD":/check \ mstruebing/editorconfig-checker:$image_version \ - sh -c "ec --exclude '.git/' $dry_run_opt \$($filter) /dev/null" + sh -c "ec --exclude '.git/' ${dry_run_opt:-} \$($filter) /dev/null" } +# ============================================================================== + function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then @@ -89,9 +92,8 @@ function is-arg-true() { # ============================================================================== +is-arg-true "${VERBOSE:-false}" && set -x -is-arg-true "$VERBOSE" && set -x - -main $* +main "$@" exit 0 diff --git a/scripts/githooks/check-markdown-format.sh b/scripts/githooks/check-markdown-format.sh index 9fcb8bc..23e1453 100755 --- a/scripts/githooks/check-markdown-format.sh +++ b/scripts/githooks/check-markdown-format.sh @@ -1,16 +1,17 @@ #!/bin/bash -set -e +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail # Pre-commit git hook to check the Markdown file formatting rules compliance # over changed files. # # Usage: -# $ ./check-markdown-format.sh +# $ check={all,staged-changes,working-tree-changes,branch} ./check-markdown-format.sh # # Options: # BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` -# ALL_FILES=true # Check all files, default is `false` # VERBOSE=true # Show all the executed commands, default is `false` # # Exit codes: @@ -27,31 +28,42 @@ set -e # ============================================================================== # SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli, use the `linux/amd64` os/arch -image_version=v0.35.0@sha256:4ec089301e2e3e1298424f4d2b5d9e18af3aa005402590770c339b6637100dc6 +image_version=v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d # ============================================================================== function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" - if is-arg-true "$ALL_FILES"; then - # Check all files - files="$(find ./ -type f -name "*.md")" - else - # Check changed files only - files="$( (git diff --diff-filter=ACMRT --name-only ${BRANCH_NAME:-origin/main} "*.md"; git diff --name-only "*.md") | sort | uniq )" - fi + check=${check:-working-tree-changes} + case $check in + "all") + files="$(find ./ -type f -name "*.md")" + ;; + "staged-changes") + files="$(git diff --diff-filter=ACMRT --name-only --cached "*.md")" + ;; + "working-tree-changes") + files="$(git diff --diff-filter=ACMRT --name-only "*.md")" + ;; + "branch") + files="$( (git diff --diff-filter=ACMRT --name-only "${BRANCH_NAME:-origin/main}" "*.md"; git diff --name-only "*.md") | sort | uniq )" + ;; + esac if [ -n "$files" ]; then + # shellcheck disable=SC2086 docker run --rm --platform linux/amd64 \ - --volume $PWD:/workdir \ + --volume "$PWD":/workdir \ ghcr.io/igorshubovych/markdownlint-cli:$image_version \ $files \ --config /workdir/scripts/config/markdownlint.yaml fi } +# ============================================================================== + function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then @@ -63,8 +75,8 @@ function is-arg-true() { # ============================================================================== -is-arg-true "$VERBOSE" && set -x +is-arg-true "${VERBOSE:-false}" && set -x -main $* +main "$@" exit 0 diff --git a/scripts/githooks/check-terraform-format.sh b/scripts/githooks/check-terraform-format.sh index b76d469..b387c33 100755 --- a/scripts/githooks/check-terraform-format.sh +++ b/scripts/githooks/check-terraform-format.sh @@ -18,7 +18,7 @@ set -euo pipefail function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" local check_only=${check_only:-false} check_only=$check_only terraform-fmt @@ -36,6 +36,8 @@ function terraform-fmt() { opts=$opts make terraform-fmt } +# ============================================================================== + function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh index 5f4a570..9e7f3dc 100755 --- a/scripts/githooks/scan-secrets.sh +++ b/scripts/githooks/scan-secrets.sh @@ -1,15 +1,18 @@ #!/bin/bash -set -e +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. -# Pre-commit git hook to scan for secrets hardcoded in the codebase. +set -euo pipefail + +# Pre-commit git hook to scan for secrets hard-coded in the codebase. # # Usage: # $ ./scan-secrets.sh # # Options: -# ALL_FILES=true # Scan whole git history or 'last-commit', default is `false` -# VERBOSE=true # Show all the executed commands, default is `false` +# check={whole-history,last-commit,staged-changes} # Type of the check to run, default is 'staged-changes' +# VERBOSE=true # Show all the executed commands, default is 'false' +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' # # Exit codes: # 0 - No leaks present @@ -18,39 +21,77 @@ set -e # ============================================================================== -# SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks, use the `linux/amd64` os/arch -image_version=v8.17.0@sha256:99e40155529614d09d264cc886c1326c9a4593ad851ccbeaaed8dcf03ff3d3d7 - -# ============================================================================== - function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" - if is_arg_true "$ALL_FILES"; then - # Scan whole git history - cmd="detect --source /scan --verbose --redact" - elif [ "$ALL_FILES" == "last-commit" ]; then - # Scan the last commit - cmd="detect --source /scan --verbose --redact --log-opts -1" + if command -v gitleaks > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + dir="$PWD" + cmd="$(get-cmd-to-run)" cli-run-gitleaks else - # Scan staged files only - cmd="protect --source /scan --verbose --staged" + dir="/workdir" + cmd="$(get-cmd-to-run)" docker-run-gitleaks fi +} + +# Get Gitleaks command to execute and configuration. +# Arguments (provided as environment variables): +# dir=[project's top-level directory] +function get-cmd-to-run() { + + check=${check:-staged-changes} + case $check in + "whole-history") + cmd="detect --source $dir --verbose --redact" + ;; + "last-commit") + cmd="detect --source $dir --verbose --redact --log-opts -1" + ;; + "staged-changes") + cmd="protect --source $dir --verbose --staged" + ;; + esac # Include base line file if it exists - if [ -f $PWD/scripts/config/.gitleaks-baseline.json ]; then - cmd="$cmd --baseline-path /scan/scripts/config/.gitleaks-baseline.json" + if [ -f "$dir/scripts/config/.gitleaks-baseline.json" ]; then + cmd="$cmd --baseline-path $dir/scripts/config/.gitleaks-baseline.json" fi + # Include the config file + cmd="$cmd --config $dir/scripts/config/gitleaks.toml" + + echo "$cmd" +} + +# Run Gitleaks natively. +# Arguments (provided as environment variables): +# cmd=[command to run] +function cli-run-gitleaks() { + + # shellcheck disable=SC2086 + gitleaks $cmd +} +# Run Gitleaks in a Docker container. +# Arguments (provided as environment variables): +# cmd=[command to run] +# dir=[directory to mount as a volume] +function docker-run-gitleaks() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/gitleaks/gitleaks docker-get-image-version-and-pull) + # shellcheck disable=SC2086 docker run --rm --platform linux/amd64 \ - --volume $PWD:/scan \ - --workdir /scan \ - ghcr.io/gitleaks/gitleaks:$image_version \ - $cmd \ - --config /scan/scripts/config/gitleaks.toml + --volume "$PWD:$dir" \ + --workdir $dir \ + "$image" \ + $cmd } -function is_arg_true() { +# ============================================================================== + +function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then return 0 @@ -61,8 +102,8 @@ function is_arg_true() { # ============================================================================== -is_arg_true "$VERBOSE" && set -x +is-arg-true "${VERBOSE:-false}" && set -x -main $* +main "$@" exit 0 diff --git a/scripts/init.mk b/scripts/init.mk index 4f37b31..5b61250 100644 --- a/scripts/init.mk +++ b/scripts/init.mk @@ -29,14 +29,6 @@ shellscript-lint-all: # Lint all shell scripts in this project, do not fail on e file=$${file} scripts/shellscript-linter.sh ||: done -nodejs-install: # Install Node.js @Installation - make _install-dependency name="nodejs" - make _install-dependency name="yarn" version=latest - -python-install: # Install Python @Installation - make _install-dependency name="python" - make _install-dependency name="poetry" version=latest - githooks-config: # Trigger Git hooks on commit that are defined in this repository @Configuration make _install-dependency name="pre-commit" pre-commit install \ @@ -48,14 +40,6 @@ githooks-run: # Run git hooks configured in this repository @Operations --config scripts/config/pre-commit.yaml \ --all-files -asdf-install: # Install asdf @Installation - if [ -d "${HOME}/.asdf" ]; then - ( cd "${HOME}/.asdf"; git pull ) - else - git clone --depth=1 https://github.com/asdf-vm/asdf.git "${HOME}/.asdf" ||: - fi - asdf plugin update --all - _install-dependency: # Install asdf dependency - mandatory: name=[listed in the '.tool-versions' file]; optional: version=[if not listed] asdf plugin add ${name} ||: asdf install ${name} $(or ${version},) @@ -73,7 +57,6 @@ clean:: # Remove all generated and temporary files (common) @Operations config:: # Configure development environment (common) @Configuration make \ - asdf-install \ githooks-config help: # Print help @Others @@ -158,15 +141,12 @@ HELP_SCRIPT = \ ${VERBOSE}.SILENT: \ _install-dependency \ - asdf-install \ clean \ config \ githooks-config \ githooks-run \ help \ list-variables \ - nodejs-install \ - python-install \ runner-act \ shellscript-lint-all \ version-create-effective-file \ diff --git a/scripts/reports/cloc-repository.sh b/scripts/reports/cloc-repository.sh index 34bfeba..062b835 100755 --- a/scripts/reports/cloc-repository.sh +++ b/scripts/reports/cloc-repository.sh @@ -1,6 +1,8 @@ #!/bin/bash -set -e +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail # Count lines of code of this repository. # @@ -20,7 +22,8 @@ image_version=latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b5 function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" + create-report enrich-report } @@ -28,12 +31,13 @@ function main() { function create-report() { docker run --rm --platform linux/amd64 \ - --volume $PWD:/workdir \ + --volume "$PWD":/workdir \ ghcr.io/make-ops-tools/gocloc:$image_version \ --output-type=json \ . \ > cloc-report.tmp.json if which jq > /dev/null && which column > /dev/null; then + # shellcheck disable=SC2002 cat cloc-report.tmp.json | jq -r '["Language","files","blank","comment","code"],["--------"],(.languages[]|[.name,.files,.blank,.comment,.code]),["-----"],(.total|["TOTAL",.files,.blank,.comment,.code])|@tsv' | column -t fi } @@ -44,21 +48,21 @@ function enrich-report() { git_url=$(git config --get remote.origin.url) git_branch=$(git rev-parse --abbrev-ref HEAD) git_commit_hash=$(git rev-parse HEAD) - git_tags=$(echo \"$(git tag | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')\" | sed 's/""//g') + git_tags=$(echo \""$(git tag | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')"\" | sed 's/""//g') pipeline_run_id=${GITHUB_RUN_ID:-0} pipeline_run_number=${GITHUB_RUN_NUMBER:-0} pipeline_run_attempt=${GITHUB_RUN_ATTEMPT:-0} - docker run --rm --platform linux/amd64 \ - --volume $PWD:/repo \ - --workdir /repo \ - ghcr.io/make-ops-tools/jq:latest \ - '.creationInfo |= . + {"created":"'${build_datetime}'","repository":{"url":"'${git_url}'","branch":"'${git_branch}'","tags":['${git_tags}'],"commitHash":"'${git_commit_hash}'"},"pipeline":{"id":'${pipeline_run_id}',"number":'${pipeline_run_number}',"attempt":'${pipeline_run_attempt}'}}' \ - cloc-report.tmp.json \ - > cloc-report.json + # shellcheck disable=SC2086 + jq \ + '.creationInfo |= . + {"created":"'${build_datetime}'","repository":{"url":"'${git_url}'","branch":"'${git_branch}'","tags":['${git_tags}'],"commitHash":"'${git_commit_hash}'"},"pipeline":{"id":'${pipeline_run_id}',"number":'${pipeline_run_number}',"attempt":'${pipeline_run_attempt}'}}' \ + cloc-report.tmp.json \ + > cloc-report.json rm -f cloc-report.tmp.json } +# ============================================================================== + function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then @@ -70,8 +74,8 @@ function is-arg-true() { # ============================================================================== -is-arg-true "$VERBOSE" && set -x +is-arg-true "${VERBOSE:-false}" && set -x -main $* +main "$@" exit 0 diff --git a/scripts/reports/generate-sbom.sh b/scripts/reports/generate-sbom.sh index 62a0428..39fd0b5 100755 --- a/scripts/reports/generate-sbom.sh +++ b/scripts/reports/generate-sbom.sh @@ -1,6 +1,8 @@ #!/bin/bash -set -e +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail # Script to generate SBOM (Software Bill of Materials) for the repository # content and any artefact created by the CI/CD pipeline. @@ -15,13 +17,14 @@ set -e # ============================================================================== # SEE: https://github.com/anchore/syft/pkgs/container/syft, use the `linux/amd64` os/arch -image_version=v0.85.0@sha256:c4f8ac5bb873738d00019cfed80f80b60fa3a8773d4053f415972eb133284d0e +image_version=v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 # ============================================================================== function main() { - cd $(git rev-parse --show-toplevel) + cd "$(git rev-parse --show-toplevel)" + create-report enrich-report } @@ -29,7 +32,7 @@ function main() { function create-report() { docker run --rm --platform linux/amd64 \ - --volume $PWD:/scan \ + --volume "$PWD":/scan \ ghcr.io/anchore/syft:$image_version \ packages dir:/scan \ --config /scan/scripts/config/syft.yaml \ @@ -42,22 +45,22 @@ function enrich-report() { git_url=$(git config --get remote.origin.url) git_branch=$(git rev-parse --abbrev-ref HEAD) git_commit_hash=$(git rev-parse HEAD) - git_tags=$(echo \"$(git tag | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')\" | sed 's/""//g') + git_tags=$(echo \""$(git tag | tr '\n' ',' | sed 's/,$//' | sed 's/,/","/g')"\" | sed 's/""//g') pipeline_run_id=${GITHUB_RUN_ID:-0} pipeline_run_number=${GITHUB_RUN_NUMBER:-0} pipeline_run_attempt=${GITHUB_RUN_ATTEMPT:-0} - docker run --rm --platform linux/amd64 \ - --volume $PWD:/repo \ - --workdir /repo \ - ghcr.io/make-ops-tools/jq:latest \ - '.creationInfo |= . + {"created":"'${build_datetime}'","repository":{"url":"'${git_url}'","branch":"'${git_branch}'","tags":['${git_tags}'],"commitHash":"'${git_commit_hash}'"},"pipeline":{"id":'${pipeline_run_id}',"number":'${pipeline_run_number}',"attempt":'${pipeline_run_attempt}'}}' \ - sbom-repository-report.tmp.json \ - > sbom-repository-report.json + # shellcheck disable=SC2086 + jq \ + '.creationInfo |= . + {"created":"'${build_datetime}'","repository":{"url":"'${git_url}'","branch":"'${git_branch}'","tags":['${git_tags}'],"commitHash":"'${git_commit_hash}'"},"pipeline":{"id":'${pipeline_run_id}',"number":'${pipeline_run_number}',"attempt":'${pipeline_run_attempt}'}}' \ + sbom-repository-report.tmp.json \ + > sbom-repository-report.json rm -f sbom-repository-report.tmp.json } -function is_arg_true() { +# ============================================================================== + +function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then return 0 @@ -68,8 +71,8 @@ function is_arg_true() { # ============================================================================== -is_arg_true "$VERBOSE" && set -x +is-arg-true "${VERBOSE:-false}" && set -x -main $* +main "$@" exit 0 diff --git a/scripts/shellscript-linter.sh b/scripts/shellscript-linter.sh index d97b7c9..108bc2c 100755 --- a/scripts/shellscript-linter.sh +++ b/scripts/shellscript-linter.sh @@ -21,8 +21,8 @@ function main() { cd "$(git rev-parse --show-toplevel)" + [ -z "${file:-}" ] && echo "WARNING: 'file' variable not set, defaulting to itself" local file=${file:-scripts/shellscript-linter.sh} - [ "$file" == "scripts/shellscript-linter.sh" ] && echo "WARNING: 'file' variable not set, defaulting to itself" if command -v shellcheck > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then file="$file" cli-run-shellcheck else diff --git a/scripts/terraform/terraform.mk b/scripts/terraform/terraform.mk index 5740392..f62a02f 100644 --- a/scripts/terraform/terraform.mk +++ b/scripts/terraform/terraform.mk @@ -90,6 +90,7 @@ ${VERBOSE}.SILENT: \ terraform-example-provision-aws-infrastructure \ terraform-fmt \ terraform-init \ + terraform-install \ terraform-plan \ terraform-shellscript-lint \ terraform-validate \ diff --git a/scripts/terraform/terraform.sh b/scripts/terraform/terraform.sh index fc93530..7d61de4 100755 --- a/scripts/terraform/terraform.sh +++ b/scripts/terraform/terraform.sh @@ -46,6 +46,7 @@ function docker-run-terraform() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh + # shellcheck disable=SC2155 local image=$(name=hashicorp/terraform docker-get-image-version-and-pull) # shellcheck disable=SC2086 docker run --rm --platform linux/amd64 \ @@ -55,6 +56,8 @@ function docker-run-terraform() { $cmd } +# ============================================================================== + function is-arg-true() { if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then