From 42206a274063f4a3f178bb7e15c0bd943e655e79 Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 14:12:43 +0100 Subject: [PATCH 01/36] OPSEXP-2849 Allow building and testing multiple versions of ACS --- .github/workflows/build_and_test.yml | 287 +--------------- .../workflows/reusuable_build_and_test.yml | 325 ++++++++++++++++++ adf-apps/acc/artifacts.json | 10 + adf-apps/adw/artifacts.json | 10 + ats/sfs/artifacts.json | 10 + ats/trouter/artifacts.json | 10 + connector/ms365/artifacts.json | 10 + connector/msteams/artifacts.json | 10 + repository/artifacts.json | 58 ++++ scripts/fetch-artifacts.sh | 3 +- search/enterprise/all-in-one/artifacts.json | 10 + search/enterprise/common/artifacts.json | 42 +++ search/enterprise/reindexing/artifacts.json | 10 + search/service/artifacts.json | 10 + share/artifacts.json | 26 ++ sync/artifacts.json | 18 + tengine/aio/artifacts.json | 10 + tengine/imagemagick/artifacts.json | 42 +++ tengine/libreoffice/artifacts.json | 18 + tengine/misc/artifacts.json | 10 + tengine/pdfrenderer/artifacts.json | 26 ++ tengine/tika/artifacts.json | 10 + 22 files changed, 686 insertions(+), 279 deletions(-) create mode 100644 .github/workflows/reusuable_build_and_test.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index aa48b17f..68b14182 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -18,13 +18,6 @@ on: - '.github/workflows/kics.yml' - '**/*.md' -env: - ACS_DEPLOYMENT_VERSION: 78132d95e29d7126025e127740d10cf958164947 # 8.6.0-alpha.0 with compose healthcheck - ARTIFACT_NAME: alfresco-docker-images - REGISTRY: ghcr.io - REGISTRY_NAMESPACE: alfresco - TAG: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }} - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} cancel-in-progress: true @@ -48,277 +41,17 @@ jobs: - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v8.2.0 with: auto-commit: "true" - - build: + CI: needs: pre-commit - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v8.2.0 - - - name: Setup nexus authentication - run: | - echo "machine nexus.alfresco.com" >> ~/.netrc - echo "login ${{ secrets.NEXUS_USERNAME }}" >> ~/.netrc - echo "password ${{ secrets.NEXUS_PASSWORD }}" >> ~/.netrc - - - name: Restore packages artifacts - uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - id: artifacts-cache - with: - key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }} - path: artifacts_cache/** - - - name: Fetch artifacts from nexus - run: ./scripts/fetch-artifacts.sh - - - name: Save packages artifacts - if: steps.artifacts-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} - path: artifacts_cache/** - - - name: Set up QEMU - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Login to Quay.io - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - if: github.actor != 'dependabot[bot]' - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - if: github.actor != 'dependabot[bot]' - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Collect Workflow Telemetry - uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 - with: - comment_on_pr: false - - - name: Enumerate bake targets - id: bake-targets - uses: docker/bake-action/subaction/list-targets@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 - with: - target: default - - - name: Enumerate registry cache targets - id: cache-targets - env: - CACHE_TARGET: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/bakery-cache - run: | - echo 'cache-set<> $GITHUB_OUTPUT - echo '${{ steps.bake-targets.outputs.targets }}' | jq -r '.[] | '\ - '"\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)\n'\ - '\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ github.event.repository.default_branch }}-\(.)\n'\ - '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)"' >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT - - - name: Bake Docker images - env: - TARGETARCH: linux/amd64,linux/arm64 - DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} - uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 - with: - set: | - *.output=type=registry,push=${{ env.DOCKER_PUSH }} - ${{ steps.cache-targets.outputs.cache-set }} - - compose-test: - needs: build - runs-on: ${{ matrix.edition == 'enterprise' && 'alfrescoPub-ubuntu2204-16G-4CPU' || 'ubuntu-latest' }} - if: github.actor != 'dependabot[bot]' + uses: ./.github/workflows/reusuable_build_and_test.yml strategy: fail-fast: false matrix: - edition: [enterprise, community] - env: - MERGED_COMPOSE_PATH: test/merged-compose.yaml - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Fetch compose from acs-deployment - run: ./scripts/fetch-compose.sh ${{ env.ACS_DEPLOYMENT_VERSION }} - - - name: Merge Docker Compose files - env: - UPSTREAM_COMPOSE_PATH: test/${{ matrix.edition == 'community' && 'community-' || '' }}compose.yaml - OVERRIDE_COMPOSE_PATH: test/${{ matrix.edition }}-override.yaml - run: docker compose -f ${{ env.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} - - - name: Verify docker-compose (${{ matrix.edition }}) - id: verify_compose - uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v8.2.0 - timeout-minutes: 10 - with: - postman_path: test/postman/docker-compose - postman_json: acs-test-docker-compose-collection.json - compose_file_path: ${{ env.MERGED_COMPOSE_PATH }} - quay_username: ${{ secrets.QUAY_USERNAME }} - quay_password: ${{ secrets.QUAY_PASSWORD }} - - - name: Save containers logs (${{ matrix.edition }}) - if: always() && steps.verify_compose.outcome != 'skipped' - uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v8.2.0 - with: - output-archive-name: ${{ matrix.edition }}-logs - - helm-test: - needs: build - runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} - if: github.actor != 'dependabot[bot]' - strategy: - fail-fast: false - matrix: - arch: [arm64, amd64] - edition: [enterprise, community] - steps: - - name: Setup cluster - uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v8.2.0 - with: - ingress-nginx-ref: controller-v1.8.2 - metrics: "true" - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: "3.15.2" - - - name: Login to Quay.io - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create registries auth secret - run: >- - kubectl create secret generic regcred - --from-file=.dockerconfigjson=$HOME/.docker/config.json - --type=kubernetes.io/dockerconfigjson - - - name: Create configmaps for adf apps - run: | - kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json - kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json - - - name: Checkout acs-deployment sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: Alfresco/acs-deployment - ref: ${{ env.ACS_DEPLOYMENT_VERSION }} - path: acs-deployment - - - name: Setup helm repository - working-directory: acs-deployment/helm/alfresco-content-services - run: | - helm repo add self https://alfresco.github.io/alfresco-helm-charts/ - helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts - helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ - helm repo add elastic https://helm.elastic.co - helm dependency build - - - name: Preprocess test-overrides.yaml - env: - OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml - run: | - sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} - - if [ "${{ matrix.edition }}" = "community" ]; then - sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - fi - - cat ${{ env.OVERRIDES_VALUES_FILE }} - - - name: Helm install - id: helm_install - run: | - helm install acs ./acs-deployment/helm/alfresco-content-services \ - --set global.search.sharedSecret="$(openssl rand -hex 24)" \ - --set global.known_urls=http://localhost \ - --set global.alfrescoRegistryPullSecrets=regcred \ - --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ - --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ - --values test/helm/test-overrides.yaml \ - --values test/helm/test-overrides-${{ matrix.edition }}.yaml - - - name: Watch Helm deployment - env: - HELM_INSTALL_TIMEOUT: 10m - run: | - kubectl get pods --watch & - KWPID=$! - kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods - kill $KWPID - - if [ "${{ matrix.edition }}" = "enterprise" ]; then - echo "Waiting for Enterprise Search Reindexing job to complete... " - kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing - fi - - - name: Debug cluster status after install - if: always() && steps.helm_install.outcome != 'skipped' - run: | - helm ls --all-namespaces --all - kubectl get all --all-namespaces - kubectl describe pod - - - name: Run helm test - id: helm_test - run: helm test acs - - - name: Debug cluster status after helm test - if: always() && steps.helm_test.outcome != 'skipped' - run: | - kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 - kubectl get all --all-namespaces - kubectl describe pod - - - name: Collect logs from all containers - if: always() && steps.helm_install.outcome != 'skipped' - run: | - mkdir -p logs - for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do - kubectl logs $pod -n default > logs/${pod}.log - done - - - name: Upload logs as artifact - if: always() && steps.helm_install.outcome != 'skipped' - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 - with: - name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }} - path: logs + version: [23, 74] + with: + registry: ghcr.io + registry_namespace: alfresco + tag: ${{ github.event_name == 'pull_request' && format('pr-{0}-v{1}', github.event.pull_request.number, matrix.version) || format('{0}-v{1}', github.ref_name, matrix.version) }} + acs_deployment_version: 78132d95e29d7126025e127740d10cf958164947 # 8.6.0-alpha.0 with compose healthcheck + acs_version: ${{ matrix.version }} + secrets: inherit diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml new file mode 100644 index 00000000..0265880c --- /dev/null +++ b/.github/workflows/reusuable_build_and_test.yml @@ -0,0 +1,325 @@ +name: CI + +on: + workflow_call: + inputs: + registry: + description: 'Container registry' + required: true + type: string + registry_namespace: + description: 'Registry namespace' + required: true + type: string + tag: + description: 'Tag for the Docker images' + required: true + type: string + acs_version: + description: 'Version of ACS to build' + required: true + type: string + acs_deployment_version: + description: 'ACS deployment version' + required: true + type: string + secrets: + QUAY_USERNAME: + description: 'Quay.io username' + required: true + QUAY_PASSWORD: + description: 'Quay.io password' + required: true + GITHUB_TOKEN: + description: 'GitHub token' + required: true + NEXUS_USERNAME: + description: 'Nexus username' + required: true + NEXUS_PASSWORD: + description: 'Nexus password' + required: true + +jobs: + pre-commit: + runs-on: ubuntu-latest + if: '! github.event.pull_request.head.repo.fork' + permissions: + contents: write + steps: + - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.1.1 + with: + auto-commit: "true" + + build: + needs: pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.1.1 + + - name: Setup nexus authentication + run: | + echo "machine nexus.alfresco.com" >> ~/.netrc + echo "login ${{ secrets.NEXUS_USERNAME }}" >> ~/.netrc + echo "password ${{ secrets.NEXUS_PASSWORD }}" >> ~/.netrc + + - name: Restore packages artifacts + uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + id: artifacts-cache + with: + key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }} + path: artifacts_cache/** + + - name: Fetch artifacts from nexus + run: ./scripts/fetch-artifacts.sh ${{ inputs.acs_version }} + + - name: Save packages artifacts + if: steps.artifacts-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + with: + key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} + path: artifacts_cache/** + + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Login to Quay.io + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + if: github.actor != 'dependabot[bot]' + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + if: github.actor != 'dependabot[bot]' + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Collect Workflow Telemetry + uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 + with: + comment_on_pr: false + + - name: Enumerate bake targets + id: bake-targets + uses: docker/bake-action/subaction/list-targets@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 + with: + target: default + + - name: Enumerate registry cache targets + id: cache-targets + env: + CACHE_TARGET: ${{ inputs.registry }}/${{ inputs.registry_namespace }}/bakery-cache + run: | + echo 'cache-set<> $GITHUB_OUTPUT + echo '${{ steps.bake-targets.outputs.targets }}' | jq -r '.[] | '\ + '"\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)\n'\ + '\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ github.event.repository.default_branch }}-\(.)\n'\ + '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)"' >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Bake Docker images + env: + TARGETARCH: linux/amd64,linux/arm64 + DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} + uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 + with: + set: | + *.output=type=registry,push=${{ env.DOCKER_PUSH }} + ${{ steps.cache-targets.outputs.cache-set }} + + compose-test: + needs: build + runs-on: ${{ matrix.edition == 'enterprise' && 'alfrescoPub-ubuntu2204-16G-4CPU' || 'ubuntu-latest' }} + if: github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: + edition: [enterprise, community] + env: + UPSTREAM_COMPOSE_PATH: test/${{ matrix.edition }}.yaml + OVERRIDE_COMPOSE_PATH: test/${{ matrix.edition }}-override.yaml + MERGED_COMPOSE_PATH: test/merged-compose.yaml + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Fetch compose from acs-deployment + run: ./scripts/fetch-compose.sh ${{ matrix.edition }} ${{ env.UPSTREAM_COMPOSE_PATH }} ${{ inputs.acs_deployment_version }} + + - name: Merge Docker Compose files + run: docker compose -f ${{ env.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} + + - name: Verify docker-compose (${{ matrix.edition }}) + id: verify_compose + uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v8.1.0 + timeout-minutes: 10 + with: + postman_path: test/postman/docker-compose + postman_json: acs-test-docker-compose-collection.json + compose_file_path: ${{ env.MERGED_COMPOSE_PATH }} + quay_username: ${{ secrets.QUAY_USERNAME }} + quay_password: ${{ secrets.QUAY_PASSWORD }} + + - name: Save containers logs (${{ matrix.edition }}) + if: always() && steps.verify_compose.outcome != 'skipped' + uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v8.1.0 + with: + output-archive-name: ${{ matrix.edition }}-logs-${{ inputs.acs_version }} + + helm-test: + needs: build + runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} + if: github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: + arch: [arm64, amd64] + edition: [enterprise, community] + steps: + - name: Setup cluster + uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 + with: + ingress-nginx-ref: controller-v1.8.2 + metrics: "true" + + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: "3.15.2" + + - name: Login to Quay.io + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create registries auth secret + run: >- + kubectl create secret generic regcred + --from-file=.dockerconfigjson=$HOME/.docker/config.json + --type=kubernetes.io/dockerconfigjson + + - name: Create configmaps for adf apps + run: | + kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json + kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json + + - name: Checkout acs-deployment sources + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + repository: Alfresco/acs-deployment + ref: ${{ inputs.acs_deployment_version }} + path: acs-deployment + + - name: Setup helm repository + working-directory: acs-deployment/helm/alfresco-content-services + run: | + helm repo add self https://alfresco.github.io/alfresco-helm-charts/ + helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts + helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ + helm repo add elastic https://helm.elastic.co + helm dependency build + + - name: Preprocess test-overrides.yaml + env: + OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml + run: | + sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} + + if [ "${{ matrix.edition }}" = "community" ]; then + sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + fi + + cat ${{ env.OVERRIDES_VALUES_FILE }} + + - name: Helm install + id: helm_install + run: | + helm install acs ./acs-deployment/helm/alfresco-content-services \ + --set global.search.sharedSecret="$(openssl rand -hex 24)" \ + --set global.known_urls=http://localhost \ + --set global.alfrescoRegistryPullSecrets=regcred \ + --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ + --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ + --values test/helm/test-overrides.yaml \ + --values test/helm/test-overrides-${{ matrix.edition }}.yaml + + - name: Watch Helm deployment + env: + HELM_INSTALL_TIMEOUT: 10m + run: | + kubectl get pods --watch & + KWPID=$! + kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods + kill $KWPID + + if [ "${{ matrix.edition }}" = "enterprise" ]; then + echo "Waiting for Enterprise Search Reindexing job to complete... " + kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing + fi + + - name: Debug cluster status after install + if: always() && steps.helm_install.outcome != 'skipped' + run: | + helm ls --all-namespaces --all + kubectl get all --all-namespaces + kubectl describe pod + + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod + + - name: Collect logs from all containers + if: always() && steps.helm_install.outcome != 'skipped' + run: | + mkdir -p logs + for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do + kubectl logs $pod -n default > logs/${pod}.log + done + + - name: Upload logs as artifact + if: always() && steps.helm_install.outcome != 'skipped' + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 + with: + name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }}-${{ inputs.acs_version}} + path: logs diff --git a/adf-apps/acc/artifacts.json b/adf-apps/acc/artifacts.json index 4df6cb0a..960f10c8 100644 --- a/adf-apps/acc/artifacts.json +++ b/adf-apps/acc/artifacts.json @@ -9,6 +9,16 @@ "group": "org.alfresco", "repository": "releases" } + ], + "acs74": [ + { + "name": "alfresco-control-center", + "version": "8.3.0", + "path": "adf-apps/acc", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "releases" + } ] } } diff --git a/adf-apps/adw/artifacts.json b/adf-apps/adw/artifacts.json index 8f7d87e3..5dca7bce 100644 --- a/adf-apps/adw/artifacts.json +++ b/adf-apps/adw/artifacts.json @@ -9,6 +9,16 @@ "group": "org.alfresco", "repository": "enterprise-releases" } + ], + "acs74": [ + { + "name": "alfresco-digital-workspace", + "version": "4.4.1", + "path": "adf-apps/adw", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "public" + } ] } } diff --git a/ats/sfs/artifacts.json b/ats/sfs/artifacts.json index f23d7655..6a834f59 100644 --- a/ats/sfs/artifacts.json +++ b/ats/sfs/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-shared-file-store-controller", + "version": "4.1.4", + "path": "ats/sfs", + "classifier": ".jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/ats/trouter/artifacts.json b/ats/trouter/artifacts.json index 5cc285b3..c753b9b2 100644 --- a/ats/trouter/artifacts.json +++ b/ats/trouter/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-transform-router", + "version": "4.1.4", + "path": "ats/trouter", + "classifier": ".jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/connector/ms365/artifacts.json b/connector/ms365/artifacts.json index a03735ab..65e69b53 100644 --- a/connector/ms365/artifacts.json +++ b/connector/ms365/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "onedrive-springboot", + "version": "2.0.3", + "path": "connector/ms365", + "classifier": ".jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/connector/msteams/artifacts.json b/connector/msteams/artifacts.json index e270c353..2bcefd85 100644 --- a/connector/msteams/artifacts.json +++ b/connector/msteams/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-ms-teams-springboot", + "version": "2.0.3", + "path": "connector/msteams", + "classifier": ".jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/repository/artifacts.json b/repository/artifacts.json index 97958a0b..e982cbc9 100644 --- a/repository/artifacts.json +++ b/repository/artifacts.json @@ -57,6 +57,64 @@ "group": "org.alfresco", "repository": "public" } + ], + "acs74": [ + { + "name": "postgresql", + "version": "42.6.1", + "classifier": ".jar", + "group": "org.postgresql", + "repository": "public", + "path": "repository/libs" + }, + { + "name": "alfresco-share-services", + "version": "22.23", + "path": "repository/amps", + "classifier": ".amp", + "group": "org.alfresco", + "repository": "releases" + }, + { + "name": "alfresco-aos-module", + "version": "1.6.3", + "path": "repository/amps", + "classifier": ".amp", + "group": "org.alfresco.aos-module", + "repository": "releases" + }, + { + "name": "alfresco-device-sync-repo", + "version": "3.11.3", + "path": "repository/amps_enterprise", + "classifier": ".amp", + "group": "org.alfresco.services.sync", + "repository": "enterprise-releases" + }, + { + "name": "alfresco-googledrive-repo-enterprise", + "version": "3.4.2", + "path": "repository/amps_enterprise", + "classifier": ".amp", + "group": "org.alfresco.integrations", + "repository": "enterprise-releases" + }, + { + "name": "alfresco-content-services-distribution", + "version": "7.4.2.1", + "path": "repository/distribution", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "enterprise-releases" + }, + { + "name": "alfresco-content-services-community-distribution", + "version": "7.4.2", + "path": "repository/distribution", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "public" + } ] } } diff --git a/scripts/fetch-artifacts.sh b/scripts/fetch-artifacts.sh index c556ddad..72b2c1f9 100755 --- a/scripts/fetch-artifacts.sh +++ b/scripts/fetch-artifacts.sh @@ -4,8 +4,7 @@ REPO_ROOT="$(dirname $0)/.." TEMP_DIR=$(mktemp -d) -ACS_VERSION=${ACS_VERSION:=23.2.2} -INDEX_KEY=${ACS_VERSION%%.*} +INDEX_KEY=$1 do_fetch_mvn() { for i in $(jq -r ".artifacts.acs${INDEX_KEY} | keys | .[]" $1); do diff --git a/search/enterprise/all-in-one/artifacts.json b/search/enterprise/all-in-one/artifacts.json index ecc1cc5b..f1590ff4 100644 --- a/search/enterprise/all-in-one/artifacts.json +++ b/search/enterprise/all-in-one/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-elasticsearch-live-indexing", + "version": "3.3.3", + "path": "search/enterprise/all-in-one", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/search/enterprise/common/artifacts.json b/search/enterprise/common/artifacts.json index 0774151b..dd4794e6 100644 --- a/search/enterprise/common/artifacts.json +++ b/search/enterprise/common/artifacts.json @@ -41,6 +41,48 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-elasticsearch-live-indexing-mediation", + "version": "3.3.3", + "path": "search/enterprise/common", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-elasticsearch-live-indexing-metadata", + "version": "3.3.3", + "path": "search/enterprise/common", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-elasticsearch-live-indexing-path", + "version": "3.3.3", + "path": "search/enterprise/common", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-elasticsearch-live-indexing-content", + "version": "3.3.3", + "path": "search/enterprise/common", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-elasticsearch-live-indexing", + "version": "3.3.3", + "path": "search/enterprise/common", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/search/enterprise/reindexing/artifacts.json b/search/enterprise/reindexing/artifacts.json index cf832683..eb7c1832 100644 --- a/search/enterprise/reindexing/artifacts.json +++ b/search/enterprise/reindexing/artifacts.json @@ -9,6 +9,16 @@ "repository": "enterprise-releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-elasticsearch-reindexing", + "version": "3.3.3", + "path": "search/enterprise/reindexing", + "classifier": "-app.jar", + "repository": "enterprise-releases", + "group": "org.alfresco" + } ] } } diff --git a/search/service/artifacts.json b/search/service/artifacts.json index 6a7cccce..ab299895 100644 --- a/search/service/artifacts.json +++ b/search/service/artifacts.json @@ -9,6 +9,16 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-search-services", + "version": "2.0.12", + "path": "search/service/distribution", + "classifier": ".zip", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/share/artifacts.json b/share/artifacts.json index 8b232543..5ec2f7ea 100644 --- a/share/artifacts.json +++ b/share/artifacts.json @@ -25,6 +25,32 @@ "group": "org.alfresco.integrations", "repository": "releases" } + ], + "acs74": [ + { + "name": "alfresco-content-services-share-distribution", + "version": "7.4.2.1", + "path": "share/distribution", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "enterprise-releases" + }, + { + "name": "alfresco-content-services-community-distribution", + "version": "7.4.2", + "path": "share/distribution", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "public" + }, + { + "name": "alfresco-googledrive-share", + "version": "3.4.2", + "path": "share/amps", + "classifier": ".amp", + "group": "org.alfresco.integrations", + "repository": "releases" + } ] } } diff --git a/sync/artifacts.json b/sync/artifacts.json index baabeee2..17d41464 100644 --- a/sync/artifacts.json +++ b/sync/artifacts.json @@ -17,6 +17,24 @@ "repository": "public", "path": "sync/configs/connectors" } + ], + "acs74": [ + { + "name": "sync-dist-6.x", + "version": "3.11.3", + "path": "sync/distribution", + "classifier": ".zip", + "group": "org.alfresco.services.sync", + "repository": "enterprise-releases" + }, + { + "name": "postgresql", + "version": "42.6.1", + "classifier": ".jar", + "group": "org.postgresql", + "repository": "public", + "path": "sync/configs/connectors" + } ] } } diff --git a/tengine/aio/artifacts.json b/tengine/aio/artifacts.json index 11740a08..9fd32092 100644 --- a/tengine/aio/artifacts.json +++ b/tengine/aio/artifacts.json @@ -9,6 +9,16 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-transform-core-aio", + "version": "5.1.4", + "path": "tengine/aio", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/tengine/imagemagick/artifacts.json b/tengine/imagemagick/artifacts.json index 46fe945f..e4b34ead 100644 --- a/tengine/imagemagick/artifacts.json +++ b/tengine/imagemagick/artifacts.json @@ -41,6 +41,48 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "imagemagick-distribution", + "version": "7.1.0-16-ci-10", + "path": "tengine/imagemagick/libs", + "classifier": "-libs-el9.rpm", + "repository": "thirdparty", + "group": "org.imagemagick" + }, + { + "name": "imagemagick-distribution", + "version": "7.1.0-16-ci-10", + "path": "tengine/imagemagick", + "classifier": "-el9.rpm", + "repository": "thirdparty", + "group": "org.imagemagick" + }, + { + "name": "imagemagick-distribution", + "version": "7.1.0-16-ci-10", + "path": "tengine/imagemagick/libs", + "classifier": "-libs-el9-aarch64.rpm", + "repository": "thirdparty", + "group": "org.imagemagick" + }, + { + "name": "imagemagick-distribution", + "version": "7.1.0-16-ci-10", + "path": "tengine/imagemagick", + "classifier": "-el9-aarch64.rpm", + "repository": "thirdparty", + "group": "org.imagemagick" + }, + { + "name": "alfresco-transform-imagemagick", + "version": "5.1.4", + "path": "tengine/imagemagick", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/tengine/libreoffice/artifacts.json b/tengine/libreoffice/artifacts.json index 5bcbd54a..c720b08f 100644 --- a/tengine/libreoffice/artifacts.json +++ b/tengine/libreoffice/artifacts.json @@ -17,6 +17,24 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "libreoffice-dist", + "version": "7.2.5", + "path": "tengine/libreoffice", + "classifier": "-linux.gz", + "repository": "thirdparty", + "group": "org.libreoffice" + }, + { + "name": "alfresco-transform-libreoffice", + "version": "5.1.4", + "path": "tengine/libreoffice", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/tengine/misc/artifacts.json b/tengine/misc/artifacts.json index a7cecca8..42d2c9ec 100644 --- a/tengine/misc/artifacts.json +++ b/tengine/misc/artifacts.json @@ -9,6 +9,16 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-transform-misc", + "version": "5.1.4", + "path": "tengine/misc", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/tengine/pdfrenderer/artifacts.json b/tengine/pdfrenderer/artifacts.json index e686fde3..a8481c2b 100644 --- a/tengine/pdfrenderer/artifacts.json +++ b/tengine/pdfrenderer/artifacts.json @@ -25,6 +25,32 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-pdf-renderer", + "version": "1.2", + "path": "tengine/pdfrenderer", + "classifier": "-linux.tgz", + "repository": "releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-pdf-renderer", + "version": "1.2", + "path": "tengine/pdfrenderer", + "classifier": "-linux-arm.tgz", + "repository": "releases", + "group": "org.alfresco" + }, + { + "name": "alfresco-transform-pdf-renderer", + "version": "5.1.4", + "path": "tengine/pdfrenderer", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } diff --git a/tengine/tika/artifacts.json b/tengine/tika/artifacts.json index 132f24a3..d5ca131d 100644 --- a/tengine/tika/artifacts.json +++ b/tengine/tika/artifacts.json @@ -9,6 +9,16 @@ "repository": "releases", "group": "org.alfresco" } + ], + "acs74": [ + { + "name": "alfresco-transform-tika", + "version": "5.1.4", + "path": "tengine/tika", + "classifier": ".jar", + "repository": "releases", + "group": "org.alfresco" + } ] } } From 54309ab709dbb9cd55ec97dda54f1ad7cc809e70 Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 14:15:34 +0100 Subject: [PATCH 02/36] test --- .github/workflows/reusuable_build_and_test.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 0265880c..a8480bc1 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -23,22 +23,6 @@ on: description: 'ACS deployment version' required: true type: string - secrets: - QUAY_USERNAME: - description: 'Quay.io username' - required: true - QUAY_PASSWORD: - description: 'Quay.io password' - required: true - GITHUB_TOKEN: - description: 'GitHub token' - required: true - NEXUS_USERNAME: - description: 'Nexus username' - required: true - NEXUS_PASSWORD: - description: 'Nexus password' - required: true jobs: pre-commit: From df79eb9c03edd2790e06cc2cefeb9d7cd0335b8f Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 14:18:11 +0100 Subject: [PATCH 03/36] move permissions --- .github/workflows/build_and_test.yml | 9 --------- .github/workflows/reusuable_build_and_test.yml | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 68b14182..9d78e647 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -22,15 +22,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} cancel-in-progress: true -permissions: - contents: read - # telemetry - actions: read - # ghcr push - packages: write - attestations: write - id-token: write - jobs: pre-commit: runs-on: ubuntu-latest diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index a8480bc1..46b8584d 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -24,6 +24,15 @@ on: required: true type: string +permissions: + contents: read + # telemetry + actions: read + # ghcr push + packages: write + attestations: write + id-token: write + jobs: pre-commit: runs-on: ubuntu-latest From b779e896801372f3686a2f157ad88867f94baf7f Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 14:25:18 +0100 Subject: [PATCH 04/36] move precommit to parent --- .github/workflows/build_and_test.yml | 9 +++++++++ .../workflows/reusuable_build_and_test.yml | 20 ------------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9d78e647..68b14182 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -22,6 +22,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} cancel-in-progress: true +permissions: + contents: read + # telemetry + actions: read + # ghcr push + packages: write + attestations: write + id-token: write + jobs: pre-commit: runs-on: ubuntu-latest diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 46b8584d..06306fe8 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -24,28 +24,8 @@ on: required: true type: string -permissions: - contents: read - # telemetry - actions: read - # ghcr push - packages: write - attestations: write - id-token: write - jobs: - pre-commit: - runs-on: ubuntu-latest - if: '! github.event.pull_request.head.repo.fork' - permissions: - contents: write - steps: - - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.1.1 - with: - auto-commit: "true" - build: - needs: pre-commit runs-on: ubuntu-latest steps: - name: Checkout From b63d108d209efe3725699c3418ea1fddfdb0e719 Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 14:43:08 +0100 Subject: [PATCH 05/36] fetching script path to dir as a 2nd argument --- scripts/fetch-artifacts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fetch-artifacts.sh b/scripts/fetch-artifacts.sh index 72b2c1f9..fbe12acd 100755 --- a/scripts/fetch-artifacts.sh +++ b/scripts/fetch-artifacts.sh @@ -41,7 +41,7 @@ do_fetch_mvn() { done } -TARGETS=$(find "${REPO_ROOT}" -regex "${REPO_ROOT}/${1:+$1/}.*" -name artifacts.json -mindepth 2 -print) +TARGETS=$(find "${REPO_ROOT}" -regex "${REPO_ROOT}/${2:+$2/}.*" -name artifacts.json -mindepth 2 -print) for i in $TARGETS ; do do_fetch_mvn $i From 0dbd571bad86a653a9b4b37acedce836b3a375a5 Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 15:06:21 +0100 Subject: [PATCH 06/36] add few fixes --- .github/workflows/reusuable_build_and_test.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 06306fe8..fc239c50 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -24,6 +24,11 @@ on: required: true type: string +env: + REGISTRY: ${{ inputs.registry }} + REGISTRY_NAMESPACE: ${{ inputs.registry_namespace }} + TAG: ${{ inputs.tag }} + jobs: build: runs-on: ubuntu-latest @@ -74,7 +79,7 @@ jobs: uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 if: github.actor != 'dependabot[bot]' with: - registry: ${{ inputs.registry }} + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -92,7 +97,7 @@ jobs: - name: Enumerate registry cache targets id: cache-targets env: - CACHE_TARGET: ${{ inputs.registry }}/${{ inputs.registry_namespace }}/bakery-cache + CACHE_TARGET: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/bakery-cache run: | echo 'cache-set<> $GITHUB_OUTPUT echo '${{ steps.bake-targets.outputs.targets }}' | jq -r '.[] | '\ @@ -130,7 +135,7 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: - registry: ${{ inputs.registry }} + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -193,7 +198,7 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: - registry: ${{ inputs.registry }} + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} From 8072fafdf46dbc2097f9541afb9a805641c461d8 Mon Sep 17 00:00:00 2001 From: pmacius Date: Mon, 4 Nov 2024 15:11:33 +0100 Subject: [PATCH 07/36] fix artifacts cache and fix artifacts for adw --- .github/workflows/reusuable_build_and_test.yml | 2 +- adf-apps/adw/artifacts.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index fc239c50..9945f35c 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -48,7 +48,7 @@ jobs: uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 id: artifacts-cache with: - key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }} + key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }}-${{ inputs.acs_version }} path: artifacts_cache/** - name: Fetch artifacts from nexus diff --git a/adf-apps/adw/artifacts.json b/adf-apps/adw/artifacts.json index 5dca7bce..2102b26c 100644 --- a/adf-apps/adw/artifacts.json +++ b/adf-apps/adw/artifacts.json @@ -17,7 +17,7 @@ "path": "adf-apps/adw", "classifier": ".zip", "group": "org.alfresco", - "repository": "public" + "repository": "enterprise-releases" } ] } From ca67b0c7f61f90579a9bbdba84f706d6c55e495c Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 10:21:42 +0100 Subject: [PATCH 08/36] compose multiversion testing wip --- .../workflows/reusuable_build_and_test.yml | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 9945f35c..1011849b 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -116,8 +116,26 @@ jobs: *.output=type=registry,push=${{ env.DOCKER_PUSH }} ${{ steps.cache-targets.outputs.cache-set }} - compose-test: + build-matrix: needs: build + runs-on: ubuntu-latest + if: github.actor != 'dependabot[bot]' + steps: + - name: Calculate values for compose + id: compose-values + run: | + echo "matrix_json=$( + if [[ '${{ inputs.acs_version }}' == '23' ]]; then + jq -n '{"edition": ["enterprise", "community"]}' + elif [[ '${{ inputs.acs_version }}' == '74' ]]; then + jq -n '{"edition": ["74"]}' + else + jq -n '{"edition": []}' + fi + )" >> $GITHUB_OUTPUT + + compose-test: + needs: build-matrix runs-on: ${{ matrix.edition == 'enterprise' && 'alfrescoPub-ubuntu2204-16G-4CPU' || 'ubuntu-latest' }} if: github.actor != 'dependabot[bot]' strategy: @@ -166,7 +184,7 @@ jobs: output-archive-name: ${{ matrix.edition }}-logs-${{ inputs.acs_version }} helm-test: - needs: build + needs: build-matrix runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} if: github.actor != 'dependabot[bot]' strategy: From ee4f495a0e58b0402665de14ed8123aee759f7d6 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 11:02:45 +0100 Subject: [PATCH 09/36] try only compose testing --- .../workflows/reusuable_build_and_test.yml | 297 +++++++++--------- .gitignore | 1 + scripts/fetch-compose.sh | 2 +- test/74-override.yaml | 39 +++ 4 files changed, 195 insertions(+), 144 deletions(-) create mode 100644 test/74-override.yaml diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 1011849b..5a410c6b 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -108,7 +108,7 @@ jobs: - name: Bake Docker images env: - TARGETARCH: linux/amd64,linux/arm64 + TARGETARCH: ${{ inputs.acs_version == '23' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 with: @@ -120,6 +120,8 @@ jobs: needs: build runs-on: ubuntu-latest if: github.actor != 'dependabot[bot]' + outputs: + matrix_json: ${{ steps.compose-values.outputs.matrix_json }} steps: - name: Calculate values for compose id: compose-values @@ -140,11 +142,8 @@ jobs: if: github.actor != 'dependabot[bot]' strategy: fail-fast: false - matrix: - edition: [enterprise, community] + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix_json) }} env: - UPSTREAM_COMPOSE_PATH: test/${{ matrix.edition }}.yaml - OVERRIDE_COMPOSE_PATH: test/${{ matrix.edition }}-override.yaml MERGED_COMPOSE_PATH: test/merged-compose.yaml steps: - name: Checkout @@ -161,10 +160,22 @@ jobs: uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Fetch compose from acs-deployment - run: ./scripts/fetch-compose.sh ${{ matrix.edition }} ${{ env.UPSTREAM_COMPOSE_PATH }} ${{ inputs.acs_deployment_version }} + run: ./scripts/fetch-compose.sh ${{ inputs.acs_deployment_version }} + + - name: Set path to compose files + id: set-compose-path + run: | + if [[ "${{ matrix.edition }}" == "74" ]]; then + echo "UPSTREAM_COMPOSE_PATH=test/7.4.N-compose.yaml" >> $GITHUB_ENV + elif [[ "${{ matrix.edition }}" == "community" ]]; then + echo "UPSTREAM_COMPOSE_PATH=test/community-compose.yaml" >> $GITHUB_ENV + else + echo "UPSTREAM_COMPOSE_PATH=test/compose.yaml" >> $GITHUB_ENV - name: Merge Docker Compose files - run: docker compose -f ${{ env.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} + env: + OVERRIDE_COMPOSE_PATH: test/${{ matrix.edition }}-override.yaml + run: docker compose -f ${{ steps.set-compose-path.outputs.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} - name: Verify docker-compose (${{ matrix.edition }}) id: verify_compose @@ -183,139 +194,139 @@ jobs: with: output-archive-name: ${{ matrix.edition }}-logs-${{ inputs.acs_version }} - helm-test: - needs: build-matrix - runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} - if: github.actor != 'dependabot[bot]' - strategy: - fail-fast: false - matrix: - arch: [arm64, amd64] - edition: [enterprise, community] - steps: - - name: Setup cluster - uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 - with: - ingress-nginx-ref: controller-v1.8.2 - metrics: "true" - - - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: "3.15.2" - - - name: Login to Quay.io - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create registries auth secret - run: >- - kubectl create secret generic regcred - --from-file=.dockerconfigjson=$HOME/.docker/config.json - --type=kubernetes.io/dockerconfigjson - - - name: Create configmaps for adf apps - run: | - kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json - kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json - - - name: Checkout acs-deployment sources - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - with: - repository: Alfresco/acs-deployment - ref: ${{ inputs.acs_deployment_version }} - path: acs-deployment - - - name: Setup helm repository - working-directory: acs-deployment/helm/alfresco-content-services - run: | - helm repo add self https://alfresco.github.io/alfresco-helm-charts/ - helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts - helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ - helm repo add elastic https://helm.elastic.co - helm dependency build - - - name: Preprocess test-overrides.yaml - env: - OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml - run: | - sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} - - if [ "${{ matrix.edition }}" = "community" ]; then - sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - fi - - cat ${{ env.OVERRIDES_VALUES_FILE }} - - - name: Helm install - id: helm_install - run: | - helm install acs ./acs-deployment/helm/alfresco-content-services \ - --set global.search.sharedSecret="$(openssl rand -hex 24)" \ - --set global.known_urls=http://localhost \ - --set global.alfrescoRegistryPullSecrets=regcred \ - --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ - --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ - --values test/helm/test-overrides.yaml \ - --values test/helm/test-overrides-${{ matrix.edition }}.yaml - - - name: Watch Helm deployment - env: - HELM_INSTALL_TIMEOUT: 10m - run: | - kubectl get pods --watch & - KWPID=$! - kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods - kill $KWPID - - if [ "${{ matrix.edition }}" = "enterprise" ]; then - echo "Waiting for Enterprise Search Reindexing job to complete... " - kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing - fi - - - name: Debug cluster status after install - if: always() && steps.helm_install.outcome != 'skipped' - run: | - helm ls --all-namespaces --all - kubectl get all --all-namespaces - kubectl describe pod - - - name: Run helm test - id: helm_test - run: helm test acs - - - name: Debug cluster status after helm test - if: always() && steps.helm_test.outcome != 'skipped' - run: | - kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 - kubectl get all --all-namespaces - kubectl describe pod - - - name: Collect logs from all containers - if: always() && steps.helm_install.outcome != 'skipped' - run: | - mkdir -p logs - for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do - kubectl logs $pod -n default > logs/${pod}.log - done - - - name: Upload logs as artifact - if: always() && steps.helm_install.outcome != 'skipped' - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 - with: - name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }}-${{ inputs.acs_version}} - path: logs +# helm-test: +# needs: build-matrix +# runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} +# if: github.actor != 'dependabot[bot]' +# strategy: +# fail-fast: false +# matrix: +# arch: [arm64, amd64] +# edition: [enterprise, community] +# steps: +# - name: Setup cluster +# uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 +# with: +# ingress-nginx-ref: controller-v1.8.2 +# metrics: "true" + +# - name: Checkout +# uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + +# - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 +# with: +# version: "3.15.2" + +# - name: Login to Quay.io +# uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 +# with: +# registry: quay.io +# username: ${{ secrets.QUAY_USERNAME }} +# password: ${{ secrets.QUAY_PASSWORD }} + +# - name: Log in to the Container registry +# uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 +# with: +# registry: ${{ env.REGISTRY }} +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Create registries auth secret +# run: >- +# kubectl create secret generic regcred +# --from-file=.dockerconfigjson=$HOME/.docker/config.json +# --type=kubernetes.io/dockerconfigjson + +# - name: Create configmaps for adf apps +# run: | +# kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json +# kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json + +# - name: Checkout acs-deployment sources +# uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 +# with: +# repository: Alfresco/acs-deployment +# ref: ${{ inputs.acs_deployment_version }} +# path: acs-deployment + +# - name: Setup helm repository +# working-directory: acs-deployment/helm/alfresco-content-services +# run: | +# helm repo add self https://alfresco.github.io/alfresco-helm-charts/ +# helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts +# helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ +# helm repo add elastic https://helm.elastic.co +# helm dependency build + +# - name: Preprocess test-overrides.yaml +# env: +# OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml +# run: | +# sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} +# sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} + +# if [ "${{ matrix.edition }}" = "community" ]; then +# sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} +# sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} +# fi + +# cat ${{ env.OVERRIDES_VALUES_FILE }} + +# - name: Helm install +# id: helm_install +# run: | +# helm install acs ./acs-deployment/helm/alfresco-content-services \ +# --set global.search.sharedSecret="$(openssl rand -hex 24)" \ +# --set global.known_urls=http://localhost \ +# --set global.alfrescoRegistryPullSecrets=regcred \ +# --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ +# --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ +# --values test/helm/test-overrides.yaml \ +# --values test/helm/test-overrides-${{ matrix.edition }}.yaml + +# - name: Watch Helm deployment +# env: +# HELM_INSTALL_TIMEOUT: 10m +# run: | +# kubectl get pods --watch & +# KWPID=$! +# kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods +# kill $KWPID + +# if [ "${{ matrix.edition }}" = "enterprise" ]; then +# echo "Waiting for Enterprise Search Reindexing job to complete... " +# kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing +# fi + +# - name: Debug cluster status after install +# if: always() && steps.helm_install.outcome != 'skipped' +# run: | +# helm ls --all-namespaces --all +# kubectl get all --all-namespaces +# kubectl describe pod + +# - name: Run helm test +# id: helm_test +# run: helm test acs + +# - name: Debug cluster status after helm test +# if: always() && steps.helm_test.outcome != 'skipped' +# run: | +# kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 +# kubectl get all --all-namespaces +# kubectl describe pod + +# - name: Collect logs from all containers +# if: always() && steps.helm_install.outcome != 'skipped' +# run: | +# mkdir -p logs +# for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do +# kubectl logs $pod -n default > logs/${pod}.log +# done + +# - name: Upload logs as artifact +# if: always() && steps.helm_install.outcome != 'skipped' +# uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 +# with: +# name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }}-${{ inputs.acs_version}} +# path: logs diff --git a/.gitignore b/.gitignore index 6bce344c..f6c6333f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ artifacts_cache # Files fetched by scripts/fetch_compose.sh test/compose.yaml test/community-compose.yaml +test/7.4.N-compose.yaml # Files fetched by scripts/fetch_helm.sh test/helm/enterprise-integration-test-values.yaml test/helm/community-integration-test-values.yaml diff --git a/scripts/fetch-compose.sh b/scripts/fetch-compose.sh index d9c051ad..f146ce0a 100755 --- a/scripts/fetch-compose.sh +++ b/scripts/fetch-compose.sh @@ -10,7 +10,7 @@ if [ "$#" -ne 1 ]; then fi ACS_DEPLOYMENT_VERSION="$1" -COMPOSE_FILES=(compose.yaml community-compose.yaml) +COMPOSE_FILES=(compose.yaml community-compose.yaml 7.4.N-compose.yaml 7.3.N-compose.yaml) DESTINATION_DIR="$(dirname "$0")/../test" for COMPOSE_FILE in "${COMPOSE_FILES[@]}"; do diff --git a/test/74-override.yaml b/test/74-override.yaml new file mode 100644 index 00000000..8e571047 --- /dev/null +++ b/test/74-override.yaml @@ -0,0 +1,39 @@ +services: + alfresco: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-content-repository:${TAG} + transform-router: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-transform-router:${TAG} + transform-core-aio: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-transform-core-aio:${TAG} + shared-file-store: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-shared-file-store:${TAG} + share: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-share:${TAG} + solr6: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-search-service:${TAG} + environment: + # variables specific to image produced by bakery + ALFRESCO_HOST: alfresco + ALFRESCO_PORT: 8080 + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + ALFRESCO_SECURECOMMS: secret + ALFRESCO_SECURECOMMS_SECRET: secret + digital-workspace: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-digital-workspace:${TAG} + environment: + BASE_PATH: / + volumes: + - ./configs/adw.json:/usr/share/nginx/html/app.config.json + control-center: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-control-center:${TAG} + environment: + BASE_PATH: / + volumes: + - ./configs/acc.json:/usr/share/nginx/html/app.config.json + sync-service: + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-sync-service:${TAG} +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs From 728d02a8001d3dceb6fca359b58d647671be8b1d Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 11:11:45 +0100 Subject: [PATCH 10/36] try to fix build matrix --- .github/workflows/reusuable_build_and_test.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 5a410c6b..258d8693 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -121,20 +121,21 @@ jobs: runs-on: ubuntu-latest if: github.actor != 'dependabot[bot]' outputs: - matrix_json: ${{ steps.compose-values.outputs.matrix_json }} + matrix_json: ${{ steps.compose-values.outputs.matrix_json }} steps: - name: Calculate values for compose id: compose-values run: | - echo "matrix_json=$( - if [[ '${{ inputs.acs_version }}' == '23' ]]; then + matrix_json=$( + if [[ '${{ inputs.acs_version }}' == '23' ]]; then jq -n '{"edition": ["enterprise", "community"]}' - elif [[ '${{ inputs.acs_version }}' == '74' ]]; then + elif [[ '${{ inputs.acs_version }}' == '74' ]]; then jq -n '{"edition": ["74"]}' - else + else jq -n '{"edition": []}' - fi - )" >> $GITHUB_OUTPUT + fi + ) + echo "matrix_json=$matrix_json" >> $GITHUB_OUTPUT compose-test: needs: build-matrix From 627d3834aff670250399e35836ba821d7e617513 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 11:20:45 +0100 Subject: [PATCH 11/36] use simpler way --- .github/workflows/reusuable_build_and_test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 258d8693..2e3955fe 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -126,15 +126,13 @@ jobs: - name: Calculate values for compose id: compose-values run: | - matrix_json=$( if [[ '${{ inputs.acs_version }}' == '23' ]]; then - jq -n '{"edition": ["enterprise", "community"]}' + matrix_json='{"edition": ["enterprise", "community"]}' elif [[ '${{ inputs.acs_version }}' == '74' ]]; then - jq -n '{"edition": ["74"]}' + matrix_json='{"edition": ["74"]}' else - jq -n '{"edition": []}' + matrix_json='{"edition": []}' fi - ) echo "matrix_json=$matrix_json" >> $GITHUB_OUTPUT compose-test: From f69dbfc92f56d8ee804f8b57e77017fa20ff0389 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 11:29:42 +0100 Subject: [PATCH 12/36] missing fi --- .github/workflows/reusuable_build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 2e3955fe..44aa8919 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -170,6 +170,7 @@ jobs: echo "UPSTREAM_COMPOSE_PATH=test/community-compose.yaml" >> $GITHUB_ENV else echo "UPSTREAM_COMPOSE_PATH=test/compose.yaml" >> $GITHUB_ENV + fi - name: Merge Docker Compose files env: From 187c0cf656c74317904047cd0bc1aaf7fd4ddfad Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 11:35:08 +0100 Subject: [PATCH 13/36] fix env --- .github/workflows/reusuable_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 44aa8919..7cde4489 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -175,7 +175,7 @@ jobs: - name: Merge Docker Compose files env: OVERRIDE_COMPOSE_PATH: test/${{ matrix.edition }}-override.yaml - run: docker compose -f ${{ steps.set-compose-path.outputs.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} + run: docker compose -f ${{ env.UPSTREAM_COMPOSE_PATH }} -f ${{ env.OVERRIDE_COMPOSE_PATH }} config > ${{ env.MERGED_COMPOSE_PATH }} - name: Verify docker-compose (${{ matrix.edition }}) id: verify_compose From d77a5b4303f43cda7ec60ca36b84be2ba1f2f4e2 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 14:28:20 +0100 Subject: [PATCH 14/36] set tomcat to 9 when version is different than 23 --- .github/workflows/reusuable_build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 7cde4489..68b624fd 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -106,6 +106,14 @@ jobs: '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)"' >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT + - name: Set Tomcat environment variables + run: | + if [[ '${{ inputs.acs_version }}' != '23' ]]; then + echo "TOMCAT_MAJOR=9" >> $GITHUB_ENV + echo "TOMCAT_VERSION=9.0.96" >> $GITHUB_ENV + echo "TOMCAT_SHA512=ef3ac81debbc3a519c43d1fdb1c88ab26a8052af424d81bceccfbd6e663050a06d7aad7960fd5d11c17849829daebbebf33d92ac1158902283d0e534514aab93" >> $GITHUB_ENV + fi + - name: Bake Docker images env: TARGETARCH: ${{ inputs.acs_version == '23' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} From bb7267385bf61142961533cc5daff415463634e6 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 14:32:53 +0100 Subject: [PATCH 15/36] prepare rebase --- .../workflows/reusuable_build_and_test.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 68b624fd..0d1b3e93 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -34,9 +34,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.1.1 + - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v8.2.0 - name: Setup nexus authentication run: | @@ -45,7 +45,7 @@ jobs: echo "password ${{ secrets.NEXUS_PASSWORD }}" >> ~/.netrc - name: Restore packages artifacts - uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: artifacts-cache with: key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }}-${{ inputs.acs_version }} @@ -56,7 +56,7 @@ jobs: - name: Save packages artifacts if: steps.artifacts-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} path: artifacts_cache/** @@ -154,7 +154,7 @@ jobs: MERGED_COMPOSE_PATH: test/merged-compose.yaml steps: - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Log in to the Container registry uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 @@ -187,7 +187,7 @@ jobs: - name: Verify docker-compose (${{ matrix.edition }}) id: verify_compose - uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v8.1.0 + uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v8.2.0 timeout-minutes: 10 with: postman_path: test/postman/docker-compose @@ -198,7 +198,7 @@ jobs: - name: Save containers logs (${{ matrix.edition }}) if: always() && steps.verify_compose.outcome != 'skipped' - uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v8.1.0 + uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v8.2.0 with: output-archive-name: ${{ matrix.edition }}-logs-${{ inputs.acs_version }} @@ -213,13 +213,13 @@ jobs: # edition: [enterprise, community] # steps: # - name: Setup cluster -# uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 +# uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v8.2.0 # with: # ingress-nginx-ref: controller-v1.8.2 # metrics: "true" # - name: Checkout -# uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 +# uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 # with: @@ -251,7 +251,7 @@ jobs: # kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json # - name: Checkout acs-deployment sources -# uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 +# uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # with: # repository: Alfresco/acs-deployment # ref: ${{ inputs.acs_deployment_version }} @@ -334,7 +334,7 @@ jobs: # - name: Upload logs as artifact # if: always() && steps.helm_install.outcome != 'skipped' -# uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 +# uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 # with: # name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }}-${{ inputs.acs_version}} # path: logs From 42c712fbb1b889fa87f7580de2152c0ef8330f56 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 15:05:43 +0100 Subject: [PATCH 16/36] enable helm --- .../workflows/reusuable_build_and_test.yml | 293 +++++++++--------- test/helm/test-overrides-74.yaml | 39 +++ test/helm/test-overrides.yaml | 5 + 3 files changed, 195 insertions(+), 142 deletions(-) create mode 100644 test/helm/test-overrides-74.yaml diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 0d1b3e93..d943c436 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -129,19 +129,30 @@ jobs: runs-on: ubuntu-latest if: github.actor != 'dependabot[bot]' outputs: - matrix_json: ${{ steps.compose-values.outputs.matrix_json }} + compose_matrix_json: ${{ steps.compose-values.outputs.compose_matrix_json }} + helm_matrix_json: ${{ steps.helm-values.outputs.helm_matrix_json }} steps: - name: Calculate values for compose id: compose-values run: | if [[ '${{ inputs.acs_version }}' == '23' ]]; then - matrix_json='{"edition": ["enterprise", "community"]}' + compose_matrix_json='{"edition": ["enterprise", "community"]}' elif [[ '${{ inputs.acs_version }}' == '74' ]]; then - matrix_json='{"edition": ["74"]}' + compose_matrix_json='{"edition": ["74"]}' else - matrix_json='{"edition": []}' + compose_matrix_json='{"edition": []}' fi - echo "matrix_json=$matrix_json" >> $GITHUB_OUTPUT + echo "compose_matrix_json=$compose_matrix_json" >> $GITHUB_OUTPUT + + - name: Calculate values for helm + id: helm-values + run: | + if [[ '${{ inputs.acs_version }}' == '23' ]]; then + helm_matrix_json='{"arch": ["arm64", "amd64"], "edition": ["enterprise", "community"]}' + elif [[ '${{ inputs.acs_version }}' == '74' ]]; then + helm_matrix_json='{"arch": ["amd64"], "edition": ["enterprise"]}' + fi + echo "helm_matrix_json=$helm_matrix_json" >> $GITHUB_OUTPUT compose-test: needs: build-matrix @@ -149,7 +160,7 @@ jobs: if: github.actor != 'dependabot[bot]' strategy: fail-fast: false - matrix: ${{ fromJson(needs.build-matrix.outputs.matrix_json) }} + matrix: ${{ fromJson(needs.build-matrix.outputs.compose_matrix_json) }} env: MERGED_COMPOSE_PATH: test/merged-compose.yaml steps: @@ -202,139 +213,137 @@ jobs: with: output-archive-name: ${{ matrix.edition }}-logs-${{ inputs.acs_version }} -# helm-test: -# needs: build-matrix -# runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} -# if: github.actor != 'dependabot[bot]' -# strategy: -# fail-fast: false -# matrix: -# arch: [arm64, amd64] -# edition: [enterprise, community] -# steps: -# - name: Setup cluster -# uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v8.2.0 -# with: -# ingress-nginx-ref: controller-v1.8.2 -# metrics: "true" - -# - name: Checkout -# uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - -# - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 -# with: -# version: "3.15.2" - -# - name: Login to Quay.io -# uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 -# with: -# registry: quay.io -# username: ${{ secrets.QUAY_USERNAME }} -# password: ${{ secrets.QUAY_PASSWORD }} - -# - name: Log in to the Container registry -# uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 -# with: -# registry: ${{ env.REGISTRY }} -# username: ${{ github.actor }} -# password: ${{ secrets.GITHUB_TOKEN }} - -# - name: Create registries auth secret -# run: >- -# kubectl create secret generic regcred -# --from-file=.dockerconfigjson=$HOME/.docker/config.json -# --type=kubernetes.io/dockerconfigjson - -# - name: Create configmaps for adf apps -# run: | -# kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json -# kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json - -# - name: Checkout acs-deployment sources -# uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 -# with: -# repository: Alfresco/acs-deployment -# ref: ${{ inputs.acs_deployment_version }} -# path: acs-deployment - -# - name: Setup helm repository -# working-directory: acs-deployment/helm/alfresco-content-services -# run: | -# helm repo add self https://alfresco.github.io/alfresco-helm-charts/ -# helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts -# helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ -# helm repo add elastic https://helm.elastic.co -# helm dependency build - -# - name: Preprocess test-overrides.yaml -# env: -# OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml -# run: | -# sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} -# sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} - -# if [ "${{ matrix.edition }}" = "community" ]; then -# sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} -# sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} -# fi - -# cat ${{ env.OVERRIDES_VALUES_FILE }} - -# - name: Helm install -# id: helm_install -# run: | -# helm install acs ./acs-deployment/helm/alfresco-content-services \ -# --set global.search.sharedSecret="$(openssl rand -hex 24)" \ -# --set global.known_urls=http://localhost \ -# --set global.alfrescoRegistryPullSecrets=regcred \ -# --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ -# --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ -# --values test/helm/test-overrides.yaml \ -# --values test/helm/test-overrides-${{ matrix.edition }}.yaml - -# - name: Watch Helm deployment -# env: -# HELM_INSTALL_TIMEOUT: 10m -# run: | -# kubectl get pods --watch & -# KWPID=$! -# kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods -# kill $KWPID - -# if [ "${{ matrix.edition }}" = "enterprise" ]; then -# echo "Waiting for Enterprise Search Reindexing job to complete... " -# kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing -# fi - -# - name: Debug cluster status after install -# if: always() && steps.helm_install.outcome != 'skipped' -# run: | -# helm ls --all-namespaces --all -# kubectl get all --all-namespaces -# kubectl describe pod - -# - name: Run helm test -# id: helm_test -# run: helm test acs - -# - name: Debug cluster status after helm test -# if: always() && steps.helm_test.outcome != 'skipped' -# run: | -# kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 -# kubectl get all --all-namespaces -# kubectl describe pod - -# - name: Collect logs from all containers -# if: always() && steps.helm_install.outcome != 'skipped' -# run: | -# mkdir -p logs -# for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do -# kubectl logs $pod -n default > logs/${pod}.log -# done - -# - name: Upload logs as artifact -# if: always() && steps.helm_install.outcome != 'skipped' -# uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 -# with: -# name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }}-${{ inputs.acs_version}} -# path: logs + helm-test: + needs: build-matrix + runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} + if: github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.helm_matrix_json) }} + steps: + - name: Setup cluster + uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v8.2.0 + with: + ingress-nginx-ref: controller-v1.8.2 + metrics: "true" + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: "3.15.2" + + - name: Login to Quay.io + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create registries auth secret + run: >- + kubectl create secret generic regcred + --from-file=.dockerconfigjson=$HOME/.docker/config.json + --type=kubernetes.io/dockerconfigjson + + - name: Create configmaps for adf apps + run: | + kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json + kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json + + - name: Checkout acs-deployment sources + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: Alfresco/acs-deployment + ref: ${{ inputs.acs_deployment_version }} + path: acs-deployment + + - name: Setup helm repository + working-directory: acs-deployment/helm/alfresco-content-services + run: | + helm repo add self https://alfresco.github.io/alfresco-helm-charts/ + helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts + helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ + helm repo add elastic https://helm.elastic.co + helm dependency build + + - name: Preprocess test-overrides.yaml + env: + OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml + run: | + sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} + + if [ "${{ matrix.edition }}" = "community" ]; then + sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + fi + + cat ${{ env.OVERRIDES_VALUES_FILE }} + + - name: Helm install + id: helm_install + run: | + helm install acs ./acs-deployment/helm/alfresco-content-services \ + --set global.search.sharedSecret="$(openssl rand -hex 24)" \ + --set global.known_urls=http://localhost \ + --set global.alfrescoRegistryPullSecrets=regcred \ + --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ + --values ./acs-deployment/test/${{ matrix.edition == 'community' && 'community' || 'enterprise' }}-integration-test-values.yaml \ + --values test/helm/test-overrides.yaml \ + --values test/helm/test-overrides-${{ matrix.edition }}.yaml + + - name: Watch Helm deployment + env: + HELM_INSTALL_TIMEOUT: 10m + run: | + kubectl get pods --watch & + KWPID=$! + kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods + kill $KWPID + + if [ "${{ matrix.edition }}" = "enterprise" ]; then + echo "Waiting for Enterprise Search Reindexing job to complete... " + kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing + fi + + - name: Debug cluster status after install + if: always() && steps.helm_install.outcome != 'skipped' + run: | + helm ls --all-namespaces --all + kubectl get all --all-namespaces + kubectl describe pod + + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod + + - name: Collect logs from all containers + if: always() && steps.helm_install.outcome != 'skipped' + run: | + mkdir -p logs + for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do + kubectl logs $pod -n default > logs/${pod}.log + done + + - name: Upload logs as artifact + if: always() && steps.helm_install.outcome != 'skipped' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 + with: + name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }} + path: logs diff --git a/test/helm/test-overrides-74.yaml b/test/helm/test-overrides-74.yaml new file mode 100644 index 00000000..1ba081c5 --- /dev/null +++ b/test/helm/test-overrides-74.yaml @@ -0,0 +1,39 @@ +# This values file can be used to install ACS 7.4.x using the latest version of +# the chart +activemq: + image: + repository: quay.io/alfresco/alfresco-activemq + tag: 5.17.6-jre17-rockylinux8 +alfresco-ai-transformer: + image: + tag: 3.1.6 +alfresco-search: + enabled: true +alfresco-search-enterprise: + enabled: false +elasticsearch: + enabled: false +postgresql: + image: + tag: 14.10.0 +postgresql-sync: + image: + tag: 14.10.0 +elasticsearch-audit: + enabled: false +kibana-audit: + enabled: false +alfresco-audit-storage: + enabled: false +dtas: + config: + assertions: + acs: + version: 7.4.2 + modules: + - id: org.alfresco.integrations.google.docs + version: 3.4.2 + installed: true + - id: alfresco-aos-module + version: 1.6.2 + installed: true diff --git a/test/helm/test-overrides.yaml b/test/helm/test-overrides.yaml index b4857a2a..3137909c 100644 --- a/test/helm/test-overrides.yaml +++ b/test/helm/test-overrides.yaml @@ -7,6 +7,11 @@ share: image: repository: localhost/alfresco/alfresco-share tag: latest +alfresco-search: + enabled: false + image: + repository: localhost/alfresco/alfresco-search-service + tag: latest alfresco-search-enterprise: reindexing: image: From 4e4f2d50846e4c9ffb047ede2bdadf4ef0c51636 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 15:26:21 +0100 Subject: [PATCH 17/36] change base helm values --- .github/workflows/reusuable_build_and_test.yml | 15 +++++++++++++-- test/helm/test-overrides-74.yaml | 15 --------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index d943c436..93ab9152 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -150,7 +150,7 @@ jobs: if [[ '${{ inputs.acs_version }}' == '23' ]]; then helm_matrix_json='{"arch": ["arm64", "amd64"], "edition": ["enterprise", "community"]}' elif [[ '${{ inputs.acs_version }}' == '74' ]]; then - helm_matrix_json='{"arch": ["amd64"], "edition": ["enterprise"]}' + helm_matrix_json='{"arch": ["amd64"], "edition": ["74"]}' fi echo "helm_matrix_json=$helm_matrix_json" >> $GITHUB_OUTPUT @@ -289,6 +289,17 @@ jobs: cat ${{ env.OVERRIDES_VALUES_FILE }} + - name: Set path to compose files + id: set-helm-values + run: | + if [[ "${{ matrix.edition }}" == "74" ]]; then + echo "UPSTREAM_HELM_VALUES=7.4.N_values.yaml" >> $GITHUB_ENV + elif [[ "${{ matrix.edition }}" == "community" ]]; then + echo "UPSTREAM_HELM_VALUES=community_values.yaml" >> $GITHUB_ENV + else + echo "UPSTREAM_HELM_VALUES=values.yaml" >> $GITHUB_ENV + fi + - name: Helm install id: helm_install run: | @@ -296,7 +307,7 @@ jobs: --set global.search.sharedSecret="$(openssl rand -hex 24)" \ --set global.known_urls=http://localhost \ --set global.alfrescoRegistryPullSecrets=regcred \ - --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ + --values ./acs-deployment/helm/alfresco-content-services/${{ env.UPSTREAM_HELM_VALUES }} \ --values ./acs-deployment/test/${{ matrix.edition == 'community' && 'community' || 'enterprise' }}-integration-test-values.yaml \ --values test/helm/test-overrides.yaml \ --values test/helm/test-overrides-${{ matrix.edition }}.yaml diff --git a/test/helm/test-overrides-74.yaml b/test/helm/test-overrides-74.yaml index 1ba081c5..2ea55065 100644 --- a/test/helm/test-overrides-74.yaml +++ b/test/helm/test-overrides-74.yaml @@ -1,24 +1,9 @@ -# This values file can be used to install ACS 7.4.x using the latest version of -# the chart -activemq: - image: - repository: quay.io/alfresco/alfresco-activemq - tag: 5.17.6-jre17-rockylinux8 -alfresco-ai-transformer: - image: - tag: 3.1.6 alfresco-search: enabled: true alfresco-search-enterprise: enabled: false elasticsearch: enabled: false -postgresql: - image: - tag: 14.10.0 -postgresql-sync: - image: - tag: 14.10.0 elasticsearch-audit: enabled: false kibana-audit: From fb7b15f7a053f0a095f0fbb3261272912396c3e1 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 15:35:07 +0100 Subject: [PATCH 18/36] fix dtas version check --- test/helm/test-overrides-74.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helm/test-overrides-74.yaml b/test/helm/test-overrides-74.yaml index 2ea55065..0f2475ca 100644 --- a/test/helm/test-overrides-74.yaml +++ b/test/helm/test-overrides-74.yaml @@ -20,5 +20,5 @@ dtas: version: 3.4.2 installed: true - id: alfresco-aos-module - version: 1.6.2 + version: 1.6.3 installed: true From 5a61d9ea4f173f9e8124143b9c81cb62e4d3a9ef Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 16:04:28 +0100 Subject: [PATCH 19/36] fix --- test/helm/test-overrides.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helm/test-overrides.yaml b/test/helm/test-overrides.yaml index 3137909c..a494b03d 100644 --- a/test/helm/test-overrides.yaml +++ b/test/helm/test-overrides.yaml @@ -8,7 +8,6 @@ share: repository: localhost/alfresco/alfresco-share tag: latest alfresco-search: - enabled: false image: repository: localhost/alfresco/alfresco-search-service tag: latest From ab5acdf04594e87fcefcca92386be30b4c6cb180 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 16:29:59 +0100 Subject: [PATCH 20/36] try without completly changing functionality of fetch-artifacts script --- .github/workflows/reusuable_build_and_test.yml | 4 +++- scripts/fetch-artifacts.sh | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index 93ab9152..bcece991 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -52,7 +52,9 @@ jobs: path: artifacts_cache/** - name: Fetch artifacts from nexus - run: ./scripts/fetch-artifacts.sh ${{ inputs.acs_version }} + env: + ACS_VERSION: ${{ inputs.acs_version }} + run: ./scripts/fetch-artifacts.sh - name: Save packages artifacts if: steps.artifacts-cache.outputs.cache-hit != 'true' diff --git a/scripts/fetch-artifacts.sh b/scripts/fetch-artifacts.sh index fbe12acd..3675defe 100755 --- a/scripts/fetch-artifacts.sh +++ b/scripts/fetch-artifacts.sh @@ -4,7 +4,8 @@ REPO_ROOT="$(dirname $0)/.." TEMP_DIR=$(mktemp -d) -INDEX_KEY=$1 +ACS_VERSION=${ACS_VERSION:=23} +INDEX_KEY=${ACS_VERSION} do_fetch_mvn() { for i in $(jq -r ".artifacts.acs${INDEX_KEY} | keys | .[]" $1); do @@ -41,7 +42,7 @@ do_fetch_mvn() { done } -TARGETS=$(find "${REPO_ROOT}" -regex "${REPO_ROOT}/${2:+$2/}.*" -name artifacts.json -mindepth 2 -print) +TARGETS=$(find "${REPO_ROOT}" -regex "${REPO_ROOT}/${1:+$1/}.*" -name artifacts.json -mindepth 2 -print) for i in $TARGETS ; do do_fetch_mvn $i From 1e6654238e5c053eaa043323a4b85034ba459e2d Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 17:00:33 +0100 Subject: [PATCH 21/36] fine tune build matrix job --- .../workflows/reusuable_build_and_test.yml | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusuable_build_and_test.yml index bcece991..f3f321d9 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusuable_build_and_test.yml @@ -131,30 +131,19 @@ jobs: runs-on: ubuntu-latest if: github.actor != 'dependabot[bot]' outputs: - compose_matrix_json: ${{ steps.compose-values.outputs.compose_matrix_json }} - helm_matrix_json: ${{ steps.helm-values.outputs.helm_matrix_json }} + compose_matrix_json: ${{ steps.set-matrix.outputs.compose_matrix_json }} + helm_matrix_json: ${{ steps.set-matrix.outputs.helm_matrix_json }} steps: - - name: Calculate values for compose - id: compose-values + - name: Set matrix values for Compose and Helm + id: set-matrix run: | - if [[ '${{ inputs.acs_version }}' == '23' ]]; then - compose_matrix_json='{"edition": ["enterprise", "community"]}' - elif [[ '${{ inputs.acs_version }}' == '74' ]]; then - compose_matrix_json='{"edition": ["74"]}' - else - compose_matrix_json='{"edition": []}' - fi - echo "compose_matrix_json=$compose_matrix_json" >> $GITHUB_OUTPUT - - - name: Calculate values for helm - id: helm-values - run: | - if [[ '${{ inputs.acs_version }}' == '23' ]]; then - helm_matrix_json='{"arch": ["arm64", "amd64"], "edition": ["enterprise", "community"]}' - elif [[ '${{ inputs.acs_version }}' == '74' ]]; then - helm_matrix_json='{"arch": ["amd64"], "edition": ["74"]}' + if [[ "${{ inputs.acs_version }}" == "23" ]]; then + echo 'compose_matrix_json={"edition": ["enterprise", "community"]}' >> $GITHUB_OUTPUT + echo 'helm_matrix_json={"arch": ["arm64", "amd64"], "edition": ["enterprise", "community"]}' >> $GITHUB_OUTPUT + elif [[ "${{ inputs.acs_version }}" == "74" ]]; then + echo 'compose_matrix_json={"edition": ["74"]}' >> $GITHUB_OUTPUT + echo 'helm_matrix_json={"arch": ["amd64"], "edition": ["74"]}' >> $GITHUB_OUTPUT fi - echo "helm_matrix_json=$helm_matrix_json" >> $GITHUB_OUTPUT compose-test: needs: build-matrix From e174ea12b7ef58e7b9f8a77e17e257de5dac4d75 Mon Sep 17 00:00:00 2001 From: pmacius Date: Tue, 5 Nov 2024 17:29:31 +0100 Subject: [PATCH 22/36] dont fetch 73 for now --- scripts/fetch-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fetch-compose.sh b/scripts/fetch-compose.sh index f146ce0a..4812d13e 100755 --- a/scripts/fetch-compose.sh +++ b/scripts/fetch-compose.sh @@ -10,7 +10,7 @@ if [ "$#" -ne 1 ]; then fi ACS_DEPLOYMENT_VERSION="$1" -COMPOSE_FILES=(compose.yaml community-compose.yaml 7.4.N-compose.yaml 7.3.N-compose.yaml) +COMPOSE_FILES=(compose.yaml community-compose.yaml 7.4.N-compose.yaml) DESTINATION_DIR="$(dirname "$0")/../test" for COMPOSE_FILE in "${COMPOSE_FILES[@]}"; do From 7a55bead6aecf632c2035105de1c9af1bf08f1ac Mon Sep 17 00:00:00 2001 From: pmacius Date: Wed, 6 Nov 2024 16:11:45 +0100 Subject: [PATCH 23/36] addres the review --- .github/workflows/build_and_test.yml | 5 +---- ...d_test.yml => reusable_build_and_test.yml} | 21 +++++-------------- 2 files changed, 6 insertions(+), 20 deletions(-) rename .github/workflows/{reusuable_build_and_test.yml => reusable_build_and_test.yml} (96%) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 68b14182..15cf29aa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -43,15 +43,12 @@ jobs: auto-commit: "true" CI: needs: pre-commit - uses: ./.github/workflows/reusuable_build_and_test.yml + uses: ./.github/workflows/reusable_build_and_test.yml strategy: fail-fast: false matrix: version: [23, 74] with: - registry: ghcr.io - registry_namespace: alfresco tag: ${{ github.event_name == 'pull_request' && format('pr-{0}-v{1}', github.event.pull_request.number, matrix.version) || format('{0}-v{1}', github.ref_name, matrix.version) }} - acs_deployment_version: 78132d95e29d7126025e127740d10cf958164947 # 8.6.0-alpha.0 with compose healthcheck acs_version: ${{ matrix.version }} secrets: inherit diff --git a/.github/workflows/reusuable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml similarity index 96% rename from .github/workflows/reusuable_build_and_test.yml rename to .github/workflows/reusable_build_and_test.yml index f3f321d9..5afbe865 100644 --- a/.github/workflows/reusuable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -3,14 +3,6 @@ name: CI on: workflow_call: inputs: - registry: - description: 'Container registry' - required: true - type: string - registry_namespace: - description: 'Registry namespace' - required: true - type: string tag: description: 'Tag for the Docker images' required: true @@ -19,15 +11,12 @@ on: description: 'Version of ACS to build' required: true type: string - acs_deployment_version: - description: 'ACS deployment version' - required: true - type: string env: - REGISTRY: ${{ inputs.registry }} - REGISTRY_NAMESPACE: ${{ inputs.registry_namespace }} + REGISTRY: ghcr.io + REGISTRY_NAMESPACE: alfresco TAG: ${{ inputs.tag }} + ACS_DEPLOYMENT_VERSION: 78132d95e29d7126025e127740d10cf958164947 # 8.6.0-alpha.0 with compose healthcheck jobs: build: @@ -169,7 +158,7 @@ jobs: uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Fetch compose from acs-deployment - run: ./scripts/fetch-compose.sh ${{ inputs.acs_deployment_version }} + run: ./scripts/fetch-compose.sh ${{ env.ACS_DEPLOYMENT_VERSION }} - name: Set path to compose files id: set-compose-path @@ -254,7 +243,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: Alfresco/acs-deployment - ref: ${{ inputs.acs_deployment_version }} + ref: ${{ env.ACS_DEPLOYMENT_VERSION }} path: acs-deployment - name: Setup helm repository From 1051630e7c03b94d3180297ef1a83106527cd525 Mon Sep 17 00:00:00 2001 From: pmacius Date: Wed, 6 Nov 2024 17:01:02 +0100 Subject: [PATCH 24/36] use the tomcat function for now --- .github/workflows/reusable_build_and_test.yml | 9 +-------- docker-bake.hcl | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 5afbe865..77593914 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -97,16 +97,9 @@ jobs: '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)"' >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT - - name: Set Tomcat environment variables - run: | - if [[ '${{ inputs.acs_version }}' != '23' ]]; then - echo "TOMCAT_MAJOR=9" >> $GITHUB_ENV - echo "TOMCAT_VERSION=9.0.96" >> $GITHUB_ENV - echo "TOMCAT_SHA512=ef3ac81debbc3a519c43d1fdb1c88ab26a8052af424d81bceccfbd6e663050a06d7aad7960fd5d11c17849829daebbebf33d92ac1158902283d0e534514aab93" >> $GITHUB_ENV - fi - - name: Bake Docker images env: + ACS_VERSION: ${{ inputs.acs_version }} TARGETARCH: ${{ inputs.acs_version == '23' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 diff --git a/docker-bake.hcl b/docker-bake.hcl index aba989b8..44b2f67e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -150,16 +150,27 @@ target "java_base" { platforms = split(",", "${TARGETARCH}") } +variable "ACS_VERSION" { + default = "23" +} + +function "determine_tomcat_version" { + params = [acs_version, tomcat10, tomcat9] + result = acs_version == "23" ? tomcat10 : tomcat9 +} + variable "TOMCAT_MAJOR" { - default = "10" + default = determine_tomcat_version("${ACS_VERSION}", "10", "9") } variable "TOMCAT_VERSION" { - default = "10.1.31" + default = determine_tomcat_version("${ACS_VERSION}", "10.1.31", "9.0.96") } variable "TOMCAT_SHA512" { - default = "0e3d423a843e2d9ba4f28a9f0a2f1073d5a1389557dfda041759f8df968bace63cd6948bd76df2727b5133ddb7c33e05dab43cea1d519ca0b6d519461152cce9" + default = determine_tomcat_version("${ACS_VERSION}", + "0e3d423a843e2d9ba4f28a9f0a2f1073d5a1389557dfda041759f8df968bace63cd6948bd76df2727b5133ddb7c33e05dab43cea1d519ca0b6d519461152cce9", + "ef3ac81debbc3a519c43d1fdb1c88ab26a8052af424d81bceccfbd6e663050a06d7aad7960fd5d11c17849829daebbebf33d92ac1158902283d0e534514aab93") } variable "TCNATIVE_VERSION" { From ad498e3870eca5b13d80022457d1ad231dd1720c Mon Sep 17 00:00:00 2001 From: pmacius Date: Wed, 6 Nov 2024 18:00:23 +0100 Subject: [PATCH 25/36] try script on workflow --- .github/workflows/reusable_build_and_test.yml | 8 ++++++- docker-bake.hcl | 17 +++---------- scripts/determine-tomcat-version.sh | 24 +++++++++++++++++++ tomcat/tomcat_versions.yaml | 8 +++++++ 4 files changed, 42 insertions(+), 15 deletions(-) create mode 100755 scripts/determine-tomcat-version.sh create mode 100644 tomcat/tomcat_versions.yaml diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 77593914..b140d8bc 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -97,9 +97,15 @@ jobs: '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)"' >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT + - name: Determine the tomcat version using script + run: | + ./scripts/determine-tomcat-version.sh ${{ inputs.acs_version }} + echo "TOMCAT_MAJOR=${TOMCAT_MAJOR}" >> $GITHUB_ENV + echo "TOMCAT_VERSION=${TOMCAT_VERSION}" >> $GITHUB_ENV + echo "TOMCAT_SHA512=${TOMCAT_SHA512}" >> $GITHUB_ENV + - name: Bake Docker images env: - ACS_VERSION: ${{ inputs.acs_version }} TARGETARCH: ${{ inputs.acs_version == '23' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 diff --git a/docker-bake.hcl b/docker-bake.hcl index 44b2f67e..aba989b8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -150,27 +150,16 @@ target "java_base" { platforms = split(",", "${TARGETARCH}") } -variable "ACS_VERSION" { - default = "23" -} - -function "determine_tomcat_version" { - params = [acs_version, tomcat10, tomcat9] - result = acs_version == "23" ? tomcat10 : tomcat9 -} - variable "TOMCAT_MAJOR" { - default = determine_tomcat_version("${ACS_VERSION}", "10", "9") + default = "10" } variable "TOMCAT_VERSION" { - default = determine_tomcat_version("${ACS_VERSION}", "10.1.31", "9.0.96") + default = "10.1.31" } variable "TOMCAT_SHA512" { - default = determine_tomcat_version("${ACS_VERSION}", - "0e3d423a843e2d9ba4f28a9f0a2f1073d5a1389557dfda041759f8df968bace63cd6948bd76df2727b5133ddb7c33e05dab43cea1d519ca0b6d519461152cce9", - "ef3ac81debbc3a519c43d1fdb1c88ab26a8052af424d81bceccfbd6e663050a06d7aad7960fd5d11c17849829daebbebf33d92ac1158902283d0e534514aab93") + default = "0e3d423a843e2d9ba4f28a9f0a2f1073d5a1389557dfda041759f8df968bace63cd6948bd76df2727b5133ddb7c33e05dab43cea1d519ca0b6d519461152cce9" } variable "TCNATIVE_VERSION" { diff --git a/scripts/determine-tomcat-version.sh b/scripts/determine-tomcat-version.sh new file mode 100755 index 00000000..d71e75ea --- /dev/null +++ b/scripts/determine-tomcat-version.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e + +# Determine the Tomcat version to use based on the acs version +# The acs version is passed as an argument to this script +# Example: ./scripts/determine-tomcat-version.sh 23 + +# Check if the acs version is passed as an argument +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +ACS_VERSION=$1 +TOMCAT_VERSIONS_FILE="tomcat/tomcat_versions.yaml" + +if [[ $ACS_VERSION == "23" ]]; then + tomcat_field="tomcat10" +else + tomcat_field="tomcat9" +fi + +export TOMCAT_MAJOR=$(yq e ".${tomcat_field}.major" $TOMCAT_VERSIONS_FILE) +export TOMCAT_VERSION=$(yq e ".${tomcat_field}.version" $TOMCAT_VERSIONS_FILE) +export TOMCAT_SHA512=$(yq e ".${tomcat_field}.sha512" $TOMCAT_VERSIONS_FILE) diff --git a/tomcat/tomcat_versions.yaml b/tomcat/tomcat_versions.yaml new file mode 100644 index 00000000..aa4f5c8b --- /dev/null +++ b/tomcat/tomcat_versions.yaml @@ -0,0 +1,8 @@ +tomcat9: + major: 9 + version: 9.0.96 + sha512: ef3ac81debbc3a519c43d1fdb1c88ab26a8052af424d81bceccfbd6e663050a06d7aad7960fd5d11c17849829daebbebf33d92ac1158902283d0e534514aab93 +tomcat10: + major: 10 + version: 10.1.31 + sha512: 0e3d423a843e2d9ba4f28a9f0a2f1073d5a1389557dfda041759f8df968bace63cd6948bd76df2727b5133ddb7c33e05dab43cea1d519ca0b6d519461152cce9 From 75f611ac9c0b9503afcaf28fa5ea6c89509e3f1a Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 09:12:02 +0100 Subject: [PATCH 26/36] run source --- .github/workflows/reusable_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index b140d8bc..bcf042c7 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -99,7 +99,7 @@ jobs: - name: Determine the tomcat version using script run: | - ./scripts/determine-tomcat-version.sh ${{ inputs.acs_version }} + source ./scripts/determine-tomcat-version.sh ${{ inputs.acs_version }} echo "TOMCAT_MAJOR=${TOMCAT_MAJOR}" >> $GITHUB_ENV echo "TOMCAT_VERSION=${TOMCAT_VERSION}" >> $GITHUB_ENV echo "TOMCAT_SHA512=${TOMCAT_SHA512}" >> $GITHUB_ENV From 42e5bfdeb339d5377f73c9bf1213883e9fd2a20b Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 10:45:22 +0100 Subject: [PATCH 27/36] remove the script add make tomcat versions determine change the step tomcat versions --- .github/workflows/reusable_build_and_test.yml | 16 +++++++++---- Makefile | 13 ++++++++++ artifacts_cache/.gitkeep | 0 scripts/determine-tomcat-version.sh | 24 ------------------- 4 files changed, 24 insertions(+), 29 deletions(-) delete mode 100644 artifacts_cache/.gitkeep delete mode 100755 scripts/determine-tomcat-version.sh diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index bcf042c7..211e8f41 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -97,12 +97,18 @@ jobs: '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ inputs.tag }}-\(.)"' >> $GITHUB_OUTPUT echo 'EOF' >> $GITHUB_OUTPUT - - name: Determine the tomcat version using script + - name: Determine the tomcat version + env: + TOMCAT_VERSIONS_FILE: "tomcat/tomcat_versions.yaml" run: | - source ./scripts/determine-tomcat-version.sh ${{ inputs.acs_version }} - echo "TOMCAT_MAJOR=${TOMCAT_MAJOR}" >> $GITHUB_ENV - echo "TOMCAT_VERSION=${TOMCAT_VERSION}" >> $GITHUB_ENV - echo "TOMCAT_SHA512=${TOMCAT_SHA512}" >> $GITHUB_ENV + if [[ ${{ inputs.acs_version }} == "23" ]]; then + tomcat_field="tomcat10" + else + tomcat_field="tomcat9" + fi + echo "TOMCAT_MAJOR=$(yq e ".${tomcat_field}.major" ${{ env.TOMCAT_VERSIONS_FILE }})" >> $GITHUB_ENV + echo "TOMCAT_VERSION=$(yq e ".${tomcat_field}.version" ${{ env.TOMCAT_VERSIONS_FILE }})" >> $GITHUB_ENV + echo "TOMCAT_SHA512=$(yq e ".${tomcat_field}.sha512" ${{ env.TOMCAT_VERSIONS_FILE }})" >> $GITHUB_ENV - name: Bake Docker images env: diff --git a/Makefile b/Makefile index fe92bd81..690ed717 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,19 @@ help: @echo " all_ci Build all images including cleanup for Continuous Integration" @echo " help Display this help message" +ACS_VERSION ?= 23 +TOMCAT_VERSIONS_FILE := tomcat/tomcat_versions.yaml + +ifeq ($(ACS_VERSION), 23) + TOMCAT_FIELD := "tomcat10" +else + TOMCAT_FIELD := "tomcat9" +endif + +TOMCAT_MAJOR := $(shell yq e '.${TOMCAT_FIELD}.major' $(TOMCAT_VERSIONS_FILE)) +TOMCAT_VERSION := $(shell yq e '.${TOMCAT_FIELD}.version' $(TOMCAT_VERSIONS_FILE)) +TOMCAT_SHA512 := $(shell yq e '.${TOMCAT_FIELD}.sha512' $(TOMCAT_VERSIONS_FILE)) + setenv: auth ifdef BAKE_NO_CACHE DOCKER_BAKE_ARGS += --no-cache diff --git a/artifacts_cache/.gitkeep b/artifacts_cache/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/scripts/determine-tomcat-version.sh b/scripts/determine-tomcat-version.sh deleted file mode 100755 index d71e75ea..00000000 --- a/scripts/determine-tomcat-version.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -e - -# Determine the Tomcat version to use based on the acs version -# The acs version is passed as an argument to this script -# Example: ./scripts/determine-tomcat-version.sh 23 - -# Check if the acs version is passed as an argument -if [ -z "$1" ]; then - echo "Usage: $0 " - exit 1 -fi - -ACS_VERSION=$1 -TOMCAT_VERSIONS_FILE="tomcat/tomcat_versions.yaml" - -if [[ $ACS_VERSION == "23" ]]; then - tomcat_field="tomcat10" -else - tomcat_field="tomcat9" -fi - -export TOMCAT_MAJOR=$(yq e ".${tomcat_field}.major" $TOMCAT_VERSIONS_FILE) -export TOMCAT_VERSION=$(yq e ".${tomcat_field}.version" $TOMCAT_VERSIONS_FILE) -export TOMCAT_SHA512=$(yq e ".${tomcat_field}.sha512" $TOMCAT_VERSIONS_FILE) From cb525e81e71925e00e794f9f4d07786db0d2a051 Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 10:47:16 +0100 Subject: [PATCH 28/36] missing folder From ca819469e00bac49ffdedfd6d9cd3c9864d64bc0 Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 10:50:52 +0100 Subject: [PATCH 29/36] add gikeep --- artifacts_cache/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 artifacts_cache/.gitkeep diff --git a/artifacts_cache/.gitkeep b/artifacts_cache/.gitkeep new file mode 100644 index 00000000..e69de29b From 7a064910f348d579738ea53ccf14d1678a14da7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Maciusiak?= <158472457+pmacius@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:43:47 +0100 Subject: [PATCH 30/36] Update .github/workflows/reusable_build_and_test.yml Co-authored-by: Giovanni Toraldo <71768+gionn@users.noreply.github.com> --- .github/workflows/reusable_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 211e8f41..9fda6264 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -1,4 +1,5 @@ -name: CI +name: Build and Test +run-name: Build v${{ inputs.acs_version }} on: workflow_call: From 3544c611a015a2b360a31bceb92972670f739b5a Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 12:57:50 +0100 Subject: [PATCH 31/36] Revert "Update .github/workflows/reusable_build_and_test.yml" This reverts commit 7a064910f348d579738ea53ccf14d1678a14da7d. --- .github/workflows/reusable_build_and_test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/reusable_build_and_test.yml b/.github/workflows/reusable_build_and_test.yml index 9fda6264..211e8f41 100644 --- a/.github/workflows/reusable_build_and_test.yml +++ b/.github/workflows/reusable_build_and_test.yml @@ -1,5 +1,4 @@ -name: Build and Test -run-name: Build v${{ inputs.acs_version }} +name: CI on: workflow_call: From f1fe9e56876b369acf8973cc4c93543b0d2cf1fc Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 12:59:22 +0100 Subject: [PATCH 32/36] change name --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 15cf29aa..be74e890 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -42,6 +42,7 @@ jobs: with: auto-commit: "true" CI: + name: Build v${{ matrix.version }} needs: pre-commit uses: ./.github/workflows/reusable_build_and_test.yml strategy: From cbdbc030fe288500d3b529aae1e0611181b4586c Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 15:11:02 +0100 Subject: [PATCH 33/36] fix cleanup --- .github/workflows/cleanup_images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cleanup_images.yml b/.github/workflows/cleanup_images.yml index 15231076..6f8c9c52 100644 --- a/.github/workflows/cleanup_images.yml +++ b/.github/workflows/cleanup_images.yml @@ -53,7 +53,7 @@ jobs: uses: dataaxiom/ghcr-cleanup-action@98b4022383d6ddb70ccbf6a378b4d8c67a60f066 # v1.0.13 if: github.event_name == 'pull_request' env: - PR_TAG: ${{ format('pr-{0}', github.event.pull_request.number) }} + PR_TAGs: ${{ format('pr-{0}*', github.event.pull_request.number) }} with: token: ${{ secrets.DELETE_PACKAGES_GITHUB_TOKEN }} owner: ${{ env.ORG }} @@ -61,6 +61,7 @@ jobs: packages: ${{ env.PACKAGE_NAMES }} delete-tags: ${{ env.PR_TAG }} dry-run: false + use-regex: true - name: Remove images when requested uses: dataaxiom/ghcr-cleanup-action@98b4022383d6ddb70ccbf6a378b4d8c67a60f066 # v1.0.13 From 01aea9f0e02d03a257bec2c923d95f88a12a44f2 Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 15:13:25 +0100 Subject: [PATCH 34/36] fix --- .github/workflows/cleanup_images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup_images.yml b/.github/workflows/cleanup_images.yml index 6f8c9c52..1dfd5eee 100644 --- a/.github/workflows/cleanup_images.yml +++ b/.github/workflows/cleanup_images.yml @@ -53,13 +53,13 @@ jobs: uses: dataaxiom/ghcr-cleanup-action@98b4022383d6ddb70ccbf6a378b4d8c67a60f066 # v1.0.13 if: github.event_name == 'pull_request' env: - PR_TAGs: ${{ format('pr-{0}*', github.event.pull_request.number) }} + PR_TAGS: ${{ format('pr-{0}*', github.event.pull_request.number) }} with: token: ${{ secrets.DELETE_PACKAGES_GITHUB_TOKEN }} owner: ${{ env.ORG }} repository: ${{ env.REPO }} packages: ${{ env.PACKAGE_NAMES }} - delete-tags: ${{ env.PR_TAG }} + delete-tags: ${{ env.PR_TAGS }} dry-run: false use-regex: true From 3e96078d37cd27d67a8edf4d7c2505faf357f6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Maciusiak?= <158472457+pmacius@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:25:51 +0100 Subject: [PATCH 35/36] Update .github/workflows/cleanup_images.yml Co-authored-by: Giovanni Toraldo <71768+gionn@users.noreply.github.com> --- .github/workflows/cleanup_images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cleanup_images.yml b/.github/workflows/cleanup_images.yml index 1dfd5eee..27a7522a 100644 --- a/.github/workflows/cleanup_images.yml +++ b/.github/workflows/cleanup_images.yml @@ -61,7 +61,6 @@ jobs: packages: ${{ env.PACKAGE_NAMES }} delete-tags: ${{ env.PR_TAGS }} dry-run: false - use-regex: true - name: Remove images when requested uses: dataaxiom/ghcr-cleanup-action@98b4022383d6ddb70ccbf6a378b4d8c67a60f066 # v1.0.13 From 6bdbe878fe1c48755d1401e604664d504a23f30b Mon Sep 17 00:00:00 2001 From: pmacius Date: Thu, 7 Nov 2024 17:34:46 +0100 Subject: [PATCH 36/36] review --- README.md | 2 +- scripts/fetch-artifacts.sh | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 65b050e3..5d77d4de 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Building images requires the following tools: - A recent enough Docker installation (with buildx support) - Credentials to access the Alfresco artifacts (Nexus server), if building Enterprise images -- Some common unix tools: `jq`, `wget`, `make` +- Some common unix tools: `jq`, `yq`, `wget`, `make` ### Nexus authentication diff --git a/scripts/fetch-artifacts.sh b/scripts/fetch-artifacts.sh index 3675defe..1380060f 100755 --- a/scripts/fetch-artifacts.sh +++ b/scripts/fetch-artifacts.sh @@ -5,16 +5,15 @@ REPO_ROOT="$(dirname $0)/.." TEMP_DIR=$(mktemp -d) ACS_VERSION=${ACS_VERSION:=23} -INDEX_KEY=${ACS_VERSION} do_fetch_mvn() { - for i in $(jq -r ".artifacts.acs${INDEX_KEY} | keys | .[]" $1); do - ARTIFACT_REPO=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].repository" $1) - ARTIFACT_NAME=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].name" $1) - ARTIFACT_VERSION=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].version" $1) - ARTIFACT_EXT=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].classifier" $1) - ARTIFACT_GROUP=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].group" $1) - ARTIFACT_PATH=$(jq -r ".artifacts.acs${INDEX_KEY}[$i].path" $1) + for i in $(jq -r ".artifacts.acs${ACS_VERSION} | keys | .[]" $1); do + ARTIFACT_REPO=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].repository" $1) + ARTIFACT_NAME=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].name" $1) + ARTIFACT_VERSION=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].version" $1) + ARTIFACT_EXT=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].classifier" $1) + ARTIFACT_GROUP=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].group" $1) + ARTIFACT_PATH=$(jq -r ".artifacts.acs${ACS_VERSION}[$i].path" $1) ARTIFACT_BASEURL="https://nexus.alfresco.com/nexus/repository/${ARTIFACT_REPO}" ARTIFACT_TMP_PATH="${TEMP_DIR}/${ARTIFACT_NAME}-${ARTIFACT_VERSION}${ARTIFACT_EXT}" ARTIFACT_CACHE_PATH="${REPO_ROOT}/artifacts_cache/${ARTIFACT_NAME}-${ARTIFACT_VERSION}${ARTIFACT_EXT}"