From 6eacb3a8de2ce49ec9d640ab3ef937cdb85ac980 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 10:31:35 +0200 Subject: [PATCH 01/24] workflow for building from forks --- .github/workflows/build_forks.yml | 182 ++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 .github/workflows/build_forks.yml diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml new file mode 100644 index 00000000..aeb68c69 --- /dev/null +++ b/.github/workflows/build_forks.yml @@ -0,0 +1,182 @@ +name: CI + +on: + pull_request: + branches: + - main + paths: + - '.github/workflows/build_forks.yml' + +env: + ACS_CHART_VERSION: 8a819c9b90b9015e5ab2654c47961fef4183cbcf # v8.5.1 plus the fix for community test values + ARTIFACT_NAME: alfresco-docker-images + REGISTRY: localhost + REGISTRY_NAMESPACE: alfresco + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} + cancel-in-progress: true + +jobs: + pre-commit: + runs-on: ubuntu-latest + # if github.event.pull_request.head.repo.fork TODO: enable when repo is public + steps: + - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.1.1 + + build: + needs: pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.1.1 + + - name: Restore packages artifacts + uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + id: artifacts-cache + with: + key: ${{ runner.os }}-packages-community-${{ hashFiles('**/artifacts.json') }} + path: | + **/*.jar + **/*.zip + **/*.amp + **/*.tgz + **/*.gz + **/*.rpm + + - name: Fetch artifacts from nexus + run: ./scripts/fetch-artifacts.sh + + - name: Save packages artifacts + id: cache-primes-save + uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + with: + key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} + path: | + **/*.jar + **/*.zip + **/*.amp + **/*.tgz + **/*.gz + **/*.rpm + + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Bake Docker images + env: + TARGETARCH: linux/amd64,linux/arm64 + DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} + uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 + with: + targets: community + + - name: Verify docker-compose + id: verify_compose + uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v7.1.1 + timeout-minutes: 10 + with: + compose_file_path: test/community-docker-compose.yml + + - name: Save containers logs + if: always() && steps.verify_compose.outcome != 'skipped' + uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v7.1.1 + with: + output-archive-name: community-logs + + - name: Setup KinD cluster + uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 + with: + ingress-nginx-ref: controller-v1.8.2 + metrics: "true" + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: "3.15.2" + + - name: Create configmaps for adf apps + run: | + kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json + kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json + + - name: Checkout acs-deployment sources + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + repository: Alfresco/acs-deployment + ref: ${{ env.ACS_CHART_VERSION }} + path: acs-deployment + + - name: Setup helm repository + working-directory: acs-deployment/helm/alfresco-content-services + run: | + helm repo add self https://alfresco.github.io/alfresco-helm-charts/ + helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts + helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ + helm repo add elastic https://helm.elastic.co + helm dependency build + + - name: Preprocess test-overrides.yaml + env: + OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml + run: | + sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + cat ${{ env.OVERRIDES_VALUES_FILE }} + + - name: Helm install + id: helm_install + run: | + helm install acs ./acs-deployment/helm/alfresco-content-services \ + --set global.search.sharedSecret="$(openssl rand -hex 24)" \ + --set global.known_urls=http://localhost \ + --set global.alfrescoRegistryPullSecrets=regcred \ + --values ./acs-deployment/helm/alfresco-content-services/community_values.yaml \ + --values ./acs-deployment/test/community-integration-test-values.yaml \ + --values test/helm/test-overrides.yaml \ + --values test/helm/test-overrides-community.yaml + + - name: Watch Helm deployment + env: + HELM_INSTALL_TIMEOUT: 10m + run: | + kubectl get pods --watch & + KWPID=$! + kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods + kill $KWPID + + - name: Debug cluster status after install + if: always() && steps.helm_install.outcome != 'skipped' + run: | + helm ls --all-namespaces --all + kubectl get all --all-namespaces + kubectl describe pod + + - name: Collect logs from all containers + if: always() && steps.helm_install.outcome != 'skipped' + run: | + mkdir -p logs + for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do + kubectl logs $pod -n default > logs/${pod}.log + done + + - name: Upload logs as artifact + if: always() && steps.helm_install.outcome != 'skipped' + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 + with: + name: k8s-logs-community + path: logs + + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod From 5012fae616759124216819a38a53a6fa3319db2a Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 10:31:49 +0200 Subject: [PATCH 02/24] TO REVERT --- .github/workflows/build_and_test.yml | 319 --------------------------- 1 file changed, 319 deletions(-) delete mode 100644 .github/workflows/build_and_test.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index 29cb8780..00000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,319 +0,0 @@ -name: CI - -on: - push: - branches: - - main - paths-ignore: - - 'Makefile' - - '.github/workflows/test-make.yml' - - '.github/workflows/kics.yml' - - 'README.md' - pull_request: - branches: - - main - paths-ignore: - - 'Makefile' - - '.github/workflows/test-make.yml' - - '.github/workflows/kics.yml' - - 'README.md' - -env: - ACS_CHART_VERSION: 8a819c9b90b9015e5ab2654c47961fef4183cbcf # v8.5.1 plus the fix for community test values - ARTIFACT_NAME: alfresco-docker-images - REGISTRY: ghcr.io - REGISTRY_NAMESPACE: alfresco - TAG: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }} - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - # telemetry - actions: read - # ghcr push - packages: write - attestations: write - id-token: write - -jobs: - pre-commit: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.1.1 - with: - auto-commit: "true" - - build: - needs: pre-commit - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.1.1 - - - name: Setup nexus authentication - run: | - echo "machine nexus.alfresco.com" >> ~/.netrc - echo "login ${{ secrets.NEXUS_USERNAME }}" >> ~/.netrc - echo "password ${{ secrets.NEXUS_PASSWORD }}" >> ~/.netrc - - - name: Restore packages artifacts - uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 - id: artifacts-cache - with: - key: ${{ runner.os }}-packages-${{ hashFiles('**/artifacts.json') }} - restore-keys: ${{ runner.os }}-packages- - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm - - - name: Fetch artifacts from nexus - run: ./scripts/fetch-artifacts.sh - - - name: Save packages artifacts - if: steps.artifacts-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 - with: - key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm - - - name: Set up QEMU - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Login to Quay.io - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - if: github.actor != 'dependabot[bot]' - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - if: github.actor != 'dependabot[bot]' - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Collect Workflow Telemetry - uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 - with: - comment_on_pr: false - - - name: Enumerate bake targets - id: bake-targets - uses: docker/bake-action/subaction/list-targets@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 - with: - target: default - - - name: Enumerate registry cache targets - id: cache-targets - env: - CACHE_TARGET: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/bakery-cache - run: | - echo 'cache-set<> $GITHUB_OUTPUT - echo '${{ steps.bake-targets.outputs.targets }}' | jq -r '.[] | '\ - '"\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)\n'\ - '\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ github.event.repository.default_branch }}-\(.)\n'\ - '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)"' >> $GITHUB_OUTPUT - echo 'EOF' >> $GITHUB_OUTPUT - - - name: Bake Docker images - env: - TARGETARCH: linux/amd64,linux/arm64 - DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} - uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 - with: - set: | - *.output=type=registry,push=${{ env.DOCKER_PUSH }} - ${{ steps.cache-targets.outputs.cache-set }} - - compose-test: - needs: build - runs-on: ubuntu-latest - if: github.actor != 'dependabot[bot]' - strategy: - fail-fast: false - matrix: - edition: [enterprise, community] - steps: - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Verify docker-compose (${{ matrix.edition }}) - id: verify_compose - uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v7.1.1 - timeout-minutes: 10 - with: - compose_file_path: test/${{ matrix.edition }}-docker-compose.yml - quay_username: ${{ secrets.QUAY_USERNAME }} - quay_password: ${{ secrets.QUAY_PASSWORD }} - - - name: Save containers logs (${{ matrix.edition }}) - if: always() && steps.verify_compose.outcome != 'skipped' - uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v7.1.1 - with: - output-archive-name: ${{ matrix.edition }}-logs - - helm-test: - needs: build - runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} - if: github.actor != 'dependabot[bot]' - strategy: - fail-fast: false - matrix: - arch: [arm64, amd64] - edition: [enterprise, community] - steps: - - name: Setup cluster - uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 - with: - ingress-nginx-ref: controller-v1.8.2 - metrics: "true" - - - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - with: - version: "3.15.2" - - - name: Login to Quay.io - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create registries auth secret - run: >- - kubectl create secret generic regcred - --from-file=.dockerconfigjson=$HOME/.docker/config.json - --type=kubernetes.io/dockerconfigjson - - - name: Create configmaps for adf apps - run: | - kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json - kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json - - - name: Checkout acs-deployment sources - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - with: - repository: Alfresco/acs-deployment - ref: ${{ env.ACS_CHART_VERSION }} - path: acs-deployment - - - name: Setup helm repository - working-directory: acs-deployment/helm/alfresco-content-services - run: | - helm repo add self https://alfresco.github.io/alfresco-helm-charts/ - helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts - helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ - helm repo add elastic https://helm.elastic.co - helm dependency build - - - name: Preprocess test-overrides.yaml - env: - OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml - run: | - sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} - - if [ "${{ matrix.edition }}" = "community" ]; then - sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - fi - - cat ${{ env.OVERRIDES_VALUES_FILE }} - - - name: Helm install - id: helm_install - run: | - helm install acs ./acs-deployment/helm/alfresco-content-services \ - --set global.search.sharedSecret="$(openssl rand -hex 24)" \ - --set global.known_urls=http://localhost \ - --set global.alfrescoRegistryPullSecrets=regcred \ - --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ - --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ - --values test/helm/test-overrides.yaml \ - --values test/helm/test-overrides-${{ matrix.edition }}.yaml - - - name: Watch Helm deployment - env: - HELM_INSTALL_TIMEOUT: 10m - run: | - kubectl get pods --watch & - KWPID=$! - kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods - kill $KWPID - - if [ "${{ matrix.edition }}" = "enterprise" ]; then - echo "Waiting for Enterprise Search Reindexing job to complete... " - kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing - fi - - - name: Debug cluster status after install - if: always() && steps.helm_install.outcome != 'skipped' - run: | - helm ls --all-namespaces --all - kubectl get all --all-namespaces - kubectl describe pod - - - name: Run helm test - id: helm_test - run: helm test acs - - - name: Debug cluster status after helm test - if: always() && steps.helm_test.outcome != 'skipped' - run: | - kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 - kubectl get all --all-namespaces - kubectl describe pod - - - name: Collect logs from all containers - if: always() && steps.helm_install.outcome != 'skipped' - run: | - mkdir -p logs - for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do - kubectl logs $pod -n default > logs/${pod}.log - done - - - name: Upload logs as artifact - if: always() && steps.helm_install.outcome != 'skipped' - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 - with: - name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }} - path: logs From 909cf880c72867d405f703c1304b037f06801069 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 11:22:38 +0200 Subject: [PATCH 03/24] build single arch to avoid docker load limitation --- .github/workflows/build_forks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index aeb68c69..1387022a 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -70,7 +70,7 @@ jobs: - name: Bake Docker images env: - TARGETARCH: linux/amd64,linux/arm64 + TARGETARCH: linux/amd64 DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 with: From c67097d12c9b7e309faf9d938ec0fd24b8af87d2 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 13:53:24 +0200 Subject: [PATCH 04/24] refetch community distribution also for share --- share/artifacts.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/share/artifacts.json b/share/artifacts.json index b0906f9b..a367aac8 100644 --- a/share/artifacts.json +++ b/share/artifacts.json @@ -9,6 +9,14 @@ "group": "org.alfresco", "repository": "enterprise-releases" }, + { + "name": "alfresco-content-services-community-distribution", + "version": "23.2.0", + "path": "share/distribution", + "classifier": ".zip", + "group": "org.alfresco", + "repository": "public" + }, { "name": "alfresco-googledrive-share", "version": "4.1.0", From 8f5dc944a96c343152d781a81bc1c3c0cd9f50ec Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 13:55:57 +0200 Subject: [PATCH 05/24] use stale cache and fix warning on esave --- .github/workflows/build_forks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 1387022a..9f524d9a 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -38,6 +38,7 @@ jobs: id: artifacts-cache with: key: ${{ runner.os }}-packages-community-${{ hashFiles('**/artifacts.json') }} + restore-keys: ${{ runner.os }}-packages-community- path: | **/*.jar **/*.zip @@ -50,7 +51,7 @@ jobs: run: ./scripts/fetch-artifacts.sh - name: Save packages artifacts - id: cache-primes-save + if: steps.artifacts-cache.outputs.cache-hit != 'true' uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 with: key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} From ff946bd2fcf51072bb880d13b934dd3954fc9e99 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Mon, 14 Oct 2024 17:57:59 +0200 Subject: [PATCH 06/24] fix share in a very ugly way --- share/Dockerfile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/share/Dockerfile b/share/Dockerfile index 02d452ea..62c7de3d 100644 --- a/share/Dockerfile +++ b/share/Dockerfile @@ -14,18 +14,23 @@ ADD amps /tmp/amps ADD distribution /tmp/ ENV DISTDIR="/tmp/distribution" -RUN yum install -y unzip -RUN unzip /tmp/*.zip -d ${DISTDIR} -RUN unzip ${DISTDIR}/alfresco*/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share/ -RUN cp -r ${DISTDIR}/alfresco*/amps/ /tmp/amps/ +RUN yum install -y unzip +RUN unzip /tmp/*.zip -d ${DISTDIR} +RUN unzip ${DISTDIR}/alfresco*/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share/ || \ + unzip ${DISTDIR}/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share +RUN cp -r ${DISTDIR}/alfresco*/amps/ /tmp/amps/ || \ + cp -r ${DISTDIR}/amps/ /tmp/amps/ RUN cp /tmp/share-config-custom.xml ${CATALINA_HOME}/shared/classes/alfresco/web-extension -RUN cp ${DISTDIR}/alfresco*/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost +RUN cp ${DISTDIR}/alfresco*/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost || \ + cp ${DISTDIR}/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost RUN sed -i 's|../modules/share|modules/share|' ${CATALINA_HOME}/conf/Catalina/localhost/share.xml RUN sed -i "s/shared.loader=/shared.loader=\${catalina.base}\/shared\/classes/" ${CATALINA_HOME}/conf/catalina.properties RUN chmod +x ${CATALINA_HOME}/shared/classes/alfresco/entrypoint.sh RUN java -jar ${DISTDIR}/alfresco*/bin/alfresco-mmt.jar install \ - /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force + /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force || \ + java -jar ${DISTDIR}/bin/alfresco-mmt.jar install \ + /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force FROM tomcat_base AS share-rhlike ARG ALFRESCO_SHARE_USER_ID From 9c9ca263fd93b3c99c713d6751ef6c386aef3f5c Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 09:30:03 +0200 Subject: [PATCH 07/24] simplify --- share/Dockerfile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/share/Dockerfile b/share/Dockerfile index 62c7de3d..2fd71810 100644 --- a/share/Dockerfile +++ b/share/Dockerfile @@ -15,21 +15,17 @@ ADD distribution /tmp/ ENV DISTDIR="/tmp/distribution" RUN yum install -y unzip -RUN unzip /tmp/*.zip -d ${DISTDIR} -RUN unzip ${DISTDIR}/alfresco*/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share/ || \ - unzip ${DISTDIR}/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share -RUN cp -r ${DISTDIR}/alfresco*/amps/ /tmp/amps/ || \ - cp -r ${DISTDIR}/amps/ /tmp/amps/ +RUN unzip /tmp/*.zip -d ${DISTDIR} +RUN if ls ${DISTDIR}/alfresco* > /dev/null 2>&1; then mv ${DISTDIR}/alfresco* ${DISTDIR}/; fi +RUN unzip ${DISTDIR}/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share +RUN cp -r ${DISTDIR}/amps/ /tmp/amps/ RUN cp /tmp/share-config-custom.xml ${CATALINA_HOME}/shared/classes/alfresco/web-extension -RUN cp ${DISTDIR}/alfresco*/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost || \ - cp ${DISTDIR}/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost +RUN cp ${DISTDIR}/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/conf/Catalina/localhost RUN sed -i 's|../modules/share|modules/share|' ${CATALINA_HOME}/conf/Catalina/localhost/share.xml RUN sed -i "s/shared.loader=/shared.loader=\${catalina.base}\/shared\/classes/" ${CATALINA_HOME}/conf/catalina.properties RUN chmod +x ${CATALINA_HOME}/shared/classes/alfresco/entrypoint.sh - -RUN java -jar ${DISTDIR}/alfresco*/bin/alfresco-mmt.jar install \ - /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force || \ - java -jar ${DISTDIR}/bin/alfresco-mmt.jar install \ + +RUN java -jar ${DISTDIR}/bin/alfresco-mmt.jar install \ /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force FROM tomcat_base AS share-rhlike From 60ace75f28d9e926f4c405b0187b1937f5335e38 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 09:31:09 +0200 Subject: [PATCH 08/24] add tag latest --- .github/workflows/build_forks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 9f524d9a..1634a646 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -12,6 +12,7 @@ env: ARTIFACT_NAME: alfresco-docker-images REGISTRY: localhost REGISTRY_NAMESPACE: alfresco + TAG: latest concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} From 58a327150660a51797281df797fde7f7b5a2aaf7 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 09:41:15 +0200 Subject: [PATCH 09/24] multi job --- .github/workflows/build_forks.yml | 53 +++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 1634a646..00b526f0 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -64,9 +64,6 @@ jobs: **/*.gz **/*.rpm - - name: Set up QEMU - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 @@ -78,11 +75,43 @@ jobs: with: targets: community + - name: Export all baked images whose name include `alfresco` + run: | + docker save -o /tmp/${{ env.ARTIFACT_NAME }}.tar $(docker images --format "{{.Repository}}:{{.Tag}}" | grep alfresco) + + - name: Upload images as artifact + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: /tmp/${{ env.ARTIFACT_NAME }}.tar + retention-days: 1 + compression-level: 0 + + compose-test: + name: compose-test + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: /tmp + name: ${{ env.ARTIFACT_NAME }} + + - name: Load docker images from artifact + run: | + docker load -i /tmp/${{ env.ARTIFACT_NAME }}.tar + docker image ls -a + - name: Verify docker-compose id: verify_compose uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v7.1.1 timeout-minutes: 10 with: + compose_pull: false compose_file_path: test/community-docker-compose.yml - name: Save containers logs @@ -91,12 +120,30 @@ jobs: with: output-archive-name: community-logs + helm-test: + name: helm test + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + path: /tmp + name: ${{ env.ARTIFACT_NAME }} + - name: Setup KinD cluster uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 with: ingress-nginx-ref: controller-v1.8.2 metrics: "true" + - name: Load Docker images + run: | + kind load image-archive -n chart-testing /tmp/${{ env.ARTIFACT_NAME }}.tar + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 with: version: "3.15.2" From 48c0d275c6ff5142ce8949f878e544584011722a Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 11:27:17 +0200 Subject: [PATCH 10/24] fix tengines for helm --- docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 80aa66f2..4f6c5138 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -3,7 +3,7 @@ group "default" { } group "community" { - targets = ["repository_community", "share", "search_service", "tengine_aio", "acc"] + targets = ["repository_community", "share", "search_service", "tengines", "acc"] } group "content_service" { From 2b76ed2367eed10534451a5d08fda9a704b15389 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 11:28:50 +0200 Subject: [PATCH 11/24] cleanup --- share/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/Dockerfile b/share/Dockerfile index 2fd71810..ee3a7bd3 100644 --- a/share/Dockerfile +++ b/share/Dockerfile @@ -14,7 +14,7 @@ ADD amps /tmp/amps ADD distribution /tmp/ ENV DISTDIR="/tmp/distribution" -RUN yum install -y unzip +RUN yum install -y unzip RUN unzip /tmp/*.zip -d ${DISTDIR} RUN if ls ${DISTDIR}/alfresco* > /dev/null 2>&1; then mv ${DISTDIR}/alfresco* ${DISTDIR}/; fi RUN unzip ${DISTDIR}/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share @@ -24,7 +24,7 @@ RUN cp ${DISTDIR}/web-server/conf/Catalina/localhost/share.xml ${CATALINA_HOME}/ RUN sed -i 's|../modules/share|modules/share|' ${CATALINA_HOME}/conf/Catalina/localhost/share.xml RUN sed -i "s/shared.loader=/shared.loader=\${catalina.base}\/shared\/classes/" ${CATALINA_HOME}/conf/catalina.properties RUN chmod +x ${CATALINA_HOME}/shared/classes/alfresco/entrypoint.sh - + RUN java -jar ${DISTDIR}/bin/alfresco-mmt.jar install \ /tmp/amps/ ${CATALINA_HOME}/webapps/share -directory -nobackup -force From dd4551c9769a7c063bb5ed8df299c61d022295b8 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 12:09:20 +0200 Subject: [PATCH 12/24] increase probe delays --- test/helm/test-overrides-community.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/helm/test-overrides-community.yaml b/test/helm/test-overrides-community.yaml index 7bce092f..d18bbacc 100644 --- a/test/helm/test-overrides-community.yaml +++ b/test/helm/test-overrides-community.yaml @@ -15,3 +15,22 @@ dtas: - id: alfresco-aos-module version: 3.0.0 installed: true +alfresco-repository: + startupProbe: + initialDelaySeconds: 120 +alfresco-transform-service: + pdfrenderer: + livenessProbe: + initialDelaySeconds: 60 + imagemagick: + livenessProbe: + initialDelaySeconds: 60 + libreoffice: + livenessProbe: + initialDelaySeconds: 60 + tika: + livenessProbe: + initialDelaySeconds: 60 + transformmisc: + livenessProbe: + initialDelaySeconds: 60 From 0ff226ced7600a588ed2b6906f5ec6b0c42d1b8d Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Tue, 15 Oct 2024 12:14:11 +0200 Subject: [PATCH 13/24] fetch postgres from public repository --- repository/artifacts.json | 2 +- sync/artifacts.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/artifacts.json b/repository/artifacts.json index c19b6b7e..5f52eedd 100644 --- a/repository/artifacts.json +++ b/repository/artifacts.json @@ -6,7 +6,7 @@ "version": "42.6.0", "classifier": ".jar", "group": "org.postgresql", - "repository": "central", + "repository": "public", "path": "repository/libs" }, { diff --git a/sync/artifacts.json b/sync/artifacts.json index 49d704dc..957f39d9 100644 --- a/sync/artifacts.json +++ b/sync/artifacts.json @@ -14,7 +14,7 @@ "version": "42.6.0", "classifier": ".jar", "group": "org.postgresql", - "repository": "central", + "repository": "public", "path": "sync/configs/connectors" } ] From 02b0683026531cf8cb1d90767f3249e66191d6ec Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Wed, 16 Oct 2024 14:11:13 +0200 Subject: [PATCH 14/24] Revert "TO REVERT" This reverts commit a10945a90bde1ae6462fad2d9282febb31db4a89. --- .github/workflows/build_and_test.yml | 319 +++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 .github/workflows/build_and_test.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 00000000..29cb8780 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,319 @@ +name: CI + +on: + push: + branches: + - main + paths-ignore: + - 'Makefile' + - '.github/workflows/test-make.yml' + - '.github/workflows/kics.yml' + - 'README.md' + pull_request: + branches: + - main + paths-ignore: + - 'Makefile' + - '.github/workflows/test-make.yml' + - '.github/workflows/kics.yml' + - 'README.md' + +env: + ACS_CHART_VERSION: 8a819c9b90b9015e5ab2654c47961fef4183cbcf # v8.5.1 plus the fix for community test values + ARTIFACT_NAME: alfresco-docker-images + REGISTRY: ghcr.io + REGISTRY_NAMESPACE: alfresco + TAG: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + # telemetry + actions: read + # ghcr push + packages: write + attestations: write + id-token: write + +jobs: + pre-commit: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: Alfresco/alfresco-build-tools/.github/actions/pre-commit@v7.1.1 + with: + auto-commit: "true" + + build: + needs: pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - uses: Alfresco/alfresco-build-tools/.github/actions/free-hosted-runner-disk-space@v7.1.1 + + - name: Setup nexus authentication + run: | + echo "machine nexus.alfresco.com" >> ~/.netrc + echo "login ${{ secrets.NEXUS_USERNAME }}" >> ~/.netrc + echo "password ${{ secrets.NEXUS_PASSWORD }}" >> ~/.netrc + + - name: Restore packages artifacts + uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + id: artifacts-cache + with: + key: ${{ runner.os }}-packages-${{ hashFiles('**/artifacts.json') }} + restore-keys: ${{ runner.os }}-packages- + path: | + **/*.jar + **/*.zip + **/*.amp + **/*.tgz + **/*.gz + **/*.rpm + + - name: Fetch artifacts from nexus + run: ./scripts/fetch-artifacts.sh + + - name: Save packages artifacts + if: steps.artifacts-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 + with: + key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} + path: | + **/*.jar + **/*.zip + **/*.amp + **/*.tgz + **/*.gz + **/*.rpm + + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Login to Quay.io + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + if: github.actor != 'dependabot[bot]' + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + if: github.actor != 'dependabot[bot]' + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Collect Workflow Telemetry + uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0 + with: + comment_on_pr: false + + - name: Enumerate bake targets + id: bake-targets + uses: docker/bake-action/subaction/list-targets@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 + with: + target: default + + - name: Enumerate registry cache targets + id: cache-targets + env: + CACHE_TARGET: ${{ env.REGISTRY }}/${{ env.REGISTRY_NAMESPACE }}/bakery-cache + run: | + echo 'cache-set<> $GITHUB_OUTPUT + echo '${{ steps.bake-targets.outputs.targets }}' | jq -r '.[] | '\ + '"\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)\n'\ + '\(.).cache-from=type=registry,ref=${{ env.CACHE_TARGET }}:${{ github.event.repository.default_branch }}-\(.)\n'\ + '\(.).cache-to=type=registry,ref=${{ env.CACHE_TARGET }}:${{ env.TAG }}-\(.)"' >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + - name: Bake Docker images + env: + TARGETARCH: linux/amd64,linux/arm64 + DOCKER_PUSH: ${{ github.actor != 'dependabot[bot]'}} + uses: docker/bake-action@2e3d19baedb14545e5d41222653874f25d5b4dfb # v5.10.0 + with: + set: | + *.output=type=registry,push=${{ env.DOCKER_PUSH }} + ${{ steps.cache-targets.outputs.cache-set }} + + compose-test: + needs: build + runs-on: ubuntu-latest + if: github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: + edition: [enterprise, community] + steps: + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Verify docker-compose (${{ matrix.edition }}) + id: verify_compose + uses: Alfresco/alfresco-build-tools/.github/actions/dbp-charts/verify-compose@v7.1.1 + timeout-minutes: 10 + with: + compose_file_path: test/${{ matrix.edition }}-docker-compose.yml + quay_username: ${{ secrets.QUAY_USERNAME }} + quay_password: ${{ secrets.QUAY_PASSWORD }} + + - name: Save containers logs (${{ matrix.edition }}) + if: always() && steps.verify_compose.outcome != 'skipped' + uses: Alfresco/alfresco-build-tools/.github/actions/docker-dump-containers-logs@v7.1.1 + with: + output-archive-name: ${{ matrix.edition }}-logs + + helm-test: + needs: build + runs-on: ${{ matrix.arch == 'arm64' && 'alfrescoARM-ubuntu2404-16G-4CPU' || 'alfrescoPub-ubuntu2204-16G-4CPU' }} + if: github.actor != 'dependabot[bot]' + strategy: + fail-fast: false + matrix: + arch: [arm64, amd64] + edition: [enterprise, community] + steps: + - name: Setup cluster + uses: Alfresco/alfresco-build-tools/.github/actions/setup-kind@v7.1.1 + with: + ingress-nginx-ref: controller-v1.8.2 + metrics: "true" + + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + with: + version: "3.15.2" + + - name: Login to Quay.io + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create registries auth secret + run: >- + kubectl create secret generic regcred + --from-file=.dockerconfigjson=$HOME/.docker/config.json + --type=kubernetes.io/dockerconfigjson + + - name: Create configmaps for adf apps + run: | + kubectl create configmap acc-config --from-file=app.config.json=test/configs/acc.json + kubectl create configmap adw-config --from-file=app.config.json=test/configs/adw.json + + - name: Checkout acs-deployment sources + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + repository: Alfresco/acs-deployment + ref: ${{ env.ACS_CHART_VERSION }} + path: acs-deployment + + - name: Setup helm repository + working-directory: acs-deployment/helm/alfresco-content-services + run: | + helm repo add self https://alfresco.github.io/alfresco-helm-charts/ + helm repo add activiti https://activiti.github.io/activiti-cloud-helm-charts + helm repo add bitnami https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami/ + helm repo add elastic https://helm.elastic.co + helm dependency build + + - name: Preprocess test-overrides.yaml + env: + OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml + run: | + sed -i "s|localhost/alfresco/|${REGISTRY}/${REGISTRY_NAMESPACE}/|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|tag: latest|tag: ${TAG}|g" ${{ env.OVERRIDES_VALUES_FILE }} + + if [ "${{ matrix.edition }}" = "community" ]; then + sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + fi + + cat ${{ env.OVERRIDES_VALUES_FILE }} + + - name: Helm install + id: helm_install + run: | + helm install acs ./acs-deployment/helm/alfresco-content-services \ + --set global.search.sharedSecret="$(openssl rand -hex 24)" \ + --set global.known_urls=http://localhost \ + --set global.alfrescoRegistryPullSecrets=regcred \ + --values ./acs-deployment/helm/alfresco-content-services/${{ matrix.edition == 'community' && 'community_' || '' }}values.yaml \ + --values ./acs-deployment/test/${{ matrix.edition }}-integration-test-values.yaml \ + --values test/helm/test-overrides.yaml \ + --values test/helm/test-overrides-${{ matrix.edition }}.yaml + + - name: Watch Helm deployment + env: + HELM_INSTALL_TIMEOUT: 10m + run: | + kubectl get pods --watch & + KWPID=$! + kubectl wait --timeout=${{ env.HELM_INSTALL_TIMEOUT }} --all=true --for=condition=Ready pods + kill $KWPID + + if [ "${{ matrix.edition }}" = "enterprise" ]; then + echo "Waiting for Enterprise Search Reindexing job to complete... " + kubectl wait --timeout=5m --for=condition=complete job/acs-alfresco-search-enterprise-reindexing + fi + + - name: Debug cluster status after install + if: always() && steps.helm_install.outcome != 'skipped' + run: | + helm ls --all-namespaces --all + kubectl get all --all-namespaces + kubectl describe pod + + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod + + - name: Collect logs from all containers + if: always() && steps.helm_install.outcome != 'skipped' + run: | + mkdir -p logs + for pod in $(kubectl get pods -n default -o jsonpath='{.items[*].metadata.name}'); do + kubectl logs $pod -n default > logs/${pod}.log + done + + - name: Upload logs as artifact + if: always() && steps.helm_install.outcome != 'skipped' + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 #v4.4.0 + with: + name: k8s-logs-${{ matrix.arch }}-${{ matrix.edition }} + path: logs From 1f1712d1f32c10166ca437d5d7f0f9f0ddbb32fd Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Wed, 16 Oct 2024 14:17:49 +0200 Subject: [PATCH 15/24] change name for new workflow --- .github/workflows/build_forks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 00b526f0..e2d53225 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -1,4 +1,4 @@ -name: CI +name: CI from forks on: pull_request: From 81eb30d3ec9ec504bbb010d2b9c64de72ce190dc Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Wed, 16 Oct 2024 14:25:20 +0200 Subject: [PATCH 16/24] gh cache only for artifacts_cache folder --- .github/workflows/build_and_test.yml | 20 ++++---------------- .github/workflows/build_forks.yml | 20 ++++---------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 29cb8780..f062829f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -67,15 +67,9 @@ jobs: uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 id: artifacts-cache with: - key: ${{ runner.os }}-packages-${{ hashFiles('**/artifacts.json') }} - restore-keys: ${{ runner.os }}-packages- - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm + key: ${{ runner.os }}-packages-v2-${{ hashFiles('**/artifacts.json') }} + restore-keys: ${{ runner.os }}-packages-v2- + path: artifacts_cache/** - name: Fetch artifacts from nexus run: ./scripts/fetch-artifacts.sh @@ -85,13 +79,7 @@ jobs: uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 with: key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm + path: artifacts_cache/** - name: Set up QEMU uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index e2d53225..27ae0d30 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -38,15 +38,9 @@ jobs: uses: actions/cache/restore@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 id: artifacts-cache with: - key: ${{ runner.os }}-packages-community-${{ hashFiles('**/artifacts.json') }} - restore-keys: ${{ runner.os }}-packages-community- - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm + key: ${{ runner.os }}-packages-community-v2-${{ hashFiles('**/artifacts.json') }} + restore-keys: ${{ runner.os }}-packages-community-v2 + path: artifacts_cache/** - name: Fetch artifacts from nexus run: ./scripts/fetch-artifacts.sh @@ -56,13 +50,7 @@ jobs: uses: actions/cache/save@2cdf405574d6ef1f33a1d12acccd3ae82f47b3f2 # v4.1.0 with: key: ${{ steps.artifacts-cache.outputs.cache-primary-key }} - path: | - **/*.jar - **/*.zip - **/*.amp - **/*.tgz - **/*.gz - **/*.rpm + path: artifacts_cache/** - name: Set up Docker Buildx uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 From ffc89fb17202a45b6e86c95e45eda3f648554f23 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Wed, 16 Oct 2024 15:02:07 +0200 Subject: [PATCH 17/24] differentiate share for enterprise and community --- .github/workflows/build_forks.yml | 1 + docker-bake.hcl | 22 ++++++++++++++++++++-- share/Dockerfile | 13 +++++++++---- test/community-docker-compose.yml | 2 +- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 27ae0d30..9c9243a8 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -162,6 +162,7 @@ jobs: OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml run: | sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} cat ${{ env.OVERRIDES_VALUES_FILE }} - name: Helm install diff --git a/docker-bake.hcl b/docker-bake.hcl index 4f6c5138..44a36da4 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -3,7 +3,7 @@ group "default" { } group "community" { - targets = ["repository_community", "share", "search_service", "tengines", "acc"] + targets = ["repository_community", "share_community", "search_service", "tengines", "acc"] } group "content_service" { @@ -658,15 +658,33 @@ target "share" { ALFRESCO_SHARE_GROUP_ID = "${ALFRESCO_GROUP_ID}" ALFRESCO_SHARE_USER_NAME = "${ALFRESCO_SHARE_USER_NAME}" ALFRESCO_SHARE_USER_ID = "${ALFRESCO_SHARE_USER_ID}" + ALFRESCO_SHARE_ARTIFACT = "${share_editions.artifact}" } labels = { "org.label-schema.name" = "${PRODUCT_LINE} Share" "org.opencontainers.image.title" = "${PRODUCT_LINE} Share" "org.opencontainers.image.description" = "Alfresco Share" } - tags = ["${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-share:${TAG}"] + tags = ["${REGISTRY}/${REGISTRY_NAMESPACE}/${share_editions.image_name}:${TAG}"] output = ["type=docker"] platforms = split(",", "${TARGETARCH}") + + name = "share_${share_editions.name}" + + matrix = { + share_editions = [ + { + artifact = "alfresco-content-services-community-distribution", + image_name = "alfresco-share-community", + name = "community" + }, + { + artifact = "alfresco-content-services-share-distribution", + image_name = "alfresco-share", + name = "enterprise" + } + ] + } } variable "ALFRESCO_SOLR_DIST_DIR" { diff --git a/share/Dockerfile b/share/Dockerfile index ee3a7bd3..7322fb9f 100644 --- a/share/Dockerfile +++ b/share/Dockerfile @@ -8,15 +8,20 @@ EXPOSE 8000 RUN mkdir -p ${CATALINA_HOME}/shared/classes/alfresco/web-extension \ ${CATALINA_HOME}/conf/Catalina/localhost +RUN yum install -y unzip + COPY entrypoint.sh ${CATALINA_HOME}/shared/classes/alfresco -ADD amps /tmp/amps -ADD distribution /tmp/ +ARG ALFRESCO_SHARE_ARTIFACT + +COPY distribution/${ALFRESCO_SHARE_ARTIFACT}-*.zip /tmp/ +COPY distribution/share-config-custom.xml /tmp/ +COPY amps /tmp/amps + ENV DISTDIR="/tmp/distribution" -RUN yum install -y unzip RUN unzip /tmp/*.zip -d ${DISTDIR} -RUN if ls ${DISTDIR}/alfresco* > /dev/null 2>&1; then mv ${DISTDIR}/alfresco* ${DISTDIR}/; fi +RUN if ls ${DISTDIR}/alfresco* > /dev/null 2>&1; then mv ${DISTDIR}/alfresco*/* ${DISTDIR}/; fi RUN unzip ${DISTDIR}/web-server/webapps/share.war -d ${CATALINA_HOME}/webapps/share RUN cp -r ${DISTDIR}/amps/ /tmp/amps/ RUN cp /tmp/share-config-custom.xml ${CATALINA_HOME}/shared/classes/alfresco/web-extension diff --git a/test/community-docker-compose.yml b/test/community-docker-compose.yml index 72682822..e5d30cb2 100644 --- a/test/community-docker-compose.yml +++ b/test/community-docker-compose.yml @@ -52,7 +52,7 @@ services: ports: - "8090:8090" share: - image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-share:${TAG} + image: ${REGISTRY}/${REGISTRY_NAMESPACE}/alfresco-share-community:${TAG} mem_limit: 1g environment: CSRF_FILTER_ORIGIN: http://localhost:8080 From fe634f708b30e1a55fd900acb28a2de3c6ff1f4e Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Thu, 17 Oct 2024 10:34:50 +0200 Subject: [PATCH 18/24] fix warning --- search/service/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/service/Dockerfile b/search/service/Dockerfile index fe43e3fc..53519de1 100644 --- a/search/service/Dockerfile +++ b/search/service/Dockerfile @@ -1,7 +1,7 @@ ARG DISTRIB_NAME ARG DISTRIB_MAJOR -FROM java_base as solr_build +FROM java_base AS solr_build ARG ALFRESCO_SOLR_DIST_DIR From fdd9cf0e163906a7b8326c825443edcb9fa060cb Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Thu, 17 Oct 2024 10:36:28 +0200 Subject: [PATCH 19/24] collect logs at the end --- .github/workflows/build_forks.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index 9c9243a8..ead25335 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -193,6 +193,17 @@ jobs: kubectl get all --all-namespaces kubectl describe pod + - name: Run helm test + id: helm_test + run: helm test acs + + - name: Debug cluster status after helm test + if: always() && steps.helm_test.outcome != 'skipped' + run: | + kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 + kubectl get all --all-namespaces + kubectl describe pod + - name: Collect logs from all containers if: always() && steps.helm_install.outcome != 'skipped' run: | @@ -207,14 +218,3 @@ jobs: with: name: k8s-logs-community path: logs - - - name: Run helm test - id: helm_test - run: helm test acs - - - name: Debug cluster status after helm test - if: always() && steps.helm_test.outcome != 'skipped' - run: | - kubectl logs -l app.kubernetes.io/component=dtas --tail=-1 - kubectl get all --all-namespaces - kubectl describe pod From c966d958c390c44e2d375c2e14a4297b63d88132 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Thu, 17 Oct 2024 10:38:12 +0200 Subject: [PATCH 20/24] drop unnecessary regcred --- .github/workflows/build_forks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index ead25335..fab324fd 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -171,7 +171,6 @@ jobs: helm install acs ./acs-deployment/helm/alfresco-content-services \ --set global.search.sharedSecret="$(openssl rand -hex 24)" \ --set global.known_urls=http://localhost \ - --set global.alfrescoRegistryPullSecrets=regcred \ --values ./acs-deployment/helm/alfresco-content-services/community_values.yaml \ --values ./acs-deployment/test/community-integration-test-values.yaml \ --values test/helm/test-overrides.yaml \ From b0e483a3fde13e4c420135ec84cfefb9b1c7bbb4 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Thu, 17 Oct 2024 10:38:37 +0200 Subject: [PATCH 21/24] tengine wait probes --- test/helm/test-overrides-community.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/helm/test-overrides-community.yaml b/test/helm/test-overrides-community.yaml index d18bbacc..87d54f2d 100644 --- a/test/helm/test-overrides-community.yaml +++ b/test/helm/test-overrides-community.yaml @@ -21,16 +21,16 @@ alfresco-repository: alfresco-transform-service: pdfrenderer: livenessProbe: - initialDelaySeconds: 60 + initialDelaySeconds: 120 imagemagick: livenessProbe: - initialDelaySeconds: 60 + initialDelaySeconds: 120 libreoffice: livenessProbe: - initialDelaySeconds: 60 + initialDelaySeconds: 120 tika: livenessProbe: - initialDelaySeconds: 60 + initialDelaySeconds: 120 transformmisc: livenessProbe: - initialDelaySeconds: 60 + initialDelaySeconds: 120 From 121034880974f5f5f9e7df4584fc2db0b3a46bb1 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Fri, 18 Oct 2024 09:13:23 +0200 Subject: [PATCH 22/24] refactor bake targets --- Makefile | 2 +- docker-bake.hcl | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0cf3ed92..b605f168 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ ats: prepare_ats prepare_tengines search_enterprise: prepare_search_enterprise @echo "Building Search Enterprise images" - docker buildx bake ${DOCKER_BAKE_ARGS} enterprise-search + docker buildx bake ${DOCKER_BAKE_ARGS} enterprise_search connectors: prepare_connectors @echo "Building Connectors images" diff --git a/docker-bake.hcl b/docker-bake.hcl index 44a36da4..81d3bcfb 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,16 +1,24 @@ group "default" { - targets = ["content_service", "enterprise-search", "ats", "tengines", "connectors", "search_service", "adf_apps", "sync"] + targets = ["enterprise", "community"] +} + +group "enterprise" { + targets = ["content_service_enterprise", "enterprise_search", "ats", "tengines", "connectors", "adf_apps", "sync"] } group "community" { - targets = ["repository_community", "share_community", "search_service", "tengines", "acc"] + targets = ["content_service_community", "search_service", "tengines", "acc"] +} + +group "content_service_enterprise" { + targets = ["repository_enterprise", "share_enterprise"] } -group "content_service" { - targets = ["repository", "share"] +group "content_service_community" { + targets = ["repository_community", "share_community"] } -group "enterprise-search" { +group "enterprise_search" { targets = ["search_liveindexing", "search_reindexing"] } From 2ebc7284c2e3e7f1921f92e07487f437a8129647 Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Fri, 18 Oct 2024 09:13:54 +0200 Subject: [PATCH 23/24] simplify image overrides --- .github/workflows/build_forks.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_forks.yml b/.github/workflows/build_forks.yml index fab324fd..88bc4526 100644 --- a/.github/workflows/build_forks.yml +++ b/.github/workflows/build_forks.yml @@ -157,20 +157,14 @@ jobs: helm repo add elastic https://helm.elastic.co helm dependency build - - name: Preprocess test-overrides.yaml - env: - OVERRIDES_VALUES_FILE: test/helm/test-overrides.yaml - run: | - sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} - cat ${{ env.OVERRIDES_VALUES_FILE }} - - name: Helm install id: helm_install run: | helm install acs ./acs-deployment/helm/alfresco-content-services \ --set global.search.sharedSecret="$(openssl rand -hex 24)" \ --set global.known_urls=http://localhost \ + --set alfresco-repository.image.repository=localhost/alfresco/alfresco-content-repository-community \ + --set share.image.repository=localhost/alfresco/alfresco-share-community \ --values ./acs-deployment/helm/alfresco-content-services/community_values.yaml \ --values ./acs-deployment/test/community-integration-test-values.yaml \ --values test/helm/test-overrides.yaml \ From 917ed0ad7c5050d4f031df0a61aae310b9c895de Mon Sep 17 00:00:00 2001 From: Giovanni Toraldo Date: Fri, 18 Oct 2024 09:16:07 +0200 Subject: [PATCH 24/24] test against community share in main wf --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f062829f..5f9ad2c4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -243,6 +243,7 @@ jobs: if [ "${{ matrix.edition }}" = "community" ]; then sed -i "s|/alfresco-content-repository|/alfresco-content-repository-community|g" ${{ env.OVERRIDES_VALUES_FILE }} + sed -i "s|/alfresco-share|/alfresco-share-community|g" ${{ env.OVERRIDES_VALUES_FILE }} fi cat ${{ env.OVERRIDES_VALUES_FILE }}