diff --git a/.tekton/odh-trustyai-service-v2-16-pull-request.yaml b/.tekton/odh-trustyai-service-v2-16-pull-request.yaml index 409a22f5..ac64d2c7 100644 --- a/.tekton/odh-trustyai-service-v2-16-pull-request.yaml +++ b/.tekton/odh-trustyai-service-v2-16-pull-request.yaml @@ -1,5 +1,6 @@ apiVersion: tekton.dev/v1 kind: PipelineRun +#test metadata: annotations: build.appstudio.openshift.io/repo: https://github.com/red-hat-data-services/trustyai-explainability?rev={{revision}} @@ -30,6 +31,20 @@ spec: value: Dockerfile.konflux - name: path-context value: . + - name: prefetch-input + value: [{"path": ".", "type": "rpm"}, {"path": ".", "type": "generic"}] + - name: build-config-git-url + value: 'https://github.com/red-hat-data-services/RHOAI-Build-Config.git' + - name: build-config-revision + value: 'rhoai-2.16' + - name: build-config-path + value: 'trustyai-pig-builds/build-config.yaml' + - name: pnc-cli-config-git-url + value: 'https://gitlab.cee.redhat.com/project-ncl/utils.git' + - name: pnc-cli-config-revision + value: 'master' + - name: pnc-profile + value: 'prod' pipelineSpec: description: | This pipeline is ideal for building container images from a Containerfile while reducing network traffic. @@ -73,6 +88,27 @@ spec: - name: workspace workspace: workspace params: + - name: build-config-git-url + description: Source Repository URL containing the build configuration + type: string + - name: build-config-revision + default: "" + description: Revision of the Source Repository containing the build configuration + type: string + - name: build-config-path + default: "build-config.yaml" + description: Full path of the build configuration file + type: string + - description: Source Repository URL containing the PNC CLI configuration + name: pnc-cli-config-git-url + type: string + - default: "" + description: Revision of the Source Repository containing the PNC CLI configuration + name: pnc-cli-config-revision + type: string + - name: pnc-profile + value: 'prod' + - description: Source Repository URL name: git-url type: string @@ -101,7 +137,7 @@ spec: description: Skip checks against built image name: skip-checks type: string - - default: "false" + - default: "true" description: Execute the build with network isolation name: hermetic type: string @@ -187,18 +223,130 @@ spec: workspace: workspace - name: basic-auth workspace: git-auth + + - name: clone-build-config-repository + params: + - name: url + value: $(params.build-config-git-url) + - name: revision + value: $(params.build-config-revision) + - name: subdirectory + value: build-config + - name: sslVerify + value: false + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + + - name: clone-pnc-cli-config-repository + params: + - name: url + value: $(params.pnc-cli-config-git-url) + - name: revision + value: $(params.pnc-cli-config-revision) + - name: subdirectory + value: cli-config + - name: sslVerify + value: false + runAfter: + - clone-build-config-repository + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + + - name: pnc-cli-build + runAfter: + - clone-pnc-cli-config-repository + taskSpec: + results: + - description: The file containing the list of pnc built artifacts to be later installed inside the container build + name: pnc-file-list + steps: + - name: run-pnc-build + image: quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/pnc/pnc-cli@sha256:00f29bb186e3afedbffeb57942038f7d9ad747640e0a4ae2c53c573a8cdece74 + env: + - name: SSO_SERVICE_ACCOUNT_NAME + valueFrom: + secretKeyRef: + name: pnc-secret + key: svc.account.username + - name: SSO_SERVICE_ACCOUNT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: pnc-secret + key: svc.account.password + - name: PNC_PROFILE + valueFrom: + secretKeyRef: + name: pnc-secret + key: pnc.profile + + script: | + #!/bin/bash + set -e + + echo -e "\n=== Build config ===" + cat /workspace/source/build-config/$(params.build-config-path) + cp /workspace/source/build-config/$(params.build-config-path) /workspace/source + echo -e "====================" + + echo -e "\nReplacing credentials in the CLI config ..." + envsubst '${SSO_SERVICE_ACCOUNT_NAME} ${SSO_SERVICE_ACCOUNT_CLIENT_SECRET}' < /workspace/source/cli-config/konflux/configs/pnc_cli/config.yaml > /workspace/source/config.yaml + + echo -e "\nStarting the PNC build ..." + PNC_CLI_OUTPUT_FILE="/workspace/source/pnc-cli-run-output.json" + java -jar /home/jboss/bacon.jar pig run --mode=FORCE --downloadAttempts=3 /workspace/source -p /workspace/source --profile ${PNC_PROFILE} --jsonOutput > $PNC_CLI_OUTPUT_FILE + + echo -e "\nFinished the PNC build!" + echo -e "\n=== PNC build output ===" + cat $PNC_CLI_OUTPUT_FILE + echo -e "====================" + + echo -e "\nGetting the list of built files (to be used later in the container build) ..." + jq '[.builds[].builtArtifacts[]?.downloadUrl ]' "$PNC_CLI_OUTPUT_FILE" > "$(results.pnc-file-list.path)" + + jq '{metadata: {version: "1.0"}, artifacts: [.builds[].builtArtifacts[]? | select(.downloadUrl | endswith(".zip")) | {download_url: .downloadUrl , checksums: {sha256: .sha256}}] }' $PNC_CLI_OUTPUT_FILE | yq -p json > "$(workspaces.source.path)/source/generic_lockfile.yaml" + cat $(results.pnc-file-list.path) + + workspaces: + - name: source + workspace: workspace - name: prefetch-dependencies params: - name: input value: $(params.prefetch-input) + - name: dev-package-managers + value: "true" runAfter: - - clone-repository + - pnc-cli-build taskRef: params: - name: name value: prefetch-dependencies - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:fe7234e3824d1e65d6a7aac352e7a6bbce623d90d8d7da9aceeee108ad2c61be + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies@sha256:f53fe5482599b39ae2d1004cf09a2026fd9dd3822ab6ef46b51b4a398b0a3232 - name: kind value: task resolver: bundles @@ -232,6 +380,7 @@ spec: value: $(tasks.clone-repository.results.commit) - name: BUILD_ARGS value: + - PNC_FILES_JSON=$(tasks.pnc-cli-build.results.pnc-file-list) - $(params.build-args[*]) - name: BUILD_ARGS_FILE value: $(params.build-args-file) diff --git a/.tekton/odh-trustyai-service-v2-16-push.yaml b/.tekton/odh-trustyai-service-v2-16-push.yaml index 5d9c2ab9..58adea00 100644 --- a/.tekton/odh-trustyai-service-v2-16-push.yaml +++ b/.tekton/odh-trustyai-service-v2-16-push.yaml @@ -43,6 +43,8 @@ spec: value: 'master' - name: pnc-profile value: 'prod' + - name: prefetch-input + value: [{"path": ".", "type": "rpm"}, {"path": ".", "type": "generic"}] pipelineSpec: description: | This pipeline is ideal for building container images from a Containerfile while reducing network traffic. @@ -157,7 +159,7 @@ spec: description: Skip checks against built image name: skip-checks type: string - - default: "false" + - default: "true" description: Execute the build with network isolation name: hermetic type: string @@ -342,7 +344,7 @@ spec: name: pnc-file-list steps: - name: run-pnc-build - image: quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/pnc/pnc-cli:48a761b5093bc474850b2ebcf06641825940deb7 + image: quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/pnc/pnc-cli@sha256:00f29bb186e3afedbffeb57942038f7d9ad747640e0a4ae2c53c573a8cdece74 env: - name: SSO_SERVICE_ACCOUNT_NAME valueFrom: @@ -384,6 +386,8 @@ spec: echo -e "\nGetting the list of built files (to be used later in the container build) ..." jq '[.builds[].builtArtifacts[]?.downloadUrl ]' "$PNC_CLI_OUTPUT_FILE" > "$(results.pnc-file-list.path)" + jq '{metadata: {version: "1.0"}, artifacts: [.builds[].builtArtifacts[]? | select(.downloadUrl | endswith(".zip")) | {download_url: .downloadUrl , checksums: {sha256: .sha256}}] }' $PNC_CLI_OUTPUT_FILE | yq -p json > /workspace/output/generic_lockfile.yaml + cat $(results.pnc-file-list.path) workspaces: @@ -394,6 +398,8 @@ spec: params: - name: input value: $(params.prefetch-input) + - name: dev-package-managers + value: "true" runAfter: - pnc-cli-build taskRef: @@ -401,7 +407,7 @@ spec: - name: name value: prefetch-dependencies - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:fe7234e3824d1e65d6a7aac352e7a6bbce623d90d8d7da9aceeee108ad2c61be + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies@sha256:f53fe5482599b39ae2d1004cf09a2026fd9dd3822ab6ef46b51b4a398b0a3232 - name: kind value: task resolver: bundles diff --git a/Dockerfile.konflux b/Dockerfile.konflux index 2ec27a88..90a90b78 100644 --- a/Dockerfile.konflux +++ b/Dockerfile.konflux @@ -18,23 +18,16 @@ ARG CI_CONTAINER_VERSION="unknown" FROM registry.redhat.io/ubi8/ubi-minimal@sha256:7583ca0ea52001562bd81a961da3f75222209e6192e4e413ee226cff97dbd48c AS stage -# Define a build argument for the PNC list of built files -ARG PNC_FILES_JSON -RUN echo "Files to download: $PNC_FILES_JSON" - # Install packages for the install script and extract archives RUN microdnf --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install -y unzip jq wget ENV STAGE_DIR="/tmp/artifacts" WORKDIR $STAGE_DIR -# Filter the zip files only and unzip them in /root/ -RUN echo "$PNC_FILES_JSON" | jq -r '.[] | select(test("\\.zip$"))' | \ - while read url; do wget --no-check-certificate "$url"; done && \ +# Change to the output directory and unzip .zip files into /root/ +RUN cd /cachi2/source/deps/generic && \ for file in *.zip; do unzip -d /root/ "$file"; done - - ############################################################################### FROM registry.redhat.io/ubi8/openjdk-17-runtime@sha256:e2f33a6c60db4f4e70882a4a557eec5890997f8a1be7e3eb8971a0ff8a45a1a8 as runtime ENV LANGUAGE='en_US:en' diff --git a/rpms.in.yaml b/rpms.in.yaml new file mode 100644 index 00000000..0086f431 --- /dev/null +++ b/rpms.in.yaml @@ -0,0 +1,8 @@ +contentOrigin: + repofiles: + - ubi.repo +packages: + - unzip + - jq + - wget + - shadow-utils diff --git a/rpms.lock.yaml b/rpms.lock.yaml new file mode 100644 index 00000000..17bdcdf5 --- /dev/null +++ b/rpms.lock.yaml @@ -0,0 +1,64 @@ +--- +lockfileVersion: 1 +lockfileVendor: redhat +arches: +- arch: x86_64 + packages: + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/appstream/os/Packages/j/jq-1.6-9.el8_10.x86_64.rpm + repoid: ubi-8-appstream-rpms + size: 207896 + checksum: sha256:baaa3660d87c4f3c12776e051b7f13835fee8918389a57673519e3389eb7aa3b + name: jq + evr: 1.6-9.el8_10 + sourcerpm: jq-1.6-9.el8_10.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/appstream/os/Packages/o/oniguruma-6.8.2-3.el8.x86_64.rpm + repoid: ubi-8-appstream-rpms + size: 192632 + checksum: sha256:1c5c91d8a33987892ec7320c08311a31245be91800aa5879e20d137971bd053f + name: oniguruma + evr: 6.8.2-3.el8 + sourcerpm: oniguruma-6.8.2-3.el8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/appstream/os/Packages/w/wget-1.19.5-12.el8_10.x86_64.rpm + repoid: ubi-8-appstream-rpms + size: 751872 + checksum: sha256:8d609774711cea9728faf684ff7e9b389f0f3c2052aa04bcf3061950a830058b + name: wget + evr: 1.19.5-12.el8_10 + sourcerpm: wget-1.19.5-12.el8_10.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/e/expat-2.2.5-15.el8_10.x86_64.rpm + repoid: ubi-8-baseos-rpms + size: 117148 + checksum: sha256:1ae9138f9947f182ad077393c16522941bce2f225eee687a0387f0f71c416bc0 + name: expat + evr: 2.2.5-15.el8_10 + sourcerpm: expat-2.2.5-15.el8_10.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/l/libmetalink-0.1.3-7.el8.x86_64.rpm + repoid: ubi-8-baseos-rpms + size: 32784 + checksum: sha256:cd7c30d21e7240f60f0861c229e17fda43e855ab4c78fab39f47f7ae2be5720e + name: libmetalink + evr: 0.1.3-7.el8 + sourcerpm: libmetalink-0.1.3-7.el8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/l/libsemanage-2.9-9.el8_6.x86_64.rpm + repoid: ubi-8-baseos-rpms + size: 172128 + checksum: sha256:1f686a73273028ca85b5a6ac858292d7b7d2fcbe379d6912ba12fc0a49ac4cce + name: libsemanage + evr: 2.9-9.el8_6 + sourcerpm: libsemanage-2.9-9.el8_6.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/s/shadow-utils-4.6-22.el8.x86_64.rpm + repoid: ubi-8-baseos-rpms + size: 1292332 + checksum: sha256:ea73ee201451bbca0d6d14ca434c93800f01c8fb1b9daef727a5af1a27356d07 + name: shadow-utils + evr: 2:4.6-22.el8 + sourcerpm: shadow-utils-4.6-22.el8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/u/unzip-6.0-46.el8.x86_64.rpm + repoid: ubi-8-baseos-rpms + size: 200568 + checksum: sha256:af382b8bbb076e8662abf092c89ddd233796c6b8ed201d37fbcfcc07824295dd + name: unzip + evr: 6.0-46.el8 + sourcerpm: unzip-6.0-46.el8.src.rpm + source: [] + module_metadata: [] diff --git a/ubi.repo b/ubi.repo new file mode 100644 index 00000000..bdc28e0a --- /dev/null +++ b/ubi.repo @@ -0,0 +1,70 @@ +[ubi-8-baseos-rpms] +name = Red Hat Universal Base Image 8 (RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/baseos/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-baseos-debug-rpms] +name = Red Hat Universal Base Image 8 (Debug RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/baseos/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-baseos-source] +name = Red Hat Universal Base Image 8 (Source RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/baseos/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-appstream-rpms] +name = Red Hat Universal Base Image 8 (RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/appstream/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-appstream-debug-rpms] +name = Red Hat Universal Base Image 8 (Debug RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/appstream/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-appstream-source] +name = Red Hat Universal Base Image 8 (Source RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/appstream/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-codeready-builder-rpms] +name = Red Hat Universal Base Image 8 (RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/codeready-builder/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-codeready-builder] +name = Red Hat Universal Base Image 8 (RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/codeready-builder/os +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + + +[ubi-8-codeready-builder-debug-rpms] +name = Red Hat Universal Base Image 8 (Debug RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/codeready-builder/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-8-codeready-builder-source] +name = Red Hat Universal Base Image 8 (Source RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/$basearch/codeready-builder/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1