From 890ebc00acc66f4ccfb83e7e36a4f77ab1f1f56a Mon Sep 17 00:00:00 2001 From: red-hat-konflux Date: Mon, 9 Sep 2024 07:21:13 +0000 Subject: [PATCH 1/8] Red Hat Konflux update ray-cuda Signed-off-by: red-hat-konflux --- .tekton/ray-cuda-pull-request.yaml | 478 +++++++++++++++++++++++++++++ .tekton/ray-cuda-push.yaml | 475 ++++++++++++++++++++++++++++ 2 files changed, 953 insertions(+) create mode 100644 .tekton/ray-cuda-pull-request.yaml create mode 100644 .tekton/ray-cuda-push.yaml diff --git a/.tekton/ray-cuda-pull-request.yaml b/.tekton/ray-cuda-pull-request.yaml new file mode 100644 index 00000000..d827752e --- /dev/null +++ b/.tekton/ray-cuda-pull-request.yaml @@ -0,0 +1,478 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/red-hat-data-services/distributed-workloads?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: runtime-ray + appstudio.openshift.io/component: ray-cuda + pipelines.appstudio.openshift.io/type: build + name: ray-cuda-on-pull-request + namespace: rhoai-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-cuda:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: dockerfile + value: /images/runtime/ray/cuda/Dockerfile + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-image-index.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-summary:0.2@sha256:d97c04ab42f277b1103eb6f3a053b247849f4f5b3237ea302a8ecada3b24e15b + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:058a59f72997c9cf1be20978eb6a145d8d4d436c6098f2460bd96766bb363b20 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.prefetch-input) + operator: notin + values: + - "" + workspaces: + - name: source + workspace: workspace + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:18eecec92fcdb96dc346aecbbe88fb5fd95e34ee6ef4ad714dc1303723a8e4ea + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.1@sha256:21cb5ebaff7a9216903cf78933dc4ec4dd6283a52636b16590a5f52ceb278269 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:d98fa9daf5ee12dfbf00880b83d092d01ce9994d79836548d2f82748bb0c64a2 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.1@sha256:baea4be429cf8d91f7c758378cea42819fe324f25a7f957bf9805409cab6d123 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.2@sha256:82c42d27c9c59db6cf6c235e89f7b37f5cdfc75d0d361ca0ee91ae703ba72301 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:7bb17b937c9342f305468e8a6d0a22493e3ecde58977bd2ffc8b50e2fa234d58 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:e6beb161ed59d7be26317da03e172137b31b26648d3e139558e9a457bc56caff + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:92d63edd09636f97961ca18fac14b67935179d2c14b4a4d5f8087c614e8c2bd9 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + workspaces: + - name: workspace + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/ray-cuda-push.yaml b/.tekton/ray-cuda-push.yaml new file mode 100644 index 00000000..b3c972a4 --- /dev/null +++ b/.tekton/ray-cuda-push.yaml @@ -0,0 +1,475 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/red-hat-data-services/distributed-workloads?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: runtime-ray + appstudio.openshift.io/component: ray-cuda + pipelines.appstudio.openshift.io/type: build + name: ray-cuda-on-push + namespace: rhoai-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-cuda:{{revision}} + - name: dockerfile + value: /images/runtime/ray/cuda/Dockerfile + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-image-index.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-summary:0.2@sha256:d97c04ab42f277b1103eb6f3a053b247849f4f5b3237ea302a8ecada3b24e15b + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:058a59f72997c9cf1be20978eb6a145d8d4d436c6098f2460bd96766bb363b20 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.prefetch-input) + operator: notin + values: + - "" + workspaces: + - name: source + workspace: workspace + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:18eecec92fcdb96dc346aecbbe88fb5fd95e34ee6ef4ad714dc1303723a8e4ea + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.1@sha256:21cb5ebaff7a9216903cf78933dc4ec4dd6283a52636b16590a5f52ceb278269 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:d98fa9daf5ee12dfbf00880b83d092d01ce9994d79836548d2f82748bb0c64a2 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.1@sha256:baea4be429cf8d91f7c758378cea42819fe324f25a7f957bf9805409cab6d123 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.2@sha256:82c42d27c9c59db6cf6c235e89f7b37f5cdfc75d0d361ca0ee91ae703ba72301 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:7bb17b937c9342f305468e8a6d0a22493e3ecde58977bd2ffc8b50e2fa234d58 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:e6beb161ed59d7be26317da03e172137b31b26648d3e139558e9a457bc56caff + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:92d63edd09636f97961ca18fac14b67935179d2c14b4a4d5f8087c614e8c2bd9 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + workspaces: + - name: workspace + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} From cf98f9e3825f394e7fec6599be12a0df0e927b7d Mon Sep 17 00:00:00 2001 From: red-hat-konflux Date: Mon, 9 Sep 2024 07:23:14 +0000 Subject: [PATCH 2/8] Red Hat Konflux update ray-rocm Signed-off-by: red-hat-konflux --- .tekton/ray-rocm-pull-request.yaml | 478 +++++++++++++++++++++++++++++ .tekton/ray-rocm-push.yaml | 475 ++++++++++++++++++++++++++++ 2 files changed, 953 insertions(+) create mode 100644 .tekton/ray-rocm-pull-request.yaml create mode 100644 .tekton/ray-rocm-push.yaml diff --git a/.tekton/ray-rocm-pull-request.yaml b/.tekton/ray-rocm-pull-request.yaml new file mode 100644 index 00000000..55a29a94 --- /dev/null +++ b/.tekton/ray-rocm-pull-request.yaml @@ -0,0 +1,478 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/red-hat-data-services/distributed-workloads?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: runtime-ray + appstudio.openshift.io/component: ray-rocm + pipelines.appstudio.openshift.io/type: build + name: ray-rocm-on-pull-request + namespace: rhoai-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-rocm:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: dockerfile + value: images/runtime/ray/rocm/Dockerfile + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-image-index.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-summary:0.2@sha256:d97c04ab42f277b1103eb6f3a053b247849f4f5b3237ea302a8ecada3b24e15b + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:058a59f72997c9cf1be20978eb6a145d8d4d436c6098f2460bd96766bb363b20 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.prefetch-input) + operator: notin + values: + - "" + workspaces: + - name: source + workspace: workspace + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:18eecec92fcdb96dc346aecbbe88fb5fd95e34ee6ef4ad714dc1303723a8e4ea + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.1@sha256:21cb5ebaff7a9216903cf78933dc4ec4dd6283a52636b16590a5f52ceb278269 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:d98fa9daf5ee12dfbf00880b83d092d01ce9994d79836548d2f82748bb0c64a2 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.1@sha256:baea4be429cf8d91f7c758378cea42819fe324f25a7f957bf9805409cab6d123 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.2@sha256:82c42d27c9c59db6cf6c235e89f7b37f5cdfc75d0d361ca0ee91ae703ba72301 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:7bb17b937c9342f305468e8a6d0a22493e3ecde58977bd2ffc8b50e2fa234d58 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:e6beb161ed59d7be26317da03e172137b31b26648d3e139558e9a457bc56caff + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:92d63edd09636f97961ca18fac14b67935179d2c14b4a4d5f8087c614e8c2bd9 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + workspaces: + - name: workspace + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/ray-rocm-push.yaml b/.tekton/ray-rocm-push.yaml new file mode 100644 index 00000000..27fa26a1 --- /dev/null +++ b/.tekton/ray-rocm-push.yaml @@ -0,0 +1,475 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/red-hat-data-services/distributed-workloads?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "main" + creationTimestamp: null + labels: + appstudio.openshift.io/application: runtime-ray + appstudio.openshift.io/component: ray-rocm + pipelines.appstudio.openshift.io/type: build + name: ray-rocm-on-push + namespace: rhoai-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-rocm:{{revision}} + - name: dockerfile + value: images/runtime/ray/rocm/Dockerfile + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:9bfc6b99ef038800fe131d7b45ff3cd4da3a415dd536f7c657b3527b01c4a13b + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-image-index.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-summary:0.2@sha256:d97c04ab42f277b1103eb6f3a053b247849f4f5b3237ea302a8ecada3b24e15b + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:0bb1be8363557e8e07ec34a3c5daaaaa23c9d533f0bb12f00dc604d00de50814 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.1@sha256:058a59f72997c9cf1be20978eb6a145d8d4d436c6098f2460bd96766bb363b20 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.prefetch-input) + operator: notin + values: + - "" + workspaces: + - name: source + workspace: workspace + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:18eecec92fcdb96dc346aecbbe88fb5fd95e34ee6ef4ad714dc1303723a8e4ea + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.1@sha256:21cb5ebaff7a9216903cf78933dc4ec4dd6283a52636b16590a5f52ceb278269 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:d98fa9daf5ee12dfbf00880b83d092d01ce9994d79836548d2f82748bb0c64a2 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.1@sha256:baea4be429cf8d91f7c758378cea42819fe324f25a7f957bf9805409cab6d123 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.2@sha256:82c42d27c9c59db6cf6c235e89f7b37f5cdfc75d0d361ca0ee91ae703ba72301 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:7bb17b937c9342f305468e8a6d0a22493e3ecde58977bd2ffc8b50e2fa234d58 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:e6beb161ed59d7be26317da03e172137b31b26648d3e139558e9a457bc56caff + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:92d63edd09636f97961ca18fac14b67935179d2c14b4a4d5f8087c614e8c2bd9 + - name: kind + value: task + resolver: bundles + workspaces: + - name: workspace + workspace: workspace + workspaces: + - name: workspace + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} From 8a179bf2a1598dfccd87d4eb20e15faa23d4e612 Mon Sep 17 00:00:00 2001 From: Moulali Date: Mon, 9 Sep 2024 16:41:51 +0530 Subject: [PATCH 3/8] onboard ray cuda on konflux --- .tekton/ray-cuda-pull-request.yaml | 14 +++++++++----- .tekton/ray-cuda-push.yaml | 16 ++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.tekton/ray-cuda-pull-request.yaml b/.tekton/ray-cuda-pull-request.yaml index d827752e..30da8c19 100644 --- a/.tekton/ray-cuda-pull-request.yaml +++ b/.tekton/ray-cuda-pull-request.yaml @@ -7,8 +7,10 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "main" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" + && target_branch == "main" + && ( !".tekton/**".pathChanged() || ".tekton/ray-cuda-pull-request.yaml".pathChanged() ) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray @@ -27,7 +29,9 @@ spec: - name: image-expires-after value: 5d - name: dockerfile - value: /images/runtime/ray/cuda/Dockerfile + value: Dockerfile + - name: path-context + value: images/runtime/ray/cuda pipelineSpec: finally: - name: show-sbom @@ -233,9 +237,9 @@ spec: taskRef: params: - name: name - value: buildah + value: buildah-8gb - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + value: quay.io/konflux-ci/tekton-catalog/task-buildah-8gb:0.2 - name: kind value: task resolver: bundles diff --git a/.tekton/ray-cuda-push.yaml b/.tekton/ray-cuda-push.yaml index b3c972a4..be52adfb 100644 --- a/.tekton/ray-cuda-push.yaml +++ b/.tekton/ray-cuda-push.yaml @@ -6,8 +6,10 @@ metadata: build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "main" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" + && target_branch == "main" + && ( !".tekton/**".pathChanged() || ".tekton/ray-cuda-push.yaml".pathChanged() ) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray @@ -22,9 +24,11 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-cuda:{{revision}} + value: quay.io/modh/ray:cuda - name: dockerfile - value: /images/runtime/ray/cuda/Dockerfile + value: Dockerfile + - name: path-context + value: images/runtime/ray/cuda pipelineSpec: finally: - name: show-sbom @@ -230,9 +234,9 @@ spec: taskRef: params: - name: name - value: buildah + value: buildah-8gb - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + value: quay.io/konflux-ci/tekton-catalog/task-buildah-8gb:0.2 - name: kind value: task resolver: bundles From 4c7ab5c1992786b8ac0a13a705b3845e18cca6a5 Mon Sep 17 00:00:00 2001 From: Moulali Date: Mon, 9 Sep 2024 18:04:40 +0530 Subject: [PATCH 4/8] onboard ray rocm on konflux --- .tekton/ray-rocm-pull-request.yaml | 14 +++++++++----- .tekton/ray-rocm-push.yaml | 16 ++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.tekton/ray-rocm-pull-request.yaml b/.tekton/ray-rocm-pull-request.yaml index 55a29a94..7c0d54bd 100644 --- a/.tekton/ray-rocm-pull-request.yaml +++ b/.tekton/ray-rocm-pull-request.yaml @@ -7,8 +7,10 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "main" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" + && target_branch == "main" + && ( !".tekton/**".pathChanged() || ".tekton/ray-rocm-pull-request.yaml".pathChanged() ) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray @@ -27,7 +29,9 @@ spec: - name: image-expires-after value: 5d - name: dockerfile - value: images/runtime/ray/rocm/Dockerfile + value: Dockerfile + - name: path-context + value: images/runtime/ray/rocm pipelineSpec: finally: - name: show-sbom @@ -233,9 +237,9 @@ spec: taskRef: params: - name: name - value: buildah + value: buildah-8gb - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + value: quay.io/konflux-ci/tekton-catalog/task-buildah-8gb:0.2 - name: kind value: task resolver: bundles diff --git a/.tekton/ray-rocm-push.yaml b/.tekton/ray-rocm-push.yaml index 27fa26a1..92e93832 100644 --- a/.tekton/ray-rocm-push.yaml +++ b/.tekton/ray-rocm-push.yaml @@ -6,8 +6,10 @@ metadata: build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "main" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "pull_request" + && target_branch == "main" + && ( !".tekton/**".pathChanged() || ".tekton/ray-rocm-push.yaml".pathChanged() ) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray @@ -22,9 +24,11 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/rhoai-tenant/runtime-ray/ray-rocm:{{revision}} + value: quay.io/modh/ray:2.35.0-py39-rocm61 - name: dockerfile - value: images/runtime/ray/rocm/Dockerfile + value: Dockerfile + - name: path-context + value: images/runtime/ray/rocm pipelineSpec: finally: - name: show-sbom @@ -230,9 +234,9 @@ spec: taskRef: params: - name: name - value: buildah + value: buildah-8gb - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.2@sha256:a523f60203d90e149f96ec776b47ce85a7acfd6d634ddfc18f4a03f14e08ea0e + value: quay.io/konflux-ci/tekton-catalog/task-buildah-8gb:0.2 - name: kind value: task resolver: bundles From 0217d6a0bb22b82849d06b8851090535f617856b Mon Sep 17 00:00:00 2001 From: Moulali Date: Mon, 9 Sep 2024 18:18:36 +0530 Subject: [PATCH 5/8] changes added --- .tekton/ray-cuda-pull-request.yaml | 1 + .tekton/ray-cuda-push.yaml | 3 ++- .tekton/ray-rocm-pull-request.yaml | 1 + .tekton/ray-rocm-push.yaml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.tekton/ray-cuda-pull-request.yaml b/.tekton/ray-cuda-pull-request.yaml index 30da8c19..7989ce38 100644 --- a/.tekton/ray-cuda-pull-request.yaml +++ b/.tekton/ray-cuda-pull-request.yaml @@ -11,6 +11,7 @@ metadata: event == "pull_request" && target_branch == "main" && ( !".tekton/**".pathChanged() || ".tekton/ray-cuda-pull-request.yaml".pathChanged() ) + && !(".tekton/ray-rcom-pull.yaml".pathChanged() && !".tekton/ray-rcom-push.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray diff --git a/.tekton/ray-cuda-push.yaml b/.tekton/ray-cuda-push.yaml index be52adfb..37f3b418 100644 --- a/.tekton/ray-cuda-push.yaml +++ b/.tekton/ray-cuda-push.yaml @@ -10,6 +10,7 @@ metadata: event == "pull_request" && target_branch == "main" && ( !".tekton/**".pathChanged() || ".tekton/ray-cuda-push.yaml".pathChanged() ) + && !(".tekton/ray-rcom-pull.yaml".pathChanged() && !".tekton/ray-rcom-push.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray @@ -24,7 +25,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/modh/ray:cuda + value: quay.io/modh/ray:2.35.0-py39-cu121 - name: dockerfile value: Dockerfile - name: path-context diff --git a/.tekton/ray-rocm-pull-request.yaml b/.tekton/ray-rocm-pull-request.yaml index 7c0d54bd..d541d9b1 100644 --- a/.tekton/ray-rocm-pull-request.yaml +++ b/.tekton/ray-rocm-pull-request.yaml @@ -11,6 +11,7 @@ metadata: event == "pull_request" && target_branch == "main" && ( !".tekton/**".pathChanged() || ".tekton/ray-rocm-pull-request.yaml".pathChanged() ) + && !(".tekton/ray-cuda-pull.yaml".pathChanged() && !".tekton/ray-cuda-push.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray diff --git a/.tekton/ray-rocm-push.yaml b/.tekton/ray-rocm-push.yaml index 92e93832..3726b0fc 100644 --- a/.tekton/ray-rocm-push.yaml +++ b/.tekton/ray-rocm-push.yaml @@ -10,6 +10,7 @@ metadata: event == "pull_request" && target_branch == "main" && ( !".tekton/**".pathChanged() || ".tekton/ray-rocm-push.yaml".pathChanged() ) + && !(".tekton/ray-cuda-pull.yaml".pathChanged() && !".tekton/ray-cuda-push.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: runtime-ray From 278849940c367a444e87ac686569b7eb4be2b56c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 5 Sep 2024 14:21:58 +0200 Subject: [PATCH 6/8] Align the CUDA Ray custom image with the ROCm image --- images/runtime/examples/ray-torch-cuda/Dockerfile | 12 ++++++++++-- images/runtime/examples/ray-torch-rocm/Dockerfile | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/images/runtime/examples/ray-torch-cuda/Dockerfile b/images/runtime/examples/ray-torch-cuda/Dockerfile index d6dafec4..66aa7354 100644 --- a/images/runtime/examples/ray-torch-cuda/Dockerfile +++ b/images/runtime/examples/ray-torch-cuda/Dockerfile @@ -1,3 +1,11 @@ -FROM quay.io/rhoai/ray:2.23.0-py39-cu121 +FROM quay.io/rhoai/ray:2.35.0-py39-cu121 -RUN python3 -m pip install torch==2.3.1 +# Install PyTorch +RUN python3 -m pip install torch==2.4.0 + +# Install Flash Attention +RUN python3 -m pip install wheel ninja +RUN python3 -m pip install flash-attn==2.6.3 --no-build-isolation + +# Install DeepSpeed +RUN DS_ACCELERATOR=cuda DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed==0.15.0 --global-option="build_ext" --global-option="-j8" diff --git a/images/runtime/examples/ray-torch-rocm/Dockerfile b/images/runtime/examples/ray-torch-rocm/Dockerfile index 415b203f..67982d92 100644 --- a/images/runtime/examples/ray-torch-rocm/Dockerfile +++ b/images/runtime/examples/ray-torch-rocm/Dockerfile @@ -8,7 +8,7 @@ RUN python3 -m pip install torch==2.4.0 --index-url https://download.pytorch.org # Install Flash Attention RUN python3 -m pip install wheel ninja -run export TMP_DIR=$(mktemp -d) \ +RUN export TMP_DIR=$(mktemp -d) \ && cd $TMP_DIR \ && git clone --depth 1 --branch v2.6.3 https://github.com/Dao-AILab/flash-attention.git \ && cd flash-attention \ From 05858ac7f7ed500bd87a0298884b51cbf82490d3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 5 Sep 2024 14:23:23 +0200 Subject: [PATCH 7/8] Use latest Ray custom images in the LLM fine-tuning with Ray example --- .../ray_finetune_llm_deepspeed.ipynb | 6 +++--- examples/ray-finetune-llm-deepspeed/requirements.txt | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/examples/ray-finetune-llm-deepspeed/ray_finetune_llm_deepspeed.ipynb b/examples/ray-finetune-llm-deepspeed/ray_finetune_llm_deepspeed.ipynb index 3bf27657..d81da032 100644 --- a/examples/ray-finetune-llm-deepspeed/ray_finetune_llm_deepspeed.ipynb +++ b/examples/ray-finetune-llm-deepspeed/ray_finetune_llm_deepspeed.ipynb @@ -67,12 +67,12 @@ " worker_memory_requests=128,\n", " worker_memory_limits=256,\n", " head_memory=128,\n", - " # Use the following parameters with NVIDIA GPUs \n", - " image=\"quay.io/rhoai/ray:2.23.0-py39-cu121\",\n", + " # Use the following parameters with NVIDIA GPUs\n", + " image=\"quay.io/rhoai/ray:2.35.0-py39-cu121-torch24-fa26\",\n", " head_extended_resource_requests={'nvidia.com/gpu':1},\n", " worker_extended_resource_requests={'nvidia.com/gpu':1},\n", " # Or replace them with these parameters for AMD GPUs\n", - " # image=\"quay.io/rhoai/ray:2.35.0-py39-rocm61-torch24\",\n", + " # image=\"quay.io/rhoai/ray:2.35.0-py39-rocm61-torch24-fa26\",\n", " # head_extended_resource_requests={'amd.com/gpu':1},\n", " # worker_extended_resource_requests={'amd.com/gpu':1},\n", "))" diff --git a/examples/ray-finetune-llm-deepspeed/requirements.txt b/examples/ray-finetune-llm-deepspeed/requirements.txt index 47e21fdd..735afc1e 100644 --- a/examples/ray-finetune-llm-deepspeed/requirements.txt +++ b/examples/ray-finetune-llm-deepspeed/requirements.txt @@ -1,11 +1,5 @@ accelerate==0.31.0 awscliv2==2.3.0 datasets==2.19.2 -deepspeed==0.14.4 -# Flash Attention 2 requires PyTorch to be installed first -# See https://github.com/Dao-AILab/flash-attention/issues/453 -https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.3/flash_attn-2.6.3+cu123torch2.3cxx11abiFALSE-cp39-cp39-linux_x86_64.whl peft==0.11.1 -ray[train]==2.23.0 -torch==2.3.1 transformers==4.44.0 From 6ae4612803b100685e51dbb6635b78dc946283de Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 5 Sep 2024 14:26:24 +0200 Subject: [PATCH 8/8] Remove ROCm specific requirements file from LLM fine-tuning example --- examples/ray-finetune-llm-deepspeed/requirements_rocm.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 examples/ray-finetune-llm-deepspeed/requirements_rocm.txt diff --git a/examples/ray-finetune-llm-deepspeed/requirements_rocm.txt b/examples/ray-finetune-llm-deepspeed/requirements_rocm.txt deleted file mode 100644 index 735afc1e..00000000 --- a/examples/ray-finetune-llm-deepspeed/requirements_rocm.txt +++ /dev/null @@ -1,5 +0,0 @@ -accelerate==0.31.0 -awscliv2==2.3.0 -datasets==2.19.2 -peft==0.11.1 -transformers==4.44.0