diff --git a/.tekton/universal-image-py312-cuda128-torch280-pull-request.yaml b/.tekton/odh-training-th03-cuda128-torch28-py312-rhel9-pull-request.yaml similarity index 74% rename from .tekton/universal-image-py312-cuda128-torch280-pull-request.yaml rename to .tekton/odh-training-th03-cuda128-torch28-py312-rhel9-pull-request.yaml index 2fcbfadf7..bb5279313 100644 --- a/.tekton/universal-image-py312-cuda128-torch280-pull-request.yaml +++ b/.tekton/odh-training-th03-cuda128-torch28-py312-rhel9-pull-request.yaml @@ -9,13 +9,13 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "true" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch == "main" && - ( "images/universal/training/py312-cuda128-torch280/***".pathChanged() || ".tekton/universal-image-py312-cuda128-torch280-pull-request.yaml".pathChanged() ) + ("images/universal/training/py312-cuda128-torch280/***".pathChanged() || ".tekton/odh-training-th03-cuda128-torch28-py312-rhel9-pull-request.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: odh-universal-image - appstudio.openshift.io/component: universal-image-py312-cuda128-torch280 + appstudio.openshift.io/component: odh-training-th03-cuda128-torch28-py312-rhel9 pipelines.appstudio.openshift.io/type: build - name: universal-image-py312-cuda128-torch280-on-pull-request + name: odh-training-th03-cuda128-torch28-py312-rhel9-on-pull-request namespace: open-data-hub-tenant spec: params: @@ -24,20 +24,21 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/opendatahub/odh-training-cuda128-torch28-py312-rhel9:on-pr-{{revision}} + value: quay.io/opendatahub/odh-training-th03-cuda128-torch28-py312-rhel9:on-pr-{{revision}} - name: image-expires-after value: 5d - - name: dockerfile - value: /images/universal/training/py312-cuda128-torch280/Dockerfile - name: build-platforms value: - linux-extra-fast/amd64 + - name: dockerfile + value: Dockerfile + - name: path-context + value: images/universal/training/py312-cuda128-torch280 pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while reducing network traffic. - - _Uses `buildah` to create a container image. It also optionally creates a source image and runs some build-time tests. EC will flag a violation for [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) if any tasks are added to the pipeline. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build?tab=tags)_ + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -84,7 +85,7 @@ spec: description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string @@ -106,10 +107,12 @@ spec: VMs name: privileged-nested type: string - - name: build-platforms - default: - - linux-extra-fast/amd64 - description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -147,14 +150,18 @@ spec: value: $(params.git-url) - name: revision value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) runAfter: - init taskRef: params: - name: name - value: git-clone + value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:33e03611639687a832c7ae388c244bd8c1f52ac44568aa6b949e3439e80d978b + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:3dc39eae48745a96097c07c577b944d6203a91c35d3f71d9ed5feab41d327a6a - name: kind value: task resolver: bundles @@ -164,41 +171,45 @@ spec: values: - "true" workspaces: - - name: output - workspace: workspace - name: basic-auth workspace: git-auth - name: prefetch-dependencies params: - name: input value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) runAfter: - clone-repository taskRef: params: - name: name - value: prefetch-dependencies + value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.2@sha256:a18a33aa577ac1b8f0c9ca6cd74c4c73a30cfd48a7b959c86390bc04066d1fb1 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:5946ca57aa29f162e11b74984ec58960f55f9fb6a0e97c6c9215c4161f768726 - name: kind value: task resolver: bundles workspaces: - - name: source - workspace: workspace - name: git-basic-auth workspace: git-auth - name: netrc workspace: netrc - - name: build-container - timeout: 8h + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) - name: DOCKERFILE value: $(params.dockerfile) - - name: PLATFORM - value: linux-extra-fast/amd64 - name: CONTEXT value: $(params.path-context) - name: HERMETIC @@ -220,14 +231,20 @@ spec: value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.6@sha256:a6fbfc569c3ebeee9ab032e06a456a1aa8c2d45476a49d03e8bcacb71bd03aac + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.6@sha256:17b267b5ae3deca5905d930e54337b89df45d3579f33b7fab4df74ee644cded4 - name: kind value: task resolver: bundles @@ -236,9 +253,6 @@ spec: operator: in values: - "true" - workspaces: - - name: source - workspace: workspace - name: build-image-index params: - name: IMAGE @@ -251,17 +265,17 @@ spec: value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:60ca793e71ba7abdf024278d6ffe12f35f292ddca3031e3ef2a3f2711203448d + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:31197f4ee71be47c6f491e888ff266cbbb8ad5ed1c7c4141cc14f002d1802a50 - name: kind value: task resolver: bundles @@ -276,14 +290,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: BINARY_IMAGE_DIGEST value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: source-build + value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.3@sha256:a681bd43f79482a8326288e304b33f25bdcb48d71f28ce2112569d89d9558348 + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:282cb5a9119a87e88559444feff67d76d6f356d03654b4845632c049b2314735 - name: kind value: task resolver: bundles @@ -296,9 +314,6 @@ spec: operator: in values: - "true" - workspaces: - - name: workspace - workspace: workspace - name: deprecated-base-image-check params: - name: IMAGE_URL @@ -321,7 +336,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -343,7 +363,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -369,14 +394,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-snyk-check + value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.4@sha256:25fefacfa0cbf453b8b7b66235f23c98f082f0deb45e064da3e622a4b2e3117f + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:60f2dac41844d222086ff7f477e51f3563716b183d87db89f603d6f604c21760 - name: kind value: task resolver: bundles @@ -385,10 +414,12 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - - name: clamav-scan + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -435,14 +466,18 @@ spec: - $(params.build-args[*]) - name: BUILD_ARGS_FILE value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - coverity-availability-check taskRef: params: - name: name - value: sast-coverity-check + value: sast-coverity-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check:0.3@sha256:9d572d7f7486224318d59de1b166efa68e59e17bac0785e2ecdcd014fe8e44d5 + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:ae62d14c999fd93246fef4e57d28570fa5200c3266b9a3263a39965e5a5b02d7 - name: kind value: task resolver: bundles @@ -455,9 +490,6 @@ spec: operator: in values: - success - workspaces: - - name: source - workspace: workspace - name: coverity-availability-check runAfter: - build-image-index @@ -481,14 +513,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-shell-check + value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check:0.1@sha256:4364876ab45148ddfe729a29de69323e12cddd5a43e85d16f36ad4a88c3794f7 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:1f0fcba24ebc447d9f8a2ea2e8f262fa435d6c523ca6b0346cd67261551fc9ed - name: kind value: task resolver: bundles @@ -497,23 +533,24 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - name: sast-unicode-check params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-unicode-check + value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check:0.3@sha256:d72abe93b4714b1b8faaea9fceb9acfaeca29c27eac8ff27ebc87622d75c871f + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:1833c618170ab9deb8455667f220df8e88d16ccd630a2361366f594e2bdcb712 - name: kind value: task resolver: bundles @@ -522,9 +559,6 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - name: apply-tags params: - name: IMAGE_URL @@ -552,20 +586,19 @@ spec: value: $(params.dockerfile) - name: CONTEXT value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: push-dockerfile + value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:74e982c07a808eaa5b1d8c126cafcbf3cc6ce94c883cf0845b55ce8064674b45 + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:13633d5ba8445c0f732a0a5d1b33ffbb708398e45ef1647542b0ab22fee25a6a - name: kind value: task resolver: bundles - workspaces: - - name: workspace - workspace: workspace - name: rpms-signature-scan params: - name: image-url @@ -579,7 +612,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:9c3a523814410046412abc09beb6f10082fecc56eb6d3386f6a6df305e5e011e + value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:78c8d7960c6db284356d94aaae64d1fca34fff4de6a6e20d897a088af0c81cf5 - name: kind value: task resolver: bundles @@ -589,26 +622,14 @@ spec: values: - "false" workspaces: - - name: workspace - name: git-auth optional: true - name: netrc optional: true taskRunTemplate: - serviceAccountName: build-pipeline-universal-image-py312-cuda128-torch280 + serviceAccountName: build-pipeline-odh-training-th03-cuda128-torch28-py312-rhel9 workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} - name: git-auth secret: secretName: '{{ git_auth_secret }}' -status: {} +status: {} \ No newline at end of file diff --git a/.tekton/universal-image-py312-cuda128-torch280-push.yaml b/.tekton/odh-training-th03-cuda128-torch28-py312-rhel9-push.yaml similarity index 74% rename from .tekton/universal-image-py312-cuda128-torch280-push.yaml rename to .tekton/odh-training-th03-cuda128-torch28-py312-rhel9-push.yaml index 7e471edc1..7ca384f04 100644 --- a/.tekton/universal-image-py312-cuda128-torch280-push.yaml +++ b/.tekton/odh-training-th03-cuda128-torch28-py312-rhel9-push.yaml @@ -8,13 +8,13 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "false" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch == "main" && - ( "images/universal/training/py312-cuda128-torch280/***".pathChanged() || ".tekton/universal-image-py312-cuda128-torch280-push.yaml".pathChanged() ) + ("images/universal/training/py312-cuda128-torch280/***".pathChanged() || ".tekton/odh-training-th03-cuda128-torch28-py312-rhel9-push.yaml".pathChanged()) creationTimestamp: null labels: appstudio.openshift.io/application: odh-universal-image - appstudio.openshift.io/component: universal-image-py312-cuda128-torch280 + appstudio.openshift.io/component: odh-training-th03-cuda128-torch28-py312-rhel9 pipelines.appstudio.openshift.io/type: build - name: universal-image-py312-cuda128-torch280-on-push + name: odh-training-th03-cuda128-torch28-py312-rhel9-on-push namespace: open-data-hub-tenant spec: params: @@ -23,18 +23,19 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/opendatahub/odh-training-cuda128-torch28-py312-rhel9:latest - - name: dockerfile - value: /images/universal/training/py312-cuda128-torch280/Dockerfile + value: quay.io/opendatahub/odh-training-th03-cuda128-torch28-py312-rhel9:latest - name: build-platforms value: - linux-extra-fast/amd64 + - name: dockerfile + value: Dockerfile + - name: path-context + value: images/universal/training/py312-cuda128-torch280 pipelineSpec: description: | - This pipeline is ideal for building container images from a Containerfile while reducing network traffic. - - _Uses `buildah` to create a container image. It also optionally creates a source image and runs some build-time tests. EC will flag a violation for [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) if any tasks are added to the pipeline. - This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build?tab=tags)_ + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ params: - description: Source Repository URL name: git-url @@ -81,7 +82,7 @@ spec: description: Build a source image. name: build-source-image type: string - - default: "false" + - default: "true" description: Add built image into an OCI image index name: build-image-index type: string @@ -103,10 +104,12 @@ spec: VMs name: privileged-nested type: string - - name: build-platforms - default: - - linux-extra-fast/amd64 - description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array results: - description: "" name: IMAGE_URL @@ -144,14 +147,18 @@ spec: value: $(params.git-url) - name: revision value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) runAfter: - init taskRef: params: - name: name - value: git-clone + value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone:0.1@sha256:33e03611639687a832c7ae388c244bd8c1f52ac44568aa6b949e3439e80d978b + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:3dc39eae48745a96097c07c577b944d6203a91c35d3f71d9ed5feab41d327a6a - name: kind value: task resolver: bundles @@ -161,41 +168,45 @@ spec: values: - "true" workspaces: - - name: output - workspace: workspace - name: basic-auth workspace: git-auth - name: prefetch-dependencies params: - name: input value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) runAfter: - clone-repository taskRef: params: - name: name - value: prefetch-dependencies + value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies:0.2@sha256:a18a33aa577ac1b8f0c9ca6cd74c4c73a30cfd48a7b959c86390bc04066d1fb1 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:5946ca57aa29f162e11b74984ec58960f55f9fb6a0e97c6c9215c4161f768726 - name: kind value: task resolver: bundles workspaces: - - name: source - workspace: workspace - name: git-basic-auth workspace: git-auth - name: netrc workspace: netrc - - name: build-container - timeout: 8h + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images params: - name: IMAGE value: $(params.output-image) - name: DOCKERFILE value: $(params.dockerfile) - - name: PLATFORM - value: linux-extra-fast/amd64 - name: CONTEXT value: $(params.path-context) - name: HERMETIC @@ -217,14 +228,20 @@ spec: value: $(tasks.clone-repository.results.url) - name: BUILDAH_FORMAT value: $(params.buildah-format) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" runAfter: - prefetch-dependencies taskRef: params: - name: name - value: buildah + value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah:0.6@sha256:a6fbfc569c3ebeee9ab032e06a456a1aa8c2d45476a49d03e8bcacb71bd03aac + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.6@sha256:17b267b5ae3deca5905d930e54337b89df45d3579f33b7fab4df74ee644cded4 - name: kind value: task resolver: bundles @@ -233,9 +250,6 @@ spec: operator: in values: - "true" - workspaces: - - name: source - workspace: workspace - name: build-image-index params: - name: IMAGE @@ -248,17 +262,17 @@ spec: value: $(params.build-image-index) - name: IMAGES value: - - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + - $(tasks.build-images.results.IMAGE_REF[*]) - name: BUILDAH_FORMAT value: $(params.buildah-format) runAfter: - - build-container + - build-images taskRef: params: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:60ca793e71ba7abdf024278d6ffe12f35f292ddca3031e3ef2a3f2711203448d + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:31197f4ee71be47c6f491e888ff266cbbb8ad5ed1c7c4141cc14f002d1802a50 - name: kind value: task resolver: bundles @@ -273,14 +287,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_URL) - name: BINARY_IMAGE_DIGEST value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: source-build + value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build:0.3@sha256:a681bd43f79482a8326288e304b33f25bdcb48d71f28ce2112569d89d9558348 + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:282cb5a9119a87e88559444feff67d76d6f356d03654b4845632c049b2314735 - name: kind value: task resolver: bundles @@ -293,9 +311,6 @@ spec: operator: in values: - "true" - workspaces: - - name: workspace - workspace: workspace - name: deprecated-base-image-check params: - name: IMAGE_URL @@ -318,7 +333,12 @@ spec: operator: in values: - "false" - - name: clair-scan + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -340,7 +360,12 @@ spec: operator: in values: - "false" - - name: ecosystem-cert-preflight-checks + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks params: - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) @@ -366,14 +391,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-snyk-check + value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check:0.4@sha256:25fefacfa0cbf453b8b7b66235f23c98f082f0deb45e064da3e622a4b2e3117f + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:60f2dac41844d222086ff7f477e51f3563716b183d87db89f603d6f604c21760 - name: kind value: task resolver: bundles @@ -382,10 +411,12 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - - name: clamav-scan + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) @@ -432,14 +463,18 @@ spec: - $(params.build-args[*]) - name: BUILD_ARGS_FILE value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - coverity-availability-check taskRef: params: - name: name - value: sast-coverity-check + value: sast-coverity-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check:0.3@sha256:9d572d7f7486224318d59de1b166efa68e59e17bac0785e2ecdcd014fe8e44d5 + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:ae62d14c999fd93246fef4e57d28570fa5200c3266b9a3263a39965e5a5b02d7 - name: kind value: task resolver: bundles @@ -452,9 +487,6 @@ spec: operator: in values: - success - workspaces: - - name: source - workspace: workspace - name: coverity-availability-check runAfter: - build-image-index @@ -478,14 +510,18 @@ spec: value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-shell-check + value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check:0.1@sha256:4364876ab45148ddfe729a29de69323e12cddd5a43e85d16f36ad4a88c3794f7 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:1f0fcba24ebc447d9f8a2ea2e8f262fa435d6c523ca6b0346cd67261551fc9ed - name: kind value: task resolver: bundles @@ -494,23 +530,24 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - name: sast-unicode-check params: - name: image-digest value: $(tasks.build-image-index.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: sast-unicode-check + value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check:0.3@sha256:d72abe93b4714b1b8faaea9fceb9acfaeca29c27eac8ff27ebc87622d75c871f + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:1833c618170ab9deb8455667f220df8e88d16ccd630a2361366f594e2bdcb712 - name: kind value: task resolver: bundles @@ -519,9 +556,6 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - workspace: workspace - name: apply-tags params: - name: IMAGE_URL @@ -549,20 +583,19 @@ spec: value: $(params.dockerfile) - name: CONTEXT value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) runAfter: - build-image-index taskRef: params: - name: name - value: push-dockerfile + value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile:0.1@sha256:74e982c07a808eaa5b1d8c126cafcbf3cc6ce94c883cf0845b55ce8064674b45 + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:13633d5ba8445c0f732a0a5d1b33ffbb708398e45ef1647542b0ab22fee25a6a - name: kind value: task resolver: bundles - workspaces: - - name: workspace - workspace: workspace - name: rpms-signature-scan params: - name: image-url @@ -576,7 +609,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:9c3a523814410046412abc09beb6f10082fecc56eb6d3386f6a6df305e5e011e + value: quay.io/konflux-ci/konflux-vanguard/task-rpms-signature-scan:0.2@sha256:78c8d7960c6db284356d94aaae64d1fca34fff4de6a6e20d897a088af0c81cf5 - name: kind value: task resolver: bundles @@ -586,26 +619,14 @@ spec: values: - "false" workspaces: - - name: workspace - name: git-auth optional: true - name: netrc optional: true taskRunTemplate: - serviceAccountName: build-pipeline-universal-image-py312-cuda128-torch280 + serviceAccountName: build-pipeline-odh-training-th03-cuda128-torch28-py312-rhel9 workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} - name: git-auth secret: secretName: '{{ git_auth_secret }}' -status: {} +status: {} \ No newline at end of file diff --git a/images/universal/training/py312-cuda128-torch280/Dockerfile b/images/universal/training/py312-cuda128-torch280/Dockerfile index ca8d30b49..7b97c4482 100644 --- a/images/universal/training/py312-cuda128-torch280/Dockerfile +++ b/images/universal/training/py312-cuda128-torch280/Dockerfile @@ -19,7 +19,8 @@ LABEL name="universal:py312-cuda128-torch280" \ io.k8s.display-name="Universal CUDA 12.8 Python 3.12 (Workbench + Runtime)" \ io.k8s.description="Universal image: Jupyter workbench by default; runtime when command provided. Includes RDMA/IB libs, Torch 2.8.0 cu128, FlashAttention 2.8.3." -COPY ./images/universal/training/py312-cuda128-torch280/LICENSE.md /licenses/cuda-license.md +## TODO: Add license file +# COPY LICENSE.md /licenses/cuda-license.md # For OS installs we need elevated privileges; base may default to 1001 USER 0 @@ -108,10 +109,10 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir numpy==2.3.3 # Install build backend for VCS package and the SDK itself (no build isolation so backend is visible) RUN pip install --retries 5 --timeout 300 --no-cache-dir hatchling hatch-vcs -RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation "git+https://github.com/briangallagher/sdk@training-hub" +RUN pip install --retries 5 --timeout 300 --no-cache-dir --no-build-isolation "git+https://github.com/briangallagher/sdk@add-training-hub" # Provide ninja via pip (RHEL/UBI repo ninja-build may be unavailable) -RUN pip install --retries 5 --timeout 300 --no-cache-dir ninja +# RUN pip install --retries 5 --timeout 300 --no-cache-dir ninja # Install remaining runtime packages (resolved from default PyPI), including FlashAttention # Note: We intentionally do not use a Pipfile/lock here to avoid mixing resolvers with the base (uv lock), @@ -119,26 +120,29 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir ninja RUN pip install --retries 5 --timeout 300 --no-cache-dir \ flash-attn==2.8.3 --no-build-isolation \ accelerate==1.10.0 \ - transformers==4.55.2 \ + transformers==4.57.1 \ peft==0.17.0 \ tqdm==4.67.1 \ datasets==4.0.0 \ pydantic>=2.11.7 \ aiofiles==24.1.0 \ + deprecated==1.2.18 \ + typer==0.19.2 \ "protobuf>=5.28.0,<6.0.0" \ "simpleeval>=0.9.13,<1.0" \ safetensors==0.6.2 \ packaging==25.0 \ pyyaml==6.0.2 \ py-cpuinfo==9.0.0 \ - numba==0.61.2 \ + numba==0.62.1 \ rich==14.1.0 \ tensorboard==2.19.0 \ - bitsandbytes>=0.45.3 \ - liger-kernel==0.5.10 \ + bitsandbytes==0.48.1 \ + liger-kernel==0.6.2 \ + einops==0.8.1 \ + kernels==0.10.3 \ "sentencepiece>=0.1.99,<0.3" \ - tokenizers==0.21.4 \ - training-hub==0.2.0 \ + tokenizers==0.22.1 \ trl==0.21.0 \ deepspeed>=0.14.3 \ async-timeout==4.0.3 \ @@ -147,13 +151,28 @@ RUN pip install --retries 5 --timeout 300 --no-cache-dir \ huggingface-hub==0.34.4 \ mlflow==3.4.0 \ psutil==7.0.0 \ + training-hub==0.3.0 \ + instructlab-training==0.12.1 \ + rhai-innovation-mini-trainer==0.3.0 \ && chmod -R g+w /opt/app-root/lib/python3.12/site-packages \ && fix-permissions /opt/app-root -P + +# Build helpers to compile PyTorch extensions +RUN pip install -U pip setuptools wheel ninja cmake + +# Optional: set GPU archs if you hit arch issues +# ENV TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0" + +# Deterministic 2-step: sub-dep first, then parent without deps +RUN pip install --no-build-isolation --no-cache-dir causal-conv1d==1.5.3.post1 && \ + pip install --no-build-isolation --no-cache-dir mamba-ssm==2.2.6.post3 --no-deps && \ + fix-permissions /opt/app-root -P + # Provide a POSIX entrypoint wrapper to choose behavior based on invocation -COPY --chmod=0755 ./images/universal/training/py312-cuda128-torch280/entrypoint-universal.sh /usr/local/bin/entrypoint-universal.sh +COPY --chmod=0755 ./entrypoint-universal.sh /usr/local/bin/entrypoint-universal.sh # Set ENTRYPOINT to the wrapper so that providing a command runs headless. # Default CMD maintains workbench behavior (no args → start-notebook.sh) ENTRYPOINT ["/usr/local/bin/entrypoint-universal.sh"] -CMD ["start-notebook.sh"] +CMD ["start-notebook.sh"] \ No newline at end of file