Skip to content

Commit 7fe84f5

Browse files
HumairAKHumairAK
authored
Merge pull request opendatahub-io#777 from HumairAK/runtime_img
add runtime image
2 parents 61dc233 + 823a012 commit 7fe84f5

File tree

36 files changed

+764
-221
lines changed

36 files changed

+764
-221
lines changed

‎.github/scripts/release_prep/create_branches.sh

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,21 @@ git clone \
1818
--branch=master \
1919
https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${DSP_REPOSITORY_FULL} \
2020
${DSP_DIR}
21-
cd ${DSP_DIR}
21+
pushd ${DSP_DIR}
2222
git checkout -B ${MINOR_RELEASE_BRANCH}
2323
git push origin ${MINOR_RELEASE_BRANCH}
2424
echo "::notice:: Created DSP ${MINOR_RELEASE_BRANCH} branch"
25+
popd
26+
27+
echo "Current branches in ${DSP_PIPELINES_REPOSITORY_FULL}"
28+
DSP_PIPELINES_DIR=$(dirname ${WORKING_DIR})/ilab-on-ocp
29+
git clone \
30+
--depth=1 \
31+
--branch=main \
32+
https://${GH_USER_NAME}:${GH_TOKEN}@github.com/${DSP_PIPELINES_REPOSITORY_FULL} \
33+
${DSP_PIPELINES_DIR}
34+
pushd ${DSP_PIPELINES_DIR}
35+
git checkout -B ${MINOR_RELEASE_BRANCH}
36+
git push origin ${MINOR_RELEASE_BRANCH}
37+
echo "::notice:: Created DSP Pipelines ${MINOR_RELEASE_BRANCH} branch"
38+
popd

‎.github/scripts/release_prep/prereqs.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ check_branch_exists(){
1414

1515
check_branch_exists ${DSPO_REPOSITORY_FULL} ${MINOR_RELEASE_BRANCH}
1616
check_branch_exists ${DSP_REPOSITORY_FULL} ${MINOR_RELEASE_BRANCH}
17+
check_branch_exists ${DSP_PIPELINES_REPOSITORY_FULL} ${MINOR_RELEASE_BRANCH}
1718

1819
echo "Ensure compatibility.yaml is upto date, and generate a new compatibility.md. Use [release-tools] to accomplish this"
1920

@@ -27,7 +28,8 @@ git checkout -B ${BRANCH_NAME}
2728
echo "Created branch: ${BRANCH_NAME}"
2829
echo "Checking if compatibility.yaml contains ${TARGET_RELEASE} release...."
2930

30-
contains_rel=$(cat docs/release/compatibility.yaml | rel=${MINOR_RELEASE_WILDCARD} yq '[.[].dsp] | contains([env(rel)])')
31+
# convert rel to string in env(rel) explicitly to avoid comparing str to yq float
32+
contains_rel=$(cat docs/release/compatibility.yaml | rel=${MINOR_RELEASE_WILDCARD} yq '[.[].dsp] | contains([""+env(rel)])')
3133

3234
if [[ "$contains_rel" == "false" ]]; then
3335

‎.github/workflows/build-tags.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ on:
1313
default: 'v1.0.x'
1414
description: 'Source branch to build DSP from'
1515
required: true
16+
src_pipelines_branch:
17+
type: string
18+
default: 'v1.0.x'
19+
description: 'Source branch to build DSP Pipelines Generic Image from'
20+
required: true
1621
target_tag:
1722
type: string
1823
default: 'vx.y.z'
@@ -28,6 +33,11 @@ on:
2833
default: 'opendatahub-io/data-science-pipelines'
2934
description: 'DSP org/repo'
3035
required: true
36+
pipelines_org_repo:
37+
type: string
38+
default: 'opendatahub-io/ilab-on-ocp'
39+
description: 'DSP Pipelines org/repo'
40+
required: true
3141
overwrite_imgs:
3242
type: string
3343
default: 'true'
@@ -45,6 +55,11 @@ on:
4555
default: 'v1.0.x'
4656
description: 'Source branch to build DSP from'
4757
required: true
58+
src_pipelines_branch:
59+
type: string
60+
default: 'v1.0.x'
61+
description: 'Source branch to build DSP Pipelines Generic Image from'
62+
required: true
4863
target_tag:
4964
default: 'vx.y.z'
5065
description: 'Target Image Tag'
@@ -57,6 +72,11 @@ on:
5772
default: 'opendatahub-io/data-science-pipelines'
5873
description: 'DSP org/repo'
5974
required: true
75+
pipelines_org_repo:
76+
type: string
77+
default: 'opendatahub-io/ilab-on-ocp'
78+
description: 'DSP Pipelines org/repo'
79+
required: true
6080
overwrite_imgs:
6181
type: string
6282
default: 'false'
@@ -70,8 +90,10 @@ env:
7090
IMAGE_REPO_SWF: ds-pipelines-scheduledworkflow
7191
IMAGE_REPO_LAUNCHER: ds-pipelines-launcher
7292
IMAGE_REPO_DRIVER: ds-pipelines-driver
93+
IMAGE_REPO_PIPELINES_RUNTIME_GENERIC: ds-pipelines-runtime-generic
7394
SOURCE_DSPO_BRANCH: ${{ inputs.src_dspo_branch }}
7495
SOURCE_DSP_BRANCH: ${{ inputs.src_dsp_branch }}
96+
SOURCE_DSP_PIPELINES_BRANCH: ${{ inputs.src_pipelines_branch }}
7597
QUAY_ORG: ${{ inputs.quay_org }}
7698
QUAY_ID: ${{ secrets.QUAY_ID }}
7799
QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
@@ -189,3 +211,19 @@ jobs:
189211
DOCKERFILE: backend/Dockerfile.launcher
190212
GH_REPO: ${{ inputs.dsp_org_repo }}
191213
OVERWRITE: ${{ env.OVERWRITE_IMAGES }}
214+
215+
RUNTIME-GENERIC-build:
216+
runs-on: ubuntu-latest
217+
permissions:
218+
contents: read
219+
steps:
220+
- uses: actions/checkout@v3
221+
- uses: ./.github/actions/build
222+
name: Build Image
223+
env:
224+
SOURCE_BRANCH: ${{ env.SOURCE_DSP_PIPELINES_BRANCH }}
225+
with:
226+
IMAGE_REPO: ${{ env.IMAGE_REPO_PIPELINES_RUNTIME_GENERIC }}
227+
DOCKERFILE: Dockerfile
228+
GH_REPO: ${{ inputs.pipelines_org_repo }}
229+
OVERWRITE: ${{ env.OVERWRITE_IMAGES }}

‎.github/workflows/release_prep.yaml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ env:
4141
DSPO_REPOSITORY_FULL: ${{ inputs.gh_org }}/data-science-pipelines-operator
4242
DSP_REPOSITORY: data-science-pipelines
4343
DSP_REPOSITORY_FULL: ${{ inputs.gh_org }}/data-science-pipelines
44+
DSP_PIPELINES_REPOSITORY_FULL: ${{ inputs.gh_org }}/ilab-on-ocp
4445
PREVIOUS_RELEASE_TAG: ${{ inputs.previous_release_tag }}
4546
OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }}
4647
CONFIG_TEMPLATE: "./.github/scripts/release_prep/templates/config.yaml"
@@ -66,6 +67,7 @@ jobs:
6667
MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }}
6768
DSPO_REPOSITORY_FULL: ${{ env.DSPO_REPOSITORY_FULL }}
6869
DSPO_REPOSITORY: ${{ env.DSPO_REPOSITORY }}
70+
DSP_PIPELINES_REPOSITORY_FULL: ${{ env.DSP_PIPELINES_REPOSITORY_FULL }}
6971
run: ./.github/scripts/release_prep/prereqs.sh
7072

7173
create_branches:
@@ -86,6 +88,7 @@ jobs:
8688
MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }}
8789
DSPO_REPOSITORY_FULL: ${{ env.DSPO_REPOSITORY_FULL }}
8890
DSP_REPOSITORY_FULL: ${{ env.DSP_REPOSITORY_FULL }}
91+
DSP_PIPELINES_REPOSITORY_FULL: ${{ env.DSP_PIPELINES_REPOSITORY_FULL }}
8992
WORKING_DIR: ${{ github.workspace }}
9093
run: ./.github/scripts/release_prep/create_branches.sh
9194

@@ -100,7 +103,8 @@ jobs:
100103
MINOR_RELEASE_BRANCH: ${{ env.MINOR_RELEASE_BRANCH }}
101104
MINOR_RELEASE_TAG: ${{ env.MINOR_RELEASE_TAG }}
102105
QUAY_ORG: ${{ env.QUAY_ORG }}
103-
DSPO_GH_ORG: ${{ env.DSP_REPOSITORY_FULL }}
106+
DSP_GH_ORG_REPO: ${{ env.DSP_REPOSITORY_FULL }}
107+
DSP_PIPELINES_GH_ORG_REPO: ${{ env.DSP_PIPELINES_REPOSITORY_FULL }}
104108
OVERWRITE_IMAGES: ${{ env.OVERWRITE_IMAGES }}
105109
steps:
106110
- run: echo "Storing env vars for re-usable workflow."
@@ -113,9 +117,11 @@ jobs:
113117
with:
114118
src_dspo_branch: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_BRANCH }}
115119
src_dsp_branch: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_BRANCH }}
120+
src_pipelines_branch: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_BRANCH }}
116121
target_tag: ${{ needs.get-env-vars.outputs.MINOR_RELEASE_TAG }}
117122
quay_org: ${{ needs.get-env-vars.outputs.QUAY_ORG }}
118-
dsp_org_repo: ${{ needs.get-env-vars.outputs.DSPO_GH_ORG }}
123+
dsp_org_repo: ${{ needs.get-env-vars.outputs.DSP_GH_ORG_REPO }}
124+
pipelines_org_repo: ${{ needs.get-env-vars.outputs.DSP_PIPELINES_GH_ORG_REPO }}
119125
overwrite_imgs: ${{ needs.get-env-vars.outputs.OVERWRITE_IMAGES }}
120126
secrets: inherit
121127

‎api/v1/dspipeline_types.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,24 @@ type APIServer struct {
6969
// Include sample pipelines with the deployment of this DSP API Server. Default: true
7070
// +kubebuilder:default:=false
7171
// +kubebuilder:validation:Optional
72-
EnableSamplePipeline bool `json:"enableSamplePipeline"`
73-
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
74-
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
72+
EnableSamplePipeline bool `json:"enableSamplePipeline"`
73+
// Launcher/Executor image used during pipeline execution.
74+
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
75+
// Driver image used during pipeline execution.
76+
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
77+
// Generic runtime image used for building managed pipelines during
78+
// api server init, and for basic runtime operations.
79+
RuntimeGenericImage string `json:"runtimeGenericImage,omitempty"`
80+
// Toolbox image used for basic container spec runtime operations
81+
// in managed pipelines.
82+
ToolboxImage string `json:"toolboxImage,omitempty"`
83+
// RhelAI image used for ilab tasks in managed pipelines.
84+
RHELAIImage string `json:"rhelAIImage,omitempty"`
7585
// Specify custom Pod resource requirements for this component.
7686
Resources *ResourceRequirements `json:"resources,omitempty"`
87+
// Specify init container resource requirements. The init container
88+
// is used to build managed-pipelines and store them in a shared volume.
89+
InitResources *ResourceRequirements `json:"initResources,omitempty"`
7790

7891
// If the Object store/DB is behind a TLS secured connection that is
7992
// unrecognized by the host OpenShift/K8s cluster, then you can

‎api/v1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1alpha1/dspipeline_types.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,24 @@ type APIServer struct {
6969
// Include sample pipelines with the deployment of this DSP API Server. Default: true
7070
// +kubebuilder:default:=false
7171
// +kubebuilder:validation:Optional
72-
EnableSamplePipeline bool `json:"enableSamplePipeline"`
73-
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
74-
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
72+
EnableSamplePipeline bool `json:"enableSamplePipeline"`
73+
// Launcher/Executor image used during pipeline execution.
74+
ArgoLauncherImage string `json:"argoLauncherImage,omitempty"`
75+
// Driver image used during pipeline execution.
76+
ArgoDriverImage string `json:"argoDriverImage,omitempty"`
77+
// Generic runtime image used for building managed pipelines during
78+
// api server init, and for basic runtime operations.
79+
RuntimeGenericImage string `json:"runtimeGenericImage,omitempty"`
80+
// Toolbox image used for basic container spec runtime operations
81+
// in managed pipelines.
82+
ToolboxImage string `json:"toolboxImage,omitempty"`
83+
// RhelAI image used for ilab tasks in managed pipelines.
84+
RHELAIImage string `json:"rhelAIImage,omitempty"`
7585
// Specify custom Pod resource requirements for this component.
7686
Resources *ResourceRequirements `json:"resources,omitempty"`
87+
// Specify init container resource requirements. The init container
88+
// is used to build managed-pipelines and store them in a shared volume.
89+
InitResources *ResourceRequirements `json:"initResources,omitempty"`
7790

7891
// If the Object store/DB is behind a TLS secured connection that is
7992
// unrecognized by the host OpenShift/K8s cluster, then you can

‎api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/base/kustomization.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,27 @@ vars:
102102
apiVersion: v1
103103
fieldref:
104104
fieldpath: data.IMAGES_ARGO_WORKFLOWCONTROLLER
105+
- name: IMAGES_PIPELINESRUNTIMEGENERIC
106+
objref:
107+
kind: ConfigMap
108+
name: dspo-parameters
109+
apiVersion: v1
110+
fieldref:
111+
fieldpath: data.IMAGES_PIPELINESRUNTIMEGENERIC
112+
- name: IMAGES_TOOLBOX
113+
objref:
114+
kind: ConfigMap
115+
name: dspo-parameters
116+
apiVersion: v1
117+
fieldref:
118+
fieldpath: data.IMAGES_TOOLBOX
119+
- name: IMAGES_RHELAI
120+
objref:
121+
kind: ConfigMap
122+
name: dspo-parameters
123+
apiVersion: v1
124+
fieldref:
125+
fieldpath: data.IMAGES_RHELAI
105126

106127
# DSPO level configs
107128
- name: ZAP_LOG_LEVEL

‎config/base/params.env

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ IMAGES_ARGO_WORKFLOWCONTROLLER=quay.io/opendatahub/ds-pipelines-argo-workflowcon
77
IMAGES_LAUNCHER=quay.io/opendatahub/ds-pipelines-launcher:latest
88
IMAGES_DRIVER=quay.io/opendatahub/ds-pipelines-driver:latest
99
IMAGES_MLMDGRPC=quay.io/opendatahub/mlmd-grpc-server:latest
10+
IMAGES_PIPELINESRUNTIMEGENERIC=quay.io/opendatahub/ds-pipelines-runtime-generic:latest
11+
IMAGES_TOOLBOX=registry.redhat.io/ubi9/toolbox@sha256:da31dee8904a535d12689346e65e5b00d11a6179abf1fa69b548dbd755fa2770
12+
IMAGES_RHELAI=registry.redhat.io/rhelai1/instructlab-nvidia-rhel9@sha256:05cfba1fb13ed54b1de4d021da2a31dd78ba7d8cc48e10c7fe372815899a18ae
1013
IMAGES_MLMDENVOY=registry.redhat.io/openshift-service-mesh/proxyv2-rhel8@sha256:b30d60cd458133430d4c92bf84911e03cecd02f60e88a58d1c6c003543cf833a
1114
IMAGES_MARIADB=registry.redhat.io/rhel8/mariadb-103@sha256:f0ee0d27bb784e289f7d88cc8ee0e085ca70e88a5d126562105542f259a1ac01
1215
IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:8ce44de8c683f198bf24ba36cd17e89708153d11f5b42c0a27e77f8fdb233551

0 commit comments

Comments
 (0)