Skip to content

Commit 23041ca

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request opendatahub-io#930 from atheo89/runtimes-imagestreams
Include runtimes ImageStreams on manifests + gha automation
2 parents 5d8d9a7 + 61b9656 commit 23041ca

9 files changed

+404
-0
lines changed

‎.github/workflows/runtimes-digest-updater-upstream-v2.yaml

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
---
2+
# The aim of this GitHub workflow is to update the runtimes ImageStreams
3+
name: Update runtime ImageStreams SHA digests
4+
on: # yamllint disable-line rule:truthy
5+
workflow_dispatch:
6+
inputs:
7+
branch:
8+
required: true
9+
description: "Which branch do you want to update?"
10+
tag_version:
11+
required: true
12+
description: "Provide tag version: main or YYYYx"
13+
user-hash:
14+
required: false
15+
description: "Optional: Specify a Git hash (it should exist in the provided branch's history)"
16+
17+
env:
18+
TMP_BRANCH: tmp-branch-${{ github.run_id }}
19+
BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
20+
TAG_VERSION: ${{ github.event.inputs.tag_version || 'main' }}
21+
USER_HASH: ${{ github.event.inputs.user-hash }}
22+
23+
jobs:
24+
initialize:
25+
runs-on: ubuntu-latest
26+
permissions:
27+
contents: write
28+
steps:
29+
- name: Install Skopeo CLI
30+
shell: bash
31+
run: |
32+
sudo apt-get -y update
33+
sudo apt-get -y install skopeo yq jq
34+
35+
- name: Checkout branch
36+
uses: actions/checkout@v4
37+
with:
38+
ref: ${{ env.BRANCH_NAME }}
39+
40+
- name: Create a new branch
41+
run: |
42+
echo ${{ env.TMP_BRANCH }}
43+
git checkout -b ${{ env.TMP_BRANCH }}
44+
git push --set-upstream origin ${{ env.TMP_BRANCH }}
45+
46+
update-runtimes:
47+
needs: [initialize]
48+
runs-on: ubuntu-latest
49+
permissions:
50+
contents: write
51+
steps:
52+
- name: Configure Git
53+
run: |
54+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
55+
git config --global user.name "GitHub Actions"
56+
57+
- name: Checkout release branch
58+
uses: actions/checkout@v4
59+
with:
60+
ref: ${{ env.TMP_BRANCH }}
61+
fetch-depth: 0
62+
63+
- name: Invoke script to handle the update
64+
shell: bash
65+
run: |
66+
bash ${GITHUB_WORKSPACE}/ci/runtimes-digest-updater.sh ${{ env.TAG_VERSION }} ${{ env.USER_HASH }}
67+
68+
- name: Commit the changes
69+
run: |
70+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
71+
git fetch origin "${{ env.TMP_BRANCH }}" && \
72+
git pull origin "${{ env.TMP_BRANCH }}" && \
73+
git add "manifests/base/runtime-*" && \
74+
git commit -m "Update file via ${{ env.TMP_BRANCH }} GitHub action" && \
75+
git push origin "${{ env.TMP_BRANCH }}"
76+
else
77+
echo "There were no changes detected in the images for the ${{ env.BRANCH_NAME }}"
78+
fi
79+
80+
open-pull-request:
81+
needs: [update-runtimes]
82+
runs-on: ubuntu-latest
83+
permissions:
84+
pull-requests: write
85+
steps:
86+
- name: Checkout repo
87+
uses: actions/checkout@v4
88+
89+
- name: pull-request
90+
uses: repo-sync/pull-request@v2
91+
with:
92+
source_branch: ${{ env.TMP_BRANCH }}
93+
destination_branch: ${{ env.BRANCH_NAME }}
94+
github_token: ${{ secrets.GITHUB_TOKEN }}
95+
pr_label: "automated pr"
96+
pr_title: "[Digest Updater Action] Update Runtimes ImageStreams"
97+
pr_body: |
98+
:rocket: This is an automated Pull Request.
99+
Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml`
100+
101+
:exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.TMP_BRANCH }}` branch after merging the changes

‎ci/runtimes-digest-updater.sh

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
3+
TAG_VERSION=$1
4+
USER_HASH=$2
5+
6+
REPO_OWNER="opendatahub-io"
7+
REPO_NAME="notebooks"
8+
GITHUB_API_URL="https://api.github.com/repos/$REPO_OWNER/$REPO_NAME"
9+
10+
if [[ -n "$USER_HASH" ]]; then
11+
HASH=$USER_HASH
12+
echo "Using user-provided HASH: $HASH"
13+
else
14+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' "$GITHUB_API_URL/commits?sha=$TAG_VERSION&per_page=1")
15+
HASH=$(echo "$PAYLOAD" | jq -r '.[0].sha' | cut -c1-7)
16+
echo "Extracted HASH: $HASH"
17+
fi
18+
19+
REPO_ROOT=$(git rev-parse --show-toplevel)
20+
MANIFEST_DIR="$REPO_ROOT/manifests/base"
21+
# Find matching files
22+
files=$(find "$MANIFEST_DIR" -type f -name "runtime-*.yaml")
23+
for file in $files; do
24+
echo "PROCESSING: $file"
25+
26+
# Extract values
27+
img=$(yq e '.spec.tags[].annotations."opendatahub.io/runtime-image-metadata" | fromjson | .[].metadata.image_name' "$file" 2>/dev/null)
28+
name=$(yq e '.spec.tags[].name' "$file" 2>/dev/null)
29+
ubi=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'UBI[0-9]+' | tr '[:upper:]' '[:lower:]')
30+
py_version=$(yq e '.metadata.annotations."opendatahub.io/runtime-image-name"' "$file" 2>/dev/null | grep -oE 'Python [0-9]+\.[0-9]+' | sed 's/ /-/g' | tr '[:upper:]' '[:lower:]')
31+
registry=$(echo "$img" | cut -d '@' -f1)
32+
33+
# Handling specific cases
34+
[[ $name == tensorflow* ]] && name="cuda-$name"
35+
36+
if [[ $TAG_VERSION == main ]]; then
37+
# This should match with the runtime-image tag name as is on quay.io registry
38+
regex="^runtime-$name-$ubi-$py_version-[0-9]{8}-$HASH$"
39+
else
40+
# This should match with the runtime-image tag name as is on quay.io registry
41+
regex="^runtime-$name-$ubi-$py_version-$TAG_VERSION-[0-9]{8}-$HASH$"
42+
fi
43+
44+
latest_tag=$(skopeo inspect --retry-times 3 "docker://$img" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
45+
echo "CHECKING: ${latest_tag}"
46+
47+
if [[ -z "$latest_tag" || "$latest_tag" == "null" ]]; then
48+
echo "No matching tag found on registry for $file. Skipping."
49+
continue
50+
fi
51+
52+
# Extract the digest sha from the latest tag
53+
digest=$(skopeo inspect --retry-times 3 "docker://$registry:$latest_tag" | jq .Digest | tr -d '"')
54+
output="${registry}@${digest}"
55+
echo "NEW: ${output}"
56+
57+
# Updates the ImageStream with the new SHAs
58+
yq e -i '(.spec.tags[] | .from.name) = "'"$output"'"' "$file"
59+
sed -i "s|\(\"image_name\": \"\)[^\"]*|\1${output}|" "$file"
60+
61+
done

‎manifests/base/kustomization.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ resources:
1414
- jupyter-rocm-minimal-notebook-imagestream.yaml
1515
- jupyter-rocm-pytorch-notebook-imagestream.yaml
1616
- jupyter-rocm-tensorflow-notebook-imagestream.yaml
17+
- runtime-datascience-imagestream.yaml
18+
- runtime-minimal-imagestream.yaml
19+
- runtime-pytorch-imagestream.yaml
20+
- runtime-rocm-pytorch-imagestream.yaml
21+
- runtime-rocm-tensorflow-imagestream.yaml
22+
- runtime-tensorflow-imagestream.yaml
1723

1824
commonLabels:
1925
opendatahub.io/component: "true"

‎manifests/base/runtime-datascience-imagestream.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
apiVersion: image.openshift.io/v1
3+
kind: ImageStream
4+
metadata:
5+
labels:
6+
opendatahub.io/runtime-image: "true"
7+
annotations:
8+
opendatahub.io/runtime-image-url: "https://github.com//opendatahub-io/notebooks/tree/main/runtimes"
9+
opendatahub.io/runtime-image-name: "Datascience with Python 3.11 (UBI9)"
10+
opendatahub.io/runtime-image-desc: "Datascience runtime image for Elyra, enabling pipeline execution from Workbenches with a set of advanced AI/ML data science libraries, supporting different runtimes for various pipeline nodes."
11+
name: runtime-datascience
12+
spec:
13+
lookupPolicy:
14+
local: true
15+
tags:
16+
- annotations:
17+
# language=json
18+
opendatahub.io/runtime-image-metadata: |
19+
[
20+
{
21+
"display_name": "Datascience with Python 3.11 (UBI9)",
22+
"metadata": {
23+
"tags": [
24+
"datascience"
25+
],
26+
"display_name": "Datascience with Python 3.11 (UBI9)",
27+
"image_name": "quay.io/opendatahub/workbench-images@sha256:304d3b2ea846832f27312ef6776064a1bf3797c645b6fea0b292a7ef6416458e",
28+
"pull_policy": "IfNotPresent"
29+
},
30+
"schema_name": "runtime-image"
31+
}
32+
]
33+
openshift.io/imported-from: quay.io/opendatahub/workbench-images
34+
from:
35+
kind: DockerImage
36+
name: quay.io/opendatahub/workbench-images@sha256:304d3b2ea846832f27312ef6776064a1bf3797c645b6fea0b292a7ef6416458e
37+
name: "datascience"
38+
referencePolicy:
39+
type: Source

‎manifests/base/runtime-minimal-imagestream.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
apiVersion: image.openshift.io/v1
3+
kind: ImageStream
4+
metadata:
5+
labels:
6+
opendatahub.io/runtime-image: "true"
7+
annotations:
8+
opendatahub.io/runtime-image-url: "https://github.com//opendatahub-io/notebooks/tree/main/runtimes"
9+
opendatahub.io/runtime-image-name: "Python 3.11 (UBI9)"
10+
opendatahub.io/runtime-image-desc: "Minimal runtime image for Elyra, enabling pipeline execution from Workbenches with minimal dependency set to start experimenting with, for various pipeline nodes."
11+
name: runtime-minimal
12+
spec:
13+
lookupPolicy:
14+
local: true
15+
tags:
16+
- annotations:
17+
# language=json
18+
opendatahub.io/runtime-image-metadata: |
19+
[
20+
{
21+
"display_name": "Python 3.11 (UBI9)",
22+
"metadata": {
23+
"tags": [
24+
"minimal"
25+
],
26+
"display_name": "Python 3.11 (UBI9)",
27+
"image_name": "quay.io/opendatahub/workbench-images@sha256:a2b1bf59f25fd0694394ad927e5eba93e32df9c2c11d8b54412564a9fc736ab8",
28+
"pull_policy": "IfNotPresent"
29+
},
30+
"schema_name": "runtime-image"
31+
}
32+
]
33+
openshift.io/imported-from: quay.io/opendatahub/workbench-images
34+
from:
35+
kind: DockerImage
36+
name: quay.io/opendatahub/workbench-images@sha256:a2b1bf59f25fd0694394ad927e5eba93e32df9c2c11d8b54412564a9fc736ab8
37+
name: "minimal"
38+
referencePolicy:
39+
type: Source

‎manifests/base/runtime-pytorch-imagestream.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
apiVersion: image.openshift.io/v1
3+
kind: ImageStream
4+
metadata:
5+
labels:
6+
opendatahub.io/runtime-image: "true"
7+
annotations:
8+
opendatahub.io/runtime-image-url: "https://github.com//opendatahub-io/notebooks/tree/main/runtimes"
9+
opendatahub.io/runtime-image-name: "PyTorch with CUDA and Python 3.11 (UBI9)"
10+
opendatahub.io/runtime-image-desc: "PyTorch runtime image for Elyra, enabling pipeline execution from Workbenches with PyTorch libraries and dependencies, supporting different runtimes for various pipeline nodes."
11+
name: runtime-pytorch
12+
spec:
13+
lookupPolicy:
14+
local: true
15+
tags:
16+
- annotations:
17+
# language=json
18+
opendatahub.io/runtime-image-metadata: |
19+
[
20+
{
21+
"display_name": "PyTorch with CUDA and Python 3.11 (UBI9)",
22+
"metadata": {
23+
"tags": [
24+
"pytorch"
25+
],
26+
"display_name": "PyTorch with CUDA and Python 3.11 (UBI9)",
27+
"image_name": "quay.io/opendatahub/workbench-images@sha256:6a2806bf3cd9b00f60f3c7fe907727fb954c9776a075d9d58df26b5119d7afe6",
28+
"pull_policy": "IfNotPresent"
29+
},
30+
"schema_name": "runtime-image"
31+
}
32+
]
33+
openshift.io/imported-from: quay.io/opendatahub/workbench-images
34+
from:
35+
kind: DockerImage
36+
name: quay.io/opendatahub/workbench-images@sha256:6a2806bf3cd9b00f60f3c7fe907727fb954c9776a075d9d58df26b5119d7afe6
37+
name: "pytorch"
38+
referencePolicy:
39+
type: Source

‎manifests/base/runtime-rocm-pytorch-imagestream.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
---
2+
apiVersion: image.openshift.io/v1
3+
kind: ImageStream
4+
metadata:
5+
labels:
6+
opendatahub.io/runtime-image: "true"
7+
annotations:
8+
opendatahub.io/runtime-image-url: "https://github.com//opendatahub-io/notebooks/tree/main/runtimes"
9+
opendatahub.io/runtime-image-name: "PyTorch with ROCm and Python 3.11 (UBI9)"
10+
opendatahub.io/runtime-image-desc: "ROCm optimized PyTorch runtime image for Elyra, enabling pipeline execution from Workbenches with PyTorch libraries and dependencies, supporting different runtimes for various pipeline nodes."
11+
name: runtime-rocm-pytorch
12+
spec:
13+
lookupPolicy:
14+
local: true
15+
tags:
16+
- annotations:
17+
# language=json
18+
opendatahub.io/runtime-image-metadata: |
19+
[
20+
{
21+
"display_name": "PyTorch with ROCm and Python 3.11 (UBI9)",
22+
"metadata": {
23+
"tags": [
24+
"rocm-pytorch"
25+
],
26+
"display_name": "PyTorch with ROCm and Python 3.11 (UBI9)",
27+
"image_name": "quay.io/opendatahub/workbench-images@sha256:c6d74d73b835d0f97040ca99bb25065a949b817b52179f239a5361c3299352ba",
28+
"pull_policy": "IfNotPresent"
29+
},
30+
"schema_name": "runtime-image"
31+
}
32+
]
33+
openshift.io/imported-from: quay.io/opendatahub/workbench-images
34+
from:
35+
kind: DockerImage
36+
name: quay.io/opendatahub/workbench-images@sha256:c6d74d73b835d0f97040ca99bb25065a949b817b52179f239a5361c3299352ba
37+
name: "rocm-pytorch"
38+
referencePolicy:
39+
type: Source

‎manifests/base/runtime-rocm-tensorflow-imagestream.yaml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
apiVersion: image.openshift.io/v1
3+
kind: ImageStream
4+
metadata:
5+
labels:
6+
opendatahub.io/runtime-image: "true"
7+
annotations:
8+
# TODO: once the restraction takes a final shape need to update that url
9+
opendatahub.io/runtime-image-url: "https://github.com//opendatahub-io/notebooks/tree/main/runtimes"
10+
opendatahub.io/runtime-image-name: "TensorFlow with ROCm and Python 3.11 (UBI9)"
11+
opendatahub.io/runtime-image-desc: "ROCm optimized TensorFlow runtime image for Elyra, enabling pipeline execution from Workbenches with TensorFlow libraries and dependencies, supporting different runtimes for various pipeline nodes."
12+
name: runtime-rocm-tensorflow
13+
spec:
14+
lookupPolicy:
15+
local: true
16+
tags:
17+
- annotations:
18+
# language=json
19+
opendatahub.io/runtime-image-metadata: |
20+
[
21+
{
22+
"display_name": "TensorFlow with ROCm and Python 3.11 (UBI9)",
23+
"metadata": {
24+
"tags": [
25+
"rocm-tensorflow"
26+
],
27+
"display_name": "TensorFlow with ROCm and Python 3.11 (UBI9)",
28+
"image_name": "quay.io/opendatahub/workbench-images@sha256:6a2da12a8bdc9cfcda27f4189827be5fbde4b4b4c4f7d92ac694d9360e3562dc",
29+
"pull_policy": "IfNotPresent"
30+
},
31+
"schema_name": "runtime-image"
32+
}
33+
]
34+
openshift.io/imported-from: quay.io/opendatahub/workbench-images
35+
from:
36+
kind: DockerImage
37+
name: quay.io/opendatahub/workbench-images@sha256:6a2da12a8bdc9cfcda27f4189827be5fbde4b4b4c4f7d92ac694d9360e3562dc
38+
name: "rocm-tensorflow"
39+
referencePolicy:
40+
type: Source

0 commit comments

Comments
 (0)