Skip to content

Commit dca18d1

Browse files
harshad16harshad16
authored
Merge pull request #594 from opendatahub-io/sync__main__2024a__20240627194556
Sync `2024a` branch with `main` branch
2 parents 53d6383 + 9f0a837 commit dca18d1

33 files changed

+9643
-186
lines changed

‎.github/workflows/build-notebooks-TEMPLATE.yaml

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,38 @@ jobs:
3434
username: ${{ github.actor }}
3535
password: ${{ secrets.GITHUB_TOKEN }}
3636

37+
- name: Free up additional disk space
38+
# https://docs.github.com/en/actions/learn-github-actions/expressions
39+
if: "${{ contains(inputs.target, 'amd') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
40+
contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
41+
run: |
42+
set -x
43+
44+
df -h
45+
46+
sudo rm -rf /usr/local/lib/android &
47+
sudo rm -rf /usr/local/share/boost &
48+
sudo rm -rf /usr/local/lib/node_modules &
49+
sudo rm -rf /usr/share/dotnet &
50+
sudo rm -rf /opt/ghc &
51+
sudo rm -rf /opt/hostedtoolcache/CodeQL &
52+
53+
sudo docker image prune --all --force &
54+
55+
wait
56+
57+
df -h
58+
59+
- name: Mount lvm overlay for podman builds
60+
run: |
61+
df -h
62+
free -h
63+
64+
bash ./ci/cached-builds/gha_lvm_overlay.bash
65+
66+
df -h
67+
free -h
68+
3769
# https://github.com/containers/buildah/issues/2521#issuecomment-884779112
3870
- name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
3971
run: sudo apt-get -qq remove podman crun
@@ -58,12 +90,10 @@ jobs:
5890
mkdir -p $HOME/.config/containers/
5991
cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
6092
cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
93+
6194
# should at least reset storage when touching storage.conf
62-
sudo mkdir -p /mnt/containers/
63-
sudo chown -R $USER:$USER /mnt/containers
6495
podman system reset --force
65-
# podman bug? need to create this _after_ doing the reset
66-
mkdir -p /mnt/containers/tmp
96+
mkdir -p $HOME/.local/share/containers/storage/tmp
6797
6898
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
6999
- name: "push: make ${{ inputs.target }}"

‎.github/workflows/build-notebooks-pr.yaml

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,40 @@
11
---
22
"name": "Build Notebooks"
3-
"permissions":
4-
"packages": "read"
53
"on":
64
"pull_request":
75

6+
permissions:
7+
contents: read
8+
packages: read
9+
pull-requests: read
10+
811
jobs:
912
gen:
1013
name: Generate job matrix
1114
runs-on: ubuntu-latest
1215
outputs:
1316
matrix: ${{ steps.gen.outputs.matrix }}
17+
has_jobs: ${{ steps.gen.outputs.has_jobs }}
1418
steps:
1519
- uses: actions/checkout@v4
16-
- run: python3 ci/cached-builds/gen_gha_matrix_jobs.py
20+
21+
- run: |
22+
python3 ci/cached-builds/gen_gha_matrix_jobs.py \
23+
--owner=${{ github.repository_owner }} \
24+
--repo=${{ github.event.pull_request.base.repo.name }} \
25+
--pr-number=${{ github.event.pull_request.number }} \
26+
--skip-unchanged
1727
id: gen
28+
env:
29+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1830
19-
# base images
2031
build:
2132
needs: ["gen"]
2233
strategy:
2334
fail-fast: false
2435
matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
2536
uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
37+
if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
2638
with:
2739
target: "${{ matrix.target }}"
2840
github: "${{ toJSON(github) }}"

‎.github/workflows/notebooks-digest-updater-upstream.yaml

Lines changed: 91 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -67,57 +67,61 @@ jobs:
6767

6868
- name: Update the param.env file
6969
run: |
70-
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
71-
IMAGES=("odh-minimal-notebook-image-n"
72-
"odh-minimal-gpu-notebook-image-n"
73-
"odh-pytorch-gpu-notebook-image-n"
74-
"odh-generic-data-science-notebook-image-n"
75-
"odh-tensorflow-gpu-notebook-image-n"
76-
"odh-trustyai-notebook-image-n"
77-
"odh-codeserver-notebook-image-n"
78-
"odh-rstudio-notebook-image-n"
79-
"odh-rstudio-gpu-notebook-image-n")
80-
81-
for ((i=0;i<${#IMAGES[@]};++i)); do
82-
image=${IMAGES[$i]}
83-
echo "CHECKING: " $image
84-
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
85-
registry=$(echo $img | cut -d '@' -f1)
86-
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
87-
regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
88-
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
89-
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
90-
output=$registry@$digest
91-
echo "NEW:" $output
92-
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
70+
PARAMS_ENV_PATH="manifests/base/params.env"
71+
72+
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
73+
74+
# Get the complete list of images N-version to update
75+
IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n=" | cut -d "=" -f 1)
76+
77+
for image in ${IMAGES}; do
78+
echo "CHECKING: '${image}'"
79+
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
80+
registry=$(echo "${img}" | cut -d '@' -f1)
81+
82+
skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
83+
84+
src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
85+
regex="^$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
86+
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
87+
# use `--no-tags` for skopeo once available in newer version
88+
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
89+
output="${registry}@${digest}"
90+
echo "NEW: ${output}"
91+
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
9392
done
93+
9494
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
95-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
95+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
96+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
97+
git add "${PARAMS_ENV_PATH}" && \
98+
git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
99+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
96100
else
97-
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
101+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
98102
fi
99103
100104
- name: Update the commit.env file
101105
run: |
102-
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
103-
COMMIT=("odh-minimal-notebook-image-commit-n"
104-
"odh-minimal-gpu-notebook-image-commit-n"
105-
"odh-pytorch-gpu-notebook-image-commit-n"
106-
"odh-generic-data-science-notebook-image-commit-n"
107-
"odh-tensorflow-gpu-notebook-image-commit-n"
108-
"odh-trustyai-notebook-image-commit-n"
109-
"odh-codeserver-notebook-image-commit-n"
110-
"odh-rstudio-notebook-image-commit-n"
111-
"odh-rstudio-gpu-notebook-image-commit-n")
112-
113-
for val in "${COMMIT[@]}"; do
114-
echo $val
115-
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
106+
COMMIT_ENV_PATH="manifests/base/commit.env"
107+
108+
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
109+
# Get the complete list of images N-1-version to update
110+
COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
111+
112+
for val in ${COMMIT}; do
113+
echo "${val}"
114+
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" "${COMMIT_ENV_PATH}"
116115
done
116+
117117
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
118-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
118+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
119+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
120+
git add "${COMMIT_ENV_PATH}" && \
121+
git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
122+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
119123
else
120-
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
124+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
121125
fi
122126
123127
update-n-1-version:
@@ -147,56 +151,61 @@ jobs:
147151

148152
- name: Update the param.env file
149153
run: |
150-
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
151-
IMAGES=("odh-minimal-notebook-image-n-1"
152-
"odh-minimal-gpu-notebook-image-n-1"
153-
"odh-pytorch-gpu-notebook-image-n-1"
154-
"odh-generic-data-science-notebook-image-n-1"
155-
"odh-tensorflow-gpu-notebook-image-n-1"
156-
"odh-trustyai-notebook-image-n-1"
157-
"odh-codeserver-notebook-image-n-1"
158-
"odh-rstudio-notebook-image-n-1"
159-
"odh-rstudio-gpu-notebook-image-n-1")
160-
161-
for ((i=0;i<${#IMAGES[@]};++i)); do
162-
image=${IMAGES[$i]}
163-
echo "CHECKING: " $image
164-
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
165-
registry=$(echo $img | cut -d '@' -f1)
166-
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
167-
regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
168-
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
169-
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
170-
output=$registry@$digest
171-
echo "NEW:" $output
172-
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
154+
PARAMS_ENV_PATH="manifests/base/params.env"
155+
156+
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
157+
158+
# Get the complete list of images N-1-version to update
159+
IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n-1=" | cut -d "=" -f 1)
160+
161+
for image in ${IMAGES}; do
162+
echo "CHECKING: '${image}'"
163+
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
164+
registry=$(echo "${img}" | cut -d '@' -f1)
165+
166+
skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
167+
168+
src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
169+
regex="^$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}\$"
170+
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
171+
# use `--no-tags` for skopeo once available in newer version
172+
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
173+
output="${registry}@${digest}"
174+
echo "NEW: ${output}"
175+
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
173176
done
177+
174178
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
175-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
179+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
180+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
181+
git add "${PARAMS_ENV_PATH}" && \
182+
git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
183+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
176184
else
177-
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
185+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1 }}"
178186
fi
187+
179188
- name: Update the commit.env file
180189
run: |
181-
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
182-
COMMIT=("odh-minimal-notebook-image-commit-n-1"
183-
"odh-minimal-gpu-notebook-image-commit-n-1"
184-
"odh-pytorch-gpu-notebook-image-commit-n-1"
185-
"odh-generic-data-science-notebook-image-commit-n-1"
186-
"odh-tensorflow-gpu-notebook-image-commit-n-1"
187-
"odh-trustyai-notebook-image-commit-n-1"
188-
"odh-codeserver-notebook-image-commit-n-1"
189-
"odh-rstudio-notebook-image-commit-n-1"
190-
"odh-rstudio-gpu-notebook-image-commit-n-1")
191-
192-
for val in "${COMMIT[@]}"; do
193-
echo $val
194-
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
190+
COMMIT_ENV_PATH="manifests/base/commit.env"
191+
192+
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
193+
# Get the complete list of images N-1-version to update
194+
COMMIT=$(grep "\-n-1=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
195+
196+
for val in ${COMMIT}; do
197+
echo "${val}"
198+
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" "${COMMIT_ENV_PATH}"
195199
done
200+
196201
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
197-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
202+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
203+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
204+
git add "${COMMIT_ENV_PATH}" && \
205+
git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
206+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
198207
else
199-
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
208+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
200209
fi
201210
202211
open-pull-request:

‎.github/workflows/runtimes-digest-updater-upstream.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ jobs:
9292
name="minimal-$name"
9393
fi
9494
registry=$(echo $img | cut -d '@' -f1)
95-
regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
95+
regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
9696
echo "CHECKING: " $regex
9797
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
9898
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')

‎.github/workflows/sync-branches-through-pr.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
---
2+
name: Sync branches through Pull Request
3+
4+
on: # yamllint disable-line rule:truthy
5+
workflow_dispatch:
6+
inputs:
7+
source:
8+
description: Source branch
9+
required: true
10+
target:
11+
description: Target branch
12+
required: true
13+
14+
jobs:
15+
sync:
16+
permissions:
17+
contents: write
18+
pull-requests: write
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout repository
22+
uses: actions/checkout@v4
23+
with:
24+
ref: ${{ github.event.inputs.target }}
25+
fetch-depth: 0
26+
27+
- name: Prepare sync branch
28+
id: prepare
29+
run: |
30+
git fetch origin ${{ github.event.inputs.source }}
31+
git reset --hard origin/${{ github.event.inputs.source }}
32+
33+
TIMESTAMP=$(date +'%Y%m%d%H%M%S')
34+
SYNC_BRANCH=sync__${{ github.event.inputs.source }}__${{ github.event.inputs.target }}__${TIMESTAMP}
35+
echo "branch=$SYNC_BRANCH" >> $GITHUB_OUTPUT
36+
37+
- name: Create pull request
38+
uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
39+
with:
40+
branch: ${{ steps.prepare.outputs.branch }}
41+
title: "Sync `${{ github.event.inputs.target }}` branch with `${{ github.event.inputs.source }}` branch"
42+
body: |
43+
:robot: This is an automated Pull Request created by `/.github/workflows/sync-branches-through-pr.yml`.
44+
45+
It merges all commits from `${{ github.event.inputs.source }}` branch into `${{ github.event.inputs.target }}` branch.
46+
47+
:warning: **IMPORTANT NOTE**: Remember to delete the `${{ steps.prepare.outputs.branch }}` branch after merging the changes.

0 commit comments

Comments
 (0)