Skip to content

Commit 8463c27

Browse files
harshad16harshad16
authored
Merge pull request #413 from atheo89/rsync-2024b
R-sync release-2024b
2 parents 69688c1 + cbcc090 commit 8463c27

File tree

57 files changed

+29570
-26032
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+29570
-26032
lines changed

‎.github/workflows/build-notebooks-TEMPLATE.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ name: Build & Publish Notebook Servers (TEMPLATE)
1818

1919
jobs:
2020
build:
21-
runs-on: ubuntu-latest
21+
runs-on: ubuntu-22.04
2222
env:
2323
# GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
2424
CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
@@ -97,6 +97,7 @@ jobs:
9797

9898
- name: Configure Podman
9999
run: |
100+
set -x
100101
mkdir -p $HOME/.config/containers/
101102
cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
102103
cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf

‎.github/workflows/code-quality.yaml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,13 @@ jobs:
4545

4646
- name: Install poetry
4747
if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
48-
run: pip install poetry==${{ env.poetry_version }}
48+
run: pipx install poetry==${{ env.poetry_version }}
49+
env:
50+
PIPX_HOME: /home/runner/.local/pipx
51+
PIPX_BIN_DIR: /home/runner/.local/bin
52+
53+
- name: Check poetry is installed correctly
54+
run: poetry env info
4955

5056
- name: Save cache
5157
if: steps.cache-poetry-restore.outputs.cache-hit != 'true'

‎.github/workflows/notebooks-digest-updater-upstream.yaml

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
---
2+
# The aim of this GitHub workflow is to update the params.env file with the latest digest.
3+
name: Update notebook image build commit hashes
4+
on: # yamllint disable-line rule:truthy
5+
workflow_dispatch:
6+
inputs:
7+
branch:
8+
required: true
9+
description: "Provide the name of the branch you want to update ex main, vYYYYx etc: "
10+
# Put the scheduler on comment until automate the full release procedure
11+
# schedule:
12+
# - cron: "0 0 * * 5" #Scheduled every Friday
13+
env:
14+
DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }}
15+
BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
16+
RELEASE_VERSION_N: 2024b
17+
RELEASE_VERSION_N_1: 2024a
18+
jobs:
19+
initialize:
20+
runs-on: ubuntu-latest
21+
permissions:
22+
contents: write
23+
steps:
24+
- name: Install Skopeo CLI
25+
shell: bash
26+
run: |
27+
sudo apt-get -y update
28+
sudo apt-get -y install skopeo
29+
30+
# Checkout the branch
31+
- name: Checkout branch
32+
uses: actions/checkout@v4
33+
with:
34+
ref: ${{ env.BRANCH_NAME }}
35+
36+
# Create a new branch
37+
- name: Create a new branch
38+
run: |
39+
echo ${{ env.DIGEST_UPDATER_BRANCH }}
40+
git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }}
41+
git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }}
42+
43+
update-n-version:
44+
needs: [initialize]
45+
runs-on: ubuntu-latest
46+
permissions:
47+
contents: write
48+
steps:
49+
- name: Configure Git
50+
run: |
51+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
52+
git config --global user.name "GitHub Actions"
53+
54+
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
55+
- name: Retrive latest commit hash from the release branch
56+
id: hash-n
57+
shell: bash
58+
run: |
59+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
60+
echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
61+
62+
# Checkout the release branch to apply the updates
63+
- name: Checkout release branch
64+
uses: actions/checkout@v4
65+
with:
66+
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
67+
68+
- name: Update the params.env file
69+
run: |
70+
PARAMS_ENV_PATH="manifests/base/params.env"
71+
72+
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
73+
74+
# Get the complete list of images N-version to update
75+
IMAGES=$(grep "\-n=" "${PARAMS_ENV_PATH}" | cut -d "=" -f 1)
76+
77+
for image in ${IMAGES}; do
78+
echo "CHECKING: '${image}'"
79+
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
80+
registry=$(echo "${img}" | cut -d '@' -f1)
81+
82+
skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
83+
84+
src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
85+
regex="^$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
86+
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
87+
# use `--no-tags` for skopeo once available in newer version
88+
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
89+
output="${registry}@${digest}"
90+
echo "NEW: ${output}"
91+
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
92+
done
93+
94+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
95+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
96+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
97+
git add "${PARAMS_ENV_PATH}" && \
98+
git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
99+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
100+
else
101+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
102+
fi
103+
104+
- name: Update the commit.env file
105+
run: |
106+
COMMIT_ENV_PATH="manifests/base/commit.env"
107+
108+
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
109+
# Get the complete list of commits N-version to update
110+
COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
111+
112+
for val in ${COMMIT}; do
113+
echo "${val}"
114+
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" "${COMMIT_ENV_PATH}"
115+
done
116+
117+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
118+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
119+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
120+
git add "${COMMIT_ENV_PATH}" && \
121+
git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
122+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
123+
else
124+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
125+
fi
126+
127+
update-n-1-version:
128+
needs: [initialize, update-n-version]
129+
runs-on: ubuntu-latest
130+
permissions:
131+
contents: write
132+
steps:
133+
- name: Configure Git
134+
run: |
135+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
136+
git config --global user.name "GitHub Actions"
137+
138+
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
139+
- name: Retrive latest commit hash from the release branch
140+
id: hash-n-1
141+
shell: bash
142+
run: |
143+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N_1&per_page=1)
144+
echo "HASH_N_1=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
145+
146+
# Checkout the release branch to apply the updates
147+
- name: Checkout release branch
148+
uses: actions/checkout@v4
149+
with:
150+
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
151+
152+
- name: Update the param.env file
153+
run: |
154+
PARAMS_ENV_PATH="manifests/base/params.env"
155+
156+
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
157+
158+
# Get the complete list of images N-1-version to update
159+
IMAGES=$(grep "\-n-1=" "${PARAMS_ENV_PATH}" | cut -d "=" -f 1)
160+
161+
for image in ${IMAGES}; do
162+
echo "CHECKING: '${image}'"
163+
img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
164+
registry=$(echo "${img}" | cut -d '@' -f1)
165+
166+
skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
167+
168+
src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
169+
regex="^$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}\$"
170+
latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
171+
# use `--no-tags` for skopeo once available in newer version
172+
digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
173+
output="${registry}@${digest}"
174+
echo "NEW: ${output}"
175+
sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
176+
done
177+
178+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
179+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
180+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
181+
git add "${PARAMS_ENV_PATH}" && \
182+
git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
183+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
184+
else
185+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1 }}"
186+
fi
187+
188+
- name: Update the commit.env file
189+
run: |
190+
COMMIT_ENV_PATH="manifests/base/commit.env"
191+
192+
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
193+
# Get the complete list of images N-1-version to update
194+
COMMIT=$(grep "\-n-1=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
195+
196+
for val in ${COMMIT}; do
197+
echo "${val}"
198+
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" "${COMMIT_ENV_PATH}"
199+
done
200+
201+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
202+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
203+
git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
204+
git add "${COMMIT_ENV_PATH}" && \
205+
git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
206+
git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
207+
else
208+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1 }}"
209+
fi
210+
211+
open-pull-request:
212+
needs: [update-n-version, update-n-1-version]
213+
runs-on: ubuntu-latest
214+
permissions:
215+
pull-requests: write
216+
steps:
217+
- name: Checkout repo
218+
uses: actions/checkout@v4
219+
220+
- name: pull-request
221+
uses: repo-sync/pull-request@v2
222+
with:
223+
source_branch: ${{ env.DIGEST_UPDATER_BRANCH }}
224+
destination_branch: ${{ env.BRANCH_NAME }}
225+
github_token: ${{ secrets.GITHUB_TOKEN }}
226+
pr_label: "automated pr"
227+
pr_title: "[Digest Updater Action] Update Notebook Images"
228+
pr_body: |
229+
:rocket: This is an automated Pull Request.
230+
Created by `/.github/workflows/notebooks-digest-updater-upstream.yaml`
231+
232+
This PR updates the following files:
233+
- `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
234+
- `manifests/base/commit.env` file with the latest commit (N & N-1).
235+
236+
:exclamation: **IMPORTANT NOTE**: Remember to delete the `${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes

‎ci/check-params-env.sh

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -216,44 +216,41 @@ function check_image_variable_matches_name_and_commitref() {
216216
expected_build_name="codeserver-ubi9-python-3.9-amd64"
217217
;;
218218
odh-rstudio-notebook-image-n)
219-
expected_name="odh-notebook-rstudio-server-c9s-python-3.9"
220-
expected_commitref="2024a"
221-
expected_build_name="rstudio-c9s-python-3.9-amd64"
219+
expected_name="odh-notebook-rstudio-server-c9s-python-3.11"
220+
expected_commitref="2024b"
221+
expected_build_name="rstudio-c9s-python-3.11-amd64"
222222
;;
223223
odh-rstudio-notebook-image-n-1)
224224
expected_name="odh-notebook-rstudio-server-c9s-python-3.9"
225-
expected_commitref="2023b"
225+
expected_commitref="2024a"
226226
expected_build_name="rstudio-c9s-python-3.9-amd64"
227227
;;
228228
# For both RStudio GPU workbenches - the final name labels are identical to plain RStudio ones
229229
# This is because the very same RStudio Dockerfile is used but different base images in both cases
230230
# We should consider what to do with this - in ideal case, we should have different labels for these cases.
231231
odh-rstudio-gpu-notebook-image-n)
232-
expected_name="odh-notebook-rstudio-server-c9s-python-3.9"
233-
expected_commitref="2024a"
234-
expected_build_name="cuda-rstudio-c9s-python-3.9-amd64"
232+
expected_name="odh-notebook-rstudio-server-c9s-python-3.11"
233+
expected_commitref="2024b"
234+
expected_build_name="cuda-rstudio-c9s-python-3.11-amd64"
235235
;;
236236
odh-rstudio-gpu-notebook-image-n-1)
237237
expected_name="odh-notebook-rstudio-server-c9s-python-3.9"
238-
expected_commitref="2023b"
238+
expected_commitref="2024a"
239239
expected_build_name="cuda-rstudio-c9s-python-3.9-amd64"
240240
;;
241241
odh-rocm-minimal-notebook-image-n)
242242
expected_name="odh-notebook-jupyter-minimal-ubi9-python-3.11"
243-
expected_commitref="main"
244-
# expected_commitref="2024b"
243+
expected_commitref="2024b"
245244
expected_build_name="rocm-jupyter-minimal-ubi9-python-3.11-amd64"
246245
;;
247246
odh-rocm-pytorch-notebook-image-n)
248247
expected_name="odh-notebook-jupyter-rocm-pytorch-ubi9-python-3.11"
249-
expected_commitref="main"
250-
# expected_commitref="2024b"
248+
expected_commitref="2024b"
251249
expected_build_name="rocm-jupyter-pytorch-ubi9-python-3.11-amd64"
252250
;;
253251
odh-rocm-tensorflow-notebook-image-n)
254252
expected_name="odh-notebook-jupyter-rocm-tensorflow-ubi9-python-3.11"
255-
expected_commitref="main"
256-
# expected_commitref="2024b"
253+
expected_commitref="2024b"
257254
expected_build_name="rocm-jupyter-tensorflow-ubi9-python-3.11-amd64"
258255
;;
259256
*)

‎codeserver/ubi9-python-3.11/run-code-server.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ universal_json_settings='{
4646
"python.defaultInterpreterPath": "/opt/app-root/bin/python3",
4747
"telemetry.telemetryLevel": "off",
4848
"telemetry.enableTelemetry": false,
49-
"workbench.enableExperiments": false
49+
"workbench.enableExperiments": false,
50+
"extensions.autoCheckUpdates": false,
51+
"extensions.autoUpdate": false
5052
}'
5153

5254
# Define python debuger settings

‎codeserver/ubi9-python-3.9/run-code-server.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ universal_json_settings='{
4646
"python.defaultInterpreterPath": "/opt/app-root/bin/python3",
4747
"telemetry.telemetryLevel": "off",
4848
"telemetry.enableTelemetry": false,
49-
"workbench.enableExperiments": false
49+
"workbench.enableExperiments": false,
50+
"extensions.autoCheckUpdates": false,
51+
"extensions.autoUpdate": false
5052
}'
5153

5254
# Define python debuger settings

‎intel/runtimes/ml/ubi9-python-3.11/Pipfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ plotly = "~=5.16.1"
1919
scipy = "~=1.11.2"
2020
scikit-learn = "~=1.3.1"
2121
skl2onnx = "~=1.15.0"
22-
codeflare-sdk = "~=0.19.1"
22+
codeflare-sdk = "~=0.22.0"
2323
# DB connectors
2424
pymongo = "~=4.5.0"
2525
psycopg = "~=3.1.10"

0 commit comments

Comments
 (0)