Skip to content

Commit 65049cb

Browse files
hbelmirohbelmiro
committed
Rebase on top of kubeflow@4c966f7
Signed-off-by: Helber Belmiro <[email protected]>
2 parents 4c966f7 + 30d0c45 commit 65049cb

File tree

136 files changed

+9652
-1935
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

136 files changed

+9652
-1935
lines changed

‎.github/actions/build/action.yaml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: "Create a podman build"
2+
description: "This workflow can be used to create a podman build and push to quay.io from source branches."
3+
inputs:
4+
IMAGE_REPO:
5+
description: "Quay image repo name."
6+
required: true
7+
DOCKERFILE:
8+
description: "Path to Dockerfile."
9+
required: true
10+
GH_REPO:
11+
description: "GH org/repo that contains the dockerfile to source."
12+
required: true
13+
OVERWRITE:
14+
default: "false"
15+
description: "GH org/repo that contains the dockerfile to source."
16+
required: true
17+
runs:
18+
using: "composite"
19+
steps:
20+
- uses: actions/checkout@v3
21+
with:
22+
repository: ${{ inputs.GH_REPO }}
23+
ref: ${{ env.SOURCE_BRANCH }}
24+
path: build
25+
- name: Login to Quay.io
26+
uses: redhat-actions/podman-login@v1
27+
with:
28+
username: ${{ env.QUAY_ID }}
29+
password: ${{ env.QUAY_TOKEN }}
30+
registry: quay.io
31+
# Tags in quay stick around as objects in api, when deleted quay adds "end_ts" time stamp on the tag.
32+
# To determine a tag is deleted, we need to check for the presence of this tag.
33+
# Also note there can be multiple tags created/deleted, thus we have 4 cases:
34+
# Case 1: Only 1 tag was ever created "tags: [{name:..},]" -- no end_ts field
35+
# Case 2: No tag was ever created "tags: []"
36+
# Case 3: >1 tags were created, but they were all deleted at some point [{name:..., end_ts,..},....] -- note they all have "end_ts" field.
37+
# Case 4: >1 tags were created, but the most recent one was never deleted (same as case 3, but the latest tag does not have "end_ts".
38+
- name: Check if Image already exists
39+
shell: bash
40+
if: inputs.OVERWRITE == 'false'
41+
env:
42+
IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
43+
run: |
44+
echo ${{ inputs.OVERWRITE }}
45+
46+
tags=$(curl --request GET 'https://quay.io/api/v1/repository/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}/tag/?specificTag=${{ env.TARGET_IMAGE_TAG }}')
47+
latest_tag_has_end_ts=$(echo $tags | yq .tags - | yq 'sort_by(.start_ts) | reverse' - -P | yq .[0].end_ts -)
48+
notempty=$(echo ${tags} | yq .tags - | yq any)
49+
50+
# Image only exists if there is a tag that does not have "end_ts" (i.e. it is still present).
51+
if [[ "$notempty" == "true" && $latest_tag_has_end_ts == "null" ]]; then
52+
echo "::error::The image ${{ env.IMAGE }} already exists"
53+
exit 1
54+
else
55+
echo "Image does not exist...proceeding with build & push."
56+
fi
57+
- name: Build image
58+
shell: bash
59+
working-directory: build
60+
env:
61+
IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
62+
run: |
63+
podman build . -f ${{ inputs.DOCKERFILE }} -t ${{ env.IMAGE }} && podman push ${{ env.IMAGE }}

‎.github/resources/scripts/helper-functions.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ collect_artifacts() {
206206

207207
local log_dir=$(mktemp -d)
208208

209+
kubectl get events -n $kubeflow_ns > $log_dir/events.log
210+
209211
pods_kubeflow=$(kubectl get pods -n $kubeflow_ns --no-headers -o custom-columns=NAME:.metadata.name)
210212

211213
for pod in $pods_kubeflow; do

‎.github/resources/unittests/application_default_credentials.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "service_account",
3+
"project_id": "your-project-id",
4+
"private_key_id": "your-private-key-id",
5+
"private_key": "-----BEGIN PRIVATE KEY-----\nYour private key content\n-----END PRIVATE KEY-----\n",
6+
"client_email": "[email protected]",
7+
"client_id": "your-client-id",
8+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
9+
"token_uri": "https://accounts.google.com/o/oauth2/token",
10+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
11+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/[email protected]"
12+
}

‎.github/workflows/backend-visualization.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ name: KFP backend visualization tests
22

33
on:
44
push:
5-
branches: [ master ]
5+
branches:
6+
- master
7+
- main
8+
- stable
9+
- 'rhoai-*'
610

711
pull_request:
812
paths:

‎.github/workflows/build-images.yaml

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Build images from sources.
2+
run-name: Build images from sources.
3+
on:
4+
workflow_call:
5+
inputs:
6+
src_branch:
7+
type: string
8+
default: 'v1.0.x'
9+
description: 'Source branch to build DSP from'
10+
required: true
11+
target_tag:
12+
type: string
13+
default: 'vx.y.z'
14+
description: 'Target Image Tag'
15+
required: true
16+
quay_org:
17+
type: string
18+
default: 'opendatahub'
19+
description: 'Quay Organization'
20+
required: true
21+
overwrite_imgs:
22+
type: string
23+
default: 'true'
24+
description: 'Overwrite images in quay if they already exist for this release.'
25+
required: true
26+
fail_fast:
27+
type: string
28+
default: 'true'
29+
description: 'Stop running entire Workflow if a single build fails'
30+
required: true
31+
32+
workflow_dispatch:
33+
inputs:
34+
src_branch:
35+
type: string
36+
default: 'v1.0.x'
37+
description: 'Source branch to build DSP from'
38+
required: true
39+
target_tag:
40+
type: string
41+
default: 'vx.y.z'
42+
description: 'Target Image Tag'
43+
required: true
44+
quay_org:
45+
type: string
46+
default: 'opendatahub'
47+
description: 'Quay Organization'
48+
required: true
49+
overwrite_imgs:
50+
type: string
51+
default: 'true'
52+
description: 'Overwrite images in quay if they already exist for this release.'
53+
required: true
54+
fail_fast:
55+
type: string
56+
default: 'true'
57+
description: 'Stop running entire Workflow if a single build fails'
58+
required: true
59+
env:
60+
SOURCE_BRANCH: ${{ inputs.src_branch }}
61+
QUAY_ORG: ${{ inputs.quay_org }}
62+
QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
63+
QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
64+
TARGET_IMAGE_TAG: ${{ inputs.target_tag }}
65+
OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }}
66+
jobs:
67+
build-images-with-tag:
68+
continue-on-error: false
69+
runs-on: ubuntu-latest
70+
permissions:
71+
contents: read
72+
strategy:
73+
fail-fast: ${{ inputs.fail_fast == 'true' }}
74+
matrix:
75+
include:
76+
- image: ds-pipelines-api-server
77+
dockerfile: backend/Dockerfile
78+
- image: ds-pipelines-frontend
79+
dockerfile: frontend/Dockerfile
80+
- image: ds-pipelines-persistenceagent
81+
dockerfile: backend/Dockerfile.persistenceagent
82+
- image: ds-pipelines-scheduledworkflow
83+
dockerfile: backend/Dockerfile.scheduledworkflow
84+
- image: ds-pipelines-driver
85+
dockerfile: backend/Dockerfile.driver
86+
- image: ds-pipelines-launcher
87+
dockerfile: backend/Dockerfile.launcher
88+
steps:
89+
- uses: actions/checkout@v3
90+
- uses: ./.github/actions/build
91+
name: Build Image
92+
with:
93+
IMAGE_REPO: ${{ matrix.image }}
94+
DOCKERFILE: ${{ matrix.dockerfile }}
95+
GH_REPO: ${{ github.repository }}
96+
OVERWRITE: ${{ env.OVERWRITE_IMAGES }}

‎.github/workflows/build-master.yml

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: Build images for Master branch
2+
on:
3+
push:
4+
branches:
5+
- master
6+
- main
7+
- stable
8+
- 'rhoai-*'
9+
concurrency:
10+
group: ${{ github.workflow }}
11+
cancel-in-progress: true
12+
env:
13+
QUAY_ORG: opendatahub
14+
QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
15+
QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
16+
SOURCE_BRANCH: master
17+
jobs:
18+
build-master-images:
19+
continue-on-error: false
20+
runs-on: ubuntu-latest
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
include:
25+
- image: ds-pipelines-api-server
26+
dockerfile: backend/Dockerfile
27+
- image: ds-pipelines-frontend
28+
dockerfile: frontend/Dockerfile
29+
- image: ds-pipelines-persistenceagent
30+
dockerfile: backend/Dockerfile.persistenceagent
31+
- image: ds-pipelines-scheduledworkflow
32+
dockerfile: backend/Dockerfile.scheduledworkflow
33+
- image: ds-pipelines-driver
34+
dockerfile: backend/Dockerfile.driver
35+
- image: ds-pipelines-launcher
36+
dockerfile: backend/Dockerfile.launcher
37+
steps:
38+
- uses: actions/checkout@v3
39+
- name: Generate Tag
40+
shell: bash
41+
id: tags
42+
env:
43+
SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
44+
run: |
45+
commit_sha=${{ github.event.after }}
46+
tag=${SOURCE_BRANCH}-${commit_sha:0:7}
47+
echo "tag=${tag}" >> $GITHUB_OUTPUT
48+
- name: Build Image
49+
uses: ./.github/actions/build
50+
env:
51+
IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}:${{ steps.tags.outputs.tag }}
52+
TARGET_IMAGE_TAG: ${{ steps.tags.outputs.tag }}
53+
with:
54+
OVERWRITE: true
55+
IMAGE_REPO: ${{ matrix.image }}
56+
DOCKERFILE: ${{ matrix.dockerfile }}
57+
GH_REPO: ${{ github.repository }}
58+
- name: Tag latest
59+
shell: bash
60+
env:
61+
IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}
62+
NEWEST_TAG: ${{ steps.tags.outputs.tag }}
63+
SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
64+
run: |
65+
podman tag ${IMG}:${NEWEST_TAG} ${IMG}:latest
66+
podman push ${IMG}:latest
67+
podman tag ${IMG}:${NEWEST_TAG} ${IMG}:${SOURCE_BRANCH}
68+
podman push ${IMG}:${SOURCE_BRANCH}

‎.github/workflows/build-prs-trigger.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Trigger PR CI
2+
on:
3+
pull_request:
4+
paths-ignore:
5+
- 'docs/**'
6+
- 'guides/**'
7+
- 'images/**'
8+
- '**/README.md'
9+
types:
10+
- opened
11+
- reopened
12+
- closed
13+
- synchronize
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
16+
cancel-in-progress: true
17+
jobs:
18+
upload-data:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v2
22+
- name: Save PR payload
23+
shell: bash
24+
run: |
25+
mkdir -p ./pr
26+
echo ${{ github.event.pull_request.number }} >> ./pr/pr_number
27+
echo ${{ github.event.pull_request.state }} >> ./pr/pr_state
28+
echo ${{ github.event.pull_request.head.sha }} >> ./pr/head_sha
29+
echo ${{ github.event.action }} >> ./pr/event_action
30+
- uses: actions/upload-artifact@v4
31+
with:
32+
name: pr
33+
path: pr/

0 commit comments

Comments
 (0)