Skip to content

Commit 802272a

Browse files
committed
UPSTREAM <carry>: Rebase code to kfp master branch - commit: af4540e
Signed-off-by: Helber Belmiro <[email protected]>
2 parents af4540e + b8d339a commit 802272a

File tree

91 files changed

+11356
-1775
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+11356
-1775
lines changed

.github/actions/build/action.yaml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: "Create a podman build"
2+
description: "This workflow can be used to create a podman build and push to quay.io from source branches."
3+
inputs:
4+
IMAGE_REPO:
5+
description: "Quay image repo name."
6+
required: true
7+
DOCKERFILE:
8+
description: "Path to Dockerfile."
9+
required: true
10+
GH_REPO:
11+
description: "GH org/repo that contains the dockerfile to source."
12+
required: true
13+
OVERWRITE:
14+
default: "false"
15+
description: "GH org/repo that contains the dockerfile to source."
16+
required: true
17+
runs:
18+
using: "composite"
19+
steps:
20+
- uses: actions/checkout@v3
21+
with:
22+
repository: ${{ inputs.GH_REPO }}
23+
ref: ${{ env.SOURCE_BRANCH }}
24+
path: build
25+
- name: Login to Quay.io
26+
uses: redhat-actions/podman-login@v1
27+
with:
28+
username: ${{ env.QUAY_ID }}
29+
password: ${{ env.QUAY_TOKEN }}
30+
registry: quay.io
31+
# Tags in quay stick around as objects in api, when deleted quay adds "end_ts" time stamp on the tag.
32+
# To determine a tag is deleted, we need to check for the presence of this tag.
33+
# Also note there can be multiple tags created/deleted, thus we have 4 cases:
34+
# Case 1: Only 1 tag was ever created "tags: [{name:..},]" -- no end_ts field
35+
# Case 2: No tag was ever created "tags: []"
36+
# Case 3: >1 tags were created, but they were all deleted at some point [{name:..., end_ts,..},....] -- note they all have "end_ts" field.
37+
# Case 4: >1 tags were created, but the most recent one was never deleted (same as case 3, but the latest tag does not have "end_ts".
38+
- name: Check if Image already exists
39+
shell: bash
40+
if: inputs.OVERWRITE == 'false'
41+
env:
42+
IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
43+
run: |
44+
echo ${{ inputs.OVERWRITE }}
45+
46+
tags=$(curl --request GET 'https://quay.io/api/v1/repository/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}/tag/?specificTag=${{ env.TARGET_IMAGE_TAG }}')
47+
latest_tag_has_end_ts=$(echo $tags | yq .tags - | yq 'sort_by(.start_ts) | reverse' - -P | yq .[0].end_ts -)
48+
notempty=$(echo ${tags} | yq .tags - | yq any)
49+
50+
# Image only exists if there is a tag that does not have "end_ts" (i.e. it is still present).
51+
if [[ "$notempty" == "true" && $latest_tag_has_end_ts == "null" ]]; then
52+
echo "::error::The image ${{ env.IMAGE }} already exists"
53+
exit 1
54+
else
55+
echo "Image does not exist...proceeding with build & push."
56+
fi
57+
- name: Build image
58+
shell: bash
59+
working-directory: build
60+
env:
61+
IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
62+
run: |
63+
podman build . -f ${{ inputs.DOCKERFILE }} -t ${{ env.IMAGE }} && podman push ${{ env.IMAGE }}

.github/resources/scripts/deploy-kfp-tekton.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ wait_for_pods || EXIT_CODE=$?
4646
if [[ $EXIT_CODE -ne 0 ]]
4747
then
4848
echo "Deploy unsuccessful. Not all pods running."
49+
collect_artifacts kubeflow
50+
collect_artifacts tekton-pipelines
4951
exit 1
5052
fi
5153

.github/resources/scripts/helper-functions.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ collect_artifacts() {
206206

207207
local log_dir=$(mktemp -d)
208208

209+
kubectl get events -n $kubeflow_ns > $log_dir/events.log
210+
209211
pods_kubeflow=$(kubectl get pods -n $kubeflow_ns --no-headers -o custom-columns=NAME:.metadata.name)
210212

211213
for pod in $pods_kubeflow; do
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "service_account",
3+
"project_id": "your-project-id",
4+
"private_key_id": "your-private-key-id",
5+
"private_key": "-----BEGIN PRIVATE KEY-----\nYour private key content\n-----END PRIVATE KEY-----\n",
6+
"client_email": "[email protected]",
7+
"client_id": "your-client-id",
8+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
9+
"token_uri": "https://accounts.google.com/o/oauth2/token",
10+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
11+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/[email protected]"
12+
}

.github/workflows/build-images.yaml

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Build images from sources.
2+
run-name: Build images from sources.
3+
on:
4+
workflow_call:
5+
inputs:
6+
src_branch:
7+
type: string
8+
default: 'v1.0.x'
9+
description: 'Source branch to build DSP from'
10+
required: true
11+
target_tag:
12+
type: string
13+
default: 'vx.y.z'
14+
description: 'Target Image Tag'
15+
required: true
16+
quay_org:
17+
type: string
18+
default: 'opendatahub'
19+
description: 'Quay Organization'
20+
required: true
21+
overwrite_imgs:
22+
type: string
23+
default: 'true'
24+
description: 'Overwrite images in quay if they already exist for this release.'
25+
required: true
26+
fail_fast:
27+
type: string
28+
default: 'true'
29+
description: 'Stop running entire Workflow if a single build fails'
30+
required: true
31+
32+
workflow_dispatch:
33+
inputs:
34+
src_branch:
35+
type: string
36+
default: 'v1.0.x'
37+
description: 'Source branch to build DSP from'
38+
required: true
39+
target_tag:
40+
type: string
41+
default: 'vx.y.z'
42+
description: 'Target Image Tag'
43+
required: true
44+
quay_org:
45+
type: string
46+
default: 'opendatahub'
47+
description: 'Quay Organization'
48+
required: true
49+
overwrite_imgs:
50+
type: string
51+
default: 'true'
52+
description: 'Overwrite images in quay if they already exist for this release.'
53+
required: true
54+
fail_fast:
55+
type: string
56+
default: 'true'
57+
description: 'Stop running entire Workflow if a single build fails'
58+
required: true
59+
env:
60+
SOURCE_BRANCH: ${{ inputs.src_branch }}
61+
QUAY_ORG: ${{ inputs.quay_org }}
62+
QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
63+
QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
64+
TARGET_IMAGE_TAG: ${{ inputs.target_tag }}
65+
OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }}
66+
jobs:
67+
build-images-with-tag:
68+
continue-on-error: false
69+
runs-on: ubuntu-latest
70+
permissions:
71+
contents: read
72+
strategy:
73+
fail-fast: ${{ inputs.fail_fast == 'true' }}
74+
matrix:
75+
include:
76+
- image: ds-pipelines-api-server
77+
dockerfile: backend/Dockerfile
78+
- image: ds-pipelines-frontend
79+
dockerfile: frontend/Dockerfile
80+
- image: ds-pipelines-persistenceagent
81+
dockerfile: backend/Dockerfile.persistenceagent
82+
- image: ds-pipelines-scheduledworkflow
83+
dockerfile: backend/Dockerfile.scheduledworkflow
84+
- image: ds-pipelines-driver
85+
dockerfile: backend/Dockerfile.driver
86+
- image: ds-pipelines-launcher
87+
dockerfile: backend/Dockerfile.launcher
88+
steps:
89+
- uses: actions/checkout@v3
90+
- uses: ./.github/actions/build
91+
name: Build Image
92+
with:
93+
IMAGE_REPO: ${{ matrix.image }}
94+
DOCKERFILE: ${{ matrix.dockerfile }}
95+
GH_REPO: ${{ github.repository }}
96+
OVERWRITE: ${{ env.OVERWRITE_IMAGES }}

.github/workflows/build-master.yml

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Build images for Master branch
2+
on:
3+
push:
4+
branches:
5+
- master
6+
concurrency:
7+
group: ${{ github.workflow }}
8+
cancel-in-progress: true
9+
env:
10+
QUAY_ORG: opendatahub
11+
QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
12+
QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
13+
SOURCE_BRANCH: master
14+
jobs:
15+
build-master-images:
16+
continue-on-error: false
17+
runs-on: ubuntu-latest
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
include:
22+
- image: ds-pipelines-api-server
23+
dockerfile: backend/Dockerfile
24+
- image: ds-pipelines-frontend
25+
dockerfile: frontend/Dockerfile
26+
- image: ds-pipelines-persistenceagent
27+
dockerfile: backend/Dockerfile.persistenceagent
28+
- image: ds-pipelines-scheduledworkflow
29+
dockerfile: backend/Dockerfile.scheduledworkflow
30+
- image: ds-pipelines-driver
31+
dockerfile: backend/Dockerfile.driver
32+
- image: ds-pipelines-launcher
33+
dockerfile: backend/Dockerfile.launcher
34+
steps:
35+
- uses: actions/checkout@v3
36+
- name: Generate Tag
37+
shell: bash
38+
id: tags
39+
env:
40+
SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
41+
run: |
42+
commit_sha=${{ github.event.after }}
43+
tag=${SOURCE_BRANCH}-${commit_sha:0:7}
44+
echo "tag=${tag}" >> $GITHUB_OUTPUT
45+
- name: Build Image
46+
uses: ./.github/actions/build
47+
env:
48+
IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}:${{ steps.tags.outputs.tag }}
49+
TARGET_IMAGE_TAG: ${{ steps.tags.outputs.tag }}
50+
with:
51+
OVERWRITE: true
52+
IMAGE_REPO: ${{ matrix.image }}
53+
DOCKERFILE: ${{ matrix.dockerfile }}
54+
GH_REPO: ${{ github.repository }}
55+
- name: Tag latest
56+
shell: bash
57+
env:
58+
IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}
59+
NEWEST_TAG: ${{ steps.tags.outputs.tag }}
60+
SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
61+
run: |
62+
podman tag ${IMG}:${NEWEST_TAG} ${IMG}:latest
63+
podman push ${IMG}:latest
64+
podman tag ${IMG}:${NEWEST_TAG} ${IMG}:${SOURCE_BRANCH}
65+
podman push ${IMG}:${SOURCE_BRANCH}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Trigger PR CI
2+
on:
3+
pull_request:
4+
paths-ignore:
5+
- 'docs/**'
6+
- 'guides/**'
7+
- 'images/**'
8+
- '**/README.md'
9+
types:
10+
- opened
11+
- reopened
12+
- closed
13+
- synchronize
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
16+
cancel-in-progress: true
17+
jobs:
18+
upload-data:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v2
22+
- name: Save PR payload
23+
shell: bash
24+
run: |
25+
mkdir -p ./pr
26+
echo ${{ github.event.pull_request.number }} >> ./pr/pr_number
27+
echo ${{ github.event.pull_request.state }} >> ./pr/pr_state
28+
echo ${{ github.event.pull_request.head.sha }} >> ./pr/head_sha
29+
echo ${{ github.event.action }} >> ./pr/event_action
30+
- uses: actions/upload-artifact@v3
31+
with:
32+
name: pr
33+
path: pr/

0 commit comments

Comments
 (0)