Skip to content

Commit 5760c87

Browse files
committed
- Parallelizing docker builds by running a build workflow with different modules as matrix
- Uploading generated docker image tarball as an artifact (with 1 day of retention) with path as "images_${last commit sha}" - Downloading the tar ball during the deployment step and loading it to kind cluster - Fixing minor warnings by parsing the CMD command as a string array Signed-off-by: Nelesh Singla <117123879+nsingla@users.noreply.github.com>
1 parent 6ccf261 commit 5760c87

21 files changed

+445
-277
lines changed

.github/actions/kfp-cluster/action.yml

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: "Set up KFP on KinD"
2-
description: "Step to start and configure KFP on KinD"
2+
description: "Step to start and configure KFP on Kind"
33

44
inputs:
55
k8s_version:
@@ -17,14 +17,26 @@ inputs:
1717
description: "If KFP should be deployed with cache enabled globally"
1818
required: false
1919
default: 'true'
20-
20+
cluser_name:
21+
description: "Provide kind cluster name if you want to name it other than kfp"
22+
required: false
23+
default: 'kfp'
24+
image_tag:
25+
required: true
26+
description: "Provide the image tag your image was tagged with"
27+
image_path:
28+
required: true
29+
description: "Path within github artifacts where your image tarball is stored"
30+
image_registry:
31+
required: true
32+
description: "Image Registry address of the images"
2133
runs:
2234
using: "composite"
2335
steps:
2436
- name: Create k8s Kind Cluster
2537
uses: container-tools/kind-action@v2
2638
with:
27-
cluster_name: kfp
39+
cluster_name: ${{ inputs.cluser_name }}
2840
kubectl_version: ${{ inputs.k8s_version }}
2941
version: v0.25.0
3042
node_image: kindest/node:${{ inputs.k8s_version }}
@@ -35,14 +47,19 @@ runs:
3547
shell: bash
3648
run: ./.github/resources/squid/deploy-squid.sh
3749

38-
- name: Build images
50+
- name: Download Docker Images
51+
uses: actions/download-artifact@v4
52+
with:
53+
path: "images_${{ github.sha }}"
54+
55+
- name: Load Docker Images
3956
shell: bash
4057
run: |
41-
if [ "${{ inputs.proxy }}" = "true" ]; then
42-
./.github/resources/scripts/build-images.sh --proxy
43-
else
44-
./.github/resources/scripts/build-images.sh
45-
fi
58+
APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent")
59+
for app in "${APPS[@]}"; do
60+
docker image load -i ${{ inputs.image_path }}/$app/$app.tar
61+
docker push ${{ inputs.image_registry }}/$app:${{ inputs.IMAGE_TAG }}
62+
done
4663
4764
- name: Deploy KFP
4865
shell: bash

.github/resources/manifests/argo/overlays/kubernetes-native/apiserver-env.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ spec:
99
- name: ml-pipeline-api-server
1010
env:
1111
- name: V2_DRIVER_IMAGE
12-
value: kind-registry:5000/driver
12+
value: kind-registry:5000/driver:latest
1313
- name: V2_LAUNCHER_IMAGE
14-
value: kind-registry:5000/launcher
14+
value: kind-registry:5000/launcher:latest
15+
- name: LOG_LEVEL
16+
value: "debug"

.github/resources/manifests/argo/overlays/kubernetes-native/kustomization.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,5 @@ images:
1515
newName: kind-registry:5000/scheduledworkflow
1616
newTag: latest
1717

18-
patchesStrategicMerge:
19-
- apiserver-env.yaml
18+
patches:
19+
- path: apiserver-env.yaml

.github/resources/manifests/argo/overlays/no-proxy/apiserver-env.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ spec:
99
- name: ml-pipeline-api-server
1010
env:
1111
- name: V2_DRIVER_IMAGE
12-
value: kind-registry:5000/driver
12+
value: kind-registry:5000/driver:latest
1313
- name: V2_LAUNCHER_IMAGE
14-
value: kind-registry:5000/launcher
14+
value: kind-registry:5000/launcher:latest
1515
- name: LOG_LEVEL
1616
value: "debug"

.github/resources/scripts/build-images.sh

Lines changed: 0 additions & 64 deletions
This file was deleted.

.github/resources/scripts/deploy-kfp.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ while [ "$#" -gt 0 ]; do
4949
esac
5050
done
5151

52-
if [ "${USE_PROXY}" == "true" && "${PIPELINES_STORE}" == "kubernetes" ]; then
52+
if [ "${USE_PROXY}" == "true" ] && [ "${PIPELINES_STORE}" == "kubernetes" ]; then
5353
echo "ERROR: Kubernetes Pipeline store cannot be deployed with proxy support."
5454
exit 1
5555
fi
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,58 @@
1-
name: Build images from sources.
1+
name: Build and Push images
22
run-name: Build images
33
on:
44
workflow_call:
55
inputs:
66
src_branch:
77
type: string
8-
default: 'release-X.Y'
8+
default: ''
99
description: 'Source branch to build KFP from'
10-
required: true
10+
required: false
1111
target_tag:
1212
type: string
1313
default: 'X.Y.Z'
1414
description: 'Target Image Tag'
1515
required: true
16-
fail_fast:
17-
type: string
18-
default: 'true'
19-
description: 'Stop running entire Workflow if a single build fails'
20-
required: true
2116
overwrite_imgs:
2217
type: string
2318
default: 'true'
2419
description: 'Overwrite images in GHCR if they already exist for this tag.'
25-
required: true
20+
required: false
2621
set_latest:
2722
type: string
2823
default: 'true'
2924
description: 'Set latest tag on build images.'
30-
required: true
25+
required: false
3126
add_sha_tag:
3227
type: string
3328
default: 'true'
3429
description: 'Add a sha image tag.'
3530
required: false
31+
app_to_build:
32+
type: string
33+
default: ''
34+
description: 'Provide the app name to build'
35+
required: true
36+
image_context:
37+
type: string
38+
default: ''
39+
description: 'Provide the docker file path'
40+
required: true
41+
docker_file:
42+
type: string
43+
default: ''
44+
description: 'Provide the docker file name'
45+
required: true
46+
push:
47+
type: boolean
48+
default: false
49+
description: 'Whether to push image to CR or not'
50+
required: false
3651
workflow_dispatch:
3752
inputs:
3853
src_branch:
3954
type: string
40-
default: 'release-X.Y'
55+
default: ''
4156
description: 'Source branch to build KFP from'
4257
required: true
4358
target_tag:
@@ -65,71 +80,52 @@ on:
6580
default: 'true'
6681
description: 'Add a sha image tag.'
6782
required: false
83+
app_to_build:
84+
type: string
85+
default: ''
86+
description: 'Provide the app name to build'
87+
required: true
88+
image_context:
89+
type: string
90+
default: ''
91+
description: 'Provide the docker file path'
92+
required: true
93+
docker_file:
94+
type: string
95+
default: ''
96+
description: 'Provide the docker file name'
97+
required: true
98+
push:
99+
type: boolean
100+
default: false
101+
description: 'Whether to push image to CR or not'
102+
required: true
68103
env:
69104
SOURCE_BRANCH: ${{ inputs.src_branch }}
70105
TARGET_IMAGE_TAG: ${{ inputs.target_tag }}
71106
OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }}
72107
IMAGE_REGISTRY: ghcr.io
73-
IMAGE_ORG: ${{ github.repository_owner }}
108+
IMAGE_ORG: ${{ github.repository }}
74109
SET_LATEST: ${{ inputs.set_latest }}
75110
ADD_SHA_TAG: ${{ inputs.add_sha_tag }}
111+
CACHE_PATH: '/tmp/.buildx-cache'
112+
76113
jobs:
77-
build-images-with-tag:
114+
build-and-push-images:
115+
if: inputs.push && inputs.src_branch
78116
continue-on-error: false
79117
runs-on: ubuntu-latest
80118
permissions:
81119
contents: read
82120
packages: write
83121
attestations: write
84122
id-token: write
85-
strategy:
86-
fail-fast: ${{ inputs.fail_fast == 'true' }}
87-
matrix:
88-
include:
89-
- image: kfp-api-server
90-
dockerfile: backend/Dockerfile
91-
context: .
92-
- image: kfp-frontend
93-
dockerfile: frontend/Dockerfile
94-
context: .
95-
- image: kfp-persistence-agent
96-
dockerfile: backend/Dockerfile.persistenceagent
97-
context: .
98-
- image: kfp-scheduled-workflow-controller
99-
dockerfile: backend/Dockerfile.scheduledworkflow
100-
context: .
101-
- image: kfp-viewer-crd-controller
102-
dockerfile: backend/Dockerfile.viewercontroller
103-
context: .
104-
- image: kfp-visualization-server
105-
dockerfile: backend/Dockerfile.visualization
106-
context: .
107-
- image: kfp-launcher
108-
dockerfile: backend/Dockerfile.launcher
109-
context: .
110-
- image: kfp-driver
111-
dockerfile: backend/Dockerfile.driver
112-
context: .
113-
- image: kfp-cache-deployer
114-
dockerfile: backend/src/cache/deployer/Dockerfile
115-
context: .
116-
- image: kfp-cache-server
117-
dockerfile: backend/Dockerfile.cacheserver
118-
context: .
119-
- image: kfp-metadata-writer
120-
dockerfile: backend/metadata_writer/Dockerfile
121-
context: .
122-
- image: kfp-metadata-envoy
123-
dockerfile: third_party/metadata_envoy/Dockerfile
124-
context: .
125-
- image: kfp-inverse-proxy-agent
126-
dockerfile: proxy/Dockerfile
127-
context: ./proxy
128123
steps:
129124
- name: Checkout repository
130125
uses: actions/checkout@v4
131126
with:
132127
ref: ${{env.SOURCE_BRANCH}}
128+
133129
- name: Log in to the Container registry
134130
uses: docker/login-action@v3
135131
with:
@@ -140,7 +136,7 @@ jobs:
140136
- name: Check if image tag already exists
141137
id: check_tag
142138
env:
143-
IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.image }}:${{env.TARGET_IMAGE_TAG}}
139+
IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ inputs.app_to_build }}:${{env.TARGET_IMAGE_TAG}}
144140
OVERWRITE: ${{ env.OVERWRITE_IMAGES }}
145141
run: |
146142
if docker manifest inspect ${IMAGE} > /dev/null 2>&1; then
@@ -164,7 +160,7 @@ jobs:
164160
uses: docker/metadata-action@v5
165161
if: steps.check_tag.outcome == 'success'
166162
with:
167-
images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.image }}
163+
images: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ inputs.app_to_build }}
168164
tags: |
169165
type=raw,value=${{env.TARGET_IMAGE_TAG}}
170166
type=raw,value=latest,enable=${{ env.SET_LATEST == 'true'}}
@@ -178,8 +174,8 @@ jobs:
178174
uses: docker/build-push-action@v6
179175
if: steps.check_tag.outcome == 'success'
180176
with:
181-
context: ${{ matrix.context }}
182-
file: ${{ matrix.dockerfile }}
177+
context: ${{ inputs.image_context }}
178+
file: ${{ inputs.docker_file }}
183179
push: true
184180
tags: ${{ steps.meta.outputs.tags }}
185181
labels: ${{ steps.meta.outputs.labels }}
@@ -193,6 +189,5 @@ jobs:
193189
uses: actions/attest-build-provenance@v1
194190
if: steps.check_tag.outcome == 'success'
195191
with:
196-
subject-name: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ matrix.image }}
197-
subject-digest: ${{ steps.push.outputs.digest }}
198-
192+
subject-name: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_ORG }}/${{ inputs.app_to_build }}
193+
subject-digest: ${{ steps.push.outputs.digest }}

0 commit comments

Comments
 (0)