Skip to content

Commit 34c47ac

Browse files
andhreljaKernandhreljaKern
committed
perf: image cleanup
1 parent a0d7dfa commit 34c47ac

File tree

4 files changed

+57
-48
lines changed

4 files changed

+57
-48
lines changed

‎.github/workflows/pi_build.yml‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ jobs:
3131
with:
3232
checkout_ref: parent-image-updates
3333
repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
34+
edit_dockerfile: true
3435

3536
pi-build:
3637
name: 'Parent Images: Docker Build'
@@ -39,9 +40,9 @@ jobs:
3940
environment: dev
4041
env:
4142
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
42-
DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
43-
DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
44-
DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
43+
DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
44+
DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
45+
DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
4546
PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }}
4647
DOCKERFILE: ${{ vars.DOCKERFILE }}
4748
HEAD_REF: parent-image-updates
@@ -83,22 +84,22 @@ jobs:
8384
with:
8485
platforms: arm64,arm
8586

86-
- name: Log into DockerHub registry
87+
- name: Log into DEV registry
8788
uses: docker/login-action@v3
8889
with:
89-
# registry: "${{ env.DOCKERHUB_CONTAINER_REGISTRY }}"
90-
username: "${{ env.DOCKERHUB_LOGIN_USERNAME }}"
91-
password: "${{ env.DOCKERHUB_LOGIN_PASSWORD }}"
90+
registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
91+
username: "${{ env.DEV_LOGIN_USERNAME }}"
92+
password: "${{ env.DEV_LOGIN_PASSWORD }}"
9293

9394
- name: Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}
9495
uses: docker/build-push-action@v5
9596
with:
9697
context: .
97-
cache-from: type=registry,ref=${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-cache
98-
cache-to: type=registry,ref=${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
98+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:dev-${{ matrix.parent_image_type }}-cache
99+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
99100
platforms: linux/amd64
100101
file: ${{ env.DOCKERFILE }}
101-
tags: ${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}
102+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}
102103
push: true
103104
build-args: |
104105
platform=linux/amd64
@@ -108,11 +109,11 @@ jobs:
108109
uses: docker/build-push-action@v5
109110
with:
110111
context: .
111-
cache-from: type=registry,ref=${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64-cache
112-
cache-to: type=registry,ref=${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
112+
cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:dev-${{ matrix.parent_image_type }}-arm64-cache
113+
cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
113114
platforms: linux/arm64
114115
file: ${{ env.DOCKERFILE }}
115-
tags: ${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64
116+
tags: ${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64
116117
push: true
117118
build-args: |
118119
platform=linux/arm64
@@ -127,6 +128,7 @@ jobs:
127128
env:
128129
PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
129130
DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
131+
DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
130132
PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }}
131133
DOCKERFILE: ${{ vars.DOCKERFILE }}
132134
HEAD_REF: parent-image-updates
@@ -180,13 +182,15 @@ jobs:
180182
-i "${{ env.PARENT_IMAGE_NAME }}" \
181183
-l "${{ env.HEAD_REF }}" \
182184
-r "${{ env.DOCKERHUB_CONTAINER_REGISTRY }}" \
185+
-e "${{ env.DEV_CONTAINER_REGISTRY }}" \
183186
-d "${{ env.DOCKERFILE }}"
184187
185188
if [ -f "dev.${{ env.DOCKERFILE }}" ]; then
186189
bash ${{ github.workspace }}/cicd-deployment-scripts/pi/edit_dockerfile.sh \
187190
-i "${{ env.PARENT_IMAGE_NAME }}" \
188191
-l "${{ env.HEAD_REF }}" \
189192
-r "${{ env.DOCKERHUB_CONTAINER_REGISTRY }}" \
193+
-e "${{ env.DEV_CONTAINER_REGISTRY }}" \
190194
-d "dev.${{ env.DOCKERFILE }}"
191195
fi
192196
@@ -195,14 +199,14 @@ jobs:
195199
-i "${{ env.PARENT_IMAGE_NAME }}" \
196200
-l "${{ env.HEAD_REF }}" \
197201
-r "${{ env.DOCKERHUB_CONTAINER_REGISTRY }}" \
202+
-e "${{ env.DEV_CONTAINER_REGISTRY }}" \
198203
-d "gpu.${{ env.DOCKERFILE }}"
199204
fi
200205
201206
git add requirements/*.txt ${{ env.DOCKERFILE }}
202207
git add dev.${{ env.DOCKERFILE }} || true
203208
git add gpu.${{ env.DOCKERFILE }} || true
204209
git commit -m "ci: update requirements and Dockerfile" || true
205-
git push origin ${{ env.HEAD_REF }} || true
206210
echo "::notice::${{ matrix.app }} - updated requirement files and ${{ env.DOCKERFILE }}s"
207211
208212
rm -f requirements.txt gpu-requirements.txt || true
@@ -214,6 +218,7 @@ jobs:
214218
215219
if [ $exitcode -ne 0 ]; then
216220
echo "::error::pip-compile failed with exit code $exitcode"
221+
git push origin ${{ env.HEAD_REF }} || true
217222
exit $exitcode
218223
fi
219224

‎.github/workflows/pi_merge_submodule.yml‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,24 @@ jobs:
8181
--head ${{ needs.configure-branch-name.outputs.gh_head_ref }} \
8282
--repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image || true
8383
84+
call-dc-registry-delete:
85+
name: 'Docker: Delete Branch Images'
86+
needs: [pi-matrix, pi-update-submodule, configure-branch-name]
87+
if: ${{ !failure() }}
88+
uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/dc_registry_delete.yml@dev
89+
secrets: inherit
90+
strategy:
91+
matrix:
92+
parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
93+
with:
94+
delete_tag: ${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}
95+
app_name: ${{ vars.PARENT_IMAGE_NAME}}
96+
8497
gh-delete-submodule-branches:
8598
name: 'GitHub: Delete Submodule Branch'
86-
needs: [pi-matrix, pi-update-submodule]
99+
needs: [call-dc-registry-delete]
87100
if: ${{ !failure() }}
88101
runs-on: ubuntu-latest
89-
# strategy:
90-
# matrix:
91-
# parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
92102
steps:
93103
- name: Checkout repository
94104
uses: actions/checkout@v4
@@ -97,23 +107,4 @@ jobs:
97107

98108
- name: Delete Branch
99109
shell: bash
100-
run: git push origin --delete ${{ github.event.pull_request.head.ref }}
101-
102-
# gh-delete-app-branches:
103-
# name: 'GitHub: Delete App Branch'
104-
# needs: [configure-branch-name, pi-matrix, gh-delete-submodule-branches]
105-
# if: ${{ !failure() }}
106-
# runs-on: ubuntu-latest
107-
# strategy:
108-
# matrix:
109-
# include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
110-
# steps:
111-
# - name: Checkout repository
112-
# uses: actions/checkout@v4
113-
# with:
114-
# token: ${{ secrets.GH_TOKEN }}
115-
# repository: ${{ github.repository_owner }}/${{ matrix.app }}
116-
117-
# - name: Delete Branch
118-
# shell: bash
119-
# run: git push origin --delete ${{ needs.configure-branch-name.outputs.gh_head_ref }} || true
110+
run: git push origin --delete ${{ github.event.pull_request.head.ref }}

‎pi/edit_dockerfile.sh‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,38 +5,51 @@ set -e
55
PARENT_IMAGE_NAME="refinery-parent-images"
66
RELEASE_TAG="parent-image-updates"
77
DOCKER_REGISTRY="kernai"
8+
DEV_REGISTRY="registry.dev.kern.ai/code-kern-ai"
89
DOCKERFILE="Dockerfile"
910
HEAD_REF=""
1011

11-
while getopts i:l:r:d:h: flag
12+
while getopts i:l:r:e:d:h: flag
1213
do
1314
case "${flag}" in
1415
i) PARENT_IMAGE_NAME=${OPTARG};;
1516
l) RELEASE_TAG=${OPTARG};;
1617
r) DOCKER_REGISTRY=${OPTARG};;
18+
e) DEV_REGISTRY=${OPTARG};;
1719
d) DOCKERFILE=${OPTARG};;
1820
h) HEAD_REF=${OPTARG};;
1921
esac
2022
done
2123

22-
grep "${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}" $DOCKERFILE | while read -r line ; do
24+
if [ -z "$HEAD_REF" ]; then
25+
REGISTRY=$DOCKER_REGISTRY
26+
else
27+
REGISTRY=$DEV_REGISTRY
28+
fi
29+
30+
grep "${REGISTRY}/${PARENT_IMAGE_NAME}" $DOCKERFILE | while read -r line ; do
2331
PI_EXISTING_TAG=$(echo $line | sed 's|FROM ||g' | cut -d ':' -f 2)
2432
ALREADY_UPDATED=$(echo $PI_EXISTING_TAG | grep $RELEASE_TAG || true)
2533

26-
if [ -z $ALREADY_UPDATED ] && [ -z $HEAD_REF ]; then
34+
if [ -z "$ALREADY_UPDATED" ] && [ -z "$HEAD_REF" ]; then
2735
image_version=$(echo $PI_EXISTING_TAG | cut -d '-' -f 1)
2836
PARENT_IMAGE_TYPE=$(echo $PI_EXISTING_TAG | sed "s|${image_version}-||g")
29-
elif [ -n $ALREADY_UPDATED ] && [ -z $HEAD_REF ]; then
37+
elif [ -n "$ALREADY_UPDATED" ] && [ -z "$HEAD_REF" ]; then
3038
PARENT_IMAGE_TYPE=$(echo $PI_EXISTING_TAG | sed "s|${RELEASE_TAG}-||g")
31-
elif [ -n $HEAD_REF ]; then
39+
elif [ -n "$HEAD_REF" ]; then
3240
PARENT_IMAGE_TYPE=$(echo $PI_EXISTING_TAG | sed "s|${HEAD_REF}-||g")
3341
else
3442
echo "::error::Failed to determine parent image type from tag: ${PI_EXISTING_TAG}"
3543
exit 1
3644
fi
3745

38-
PI_EXISTING_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${PI_EXISTING_TAG}"
39-
PI_NEW_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${RELEASE_TAG}-${PARENT_IMAGE_TYPE}"
46+
if [ -z "$HEAD_REF" ]; then
47+
PI_EXISTING_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${PI_EXISTING_TAG}"
48+
PI_NEW_IMAGE="${DEV_REGISTRY}/${PARENT_IMAGE_NAME}:${RELEASE_TAG}-${PARENT_IMAGE_TYPE}"
49+
else
50+
PI_EXISTING_IMAGE="${DEV_REGISTRY}/${PARENT_IMAGE_NAME}:${PI_EXISTING_TAG}"
51+
PI_NEW_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${RELEASE_TAG}-${PARENT_IMAGE_TYPE}"
52+
fi
4053

4154
sed "s|${PI_EXISTING_IMAGE}|${PI_NEW_IMAGE}|g" ${DOCKERFILE} > ${DOCKERFILE}.tmp && mv ${DOCKERFILE}.tmp ${DOCKERFILE}
4255
echo "::notice::${DOCKERFILE} updated with new image: ${PI_NEW_IMAGE}"

‎pi/matrix.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44

5-
PR_NUMBER="38"
5+
PR_NUMBER=""
66
SOURCE_SCRIPT="pi/constants.sh"
77
PARENT_IMAGE_TYPE=""
88
EDIT_DOCKERFILE=false
@@ -21,7 +21,7 @@ source $SOURCE_SCRIPT
2121

2222
UPDATED_PARENT_TYPES=()
2323

24-
if [ -n $PR_NUMBER ] && [ -z $PARENT_IMAGE_TYPE ]; then
24+
if [ -n "$PR_NUMBER" ] && [ -z "$PARENT_IMAGE_TYPE" ]; then
2525
UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only --repo code-kern-ai/refinery-submodule-parent-images)
2626
while IFS= read -r file; do
2727
if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
@@ -34,7 +34,7 @@ if [ -n $PR_NUMBER ] && [ -z $PARENT_IMAGE_TYPE ]; then
3434
done <<< "$UPDATED_FILES"
3535
# TODO: UPDATED_PARENT_TYPES are not resolved correctly
3636
echo -e "::notice::Exporting matrix for parent image types: $UPDATED_PARENT_TYPES"
37-
elif [ -n $PARENT_IMAGE_TYPE ]; then
37+
elif [ -n "$PARENT_IMAGE_TYPE" ]; then
3838
echo "::notice::Exporting matrix for parent image type: $PARENT_IMAGE_TYPE"
3939
UPDATED_PARENT_TYPES=( $PARENT_IMAGE_TYPE )
4040
fi

0 commit comments

Comments
 (0)