Skip to content

Commit 48e1bc2

Browse files
Aias00Aias00
committed
chore: optimize Docker CI workflow for faster builds and multi-architecture support
1 parent 0d8fd57 commit 48e1bc2

File tree

2 files changed

+96
-56
lines changed

2 files changed

+96
-56
lines changed

‎.github/workflows/docker-publish.yml‎

Lines changed: 91 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -14,67 +14,42 @@ on:
1414
required: false
1515
type: boolean
1616
default: false
17+
skip_multiarch:
18+
description: 'Skip multi-architecture build for faster CI'
19+
required: false
20+
type: boolean
21+
default: false
1722
push:
1823
branches: [ "main" ]
1924

2025
jobs:
21-
build_and_push_extproc:
26+
# Parallel job for building both images
27+
build_and_push:
2228
runs-on: ubuntu-latest
2329
permissions:
2430
contents: read
2531
packages: write
32+
strategy:
33+
matrix:
34+
image: [extproc, llm-katan]
35+
fail-fast: false # Continue building other images if one fails
2636

2737
steps:
2838
- name: Check out the repo
2939
uses: actions/checkout@v4
3040

3141
- name: Set up Docker Buildx
3242
uses: docker/setup-buildx-action@v3
33-
34-
- name: Set up QEMU
35-
uses: docker/setup-qemu-action@v3
36-
37-
- name: Log in to GitHub Container Registry
38-
uses: docker/login-action@v3
3943
with:
40-
registry: ghcr.io
41-
username: ${{ github.actor }}
42-
password: ${{ secrets.GITHUB_TOKEN }}
43-
44-
- name: Generate date tag for nightly builds
45-
id: date
46-
if: inputs.is_nightly == true
47-
run: echo "date_tag=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
44+
driver-opts: |
45+
network=host
46+
image=moby/buildkit:v0.12.5
4847
49-
- name: Set lowercase repository owner
50-
run: echo "REPOSITORY_OWNER_LOWER=$(echo $GITHUB_REPOSITORY_OWNER | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
51-
52-
- name: Build and push extproc Docker image
53-
uses: docker/build-push-action@v5
54-
with:
55-
context: .
56-
file: ./Dockerfile.extproc
57-
platforms: linux/amd64,linux/arm64
58-
push: ${{ github.event_name != 'pull_request' }} # Only push on merge to main, not on PRs
59-
tags: |
60-
${{ inputs.is_nightly == true && format('ghcr.io/{0}/semantic-router/extproc:nightly-{1}', env.REPOSITORY_OWNER_LOWER, steps.date.outputs.date_tag) || format('ghcr.io/{0}/semantic-router/extproc:{1}', env.REPOSITORY_OWNER_LOWER, github.sha) }}
61-
${{ inputs.is_nightly != true && format('ghcr.io/{0}/semantic-router/extproc:latest', env.REPOSITORY_OWNER_LOWER) || '' }}
62-
63-
build_and_push_llm_katan:
64-
runs-on: ubuntu-latest
65-
permissions:
66-
contents: read
67-
packages: write
68-
69-
steps:
70-
- name: Check out the repo
71-
uses: actions/checkout@v4
72-
73-
- name: Set up Docker Buildx
74-
uses: docker/setup-buildx-action@v3
75-
76-
- name: Set up QEMU
48+
- name: Set up QEMU (only for multi-arch builds)
49+
if: inputs.skip_multiarch != true
7750
uses: docker/setup-qemu-action@v3
51+
with:
52+
platforms: arm64
7853

7954
- name: Log in to GitHub Container Registry
8055
uses: docker/login-action@v3
@@ -91,20 +66,84 @@ jobs:
9166
- name: Set lowercase repository owner
9267
run: echo "REPOSITORY_OWNER_LOWER=$(echo $GITHUB_REPOSITORY_OWNER | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
9368

69+
# Rust build cache for extproc
70+
- name: Cache Rust dependencies (extproc only)
71+
if: matrix.image == 'extproc'
72+
uses: actions/cache@v4
73+
with:
74+
path: |
75+
~/.cargo/bin/
76+
~/.cargo/registry/index/
77+
~/.cargo/registry/cache/
78+
~/.cargo/git/db/
79+
candle-binding/target/
80+
key: ${{ runner.os }}-cargo-extproc-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
81+
restore-keys: |
82+
${{ runner.os }}-cargo-extproc-
83+
84+
# Set build context and dockerfile based on matrix
85+
- name: Set build parameters
86+
id: build-params
87+
run: |
88+
if [ "${{ matrix.image }}" = "extproc" ]; then
89+
echo "context=." >> $GITHUB_OUTPUT
90+
echo "dockerfile=./Dockerfile.extproc" >> $GITHUB_OUTPUT
91+
echo "platforms=${{ inputs.skip_multiarch == true && 'linux/amd64' || 'linux/amd64,linux/arm64' }}" >> $GITHUB_OUTPUT
92+
elif [ "${{ matrix.image }}" = "llm-katan" ]; then
93+
echo "context=./e2e-tests/llm-katan" >> $GITHUB_OUTPUT
94+
echo "dockerfile=./e2e-tests/llm-katan/Dockerfile" >> $GITHUB_OUTPUT
95+
echo "platforms=${{ inputs.skip_multiarch == true && 'linux/amd64' || 'linux/amd64,linux/arm64' }}" >> $GITHUB_OUTPUT
96+
fi
97+
98+
# Extract version for llm-katan
9499
- name: Extract version from pyproject.toml
95100
id: version
101+
if: matrix.image == 'llm-katan'
96102
run: |
97103
VERSION=$(grep '^version = ' e2e-tests/llm-katan/pyproject.toml | sed 's/version = "\(.*\)"/\1/')
98104
echo "version=$VERSION" >> $GITHUB_OUTPUT
99105
100-
- name: Build and push llm-katan Docker image
106+
# Generate tags for extproc
107+
- name: Generate extproc tags
108+
id: extproc-tags
109+
if: matrix.image == 'extproc'
110+
run: |
111+
REPO_LOWER=$(echo $GITHUB_REPOSITORY_OWNER | tr '[:upper:]' '[:lower:]')
112+
if [ "${{ inputs.is_nightly }}" = "true" ]; then
113+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/extproc:nightly-${{ steps.date.outputs.date_tag }}" >> $GITHUB_OUTPUT
114+
else
115+
if [ "${{ github.event_name }}" != "pull_request" ]; then
116+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/extproc:${{ github.sha }},ghcr.io/${REPO_LOWER}/semantic-router/extproc:latest" >> $GITHUB_OUTPUT
117+
else
118+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/extproc:${{ github.sha }}" >> $GITHUB_OUTPUT
119+
fi
120+
fi
121+
122+
# Generate tags for llm-katan
123+
- name: Generate llm-katan tags
124+
id: llm-katan-tags
125+
if: matrix.image == 'llm-katan'
126+
run: |
127+
REPO_LOWER=$(echo $GITHUB_REPOSITORY_OWNER | tr '[:upper:]' '[:lower:]')
128+
if [ "${{ inputs.is_nightly }}" = "true" ]; then
129+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/llm-katan:nightly-${{ steps.date.outputs.date_tag }}" >> $GITHUB_OUTPUT
130+
else
131+
if [ "${{ github.event_name }}" != "pull_request" ]; then
132+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/llm-katan:${{ github.sha }},ghcr.io/${REPO_LOWER}/semantic-router/llm-katan:latest,ghcr.io/${REPO_LOWER}/semantic-router/llm-katan:v${{ steps.version.outputs.version }}" >> $GITHUB_OUTPUT
133+
else
134+
echo "tags=ghcr.io/${REPO_LOWER}/semantic-router/llm-katan:${{ github.sha }}" >> $GITHUB_OUTPUT
135+
fi
136+
fi
137+
138+
- name: Build and push ${{ matrix.image }} Docker image
101139
uses: docker/build-push-action@v5
102140
with:
103-
context: ./e2e-tests/llm-katan
104-
file: ./e2e-tests/llm-katan/Dockerfile
105-
platforms: linux/amd64,linux/arm64
106-
push: ${{ github.event_name != 'pull_request' }} # Only push on merge to main, not on PRs
107-
tags: |
108-
${{ inputs.is_nightly == true && format('ghcr.io/{0}/semantic-router/llm-katan:nightly-{1}', env.REPOSITORY_OWNER_LOWER, steps.date.outputs.date_tag) || format('ghcr.io/{0}/semantic-router/llm-katan:{1}', env.REPOSITORY_OWNER_LOWER, github.sha) }}
109-
${{ inputs.is_nightly != true && format('ghcr.io/{0}/semantic-router/llm-katan:latest', env.REPOSITORY_OWNER_LOWER) || '' }}
110-
${{ inputs.is_nightly != true && format('ghcr.io/{0}/semantic-router/llm-katan:v{1}', env.REPOSITORY_OWNER_LOWER, steps.version.outputs.version) || '' }}
141+
context: ${{ steps.build-params.outputs.context }}
142+
file: ${{ steps.build-params.outputs.dockerfile }}
143+
platforms: ${{ steps.build-params.outputs.platforms }}
144+
push: ${{ github.event_name != 'pull_request' }}
145+
cache-from: type=gha,scope=${{ matrix.image }}
146+
cache-to: type=gha,mode=max,scope=${{ matrix.image }}
147+
tags: ${{ matrix.image == 'extproc' && steps.extproc-tags.outputs.tags || steps.llm-katan-tags.outputs.tags }}
148+
build-args: |
149+
BUILDKIT_INLINE_CACHE=1

‎.github/workflows/test-and-build.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,12 +103,13 @@ jobs:
103103
run: |
104104
echo "::error::Test and build failed. Check the workflow run for details."
105105
106-
# Trigger Docker publishing on successful nightly runs
106+
# Trigger Docker publishing on successful runs
107107
publish-docker:
108108
needs: test-and-build
109-
if: success() && github.event_name == 'schedule'
109+
if: success()
110110
uses: ./.github/workflows/docker-publish.yml
111111
with:
112-
tag_suffix: nightly-$(date +'%Y%m%d')
113-
is_nightly: true
112+
tag_suffix: ${{ github.event_name == 'schedule' && format('nightly-{0}', github.run_id) || '' }}
113+
is_nightly: ${{ github.event_name == 'schedule' }}
114+
skip_multiarch: ${{ github.event_name == 'pull_request' }}
114115
secrets: inherit

0 commit comments

Comments
 (0)