Skip to content

Commit cc1e2f0

Browse files
swelbornswelborn
authored
Merge pull request #215 from NERSC/ci-build
[#214] add ci
2 parents 018dc6c + 3ca8b4f commit cc1e2f0

File tree

10 files changed

+316
-44
lines changed

10 files changed

+316
-44
lines changed

‎.github/workflows/bake.yml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
name: Docker Bake Build and Push
2+
# This is mostly taken from
3+
# here: https://docs.docker.com/build/ci/github-actions/multi-platform/#with-bake
4+
on:
5+
push:
6+
branches: [ main ]
7+
pull_request:
8+
branches: [ main ]
9+
workflow_dispatch:
10+
11+
env:
12+
REGISTRY: ghcr.io/nersc/interactem
13+
14+
jobs:
15+
prepare:
16+
runs-on: ubuntu-latest
17+
permissions:
18+
contents: read
19+
outputs:
20+
tag: ${{ steps.meta.outputs.tag }}
21+
services: ${{ steps.services.outputs.services }}
22+
all-images: ${{ steps.services.outputs.all-images }}
23+
steps:
24+
- name: Checkout
25+
uses: actions/checkout@v4
26+
with:
27+
submodules: false
28+
29+
- name: Set up Docker Buildx
30+
uses: docker/setup-buildx-action@v3
31+
32+
- name: Extract metadata
33+
id: meta
34+
run: |
35+
TAG=$(git rev-parse --short=6 HEAD)
36+
echo "tag=${TAG}" >> $GITHUB_OUTPUT
37+
echo "Generated tag: ${TAG}"
38+
39+
- name: Define services
40+
id: services
41+
run: |
42+
# Extract services from the prod group in docker-bake.hcl
43+
SERVICES=$(docker buildx bake prod --print | jq -cr '[.group.prod.targets[]]')
44+
echo "Services found: ${SERVICES}"
45+
echo "services=${SERVICES}" >> $GITHUB_OUTPUT
46+
47+
# Create combined list including base image for merge-manifests
48+
ALL_IMAGES=$(echo ${SERVICES} | jq -c '. + ["interactem"]')
49+
echo "All images (including base): ${ALL_IMAGES}"
50+
echo "all-images=${ALL_IMAGES}" >> $GITHUB_OUTPUT
51+
working-directory: ./docker
52+
53+
build-base:
54+
needs: prepare
55+
strategy:
56+
matrix:
57+
include:
58+
- platform: amd64
59+
runner: ubuntu-24.04
60+
docker-platform: linux/amd64
61+
- platform: arm64
62+
runner: ubuntu-24.04-arm
63+
docker-platform: linux/arm64
64+
runs-on: ${{ matrix.runner }}
65+
permissions:
66+
contents: read
67+
packages: write
68+
steps:
69+
- name: Checkout
70+
uses: actions/checkout@v4
71+
with:
72+
submodules: false
73+
74+
- name: Set up Docker Buildx
75+
uses: docker/setup-buildx-action@v3
76+
77+
- name: Login to GitHub Container Registry
78+
uses: docker/login-action@v3
79+
with:
80+
registry: ${{ env.REGISTRY }}
81+
username: ${{ github.actor }}
82+
password: ${{ secrets.GHCR_PAT }}
83+
84+
- name: Build and push base image by digest
85+
id: bake
86+
uses: docker/bake-action@v6
87+
with:
88+
source: .
89+
files: ./docker/docker-bake.hcl
90+
targets: base
91+
set: |
92+
*.platform=${{ matrix.docker-platform }}
93+
${{ github.event_name != 'pull_request' && '*.output=type=image,push-by-digest=true,name-canonical=true,push=true' || '' }}
94+
*.tags=${{ env.REGISTRY }}/interactem
95+
env:
96+
TAG: ${{ needs.prepare.outputs.tag }}
97+
CACHE_PLATFORM: ${{ matrix.platform }}
98+
99+
- name: Export digest
100+
if: github.event_name != 'pull_request'
101+
run: |
102+
mkdir -p /tmp/digests
103+
digest="${{ fromJSON(steps.bake.outputs.metadata).base['containerimage.digest'] }}"
104+
touch "/tmp/digests/${digest#sha256:}"
105+
106+
- name: Upload digest
107+
if: github.event_name != 'pull_request'
108+
uses: actions/upload-artifact@v4
109+
with:
110+
name: interactem-digest-${{ matrix.platform }}
111+
path: /tmp/digests/*
112+
if-no-files-found: error
113+
retention-days: 1
114+
115+
build-services:
116+
needs: [prepare, build-base]
117+
strategy:
118+
matrix:
119+
platform:
120+
- { name: amd64, runner: ubuntu-24.04, docker: linux/amd64 }
121+
- { name: arm64, runner: ubuntu-24.04-arm, docker: linux/arm64 }
122+
service: ${{ fromJson(needs.prepare.outputs.services) }}
123+
runs-on: ${{ matrix.platform.runner }}
124+
permissions:
125+
contents: read
126+
packages: write
127+
steps:
128+
- name: Checkout
129+
uses: actions/checkout@v4
130+
with:
131+
submodules: false
132+
133+
- name: Set up Docker Buildx
134+
uses: docker/setup-buildx-action@v3
135+
136+
- name: Login to GitHub Container Registry
137+
uses: docker/login-action@v3
138+
with:
139+
registry: ${{ env.REGISTRY }}
140+
username: ${{ github.actor }}
141+
password: ${{ secrets.GHCR_PAT }}
142+
143+
- name: Build and push ${{ matrix.service }} by digest
144+
id: bake
145+
uses: docker/bake-action@v6
146+
with:
147+
source: .
148+
files: ./docker/docker-bake.hcl
149+
targets: ${{ matrix.service }}
150+
set: |
151+
*.platform=${{ matrix.platform.docker }}
152+
${{ github.event_name != 'pull_request' && '*.output=type=image,push-by-digest=true,name-canonical=true,push=true' || '' }}
153+
*.tags=${{ env.REGISTRY }}/${{ matrix.service }}
154+
env:
155+
TAG: ${{ needs.prepare.outputs.tag }}
156+
CACHE_PLATFORM: ${{ matrix.platform.name }}
157+
158+
- name: Export digest
159+
if: github.event_name != 'pull_request'
160+
run: |
161+
mkdir -p /tmp/digests
162+
digest="${{ fromJSON(steps.bake.outputs.metadata)[matrix.service]['containerimage.digest'] }}"
163+
touch "/tmp/digests/${digest#sha256:}"
164+
165+
- name: Upload digest
166+
if: github.event_name != 'pull_request'
167+
uses: actions/upload-artifact@v4
168+
with:
169+
name: ${{ matrix.service }}-digest-${{ matrix.platform.name }}
170+
path: /tmp/digests/*
171+
if-no-files-found: error
172+
retention-days: 1
173+
174+
merge-manifests:
175+
if: github.event_name != 'pull_request'
176+
needs: [prepare, build-base, build-services]
177+
runs-on: ubuntu-latest
178+
permissions:
179+
contents: read
180+
packages: write
181+
strategy:
182+
matrix:
183+
service: ${{ fromJson(needs.prepare.outputs.all-images) }}
184+
steps:
185+
- name: Set up Docker Buildx
186+
uses: docker/setup-buildx-action@v3
187+
188+
- name: Login to GitHub Container Registry
189+
uses: docker/login-action@v3
190+
with:
191+
registry: ${{ env.REGISTRY }}
192+
username: ${{ github.actor }}
193+
password: ${{ secrets.GHCR_PAT }}
194+
195+
- name: Download digests
196+
uses: actions/download-artifact@v4
197+
with:
198+
path: /tmp/digests
199+
pattern: ${{ matrix.service }}-digest-*
200+
merge-multiple: true
201+
202+
- name: Create manifest list and push
203+
working-directory: /tmp/digests
204+
run: |
205+
# Create manifest
206+
docker buildx imagetools create \
207+
-t ${{ env.REGISTRY }}/${{ matrix.service }}:${{ needs.prepare.outputs.tag }} \
208+
-t ${{ env.REGISTRY }}/${{ matrix.service }}:latest \
209+
$(printf '${{ env.REGISTRY }}/${{ matrix.service }}@sha256:%s ' *)
210+
211+
- name: Inspect image
212+
run: |
213+
docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ matrix.service }}:${{ needs.prepare.outputs.tag }}

‎docker-compose.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ x-backend-base:
66
args:
77
INSTALL_DEV: ${INSTALL_DEV-false}
88
dockerfile: ../docker/Dockerfile.fastapi
9+
additional_contexts:
10+
- interactem-base=docker-image://ghcr.io/nersc/interactem/interactem
911
env_file:
1012
- .env
1113
environment:
@@ -141,6 +143,8 @@ services:
141143

142144
build:
143145
context: backend/
146+
additional_contexts:
147+
- interactem-base=docker-image://ghcr.io/nersc/interactem/interactem
144148
dockerfile: ../docker/Dockerfile.orchestrator
145149

146150

@@ -180,6 +184,8 @@ services:
180184
- "${HOME}/.superfacility/key.pem:/key.pem"
181185
build:
182186
context: backend/
187+
additional_contexts:
188+
- interactem-base=docker-image://ghcr.io/nersc/interactem/interactem
183189
dockerfile: ../docker/Dockerfile.launcher
184190

185191
metrics:
@@ -202,6 +208,8 @@ services:
202208

203209
build:
204210
context: backend/
211+
additional_contexts:
212+
- interactem-base=docker-image://ghcr.io/nersc/interactem/interactem
205213
dockerfile: ../docker/Dockerfile.metrics
206214

207215
prometheus:

‎docker/Dockerfile.fastapi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ghcr.io/nersc/interactem/interactem:latest
1+
FROM interactem-base
22

33
WORKDIR /interactem/app
44
COPY ./app/pyproject.toml ./app/poetry.lock* ./

‎docker/Dockerfile.launcher‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ghcr.io/nersc/interactem/interactem:latest
1+
FROM interactem-base
22

33
COPY ./sfapi_models/ /interactem/sfapi_models/
44

‎docker/Dockerfile.metrics‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ghcr.io/nersc/interactem/interactem:latest
1+
FROM interactem-base
22

33
WORKDIR /interactem/metrics/
44

‎docker/Dockerfile.operator‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ghcr.io/nersc/interactem/interactem:latest
1+
FROM interactem-base
22

33
# --- Install interactem operators deps ---
44
WORKDIR /interactem/operators

‎docker/Dockerfile.orchestrator‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ghcr.io/nersc/interactem/interactem:latest
1+
FROM interactem-base
22

33
WORKDIR /interactem/orchestrator/
44
COPY ./orchestrator/README.md ./orchestrator/pyproject.toml ./orchestrator/poetry.lock* ./

‎docker/bake.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ cd $ROOT_DIR
99

1010
# Build all images using Docker Bake
1111
echo "Building all images with tag $TAG..."
12-
TAG=${TAG} BUILDX_BAKE_ENTITLEMENTS_FS=0 $DOCKER buildx bake --file $ROOT_DIR/docker/docker-bake.hcl base
13-
TAG=${TAG} BUILDX_BAKE_ENTITLEMENTS_FS=0 $DOCKER buildx bake --file $ROOT_DIR/docker/docker-bake.hcl prod
12+
TAG=${TAG} BUILDX_BAKE_ENTITLEMENTS_FS=0 $DOCKER buildx bake \
13+
--set *.cache-from="" --set *.cache-to="" \
14+
--file $ROOT_DIR/docker/docker-bake.hcl prod
1415

1516
if [ $? -ne 0 ]; then
1617
echo "Failed to build images"

0 commit comments

Comments
 (0)