Skip to content

Commit f418273

Browse files
nhynesnhynes
authored
Support mixed-site scenarios (e.g., VM+docker) (#72)
1 parent 2f9f47f commit f418273

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+19148
-620
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 225 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
- main
1010

1111
concurrency:
12-
group: ${{ github.workflow }}-${{ github.ref }}
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
1313
cancel-in-progress: true
1414

1515
jobs:
@@ -184,7 +184,7 @@ jobs:
184184
name: Kubernetes Reporter Tests
185185
runs-on: ubuntu-latest
186186
if: github.event_name == 'pull_request'
187-
needs: docker-merge
187+
needs: docker-build
188188
permissions:
189189
contents: read
190190
packages: read
@@ -261,6 +261,225 @@ jobs:
261261
kind delete cluster --name "${AMBER_TEST_KIND_CLUSTER_NAME}" --kubeconfig "${AMBER_TEST_KIND_KUBECONFIG}"
262262
fi
263263
264+
mixed-site-tests:
265+
name: Mixed-Site E2E Tests
266+
runs-on: ubuntu-latest
267+
timeout-minutes: 120
268+
needs: docker-build
269+
if: ${{ always() && (github.event_name != 'pull_request' || needs.docker-build.result == 'success') }}
270+
permissions:
271+
contents: read
272+
packages: read
273+
env:
274+
SCCACHE_GHA_ENABLED: "true"
275+
RUSTC_WRAPPER: "sccache"
276+
AMBER_VM_FORCE_TCG: "1"
277+
steps:
278+
- uses: actions/checkout@v5
279+
- uses: dtolnay/rust-toolchain@stable
280+
- uses: mozilla-actions/sccache-action@v0.0.9
281+
- name: Configure mixed-site VM image path
282+
run: |
283+
set -euo pipefail
284+
case "$(uname -m)" in
285+
x86_64)
286+
echo "AMBER_MIXED_RUN_BASE_IMAGE=${RUNNER_TEMP}/ubuntu-24.04-minimal-cloudimg-amd64.img" >> "$GITHUB_ENV"
287+
echo "AMBER_MIXED_RUN_QEMU_PACKAGES=qemu-system-x86 qemu-utils xorriso" >> "$GITHUB_ENV"
288+
echo "AMBER_MIXED_RUN_IMAGE_URL=https://cloud-images.ubuntu.com/minimal/releases/noble/release-20240709/ubuntu-24.04-minimal-cloudimg-amd64.img" >> "$GITHUB_ENV"
289+
;;
290+
aarch64|arm64)
291+
echo "AMBER_MIXED_RUN_BASE_IMAGE=${RUNNER_TEMP}/ubuntu-24.04-minimal-cloudimg-arm64.img" >> "$GITHUB_ENV"
292+
echo "AMBER_MIXED_RUN_QEMU_PACKAGES=qemu-system-arm qemu-utils qemu-efi-aarch64 xorriso" >> "$GITHUB_ENV"
293+
echo "AMBER_MIXED_RUN_IMAGE_URL=https://cloud-images.ubuntu.com/minimal/releases/noble/release-20240709/ubuntu-24.04-minimal-cloudimg-arm64.img" >> "$GITHUB_ENV"
294+
;;
295+
*)
296+
echo "unsupported mixed-site runner architecture: $(uname -m)" >&2
297+
exit 1
298+
;;
299+
esac
300+
- name: Relax AppArmor user namespace restrictions when present
301+
run: |
302+
set -euo pipefail
303+
304+
if sysctl kernel.apparmor_restrict_unprivileged_unconfined >/dev/null 2>&1; then
305+
sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
306+
fi
307+
if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
308+
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
309+
fi
310+
- name: Ensure bubblewrap is available and usable
311+
run: |
312+
set -euo pipefail
313+
probe_bwrap() {
314+
bwrap \
315+
--die-with-parent \
316+
--new-session \
317+
--unshare-pid \
318+
--unshare-ipc \
319+
--unshare-net \
320+
--unshare-uts \
321+
--ro-bind / / \
322+
--proc /proc \
323+
--dev /dev \
324+
--tmpfs /tmp \
325+
--tmpfs /run \
326+
-- /usr/bin/true >/dev/null 2>&1
327+
}
328+
probe_bwrap_verbose() {
329+
bwrap \
330+
--die-with-parent \
331+
--new-session \
332+
--unshare-pid \
333+
--unshare-ipc \
334+
--unshare-net \
335+
--unshare-uts \
336+
--ro-bind / / \
337+
--proc /proc \
338+
--dev /dev \
339+
--tmpfs /tmp \
340+
--tmpfs /run \
341+
-- /usr/bin/true
342+
}
343+
344+
if ! command -v bwrap >/dev/null 2>&1 || ! probe_bwrap; then
345+
sudo apt-get update
346+
sudo apt-get install -y --reinstall bubblewrap
347+
fi
348+
349+
if ! probe_bwrap; then
350+
echo "bubblewrap is installed but unusable on this runner" >&2
351+
bwrap --version || true
352+
ls -l "$(command -v bwrap)" || true
353+
sysctl kernel.apparmor_restrict_unprivileged_unconfined || true
354+
sysctl kernel.apparmor_restrict_unprivileged_userns || true
355+
probe_bwrap_verbose || true
356+
exit 1
357+
fi
358+
- name: Ensure slirp4netns is available
359+
run: |
360+
set -euo pipefail
361+
362+
if ! command -v slirp4netns >/dev/null 2>&1; then
363+
sudo apt-get update
364+
sudo apt-get install -y slirp4netns
365+
fi
366+
367+
slirp4netns --version
368+
- name: Ensure mixed-site VM runtime packages are available
369+
run: |
370+
set -euo pipefail
371+
sudo apt-get update
372+
sudo apt-get install -y ${AMBER_MIXED_RUN_QEMU_PACKAGES}
373+
- name: Fetch Ubuntu minimal cloud image
374+
run: |
375+
set -euo pipefail
376+
if [ ! -f "$AMBER_MIXED_RUN_BASE_IMAGE" ]; then
377+
curl --fail --show-error --silent --location \
378+
--retry 5 \
379+
--retry-all-errors \
380+
--output "$AMBER_MIXED_RUN_BASE_IMAGE" \
381+
"$AMBER_MIXED_RUN_IMAGE_URL"
382+
fi
383+
qemu-img info "$AMBER_MIXED_RUN_BASE_IMAGE"
384+
- name: Ensure KinD is available
385+
run: |
386+
if ! command -v kind >/dev/null 2>&1; then
387+
curl -fsSL -o /tmp/kind https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64
388+
chmod +x /tmp/kind
389+
sudo mv /tmp/kind /usr/local/bin/kind
390+
fi
391+
- name: Ensure kubectl is available
392+
run: |
393+
if ! command -v kubectl >/dev/null 2>&1; then
394+
curl -fsSL -o /tmp/kubectl "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
395+
chmod +x /tmp/kubectl
396+
sudo mv /tmp/kubectl /usr/local/bin/kubectl
397+
fi
398+
- name: Create shared KinD cluster
399+
shell: bash
400+
run: |
401+
set -euo pipefail
402+
cluster_name="amber-mixed-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
403+
kubeconfig="${RUNNER_TEMP}/amber-mixed-kind-kubeconfig"
404+
kind create cluster --name "$cluster_name" --kubeconfig "$kubeconfig" --wait 120s
405+
echo "AMBER_TEST_KIND_CLUSTER_NAME=${cluster_name}" >> "$GITHUB_ENV"
406+
echo "AMBER_TEST_KIND_KUBECONFIG=${kubeconfig}" >> "$GITHUB_ENV"
407+
- name: Compute image registries
408+
if: github.event_name == 'pull_request'
409+
id: registry
410+
shell: bash
411+
run: |
412+
set -euo pipefail
413+
owner_lc="$(printf '%s' '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')"
414+
ci_registry="ghcr.io/${owner_lc}"
415+
code_registry="$(jq -r '.registry' docker/images.json)"
416+
if [ -z "$code_registry" ] || [ "$code_registry" = "null" ]; then
417+
echo "docker/images.json registry is missing" >&2
418+
exit 1
419+
fi
420+
code_registry="${code_registry%/}"
421+
echo "CI_IMAGE_REGISTRY=${ci_registry}" >> "$GITHUB_ENV"
422+
echo "CODE_IMAGE_REGISTRY=${code_registry}" >> "$GITHUB_ENV"
423+
- name: Compute semver tag metadata
424+
if: github.event_name == 'pull_request'
425+
id: version_tags
426+
shell: bash
427+
run: |
428+
set -euo pipefail
429+
json="$(cargo run -q -p amber-images --bin version_tags -- docker/images.json)"
430+
{
431+
echo "json<<EOF"
432+
echo "$json"
433+
echo "EOF"
434+
} >> "$GITHUB_OUTPUT"
435+
- uses: docker/login-action@v3
436+
if: github.event_name == 'pull_request'
437+
with:
438+
registry: ghcr.io
439+
username: ${{ github.actor }}
440+
password: ${{ secrets.GITHUB_TOKEN }}
441+
- name: Pull and retag images for mixed-site tests
442+
if: github.event_name == 'pull_request'
443+
shell: bash
444+
run: |
445+
set -euo pipefail
446+
version_tags_json='${{ steps.version_tags.outputs.json }}'
447+
while IFS= read -r spec; do
448+
[ -z "$spec" ] && continue
449+
name="$(jq -r '.name' <<< "$spec")"
450+
version="$(jq -r '.version' <<< "$spec")"
451+
runtime_tag="$(jq -r '.runtime_tag' <<< "$spec")"
452+
src="${CI_IMAGE_REGISTRY}/${name}:${GITHUB_SHA}"
453+
docker pull "$src"
454+
docker tag "$src" "${CODE_IMAGE_REGISTRY}/${name}:${version}"
455+
if [ "$runtime_tag" != "$version" ]; then
456+
docker tag "$src" "${CODE_IMAGE_REGISTRY}/${name}:${runtime_tag}"
457+
fi
458+
if [ "$name" = "amber-helper" ]; then
459+
docker tag "$src" "amber-helper:e2e"
460+
fi
461+
done < <(jq -c '.images[]' <<< "$version_tags_json")
462+
- name: Run mixed-site e2e tests (prebuilt images)
463+
if: github.event_name == 'pull_request'
464+
env:
465+
AMBER_TEST_USE_PREBUILT_IMAGES: "1"
466+
run: |
467+
set -euo pipefail
468+
cargo test -p amber-cli --test mixed_run mixed_run_ -- --ignored --nocapture --test-threads=1
469+
- name: Run mixed-site e2e tests
470+
if: github.event_name != 'pull_request'
471+
run: |
472+
set -euo pipefail
473+
cargo test -p amber-cli --test mixed_run mixed_run_ -- --ignored --nocapture --test-threads=1
474+
- name: Delete shared KinD cluster
475+
if: always()
476+
shell: bash
477+
run: |
478+
set -euo pipefail
479+
if [ -n "${AMBER_TEST_KIND_CLUSTER_NAME:-}" ]; then
480+
kind delete cluster --name "${AMBER_TEST_KIND_CLUSTER_NAME}" --kubeconfig "${AMBER_TEST_KIND_KUBECONFIG}"
481+
fi
482+
264483
image-matrix:
265484
name: Image Matrix
266485
runs-on: ubuntu-latest
@@ -276,13 +495,12 @@ jobs:
276495
matrix=$(jq -c '{
277496
include: [
278497
.images[] as $img |
279-
($img.arches // ["amd64", "arm64"])[] as $arch |
280498
{
281499
image: $img.name,
282500
context: $img.context,
283501
file: $img.dockerfile,
284-
arch: $arch,
285-
runner: (if $arch == "arm64" then "ubuntu-24.04-arm" else "ubuntu-latest" end)
502+
arch: "amd64",
503+
runner: "ubuntu-latest"
286504
}
287505
]
288506
}' docker/images.json)
@@ -325,65 +543,17 @@ jobs:
325543
BUILD_MODE=release
326544
platforms: linux/${{ matrix.arch }}
327545
push: true
328-
tags: ${{ steps.registry.outputs.image_registry }}/${{ matrix.image }}:${{ github.sha }}-${{ matrix.arch }}
546+
tags: ${{ steps.registry.outputs.image_registry }}/${{ matrix.image }}:${{ github.sha }}
329547
cache-from: |
330548
type=gha,scope=refs/heads/main-${{ matrix.image }}-${{ matrix.arch }}
331549
type=gha,scope=${{ matrix.image }}-pr-${{ github.event.pull_request.number }}-${{ matrix.arch }}
332550
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-pr-${{ github.event.pull_request.number }}-${{ matrix.arch }}
333551

334-
docker-merge:
335-
name: Create Multi-Arch Manifests
336-
runs-on: ubuntu-latest
337-
if: github.event_name == 'pull_request'
338-
needs: [image-matrix, docker-build]
339-
permissions:
340-
contents: read
341-
packages: write
342-
steps:
343-
- uses: actions/checkout@v5
344-
- name: Compute image registry
345-
id: registry
346-
shell: bash
347-
run: |
348-
set -euo pipefail
349-
owner_lc="$(printf '%s' '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')"
350-
image_registry="ghcr.io/${owner_lc}"
351-
echo "image_registry=${image_registry}" >> "$GITHUB_OUTPUT"
352-
echo "IMAGE_REGISTRY=${image_registry}" >> "$GITHUB_ENV"
353-
- name: List images
354-
id: images
355-
shell: bash
356-
run: |
357-
set -euo pipefail
358-
images=$(jq -r --arg reg "$IMAGE_REGISTRY" '.images[].name | "\($reg)/\(.)"' docker/images.json)
359-
{
360-
echo "images<<EOF"
361-
echo "$images"
362-
echo "EOF"
363-
} >> "$GITHUB_OUTPUT"
364-
- uses: docker/setup-buildx-action@v3
365-
- uses: docker/login-action@v3
366-
with:
367-
registry: ghcr.io
368-
username: ${{ github.actor }}
369-
password: ${{ secrets.GITHUB_TOKEN }}
370-
- name: Create multi-arch manifests
371-
run: |
372-
set -euo pipefail
373-
images="${{ steps.images.outputs.images }}"
374-
while read -r image; do
375-
[ -z "$image" ] && continue
376-
docker buildx imagetools create \
377-
-t "${image}:${{ github.sha }}" \
378-
"${image}:${{ github.sha }}-amd64" \
379-
"${image}:${{ github.sha }}-arm64"
380-
done <<< "$images"
381-
382552
docker-tests:
383553
name: Docker Smoke Tests (prebuilt images)
384554
runs-on: ubuntu-latest
385555
if: github.event_name == 'pull_request'
386-
needs: docker-merge
556+
needs: docker-build
387557
permissions:
388558
contents: read
389559
packages: read

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
/target/
22
*.img
3+
.amber-runs/
34
/npm/node_modules/

0 commit comments

Comments
 (0)