Skip to content

Commit 57d1718

Browse files
apostasieapostasie
committed
GHA: rewrite in container integration workflow
Signed-off-by: apostasie <[email protected]>
1 parent d74714f commit 57d1718

File tree

4 files changed

+290
-290
lines changed

4 files changed

+290
-290
lines changed

‎.github/workflows/job-test-dependencies.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# This job pre-heats the cache for the test image by building all dependencies
2+
name: job-test-dependencies
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
timeout:
8+
required: true
9+
type: number
10+
runner:
11+
required: true
12+
type: string
13+
containerd-version:
14+
required: false
15+
default: ''
16+
type: string
17+
18+
env:
19+
GOTOOLCHAIN: local
20+
21+
jobs:
22+
# This job builds the dependency target of the test docker image for all supported architectures and cache it in GHA
23+
build-dependencies:
24+
# Note: for whatever reason, you cannot access env.RUNNER_ARCH here
25+
name: "${{ contains(inputs.runner, 'arm') && 'arm64' || 'amd64' }}${{ inputs.containerd-version && format(' | {0}', inputs.containerd-version) || ''}}"
26+
timeout-minutes: ${{ inputs.timeout }}
27+
runs-on: "${{ inputs.runner }}"
28+
defaults:
29+
run:
30+
shell: bash
31+
32+
steps:
33+
- name: "Init: checkout"
34+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
35+
with:
36+
fetch-depth: 1
37+
38+
- name: "Init: expose GitHub Runtime variables for gha"
39+
uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0
40+
41+
- name: "Run: build dependencies for the integration test environment image"
42+
run: |
43+
# Cache is sharded per-architecture
44+
arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }}
45+
docker buildx create --name with-gha --use
46+
# Honor old containerd if requested
47+
args=()
48+
if [ "${{ inputs.containerd-version }}" != "" ]; then
49+
args=(--build-arg CONTAINERD_VERSION=${{ inputs.containerd-version }})
50+
fi
51+
docker buildx build \
52+
--cache-to type=gha,compression=zstd,mode=max,scope=test-integration-dependencies-"$arch" \
53+
--cache-from type=gha,scope=test-integration-dependencies-"$arch" \
54+
--target build-dependencies "${args[@]}" .

‎.github/workflows/job-test-in-container.yml

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# This job runs integration tests inside a container, for all supported variants (ipv6, canary, etc)
2+
# Note that it is linux and nerdctl (+/- gomodjail) only.
3+
name: job-test-in-container
4+
5+
on:
6+
workflow_call:
7+
inputs:
8+
timeout:
9+
required: true
10+
type: number
11+
runner:
12+
required: true
13+
type: string
14+
canary:
15+
required: false
16+
default: false
17+
type: boolean
18+
target:
19+
required: false
20+
default: ''
21+
type: string
22+
binary:
23+
required: false
24+
default: nerdctl
25+
type: string
26+
containerd-version:
27+
required: false
28+
default: ''
29+
type: string
30+
rootlesskit-version:
31+
required: false
32+
default: ''
33+
type: string
34+
ipv6:
35+
required: false
36+
default: false
37+
type: boolean
38+
39+
env:
40+
GOTOOLCHAIN: local
41+
42+
jobs:
43+
test:
44+
name: |
45+
${{ inputs.binary != 'nerdctl' && format('{0} < ', inputs.binary) || '' }}
46+
${{ inputs.target }}
47+
${{ contains(inputs.runner, 'arm') && '(arm)' || '' }}
48+
${{ contains(inputs.runner, '22.04') && '(old ubuntu)' || '' }}
49+
${{ inputs.ipv6 && ' (ipv6)' || '' }}
50+
${{ inputs.canary && ' (canary)' || '' }}
51+
${{ inputs.containerd-version && format(' (ctd: {0})', inputs.containerd-version) || '' }}
52+
${{ inputs.rootlesskit-version && format(' (rlk: {0})', inputs.rootlesskit-version) || '' }}
53+
timeout-minutes: ${{ inputs.timeout }}
54+
runs-on: ${{ inputs.runner }}
55+
defaults:
56+
run:
57+
shell: bash
58+
59+
steps:
60+
- name: "Init: checkout"
61+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
62+
with:
63+
fetch-depth: 1
64+
65+
- name: "Init: expose GitHub Runtime variables for gha"
66+
uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0
67+
68+
- name: "Init: register QEMU (tonistiigi/binfmt)"
69+
run: |
70+
# `--install all` will only install emulation for architectures that cannot be natively executed
71+
# Since some arm64 platforms do provide native fallback execution for 32 bits,
72+
# armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
73+
# To avoid that, we explicitly list the architectures we do want emulation for.
74+
docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
75+
docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
76+
docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
77+
- if: ${{ inputs.canary }}
78+
name: "Init (canary): prepare updated test image"
79+
run: |
80+
. ./hack/build-integration-canary.sh
81+
canary::build::integration
82+
- if: ${{ ! inputs.canary }}
83+
name: "Init: prepare test image"
84+
run: |
85+
buildargs=()
86+
# If the runner is old, use old ubuntu inside the container as well
87+
[ "${{ contains(inputs.runner, '22.04') }}" != "true" ] || buildargs=(--build-arg UBUNTU_VERSION=22.04)
88+
# Honor if we want old containerd
89+
[ "${{ inputs.containerd-version }}" == "" ] || buildargs+=(--build-arg CONTAINERD_VERSION=${{ inputs.containerd-version }})
90+
# Honor custom targets and if we want old rootlesskit
91+
target=test-integration
92+
if [ "${{ inputs.target }}" != "rootful" ]; then
93+
target+=-${{ inputs.target }}
94+
if [ "${{ inputs.rootlesskit-version }}" != "" ]; then
95+
buildargs+=(--build-arg ROOTLESSKIT_VERSION=${{ inputs.rootlesskit-version }})
96+
fi
97+
fi
98+
# Cache is sharded per-architecture
99+
arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }}
100+
docker buildx create --name with-gha --use
101+
docker buildx build \
102+
--output=type=docker \
103+
--cache-from type=gha,scope=test-integration-dependencies-"$arch" \
104+
-t "$target" --target "$target" \
105+
"${buildargs[@]}" \
106+
.
107+
# Rootful needs to disable snap
108+
- if: ${{ inputs.target == 'rootful' }}
109+
name: "Init: remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
110+
run: |
111+
sudo systemctl disable --now snapd.service snapd.socket
112+
sudo apt-get purge -qq snapd
113+
sudo losetup -Dv
114+
sudo losetup -lv
115+
# Rootless on modern ubuntu wants apparmor
116+
- if: ${{ inputs.target != 'rootful' && ! contains(inputs.runner, '22.04') }}
117+
name: "Init: prepare apparmor for rootless + ubuntu 24+"
118+
run: |
119+
cat <<EOT | sudo tee "/etc/apparmor.d/usr.local.bin.rootlesskit"
120+
abi <abi/4.0>,
121+
include <tunables/global>
122+
/usr/local/bin/rootlesskit flags=(unconfined) {
123+
userns,
124+
# Site-specific additions and overrides. See local/README for details.
125+
include if exists <local/usr.local.bin.rootlesskit>
126+
}
127+
EOT
128+
sudo systemctl restart apparmor.service
129+
# ipv6 wants... ipv6
130+
- if: ${{ inputs.ipv6 }}
131+
name: "Init: ipv6"
132+
run: |
133+
# Enable ipv4 and ipv6 forwarding
134+
sudo sysctl -w net.ipv6.conf.all.forwarding=1
135+
sudo sysctl -w net.ipv4.ip_forward=1
136+
# Enable IPv6 for Docker, and configure docker to use containerd for gha
137+
sudo mkdir -p /etc/docker
138+
echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "experimental": true, "ip6tables": true}' | sudo tee /etc/docker/daemon.json
139+
sudo systemctl restart docker
140+
# Rootless with old rootlesskit wants to disable buildkit
141+
- if: ${{ inputs.target != 'rootful' && inputs.rootlesskit-version != '' }}
142+
name: "Init: disable buildkit for old rootlesskit"
143+
run: |
144+
# https://github.com/containerd/nerdctl/issues/622
145+
WORKAROUND_ISSUE_622=
146+
if echo "${ROOTLESSKIT_VERSION}" | grep -q v1; then
147+
WORKAROUND_ISSUE_622=1
148+
fi
149+
echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
150+
- name: "Run: integration tests"
151+
run: |
152+
# IPV6 note: nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config.
153+
# Therefore, it's hard to debug why the IPv6 tests fail in such an isolation layer.
154+
# On the other side, using the host network is easier at configuration.
155+
# Besides, each job is running on a different instance, which means using host network here
156+
# is safe and has no side effects on others.
157+
[ "${{ inputs.target }}" == "rootful" ] \
158+
&& args=(test-integration ./hack/test-integration.sh) \
159+
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
160+
if [ "${{ inputs.ipv6 }}" == true ]; then
161+
docker run --network host -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
162+
else
163+
docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
164+
fi
165+
# FIXME: this NEEDS to go away
166+
- name: "Run: integration tests (flaky)"
167+
run: |
168+
[ "${{ inputs.target }}" == "rootful" ] \
169+
&& args=(test-integration ./hack/test-integration.sh) \
170+
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
171+
if [ "${{ inputs.ipv6 }}" == true ]; then
172+
docker run --network host -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
173+
else
174+
docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
175+
fi

‎.github/workflows/test-canary.yml

Lines changed: 0 additions & 47 deletions
This file was deleted.

0 commit comments

Comments
 (0)