Skip to content

Commit d74714f

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #4189 from apostasie/ci-2025-05-workflow-n1
[CI]: cleanup, breakout 5: separate jobs
2 parents ced5c80 + f4a9def commit d74714f

File tree

12 files changed

+468
-277
lines changed

12 files changed

+468
-277
lines changed

‎.github/workflows/job-test-in-host.yml

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# This currently test docker and nerdctl on windows (w/o canary)
2+
# Structure is in to allow testing nerdctl on linux as well, though more work is required to make it functional.
3+
name: job-test-in-host
4+
5+
on:
6+
workflow_call:
7+
inputs:
8+
timeout:
9+
required: true
10+
type: number
11+
runner:
12+
required: true
13+
type: string
14+
canary:
15+
required: false
16+
default: false
17+
type: boolean
18+
binary:
19+
required: false
20+
default: nerdctl
21+
type: string
22+
go-version:
23+
required: true
24+
type: string
25+
containerd-version:
26+
required: true
27+
type: string
28+
containerd-sha:
29+
required: true
30+
type: string
31+
containerd-service-sha:
32+
required: true
33+
type: string
34+
windows-cni-version:
35+
required: true
36+
type: string
37+
linux-cni-version:
38+
required: true
39+
type: string
40+
linux-cni-sha:
41+
required: true
42+
type: string
43+
44+
env:
45+
GOTOOLCHAIN: local
46+
47+
jobs:
48+
test:
49+
name: |
50+
${{ inputs.binary != 'nerdctl' && format('{0} < ', inputs.binary) || '' }}
51+
${{ contains(inputs.runner, 'ubuntu') && ' linux' || ' windows' }}
52+
${{ contains(inputs.runner, 'arm') && '(arm)' || '' }}
53+
${{ contains(inputs.runner, '22.04') && '(old ubuntu)' || '' }}
54+
${{ inputs.canary && ' (canary)' || '' }}
55+
timeout-minutes: ${{ inputs.timeout }}
56+
runs-on: "${{ inputs.runner }}"
57+
defaults:
58+
run:
59+
shell: bash
60+
61+
env:
62+
SHOULD_RUN: "yes"
63+
GO_VERSION: ${{ inputs.go-version }}
64+
# Both Docker and nerdctl on linux need rootful right now
65+
WITH_SUDO: ${{ contains(inputs.runner, 'ubuntu') }}
66+
CONTAINERD_VERSION: ${{ inputs.containerd-version }}
67+
CONTAINERD_SHA: ${{ inputs.containerd-sha }}
68+
69+
steps:
70+
- name: "Init: checkout"
71+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
72+
with:
73+
fetch-depth: 1
74+
75+
- if: ${{ inputs.canary }}
76+
name: "Init (canary): retrieve latest go and containerd"
77+
run: |
78+
latest_go="$(. ./hack/provisioning/version/fetch.sh; go::canary::for::go-setup)"
79+
latest_containerd="$(. ./hack/provisioning/version/fetch.sh; github::project::latest "containerd/containerd")"
80+
81+
[ "$latest_go" == "" ] || \
82+
printf "GO_VERSION=%s\n" "$latest_go" >> "$GITHUB_ENV"
83+
[ "${latest_containerd:1}" == "$CONTAINERD_VERSION" ] || {
84+
printf "CONTAINERD_VERSION=%s\n" "${latest_containerd:1}" >> "$GITHUB_ENV"
85+
printf "CONTAINERD_SHA=canary is volatile and I accept the risk\n" >> "$GITHUB_ENV"
86+
}
87+
if [ "$latest_go" == "" ] && [ "${latest_containerd:1}" == "$CONTAINERD_VERSION" ]; then
88+
echo "::warning title=No canary::There is currently no canary versions to test. Steps will not run.";
89+
printf "SHOULD_RUN=no\n" >> "$GITHUB_ENV"
90+
fi
91+
92+
- if: ${{ env.SHOULD_RUN == 'yes' }}
93+
name: "Init: install go"
94+
uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
95+
with:
96+
go-version: ${{ env.GO_VERSION }}
97+
check-latest: true
98+
99+
# XXX RUNNER_OS and generally env is too unreliable
100+
# - if: ${{ env.RUNNER_OS == 'Linux' }}
101+
- if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
102+
name: "Init (linux): prepare host"
103+
run: |
104+
if [ "${{ contains(inputs.binary, 'docker') }}" == true ]; then
105+
echo "::group:: configure cdi for docker"
106+
sudo mkdir -p /etc/docker
107+
sudo jq '.features.cdi = true' /etc/docker/daemon.json | sudo tee /etc/docker/daemon.json.tmp && sudo mv /etc/docker/daemon.json.tmp /etc/docker/daemon.json
108+
sudo systemctl restart docker
109+
echo "::endgroup::"
110+
else
111+
# FIXME: this is missing runc (see top level workflow note about the state of this)
112+
echo "::group:: install dependencies"
113+
sudo ./hack/provisioning/linux/containerd.sh uninstall
114+
./hack/provisioning/linux/containerd.sh rootful "$CONTAINERD_VERSION" "amd64" "$CONTAINERD_SHA" "${{ inputs.containerd-service-sha }}"
115+
sudo ./hack/provisioning/linux/cni.sh uninstall
116+
./hack/provisioning/linux/cni.sh install "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
117+
echo "::endgroup::"
118+
119+
echo "::group:: build nerctl"
120+
go install ./cmd/nerdctl
121+
echo "$HOME/go/bin" >> "$GITHUB_PATH"
122+
# Since tests are going to run root, we need nerdctl to be in a PATH that will survive `sudo`
123+
sudo cp "$(which nerdctl)" /usr/local/bin
124+
echo "::endgroup::"
125+
fi
126+
127+
# Register QEMU (tonistiigi/binfmt)
128+
# `--install all` will only install emulation for architectures that cannot be natively executed
129+
# Since some arm64 platforms do provide native fallback execution for 32 bits,
130+
# armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
131+
# To avoid that, we explicitly list the architectures we do want emulation for.
132+
docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
133+
docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
134+
docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
135+
136+
# FIXME: remove expect when we are done removing unbuffer from tests
137+
sudo apt-get install -qq expect
138+
139+
- if: ${{ contains(inputs.runner, 'windows') && env.SHOULD_RUN == 'yes' }}
140+
name: "Init (windows): prepare host"
141+
env:
142+
ctrdVersion: ${{ env.CONTAINERD_VERSION }}
143+
run: |
144+
# Install WinCNI
145+
echo "::group:: install wincni"
146+
GOPATH=$(go env GOPATH) WINCNI_VERSION=${{ inputs.windows-cni-version }} ./hack/provisioning/windows/cni.sh
147+
echo "::endgroup::"
148+
149+
# Install containerd
150+
echo "::group:: install containerd"
151+
powershell hack/provisioning/windows/containerd.ps1
152+
echo "::endgroup::"
153+
154+
# Install nerdctl
155+
echo "::group:: build nerctl"
156+
go install ./cmd/nerdctl
157+
echo "::endgroup::"
158+
159+
- if: ${{ env.SHOULD_RUN == 'yes' }}
160+
name: "Init: install dev tools"
161+
run: |
162+
echo "::group:: make install-dev-tools"
163+
make install-dev-tools
164+
echo "::endgroup::"
165+
166+
# ipv6 is tested only on linux
167+
- if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
168+
name: "Run (linux): integration tests (IPv6)"
169+
run: |
170+
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
171+
172+
- if: ${{ env.SHOULD_RUN == 'yes' }}
173+
name: "Run: integration tests"
174+
run: |
175+
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
176+
177+
# FIXME: this must go
178+
- if: ${{ env.SHOULD_RUN == 'yes' }}
179+
name: "Run: integration tests (flaky)"
180+
run: |
181+
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true

‎.github/workflows/job-test-in-lima.yml

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# Currently, Lima job test only for EL, though in the future it could be used to also test FreeBSD or other linux-es
2+
name: job-test-in-lima
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
timeout:
8+
required: true
9+
type: number
10+
runner:
11+
required: true
12+
type: string
13+
target:
14+
required: true
15+
type: string
16+
guest:
17+
required: true
18+
type: string
19+
20+
jobs:
21+
test:
22+
name: "${{ inputs.guest }} ${{ inputs.target }}"
23+
timeout-minutes: ${{ inputs.timeout }}
24+
runs-on: "${{ inputs.runner }}"
25+
env:
26+
TARGET: ${{ inputs.target }}
27+
steps:
28+
- name: "Init: checkout"
29+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
30+
with:
31+
fetch-depth: 1
32+
33+
- name: "Init: lima"
34+
uses: lima-vm/lima-actions/setup@be564a1408f84557d067b099a475652288074b2e # v1.0.0
35+
id: lima-actions-setup
36+
37+
- name: "Init: Cache"
38+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
39+
with:
40+
path: ~/.cache/lima
41+
key: lima-${{ steps.lima-actions-setup.outputs.version }}
42+
43+
- name: "Init: start the guest VM"
44+
run: |
45+
set -eux
46+
# containerd=none is set because the built-in containerd support conflicts with Docker
47+
limactl start \
48+
--name=default \
49+
--cpus=4 \
50+
--memory=12 \
51+
--containerd=none \
52+
--set '.mounts=null | .portForwards=[{"guestSocket":"/var/run/docker.sock","hostSocket":"{{.Dir}}/sock/docker.sock"}]' \
53+
template://${{ inputs.guest }}
54+
55+
# FIXME: the tests should be directly executed in the VM without nesting Docker inside it
56+
# https://github.com/containerd/nerdctl/issues/3858
57+
- name: "Init: install dockerd in the guest VM"
58+
run: |
59+
set -eux
60+
lima sudo mkdir -p /etc/systemd/system/docker.socket.d
61+
cat <<-EOF | lima sudo tee /etc/systemd/system/docker.socket.d/override.conf
62+
[Socket]
63+
SocketUser=$(whoami)
64+
EOF
65+
lima sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
66+
lima sudo dnf -q -y install docker-ce --nobest
67+
lima sudo systemctl enable --now docker
68+
69+
- name: "Init: configure the host to use dockerd in the guest VM"
70+
run: |
71+
set -eux
72+
sudo systemctl disable --now docker.service docker.socket
73+
export DOCKER_HOST="unix://$(limactl ls --format '{{.Dir}}/sock/docker.sock' default)"
74+
echo "DOCKER_HOST=${DOCKER_HOST}" >>$GITHUB_ENV
75+
docker info
76+
docker version
77+
78+
- name: "Init: expose GitHub Runtime variables for gha"
79+
uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0
80+
81+
- name: "Init: prepare integration tests"
82+
run: |
83+
set -eux
84+
85+
sudo losetup -Dv
86+
sudo losetup -lv
87+
88+
[ "$TARGET" = "rootless" ] && TARGET=test-integration-rootless || TARGET=test-integration
89+
docker buildx create --name with-gha --use
90+
docker buildx build \
91+
--output=type=docker \
92+
--cache-from type=gha,scope=test-integration-dependencies-amd64 \
93+
-t test-integration --target "${TARGET}" \
94+
.
95+
96+
- name: "Run integration tests"
97+
# Presumably, something is broken with the way docker exposes /dev to the container, as it appears to only
98+
# randomly work. Mounting /dev does workaround the issue.
99+
# This might be due to the old kernel shipped with Alma (4.18), or something else between centos/docker.
100+
run: |
101+
set -eux
102+
if [ "$TARGET" = "rootless" ]; then
103+
echo "rootless"
104+
docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
105+
else
106+
echo "rootful"
107+
docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false
108+
fi
109+
- name: "Run: integration tests (flaky)"
110+
run: |
111+
set -eux
112+
if [ "$TARGET" = "rootless" ]; then
113+
echo "rootless"
114+
docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true
115+
else
116+
echo "rootful"
117+
docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true
118+
fi

‎.github/workflows/job-test-in-vagrant.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Right now, this is testing solely FreeBSD, but could be used to test other targets.
2+
# Alternatively, this might get replaced entirely by Lima eventually.
3+
name: job-test-in-vagrant
4+
5+
on:
6+
workflow_call:
7+
inputs:
8+
timeout:
9+
required: true
10+
type: number
11+
runner:
12+
required: true
13+
type: string
14+
15+
jobs:
16+
test:
17+
# Will appear as freebsd / 14 in GitHub UI
18+
name: "14"
19+
timeout-minutes: ${{ inputs.timeout }}
20+
runs-on: "${{ inputs.runner }}"
21+
steps:
22+
- name: "Init: checkout"
23+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
24+
with:
25+
fetch-depth: 1
26+
27+
- name: "Init: setup cache"
28+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
29+
with:
30+
path: /root/.vagrant.d
31+
key: vagrant
32+
33+
- name: "Init: set up vagrant"
34+
run: |
35+
# from https://github.com/containerd/containerd/blob/v2.0.2/.github/workflows/ci.yml#L583-L596
36+
# which is based on https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
37+
# FIXME: https://github.com/containerd/nerdctl/issues/4163
38+
curl -fsSL --proto '=https' --tlsv1.2 https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
39+
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
40+
sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
41+
sudo apt-get update -qq
42+
sudo apt-get install -qq libvirt-daemon libvirt-daemon-system vagrant ovmf
43+
# https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1725#issuecomment-1454058646
44+
sudo cp /usr/share/OVMF/OVMF_VARS_4M.fd /var/lib/libvirt/qemu/nvram/
45+
sudo systemctl enable --now libvirtd
46+
sudo apt-get build-dep -qq ruby-libvirt
47+
sudo apt-get install -qq --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
48+
# Disable strict dependency enforcement to bypass gem version conflicts during the installation of the vagrant-libvirt plugin.
49+
sudo env VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 vagrant plugin install vagrant-libvirt
50+
51+
- name: "Init: boot VM"
52+
run: |
53+
ln -sf Vagrantfile.freebsd Vagrantfile
54+
sudo vagrant up --no-tty
55+
56+
- name: "Run: test-unit"
57+
run: sudo vagrant up --provision-with=test-unit
58+
59+
- name: "Run: test-integration"
60+
run: sudo vagrant up --provision-with=test-integration

‎.github/workflows/job-test-unit.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
if [ "$RUNNER_OS" == "Windows" ]; then
7474
GOPATH=$(go env GOPATH) WINCNI_VERSION=${{ inputs.windows-cni-version }} ./hack/provisioning/windows/cni.sh
7575
elif [ "$RUNNER_OS" == "Linux" ]; then
76-
./hack/provisioning/linux/cni.sh "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
76+
./hack/provisioning/linux/cni.sh install "${{ inputs.linux-cni-version }}" "amd64" "${{ inputs.linux-cni-sha }}"
7777
fi
7878
7979
- if: ${{ env.GO_VERSION != '' }}

0 commit comments

Comments
 (0)