Skip to content

Commit 38ab604

Browse files
apostasieapostasie
committed
System logs collector
Signed-off-by: apostasie <[email protected]>
1 parent 8e77757 commit 38ab604

File tree

8 files changed

+268
-31
lines changed

8 files changed

+268
-31
lines changed

‎.github/matchers/tigron.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"problemMatcher": [
3+
{
4+
"owner": "tigron",
5+
"pattern": [
6+
{
7+
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(error|warning):\\s+(.*)$",
8+
"severity": 1,
9+
"file": 2,
10+
"line": 3,
11+
"column": 4,
12+
"message": 5
13+
}
14+
]
15+
}
16+
]
17+
}

‎.github/workflows/job-test-in-container.yml

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ on:
3535
required: false
3636
default: false
3737
type: boolean
38+
outputs:
39+
artifact:
40+
description: "Artifact generated by this job"
41+
value: ${{ jobs.test.outputs.artifact }}
3842

3943
env:
4044
GOTOOLCHAIN: local
@@ -55,6 +59,8 @@ jobs:
5559
defaults:
5660
run:
5761
shell: bash
62+
outputs:
63+
artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
5864

5965
env:
6066
# https://github.com/containerd/nerdctl/issues/622
@@ -161,9 +167,9 @@ jobs:
161167
&& args=(test-integration ./hack/test-integration.sh -test.allow-modify-users=true) \
162168
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
163169
if [ "${{ inputs.ipv6 }}" == true ]; then
164-
docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
170+
docker run --name test-runner --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.only-ipv6 -test.target=${{ inputs.binary }}
165171
else
166-
docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
172+
docker run --name test-runner -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=false -test.target=${{ inputs.binary }}
167173
fi
168174
# FIXME: this NEEDS to go away
169175
- name: "Run: integration tests (flaky)"
@@ -175,7 +181,42 @@ jobs:
175181
&& args=(test-integration ./hack/test-integration.sh) \
176182
|| args=(test-integration-${{ inputs.target }} /test-integration-rootless.sh ./hack/test-integration.sh)
177183
if [ "${{ inputs.ipv6 }}" == true ]; then
178-
docker run --network host -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
184+
docker run --name test-runner-flaky --network host -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.only-ipv6 -test.target=${{ inputs.binary }}
179185
else
180-
docker run -t --rm --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
186+
docker run --name test-runner-flaky -t --privileged -e GITHUB_STEP_SUMMARY="$GITHUB_STEP_SUMMARY" -v "$GITHUB_STEP_SUMMARY":"$GITHUB_STEP_SUMMARY" -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622:-} "${args[@]}" -test.only-flaky=true -test.target=${{ inputs.binary }}
181187
fi
188+
189+
- name: "Wrap: collector"
190+
if: ${{ failure() || success() }}
191+
run: |
192+
# Get the reports from inside the containers
193+
[ "${{ inputs.target }}" == "rootful" ] && src=/root || src=/home/rootless
194+
mkdir -p ~/report
195+
docker cp test-runner:$src/nerdctl-test-report ~/report/main || true
196+
# Flaky may not have run
197+
docker cp test-runner-flaky:$src/nerdctl-test-report ~/report/flaky 2>/dev/null || true
198+
# Add metadata info to the runs
199+
. ./mod/wax/scripts/collector.sh
200+
collect::metadata \
201+
"runner=${{ inputs.runner }}" \
202+
"binary=${{ inputs.binary }}" \
203+
"canary=${{ inputs.canary }}" \
204+
"target=${{ inputs.target }}" \
205+
"ipv6=${{ inputs.ipv6 }}" \
206+
"containerd-version=${{ inputs.containerd-version }}" \
207+
"rootlesskit-version=${{ inputs.rootlesskit-version }}" \
208+
"attempt=$GITHUB_RUN_ATTEMPT" \
209+
"sha=$GITHUB_SHA" \
210+
"id=$GITHUB_RUN_ID" \
211+
"number=$GITHUB_RUN_NUMBER" \
212+
> ~/report/main/metadata.json
213+
[ ! -e ~/report/flaky ] || cp ~/report/main/metadata.json ~/report/flaky/metadata.json
214+
215+
- name: "Wrap: upload"
216+
id: artifact-upload
217+
if: ${{ failure() || success() }}
218+
uses: actions/upload-artifact@v4
219+
with:
220+
path: ~/report/*
221+
retention-days: 1
222+
name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}-${{ inputs.target }}-${{ inputs.ipv6 }}-${{ inputs.containerd-version }}-${{ inputs.rootlesskit-version }}

‎.github/workflows/job-test-in-host.yml

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ on:
4343
linux-cni-sha:
4444
required: true
4545
type: string
46+
outputs:
47+
artifact:
48+
description: "Artifact generated by this job"
49+
value: ${{ jobs.test.outputs.artifact }}
4650

4751
env:
4852
GOTOOLCHAIN: local
@@ -60,6 +64,8 @@ jobs:
6064
defaults:
6165
run:
6266
shell: bash
67+
outputs:
68+
artifact: ${{ steps.artifact-upload.outputs.artifact-url }}
6369

6470
env:
6571
SHOULD_RUN: "yes"
@@ -183,22 +189,30 @@ jobs:
183189
make install-dev-tools
184190
echo "::endgroup::"
185191
192+
193+
- if: ${{ env.SHOULD_RUN == 'yes' }}
194+
name: "Init: prepare artifacts directories"
195+
run: |
196+
mkdir -p ~/report/main
197+
mkdir -p ~/report/ipv6
198+
mkdir -p ~/report/flaky
199+
186200
# ipv6 is tested only on linux
187-
- if: ${{ contains(inputs.runner, 'ubuntu') && env.SHOULD_RUN == 'yes' }}
201+
- if: ${{ env.SHOULD_RUN == 'yes' && contains(inputs.runner, 'ubuntu' )}}
188202
name: "Run (linux): integration tests (IPv6)"
189203
run: |
190204
. ./hack/github/action-helpers.sh
191205
github::md::h2 "ipv6" >> "$GITHUB_STEP_SUMMARY"
192206
193-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
207+
WAX_REPORT_LOCATION=$HOME/report/ipv6 ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-ipv6
194208
195209
- if: ${{ env.SHOULD_RUN == 'yes' }}
196210
name: "Run: integration tests"
197211
run: |
198212
. ./hack/github/action-helpers.sh
199213
github::md::h2 "non-flaky" >> "$GITHUB_STEP_SUMMARY"
200214
201-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
215+
WAX_REPORT_LOCATION=$HOME/report/main ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=false
202216
203217
# FIXME: this must go
204218
- if: ${{ env.SHOULD_RUN == 'yes' }}
@@ -207,4 +221,42 @@ jobs:
207221
. ./hack/github/action-helpers.sh
208222
github::md::h2 "flaky" >> "$GITHUB_STEP_SUMMARY"
209223
210-
./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
224+
WAX_REPORT_LOCATION=$HOME/report/flaky ./hack/test-integration.sh -test.target=${{ inputs.binary }} -test.only-flaky=true
225+
226+
- name: "Wrap: collector"
227+
if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
228+
run: |
229+
# Add metadata info to the runs
230+
. ./mod/wax/scripts/collector.sh
231+
collect::metadata \
232+
"runner=${{ inputs.runner }}" \
233+
"binary=${{ inputs.binary }}" \
234+
"canary=${{ inputs.canary }}" \
235+
"ipv6=false" \
236+
"attempt=$GITHUB_RUN_ATTEMPT" \
237+
"sha=$GITHUB_SHA" \
238+
"id=$GITHUB_RUN_ID" \
239+
"number=$GITHUB_RUN_NUMBER" \
240+
> ~/report/main/metadata.json
241+
242+
collect::metadata \
243+
"runner=${{ inputs.runner }}" \
244+
"binary=${{ inputs.binary }}" \
245+
"canary=${{ inputs.canary }}" \
246+
"ipv6=true" \
247+
"attempt=$GITHUB_RUN_ATTEMPT" \
248+
"sha=$GITHUB_SHA" \
249+
"id=$GITHUB_RUN_ID" \
250+
"number=$GITHUB_RUN_NUMBER" \
251+
> ~/report/ipv6/metadata.json
252+
253+
cp ~/report/main/metadata.json ~/report/flaky/metadata.json
254+
255+
- name: "Wrap: upload"
256+
id: artifact-upload
257+
if: ${{ env.SHOULD_RUN == 'yes' && (failure() || success()) }}
258+
uses: actions/upload-artifact@v4
259+
with:
260+
path: ~/report/*
261+
retention-days: 1
262+
name: wax-${{ inputs.runner }}-${{ inputs.binary }}-${{ inputs.canary }}

‎.github/workflows/workflow-test.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,3 +147,23 @@ jobs:
147147
containerd-service-sha: 1941362cbaa89dd591b99c32b050d82c583d3cd2e5fa63085d7017457ec5fca8
148148
linux-cni-version: v1.7.1
149149
linux-cni-sha: 1a28a0506bfe5bcdc981caf1a49eeab7e72da8321f1119b7be85f22621013098
150+
151+
reporter:
152+
if: ${{ failure() || success() }}
153+
name: "reporter${{ inputs.hack }}"
154+
needs:
155+
- test-integration-host
156+
- test-integration-container
157+
runs-on: ubuntu-24.04
158+
steps:
159+
- name: Fetch Repository
160+
uses: actions/checkout@v4
161+
- name: Download all workflow run artifacts
162+
uses: actions/download-artifact@v4
163+
with:
164+
path: ~/report
165+
- name: Process
166+
run: |
167+
# git show --name-only --pretty=""
168+
ls -lAR ~/report || true
169+
exit 1

‎hack/github/gotestsum-reporter.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,18 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17+
# DEPRECATED: favor post-processing inside integration.sh rather than using a post-run hook.
18+
# Reasons are:
19+
# - post-run hook do not work on windows
20+
# - post-run hook is limited to a single run of gotestsum, while it is desirable to process multiple runs together
21+
1722
# shellcheck disable=SC2034,SC2015
1823
set -o errexit -o errtrace -o functrace -o nounset -o pipefail
1924
root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
2025
readonly root
2126

2227
# shellcheck source=/dev/null
23-
. "$root"/action-helpers.sh
28+
. "$root"/../github/action-helpers.sh
2429

2530
GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
2631

‎hack/test-integration.sh

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,38 +19,79 @@ set -o errexit -o errtrace -o functrace -o nounset -o pipefail
1919
root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
2020
readonly root
2121

22-
if [[ "$(id -u)" = "0" ]]; then
23-
# Ensure securityfs is mounted for apparmor to work
24-
if ! mountpoint -q /sys/kernel/security; then
25-
mount -tsecurityfs securityfs /sys/kernel/security
26-
fi
22+
# If no argument is provided, run both flaky and not-flaky test suites.
23+
if [ "$#" == 0 ]; then
24+
"$root"/integration.sh -test.only-flaky=false
25+
"$root"/integration.sh -test.only-flaky=true
26+
exit
2727
fi
2828

29+
##### Import helper libraries
30+
# shellcheck source=/dev/null
31+
. "$root"/../mod/wax/scripts/collector.sh
32+
33+
##### Configuration
34+
# Where to store report files
35+
readonly report_location="${WAX_REPORT_LOCATION:-$HOME/nerdctl-test-report}"
36+
# Where to store gotestsum log file
37+
readonly gotestsum_log_main="$report_location"/test-integration.log
38+
readonly gotestsum_log_flaky="$report_location"/test-integration-flaky.log
39+
# Total run timeout
2940
readonly timeout="60m"
41+
# Number of retries for flaky tests
3042
readonly retries="2"
31-
readonly needsudo="${WITH_SUDO:-}"
32-
33-
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
34-
args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
35-
# FIXME: not working on windows. Need to change approach: move away from --post-run-command and
36-
# just process the log file. This might also allow multi-steps/multi-target results aggregation.
37-
[ "$(uname -s)" != "Linux" ] || args+=(--post-run-command "$root"/github/gotestsum-reporter.sh)
38-
39-
if [ "$#" == 0 ]; then
40-
"$root"/test-integration.sh -test.only-flaky=false
41-
"$root"/test-integration.sh -test.only-flaky=true
42-
exit
43-
fi
43+
readonly need_sudo="${WITH_SUDO:-}"
4444

45+
##### Prepare gotestsum arguments
46+
mkdir -p "$report_location"
47+
# Format and packages to test
48+
args=(--format=testname --packages="$root"/../cmd/nerdctl/...)
49+
# Log file
50+
gotestsum_log="$gotestsum_log_main"
4551
for arg in "$@"; do
4652
if [ "$arg" == "-test.only-flaky=true" ] || [ "$arg" == "-test.only-flaky" ]; then
4753
args+=("--rerun-fails=$retries")
54+
gotestsum_log="$gotestsum_log_flaky"
4855
break
4956
fi
5057
done
58+
args+=(--jsonfile "$gotestsum_log" --)
59+
60+
##### Append go test arguments
61+
# Honor sudo
62+
[ "$need_sudo" != true ] && [ "$need_sudo" != yes ] && [ "$need_sudo" != 1 ] || args+=(-exec sudo)
63+
# About `-p 1`, see https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
64+
args+=(-timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@")
5165

52-
if [ "$needsudo" == "true" ] || [ "$needsudo" == "yes" ] || [ "$needsudo" == "1" ]; then
53-
gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -exec sudo -args -test.allow-kill-daemon "$@"
54-
else
55-
gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
66+
# FIXME: this should not be the responsibility of the test script
67+
# Instead, it should be in the Dockerfile (or other stack provisioning script) - eg: /etc/systemd/system/securityfs.service
68+
# [Unit]
69+
# Description=Kernel Security File System
70+
# DefaultDependencies=no
71+
# Before=sysinit.target
72+
# Before=apparmor.service
73+
# ConditionSecurity=apparmor
74+
# ConditionPathIsMountPoint=!/sys/kernel/security
75+
#
76+
# [Service]
77+
# Type=oneshot
78+
# ExecStart=/bin/mount -t securityfs -o nosuid,nodev,noexec securityfs /sys/kernel/security
79+
#
80+
# [Install]
81+
# WantedBy=sysinit.target
82+
if [[ "$(id -u)" = "0" ]]; then
83+
# Ensure securityfs is mounted for apparmor to work
84+
if ! mountpoint -q /sys/kernel/security; then
85+
mount -tsecurityfs securityfs /sys/kernel/security
86+
fi
5687
fi
88+
89+
##### Run it
90+
ex=0
91+
gotestsum "${args[@]}" || ex=$?
92+
93+
##### Post: collect logs into the report location
94+
collect::logs "$report_location"
95+
96+
# Honor gotestsum exit code
97+
exit "$ex"

‎mod/wax/README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Wax
2+
3+
> polish your CI
4+
5+
A tool to annotate pull requests and generate reports from gotestsum logs.
6+
7+
## TL;DR
8+
9+
TBD.

0 commit comments

Comments
 (0)