Skip to content

Commit 4ef0102

Browse files
therazixtherazix
authored
Report special events as check results (teemtee#3785)
1 parent 58a9372 commit 4ef0102

File tree

22 files changed

+405
-40
lines changed

22 files changed

+405
-40
lines changed

‎docs/releases.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@ whole component portfolio.
2626
The ``results.yaml`` file will now contain the log path for
2727
``journal.xml``.
2828

29+
New internal :ref:`checks </plugins/test-checks>` have been added
30+
to report special events that occur during test execution, such as
31+
timeouts or aborts. These internal checks run for every test, and
32+
the result of each check is included in the ``results.yaml`` file
33+
only if that specific check fails.
34+
2935

3036
tmt-1.49.0
3137
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

‎tests/core/about/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ step.finish: ansible shell
1414
step.prepare: ansible feature install shell
1515
step.provision: artemis beaker bootc connect container local virtual.testcloud
1616
step.report: display html junit polarion reportportal
17-
test.check: avc coredump dmesg watchdog
17+
test.check: avc coredump dmesg internal/abort internal/interrupt internal/invocation internal/permission internal/timeout watchdog
1818
test.framework: beakerlib shell"
1919

2020

‎tests/execute/framework/beakerlib.sh‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,18 @@ rlJournalStart
88
rlRun "run=\$(mktemp -d)" 0 "Creating run directory/id"
99
rlPhaseEnd
1010

11+
function assert_result () {
12+
name="$1"
13+
serial="$2"
14+
result="$3"
15+
guest="$4"
16+
note="$5"
17+
file="${run}/plans/execute/results.yaml"
18+
actual=$(yq -er ".[] | \"\\(.name) \\(.\"serial-number\") \\(.result) \\(.guest.name) \\(if .note == [] then \"\" else ((.note[] | select(. == \"$note\")) // .note[0]) end)\"" "$file")
19+
rlAssertEquals "Check result for $name" "$actual" "$name $serial $result $guest $note"
20+
}
21+
1122
tmt_command="tmt run --scratch -a --id ${run} provision --how local execute -vv report -vvv test --name"
12-
extract_results_command="yq -er '.[] | \"\\(.name) \\(.\"serial-number\") \\(.result) \\(.guest.name) \\(.note[0])\"'"
1323

1424
testName="/tests/pass"
1525
rlPhaseStartTest "${testName}"
@@ -21,8 +31,7 @@ rlJournalStart
2131
# tmt prints the correct result into log
2232
rlAssertGrep "pass /tests/pass" $rlRun_LOG
2333
# tmt saves the correct results, including note, into results yaml
24-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
25-
rlAssertGrep "/tests/pass 1 pass default-0 null" $rlRun_LOG
34+
assert_result "/tests/pass" "1" "pass" "default-0" ""
2635
rlPhaseEnd
2736

2837
testName="/tests/fail"
@@ -35,8 +44,7 @@ rlJournalStart
3544
# tmt prints the correct result into log
3645
rlAssertGrep "fail /tests/fail" $rlRun_LOG
3746
# tmt saves the correct results, including note, into results yaml
38-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
39-
rlAssertGrep "/tests/fail 1 fail default-0 null" $rlRun_LOG
47+
assert_result "/tests/fail" "1" "fail" "default-0" ""
4048
rlPhaseEnd
4149

4250
testName="/tests/warn"
@@ -49,8 +57,7 @@ rlJournalStart
4957
# tmt prints the correct result into log
5058
rlAssertGrep "warn /tests/warn" $rlRun_LOG
5159
# tmt saves the correct results, including note, into results yaml
52-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
53-
rlAssertGrep "/tests/warn 1 warn default-0 null" $rlRun_LOG
60+
assert_result "/tests/warn" "1" "warn" "default-0" ""
5461
rlPhaseEnd
5562

5663
testName="/tests/worst"
@@ -67,8 +74,7 @@ rlJournalStart
6774
# tmt prints the correct result into log
6875
rlAssertGrep "fail /tests/worst" $rlRun_LOG
6976
# tmt saves the correct results, including note, into results yaml
70-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
71-
rlAssertGrep "/tests/worst 1 fail default-0 null" $rlRun_LOG
77+
assert_result "/tests/worst" "1" "fail" "default-0" ""
7278
rlPhaseEnd
7379

7480
testName="/tests/timeout"
@@ -85,8 +91,7 @@ rlJournalStart
8591
rlAssertGrep "Maximum test time '5m' exceeded." $rlRun_LOG
8692
rlAssertGrep "Adjust the test 'duration' attribute if necessary." $rlRun_LOG
8793
# tmt saves the correct results, including note, into results yaml
88-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
89-
rlAssertGrep "/tests/timeout 1 error default-0 timeout" $rlRun_LOG
94+
assert_result "/tests/timeout" "1" "error" "default-0" "check 'internal/timeout' failed"
9095
rlPhaseEnd
9196

9297
testName="/tests/pidlock"
@@ -102,8 +107,7 @@ rlJournalStart
102107
rlAssertGrep "pidfile locking" $rlRun_LOG
103108
rlAssertGrep "warn: Test failed to manage its pidfile." $rlRun_LOG
104109
# tmt saves the correct results, including note, into results yaml
105-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
106-
rlAssertGrep "/tests/pidlock 1 error default-0 pidfile locking" $rlRun_LOG
110+
assert_result "/tests/pidlock" "1" "error" "default-0" "check 'internal/invocation' failed"
107111
rlPhaseEnd
108112

109113
testName="/tests/incomplete-fail"
@@ -119,8 +123,7 @@ rlJournalStart
119123
rlAssertGrep "errr /tests/incomplete-fail" $rlRun_LOG
120124
rlAssertGrep "beakerlib: State 'incomplete'" $rlRun_LOG
121125
# tmt saves the correct results, including note, into results yaml
122-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
123-
rlAssertGrep "/tests/incomplete-fail 1 error default-0 beakerlib: State 'incomplete'" $rlRun_LOG
126+
assert_result "/tests/incomplete-fail" "1" "error" "default-0" "beakerlib: State 'incomplete'"
124127
rlPhaseEnd
125128

126129
testName="/tests/incomplete-pass"
@@ -136,8 +139,7 @@ rlJournalStart
136139
rlAssertGrep "errr /tests/incomplete-pass" $rlRun_LOG
137140
rlAssertGrep "beakerlib: State 'incomplete'" $rlRun_LOG
138141
# tmt saves the correct results, including note, into results yaml
139-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
140-
rlAssertGrep "/tests/incomplete-pass 1 error default-0 beakerlib: State 'incomplete'" $rlRun_LOG
142+
assert_result "/tests/incomplete-pass" "1" "error" "default-0" "beakerlib: State 'incomplete'"
141143
rlPhaseEnd
142144

143145
testName="/tests/notfound"
@@ -152,8 +154,7 @@ rlJournalStart
152154
rlAssertGrep "beakerlib: TestResults FileError" $rlRun_LOG
153155
rlAssertGrep "No such file or directory" $rlRun_LOG
154156
# tmt saves the correct results, including note, into results yaml
155-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
156-
rlAssertGrep "/tests/notfound 1 error default-0 beakerlib: TestResults FileError" $rlRun_LOG
157+
assert_result "/tests/notfound" "1" "error" "default-0" "beakerlib: TestResults FileError"
157158
rlPhaseEnd
158159

159160
testName="/tests/notexec"
@@ -168,8 +169,7 @@ rlJournalStart
168169
rlAssertGrep "beakerlib: TestResults FileError" $rlRun_LOG
169170
rlAssertGrep "Permission denied" $rlRun_LOG
170171
# tmt saves the correct results, including note, into results yaml
171-
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
172-
rlAssertGrep "/tests/notexec 1 error default-0 beakerlib: TestResults FileError" $rlRun_LOG
172+
assert_result "/tests/notexec" "1" "error" "default-0" "beakerlib: TestResults FileError"
173173
rlPhaseEnd
174174

175175
rlPhaseStartCleanup

‎tests/execute/framework/shell.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ rlJournalStart
5050
rlAssertGrep "errr /tests/pidlock" $rlRun_LOG
5151
rlAssertGrep "pidfile locking" $rlRun_LOG
5252
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
53-
rlAssertGrep "/tests/pidlock 1 error default-0 pidfile locking" $rlRun_LOG
53+
rlAssertGrep "/tests/pidlock 1 error default-0 check 'internal/invocation' failed" $rlRun_LOG
5454
rlPhaseEnd
5555

5656
testName="/tests/timeout"
@@ -63,7 +63,7 @@ rlJournalStart
6363
rlAssertGrep "Maximum test time '5m' exceeded." $rlRun_LOG
6464
rlAssertGrep "Adjust the test 'duration' attribute if necessary." $rlRun_LOG
6565
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
66-
rlAssertGrep "/tests/timeout 1 error default-0 timeout" $rlRun_LOG
66+
rlAssertGrep "/tests/timeout 1 error default-0 check 'internal/timeout' failed" $rlRun_LOG
6767
rlPhaseEnd
6868

6969
testName="/tests/notfound"
@@ -83,7 +83,7 @@ rlJournalStart
8383
rlAssertGrep "/dev/null: Permission denied" $rlRun_LOG
8484
rlAssertGrep "errr /tests/notexec" $rlRun_LOG
8585
rlRun -s "${extract_results_command} ${run}/plans/execute/results.yaml"
86-
rlAssertGrep "/tests/notexec 1 error default-0 null" $rlRun_LOG
86+
rlAssertGrep "/tests/notexec 1 error default-0 check 'internal/permission' failed" $rlRun_LOG
8787
rlPhaseEnd
8888

8989
rlPhaseStartCleanup

‎tests/execute/ignore-duration/test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,13 @@ rlJournalStart
1212
rlPhaseStartTest "No envvar used"
1313
rlRun -s "tmt run -vv plan -n /no-option" "2"
1414
rlAssertGrep 'errr /demo/test' $rlRun_LOG '-F'
15-
rlAssertGrep 'Note: timeout' $rlRun_LOG '-F'
15+
rlAssertGrep "Note: check 'internal/timeout' failed" $rlRun_LOG '-F'
1616

1717
rlRun "tmt run -vv plan -n /via-plan-true" "0"
1818

1919
rlRun "tmt run -vv plan -n /via-plan-false" "2"
2020
rlAssertGrep 'errr /demo/test' $rlRun_LOG '-F'
21-
rlAssertGrep 'Note: timeout' $rlRun_LOG '-F'
21+
rlAssertGrep "Note: check 'internal/timeout' failed" $rlRun_LOG '-F'
2222
rlPhaseEnd
2323

2424
rlPhaseStartTest "With IGNORE_DURATION=1"

‎tests/execute/restraint/tmt-abort/test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ rlJournalStart
2929
"$(yq -r '[sort_by(.name) | .[] | "\(.name):\(.result)"] | join(" ")' ${run}/plan/execute/results.yaml)" \
3030
"/default-0/abort:error /default-0/do-not-run/1:pending /default-1/do-not-run/2:pending"
3131
rlAssertEquals "results should record the test aborted" \
32-
"$(yq -r '.[] | .note | join(", ")' ${run}/plan/execute/results.yaml)" \
33-
"beakerlib: State 'started', aborted"
32+
"$(yq -r '.[] | .note | sort | join(", ")' ${run}/plan/execute/results.yaml)" \
33+
"aborted, beakerlib: State 'started', check 'internal/abort' failed, check 'internal/interrupt' failed"
3434
rlPhaseEnd
3535

3636
rlPhaseStartCleanup

‎tests/execute/result/basic.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ rlJournalStart
9494
0
9595
test_result "errr /test/error-timeout \(on default-0\) \[7/12\]" \
9696
1 \
97-
"Note: timeout"
97+
"Note: check 'internal/timeout' failed"
9898
test_result "fail /test/fail \(on default-0\) \[8/12\]" \
9999
0
100100
test_result "pass /test/pass \(on default-0\) \[9/12\]" \

‎tests/report/html/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ rlJournalStart
3737
grep -B 1 "/test/$test_name_suffix</td>" $HTML | tee $tmp/$test_name_suffix
3838
rlAssertGrep 'class="result error">error</td>' $tmp/$test_name_suffix -F
3939
sed -e "/name\">\/test\/$test_name_suffix/,/\/tr/!d" $HTML | tee $tmp/$test_name_suffix-note
40-
rlAssertGrep '<li class="note">timeout</li>' $tmp/$test_name_suffix-note -F
40+
rlAssertGrep '<li class="note">check &#39;internal/timeout&#39; failed</li>' $tmp/$test_name_suffix-note -F
4141

4242
test_name_suffix=xfail
4343
grep -B 1 "/test/$test_name_suffix</td>" $HTML | tee $tmp/$test_name_suffix

‎tests/test/check/data/main.fmf‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,16 @@
128128

129129
ssh-ping: true
130130
ssh-ping-threshold: 3
131+
132+
/internal:
133+
/pass:
134+
test: /bin/true
135+
/timeout:
136+
test: sleep 3s
137+
duration: 1s
138+
/abort:
139+
test: "echo Start; tmt-abort; echo End"
140+
/permission:
141+
test: /dev/null
142+
/invocation:
143+
test: exit 122

‎tests/test/check/main.fmf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,9 @@ tier: 2
3636
- provision-only
3737
- provision-container
3838
- provision-virtual
39+
40+
/internal:
41+
test: ./test-internal.sh
42+
tag+:
43+
- provision-only
44+
- provision-container

0 commit comments

Comments
 (0)