Skip to content

Commit e759c14

Browse files
meisterTmeisterT
committed
Add some standalone tests for runguard.
This is not only useful in general for quicker testing of runguard, but will help to gain some confidence in runguard changes as we plan to do with https://docs.google.com/document/d/1WZRwdvJUamsczYC7CpP3ZIBU8xG6wNqYqrNJf7osxYs/edit
1 parent 8b735e5 commit e759c14

File tree

9 files changed

+402
-6
lines changed

9 files changed

+402
-6
lines changed

‎gitlab/integration.sh

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ if [ ! -d ${DIR}/chroot/domjudge/ ]; then
9393
fi
9494
section_end judgehost
9595

96-
section_start more_setup "Remaining setup (e.g. starting judgedaemon)"
96+
section_start more_setup "Remaining setup"
9797

9898
# Download yajsv and ccs-specs for API check.
9999
cd $HOME
@@ -118,9 +118,6 @@ fi
118118

119119
sudo useradd -d /nonexistent -g nogroup -s /bin/false -u $((2000+(RANDOM%1000))) $RUN_USER
120120

121-
# start judgedaemon
122-
cd /opt/domjudge/judgehost/
123-
124121
# Since ubuntu20.04 gitlabci image this is sometimes needed
125122
# It should be safe to remove this when it creates issues
126123
set +e
@@ -130,10 +127,21 @@ set -e
130127
if [ $PIN_JUDGEDAEMON -eq 1 ]; then
131128
PINNING="-n 0"
132129
fi
130+
section_end more_setup
131+
132+
section_start runguard_tests "Running isolated runguard tests"
133+
sudo addgroup domjudge-run-0
134+
sudo usermod -g domjudge-run-0 domjudge-run-0
135+
cd ${DIR}/judge/runguard_test
136+
make test
137+
section_end runguard_tests
138+
139+
section_start start_judging "Start judging"
140+
cd /opt/domjudge/judgehost/
141+
133142
sudo -u domjudge bin/judgedaemon $PINNING |& tee /tmp/judgedaemon.log &
134143
sleep 5
135-
136-
section_end more_setup
144+
section_end start_judging
137145

138146
section_start submitting "Importing Kattis examples"
139147
export SUBMITBASEURL='http://localhost/domjudge/'

‎judge/runguard_test/Makefile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
ifndef TOPDIR
2+
TOPDIR=../..
3+
endif
4+
include $(TOPDIR)/Makefile.global
5+
6+
export judgehost_tmpdir
7+
export judgehost_judgedir
8+
test: mem threads hello
9+
./runguard_test.sh
10+
11+
mem: mem.cc
12+
$(CXX) $(CXXFLAGS) -o $@ $<
13+
14+
threads: threads.cc
15+
$(CXX) $(CXXFLAGS) -o $@ $< -lpthread
16+
17+
hello: hello.cc
18+
$(CXX) $(CXXFLAGS) -static -o $@ $<
19+
20+
.PHONY: test

‎judge/runguard_test/fill-stderr.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
while true; do
4+
echo "DOMjudge" >&2
5+
done

‎judge/runguard_test/forky.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/bash
2+
3+
for (( i = 0; i < 32; i++ )); do
4+
echo $i
5+
sleep 5 &
6+
done

‎judge/runguard_test/hello.cc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#include <iostream>
2+
3+
int main() {
4+
std::cout << "Hello DOMjudge" << std::endl;
5+
return 0;
6+
}

‎judge/runguard_test/mem.cc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#include <iostream>
2+
3+
int main(int argc, char **argv) {
4+
int byteCount = atoi(argv[1]);
5+
char *mem = new char[byteCount];
6+
for (int i = 0; i < byteCount; i += 2048) {
7+
mem[i] = (char) i*i;
8+
}
9+
std::cout << "mem = " << byteCount << std::endl;
10+
}

‎judge/runguard_test/print_envvars.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/python3
2+
3+
import os
4+
5+
envvars = os.environ.items()
6+
print(f'COUNT: {len(envvars)}.')
7+
for k, v in envvars:
8+
print(f'{k}={v}')
9+

‎judge/runguard_test/runguard_test.sh

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
#!/bin/bash
2+
3+
cd "$(dirname "${BASH_SOURCE}")"
4+
5+
RUNGUARD=../runguard
6+
LOG1="$(mktemp)"
7+
LOG2="$(mktemp)"
8+
META=$(mktemp -p "$judgehost_tmpdir")
9+
10+
fail() {
11+
fail=1
12+
msg="$1"
13+
echo -e "\e[31mFAIL $msg\e[0m" >&2
14+
return 1
15+
}
16+
17+
expect_file() {
18+
file=$1
19+
token="$2"
20+
grep -q "$token" "$file" || fail "did not find expected '$token' in '$file', first few lines: $(head -n20 "$file")"
21+
}
22+
23+
expect_meta() {
24+
expect_file "$META" "$1"
25+
}
26+
27+
expect_stdout() {
28+
expect_file "$LOG1" "$1"
29+
}
30+
31+
expect_stderr() {
32+
expect_file "$LOG2" "$1"
33+
}
34+
35+
not_expect_stdout() {
36+
token="$1"
37+
grep -q "$token" "$LOG1" && fail "did find unexpected '$token' in log, first few lines: $(head "$LOG1")"
38+
}
39+
40+
exec_check_success() {
41+
all_args_string=$(printf "%s " "$@")
42+
"$@" > "$LOG1" 2> "$LOG2" || fail "expected command ('$all_args_string') to succeed, first few lines of stderr: $(head "$LOG2")"
43+
}
44+
45+
exec_check_fail() {
46+
all_args_string=$(printf "%s " "$@")
47+
"$@" > "$LOG1" 2> "$LOG2" && fail "expected command ('$all_args_string') to fail"
48+
}
49+
50+
test_no_command() {
51+
exec_check_fail $RUNGUARD
52+
expect_stderr "no command specified"
53+
}
54+
55+
test_no_sudo() {
56+
exec_check_fail $RUNGUARD ls
57+
expect_stderr "creating cgroup"
58+
}
59+
60+
test_no_sudo() {
61+
exec_check_fail sudo $RUNGUARD ls
62+
expect_stderr "root privileges not dropped"
63+
}
64+
65+
test_ls() {
66+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 ls
67+
expect_stdout "runguard_test.sh"
68+
}
69+
70+
test_walltime_limit() {
71+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -t 2 sleep 1
72+
73+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 2 sleep 3
74+
expect_stderr "timelimit exceeded"
75+
expect_stderr "hard wall time"
76+
}
77+
78+
test_cputime_limit() {
79+
# 2 threads, ~3s of CPU time, gives ~1.5s of wall time.
80+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -C 3.1 ./threads 2 3
81+
82+
# Now also limiting wall time to 2s.
83+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -C 3.1 -t 2 ./threads 2 3
84+
85+
# Some failing cases.
86+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -C 2.9 ./threads 2 3
87+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -C 3.1 -t 1.4 ./threads 2 3
88+
}
89+
90+
test_cputime_pinning() {
91+
# 2 threads, ~3s of CPU time, with one core we are out of luck...
92+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -C 3.1 -t 2 -P 1 ./threads 2 3
93+
# ...but with two cores it works.
94+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -C 3.1 -t 2 -P 0-1 ./threads 2 3
95+
}
96+
97+
test_streamsize() {
98+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 1 -s 123 yes DOMjudge
99+
expect_stdout "DOMjudge"
100+
limit=$((123*1024))
101+
actual=$(cat "$LOG1" | wc -c)
102+
[ $limit -eq $actual ] || fail "stdout not limited to ${limit}B, but wrote ${actual}B"
103+
}
104+
105+
test_streamsize_stderr() {
106+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 1 -s 42 ./fill-stderr.sh
107+
expect_stderr "DOMjudge"
108+
# Allow 100 bytes extra, for the runguard time limit message.
109+
limit=$((42*1024 + 100))
110+
actual=$(cat "$LOG2" | wc -c)
111+
[ $limit -gt $actual ] || fail "stdout not limited to ${limit}B, but wrote ${actual}B"
112+
}
113+
114+
test_redir_stdout() {
115+
stdout=$(mktemp -p "$judgehost_tmpdir")
116+
chmod go+rwx "$stdout"
117+
118+
# Basic test.
119+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -o "$stdout" echo 'foobar'
120+
grep -q "foobar" "$stdout" || fail "did not find expected 'foobar' in redirect stdout"
121+
122+
# Verify that stdout is empty.
123+
actual=$(cat "$LOG1" | wc -c)
124+
[ $actual -eq 0 ] || fail "stdout should be empty, but contains ${actual}B"
125+
126+
# This will fail because of the timeout.
127+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 1 -s 23 -o "$stdout" yes DOMjudge
128+
expect_stderr "timelimit exceeded"
129+
expect_stderr "hard wall time"
130+
131+
# Verify that stdout is empty.
132+
actual=$(cat "$LOG1" | wc -c)
133+
[ $actual -eq 0 ] || fail "stdout should be empty, but contains ${actual}B"
134+
135+
# Verify that redirected stdout has the right contents.
136+
grep -q "DOMjudge" "$stdout" || fail "did not find expected 'DOMjudge' in redirect stdout"
137+
limit=$((23*1024))
138+
actual=$(cat "$stdout" | wc -c)
139+
[ $limit -eq $actual ] || fail "redirected stdout not limited to ${limit}B, but wrote ${actual}B"
140+
141+
rm "$stdout"
142+
}
143+
144+
test_redir_stderr() {
145+
stderr=$(mktemp -p "$judgehost_tmpdir")
146+
chmod go+rwx "$stderr"
147+
148+
# This will fail because of the timeout.
149+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 1 -s 11 -e "$stderr" ./fill-stderr.sh
150+
expect_stderr "timelimit exceeded"
151+
expect_stderr "hard wall time"
152+
153+
# Verify that actual stderr does not contain DOMjudge.
154+
grep -q "DOMjudge" "$LOG2" && fail "did find unexpected 'DOMjudge' in stderr"
155+
156+
# Verify that redirected stdout has the right contents.
157+
grep -q "DOMjudge" "$stderr" || fail "did not find expected 'DOMjudge' in redirect stderr"
158+
limit=$((11*1024))
159+
actual=$(cat "$stderr" | wc -c)
160+
[ $limit -eq $actual ] || fail "redirected stdout not limited to ${limit}B, but wrote ${actual}B"
161+
162+
rm "$stderr"
163+
}
164+
165+
test_rootdir_changedir() {
166+
# Prepare test directory.
167+
almost_empty_dir="$judgehost_judgedir/runguard_tests/almost_empty"
168+
mkdir -p "$almost_empty_dir"/exists
169+
cp hello "$almost_empty_dir"/
170+
ln -sf /hello "$almost_empty_dir"/exists/foo
171+
172+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -r "$almost_empty_dir" ./hello
173+
expect_stdout "Hello DOMjudge"
174+
175+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -r "$almost_empty_dir" -d doesnotexist /hello
176+
expect_stderr "cannot chdir to \`doesnotexist' in chroot"
177+
178+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -r "$almost_empty_dir" -d exists /hello
179+
expect_stdout "Hello DOMjudge"
180+
181+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -r "$almost_empty_dir" -d exists ./foo
182+
expect_stdout "Hello DOMjudge"
183+
184+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -r "$almost_empty_dir" -d exists /exists/foo
185+
expect_stdout "Hello DOMjudge"
186+
}
187+
188+
test_memsize() {
189+
# This is slightly over the limit as there is other stuff to be allocated as well.
190+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -m 1024 ./mem $((1024*1024))
191+
192+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -m 1500 ./mem $((1024*1024))
193+
expect_stdout "mem = 1048576"
194+
195+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -m $((1024*1024)) ./mem $((1024*1024*1024))
196+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -m $((1024*1024 + 10000)) ./mem $((1024*1024*1024))
197+
expect_stdout "mem = 1073741824"
198+
}
199+
200+
test_envvars() {
201+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 ./print_envvars.py
202+
expect_stdout "COUNT: 2."
203+
expect_stdout "PATH="
204+
expect_stdout "LC_CTYPE="
205+
not_expect_stdout "DOMjudge"
206+
207+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -E ./print_envvars.py
208+
expect_stdout "HOME="
209+
expect_stdout "USER="
210+
expect_stdout "SHELL="
211+
212+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -V"DOMjudgeA=A;DOMjudgeB=BB" ./print_envvars.py
213+
expect_stdout "COUNT: 4."
214+
expect_stdout "DOMjudgeA=A"
215+
expect_stdout "DOMjudgeB=BB"
216+
not_expect_stdout "HOME="
217+
not_expect_stdout "USER="
218+
not_expect_stdout "SHELL="
219+
}
220+
221+
test_nprocs() {
222+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 ./forky.sh
223+
expect_stdout 31
224+
225+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -p 16 ./forky.sh
226+
expect_stdout 15
227+
not_expect_stdout 16
228+
not_expect_stdout 31
229+
expect_stderr "fork: retry: Resource temporarily unavailable"
230+
}
231+
232+
test_meta() {
233+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -t 2 -M "$META" sleep 1
234+
expect_meta 'wall-time: 1.0'
235+
expect_meta 'cpu-time: 0.0'
236+
expect_meta 'sys-time: 0.0'
237+
expect_meta 'time-used: wall-time'
238+
expect_meta 'exitcode: 0'
239+
expect_meta 'stdin-bytes: 0'
240+
expect_meta 'stdout-bytes: 0'
241+
expect_meta 'stderr-bytes: 0'
242+
243+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -M "$META" false
244+
expect_meta 'exitcode: 1'
245+
246+
echo "DOMjudge" | sudo $RUNGUARD -u domjudge-run-0 -t 2 -M "$META" rev > "$LOG1" 2> "$LOG2"
247+
expect_meta 'wall-time: 0.0'
248+
expect_meta 'stdout-bytes: 9'
249+
expect_stdout "egdujMOD"
250+
251+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -C 3.1 -t 1.4 -M "$META" ./threads 2 3
252+
expect_meta 'exitcode: 143'
253+
expect_meta 'signal: 14'
254+
expect_meta 'wall-time: 1.5'
255+
expect_meta 'time-result: hard-timelimit'
256+
257+
exec_check_success sudo $RUNGUARD -u domjudge-run-0 -C 1:5 -M "$META" ./threads 2 3
258+
expect_meta 'time-used: cpu-time'
259+
expect_meta 'time-result: soft-timelimit'
260+
expect_meta 'exitcode: 0'
261+
262+
exec_check_fail sudo $RUNGUARD -u domjudge-run-0 -t 1 -s 3 -M "$META" ./fill-stderr.sh
263+
# We expect stderr-bytes to have a non-zero value.
264+
expect_meta 'stderr-bytes: '
265+
grep -q 'stderr-bytes: 0' "$META" && fail ""
266+
expect_meta 'output-truncated: stderr'
267+
}
268+
269+
any_test_failed=0
270+
only_func=$1
271+
for func in $(compgen -o nosort -A function test_); do
272+
# Check whether the user requested to run a single function.
273+
if [[ -n "$only_func" && "$only_func" != "$func" ]]; then
274+
continue;
275+
fi
276+
fail=0
277+
echo -n "- $func "
278+
$func
279+
if [ $fail -eq 0 ]; then
280+
echo -e "\e[32m✔\e[0m" >&2
281+
else
282+
echo -e "\e[31m✘\e[0m" >&2
283+
any_test_failed=1
284+
fi
285+
done
286+
287+
rm -f "$LOG1" "$LOG2" "$META"
288+
289+
exit $any_test_failed

0 commit comments

Comments
 (0)