Skip to content

Commit 7ee2748

Browse files
vmcjvmcj
committed
Webstandard job has been moved to GHA
The GitLab folder will be removed so this cleanup was needed. Also fixed displaying the sections for the GHA job output. (cherry picked from commit 34c2c4d)
1 parent 2f9743b commit 7ee2748

File tree

3 files changed

+196
-2
lines changed

3 files changed

+196
-2
lines changed

‎gitlab/jsontogitlab.py renamed to ‎.github/jobs/jsontogha.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ def cleanHash(toHash):
1010
return hashlib.sha224(toHash).hexdigest()
1111

1212
def sec_start(job,header):
13-
print('section_start:{}:{}{}{}'.format(int(time.time()),cleanHash(job),'\r\033[0K',header))
13+
print('section_start\r\033[0K'+header)
1414

1515
def sec_end(job):
16-
print('section_end:{}:{}'.format(int(time.time()),cleanHash(job)+'\r\033[0K'))
16+
print('section_end\r\033[0K')
1717

1818
with open(sys.argv[1],'r') as f:
1919
data = json.load(f, encoding='utf-8')

‎.github/jobs/webstandard.sh

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/bin/bash
2+
3+
. .github/jobs/ci_settings.sh
4+
5+
DIR="$PWD"
6+
7+
if [ "$#" -ne "2" ]; then
8+
exit 2
9+
fi
10+
11+
TEST="$1"
12+
ROLE="$2"
13+
14+
cd /opt/domjudge/domserver
15+
16+
section_start "Setup pa11y"
17+
/home/domjudge/node_modules/.bin/pa11y --version
18+
section_end
19+
20+
section_start "Setup the test user"
21+
ADMINPASS=$(cat etc/initial_admin_password.secret)
22+
export COOKIEJAR
23+
COOKIEJAR=$(mktemp --tmpdir)
24+
export CURLOPTS="--fail -sq -m 30 -b $COOKIEJAR"
25+
if [ "$ROLE" = "public" ]; then
26+
ADMINPASS="failedlogin"
27+
fi
28+
29+
# Make an initial request which will get us a session id, and grab the csrf token from it
30+
CSRFTOKEN=$(curl $CURLOPTS -c $COOKIEJAR "http://localhost/domjudge/login" 2>/dev/null | sed -n 's/.*_csrf_token.*value="\(.*\)".*/\1/p')
31+
# Make a second request with our session + csrf token to actually log in
32+
# shellcheck disable=SC2086
33+
curl $CURLOPTS -c "$COOKIEJAR" -F "_csrf_token=$CSRFTOKEN" -F "_username=admin" -F "_password=$ADMINPASS" "http://localhost/domjudge/login"
34+
35+
# Move back to the default directory
36+
cd "$DIR"
37+
38+
cp "$COOKIEJAR" cookies.txt
39+
sed -i 's/#HttpOnly_//g' cookies.txt
40+
sed -i 's/\t0\t/\t1999999999\t/g' cookies.txt
41+
section_end
42+
43+
# Could try different entrypoints
44+
FOUNDERR=0
45+
URL=public
46+
mkdir "$URL"
47+
cd "$URL"
48+
cp "$DIR"/cookies.txt ./
49+
section_start "Scrape the site with the rebuild admin user"
50+
set +e
51+
wget \
52+
--reject-regex logout \
53+
--recursive \
54+
--no-clobber \
55+
--page-requisites \
56+
--html-extension \
57+
--convert-links \
58+
--restrict-file-names=windows \
59+
--domains localhost \
60+
--no-parent \
61+
--load-cookies cookies.txt \
62+
http://localhost/domjudge/"$URL"
63+
set -e
64+
RET=$?
65+
section_end
66+
67+
section_start "Archive downloaded site"
68+
cp -r localhost $ARTIFACTS/
69+
section_end
70+
71+
section_start "Analyse failures"
72+
#https://www.gnu.org/software/wget/manual/html_node/Exit-Status.html
73+
# Exit code 4 is network error which we can ignore
74+
# Exit code 8 can also be because of HTTP404 or 400
75+
if [ $RET -ne 4 ] && [ $RET -ne 0 ] && [ $RET -ne 8 ]; then
76+
exit $RET
77+
fi
78+
79+
EXPECTED_HTTP_CODES="200\|302\|400\|404\|403"
80+
if [ "$ROLE" = "public" ]; then
81+
# It's expected to encounter a 401 for the login page as we supply the wrong password
82+
EXPECTED_HTTP_CODES="$EXPECTED_HTTP_CODES\|401"
83+
fi
84+
set +e
85+
NUM_ERRORS=$(grep -v "HTTP/1.1\" \($EXPECTED_HTTP_CODES\)" /var/log/nginx/domjudge.log | grep -v "robots.txt" -c; if [ "$?" -gt 1 ]; then exit 127; fi)
86+
set -e
87+
echo "$NUM_ERRORS"
88+
89+
if [ "$NUM_ERRORS" -ne 0 ]; then
90+
grep -v "HTTP/1.1\" \($EXPECTED_HTTP_CODES\)" /var/log/nginx/domjudge.log | grep -v "robots.txt"
91+
exit 1
92+
fi
93+
section_end
94+
95+
if [ "$TEST" = "none" ]; then
96+
exit $NUM_ERRORS
97+
fi
98+
99+
if [ "$TEST" = "w3cval" ]; then
100+
section_start "Remove files from upstream with problems"
101+
rm -rf localhost/domjudge/doc
102+
rm -rf localhost/domjudge/css/fontawesome-all.min.css*
103+
rm -rf localhost/domjudge/bundles/nelmioapidoc*
104+
rm -f localhost/domjudge/css/bootstrap.min.css*
105+
rm -f localhost/domjudge/css/select2-bootstrap*.css*
106+
rm -f localhost/domjudge/css/dataTables*.css*
107+
rm -f localhost/domjudge/jury/config/check/phpinfo*
108+
section_end
109+
110+
section_start "Install testsuite"
111+
cd "$DIR"
112+
wget https://github.com/validator/validator/releases/latest/download/vnu.linux.zip
113+
unzip -q vnu.linux.zip
114+
section_end
115+
116+
FLTR='--filterpattern .*autocomplete.*|.*style.*|.*role=tab.*|.*descendant.*|.*Stray.*|.*attribute.*|.*Forbidden.*|.*stream.*|.*obsolete.*'
117+
for typ in html css svg
118+
do
119+
section_start "Analyse with $typ"
120+
# shellcheck disable=SC2086
121+
"$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format json $FLTR "$URL" 2> result.json
122+
# shellcheck disable=SC2086
123+
NEWFOUNDERRORS=$("$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format gnu $FLTR "$URL" 2>&1 | wc -l)
124+
FOUNDERR=$((NEWFOUNDERRORS+FOUNDERR))
125+
python3 -m "json.tool" < result.json > "$ARTIFACTS/w3c$typ$URL.json"
126+
trace_off; python3 .github/jobs/jsontogha.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on
127+
section_end
128+
done
129+
else
130+
section_start "Remove files from upstream with problems"
131+
rm -rf localhost/domjudge/{doc,api}
132+
section_end
133+
134+
if [ "$TEST" == "axe" ]; then
135+
STAN="-e $TEST"
136+
FLTR=""
137+
else
138+
STAN="-s $TEST"
139+
FLTR0="-E '#DataTables_Table_0 > tbody > tr > td > a','#menuDefault > a','#filter-card > div > div > div > span > span:nth-child(1) > span > ul > li > input',.problem-badge"
140+
FLTR1="'html > body > div > div > div > div > div > div > table > tbody > tr > td > a > span','html > body > div > div > div > div > div > div > form > div > div > div > label'"
141+
FLTR="$FLTR0,$FLTR1"
142+
fi
143+
chown -R domjudge:domjudge "$DIR"
144+
cd "$DIR"
145+
ACCEPTEDERR=5
146+
# shellcheck disable=SC2044,SC2035
147+
for file in $(find $URL -name "*.html")
148+
do
149+
section_start "$file"
150+
su domjudge -c "/home/domjudge/node_modules/.bin/pa11y --config .github/jobs/pa11y_config.json $STAN -r json -T $ACCEPTEDERR $FLTR $file" | python3 -m json.tool
151+
ERR=$(su domjudge -c "/home/domjudge/node_modules/.bin/pa11y --config .github/jobs/pa11y_config.json $STAN -r csv -T $ACCEPTEDERR $FLTR $file" | wc -l)
152+
FOUNDERR=$((ERR+FOUNDERR-1)) # Remove header row
153+
section_end
154+
done
155+
fi
156+
157+
echo "Found: " $FOUNDERR
158+
[ "$FOUNDERR" -eq 0 ]

‎.github/workflows/database-upgrade.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Database Upgrade Test
2+
3+
on:
4+
merge_group:
5+
pull_request:
6+
branches:
7+
- main
8+
- '[0-9]+.[0-9]+'
9+
10+
jobs:
11+
upgrade_test:
12+
runs-on: ubuntu-latest
13+
container:
14+
image: domjudge/gitlabci:24.04
15+
services:
16+
sqlserver:
17+
image: mariadb
18+
ports:
19+
- 3306:3306
20+
env:
21+
MYSQL_ROOT_PASSWORD: root
22+
MYSQL_USER: domjudge
23+
MYSQL_PASSWORD: domjudge
24+
options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
25+
steps:
26+
- uses: actions/checkout@v4
27+
- name: Import Database
28+
run: mysql -hsqlserver -uroot -proot < .github/jobs/data/dj733.sql
29+
- name: Upgrade DOMjudge
30+
run: .github/jobs/baseinstall.sh default upgrade
31+
- name: Setting initial Admin Password
32+
run: echo "pass" > /opt/domjudge/domserver/etc/initial_admin_password.secret
33+
- name: Check for Errors in the Upgrade
34+
run: mysql -hsqlserver -uroot -proot -e "SHOW TABLES FROM domjudge;"
35+
- name: Check for Errors in DOMjudge Webinterface
36+
run: .github/jobs/webstandard.sh none admin

0 commit comments

Comments
 (0)