Skip to content

Commit e30cf7f

Browse files
vmcjvmcj
committed
Revert "Remove unused folder"
This reverts commit 22c772b.
1 parent 7b9e3d7 commit e30cf7f

File tree

6 files changed

+317
-0
lines changed

6 files changed

+317
-0
lines changed

‎gitlab/base.sh‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/bin/bash
2+
3+
. gitlab/ci_settings.sh
4+
5+
# If this script is called from unit.sh, we use the test environment
6+
export APP_ENV="${1:-prod}"
7+
8+
# In the test environment, we need to use a different database
9+
[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test
10+
11+
lsb_release -a
12+
13+
# FIXME: This chicken-egg problem is annoying but let us bootstrap for now.
14+
echo "CREATE DATABASE IF NOT EXISTS \`${DATABASE_NAME}\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" | mysql
15+
echo "CREATE USER 'domjudge'@'%' IDENTIFIED BY 'domjudge';" | mysql
16+
echo "GRANT SELECT, INSERT, UPDATE, DELETE ON \`${DATABASE_NAME}\`.* TO 'domjudge'@'%';" | mysql
17+
18+
# Increase max_allowed_packet for following connections.
19+
echo "SET GLOBAL max_allowed_packet = 100*1024*1024;" | mysql
20+
21+
# Test that SQL upgrade scripts also work with this setting
22+
if [ -n "${MYSQL_REQUIRE_PRIMARY_KEY:-}" ]; then
23+
echo 'SET GLOBAL sql_require_primary_key = 1;' | mysql
24+
fi
25+
26+
# Generate a dbpasswords file
27+
# Note that this does not use ${DATABASE_NAME} since Symfony adds the _test postfix itself
28+
echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > etc/dbpasswords.secret
29+
30+
# Generate APP_SECRET for symfony
31+
# shellcheck disable=SC2164
32+
( cd etc ; ./gensymfonysecret > symfony_app.secret )
33+
34+
cat > webapp/config/static.yaml <<EOF
35+
parameters:
36+
domjudge.version: unconfigured
37+
domjudge.bindir: /bin
38+
domjudge.etcdir: /etc
39+
domjudge.wwwdir: /www
40+
domjudge.webappdir: /webapp
41+
domjudge.libdir: /lib
42+
domjudge.sqldir: /sql
43+
domjudge.vendordir: /webapp/vendor
44+
domjudge.logdir: /output/log
45+
domjudge.rundir: /output/run
46+
domjudge.tmpdir: /output/tmp
47+
domjudge.baseurl: http://localhost/domjudge
48+
EOF
49+
50+
# Composer steps
51+
cd webapp
52+
# install check if the cache might be dirty
53+
set +e
54+
composer install --no-scripts || rm -rf vendor
55+
set -e
56+
57+
# setup database and add special user
58+
# shellcheck disable=SC2164
59+
cd /opt/domjudge/domserver
60+
setfacl -m u:www-data:r etc/restapi.secret etc/initial_admin_password.secret \
61+
etc/dbpasswords.secret etc/symfony_app.secret
62+
63+
# configure and restart nginx
64+
sudo rm -f /etc/nginx/sites-enabled/*
65+
sudo cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge
66+
sudo /usr/sbin/nginx
67+
68+
# configure and restart php-fpm
69+
# shellcheck disable=SC2154
70+
php_version="${version:-}"
71+
sudo cp /opt/domjudge/domserver/etc/domjudge-fpm.conf "/etc/php/$php_version/fpm/pool.d/domjudge-fpm.conf"
72+
echo "php_admin_value[date.timezone] = Europe/Amsterdam" | sudo tee -a "/etc/php/$php_version/fpm/pool.d/domjudge-fpm.conf"
73+
sudo /usr/sbin/php-fpm${php_version}
74+
echo "date.timezone = Europe/Amsterdam" | sudo tee -a "/etc/php/$php_version/cli/php.ini"
75+
76+
passwd=$(cat etc/initial_admin_password.secret)
77+
echo "machine localhost login admin password $passwd" >> ~www-data/.netrc
78+
sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install
79+
80+
# shellcheck disable=SC2154
81+
if [ -n "${integration:-}" ]; then
82+
# Make sure admin has a team associated to insert submissions as well.
83+
echo "UPDATE user SET teamid=1 WHERE userid=1;" | mysql domjudge
84+
elif [ -n "${unit:-}" ]; then
85+
# Make sure admin has no team associated so we will not insert submissions during unit tests.
86+
echo "UPDATE user SET teamid=null WHERE userid=1;" | mysql domjudge_test
87+
fi
88+
89+
sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples

‎gitlab/ci/template.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# This placeholder job tries to start as soon as possible
2+
.clean_ordering:
3+
needs: []
4+
retry:
5+
max: 2 #Max is 2, set when gitlab is flacky
6+
when:
7+
- always
8+
script:
9+
- /bin/true
10+
11+
.tiny_job:
12+
extends: [.clean_ordering]
13+
timeout: 4m
14+
15+
.short_job:
16+
extends: [.clean_ordering]
17+
timeout: 7m
18+
19+
.normal_job:
20+
extends: [.clean_ordering]
21+
timeout: 20m
22+
23+
.long_job:
24+
extends: [.clean_ordering]
25+
timeout: 30m
26+
27+
.cached_vendor:
28+
extends: [.clean_ordering]
29+
cache:
30+
key: webappvendor-20240623
31+
paths:
32+
- webapp/vendor/
33+
34+
.mysql_job:
35+
script:
36+
- /bin/true
37+
services:
38+
- name: mysql
39+
alias: sqlserver
40+
41+
.mariadb_job:
42+
script:
43+
- /bin/true
44+
services:
45+
- name: mariadb
46+
alias: sqlserver
47+
48+
.phpsupported_job:
49+
script:
50+
- /bin/true
51+
parallel:
52+
matrix:
53+
- PHPVERSION: ["8.1","8.2","8.3", "8.4"]
54+
TEST: ["E2E","Unit"]
55+
CRAWL_SHADOW_MODE: ["0","1"]
56+
57+
.phpsupported_job_pr:
58+
script:
59+
- /bin/true
60+
parallel:
61+
matrix:
62+
- PHPVERSION: ["8.3"]
63+
TEST: ["E2E","Unit"]
64+
CRAWL_SHADOW_MODE: ["0"]

‎gitlab/ci/unit.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
.unit_job:
2+
extends: [.normal_job,.cached_vendor]
3+
stage: unit
4+
# Disabled for now as it drastically speeds up running unit tests and we don't use it yet
5+
# before_script:
6+
# - apt-get update -yqq
7+
# - apt-get install php-xdebug -yqq
8+
variables:
9+
MYSQL_ROOT_PASSWORD: password
10+
MARIADB_PORT_3306_TCP_ADDR: sqlserver
11+
script:
12+
- set -eux
13+
- if [ -z ${PHPVERSION+x} ]; then export PHPVERSION=8.1; fi
14+
- if [ -z ${TEST+x} ]; then export TEST="UNIT"; fi
15+
- if [ "$TEST" = "UNIT" ] && [ "$CRAWL_SHADOW_MODE" != "0" ]; then exit 0; fi
16+
- if [ "$TEST" = "E2E" ] && [ "$CRAWL_SHADOW_MODE" != "0" ] && [ "$CI_COMMIT_BRANCH" != "main" ]; then exit 0; fi
17+
- export CRAWL_SHADOW_MODE
18+
- ./gitlab/unit-tests.sh $PHPVERSION $TEST
19+
artifacts:
20+
when: always
21+
paths:
22+
- unit-tests.xml
23+
- coverage-html
24+
- deprecation.txt
25+
- duration
26+
- gitlabartifacts
27+
reports:
28+
junit:
29+
- unit-tests.xml
30+
31+
run unit tests:
32+
extends: [.mariadb_job,.phpsupported_job,.unit_job]
33+
34+
run unit tests (PR):
35+
extends: [.mariadb_job,.phpsupported_job_pr,.unit_job]
36+
37+
run unit tests (MySQL):
38+
extends: [.mysql_job,.unit_job]
39+
parallel:
40+
matrix:
41+
- TEST: ["E2E","Unit"]
42+
CRAWL_SHADOW_MODE: ["0"]

‎gitlab/ci_settings.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Fail pipeline when variable is not set or individual command has an non-zero exitcode.
4+
set -euo pipefail
5+
6+
shopt -s expand_aliases
7+
8+
# Shared constants between jobs
9+
DIR=$(pwd)
10+
GITSHA=$(git rev-parse HEAD || true)
11+
export DIR
12+
export GITSHA
13+
export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ '
14+
export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log"
15+
16+
CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1'
17+
18+
# Shared storage for all artifacts
19+
export GITLABARTIFACTS="$DIR/gitlabartifacts"
20+
mkdir -p "$GITLABARTIFACTS"
21+
22+
# Functions to annotate the GitLab logs
23+
alias trace_on='set -x'
24+
alias trace_off='{ set +x; } 2>/dev/null'
25+
function section_start_internal() {
26+
echo -e "section_start:$(date +%s):$2[collapsed=$1]\r\e[0K$3"
27+
trace_on
28+
}
29+
function section_end_internal() {
30+
echo -e "section_end:$(date +%s):$1\r\e[0K"
31+
trace_on
32+
}
33+
alias section_start_collap='trace_off ; section_start_internal true'
34+
alias section_start='trace_off ; section_start_internal false'
35+
alias section_end='trace_off ; section_end_internal '
36+
37+
function log_on_err() {
38+
echo -e "\\n\\n=======================================================\\n"
39+
echo "Symfony log:"
40+
if sudo test -f "$LOGFILE" ; then
41+
sudo cat "$LOGFILE"
42+
fi
43+
}
44+
45+
function show_phpinfo() {
46+
phpversion=$1
47+
section_start_collap phpinfo "Show the new PHP info"
48+
update-alternatives --set php /usr/bin/php"${phpversion}"
49+
php -v
50+
php -m
51+
section_end phpinfo
52+
}
53+
54+
# Show running command
55+
set -x

‎gitlab/default-nginx‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
server {
2+
listen 80 default_server;
3+
listen [::]:80 default_server;
4+
root /var/www/html;
5+
index index.html index.htm index.nginx-debian.html;
6+
server_name _;
7+
location / {
8+
# First attempt to serve request as file, then
9+
# as directory, then fall back to displaying a 404.
10+
try_files $uri $uri/ =404;
11+
}
12+
}

‎gitlab/jsontogitlab.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import json
2+
import sys
3+
import time
4+
import hashlib
5+
6+
storage1 ={}
7+
storage2 = {}
8+
9+
def cleanHash(toHash):
10+
return hashlib.sha224(toHash).hexdigest()
11+
12+
def sec_start(job,header):
13+
print('section_start:{}:{}{}{}'.format(int(time.time()),cleanHash(job),'\r\033[0K',header))
14+
15+
def sec_end(job):
16+
print('section_end:{}:{}'.format(int(time.time()),cleanHash(job)+'\r\033[0K'))
17+
18+
with open(sys.argv[1],'r') as f:
19+
data = json.load(f)
20+
for message in data['messages']:
21+
mtyp = message['type'].encode('utf-8', 'ignore')
22+
murl = message['url'].encode('utf-8', 'ignore')
23+
mmes = message['message'].encode('utf-8', 'ignore')
24+
if mtyp not in storage1.keys():
25+
storage1[mtyp] = {"messages":{}, "cnt":0}
26+
storage2[mtyp] = {"urls":{}, "cnt":0}
27+
if mmes not in storage1[mtyp]["messages"].keys():
28+
storage1[mtyp]["messages"][mmes] = {"urls":{}, "cnt":0}
29+
if murl not in storage2[mtyp]["urls"].keys():
30+
storage2[mtyp]["urls"][murl] = {"messages":{}, "cnt":0}
31+
if murl not in storage1[mtyp]["messages"][mmes]["urls"].keys():
32+
storage1[mtyp]["messages"][mmes]["urls"][murl] = 0
33+
if mmes not in storage2[mtyp]["urls"][murl]["messages"].keys():
34+
storage2[mtyp]["urls"][murl]["messages"][mmes] = 0
35+
storage1[mtyp]["messages"][mmes]["urls"][murl] += 1
36+
storage1[mtyp]["messages"][mmes]["cnt"] += 1
37+
storage1[mtyp]["cnt"] += 1
38+
storage2[mtyp]["urls"][murl]["messages"][mmes] += 1
39+
storage2[mtyp]["urls"][murl]["cnt"] += 1
40+
storage2[mtyp]["cnt"] += 1
41+
# Stats
42+
for key,value in sorted(storage1.items(), key=lambda x:x[1]['cnt']):
43+
print("Type: {}, Totalfound: {}".format(key, value["cnt"]))
44+
for key2,value2 in sorted(storage1[key]["messages"].items(), key=lambda x:x[1]['cnt'], reverse=True):
45+
sec_start(key+key2, key2)
46+
print("[{}] [{}%] Message: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2))
47+
for key3,value3 in sorted(storage1[key]["messages"][key2]["urls"].items(), key=lambda x:x[1], reverse=True):
48+
print("[{}%] URL: {}".format(round(100*value3/value2["cnt"],2), key3))
49+
sec_end(key+key2)
50+
for key2,value2 in sorted(storage2[key]["urls"].items(), key=lambda x:x[1]['cnt'], reverse=True):
51+
sec_start(key+key2, key2)
52+
print("[{}] [{}%] URL: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2))
53+
for key3,value3 in sorted(storage2[key]["urls"][key2]["messages"].items(), key=lambda x:x[1], reverse=True):
54+
print("[{}%] Message: {}".format(round(100*value3/value2["cnt"],2), key3))
55+
sec_end(key+key2)

0 commit comments

Comments
 (0)