diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index d292fc639c..2064339d94 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -4,11 +4,22 @@ export version="$1" db=${2:-install} +phpversion="${3:-8.1}" +# If this script is called from unit-tests.sh, we use the test environment +export APP_ENV="${4:-prod}" + +# In the test environment, we need to use a different database +[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test + +MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root} set -eux -PHPVERSION=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') -export PHPVERSION +if [ -z "$phpversion" ]; then +phpversion=$(php -r 'echo PHP_MAJOR_VERSION.".".PHP_MINOR_VERSION."\n";') +fi + +show_phpinfo "$phpversion" section_start "Run composer" export APP_ENV="dev" @@ -41,8 +52,10 @@ else --enable-judgehost-build=no | tee "$ARTIFACTS"/configure.txt make domserver make install-domserver + rm -rf /opt/domjudge/domserver/webapp/public/doc + cp -r doc /opt/domjudge/domserver/webapp/public/ + find /opt/domjudge/domserver -name DOMjudgelogo.pdf fi - section_end section_start "SQL settings" @@ -50,13 +63,13 @@ cat > ~/.my.cnf < /opt/domjudge/domserver/etc/dbpasswords.secret +mysql_root "SHOW GLOBAL STATUS LIKE 'Connection_errors_%'" +mysql_root "SHOW VARIABLES LIKE '%_timeout'" +echo "unused:sqlserver:$DATABASE_NAME:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret mysql_user "SELECT CURRENT_USER();" mysql_user "SELECT USER();" section_end if [ "${db}" = "install" ]; then section_start "Install DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot bare-install + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install section_end elif [ "${db}" = "upgrade" ]; then section_start "Upgrade DOMjudge database" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot upgrade + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} upgrade section_end fi @@ -92,7 +107,7 @@ cp /proc/cmdline "$ARTIFACTS"/cmdline.txt section_end section_start "Setup webserver" -cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$PHPVERSION"/fpm/pool.d/domjudge.conf +cp /opt/domjudge/domserver/etc/domjudge-fpm.conf /etc/php/"$phpversion"/fpm/pool.d/domjudge.conf rm -f /etc/nginx/sites-enabled/* cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge @@ -106,7 +121,7 @@ nginx -t section_end section_start "Show webserver is up" -for service in nginx php${PHPVERSION}-fpm; do +for service in nginx php${phpversion}-fpm; do service "$service" restart service "$service" status done @@ -114,30 +129,33 @@ section_end if [ "${db}" = "install" ]; then section_start "Install the example data" - /opt/domjudge/domserver/bin/dj_setup_database -uroot -proot install-examples | tee -a "$ARTIFACTS/mysql.txt" + if [ "$version" = "unit" ]; then + # Make sure admin has no team associated so we will not insert submissions during unit tests. + mysql_root "UPDATE user SET teamid=null WHERE userid=1;" $DATABASE_NAME + fi + /opt/domjudge/domserver/bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples | tee -a "$ARTIFACTS/mysql.txt" section_end fi section_start "Setup user" # We're using the admin user in all possible roles -mysql_root "DELETE FROM userrole WHERE userid=1;" domjudge +mysql_root "DELETE FROM userrole WHERE userid=1;" $DATABASE_NAME if [ "$version" = "team" ]; then # Add team to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" domjudge - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME elif [ "$version" = "jury" ]; then # Add jury to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 2);" $DATABASE_NAME elif [ "$version" = "balloon" ]; then # Add balloon to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 4);" $DATABASE_NAME elif [ "$version" = "admin" ]; then # Add admin to admin user - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" domjudge -elif [ "$version" = "all" ]; then - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" domjudge - mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" domjudge - mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" domjudge + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME +elif [ "$version" = "all" ] || [ "$version" = "unit" ]; then + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 1);" $DATABASE_NAME + mysql_root "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" $DATABASE_NAME + mysql_root "UPDATE user SET teamid = 1 WHERE userid = 1;" $DATABASE_NAME fi section_end - diff --git a/.github/jobs/ci_settings.sh b/.github/jobs/ci_settings.sh index 350b07cf09..391996752b 100644 --- a/.github/jobs/ci_settings.sh +++ b/.github/jobs/ci_settings.sh @@ -34,6 +34,15 @@ mysql_user () { echo "$1" | mysql -udomjudge -pdomjudge ${2:-} | tee -a "$ARTIFACTS"/mysql.txt } +show_phpinfo() { + phpversion=$1 + section_start "Show the new PHP info" + update-alternatives --set php /usr/bin/php"${phpversion}" + php -v + php -m + section_end +} + section_start () { if [ "$#" -ne 1 ]; then echo "Only 1 argument is needed for GHA, 2 was needed for GitLab." diff --git a/.github/jobs/jsontogha.py b/.github/jobs/jsontogha.py new file mode 100644 index 0000000000..b2916904b2 --- /dev/null +++ b/.github/jobs/jsontogha.py @@ -0,0 +1,59 @@ +import json +import sys +import time +import hashlib + +storage1 = {} +storage2 = {} + + +def cleanHash(toHash): + return hashlib.sha224(toHash).hexdigest() + + +def sec_start(job, header): + print('section_start\r\033[0K'+header) + + +def sec_end(job): + print('section_end\r\033[0K') + + +with open(sys.argv[1], 'r') as f: + data = json.load(f) + for message in data['messages']: + mtyp = message['type'].encode('utf-8', 'ignore') + murl = message['url'].encode('utf-8', 'ignore') + mmes = message['message'].encode('utf-8', 'ignore') + if mtyp not in storage1.keys(): + storage1[mtyp] = {"messages": {}, "cnt": 0} + storage2[mtyp] = {"urls": {}, "cnt": 0} + if mmes not in storage1[mtyp]["messages"].keys(): + storage1[mtyp]["messages"][mmes] = {"urls": {}, "cnt": 0} + if murl not in storage2[mtyp]["urls"].keys(): + storage2[mtyp]["urls"][murl] = {"messages": {}, "cnt": 0} + if murl not in storage1[mtyp]["messages"][mmes]["urls"].keys(): + storage1[mtyp]["messages"][mmes]["urls"][murl] = 0 + if mmes not in storage2[mtyp]["urls"][murl]["messages"].keys(): + storage2[mtyp]["urls"][murl]["messages"][mmes] = 0 + storage1[mtyp]["messages"][mmes]["urls"][murl] += 1 + storage1[mtyp]["messages"][mmes]["cnt"] += 1 + storage1[mtyp]["cnt"] += 1 + storage2[mtyp]["urls"][murl]["messages"][mmes] += 1 + storage2[mtyp]["urls"][murl]["cnt"] += 1 + storage2[mtyp]["cnt"] += 1 + +for key, value in sorted(storage1.items(), key=lambda x: x[1]['cnt']): + print("Type: {}, Totalfound: {}".format(key, value["cnt"])) + for key2, value2 in sorted(storage1[key]["messages"].items(), key=lambda x: x[1]['cnt'], reverse=True): + sec_start(key+key2, key2) + print("[{}] [{}%] Message: {}".format(key, round(100*value2["cnt"]/value["cnt"], 2), key2)) + for key3, value3 in sorted(storage1[key]["messages"][key2]["urls"].items(), key=lambda x: x[1], reverse=True): + print("[{}%] URL: {}".format(round(100*value3/value2["cnt"], 2), key3)) + sec_end(key+key2) + for key2, value2 in sorted(storage2[key]["urls"].items(), key=lambda x: x[1]['cnt'], reverse=True): + sec_start(key+key2, key2) + print("[{}] [{}%] URL: {}".format(key, round(100*value2["cnt"]/value["cnt"], 2), key2)) + for key3, value3 in sorted(storage2[key]["urls"][key2]["messages"].items(), key=lambda x: x[1], reverse=True): + print("[{}%] Message: {}".format(round(100*value3/value2["cnt"], 2), key3)) + sec_end(key+key2) diff --git a/.github/jobs/unit-tests.sh b/.github/jobs/unit-tests.sh new file mode 100755 index 0000000000..b55fb86ce3 --- /dev/null +++ b/.github/jobs/unit-tests.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +. .github/jobs/ci_settings.sh + +DIR="$PWD" + +export version=$1 +unittest=$2 +[ "$version" = "8.1" ] && CODECOVERAGE=1 || CODECOVERAGE=0 + +# Set up +export unit=1 + +# Add team to admin user +echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql domjudge_test + +# Copy the .env.test file, as this is normally not done during +# installation and we need it. +cp webapp/.env.test /opt/domjudge/domserver/webapp/ + +# We also need the composer.json for PHPunit to detect the correct directory. +cp webapp/composer.json /opt/domjudge/domserver/webapp/ + +cd /opt/domjudge/domserver + +# Run phpunit tests. +pcov="" +phpcov="" +if [ "$CODECOVERAGE" -eq 1 ]; then + phpcov="-dpcov.enabled=1 -dpcov.directory=webapp/src" + pcov="--coverage-html=${DIR}/coverage-html --coverage-clover coverage.xml" +fi +set +e +echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret +php $phpcov webapp/bin/phpunit -c webapp/phpunit.xml.dist webapp/tests/$unittest --log-junit ${ARTIFACTS}/unit-tests.xml --colors=never $pcov > "$ARTIFACTS"/phpunit.out +UNITSUCCESS=$? + +# Store the unit tests also in the root for the GHA +cp $ARTIFACTS/unit-tests.xml $DIR/ + +# Make sure the log exists before copy +touch ${DIR}/webapp/var/log/test.log +cp ${DIR}/webapp/var/log/*.log "$ARTIFACTS"/ + +set -e +CNT=0 +THRESHOLD=32 +if [ $CODECOVERAGE -eq 1 ]; then + CNT=$(sed -n '/Generating code coverage report/,$p' "$ARTIFACTS"/phpunit.out | grep -v DoctrineTestBundle | grep -cv ^$) +fi + +if [ $UNITSUCCESS -ne 0 ] || [ $CNT -gt $THRESHOLD ]; then + exit 1 +fi + +if [ $CODECOVERAGE -eq 1 ]; then + section_start "Upload code coverage" + # Only upload when we got working unit-tests. + set +u # Uses some variables which are not set + # shellcheck disable=SC1090 + . $DIR/.github/jobs/uploadcodecov.sh &>> "$ARTIFACTS"/codecov.log + section_end +fi diff --git a/.github/jobs/webstandard.sh b/.github/jobs/webstandard.sh index 66d5641aa6..c1fd988eca 100755 --- a/.github/jobs/webstandard.sh +++ b/.github/jobs/webstandard.sh @@ -123,7 +123,7 @@ if [ "$TEST" = "w3cval" ]; then NEWFOUNDERRORS=$("$DIR"/vnu-runtime-image/bin/vnu --errors-only --exit-zero-always --skip-non-$typ --format gnu $FLTR "$URL" 2>&1 | wc -l) FOUNDERR=$((NEWFOUNDERRORS+FOUNDERR)) python3 -m "json.tool" < result.json > "$ARTIFACTS/w3c$typ$URL.json" - trace_off; python3 gitlab/jsontogitlab.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on + trace_off; python3 .github/jobs/jsontogha.py "$ARTIFACTS/w3c$typ$URL.json"; trace_on section_end done else diff --git a/.github/workflows/database-upgrade.yml b/.github/workflows/database-upgrade.yml index dd3fe7a351..47b6a14ed1 100644 --- a/.github/workflows/database-upgrade.yml +++ b/.github/workflows/database-upgrade.yml @@ -32,5 +32,5 @@ jobs: run: echo "pass" > /opt/domjudge/domserver/etc/initial_admin_password.secret - name: Check for Errors in the Upgrade run: mysql -hsqlserver -uroot -proot -e "SHOW TABLES FROM domjudge;" - - name: Check for Errors in Domjudge Web + - name: Check for Errors in DOMjudge Webinterface run: .github/jobs/webstandard.sh none admin diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b26abc8118..7455125a7f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,4 +1,5 @@ name: Unit tests +# We can speedup with: https://github.com/actions/cache on: merge_group: pull_request: @@ -8,7 +9,7 @@ on: jobs: check-static-codecov: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Download latest codecov upload script @@ -16,3 +17,65 @@ jobs: - name: Detect changes to manually verify run: diff newcodecov .github/jobs/uploadcodecov.sh + unit-tests: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + container: + image: domjudge/gitlabci:24.04 + services: + sqlserver: + image: mariadb + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: domjudge + MYSQL_PASSWORD: domjudge + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 + strategy: + matrix: + PHPVERSION: [8.1, 8.4] + TEST: [Unit, E2E] + steps: + - uses: actions/checkout@v4 + - name: info + run: | + cat /proc/cmdline && echo && + cat /proc/mounts && echo && + ls -al /sys/fs/cgroup && echo && + uname -a && echo && + stat -fc %T /sys/fs/cgroup && echo && + cat /proc/self/cgroup && echo && + cat /proc/cpuinfo + - name: pstree + run: pstree -p + - name: Install DOMjudge + run: .github/jobs/baseinstall.sh unit install ${{ matrix.PHPVERSION }} test + - name: Check nginx + run: curl -v https://localhost/domjudge/ + - name: Run the unit-tests + run: .github/jobs/unit-tests.sh ${{ matrix.PHPVERSION }} ${{ matrix.TEST }} + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: ${{ !cancelled() }} + with: + comment_mode: changes in failures + check_name: unit-tests-${{ matrix.PHPVERSION }}-${{ matrix.TEST }}.xml + files: unit-tests-${{ matrix.PHPVERSION }}-${{ matrix.TEST }}.xml + - name: Get SQL logs + run: docker logs "${{ job.services.sqlserver.id }}" + - name: Collect docker logs on failure + if: ${{ !cancelled() }} + uses: jwalton/gh-docker-logs@v1 + with: + dest: '/tmp/docker-logs' + - name: Upload all logs/artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: Logs-${{ matrix.PHPVERSION }}-${{ matrix.TEST }} + path: | + /var/log/nginx + /opt/domjudge/domserver/webapp/var/log/*.log + /tmp/docker-logs + /tmp/artifacts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9f73622022..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,11 +0,0 @@ -include: - - '/gitlab/ci/unit.yml' - - '/gitlab/ci/template.yml' - -stages: - - test - - unit - - style - - ci_checks - -image: domjudge/gitlabci:24.04 diff --git a/gitlab/base.sh b/gitlab/base.sh deleted file mode 100755 index a07f026d5b..0000000000 --- a/gitlab/base.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -. gitlab/ci_settings.sh - -# If this script is called from unit.sh, we use the test environment -export APP_ENV="${1:-prod}" - -# In the test environment, we need to use a different database -[ "$APP_ENV" = "prod" ] && DATABASE_NAME=domjudge || DATABASE_NAME=domjudge_test - -lsb_release -a - -cat > ~/.my.cnf < etc/dbpasswords.secret - -# Generate APP_SECRET for symfony -# shellcheck disable=SC2164 -( cd etc ; ./gensymfonysecret > symfony_app.secret ) - -cat > webapp/config/static.yaml <> ~www-data/.netrc -sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} bare-install - -# shellcheck disable=SC2154 -if [ -n "${integration:-}" ]; then - # Make sure admin has a team associated to insert submissions as well. - echo "UPDATE user SET teamid=1 WHERE userid=1;" | mysql domjudge -elif [ -n "${unit:-}" ]; then - # Make sure admin has no team associated so we will not insert submissions during unit tests. - echo "UPDATE user SET teamid=null WHERE userid=1;" | mysql domjudge_test -fi - -sudo -Eu www-data bin/dj_setup_database -uroot -p${MYSQL_ROOT_PASSWORD} install-examples diff --git a/gitlab/ci/template.yml b/gitlab/ci/template.yml deleted file mode 100644 index e810df854c..0000000000 --- a/gitlab/ci/template.yml +++ /dev/null @@ -1,64 +0,0 @@ -# This placeholder job tries to start as soon as possible -.clean_ordering: - needs: [] - retry: - max: 2 #Max is 2, set when gitlab is flacky - when: - - always - script: - - /bin/true - -.tiny_job: - extends: [.clean_ordering] - timeout: 4m - -.short_job: - extends: [.clean_ordering] - timeout: 7m - -.normal_job: - extends: [.clean_ordering] - timeout: 20m - -.long_job: - extends: [.clean_ordering] - timeout: 30m - -.cached_vendor: - extends: [.clean_ordering] - cache: - key: webappvendor-20240623 - paths: - - webapp/vendor/ - -.mysql_job: - script: - - /bin/true - services: - - name: mysql - alias: sqlserver - -.mariadb_job: - script: - - /bin/true - services: - - name: mariadb - alias: sqlserver - -.phpsupported_job: - script: - - /bin/true - parallel: - matrix: - - PHPVERSION: ["8.1","8.2","8.3", "8.4"] - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0","1"] - -.phpsupported_job_pr: - script: - - /bin/true - parallel: - matrix: - - PHPVERSION: ["8.3"] - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0"] diff --git a/gitlab/ci/unit.yml b/gitlab/ci/unit.yml deleted file mode 100644 index 5bf7aab8c7..0000000000 --- a/gitlab/ci/unit.yml +++ /dev/null @@ -1,42 +0,0 @@ -.unit_job: - extends: [.normal_job,.cached_vendor] - stage: unit - # Disabled for now as it drastically speeds up running unit tests and we don't use it yet - # before_script: - # - apt-get update -yqq - # - apt-get install php-xdebug -yqq - variables: - MYSQL_ROOT_PASSWORD: password - MARIADB_PORT_3306_TCP_ADDR: sqlserver - script: - - set -eux - - if [ -z ${PHPVERSION+x} ]; then export PHPVERSION=8.1; fi - - if [ -z ${TEST+x} ]; then export TEST="UNIT"; fi - - if [ "$TEST" = "UNIT" ] && [ "$CRAWL_SHADOW_MODE" != "0" ]; then exit 0; fi - - if [ "$TEST" = "E2E" ] && [ "$CRAWL_SHADOW_MODE" != "0" ] && [ "$CI_COMMIT_BRANCH" != "main" ]; then exit 0; fi - - export CRAWL_SHADOW_MODE - - ./gitlab/unit-tests.sh $PHPVERSION $TEST - artifacts: - when: always - paths: - - unit-tests.xml - - coverage-html - - deprecation.txt - - duration - - gitlabartifacts - reports: - junit: - - unit-tests.xml - -run unit tests: - extends: [.mariadb_job,.phpsupported_job,.unit_job] - -run unit tests (PR): - extends: [.mariadb_job,.phpsupported_job_pr,.unit_job] - -run unit tests (MySQL): - extends: [.mysql_job,.unit_job] - parallel: - matrix: - - TEST: ["E2E","Unit"] - CRAWL_SHADOW_MODE: ["0"] diff --git a/gitlab/ci_settings.sh b/gitlab/ci_settings.sh deleted file mode 100755 index dfea8a1925..0000000000 --- a/gitlab/ci_settings.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Fail pipeline when variable is not set or individual command has an non-zero exitcode. -set -euo pipefail - -shopt -s expand_aliases - -# Shared constants between jobs -DIR=$(pwd) -GITSHA=$(git rev-parse HEAD || true) -export DIR -export GITSHA -export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ ' -export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log" - -CCS_SPECS_PINNED_SHA1='a68aff54c4e60fc2bff2fc5c36c119bffa4d30f1' - -# Shared storage for all artifacts -export GITLABARTIFACTS="$DIR/gitlabartifacts" -mkdir -p "$GITLABARTIFACTS" - -# Functions to annotate the GitLab logs -alias trace_on='set -x' -alias trace_off='{ set +x; } 2>/dev/null' -function section_start_internal() { - echo -e "section_start:$(date +%s):$2[collapsed=$1]\r\e[0K$3" - trace_on -} -function section_end_internal() { - echo -e "section_end:$(date +%s):$1\r\e[0K" - trace_on -} -alias section_start_collap='trace_off ; section_start_internal true' -alias section_start='trace_off ; section_start_internal false' -alias section_end='trace_off ; section_end_internal ' - -function log_on_err() { - echo -e "\\n\\n=======================================================\\n" - echo "Symfony log:" - if sudo test -f "$LOGFILE" ; then - sudo cat "$LOGFILE" - fi -} - -function show_phpinfo() { - phpversion=$1 - section_start_collap phpinfo "Show the new PHP info" - update-alternatives --set php /usr/bin/php"${phpversion}" - php -v - php -m - section_end phpinfo -} - -# Show running command -set -x diff --git a/gitlab/default-nginx b/gitlab/default-nginx deleted file mode 100644 index f27f32a0ec..0000000000 --- a/gitlab/default-nginx +++ /dev/null @@ -1,12 +0,0 @@ -server { - listen 80 default_server; - listen [::]:80 default_server; - root /var/www/html; - index index.html index.htm index.nginx-debian.html; - server_name _; - location / { - # First attempt to serve request as file, then - # as directory, then fall back to displaying a 404. - try_files $uri $uri/ =404; - } -} diff --git a/gitlab/jsontogitlab.py b/gitlab/jsontogitlab.py deleted file mode 100644 index a1e8e69327..0000000000 --- a/gitlab/jsontogitlab.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -import sys -import time -import hashlib - -storage1 ={} -storage2 = {} - -def cleanHash(toHash): - return hashlib.sha224(toHash).hexdigest() - -def sec_start(job,header): - print('section_start:{}:{}{}{}'.format(int(time.time()),cleanHash(job),'\r\033[0K',header)) - -def sec_end(job): - print('section_end:{}:{}'.format(int(time.time()),cleanHash(job)+'\r\033[0K')) - -with open(sys.argv[1],'r') as f: - data = json.load(f) - for message in data['messages']: - mtyp = message['type'].encode('utf-8', 'ignore') - murl = message['url'].encode('utf-8', 'ignore') - mmes = message['message'].encode('utf-8', 'ignore') - if mtyp not in storage1.keys(): - storage1[mtyp] = {"messages":{}, "cnt":0} - storage2[mtyp] = {"urls":{}, "cnt":0} - if mmes not in storage1[mtyp]["messages"].keys(): - storage1[mtyp]["messages"][mmes] = {"urls":{}, "cnt":0} - if murl not in storage2[mtyp]["urls"].keys(): - storage2[mtyp]["urls"][murl] = {"messages":{}, "cnt":0} - if murl not in storage1[mtyp]["messages"][mmes]["urls"].keys(): - storage1[mtyp]["messages"][mmes]["urls"][murl] = 0 - if mmes not in storage2[mtyp]["urls"][murl]["messages"].keys(): - storage2[mtyp]["urls"][murl]["messages"][mmes] = 0 - storage1[mtyp]["messages"][mmes]["urls"][murl] += 1 - storage1[mtyp]["messages"][mmes]["cnt"] += 1 - storage1[mtyp]["cnt"] += 1 - storage2[mtyp]["urls"][murl]["messages"][mmes] += 1 - storage2[mtyp]["urls"][murl]["cnt"] += 1 - storage2[mtyp]["cnt"] += 1 - # Stats -for key,value in sorted(storage1.items(), key=lambda x:x[1]['cnt']): - print("Type: {}, Totalfound: {}".format(key, value["cnt"])) - for key2,value2 in sorted(storage1[key]["messages"].items(), key=lambda x:x[1]['cnt'], reverse=True): - sec_start(key+key2, key2) - print("[{}] [{}%] Message: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2)) - for key3,value3 in sorted(storage1[key]["messages"][key2]["urls"].items(), key=lambda x:x[1], reverse=True): - print("[{}%] URL: {}".format(round(100*value3/value2["cnt"],2), key3)) - sec_end(key+key2) - for key2,value2 in sorted(storage2[key]["urls"].items(), key=lambda x:x[1]['cnt'], reverse=True): - sec_start(key+key2, key2) - print("[{}] [{}%] URL: {}".format(key, round(100*value2["cnt"]/value["cnt"],2), key2)) - for key3,value3 in sorted(storage2[key]["urls"][key2]["messages"].items(), key=lambda x:x[1], reverse=True): - print("[{}%] Message: {}".format(round(100*value3/value2["cnt"],2), key3)) - sec_end(key+key2) diff --git a/gitlab/unit-tests.sh b/gitlab/unit-tests.sh deleted file mode 100755 index b412a060cb..0000000000 --- a/gitlab/unit-tests.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -. gitlab/ci_settings.sh - -export version=$1 -unittest=$2 -[ "$version" = "8.1" ] && CODECOVERAGE=1 || CODECOVERAGE=0 - -show_phpinfo $version - -# Set up -export unit=1 -"$( dirname "${BASH_SOURCE[0]}" )"/base.sh test - -# Add team to admin user -echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql domjudge_test - -# Copy the .env.test file, as this is normally not done during -# installation and we need it. -cp webapp/.env.test /opt/domjudge/domserver/webapp/ - -# We also need the composer.json for PHPunit to detect the correct directory. -cp webapp/composer.json /opt/domjudge/domserver/webapp/ - -cd /opt/domjudge/domserver - -# Run phpunit tests. -pcov="" -phpcov="" -if [ "$CODECOVERAGE" -eq 1 ]; then - phpcov="-dpcov.enabled=1 -dpcov.directory=webapp/src" - pcov="--coverage-html=${CI_PROJECT_DIR}/coverage-html --coverage-clover coverage.xml" -fi -set +e -php $phpcov webapp/bin/phpunit -c webapp/phpunit.xml.dist webapp/tests/$unittest --log-junit ${CI_PROJECT_DIR}/unit-tests.xml --colors=never $pcov > "$GITLABARTIFACTS"/phpunit.out -UNITSUCCESS=$? -set -e -CNT=0 -THRESHOLD=32 -if [ $CODECOVERAGE -eq 1 ]; then - CNT=$(sed -n '/Generating code coverage report/,$p' "$GITLABARTIFACTS"/phpunit.out | grep -v DoctrineTestBundle | grep -cv ^$) - FILE=deprecation.txt - sed -n '/Generating code coverage report/,$p' "$GITLABARTIFACTS"/phpunit.out > ${CI_PROJECT_DIR}/$FILE - if [ $CNT -le $THRESHOLD ]; then - STATE=success - else - STATE=failure - fi - ORIGINAL="gitlab.com/DOMjudge" - REPLACETO="domjudge.gitlab.io/-" - # Copied from CCS - curl https://api.github.com/repos/domjudge/domjudge/statuses/$CI_COMMIT_SHA \ - -X POST \ - -H "Authorization: token $GH_BOT_TOKEN_OBSCURED" \ - -H "Accept: application/vnd.github.v3+json" \ - -d "{\"state\": \"$STATE\", \"target_url\": \"${CI_JOB_URL/$ORIGINAL/$REPLACETO}/artifacts/$FILE\", \"description\":\"Symfony deprecations ($version)\", \"context\": \"Symfony deprecation ($version)\"}" -fi -if [ $UNITSUCCESS -eq 0 ]; then - STATE=success -else - STATE=failure -fi -cp webapp/var/log/test.log "$GITLABARTIFACTS"/test.log - -curl https://api.github.com/repos/domjudge/domjudge/statuses/$CI_COMMIT_SHA \ - -X POST \ - -H "Authorization: token $GH_BOT_TOKEN_OBSCURED" \ - -H "Accept: application/vnd.github.v3+json" \ - -d "{\"state\": \"$STATE\", \"target_url\": \"${CI_PIPELINE_URL}/test_report\", \"description\":\"Unit tests\", \"context\": \"unit_tests ($version)\"}" -if [ $UNITSUCCESS -ne 0 ] || [ $CNT -gt $THRESHOLD ]; then - exit 1 -fi - -if [ $CODECOVERAGE -eq 1 ]; then - section_start_collap uploadcoverage "Upload code coverage" - # Only upload when we got working unit-tests. - set +u # Uses some variables which are not set - # shellcheck disable=SC1090 - . $DIR/.github/jobs/uploadcodecov.sh 1>/dev/zero 2>/dev/zero - section_end uploadcoverage -fi