GitHub action container

.github/jobs/base.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -x
+
+. .github/jobs/data/gha_ci_bashrc
+
+lsb_release -a
+
+cat > ~/.my.cnf <<EOF
+[client]
+host=sqlserver
+user=root
+password=domjudge
+EOF
+cat ~/.my.cnf
+
+# FIXME: This chicken-egg problem is annoying but let us bootstrap for now.
+echo "CREATE DATABASE IF NOT EXISTS \`domjudge\` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" | mysql
+echo "CREATE USER 'domjudge'@'%' IDENTIFIED BY 'domjudge';" | mysql
+echo "GRANT SELECT, INSERT, UPDATE, DELETE ON \`domjudge\`.* TO 'domjudge'@'%';" | mysql
+
+# Increase max_allowed_packet for following connections.
+echo "SET GLOBAL max_allowed_packet = 100*1024*1024;" | mysql
+
+# Test that SQL upgrade scripts also work with this setting
+if [ -n "${MYSQL_REQUIRE_PRIMARY_KEY:-}" ]; then
+    echo 'SET GLOBAL sql_require_primary_key = 1;' | mysql
+fi
+
+# Generate a dbpasswords file
+echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > etc/dbpasswords.secret
+
+# Generate APP_SECRET for symfony
+# shellcheck disable=SC2164
+( cd etc ; ./gensymfonysecret > symfony_app.secret )
+
+cat > webapp/config/static.yaml <<EOF
+parameters:
+    domjudge.version: unconfigured
+    domjudge.bindir: /bin
+    domjudge.etcdir: /etc
+    domjudge.wwwdir: /www
+    domjudge.webappdir: /webapp
+    domjudge.libdir: /lib
+    domjudge.sqldir: /sql
+    domjudge.libvendordir: /lib/vendor
+    domjudge.logdir: /output/log
+    domjudge.rundir: /output/run
+    domjudge.tmpdir: /output/tmp
+    domjudge.baseurl: http://localhost/domjudge
+EOF
+
+# install all php dependencies
+export APP_ENV="prod"
+composer install --no-scripts
+echo -e "\033[0m"
+
+# configure, make and install (but skip documentation)
+make configure
+./configure --with-baseurl='http://localhost/domjudge/' --with-domjudge-user=domjudge --with-judgehost_chrootdir=${DIR}/chroot/domjudge |& tee "$ARTIFACTS/configure.log"
+make build-scripts domserver judgehost docs |& tee "$ARTIFACTS/make.log"
+sudo make install-domserver install-judgehost install-docs |& tee -a "$ARTIFACTS/make.log"
+
+# setup database and add special user
+# shellcheck disable=SC2164
+cd /opt/domjudge/domserver
+setfacl -m u:www-data:r etc/restapi.secret etc/initial_admin_password.secret \
+                        etc/dbpasswords.secret etc/symfony_app.secret
+
+# configure and restart nginx
+sudo rm -f /etc/nginx/sites-enabled/*
+sudo cp /opt/domjudge/domserver/etc/nginx-conf /etc/nginx/sites-enabled/domjudge
+sudo /usr/sbin/nginx
+
+# configure and restart php-fpm
+# shellcheck disable=SC2154
+php_version="${version:-}"
+sudo cp /opt/domjudge/domserver/etc/domjudge-fpm.conf "/etc/php/$php_version/fpm/pool.d/domjudge-fpm.conf"
+echo "php_admin_value[date.timezone] = Europe/Amsterdam" | sudo tee -a "/etc/php/$php_version/fpm/pool.d/domjudge-fpm.conf"
+sudo /usr/sbin/php-fpm${php_version}
+
+
+passwd=$(cat etc/initial_admin_password.secret)
+echo "machine localhost login admin password $passwd" >> ~www-data/.netrc
+sudo -u www-data bin/dj_setup_database -uroot -pdomjudge bare-install
+
+# shellcheck disable=SC2154
+if [ -n "${integration:-}" ]; then
+	# Make sure admin has a team associated to insert submissions as well.
+	echo "UPDATE user SET teamid=1 WHERE userid=1;" | mysql domjudge
+fi
+
+sudo -u www-data bin/dj_setup_database -uroot -pdomjudge install-examples

.github/workflowscripts/baseinstall.sh → .github/jobs/baseinstall.sh

@@ -79,7 +79,7 @@ openssl req -nodes -new -x509 -keyout /tmp/server.key -out /tmp/server.crt -subj
 sudo cp /tmp/server.crt /usr/local/share/ca-certificates/
 sudo update-ca-certificates
 # shellcheck disable=SC2002
-cat "$(pwd)/.github/workflowscripts/nginx_extra" | sudo tee -a /etc/nginx/sites-enabled/domjudge
+cat "$(pwd)/.github/jobs/data/nginx_extra" | sudo tee -a /etc/nginx/sites-enabled/domjudge
 sudo nginx -t
 section_end
 

.github/jobs/data/gha_ci_bashrc

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Expand aliases for non-interactive shell
+shopt -s expand_aliases
+
+# Fail pipeline when variable is not set or individual command has an non-zero exitcode.
+set -euo pipefail
+
+# Show which commands are being run
+set -x
+
+# Chown the checked out files
+sudo chown -R domjudge:domjudge .
+
+# Shared constants between jobs
+export DIR=$(pwd)
+export GITSHA=$(git rev-parse HEAD || true)
+export PS4='(${BASH_SOURCE}:${LINENO}): - [$?] $ '
+export LOGFILE="/opt/domjudge/domserver/webapp/var/log/prod.log"
+
+# Functions to annotate the Github actions logs
+alias trace_on='set -x'
+alias trace_off='{ set +x; } 2>/dev/null'
+
+section_start_internal  () {
+    echo "::group::$1"
+    trace_on
+}
+
+section_end_internal () {
+    echo "::endgroup::"
+    trace_on
+}
+
+alias section_start='trace_off ; section_start_internal '
+alias section_end='trace_off ; section_end_internal '
+
+# Shared storage for all artifacts
+export ARTIFACTS="$DIR/artifacts"
+mkdir -p "$ARTIFACTS"
+
+function show_phpinfo() {
+    phpversion=$1
+    section_start phpinfo
+    update-alternatives --set php /usr/bin/php"${phpversion}"
+    php -v
+    php -m
+    section_end phpinfo
+}
+
+function log_on_err() {
+    echo -e "\\n\\n=======================================================\\n"
+    echo "Symfony log:"
+    if sudo test -f "$LOGFILE" ; then
+        sudo cat "$LOGFILE"
+    fi
+}
+
+set -eux

.github/jobs/unit-tests.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+. .github/jobs/data/gha_ci_bashrc
+
+section_start chown_checkout
+git config --global --add safe.directory /__w/domjudge/domjudge
+section_end chown_checkout
+
+section_start current_database_domjudge "Currently installed databases (domjudge)"
+set +eu
+echo "show databases" | mysql -hsqlserveranother -udomjudge -pdomjudge
+echo "show databases" | mysql -hsqlserver -udomjudge -pdomjudge
+set -eu
+section_end current_database_domjudge
+
+section_start current_database_root "Currently installed databases (root)"
+set +eu
+echo "show databases" | mysql -hsqlserver -uroot -pdomjudge
+set -eu
+section_end current_database_root
+
+export version=$1
+unittest=$2
+[ "$version" = "8.1" ] && CODECOVERAGE=1 || CODECOVERAGE=0
+
+show_phpinfo $version
+
+.github/jobs/base.sh
+
+# Add team to admin user
+echo "INSERT INTO userrole (userid, roleid) VALUES (1, 3);" | mysql domjudge
+echo "UPDATE user SET teamid = 1 WHERE userid = 1;" | mysql domjudge
+
+# Copy the .env.test file, as this is normally not done during
+# installation and we need it.
+cp webapp/.env.test /opt/domjudge/domserver/webapp/
+
+# We also need the composer.json for PHPunit to detect the correct directory.
+cp composer.json /opt/domjudge/domserver/
+
+cd /opt/domjudge/domserver
+
+export APP_ENV="test"
+
+# Run phpunit tests.
+pcov=""
+phpcov=""
+if [ "$CODECOVERAGE" -eq 1 ]; then
+    phpcov="-dpcov.enabled=1 -dpcov.directory=webapp/src"
+    pcov="--coverage-html=${PWD}/coverage-html --coverage-clover coverage.xml"
+fi
+set +e
+php $phpcov lib/vendor/bin/phpunit -c webapp/phpunit.xml.dist webapp/tests/$unittest --log-junit ${DIR}/unit-tests.xml --colors=never $pcov > "$ARTIFACTS"/phpunit.out
+UNITSUCCESS=$?
+set -e
+CNT=0
+if [ $CODECOVERAGE -eq 1 ]; then
+    CNT=$(sed -n '/Generating code coverage report/,$p' "$ARTIFACTS"/phpunit.out | grep -v DoctrineTestBundle | grep -cv ^$)
+    FILE=deprecation.txt
+    sed -n '/Generating code coverage report/,$p' "$ARTIFACTS"/phpunit.out > "$DIR/$FILE"
+    if [ $CNT -le 12 ]; then
+        STATE=success
+    else
+        STATE=failure
+    fi
+    ORIGINAL="gitlab.com/DOMjudge"
+    REPLACETO="domjudge.gitlab.io/-"
+    # Copied from CCS
+    #curl https://api.github.com/repos/domjudge/domjudge/statuses/$CI_COMMIT_SHA \
+    #  -X POST \
+    #  -H "Authorization: token $GH_BOT_TOKEN_OBSCURED" \
+    #  -H "Accept: application/vnd.github.v3+json" \
+    #  -d "{\"state\": \"$STATE\", \"target_url\": \"${CI_JOB_URL/$ORIGINAL/$REPLACETO}/artifacts/$FILE\", \"description\":\"Symfony deprecations\", \"context\": \"Symfony deprecation\"}"
+fi
+if [ $UNITSUCCESS -ne 0 ]; then
+    exit $UNITSUCCESS
+fi
+
+if [ $CODECOVERAGE -eq 1 ]; then
+    section_start uploadcoverage "Upload code coverage"
+    # Only upload when we got working unit-tests.
+    set +u # Uses some variables which are not set
+    # shellcheck disable=SC1090
+    . $DIR/.github/jobs/uploadcodecov.sh &>/dev/zero
+    set -u # Undo set dance
+    section_end uploadcoverage
+fi

.github/workflows/check-unit-codecov-update.yml

@@ -0,0 +1,21 @@
+name: Unit tests (Codecov script)
+on:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+
+jobs:
+  check-static-codecov:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download latest codecov upload script
+        run: wget https://codecov.io/bash -O newcodecov
+      - name: Detect changes to manually verify
+        run: diff newcodecov .github/jobs/uploadcodecov.sh
+

.github/workflows/codeql-analysis.yml

@@ -2,22 +2,22 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '16 12 * * 6'
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
 
 jobs:
   analyze:
+    # We can not run with our gitlab container
+    # CodeQL has missing .so files otherwise
     name: Analyze
     runs-on: ubuntu-latest
     env:
-      COMPILED: |
-        [
-          "cpp"
-        ]
+      COMPILED: "cpp"
     permissions:
       actions: read
       contents: read
@@ -27,24 +27,16 @@ jobs:
       fail-fast: false
       matrix:
         language: [ 'cpp', 'java', 'javascript', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
 
-    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Install needed tools,
     - name: Install required tools
       if: ${{ contains(env.COMPILED, matrix.language) }}
       run: |

.github/workflows/codespell.yml

@@ -1,20 +1,27 @@
 on:
-    pull_request:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+  pull_request:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
 
 name: Spell Check
 
 jobs:
-    codespell:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v2
-            - name: Rewrite Changelog to find new mistakes
-              run: awk '1;/Version 7.2.1 - 6 May 2020/{exit}' ChangeLog > latest_Changelog
-            - name: Get dirs to skip
-              id: list_to_csv
-              run: echo "::set-output name=SKIP::$(awk '{print $1}' .github/jobs/data/codespellignorefiles.txt | paste -s -d, -)"
-            - uses: codespell-project/actions-codespell@master
-              with:
-                  check_filenames: true
-                  ignore_words_file: .github/jobs/data/codespellignorewords.txt
-                  skip: ${{ steps.list_to_csv.outputs.SKIP }}
+  codespell:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Rewrite Changelog to find new mistakes
+        run: awk '1;/Version 7.2.1 - 6 May 2020/{exit}' ChangeLog > latest_Changelog
+      - name: Get dirs to skip
+        id: list_to_csv
+        run: echo "::set-output name=SKIP::$(awk '{print $1}' .github/jobs/data/codespellignorefiles.txt | paste -s -d, -)"
+      - uses: codespell-project/actions-codespell@master
+        with:
+          check_filenames: true
+          ignore_words_file: .github/jobs/data/codespellignorewords.txt
+          skip: ${{ steps.list_to_csv.outputs.SKIP }}

.github/workflows/codestyle.yml

@@ -5,6 +5,9 @@ on:
       - main
       - '[0-9]+.[0-9]+'
   pull_request:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
 
 jobs:
   syntax-job: