From eb1a56cc06bbaad74caf43359e577c518bf9e6ac Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 23 Nov 2024 09:36:06 +0100 Subject: [PATCH 01/10] Remove check for legacy multiple mysql users This was removed in version 3.3.0RC1 in 2012 so should not be relevant anymore by now. --- sql/dj_setup_database.in | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 0bbfe80e20..4ded25f528 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -351,11 +351,6 @@ bare-install|install) ;; upgrade) - # check for legacy dbpasswords.secret content - if grep -Eq ^team: $PASSWDFILE >/dev/null 2>&1 ; then - echo "Warning: please remove all non-jury users from $PASSWDFILE" - echo "You may also remove those users from MySQL." - fi read_dbpasswords # Check if we need to upgrade the Doctrine migrations table From 43a7d292a499f46ad4cf1fa1f1d43e9138a78f6f Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 23 Nov 2024 09:43:14 +0100 Subject: [PATCH 02/10] Remove some trivial helper functions They were actually leading to double password reading in the `load` command. --- sql/dj_setup_database.in | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 4ded25f528..47e8eb27bf 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -240,23 +240,6 @@ install_examples() ( cd "$EXAMPLEPROBDIR" && yes y | "$BINDIR"/import-contest ) } -uninstall_helper() -{ - read_dbpasswords - remove_db_users -} - -create_db_users_helper() -{ - read_dbpasswords - create_db_users - verbose "Created empty database and users." -} - -create_database_dump () { - mysqldump $(mysql_options) --opt --skip-lock-tables "$DBNAME" | pv | gzip > "$DATABASEDUMPDIR/${1}.sql.gz" -} - ### Script starts here ### # Parse command-line options: @@ -313,7 +296,8 @@ genpass) ;; uninstall) - uninstall_helper + read_dbpasswords + remove_db_users ;; install-examples) @@ -332,7 +316,8 @@ install-loadtest) ;; create-db-users) - create_db_users_helper + read_dbpasswords + create_db_users ;; bare-install|install) @@ -377,7 +362,8 @@ dump) exit 1 fi - if [ -f "${DATABASEDUMPDIR}/${DUMPNAME}.sql.gz" ]; then + DUMPFILE="${DATABASEDUMPDIR}/${DUMPNAME}.sql.gz" + if [ -f "$DUMPFILE" ]; then while true; do printf "Overwrite existing database dump (y/N)? " read -r yn @@ -387,7 +373,7 @@ dump) esac done fi - create_database_dump "$DUMPNAME" + mysqldump $(mysql_options) --opt --skip-lock-tables "$DBNAME" | pv | gzip > "$DUMPFILE" ;; load) @@ -429,8 +415,8 @@ load) fi read_dbpasswords - uninstall_helper - create_db_users_helper + remove_db_users + create_db_users pv "${FILE}" | gunzip | mysql "$DBNAME" ;; From 7bc194536ecc3e6e9ead8a8f0af9c3223dd8e192 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 25 Aug 2024 17:41:09 +0200 Subject: [PATCH 03/10] Add command to update DB user password to current from file There was no easy way to do this, besides also completely dropping and recreating an empty database. --- sql/dj_setup_database.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 47e8eb27bf..9725b29330 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -29,6 +29,7 @@ Commands: status check database installation status genpass generate DB,API,Symfony,admin password files create-db-users create (empty) database and users + update-password update DB user database to that in 'etc/dbpasswords.secret' install create database, example contest and users if not existing bare-install create database, setup defaults if not existing uninstall remove database users and database, INCLUDING ALL DATA! @@ -233,6 +234,17 @@ remove_db_users() verbose "DOMjudge database and user(s) removed." } +update_password() +{ + read_dbpasswords + ( + echo "ALTER USER '$domjudge_DBUSER'@'localhost' IDENTIFIED BY '$domjudge_PASSWD';" + echo "FLUSH PRIVILEGES;" + ) | mysql + verbose "ALTER USER '$domjudge_DBUSER'@'localhost' IDENTIFIED BY '$domjudge_PASSWD';" + verbose "Database user password updated from credentials file." +} + install_examples() { DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-example-data @@ -320,6 +332,10 @@ create-db-users) create_db_users ;; +update-password) + update_password + ;; + bare-install|install) read_dbpasswords create_db_users From e0067e4dbe747a42ae526fd73c0e03439903e546 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 23 Nov 2024 10:40:40 +0100 Subject: [PATCH 04/10] Also force update DB password --- misc-tools/force-passwords.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc-tools/force-passwords.in b/misc-tools/force-passwords.in index af7bf80092..0dc4331183 100755 --- a/misc-tools/force-passwords.in +++ b/misc-tools/force-passwords.in @@ -5,6 +5,8 @@ import subprocess webappdir = '@domserver_webappdir@' etcdir = '@domserver_etcdir@' +subprocess.run([bindir + '/dj_setup_database', 'update-password']) + with open(f'{etcdir}/restapi.secret', 'r') as f: while True: line = f.readline() From 8434849a8b12f3f1abc2bbc7ac3d04a817172a5f Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sat, 23 Nov 2024 10:40:19 +0100 Subject: [PATCH 05/10] Install Symfony console in domserver_bindir as dj_console This makes it easily available to admins. --- Makefile | 1 + misc-tools/force-passwords.in | 6 +++--- webapp/Makefile | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8584274dec..fcfd998979 100644 --- a/Makefile +++ b/Makefile @@ -223,6 +223,7 @@ inplace-install-l: ln -sf $(CURDIR)/judge/runpipe $(judgehost_bindir) ln -sf $(CURDIR)/judge/create_cgroups $(judgehost_bindir) ln -sf $(CURDIR)/sql/dj_setup_database $(domserver_bindir) + ln -sf $(CURDIR)/webapp/bin/console $(domserver_bindir)/dj_console # Create tmpdir and make tmpdir writable for webserver, # because judgehost-create-dirs sets wrong permissions: $(MKDIR_P) $(domserver_tmpdir) diff --git a/misc-tools/force-passwords.in b/misc-tools/force-passwords.in index 0dc4331183..505d1cd337 100755 --- a/misc-tools/force-passwords.in +++ b/misc-tools/force-passwords.in @@ -2,7 +2,7 @@ import subprocess -webappdir = '@domserver_webappdir@' +bindir = '@domserver_bindir@' etcdir = '@domserver_etcdir@' subprocess.run([bindir + '/dj_setup_database', 'update-password']) @@ -16,9 +16,9 @@ with open(f'{etcdir}/restapi.secret', 'r') as f: if len(tokens) == 4 and tokens[0] == 'default': user = tokens[2] password = tokens[3] - subprocess.run([webappdir + '/bin/console', 'domjudge:reset-user-password', user, password]) + subprocess.run([bindir + '/dj_console', 'domjudge:reset-user-password', user, password]) break with open(f'{etcdir}/initial_admin_password.secret', 'r') as f: password = f.readline().strip() - subprocess.run([webappdir + '/bin/console', 'domjudge:reset-user-password', 'admin', password]) + subprocess.run([bindir + '/dj_console', 'domjudge:reset-user-password', 'admin', password]) diff --git a/webapp/Makefile b/webapp/Makefile index f4000332c9..5eac62ea13 100644 --- a/webapp/Makefile +++ b/webapp/Makefile @@ -61,6 +61,8 @@ install-domserver: for d in bin config migrations public resources src templates tests vendor; do \ $(call install_tree,$(DESTDIR)$(domserver_webappdir),$$d) ; \ done +# Add Symlink to Symfony console that is in the standard path + ln -s $(domserver_webappdir)/bin/console $(DESTDIR)$(domserver_bindir)/dj_console # Change webapp/public/doc symlink ln -sf $(domjudge_docdir) $(DESTDIR)$(domserver_webappdir)/public/doc $(INSTALL_DATA) -t $(DESTDIR)$(domserver_webappdir) phpunit.xml.dist .env From 9bbac8ee55758252892813741c07044343b854fc Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Mon, 25 Nov 2024 09:02:18 +0100 Subject: [PATCH 06/10] Remove DB_FIRST_INSTALL environment variable This was used for only setting default configuration on first installation, but irrelevant since 77932ded5b8261ea. --- sql/dj_setup_database.in | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 9725b29330..8095fb706c 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -320,9 +320,7 @@ install-examples) install-loadtest) read_dbpasswords create_db_users - export DB_FIRST_INSTALL=1 symfony_console doctrine:migrations:migrate -n - unset DB_FIRST_INSTALL DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-default-data DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-gatling-data ;; @@ -339,9 +337,7 @@ update-password) bare-install|install) read_dbpasswords create_db_users - export DB_FIRST_INSTALL=1 symfony_console doctrine:migrations:migrate -n - unset DB_FIRST_INSTALL DBUSER=$domjudge_DBUSER PASSWD=$domjudge_PASSWD symfony_console domjudge:load-default-data if [ "$1" = "install" ]; then install_examples From 329095a850d7fa749d62f4f848bdee7e573ca12d Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Sun, 25 Aug 2024 15:44:23 +0200 Subject: [PATCH 07/10] URL encode variables that go into a MySQL credentials URI See for reference: - https://dev.mysql.com/doc/refman/8.0/en/connecting-using-uri-or-key-value-pairs.html#connecting-using-uri - https://symfony.com/doc/current/doctrine.html but note that we must use `rawurlencode` instead of `urlencode` which differ in how they encode a space (as tested). Fixes: #2651 Closes: #2502 as this is likely fixed but I couldn't reproduce it (cherry picked from commit abeefb36060a7d13c8a5a5c6aaac5e201e38bbb4) --- webapp/config/load_db_secrets.php | 6 +++++- webapp/config/packages/doctrine.yaml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/webapp/config/load_db_secrets.php b/webapp/config/load_db_secrets.php index ea4ca11b83..c3a8b3dfb5 100644 --- a/webapp/config/load_db_secrets.php +++ b/webapp/config/load_db_secrets.php @@ -36,7 +36,11 @@ function get_db_url(): string break; } - return sprintf('mysql://%s:%s@%s:%d/%s?serverVersion=5.7.0', $user, $pass, $host, $port ?? 3306, $db); + return sprintf( + 'mysql://%s:%s@%s:%d/%s?serverVersion=5.7.0', + rawurlencode($user), rawurlencode($pass), rawurlencode($host), + $port ?? 3306, rawurlencode($db) + ); } function get_app_secret(): string diff --git a/webapp/config/packages/doctrine.yaml b/webapp/config/packages/doctrine.yaml index d04de47c52..d4ba6543f6 100644 --- a/webapp/config/packages/doctrine.yaml +++ b/webapp/config/packages/doctrine.yaml @@ -7,7 +7,7 @@ doctrine: charset: utf8mb4 collate: utf8mb4_unicode_ci - url: '%env(resolve:DATABASE_URL)%' + url: '%env(DATABASE_URL)%' profiling_collect_backtrace: '%kernel.debug%' types: tinyint: App\Doctrine\DBAL\Types\TinyIntType From ce9eeda4d3e710573e10a78fbafb9c0d79c0fe32 Mon Sep 17 00:00:00 2001 From: Jaap Eldering Date: Fri, 22 Nov 2024 18:12:28 +0100 Subject: [PATCH 08/10] Fixes for supporting passwords with weird characters Use URL encoding in DATABASE_URL and return mysql_options as an array (via ugly global variable), so each element in it can be separately added to the command line using `@` for expansion. Because of changing the script to bash, also reverts some of the changes in 483200a7feb22f28b64c9fda72c. (cherry picked from commit ded9ef36c52e1611cb6f51fb3574d0313da1b2c0) --- sql/dj_setup_database.in | 46 ++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/sql/dj_setup_database.in b/sql/dj_setup_database.in index 8095fb706c..c4251484e2 100755 --- a/sql/dj_setup_database.in +++ b/sql/dj_setup_database.in @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # @configure_input@ # This script allows one to perform DOMjudge database setup actions. @@ -52,30 +52,47 @@ not have to pass any of the options above. EOF } +urlencode() +{ + php -r "echo rawurlencode('$1');" +} + +# This is global variable to be able to return the output from +# mysql_options() below as an array, which is not possible otherwise. +declare -a _mysql_options + mysql_options() { + local user pass + _mysql_options=() + # shellcheck disable=SC2153 if [ -n "$DBUSER" ]; then - _user="-u $DBUSER" - else - _user="${DBA_USER:+-u ${DBA_USER}}" + _mysql_options+=('-u' "$DBUSER") + elif [ -n "$DBA_USER" ]; then + _mysql_options+=('-u' "$DBA_USER") fi # shellcheck disable=SC2153 if [ -n "$PASSWD" ]; then - _pass="-p$PASSWD" - else - [ -n "$PROMPT_PASSWD" ] && _pass="-p" - [ -n "$DBA_PASSWD" ] && _pass="-p$DBA_PASSWD" + _mysql_options+=("-p$PASSWD") + elif [ -n "$DBA_PASSWD" ]; then + _mysql_options+=("-p$DBA_PASSWD") + elif [ -n "$PROMPT_PASSWD" ]; then + _mysql_options+=('-p') fi - [ -z "$USE_SOCKET" ] && port="-P$DBPORT" - echo $_user ${_pass:+"$_pass"} -h "$DBHOST" ${port:+"$port"} + _mysql_options+=('-h' "$DBHOST") + + if [ -z "$USE_SOCKET" ]; then + _mysql_options+=("-P$DBPORT") + fi } # Wrapper around mysql command to allow setting options, user, etc. mysql() { - command mysql $(mysql_options) --silent --skip-column-names "$@" + mysql_options + command mysql "${_mysql_options[@]}" --silent --skip-column-names "$@" } # Quick shell hack to get a key from an INI file. @@ -126,10 +143,13 @@ symfony_console() fi if [ -n "$DBA_USER" ]; then + user=$(urlencode "${DBA_USER}") + host=$(urlencode "${domjudge_DBHOST}") + db=$(urlencode "${domjudge_DBNAME}") if [ -n "$DBA_PASSWD" ]; then - DATABASE_URL=mysql://${DBA_USER}:${DBA_PASSWD}@${domjudge_DBHOST}:${domjudge_DBPORT}/${domjudge_DBNAME} + DATABASE_URL="mysql://$user:$(urlencode "${DBA_PASSWD}")@$host:${domjudge_DBPORT}/$db" else - DATABASE_URL=mysql://${DBA_USER}@${domjudge_DBHOST}:${domjudge_DBPORT}/${domjudge_DBNAME} + DATABASE_URL="mysql://$user@$host:${domjudge_DBPORT}/$db" fi fi fi From cf365e197c5451c73689ec077b977cdeacb1c5cd Mon Sep 17 00:00:00 2001 From: Tobias Werth Date: Mon, 25 Nov 2024 10:03:41 +0100 Subject: [PATCH 09/10] debug --- .github/jobs/baseinstall.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 2c61bc2a7b..3f8755a547 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -64,7 +64,8 @@ mysql_root "show databases" mysql_root "SELECT CURRENT_USER();" mysql_root "SELECT USER();" mysql_root "SELECT user,host FROM mysql.user" -echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret +#echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret +echo 'unused:sqlserver:domjudge:domjudge:domjudge_db-pw+% #$*)@(!/;,.:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret mysql_user "SELECT CURRENT_USER();" mysql_user "SELECT USER();" section_end From b65f6241859aca7acd1e515b06b6979db24b7ed0 Mon Sep 17 00:00:00 2001 From: Tobias Werth Date: Mon, 25 Nov 2024 10:11:37 +0100 Subject: [PATCH 10/10] debug --- .github/jobs/baseinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/baseinstall.sh b/.github/jobs/baseinstall.sh index 3f8755a547..59154651db 100755 --- a/.github/jobs/baseinstall.sh +++ b/.github/jobs/baseinstall.sh @@ -65,7 +65,7 @@ mysql_root "SELECT CURRENT_USER();" mysql_root "SELECT USER();" mysql_root "SELECT user,host FROM mysql.user" #echo "unused:sqlserver:domjudge:domjudge:domjudge:3306" > /opt/domjudge/domserver/etc/dbpasswords.secret -echo 'unused:sqlserver:domjudge:domjudge:domjudge_db-pw+% #$*)@(!/;,.:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret +echo 'unused:sqlserver:domjudge:domjudge:simplepw:3306' > /opt/domjudge/domserver/etc/dbpasswords.secret mysql_user "SELECT CURRENT_USER();" mysql_user "SELECT USER();" section_end