Merge remote-tracking branch 'origin/main' into K8SPS-417

Author: pooknull

api/v1alpha1/perconaservermysql_types.go

@@ -128,6 +128,8 @@ type MySQLSpec struct {
 	SidecarVolumes []corev1.Volume    `json:"sidecarVolumes,omitempty"`
 	SidecarPVCs    []SidecarPVC       `json:"sidecarPVCs,omitempty"`
 
+	VaultSecretName string `json:"vaultSecretName,omitempty"`
+
 	PodSpec `json:",inline"`
 }
 
@@ -416,6 +418,8 @@ type MySQLRouterSpec struct {
 
 	Expose ServiceExpose `json:"expose,omitempty"`
 
+	Ports []corev1.ServicePort `json:"ports,omitempty"`
+
 	PodSpec `json:",inline"`
 }
 
@@ -591,6 +595,16 @@ func (cr *PerconaServerMySQL) SetVersion() {
 	cr.Spec.CRVersion = version.Version()
 }
 
+func (cr *PerconaServerMySQL) Version() *v.Version {
+	return v.Must(v.NewVersion(cr.Spec.CRVersion))
+}
+
+// CompareVersion compares given version to current version.
+// Returns -1, 0, or 1 if given version is smaller, equal, or larger than the current version, respectively.
+func (cr *PerconaServerMySQL) CompareVersion(ver string) int {
+	return cr.Version().Compare(v.Must(v.NewVersion(ver)))
+}
+
 // CheckNSetDefaults validates and sets default values for the PerconaServerMySQL custom resource.
 func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion *platform.ServerVersion) error {
 	if len(cr.Spec.MySQL.ClusterType) == 0 {
@@ -881,6 +895,10 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion
 		cr.Spec.SSLSecretName = cr.Name + "-ssl"
 	}
 
+	if cr.Spec.MySQL.VaultSecretName == "" {
+		cr.Spec.MySQL.VaultSecretName = cr.Name + "-vault"
+	}
+
 	return nil
 }
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -43,8 +43,15 @@ fi
 
 set +o xtrace
 temp=$(mktemp)
-sed -r "s|^[#]?user=.*$|user=${TOPOLOGY_USER}|" "${ORC_CONF_PATH}/orc-topology.cnf" >"${temp}"
-sed -r "s|^[#]?password=.*$|password=${TOPOLOGY_PASSWORD:-$ORC_TOPOLOGY_PASSWORD}|" "${ORC_CONF_PATH}/orc-topology.cnf" >"${temp}"
+
+ESCAPED_PASSWORD=$(printf '%s' "${TOPOLOGY_PASSWORD:-$ORC_TOPOLOGY_PASSWORD}" | sed -e 's/[&"\\]/\\&/g')
+ESCAPED_PASSWORD="\"${ESCAPED_PASSWORD}\""  # Wrap in double quotes for .cnf
+
+sed -r \
+  -e "s|^[#]?user=.*$|user=${TOPOLOGY_USER}|" \
+  -e "s|^[#]?password=.*$|password=${ESCAPED_PASSWORD}|" \
+  "${ORC_CONF_PATH}/orc-topology.cnf" > "${temp}"
+
 cat "${temp}" >"${ORC_CONF_PATH}/config/orc-topology.cnf"
 rm "${temp}"
 set -o xtrace

build/orc-entrypoint.sh

@@ -167,6 +167,26 @@ create_default_cnf() {
 		sed -i "/\[mysqld\]/a ssl_key=${TLS_DIR}/tls.key" $CFG
 	fi
 
+	# if vault secret file exists we assume we need to turn on encryption
+	vault_secret="/etc/mysql/vault-keyring-secret/keyring_vault.conf"
+	if [[ -f "${vault_secret}" ]]; then
+		sed -i "/\[mysqld\]/a early-plugin-load=keyring_vault.so" $CFG
+		sed -i "/\[mysqld\]/a keyring_vault_config=${vault_secret}" $CFG
+
+		if [[ ${MYSQL_VERSION} =~ ^(8\.0|8\.4)$ ]]; then
+			sed -i "/\[mysqld\]/a default_table_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a table_encryption_privilege_check=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_undo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_redo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_rotate_encryption_master_key_at_startup=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_temp_tablespace_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_parallel_dblwr_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_encrypt_online_alter_logs=ON" $CFG
+			sed -i "/\[mysqld\]/a encrypt_tmp_files=ON" $CFG
+		fi
+	fi
+
 	for f in "${CUSTOM_CONFIG_FILES[@]}"; do
 		echo "${f}"
 		if [ -f "${f}" ]; then
@@ -190,6 +210,14 @@ ensure_read_only() {
 	sed -i "/\[mysqld\]/a super_read_only=ON" $CFG
 }
 
+escape_special() {
+	{ set +x; } 2>/dev/null
+	echo "$1" \
+		| sed 's/\\/\\\\/g' \
+		| sed 's/'\''/'\\\\\''/g' \
+		| sed 's/"/\\\"/g'
+}
+
 MYSQL_VERSION=$(mysqld -V | awk '{print $3}' | awk -F'.' '{print $1"."$2}')
 
 if [[ "$MYSQL_VERSION" != '8.0' ]] && [[ "${MYSQL_VERSION}" != '8.4' ]]; then
@@ -275,7 +303,7 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
 			# no, we don't care if read finds a terminating character in this heredoc
 			# https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151
 			read -r -d '' rootCreate <<-EOSQL || true
-				CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' PASSWORD EXPIRE NEVER;
+				CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '$(escape_special "${MYSQL_ROOT_PASSWORD}")' PASSWORD EXPIRE NEVER;
 				GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;
 			EOSQL
 		fi
@@ -299,38 +327,38 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
 			SET @@SESSION.SQL_LOG_BIN=0;
 
 			DELETE FROM mysql.user WHERE user NOT IN ('mysql.sys', 'mysqlxsys', 'root', 'mysql.infoschema', 'mysql.session') OR host NOT IN ('localhost') ;
-			ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
+			ALTER USER 'root'@'localhost' IDENTIFIED BY '$(escape_special "${MYSQL_ROOT_PASSWORD}")' ;
 			GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ;
 			${rootCreate}
 			/*!80016 REVOKE SYSTEM_USER ON *.* FROM root */;
 
-			CREATE USER 'operator'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${OPERATOR_ADMIN_PASSWORD}' PASSWORD EXPIRE NEVER;
+			CREATE USER 'operator'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '$(escape_special "${OPERATOR_ADMIN_PASSWORD}")' PASSWORD EXPIRE NEVER;
 			GRANT ALL ON *.* TO 'operator'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;
 
-			CREATE USER 'xtrabackup'@'localhost' IDENTIFIED BY '${XTRABACKUP_PASSWORD}' PASSWORD EXPIRE NEVER;
+			CREATE USER 'xtrabackup'@'localhost' IDENTIFIED BY '$(escape_special "${XTRABACKUP_PASSWORD}")' PASSWORD EXPIRE NEVER;
 			GRANT SYSTEM_USER, BACKUP_ADMIN, PROCESS, RELOAD, GROUP_REPLICATION_ADMIN, REPLICATION_SLAVE_ADMIN, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'xtrabackup'@'localhost';
 			GRANT SELECT ON performance_schema.replication_group_members TO 'xtrabackup'@'localhost';
 			GRANT SELECT ON performance_schema.log_status TO 'xtrabackup'@'localhost';
 			GRANT SELECT ON performance_schema.keyring_component_status TO 'xtrabackup'@'localhost';
 
-			CREATE USER 'monitor'@'${MONITOR_HOST}' IDENTIFIED BY '${MONITOR_PASSWORD}' WITH MAX_USER_CONNECTIONS 100 PASSWORD EXPIRE NEVER;
+			CREATE USER 'monitor'@'${MONITOR_HOST}' IDENTIFIED BY '$(escape_special "${MONITOR_PASSWORD}")' WITH MAX_USER_CONNECTIONS 100 PASSWORD EXPIRE NEVER;
 			GRANT SYSTEM_USER, SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD, BACKUP_ADMIN ON *.* TO 'monitor'@'${MONITOR_HOST}';
 			GRANT SELECT ON performance_schema.* TO 'monitor'@'${MONITOR_HOST}';
 			${monitorConnectGrant}
 
-			CREATE USER 'replication'@'%' IDENTIFIED BY '${REPLICATION_PASSWORD}' PASSWORD EXPIRE NEVER;
+			CREATE USER 'replication'@'%' IDENTIFIED BY '$(escape_special "${REPLICATION_PASSWORD}")' PASSWORD EXPIRE NEVER;
 			GRANT DELETE, INSERT, UPDATE ON mysql.* TO 'replication'@'%' WITH GRANT OPTION;
 			GRANT SELECT ON performance_schema.threads to 'replication'@'%';
 			GRANT SYSTEM_USER, REPLICATION SLAVE, BACKUP_ADMIN, GROUP_REPLICATION_STREAM, CLONE_ADMIN, CONNECTION_ADMIN, CREATE USER, EXECUTE, FILE, GROUP_REPLICATION_ADMIN, PERSIST_RO_VARIABLES_ADMIN, PROCESS, RELOAD, REPLICATION CLIENT, REPLICATION_APPLIER, REPLICATION_SLAVE_ADMIN, ROLE_ADMIN, SELECT, SHUTDOWN, SYSTEM_VARIABLES_ADMIN ON *.* TO 'replication'@'%' WITH GRANT OPTION;
 
-			CREATE USER 'orchestrator'@'%' IDENTIFIED BY '${ORC_TOPOLOGY_PASSWORD}' PASSWORD EXPIRE NEVER;
+			CREATE USER 'orchestrator'@'%' IDENTIFIED BY '$(escape_special "${ORC_TOPOLOGY_PASSWORD}")' PASSWORD EXPIRE NEVER;
 			GRANT SYSTEM_USER, SUPER, PROCESS, REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'orchestrator'@'%';
 			GRANT SELECT ON performance_schema.replication_group_members TO 'orchestrator'@'%';
 			GRANT SELECT ON mysql.slave_master_info TO 'orchestrator'@'%';
 			GRANT SELECT ON sys_operator.* TO 'orchestrator'@'%';
 
 			CREATE DATABASE IF NOT EXISTS sys_operator;
-			CREATE USER 'heartbeat'@'localhost' IDENTIFIED BY '${HEARTBEAT_PASSWORD}' PASSWORD EXPIRE NEVER;
+			CREATE USER 'heartbeat'@'localhost' IDENTIFIED BY '$(escape_special "${HEARTBEAT_PASSWORD}")' PASSWORD EXPIRE NEVER;
 			GRANT SYSTEM_USER, REPLICATION CLIENT ON *.* TO 'heartbeat'@'localhost';
 			GRANT SELECT, CREATE, DELETE, UPDATE, INSERT ON sys_operator.heartbeat TO 'heartbeat'@'localhost';
 

build/ps-entrypoint.sh

@@ -6,18 +6,23 @@ ROUTER_DIR=${ROUTER_DIR:-/tmp/router}
 OPERATOR_USER=${OPERATOR_USER:-operator}
 NAMESPACE=$(</var/run/secrets/kubernetes.io/serviceaccount/namespace)
 
+urlencode() {
+  python3 -c 'import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1]))' "$1"
+}
+
 if [ -f "/etc/mysql/mysql-users-secret/${OPERATOR_USER}" ]; then
 	OPERATOR_PASS=$(<"/etc/mysql/mysql-users-secret/${OPERATOR_USER}")
+	OPERATOR_PASS_ESCAPED=$(urlencode "$OPERATOR_PASS")
 fi
 
 mysqlrouter --force \
-	--bootstrap "${OPERATOR_USER}:${OPERATOR_PASS}@${MYSQL_SERVICE_NAME}-0.${MYSQL_SERVICE_NAME}.${NAMESPACE}.svc" \
+	--bootstrap "${OPERATOR_USER}:${OPERATOR_PASS_ESCAPED}@${MYSQL_SERVICE_NAME}-0.${MYSQL_SERVICE_NAME}.${NAMESPACE}.svc" \
 	--conf-bind-address 0.0.0.0 \
 	--conf-set-option http_auth_backend:default_auth_backend.backend=file \
 	--conf-set-option http_auth_backend:default_auth_backend.filename="${ROUTER_DIR}/realm.txt" \
 	--directory "${ROUTER_DIR}"
 
-echo ${OPERATOR_PASS} | mysqlrouter_passwd set "${ROUTER_DIR}/realm.txt" ${OPERATOR_USER}
+echo "${OPERATOR_PASS_ESCAPED}" | mysqlrouter_passwd set "${ROUTER_DIR}/realm.txt" "${OPERATOR_USER}"
 
 sed -i 's/logging_folder=.*/logging_folder=/g' "${ROUTER_DIR}/mysqlrouter.conf"
 sed -i "/\[logger\]/a destination=/dev/stdout" "${ROUTER_DIR}/mysqlrouter.conf"

build/router-entrypoint.sh

@@ -1,15 +1,20 @@
 #!/bin/bash
 
+urlencode() {
+  python3 -c 'import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1]))' "$1"
+}
+
 OPERATOR_PASS=$(</etc/mysql/mysql-users-secret/operator)
+OPERATOR_PASS_ESCAPED=$(urlencode "$OPERATOR_PASS")
 
-if ! curl -k -s -u operator:"${OPERATOR_PASS}" https://localhost:8443/api/20190715/routes/bootstrap_rw/health | grep true; then
+if ! curl -k -s -u operator:"${OPERATOR_PASS_ESCAPED}" https://localhost:8443/api/20190715/routes/bootstrap_rw/health | grep true; then
 	echo "Read-write route is not healthy"
 	exit 1
 fi
 
-if ! curl -k -s -u operator:"${OPERATOR_PASS}" https://localhost:8443/api/20190715/routes/bootstrap_ro/health | grep true; then
+if ! curl -k -s -u operator:"${OPERATOR_PASS_ESCAPED}" https://localhost:8443/api/20190715/routes/bootstrap_ro/health | grep true; then
 	echo "Read-only route is not healthy"
 	exit 1
 fi
 
-exit 0
+exit 0

build/router_readiness_check.sh

@@ -1,7 +1,12 @@
 #!/bin/bash
 
+urlencode() {
+  python3 -c 'import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1]))' "$1"
+}
+
 OPERATOR_PASS=$(</etc/mysql/mysql-users-secret/operator)
+OPERATOR_PASS_ESCAPED=$(urlencode "$OPERATOR_PASS")
 
-if [[ $(curl -k -s -u operator:"${OPERATOR_PASS}" -o /dev/null -w %{http_code} https://localhost:8443/api/20190715/router/status) != 200 ]]; then
+if [[ $(curl -k -s -u operator:"${OPERATOR_PASS_ESCAPED}" -o /dev/null -w %{http_code} https://localhost:8443/api/20190715/router/status) != 200 ]]; then
 	echo "Router is not ready"
-fi
+fi

build/router_startup_check.sh

@@ -41,7 +41,13 @@ main() {
 		"azure") run_azure | extract "${tmpdir}" ;;
 	esac
 
-	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}"
+	local keyring=""
+	if [[ -f ${KEYRING_VAULT_PATH} ]]; then
+		echo "Using keyring vault config: ${KEYRING_VAULT_PATH}"
+		keyring="--keyring-vault-config=${KEYRING_VAULT_PATH}"
+	fi
+
+	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}" ${keyring}
 	xtrabackup --datadir="${DATADIR}" --move-back --force-non-empty-directories --target-dir="${tmpdir}"
 
 	rm -rf "${tmpdir}"

build/run-restore.sh

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/url"
 	"os"
 	"os/exec"
 	"regexp"
@@ -54,8 +55,9 @@ func (m *mysqlsh) getURI() string {
 	if err != nil {
 		return ""
 	}
+	escapedPass := url.QueryEscape(operatorPass)
 
-	return fmt.Sprintf("%s:%s@%s", apiv1alpha1.UserOperator, operatorPass, m.host)
+	return fmt.Sprintf("%s:%s@%s", apiv1alpha1.UserOperator, escapedPass, m.host)
 }
 
 func (m *mysqlsh) run(ctx context.Context, cmd string) (bytes.Buffer, bytes.Buffer, error) {

cmd/bootstrap/group_replication.go

@@ -9,12 +9,15 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
+	"time"
 
 	"golang.org/x/sync/errgroup"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -69,10 +72,7 @@ func (s *Status) GetBackupConfig() *xb.BackupConfig {
 	return &cfg
 }
 
-func main() {
-	opts := zap.Options{Development: true}
-	logf.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
-
+func startServer() *http.Server {
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("/health/", func(w http.ResponseWriter, req *http.Request) {
@@ -81,8 +81,42 @@ func main() {
 	mux.HandleFunc("/backup/", backupHandler)
 	mux.HandleFunc("/logs/", logHandler)
 
-	log.Info("starting http server")
-	log.Error(http.ListenAndServe(":"+strconv.Itoa(mysql.SidecarHTTPPort), mux), "http server failed")
+	srv := &http.Server{Addr: ":" + strconv.Itoa(mysql.SidecarHTTPPort), Handler: mux}
+
+	go func() {
+		log.Info("starting http server")
+		// always returns error. ErrServerClosed on graceful close
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			log.Error(err, "http server failed")
+		}
+	}()
+
+	return srv
+}
+
+func main() {
+	opts := zap.Options{Development: true}
+	logf.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
+	srv := startServer()
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
+
+	<-stop
+
+	log.Info("received interrupt signal, shutting down http server")
+
+	// TODO: should this timeout use terminationGracePeriodSeconds?
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := srv.Shutdown(ctx); err != nil {
+		log.Error(err, "graceful shutdown failed")
+		os.Exit(1)
+	}
+
+	os.Exit(0)
 }
 
 func getSecret(username apiv1alpha1.SystemUser) (string, error) {