K8SPXC-1781: [bugfix] cannot use custom proxysql config

build/proxysql-entrypoint.sh

@@ -14,6 +14,10 @@
 }
 
 cp /opt/percona/proxysql.cnf /etc/proxysql
+if [[ -f /etc/proxysql/custom/proxysql.cnf ]]; then
+	cp /etc/proxysql/custom/proxysql.cnf /etc/proxysql/proxysql.cnf
+fi
+
 cp /opt/percona/proxysql-admin.cnf /etc
 
 MYSQL_INTERFACES='0.0.0.0:3306;0.0.0.0:33062'
@@ -25,7 +29,7 @@
 # Percona scheduler
 PERCONA_SCHEDULER_CFG_TMPL=/opt/percona/proxysql_scheduler_config.tmpl
 if [[ -f ${PERCONA_SCHEDULER_CFG_TMPL} && -n ${PERCONA_SCHEDULER_CFG} ]]; then
 	cp ${PERCONA_SCHEDULER_CFG_TMPL} ${PERCONA_SCHEDULER_CFG}
 fi
 
 # internal scheduler
@@ -35,9 +39,9 @@
 sed_in_place "s/threads=2/threads=${MYSQL_THREADS:-2}/g" ${PROXY_CFG}
 
 set +o xtrace # hide sensitive information
 OPERATOR_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_\~\{\}]/\\&/g' <<<"${OPERATOR_PASSWORD}")
 MONITOR_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_\~\{\}]/\\&/g' <<<"${MONITOR_PASSWORD}")
 PROXY_ADMIN_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_\~\{\}]/\\&/g' <<<"${PROXY_ADMIN_PASSWORD}")
 
 sed_in_place "s/\"admin:admin\"/\"${PROXY_ADMIN_USER:-admin}:${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}\"/g" ${PROXY_CFG}
 sed_in_place "s/cluster_username=\"admin\"/cluster_username=\"${PROXY_ADMIN_USER:-admin}\"/g" ${PROXY_CFG}
@@ -55,21 +59,21 @@
 # Percona scheduler
 if [[ -f ${PERCONA_SCHEDULER_CFG} ]]; then
 	set +o xtrace # hide sensitive information
 	sed_in_place "s/SCHEDULER_PROXYSQLPASSWORD/'${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}'/" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_CLUSTERPASSWORD/'${OPERATOR_PASSWORD_ESCAPED:-operator}'/" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_MONITORPASSWORD/'${MONITOR_PASSWORD_ESCAPED:-monitor}'/" ${PERCONA_SCHEDULER_CFG}
 	set -o xtrace # hide sensitive information
 
 	sed_in_place "s/SCHEDULER_PROXYSQLHOST/'$(hostname -f)'/" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_CLUSTERHOST/'${PXC_SERVICE}.$(hostname -f | cut -d '.' -f3-)'/" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_CLUSTERPORT/'${CLUSTER_PORT:-3306}'/" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_MAXCONNECTIONS/${SCHEDULER_MAXCONNECTIONS}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_NODECHECKINTERVAL/${SCHEDULER_NODECHECKINTERVAL}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_CHECKTIMEOUT/${SCHEDULER_CHECKTIMEOUT}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_PINGTIMEOUT/${SCHEDULER_PINGTIMEOUT}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_RETRYDOWN/${SCHEDULER_RETRYDOWN}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_RETRYUP/${SCHEDULER_RETRYUP}/g" ${PERCONA_SCHEDULER_CFG}
 	sed_in_place "s/SCHEDULER_WRITERALSOREADER/${SCHEDULER_WRITERALSOREADER}/g" ${PERCONA_SCHEDULER_CFG}
 fi
 
 ## SSL/TLS support
@@ -81,14 +85,14 @@
 if [ -f "${SSL_DIR}/ca.crt" ]; then
 	CA=${SSL_DIR}/ca.crt
 	if [[ -f ${PERCONA_SCHEDULER_CFG} ]]; then
 		sed_in_place "s:^sslCertificatePath.*= .*\"$:sslCertificatePath = \"${SSL_DIR}\":" ${PERCONA_SCHEDULER_CFG}
 	fi
 fi
 SSL_INTERNAL_DIR=${SSL_INTERNAL_DIR:-/etc/proxysql/ssl-internal}
 if [ -f "${SSL_INTERNAL_DIR}/ca.crt" ]; then
 	CA=${SSL_INTERNAL_DIR}/ca.crt
 	if [[ -f ${PERCONA_SCHEDULER_CFG} ]]; then
 		sed_in_place "s:^sslCertificatePath.*= .*\"$:sslCertificatePath = \"${SSL_INTERNAL_DIR}\":" ${PERCONA_SCHEDULER_CFG}
 	fi
 fi
 
@@ -108,9 +112,9 @@
 
 	# Percona scheduler
 	if [[ -f ${PERCONA_SCHEDULER_CFG} ]]; then
 		sed_in_place "s:^sslCa.*=.*\"$:sslCa = \"${CA##*/}\":" ${PERCONA_SCHEDULER_CFG}
 		sed_in_place "s:^sslKey.*=.*\"$:sslKey = \"${KEY##*/}\":" ${PERCONA_SCHEDULER_CFG}
 		sed_in_place "s:^sslClient.*=.*\"$:sslClient = \"${CERT##*/}\":" ${PERCONA_SCHEDULER_CFG}
 	fi
 fi
 
@@ -122,7 +126,7 @@
 	cp "${SSL_DIR}/ca.crt" /var/lib/proxysql/proxysql-ca.pem
 fi
 
 test -e /opt/percona/hookscript/hook.sh && source /opt/percona/hookscript/hook.sh
 
 # Start zombie reaper to clean up processes spawned by commands
 # This is needed because percona-scheduler-admin may not properly reap all child processes
@@ -150,7 +154,7 @@
 MAIN_PID=$!
 
 # Forward signals to main process
 forward_signal() {
        kill -"$1" "$MAIN_PID" 2>/dev/null || true
 }
 trap 'forward_signal TERM' TERM

deploy/cr.yaml

@@ -404,12 +404,14 @@ spec:
 #
 #      admin_variables =
 #      {
-#        admin_credentials="proxyadmin:admin_password"
+#        admin_credentials="admin:admin"
 #        mysql_ifaces="0.0.0.0:6032"
 #        refresh_interval=2000
+#        restapi_enabled=true
+#        restapi_port=6070
 #
-#        cluster_username="proxyadmin"
-#        cluster_password="admin_password"
+#        cluster_username="admin"
+#        cluster_password="admin"
 #        checksum_admin_variables=false
 #        checksum_ldap_variables=false
 #        checksum_mysql_variables=false
@@ -444,11 +446,12 @@ spec:
 #        ping_timeout_server=200
 #        commands_stats=true
 #        sessions_sort=true
-#        have_ssl=true
-#        ssl_p2s_ca="/etc/proxysql/ssl-internal/ca.crt"
-#        ssl_p2s_cert="/etc/proxysql/ssl-internal/tls.crt"
-#        ssl_p2s_key="/etc/proxysql/ssl-internal/tls.key"
+#        have_ssl=false
+#        ssl_p2s_ca=""
+#        ssl_p2s_cert=""
+#        ssl_p2s_key=""
 #        ssl_p2s_cipher="ECDHE-RSA-AES128-GCM-SHA256"
+#        default_authentication_plugin="caching_sha2_password"
 #      }
 #    scheduler:
 #      enabled: false

e2e-tests/proxysql-scheduler/compare/proxysql-cfg.sql

@@ -0,0 +1 @@
+4000

e2e-tests/proxysql-scheduler/compare/statefulset_some-name-proxysql.yml

@@ -110,6 +110,8 @@ spec:
               name: ssl
             - mountPath: /etc/proxysql/ssl-internal
               name: ssl-internal
+            - mountPath: /etc/proxysql/custom
+              name: config
             - mountPath: /opt/percona
               name: bin
         - args:
@@ -221,6 +223,11 @@ spec:
             defaultMode: 420
             optional: false
             secretName: some-name-ssl
+        - configMap:
+            defaultMode: 420
+            name: some-name-proxysql
+            optional: true
+          name: config
         - emptyDir: {}
           name: bin
   updateStrategy:

e2e-tests/proxysql-scheduler/conf/some-name.yml

@@ -44,3 +44,56 @@ spec:
         resources:
           requests:
             storage: 2Gi
+    configuration: |
+      datadir="/var/lib/proxysql"
+      admin_variables =
+      {
+        admin_credentials="admin:admin"
+        mysql_ifaces="0.0.0.0:6032"
+        refresh_interval=2000
+        restapi_enabled=true
+        restapi_port=6070
+
+        cluster_username="admin"
+        cluster_password="admin"
+        checksum_admin_variables=false
+        checksum_ldap_variables=false
+        checksum_mysql_variables=false
+        cluster_check_interval_ms=200
+        cluster_check_status_frequency=100
+        cluster_mysql_query_rules_save_to_disk=true
+        cluster_mysql_servers_save_to_disk=true
+        cluster_mysql_users_save_to_disk=true
+        cluster_proxysql_servers_save_to_disk=true
+        cluster_mysql_query_rules_diffs_before_sync=1
+        cluster_mysql_servers_diffs_before_sync=1
+        cluster_mysql_users_diffs_before_sync=1
+        cluster_proxysql_servers_diffs_before_sync=1
+      }
+
+      mysql_variables=
+      {
+        monitor_password="monitor"
+        monitor_galera_healthcheck_interval=1000
+        threads=2
+        max_connections=2048
+        default_query_delay=0
+        default_query_timeout=10000
+        poll_timeout=4000
+        interfaces="0.0.0.0:3306"
+        default_schema="information_schema"
+        stacksize=1048576
+        connect_timeout_server=10000
+        monitor_history=60000
+        monitor_connect_interval=20000
+        monitor_ping_interval=10000
+        ping_timeout_server=200
+        commands_stats=true
+        sessions_sort=true
+        have_ssl=false
+        ssl_p2s_ca=""
+        ssl_p2s_cert=""
+        ssl_p2s_key=""
+        ssl_p2s_cipher="ECDHE-RSA-AES128-GCM-SHA256"
+        default_authentication_plugin="caching_sha2_password"
+      }

e2e-tests/proxysql-scheduler/run

@@ -19,6 +19,15 @@ function compare_scheduler() {
 		${pod}
 }
 
+function compare_proxysql_cfg() {
+	local pod=$1
+	local compare_file=$2
+
+	local query='SELECT variable_value from global_variables WHERE variable_name='\'mysql-poll_timeout\'''
+	compare_mysql_cmd_local ${compare_file} "${query}" \
+		"-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" ${pod}
+}
+
 function compare_mysql_servers() {
 	local pod=$1
 	local compare_file=$2
@@ -59,6 +68,9 @@ desc "create PXC cluster: ${cluster}"
 spinup_pxc "${cluster}" "${test_dir}/conf/${cluster}.yml"
 wait_cluster_consistency "${cluster}" 3 2
 
+desc 'check proxysql custom config'
+compare_proxysql_cfg "${cluster}-proxysql-0" "proxysql-cfg"
+
 desc 'check if service and statefulset created with expected config'
 compare_kubectl statefulset/${cluster}-pxc
 compare_kubectl statefulset/${cluster}-proxysql

e2e-tests/users-scheduler/compare/proxysql-cfg.sql

@@ -0,0 +1 @@
+test-password

e2e-tests/users-scheduler/run

@@ -68,6 +68,8 @@ sleep 15
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-0" "" 'proxysql'
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-1" "" 'proxysql'
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-2" "" 'proxysql'
+compare_mysql_cmd_local "proxysql-cfg" "SELECT variable_value from global_variables WHERE variable_name='admin-cluster_password';" \
+	"-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-2" "" 'proxysql'
 
 desc 'test xtrabackup'
 kubectl_bin patch pxc some-name --type=merge -p="{\"spec\":{\"proxysql\":{\"size\":2}}}"
@@ -83,6 +85,8 @@ wait_for_password_propagation "my-cluster-secrets" "monitor" "$cluster"
 wait_cluster_consistency "$cluster" 3 2
 sleep 15 # give some time for proxy-admin --syncusers
 compare_mysql_cmd "select-4" "SHOW TABLES;" "-h $cluster-proxysql -umonitor -p'$newpass'"
+compare_mysql_cmd_local "proxysql-cfg" "SELECT variable_value from global_variables WHERE variable_name='mysql-monitor_password';" \
+	"-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-1" "" 'proxysql'
 
 desc 'test operator'
 patch_secret "my-cluster-secrets" "operator" "$newpassencrypted"

pkg/pxc/app/statefulset/proxysql.go

@@ -143,10 +143,14 @@ func (c *Proxy) AppContainer(ctx context.Context, _ client.Client, spec *api.Pod
 		},
 	}
 
+	proxyConfigMountPath := "/etc/proxysql"
+	if cr.CompareVersionWith("1.19.0") >= 0 {
+		proxyConfigMountPath = "/etc/proxysql/custom"
+	}
 	if api.ContainsVolume(availableVolumes, proxyConfigVolumeName) {
 		appc.VolumeMounts = append(appc.VolumeMounts, corev1.VolumeMount{
 			Name:      proxyConfigVolumeName,
-			MountPath: "/etc/proxysql/",
+			MountPath: proxyConfigMountPath,
 		})
 	}