K8SPXC-1781: Fix custom ProxySQL config (#2325)

* fix proxysql custom config errors

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* update example

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* add e2e test proxysql custom cfg

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* update e2e tests

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* check crVersion while setting mount path

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

---------

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

Author: mayankshah1607

build/proxysql-entrypoint.sh

@@ -14,6 +14,10 @@ function sed_in_place() {
 }
 
 cp /opt/percona/proxysql.cnf /etc/proxysql
+if [[ -f /etc/proxysql/custom/proxysql.cnf ]]; then
+	cp /etc/proxysql/custom/proxysql.cnf /etc/proxysql/proxysql.cnf
+fi
+
 cp /opt/percona/proxysql-admin.cnf /etc
 
 MYSQL_INTERFACES='0.0.0.0:3306;0.0.0.0:33062'

deploy/cr.yaml

@@ -404,12 +404,14 @@ spec:
 #
 #      admin_variables =
 #      {
-#        admin_credentials="proxyadmin:admin_password"
+#        admin_credentials="admin:admin"
 #        mysql_ifaces="0.0.0.0:6032"
 #        refresh_interval=2000
+#        restapi_enabled=true
+#        restapi_port=6070
 #
-#        cluster_username="proxyadmin"
-#        cluster_password="admin_password"
+#        cluster_username="admin"
+#        cluster_password="admin"
 #        checksum_admin_variables=false
 #        checksum_ldap_variables=false
 #        checksum_mysql_variables=false
@@ -444,11 +446,12 @@ spec:
 #        ping_timeout_server=200
 #        commands_stats=true
 #        sessions_sort=true
-#        have_ssl=true
-#        ssl_p2s_ca="/etc/proxysql/ssl-internal/ca.crt"
-#        ssl_p2s_cert="/etc/proxysql/ssl-internal/tls.crt"
-#        ssl_p2s_key="/etc/proxysql/ssl-internal/tls.key"
+#        have_ssl=false
+#        ssl_p2s_ca=""
+#        ssl_p2s_cert=""
+#        ssl_p2s_key=""
 #        ssl_p2s_cipher="ECDHE-RSA-AES128-GCM-SHA256"
+#        default_authentication_plugin="caching_sha2_password"
 #      }
 #    scheduler:
 #      enabled: false

e2e-tests/proxysql-scheduler/compare/proxysql-cfg.sql

@@ -0,0 +1 @@
+4000

e2e-tests/proxysql-scheduler/compare/statefulset_some-name-proxysql.yml

@@ -110,6 +110,8 @@ spec:
               name: ssl
             - mountPath: /etc/proxysql/ssl-internal
               name: ssl-internal
+            - mountPath: /etc/proxysql/custom
+              name: config
             - mountPath: /opt/percona
               name: bin
         - args:
@@ -221,6 +223,11 @@ spec:
             defaultMode: 420
             optional: false
             secretName: some-name-ssl
+        - configMap:
+            defaultMode: 420
+            name: some-name-proxysql
+            optional: true
+          name: config
         - emptyDir: {}
           name: bin
   updateStrategy:

e2e-tests/proxysql-scheduler/conf/some-name.yml

@@ -44,3 +44,56 @@ spec:
         resources:
           requests:
             storage: 2Gi
+    configuration: |
+      datadir="/var/lib/proxysql"
+      admin_variables =
+      {
+        admin_credentials="admin:admin"
+        mysql_ifaces="0.0.0.0:6032"
+        refresh_interval=2000
+        restapi_enabled=true
+        restapi_port=6070
+
+        cluster_username="admin"
+        cluster_password="admin"
+        checksum_admin_variables=false
+        checksum_ldap_variables=false
+        checksum_mysql_variables=false
+        cluster_check_interval_ms=200
+        cluster_check_status_frequency=100
+        cluster_mysql_query_rules_save_to_disk=true
+        cluster_mysql_servers_save_to_disk=true
+        cluster_mysql_users_save_to_disk=true
+        cluster_proxysql_servers_save_to_disk=true
+        cluster_mysql_query_rules_diffs_before_sync=1
+        cluster_mysql_servers_diffs_before_sync=1
+        cluster_mysql_users_diffs_before_sync=1
+        cluster_proxysql_servers_diffs_before_sync=1
+      }
+
+      mysql_variables=
+      {
+        monitor_password="monitor"
+        monitor_galera_healthcheck_interval=1000
+        threads=2
+        max_connections=2048
+        default_query_delay=0
+        default_query_timeout=10000
+        poll_timeout=4000
+        interfaces="0.0.0.0:3306"
+        default_schema="information_schema"
+        stacksize=1048576
+        connect_timeout_server=10000
+        monitor_history=60000
+        monitor_connect_interval=20000
+        monitor_ping_interval=10000
+        ping_timeout_server=200
+        commands_stats=true
+        sessions_sort=true
+        have_ssl=false
+        ssl_p2s_ca=""
+        ssl_p2s_cert=""
+        ssl_p2s_key=""
+        ssl_p2s_cipher="ECDHE-RSA-AES128-GCM-SHA256"
+        default_authentication_plugin="caching_sha2_password"
+      }

e2e-tests/proxysql-scheduler/run

@@ -19,6 +19,15 @@ function compare_scheduler() {
 		${pod}
 }
 
+function compare_proxysql_cfg() {
+	local pod=$1
+	local compare_file=$2
+
+	local query='SELECT variable_value from global_variables WHERE variable_name='\'mysql-poll_timeout\'''
+	compare_mysql_cmd_local ${compare_file} "${query}" \
+		"-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" ${pod}
+}
+
 function compare_mysql_servers() {
 	local pod=$1
 	local compare_file=$2
@@ -59,6 +68,9 @@ desc "create PXC cluster: ${cluster}"
 spinup_pxc "${cluster}" "${test_dir}/conf/${cluster}.yml"
 wait_cluster_consistency "${cluster}" 3 2
 
+desc 'check proxysql custom config'
+compare_proxysql_cfg "${cluster}-proxysql-0" "proxysql-cfg"
+
 desc 'check if service and statefulset created with expected config'
 compare_kubectl statefulset/${cluster}-pxc
 compare_kubectl statefulset/${cluster}-proxysql

e2e-tests/users-scheduler/compare/proxysql-cfg.sql

@@ -0,0 +1 @@
+test-password

e2e-tests/users-scheduler/run

@@ -68,6 +68,8 @@ sleep 15
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-0" "" 'proxysql'
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-1" "" 'proxysql'
 compare_mysql_cmd_local "$tables_cmp_file" "SHOW TABLES;" "-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-2" "" 'proxysql'
+compare_mysql_cmd_local "proxysql-cfg" "SELECT variable_value from global_variables WHERE variable_name='admin-cluster_password';" \
+	"-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-2" "" 'proxysql'
 
 desc 'test xtrabackup'
 kubectl_bin patch pxc some-name --type=merge -p="{\"spec\":{\"proxysql\":{\"size\":2}}}"
@@ -83,6 +85,8 @@ wait_for_password_propagation "my-cluster-secrets" "monitor" "$cluster"
 wait_cluster_consistency "$cluster" 3 2
 sleep 15 # give some time for proxy-admin --syncusers
 compare_mysql_cmd "select-4" "SHOW TABLES;" "-h $cluster-proxysql -umonitor -p'$newpass'"
+compare_mysql_cmd_local "proxysql-cfg" "SELECT variable_value from global_variables WHERE variable_name='mysql-monitor_password';" \
+	"-h127.0.0.1 -P6032 -uproxyadmin -p'$newpass'" "$cluster-proxysql-1" "" 'proxysql'
 
 desc 'test operator'
 patch_secret "my-cluster-secrets" "operator" "$newpassencrypted"

pkg/pxc/app/statefulset/proxysql.go

@@ -143,10 +143,14 @@ func (c *Proxy) AppContainer(ctx context.Context, _ client.Client, spec *api.Pod
 		},
 	}
 
+	proxyConfigMountPath := "/etc/proxysql"
+	if cr.CompareVersionWith("1.19.0") >= 0 {
+		proxyConfigMountPath = "/etc/proxysql/custom"
+	}
 	if api.ContainsVolume(availableVolumes, proxyConfigVolumeName) {
 		appc.VolumeMounts = append(appc.VolumeMounts, corev1.VolumeMount{
 			Name:      proxyConfigVolumeName,
-			MountPath: "/etc/proxysql/",
+			MountPath: proxyConfigMountPath,
 		})
 	}