K8SPS-163: add PMM support for HAProxy (#380)

pooknull · web-flow · commit 1a4294d69b66 · 2023-06-14T15:26:58.000+03:00
https://jira.percona.com/browse/K8SPS-163
diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go
@@ -793,7 +793,7 @@ func (cr *PerconaServerMySQL) InternalSecretName() string {
 }
 
 func (cr *PerconaServerMySQL) PMMEnabled(secret *corev1.Secret) bool {
-	if cr.Spec.PMM != nil && cr.Spec.PMM.Enabled && secret.Data != nil {
+	if cr.Spec.PMM != nil && cr.Spec.PMM.Enabled && secret != nil && secret.Data != nil {
 		return cr.Spec.PMM.HasSecret(secret)
 	}
 	return false
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -101,5 +101,6 @@ COPY build/haproxy_liveness_check.sh /opt/percona-server-mysql-operator/haproxy_
 COPY build/haproxy_readiness_check.sh /opt/percona-server-mysql-operator/haproxy_readiness_check.sh
 COPY build/haproxy.cfg /opt/percona-server-mysql-operator/haproxy.cfg
 COPY build/haproxy-global.cfg /opt/percona-server-mysql-operator/haproxy-global.cfg
+COPY build/pmm-prerun.sh /opt/percona-server-mysql-operator/pmm-prerun.sh
 
 USER 2
diff --git a/build/haproxy-global.cfg b/build/haproxy-global.cfg
@@ -25,3 +25,8 @@
       mode tcp
       option clitcpka
       default_backend mysql-replicas
+
+    frontend stats
+      bind *:8404
+      mode http
+      http-request use-service prometheus-exporter if { path /metrics }
diff --git a/build/pmm-prerun.sh b/build/pmm-prerun.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -o errexit
+
+pmm_args=()
+
+read -ra PMM_ADMIN_CUSTOM_PARAMS_ARRAY <<<"$PMM_ADMIN_CUSTOM_PARAMS"
+pmm_args+=(
+	"${PMM_ADMIN_CUSTOM_PARAMS_ARRAY[@]}"
+)
+
+if [[ $DB_TYPE != "haproxy" ]]; then
+	pmm_args+=(
+		--service-name="$PMM_AGENT_SETUP_NODE_NAME"
+		--host="$POD_NAME"
+		--port="$DB_PORT"
+	)
+fi
+
+if [[ $DB_TYPE == "mysql" ]]; then
+	read -ra DB_ARGS_ARRAY <<<"$DB_ARGS"
+	pmm_args+=(
+		"${DB_ARGS_ARRAY[@]}"
+	)
+fi
+
+if [[ $DB_TYPE == "haproxy" ]]; then
+	pmm_args+=(
+		"$PMM_AGENT_SETUP_NODE_NAME"
+	)
+fi
+
+pmm-admin status --wait=10s
+pmm-admin add "$DB_TYPE" --skip-connection-check --metrics-mode="$PMM_AGENT_SETUP_METRICS_MODE" --username="$DB_USER" --password="$DB_PASSWORD" --cluster="$CLUSTER_NAME" "${pmm_args[@]}"
+pmm-admin annotate --service-name="$PMM_AGENT_SETUP_NODE_NAME" 'Service restarted'
diff --git a/build/ps-init-entrypoint.sh b/build/ps-init-entrypoint.sh
@@ -32,3 +32,5 @@ install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy_liveness_
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy_readiness_check.sh" "${BINDIR}/haproxy_readiness_check.sh"
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy.cfg" "${BINDIR}/haproxy.cfg"
 install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/haproxy-global.cfg" "${BINDIR}/haproxy-global.cfg"
+
+install -o "$(id -u)" -g "$(id -g)" -m 0755 -D "${OPERATORDIR}/pmm-prerun.sh" "${BINDIR}/pmm-prerun.sh"
diff --git a/e2e-tests/tests/monitoring/02-assert.yaml b/e2e-tests/tests/monitoring/02-assert.yaml
@@ -74,10 +74,7 @@ spec:
           - name: PMM_AGENT_SETUP_NODE_TYPE
             value: container
           - name: PMM_AGENT_PRERUN_SCRIPT
-            value: |-
-              pmm-admin status --wait=10s;
-              pmm-admin add ${DB_TYPE} ${PMM_ADMIN_CUSTOM_PARAMS} --skip-connection-check --metrics-mode=${PMM_AGENT_SETUP_METRICS_MODE} --username=${DB_USER} --password=${DB_PASSWORD} --cluster=${CLUSTER_NAME} --service-name=${PMM_AGENT_SETUP_NODE_NAME} --host=${POD_NAME} --port=${DB_PORT} ${DB_ARGS};
-              pmm-admin annotate --service-name=${PMM_AGENT_SETUP_NODE_NAME} 'Service restarted'
+            value: /opt/percona/pmm-prerun.sh
           - name: PMM_AGENT_SIDECAR
             value: "true"
           - name: PMM_AGENT_SIDECAR_SLEEP
@@ -135,3 +132,167 @@ status:
     size: 3
     state: ready
   state: ready
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: monitoring-haproxy
+spec:
+  template:
+    spec:
+      containers:
+      - name: haproxy
+      - name: mysql-monit
+      - name: pmm-client
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+          - name: CLUSTER_NAME
+            value: monitoring
+          - name: CLIENT_PORT_LISTEN
+            value: "7777"
+          - name: CLIENT_PORT_MIN
+            value: "30100"
+          - name: CLIENT_PORT_MAX
+            value: "30105"
+          - name: PMM_AGENT_SERVER_ADDRESS
+            value: monitoring-service
+          - name: PMM_AGENT_SERVER_USERNAME
+            value: api_key
+          - name: PMM_AGENT_SERVER_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                key: pmmserverkey
+                name: internal-monitoring
+          - name: PMM_SERVER
+            value: monitoring-service
+          - name: PMM_USER
+            value: api_key
+          - name: PMM_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                key: pmmserverkey
+                name: internal-monitoring
+          - name: PMM_AGENT_LISTEN_PORT
+            value: "7777"
+          - name: PMM_AGENT_PORTS_MIN
+            value: "30100"
+          - name: PMM_AGENT_PORTS_MAX
+            value: "30105"
+          - name: PMM_AGENT_CONFIG_FILE
+            value: /usr/local/percona/pmm2/config/pmm-agent.yaml
+          - name: PMM_AGENT_SERVER_INSECURE_TLS
+            value: "1"
+          - name: PMM_AGENT_LISTEN_ADDRESS
+            value: 0.0.0.0
+          - name: PMM_AGENT_SETUP_NODE_NAME
+            value: $(POD_NAMESPACE)-$(POD_NAME)
+          - name: PMM_AGENT_SETUP_METRICS_MODE
+            value: push
+          - name: PMM_AGENT_SETUP
+            value: "1"
+          - name: PMM_AGENT_SETUP_FORCE
+            value: "1"
+          - name: PMM_AGENT_SETUP_NODE_TYPE
+            value: container
+          - name: PMM_AGENT_PRERUN_SCRIPT
+            value: /opt/percona/pmm-prerun.sh
+          - name: PMM_AGENT_SIDECAR
+            value: "true"
+          - name: PMM_AGENT_SIDECAR_SLEEP
+            value: "5"
+          - name: DB_CLUSTER
+            value: monitoring
+          - name: DB_TYPE
+            value: haproxy
+          - name: DB_HOST
+            value: localhost
+          - name: DB_PORT
+            value: "33062"
+          - name: DB_USER
+            value: monitor
+          - name: DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                key: monitor
+                name: internal-monitoring
+          - name: DB_ARGS
+            value: --query-source=perfschema
+          - name: PMM_ADMIN_CUSTOM_PARAMS
+            value: --listen-port=8404
+        image: perconalab/pmm-client:dev-latest
+        imagePullPolicy: Always
+        ports:
+          - containerPort: 7777
+            protocol: TCP
+          - containerPort: 30100
+            protocol: TCP
+          - containerPort: 30101
+            protocol: TCP
+          - containerPort: 30102
+            protocol: TCP
+          - containerPort: 30103
+            protocol: TCP
+          - containerPort: 30104
+            protocol: TCP
+          - containerPort: 30105
+            protocol: TCP
+          - containerPort: 8404
+            protocol: TCP
+        resources:
+          requests:
+            cpu: 300m
+            memory: 150M
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /opt/percona
+          name: bin
+      dnsPolicy: ClusterFirst
+      initContainers:
+      - command:
+        - /opt/percona-server-mysql-operator/ps-init-entrypoint.sh
+        imagePullPolicy: Always
+        name: haproxy-init
+        resources: {}
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /opt/percona
+          name: bin
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - emptyDir: {}
+        name: bin
+      - emptyDir: {}
+        name: haproxy-config
+      - name: users
+        secret:
+          defaultMode: 420
+          secretName: internal-monitoring
+      - name: tls
+        secret:
+          defaultMode: 420
+          secretName: test-ssl
+  updateStrategy:
+    rollingUpdate:
+      partition: 0
+    type: RollingUpdate
+status:
+  availableReplicas: 3
+  currentReplicas: 3
+  observedGeneration: 1
+  readyReplicas: 3
+  replicas: 3
+  updatedReplicas: 3
diff --git a/e2e-tests/tests/monitoring/02-create-cluster.yaml b/e2e-tests/tests/monitoring/02-create-cluster.yaml
@@ -10,5 +10,7 @@ commands:
       get_cr \
         | yq eval '.spec.mysql.clusterType="async"' - \
       	| yq eval '.spec.pmm.enabled = true' - \
+      	| yq eval '.spec.proxy.haproxy.enabled = true' - \
+      	| yq eval '.spec.proxy.haproxy.expose.type = "LoadBalancer"' - \
       	| kubectl -n "${NAMESPACE}" apply -f -
     timeout: 10
diff --git a/e2e-tests/tests/monitoring/03-check-metrics.yaml b/e2e-tests/tests/monitoring/03-check-metrics.yaml
@@ -17,4 +17,11 @@ commands:
       sleep 90 # wait for QAN
 
       get_qan20_values monitoring-mysql-0 api_key:$API_KEY
+
+      haproxy_svc=$(get_service_ip "monitoring-haproxy")
+      http_code=$(curl -s -o /dev/null -w "%{http_code}" http://${haproxy_svc}:8404/metrics)
+      if [[ $http_code != 200 ]]; then
+          echo "Error: http code is $http_code"
+          exit 1
+      fi
     timeout: 120
diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go
@@ -719,7 +719,14 @@ func (r *PerconaServerMySQLReconciler) reconcileHAProxy(ctx context.Context, cr
 		return errors.Wrap(err, "get init image")
 	}
 
-	if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, haproxy.StatefulSet(cr, initImage), r.Scheme); err != nil {
+	internalSecret := new(corev1.Secret)
+	nn = types.NamespacedName{Name: cr.InternalSecretName(), Namespace: cr.Namespace}
+	err = r.Client.Get(ctx, nn, internalSecret)
+	if client.IgnoreNotFound(err) != nil {
+		return errors.Wrapf(err, "get Secret/%s", nn.Name)
+	}
+
+	if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, haproxy.StatefulSet(cr, initImage, internalSecret), r.Scheme); err != nil {
 		return errors.Wrap(err, "reconcile StatefulSet")
 	}
 
@@ -736,8 +743,15 @@ func (r *PerconaServerMySQLReconciler) reconcileServices(ctx context.Context, cr
 			return errors.Wrap(err, "reconcile Orchestrator services")
 		}
 		if cr.HAProxyEnabled() {
+			internalSecret := new(corev1.Secret)
+			nn := types.NamespacedName{Name: cr.InternalSecretName(), Namespace: cr.Namespace}
+			err := r.Client.Get(ctx, nn, internalSecret)
+			if client.IgnoreNotFound(err) != nil {
+				return errors.Wrapf(err, "get Secret/%s", nn.Name)
+			}
+
 			expose := cr.Spec.Proxy.HAProxy.Expose
-			if err := k8s.EnsureService(ctx, r.Client, cr, haproxy.Service(cr), r.Scheme, expose.SaveOldMeta()); err != nil {
+			if err := k8s.EnsureService(ctx, r.Client, cr, haproxy.Service(cr, internalSecret), r.Scheme, expose.SaveOldMeta()); err != nil {
 				return errors.Wrap(err, "reconcile HAProxy svc")
 			}
 		}
@@ -894,7 +908,7 @@ func (r *PerconaServerMySQLReconciler) cleanupOutdatedServices(ctx context.Conte
 	log := logf.FromContext(ctx).WithName("cleanupOutdatedServices")
 
 	if !cr.HAProxyEnabled() || cr.Spec.Proxy.HAProxy.Size == 0 {
-		svc := haproxy.Service(cr)
+		svc := haproxy.Service(cr, nil)
 		if err := r.Client.Delete(ctx, svc); err != nil && !k8serrors.IsNotFound(err) {
 			return errors.Wrapf(err, "delete HAProxy svc %s", svc.Name)
 		}
@@ -942,7 +956,7 @@ func (r *PerconaServerMySQLReconciler) cleanupOutdatedStatefulSets(ctx context.C
 		}
 	}
 	if !cr.HAProxyEnabled() {
-		if err := r.Delete(ctx, haproxy.StatefulSet(cr, "")); err != nil && !k8serrors.IsNotFound(err) {
+		if err := r.Delete(ctx, haproxy.StatefulSet(cr, "", nil)); err != nil && !k8serrors.IsNotFound(err) {
 			return errors.Wrap(err, "failed to delete haproxy statefulset")
 		}
 	}
diff --git a/pkg/haproxy/haproxy.go b/pkg/haproxy/haproxy.go
diff --git a/pkg/mysql/mysql.go b/pkg/mysql/mysql.go
diff --git a/pkg/pmm/container.go b/pkg/pmm/container.go

Original file line number	Diff line number	Diff line change
`@@ -793,7 +793,7 @@ func (cr *PerconaServerMySQL) InternalSecretName() string {`
`793`	`793`	`}`
`794`	`794`
`795`	`795`	`func (cr PerconaServerMySQL) PMMEnabled(secret corev1.Secret) bool {`
`796`		`- if cr.Spec.PMM != nil && cr.Spec.PMM.Enabled && secret.Data != nil {`
	`796`	`+ if cr.Spec.PMM != nil && cr.Spec.PMM.Enabled && secret != nil && secret.Data != nil {`
`797`	`797`	`return cr.Spec.PMM.HasSecret(secret)`
`798`	`798`	`}`
`799`	`799`	`return false`
Original file line number	Diff line number	Diff line change
`@@ -719,7 +719,14 @@ func (r *PerconaServerMySQLReconciler) reconcileHAProxy(ctx context.Context, cr`
`719`	`719`	`return errors.Wrap(err, "get init image")`
`720`	`720`	`}`
`721`	`721`
`722`		`- if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, haproxy.StatefulSet(cr, initImage), r.Scheme); err != nil {`
	`722`	`+ internalSecret := new(corev1.Secret)`
	`723`	`+ nn = types.NamespacedName{Name: cr.InternalSecretName(), Namespace: cr.Namespace}`
	`724`	`+ err = r.Client.Get(ctx, nn, internalSecret)`
	`725`	`+ if client.IgnoreNotFound(err) != nil {`
	`726`	`+ return errors.Wrapf(err, "get Secret/%s", nn.Name)`
	`727`	`+ }`
	`728`	`+`
	`729`	`+ if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, haproxy.StatefulSet(cr, initImage, internalSecret), r.Scheme); err != nil {`
`723`	`730`	`return errors.Wrap(err, "reconcile StatefulSet")`
`724`	`731`	`}`
`725`	`732`
`@@ -736,8 +743,15 @@ func (r *PerconaServerMySQLReconciler) reconcileServices(ctx context.Context, cr`
`736`	`743`	`return errors.Wrap(err, "reconcile Orchestrator services")`
`737`	`744`	`}`
`738`	`745`	`if cr.HAProxyEnabled() {`
	`746`	`+ internalSecret := new(corev1.Secret)`
	`747`	`+ nn := types.NamespacedName{Name: cr.InternalSecretName(), Namespace: cr.Namespace}`
	`748`	`+ err := r.Client.Get(ctx, nn, internalSecret)`
	`749`	`+ if client.IgnoreNotFound(err) != nil {`
	`750`	`+ return errors.Wrapf(err, "get Secret/%s", nn.Name)`
	`751`	`+ }`
	`752`	`+`
`739`	`753`	`expose := cr.Spec.Proxy.HAProxy.Expose`
`740`		`- if err := k8s.EnsureService(ctx, r.Client, cr, haproxy.Service(cr), r.Scheme, expose.SaveOldMeta()); err != nil {`
	`754`	`+ if err := k8s.EnsureService(ctx, r.Client, cr, haproxy.Service(cr, internalSecret), r.Scheme, expose.SaveOldMeta()); err != nil {`
`741`	`755`	`return errors.Wrap(err, "reconcile HAProxy svc")`
`742`	`756`	`}`
`743`	`757`	`}`
`@@ -894,7 +908,7 @@ func (r *PerconaServerMySQLReconciler) cleanupOutdatedServices(ctx context.Conte`
`894`	`908`	`log := logf.FromContext(ctx).WithName("cleanupOutdatedServices")`
`895`	`909`
`896`	`910`	`if !cr.HAProxyEnabled() \|\| cr.Spec.Proxy.HAProxy.Size == 0 {`
`897`		`- svc := haproxy.Service(cr)`
	`911`	`+ svc := haproxy.Service(cr, nil)`
`898`	`912`	`if err := r.Client.Delete(ctx, svc); err != nil && !k8serrors.IsNotFound(err) {`
`899`	`913`	`return errors.Wrapf(err, "delete HAProxy svc %s", svc.Name)`
`900`	`914`	`}`
`@@ -942,7 +956,7 @@ func (r *PerconaServerMySQLReconciler) cleanupOutdatedStatefulSets(ctx context.C`
`942`	`956`	`}`
`943`	`957`	`}`
`944`	`958`	`if !cr.HAProxyEnabled() {`
`945`		`- if err := r.Delete(ctx, haproxy.StatefulSet(cr, "")); err != nil && !k8serrors.IsNotFound(err) {`
	`959`	`+ if err := r.Delete(ctx, haproxy.StatefulSet(cr, "", nil)); err != nil && !k8serrors.IsNotFound(err) {`
`946`	`960`	`return errors.Wrap(err, "failed to delete haproxy statefulset")`
`947`	`961`	`}`
`948`	`962`	`}`