Add certificate secret capability for multisite etcd configuration

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -698,6 +698,8 @@ spec:
                         type: string
                       protocol:
                         type: string
+                      certSecretName:
+                        type: string
                   ttl:
                     type: integer
                   retry_timeout:

charts/postgres-operator/crds/postgresqls.yaml

@@ -500,6 +500,8 @@ spec:
                             type: string
                           protocol:
                             type: string
+                          certSecretName:
+                            type: string
                       ttl:
                         type: integer
                       retry_timeout:

charts/postgres-operator/values.yaml

@@ -445,6 +445,7 @@ configMultisite:
     user: ""
     password: ""
     protocol: http
+    # certSecretName: ..
   # Timeout for cross site failover, and timeout for demoting to read only when accessing shared etcd cluster fails.
   # There should be adequate safety margin between the two to allow for demotion to take place.
   #ttl: 90

docs/hugo/content/en/crd/crd-postgresql.md

@@ -361,12 +361,13 @@ key, operator, value, effect and tolerationSeconds |
 
 #### etcd
 
-| Name                           | Type    | required  | Description        |
-| ------------------------------ |:-------:| ---------:| ------------------:|
-| hosts                          | string  | true      | list of etcd hosts, including etcd-client-port (default: `2379`), comma separated like in the etcd config  |
-| password                       | string  | false     | Password for the global etcd |
-| protocol                       | string  | true      | Protocol for the global etcd (http or https) |
-| user                           | string  | false     | Username for the global etcd |
+| Name           |  Type  | required |                                                                                               Description |
+|----------------|:------:|---------:|----------------------------------------------------------------------------------------------------------:|
+| hosts          | string |     true | list of etcd hosts, including etcd-client-port (default: `2379`), comma separated like in the etcd config |
+| password       | string |    false |                                                                              Password for the global etcd |
+| protocol       | string |     true |                                                              Protocol for the global etcd (http or https) |
+| user           | string |    false |                                                                              Username for the global etcd |
+| certSecretName | string |    false |                   Secret for client certificates (tls.crt/key) and server certificate validation (ca.crt) |
 
 {{< back >}}
 

manifests/operatorconfiguration.crd.yaml

@@ -696,6 +696,8 @@ spec:
                         type: string
                       protocol:
                         type: string
+                      certSecretName:
+                        type: string
                   ttl:
                     type: integer
                   retry_timeout:

manifests/postgresql.crd.yaml

@@ -498,6 +498,8 @@ spec:
                             type: string
                           protocol:
                             type: string
+                          certSecretName:
+                            type: string
                       ttl:
                         type: integer
                       retry_timeout:

pkg/apis/cpo.opensource.cybertec.at/v1/crds.go

@@ -658,6 +658,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 											"protocol": {
 												Type: "string",
 											},
+											"certSecretName": {
+												Type: "string",
+											},
 										},
 									},
 									"ttl": {
@@ -2307,14 +2310,25 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{
 							"site": {
 								Type: "string",
 							},
-							"etcd_host": {
-								Type: "string",
-							},
-							"etcd_user": {
-								Type: "string",
-							},
-							"etcd_password": {
-								Type: "string",
+							"etcd": {
+								Type: "object",
+								Properties: map[string]apiextv1.JSONSchemaProps{
+									"hosts": {
+										Type: "string",
+									},
+									"user": {
+										Type: "string",
+									},
+									"password": {
+										Type: "string",
+									},
+									"protocol": {
+										Type: "string",
+									},
+									"certSecretName": {
+										Type: "string",
+									},
+								},
 							},
 							"ttl": {
 								Type: "integer",

pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go

@@ -332,8 +332,9 @@ type Multisite struct {
 }
 
 type EtcdConfig struct {
-	Hosts    *string `json:"hosts,omitempty"`
-	User     *string `json:"user,omitempty"`
-	Password *string `json:"password,omitempty"`
-	Protocol *string `json:"protocol,omitempty"`
+	Hosts          *string `json:"hosts,omitempty"`
+	User           *string `json:"user,omitempty"`
+	Password       *string `json:"password,omitempty"`
+	Protocol       *string `json:"protocol,omitempty"`
+	CertSecretName *string `json:"certSecretName,omitempty"`
 }

pkg/cluster/cluster.go

@@ -4,6 +4,7 @@ package cluster
 
 import (
 	"context"
+	"crypto/tls"
 	"database/sql"
 	"encoding/base64"
 	"encoding/json"
@@ -2015,11 +2016,22 @@ func (c *Cluster) getPasswordForUser(username string) (string, error) {
 			util.CoalesceStrPtr(msSpec.Etcd.Hosts, c.OpConfig.Multisite.Etcd.Hosts),
 			util.CoalesceStrPtr(msSpec.Etcd.Protocol, c.OpConfig.Multisite.Etcd.Protocol),
 		)
+		certSecretName := util.CoalesceStrPtr(msSpec.Etcd.CertSecretName, c.OpConfig.Multisite.Etcd.CertSecretName)
+		var tlsConfig *tls.Config
+		if certSecretName != "" {
+			var err error
+			tlsConfig, err = c.getTlsConfigFromCertSecret(certSecretName)
+			if err != nil {
+				return "", err
+			}
+		}
+
 		client, err := clientv3.New(clientv3.Config{
 			Endpoints:   endpoints,
 			Username:    util.CoalesceStrPtr(msSpec.Etcd.User, c.OpConfig.Multisite.Etcd.User),
 			Password:    util.CoalesceStrPtr(msSpec.Etcd.Password, c.OpConfig.Multisite.Etcd.Password),
 			DialTimeout: time.Duration(2) * time.Second,
+			TLS:         tlsConfig,
 		})
 		if err != nil {
 			return "", fmt.Errorf("unable to access multisite etcd: %s", err)

pkg/cluster/k8sres.go

@@ -3,6 +3,8 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"path"
@@ -1433,7 +1435,9 @@ func (c *Cluster) generateStatefulSet(spec *cpov1.PostgresSpec) (*appsv1.Statefu
 	}
 
 	if c.multisiteEnabled() {
-		spiloEnvVars = appendEnvVars(spiloEnvVars, c.generateMultisiteEnvVars()...)
+		multisiteEnvVars, multisiteVolumes := c.generateMultisiteEnvVars()
+		spiloEnvVars = appendEnvVars(spiloEnvVars, multisiteEnvVars...)
+		additionalVolumes = append(additionalVolumes, multisiteVolumes...)
 	}
 
 	// generate the spilo container
@@ -3387,7 +3391,7 @@ func (c *Cluster) getPgbackrestJobName(repoName string, backupType string) (jobN
 	return trimCronjobName(fmt.Sprintf("%s-%s-%s-%s", "pgbackrest", c.clusterName().Name, repoName, backupType))
 }
 
-func (c *Cluster) generateMultisiteEnvVars() []v1.EnvVar {
+func (c *Cluster) generateMultisiteEnvVars() ([]v1.EnvVar, []cpov1.AdditionalVolume) {
 	site, err := c.getPrimaryLoadBalancerIp()
 	if err != nil {
 		c.logger.Errorf("Error getting primary load balancer IP for %s: %s", c.Name, err)
@@ -3410,5 +3414,81 @@ func (c *Cluster) generateMultisiteEnvVars() []v1.EnvVar {
 		{Name: "UPDATE_CRD", Value: c.Namespace + "." + c.Name},
 		{Name: "CRD_UID", Value: string(c.UID)},
 	}
-	return envVars
+
+	certSecretName := util.CoalesceStrPtr(clsConf.Etcd.CertSecretName, c.OpConfig.Multisite.Etcd.CertSecretName)
+	volumes := make([]cpov1.AdditionalVolume, 0, 1)
+	if certSecretName != "" {
+
+		etcdCertMountPath := "/etcd-tls"
+		envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_CA_CERT", Value: etcdCertMountPath + "/ca.crt"})
+		envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_CERT", Value: etcdCertMountPath + "/tls.crt"})
+		envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_KEY", Value: etcdCertMountPath + "/tls.key"})
+		defaultMode := int32(0640)
+		volumes = append(volumes, cpov1.AdditionalVolume{
+			Name:      "multisite-etcd-certs",
+			MountPath: etcdCertMountPath,
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName:  certSecretName,
+					DefaultMode: &defaultMode,
+				},
+			}})
+	}
+
+	return envVars, volumes
+}
+
+func (c *Cluster) getTlsConfigFromCertSecret(certSecretName string) (*tls.Config, error) {
+	secret := &v1.Secret{}
+	var notFoundErr error
+	err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout,
+		func() (bool, error) {
+			var err error
+			secret, err = c.KubeClient.Secrets(c.Namespace).Get(
+				context.TODO(),
+				certSecretName,
+				metav1.GetOptions{})
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					notFoundErr = err
+					return false, nil
+				}
+				return false, err
+			}
+			return true, nil
+		},
+	)
+	if notFoundErr != nil && err != nil {
+		err = errors.Wrap(notFoundErr, err.Error())
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get secret for TLS configuration")
+	}
+
+	var caPool *x509.CertPool
+	if caCert, ok := secret.Data["ca.crt"]; ok {
+		caPool = x509.NewCertPool()
+		if !caPool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to append CA cert from secret %s", certSecretName)
+		}
+	}
+	clientCert, certPresent := secret.Data["tls.crt"]
+	clientKey, keyPresent := secret.Data["tls.key"]
+	var certificates []tls.Certificate
+	if certPresent && keyPresent {
+		cert, err := tls.X509KeyPair(clientCert, clientKey)
+		if err != nil {
+			c.logger.Warningf("failed to load TLS client certificate from secret %s: %s", certSecretName, err)
+		} else {
+			certificates = append(certificates, cert)
+		}
+	}
+
+	tlsConfig := &tls.Config{
+		RootCAs:      caPool,
+		Certificates: certificates,
+		MinVersion:   tls.VersionTLS12,
+	}
+
+	return tlsConfig, nil
 }

pkg/controller/operator_config.go

@@ -288,6 +288,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *cpov1.OperatorConfigura
 	result.Multisite.Etcd.User = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.User, "")
 	result.Multisite.Etcd.Password = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.Password, "")
 	result.Multisite.Etcd.Protocol = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.Protocol, "")
+	result.Multisite.Etcd.CertSecretName = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.CertSecretName, "")
 	result.Multisite.TTL = util.CoalesceInt32(fromCRD.Multisite.TTL, k8sutil.Int32ToPointer(90))
 	result.Multisite.RetryTimeout = util.CoalesceInt32(fromCRD.Multisite.RetryTimeout, k8sutil.Int32ToPointer(40))
 

pkg/util/config/config.go

@@ -170,10 +170,11 @@ type Multisite struct {
 }
 
 type Etcd struct {
-	Hosts    string `name:"multisite_etcd_hosts" default:""`
-	User     string `name:"multisite_etcd_user" default:""`
-	Password string `name:"multisite_etcd_password" default:""`
-	Protocol string `name:"multisite_etcd_protocol" default:"http"`
+	Hosts          string `name:"multisite_etcd_hosts" default:""`
+	User           string `name:"multisite_etcd_user" default:""`
+	Password       string `name:"multisite_etcd_password" default:""`
+	Protocol       string `name:"multisite_etcd_protocol" default:"http"`
+	CertSecretName string `name:"multisite_etcd_cert_secret_name" default:""`
 }
 
 // Config describes operator config