diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 6ff2c556b..bd84fc25f 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -698,6 +698,8 @@ spec: type: string protocol: type: string + certSecretName: + type: string ttl: type: integer retry_timeout: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 2b85bd1e2..2ca3ad1b3 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -500,6 +500,8 @@ spec: type: string protocol: type: string + certSecretName: + type: string ttl: type: integer retry_timeout: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e1a9837a8..a4c41ba57 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -445,6 +445,7 @@ configMultisite: user: "" password: "" protocol: http + # certSecretName: .. # Timeout for cross site failover, and timeout for demoting to read only when accessing shared etcd cluster fails. # There should be adequate safety margin between the two to allow for demotion to take place. #ttl: 90 diff --git a/docs/hugo/content/en/crd/crd-postgresql.md b/docs/hugo/content/en/crd/crd-postgresql.md index 97ba2544a..b972b7f45 100644 --- a/docs/hugo/content/en/crd/crd-postgresql.md +++ b/docs/hugo/content/en/crd/crd-postgresql.md @@ -361,12 +361,13 @@ key, operator, value, effect and tolerationSeconds | #### etcd -| Name | Type | required | Description | -| ------------------------------ |:-------:| ---------:| ------------------:| -| hosts | string | true | list of etcd hosts, including etcd-client-port (default: `2379`), comma separated like in the etcd config | -| password | string | false | Password for the global etcd | -| protocol | string | true | Protocol for the global etcd (http or https) | -| user | string | false | Username for the global etcd | +| Name | Type | required | Description | +|----------------|:------:|---------:|----------------------------------------------------------------------------------------------------------:| +| hosts | string | true | list of etcd hosts, including etcd-client-port (default: `2379`), comma separated like in the etcd config | +| password | string | false | Password for the global etcd | +| protocol | string | true | Protocol for the global etcd (http or https) | +| user | string | false | Username for the global etcd | +| certSecretName | string | false | Secret for client certificates (tls.crt/key) and server certificate validation (ca.crt) | {{< back >}} diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 6797846f2..f24a603f8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -696,6 +696,8 @@ spec: type: string protocol: type: string + certSecretName: + type: string ttl: type: integer retry_timeout: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 20b32fe85..eb99ef2ff 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -498,6 +498,8 @@ spec: type: string protocol: type: string + certSecretName: + type: string ttl: type: integer retry_timeout: diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go index d85a0a849..ed5274cd6 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go @@ -658,6 +658,9 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "protocol": { Type: "string", }, + "certSecretName": { + Type: "string", + }, }, }, "ttl": { @@ -2307,14 +2310,25 @@ var OperatorConfigCRDResourceValidation = apiextv1.CustomResourceValidation{ "site": { Type: "string", }, - "etcd_host": { - Type: "string", - }, - "etcd_user": { - Type: "string", - }, - "etcd_password": { - Type: "string", + "etcd": { + Type: "object", + Properties: map[string]apiextv1.JSONSchemaProps{ + "hosts": { + Type: "string", + }, + "user": { + Type: "string", + }, + "password": { + Type: "string", + }, + "protocol": { + Type: "string", + }, + "certSecretName": { + Type: "string", + }, + }, }, "ttl": { Type: "integer", diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go index 4070e1e82..9e9ba2f0b 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go @@ -332,8 +332,9 @@ type Multisite struct { } type EtcdConfig struct { - Hosts *string `json:"hosts,omitempty"` - User *string `json:"user,omitempty"` - Password *string `json:"password,omitempty"` - Protocol *string `json:"protocol,omitempty"` + Hosts *string `json:"hosts,omitempty"` + User *string `json:"user,omitempty"` + Password *string `json:"password,omitempty"` + Protocol *string `json:"protocol,omitempty"` + CertSecretName *string `json:"certSecretName,omitempty"` } diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/zz_generated.deepcopy.go b/pkg/apis/cpo.opensource.cybertec.at/v1/zz_generated.deepcopy.go index 9daa74e86..7bb49d495 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/zz_generated.deepcopy.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/zz_generated.deepcopy.go @@ -215,6 +215,11 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { *out = new(string) **out = **in } + if in.CertSecretName != nil { + in, out := &in.CertSecretName, &out.CertSecretName + *out = new(string) + **out = **in + } return } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index f97611a1d..bc6f09d1c 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -4,6 +4,7 @@ package cluster import ( "context" + "crypto/tls" "database/sql" "encoding/base64" "encoding/json" @@ -2015,11 +2016,22 @@ func (c *Cluster) getPasswordForUser(username string) (string, error) { util.CoalesceStrPtr(msSpec.Etcd.Hosts, c.OpConfig.Multisite.Etcd.Hosts), util.CoalesceStrPtr(msSpec.Etcd.Protocol, c.OpConfig.Multisite.Etcd.Protocol), ) + certSecretName := util.CoalesceStrPtr(msSpec.Etcd.CertSecretName, c.OpConfig.Multisite.Etcd.CertSecretName) + var tlsConfig *tls.Config + if certSecretName != "" { + var err error + tlsConfig, err = c.getTlsConfigFromCertSecret(certSecretName) + if err != nil { + return "", err + } + } + client, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Username: util.CoalesceStrPtr(msSpec.Etcd.User, c.OpConfig.Multisite.Etcd.User), Password: util.CoalesceStrPtr(msSpec.Etcd.Password, c.OpConfig.Multisite.Etcd.Password), DialTimeout: time.Duration(2) * time.Second, + TLS: tlsConfig, }) if err != nil { return "", fmt.Errorf("unable to access multisite etcd: %s", err) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index a8878a724..4435c96c5 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -3,6 +3,8 @@ package cluster import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "path" @@ -1433,7 +1435,9 @@ func (c *Cluster) generateStatefulSet(spec *cpov1.PostgresSpec) (*appsv1.Statefu } if c.multisiteEnabled() { - spiloEnvVars = appendEnvVars(spiloEnvVars, c.generateMultisiteEnvVars()...) + multisiteEnvVars, multisiteVolumes := c.generateMultisiteEnvVars() + spiloEnvVars = appendEnvVars(spiloEnvVars, multisiteEnvVars...) + additionalVolumes = append(additionalVolumes, multisiteVolumes...) } // generate the spilo container @@ -3387,7 +3391,7 @@ func (c *Cluster) getPgbackrestJobName(repoName string, backupType string) (jobN return trimCronjobName(fmt.Sprintf("%s-%s-%s-%s", "pgbackrest", c.clusterName().Name, repoName, backupType)) } -func (c *Cluster) generateMultisiteEnvVars() []v1.EnvVar { +func (c *Cluster) generateMultisiteEnvVars() ([]v1.EnvVar, []cpov1.AdditionalVolume) { site, err := c.getPrimaryLoadBalancerIp() if err != nil { c.logger.Errorf("Error getting primary load balancer IP for %s: %s", c.Name, err) @@ -3410,5 +3414,81 @@ func (c *Cluster) generateMultisiteEnvVars() []v1.EnvVar { {Name: "UPDATE_CRD", Value: c.Namespace + "." + c.Name}, {Name: "CRD_UID", Value: string(c.UID)}, } - return envVars + + certSecretName := util.CoalesceStrPtr(clsConf.Etcd.CertSecretName, c.OpConfig.Multisite.Etcd.CertSecretName) + volumes := make([]cpov1.AdditionalVolume, 0, 1) + if certSecretName != "" { + + etcdCertMountPath := "/etcd-tls" + envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_CA_CERT", Value: etcdCertMountPath + "/ca.crt"}) + envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_CERT", Value: etcdCertMountPath + "/tls.crt"}) + envVars = append(envVars, v1.EnvVar{Name: "MULTISITE_ETCD_KEY", Value: etcdCertMountPath + "/tls.key"}) + defaultMode := int32(0640) + volumes = append(volumes, cpov1.AdditionalVolume{ + Name: "multisite-etcd-certs", + MountPath: etcdCertMountPath, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: certSecretName, + DefaultMode: &defaultMode, + }, + }}) + } + + return envVars, volumes +} + +func (c *Cluster) getTlsConfigFromCertSecret(certSecretName string) (*tls.Config, error) { + secret := &v1.Secret{} + var notFoundErr error + err := retryutil.Retry(c.OpConfig.ResourceCheckInterval, c.OpConfig.ResourceCheckTimeout, + func() (bool, error) { + var err error + secret, err = c.KubeClient.Secrets(c.Namespace).Get( + context.TODO(), + certSecretName, + metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + notFoundErr = err + return false, nil + } + return false, err + } + return true, nil + }, + ) + if notFoundErr != nil && err != nil { + err = errors.Wrap(notFoundErr, err.Error()) + } + if err != nil { + return nil, errors.Wrap(err, "could not get secret for TLS configuration") + } + + var caPool *x509.CertPool + if caCert, ok := secret.Data["ca.crt"]; ok { + caPool = x509.NewCertPool() + if !caPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to append CA cert from secret %s", certSecretName) + } + } + clientCert, certPresent := secret.Data["tls.crt"] + clientKey, keyPresent := secret.Data["tls.key"] + var certificates []tls.Certificate + if certPresent && keyPresent { + cert, err := tls.X509KeyPair(clientCert, clientKey) + if err != nil { + c.logger.Warningf("failed to load TLS client certificate from secret %s: %s", certSecretName, err) + } else { + certificates = append(certificates, cert) + } + } + + tlsConfig := &tls.Config{ + RootCAs: caPool, + Certificates: certificates, + MinVersion: tls.VersionTLS12, + } + + return tlsConfig, nil } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 12f27a536..a3ebd883e 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -288,6 +288,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *cpov1.OperatorConfigura result.Multisite.Etcd.User = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.User, "") result.Multisite.Etcd.Password = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.Password, "") result.Multisite.Etcd.Protocol = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.Protocol, "") + result.Multisite.Etcd.CertSecretName = util.CoalesceStrPtr(fromCRD.Multisite.Etcd.CertSecretName, "") result.Multisite.TTL = util.CoalesceInt32(fromCRD.Multisite.TTL, k8sutil.Int32ToPointer(90)) result.Multisite.RetryTimeout = util.CoalesceInt32(fromCRD.Multisite.RetryTimeout, k8sutil.Int32ToPointer(40)) diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2e37a0b65..a167fdb53 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -170,10 +170,11 @@ type Multisite struct { } type Etcd struct { - Hosts string `name:"multisite_etcd_hosts" default:""` - User string `name:"multisite_etcd_user" default:""` - Password string `name:"multisite_etcd_password" default:""` - Protocol string `name:"multisite_etcd_protocol" default:"http"` + Hosts string `name:"multisite_etcd_hosts" default:""` + User string `name:"multisite_etcd_user" default:""` + Password string `name:"multisite_etcd_password" default:""` + Protocol string `name:"multisite_etcd_protocol" default:"http"` + CertSecretName string `name:"multisite_etcd_cert_secret_name" default:""` } // Config describes operator config