YDBOPS-9692 move encryption config to args (#217)

Author: kobzonega

api/v1alpha1/configuration.go

@@ -2,29 +2,14 @@ package v1alpha1
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"fmt"
-	"path"
 	"strconv"
 
 	"gopkg.in/yaml.v3"
 
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/configuration/schema"
 )
 
-const (
-	DatabaseEncryptionKeyPath           = "/opt/ydb/secrets/database_encryption"
-	DatabaseEncryptionKeyFile           = "key"
-	DatastreamsIAMServiceAccountKeyPath = "/opt/ydb/secrets/datastreams"
-	DatastreamsIAMServiceAccountKeyFile = "sa_key.json"
-)
-
-func hash(text string) string {
-	h := sha256.New()
-	h.Write([]byte(text))
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
 func generateHosts(cr *Storage) []schema.Host {
 	var hosts []schema.Host
 
@@ -61,24 +46,6 @@ func generateHosts(cr *Storage) []schema.Host {
 	return hosts
 }
 
-func generateKeyConfig(cr *Storage, crDB *Database) *schema.KeyConfig {
-	var keyConfig *schema.KeyConfig
-	if crDB != nil && crDB.Spec.Encryption != nil && crDB.Spec.Encryption.Enabled {
-		keyConfig = &schema.KeyConfig{
-			Keys: []schema.Key{
-				{
-					ContainerPath: path.Join(DatabaseEncryptionKeyPath, DatabaseEncryptionKeyFile),
-					ID:            hash(cr.Name),
-					Pin:           crDB.Spec.Encryption.Pin,
-					Version:       1,
-				},
-			},
-		}
-	}
-
-	return keyConfig
-}
-
 func BuildConfiguration(cr *Storage, crDB *Database) ([]byte, error) {
 	config := make(map[string]interface{})
 
@@ -113,12 +80,6 @@ func BuildConfiguration(cr *Storage, crDB *Database) ([]byte, error) {
 		config["hosts"] = hosts
 	}
 
-	// Will be removed by YDBOPS-9692
-	keyConfig := generateKeyConfig(cr, crDB)
-	if keyConfig != nil {
-		config["key_config"] = keyConfig
-	}
-
 	return yaml.Marshal(config)
 }
 

api/v1alpha1/const.go

@@ -30,12 +30,21 @@ const (
 	ConfigDir      = "/opt/ydb/cfg"
 	ConfigFileName = "config.yaml"
 
+	DatabaseEncryptionKeySecretDir  = "encryption"
+	DatabaseEncryptionKeySecretFile = "key.pem"
+	DatabaseEncryptionKeyConfigFile = "key.txt"
+
+	DatastreamsIAMServiceAccountKeyDir  = "datastreams"
+	DatastreamsIAMServiceAccountKeyFile = "sa_key.json"
+
 	BinariesDir      = "/opt/ydb/bin"
 	DaemonBinaryName = "ydbd"
 
-	DefaultRootUsername  = "root"
-	DefaultRootPassword  = ""
-	DefaultSignAlgorithm = "RS256"
+	DefaultRootUsername          = "root"
+	DefaultRootPassword          = ""
+	DefaultDatabaseDomain        = "Root"
+	DefaultDatabaseEncryptionPin = "EmptyPin"
+	DefaultSignAlgorithm         = "RS256"
 
 	LabelDeploymentKey             = "deployment"
 	LabelDeploymentValueKubernetes = "kubernetes"

api/v1alpha1/database_webhook.go

@@ -17,10 +17,6 @@ import (
 	. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
 )
 
-const (
-	DefaultDatabaseDomain = "Root"
-)
-
 // log is for logging in this package.
 var databaselog = logf.Log.WithName("database-resource")
 
@@ -126,6 +122,13 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 		database.Spec.Encryption = &EncryptionConfig{Enabled: false}
 	}
 
+	if database.Spec.Encryption.Enabled && database.Spec.Encryption.Key == nil {
+		if database.Spec.Encryption.Pin == nil || len(*database.Spec.Encryption.Pin) == 0 {
+			encryptionPin := DefaultDatabaseEncryptionPin
+			database.Spec.Encryption.Pin = &encryptionPin
+		}
+	}
+
 	if database.Spec.Datastreams == nil {
 		database.Spec.Datastreams = &DatastreamsConfig{Enabled: false}
 	}
@@ -149,7 +152,7 @@ func (r *DatabaseDefaulter) Default(ctx context.Context, obj runtime.Object) err
 		database.Spec.StorageEndpoint = storage.GetStorageEndpointWithProto()
 	}
 
-	if database.Spec.Configuration != "" || (database.Spec.Encryption != nil && database.Spec.Encryption.Enabled) {
+	if database.Spec.Configuration != "" {
 		configuration, err := BuildConfiguration(storage, database)
 		if err != nil {
 			return err

deploy/ydb-operator/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.22
+version: 0.5.23
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.5.22"
+appVersion: "0.5.23"

e2e/tests/smoke_test.go

@@ -866,6 +866,54 @@ var _ = Describe("Operator smoke test", func() {
 		)
 	})
 
+	It("Check encryption for Database", func() {
+		By("create storage...")
+		Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed())
+		defer func() {
+			Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed())
+		}()
+		By("create database...")
+		databaseSample.Spec.Encryption = &v1alpha1.EncryptionConfig{
+			Enabled: true,
+		}
+		Expect(k8sClient.Create(ctx, databaseSample)).Should(Succeed())
+		defer func() {
+			Expect(k8sClient.Delete(ctx, databaseSample)).Should(Succeed())
+		}()
+
+		By("waiting until Storage is ready...")
+		waitUntilStorageReady(ctx, storageSample.Name, testobjects.YdbNamespace)
+
+		By("checking that all the storage pods are running and ready...")
+		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes)
+
+		By("waiting until database is ready...")
+		waitUntilDatabaseReady(ctx, databaseSample.Name, testobjects.YdbNamespace)
+
+		By("checking that all the database pods are running and ready...")
+		checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
+
+		database := v1alpha1.Database{}
+		Expect(k8sClient.Get(ctx, types.NamespacedName{
+			Name:      databaseSample.Name,
+			Namespace: testobjects.YdbNamespace,
+		}, &database)).Should(Succeed())
+		storageEndpoint := database.Spec.StorageEndpoint
+
+		databasePods := corev1.PodList{}
+		Expect(k8sClient.List(ctx, &databasePods,
+			client.InNamespace(testobjects.YdbNamespace),
+			client.MatchingLabels{"ydb-cluster": "kind-database"}),
+		).Should(Succeed())
+		podName := databasePods.Items[0].Name
+
+		By("bring YDB CLI inside ydb database pod...")
+		bringYdbCliToPod(podName, testobjects.YdbNamespace)
+
+		By("execute simple query inside ydb database pod...")
+		executeSimpleQuery(podName, testobjects.YdbNamespace, storageEndpoint)
+	})
+
 	AfterEach(func() {
 		Expect(uninstallOperatorWithHelm(testobjects.YdbNamespace)).Should(BeTrue())
 		Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())

internal/controllers/database/controller_test.go

@@ -3,7 +3,9 @@ package database_test
 import (
 	"context"
 	"errors"
+	"fmt"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -142,5 +144,77 @@ var _ = Describe("Database controller medium tests", func() {
 				}
 			}
 		})
+
+		By("Check encryption for Database...")
+		foundDatabase := v1alpha1.Database{}
+		Expect(k8sClient.Get(ctx, types.NamespacedName{
+			Name:      databaseSample.Name,
+			Namespace: testobjects.YdbNamespace,
+		}, &foundDatabase))
+
+		By("Update Database and enable encryption...")
+		foundDatabase.Spec.Encryption = &v1alpha1.EncryptionConfig{Enabled: true}
+		Expect(k8sClient.Update(ctx, &foundDatabase)).Should(Succeed())
+
+		By("Check that encryption secret was created...")
+		encryptionSecret := corev1.Secret{}
+		Eventually(func() error {
+			return k8sClient.Get(ctx, types.NamespacedName{
+				Name:      databaseSample.Name,
+				Namespace: testobjects.YdbNamespace,
+			}, &encryptionSecret)
+		}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
+		encryptionData := encryptionSecret.Data
+
+		By("Check that arg `--key-file` was added to StatefulSet...")
+		databaseStatefulSet = appsv1.StatefulSet{}
+		Eventually(func() error {
+			Expect(k8sClient.List(ctx,
+				&foundStatefulSets,
+				client.InNamespace(testobjects.YdbNamespace),
+			)).ShouldNot(HaveOccurred())
+			for idx, statefulSet := range foundStatefulSets.Items {
+				if statefulSet.Name == testobjects.DatabaseName {
+					databaseStatefulSet = foundStatefulSets.Items[idx]
+					break
+				}
+			}
+			podContainerArgs := databaseStatefulSet.Spec.Template.Spec.Containers[0].Args
+			encryptionKeyConfigPath := fmt.Sprintf("%s/%s", v1alpha1.ConfigDir, v1alpha1.DatabaseEncryptionKeyConfigFile)
+			for idx, arg := range podContainerArgs {
+				if arg == "--key-file" {
+					if podContainerArgs[idx+1] == encryptionKeyConfigPath {
+						return nil
+					}
+					return fmt.Errorf(
+						"Found arg `--key-file=%s` for encryption does not match with expected path: %s",
+						podContainerArgs[idx+1],
+						encryptionKeyConfigPath,
+					)
+				}
+			}
+			return errors.New("Failed to find arg `--key-file` for encryption in StatefulSet")
+		}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
+
+		By("Update Database encryption pin...")
+		Expect(k8sClient.Get(ctx, types.NamespacedName{
+			Name:      databaseSample.Name,
+			Namespace: testobjects.YdbNamespace,
+		}, &foundDatabase))
+		pin := "Ignore"
+		foundDatabase.Spec.Encryption = &v1alpha1.EncryptionConfig{
+			Enabled: true,
+			Pin:     &pin,
+		}
+		Expect(k8sClient.Update(ctx, &foundDatabase)).Should(Succeed())
+
+		By("Check that Secret for encryption was not changed...")
+		Consistently(func(g Gomega) bool {
+			g.Expect(k8sClient.Get(ctx, types.NamespacedName{
+				Name:      databaseSample.Name,
+				Namespace: testobjects.YdbNamespace,
+			}, &encryptionSecret))
+			return reflect.DeepEqual(encryptionData, encryptionSecret.Data)
+		}, test.Timeout, test.Interval).Should(BeTrue())
 	})
 })

internal/controllers/database/sync.go

@@ -364,6 +364,13 @@ func shouldIgnoreDatabaseChange(database *resources.DatabaseBuilder) resources.I
 				return true
 			}
 		}
+
+		if sec, ok := oldObj.(*corev1.Secret); ok {
+			// Do not update already existing secret data for encryption
+			if (len(sec.StringData) > 0) || (len(sec.Data) > 0) {
+				return true
+			}
+		}
 		return false
 	}
 }

internal/resources/configmap.go

@@ -1,19 +1,42 @@
 package resources
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
+	"html/template"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	api "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
+	"github.com/ydb-platform/ydb-kubernetes-operator/internal/configuration/schema"
 )
 
+const keyConfigTmpl = `Keys {
+    ContainerPath: "{{ .ContainerPath }}"
+    Pin: "{{ .Pin }}"
+    Id: "{{ .ID }}"
+    Version: {{ .Version }}
+}`
+
 type ConfigMapBuilder struct {
 	client.Object
 
 	Name   string
-	Data   map[string]string
 	Labels map[string]string
+
+	Data map[string]string
+}
+
+type EncryptionConfigBuilder struct {
+	client.Object
+
+	Name   string
+	Labels map[string]string
+
+	KeyConfig schema.KeyConfig
 }
 
 func (b *ConfigMapBuilder) Build(obj client.Object) error {
@@ -23,13 +46,43 @@ func (b *ConfigMapBuilder) Build(obj client.Object) error {
 	}
 
 	if cm.ObjectMeta.Name == "" {
-		cm.ObjectMeta.Name = b.GetName()
+		cm.ObjectMeta.Name = b.Name
 	}
 	cm.ObjectMeta.Namespace = b.GetNamespace()
 
+	cm.Labels = b.Labels
+
 	cm.Data = b.Data
+
+	return nil
+}
+
+func (b *EncryptionConfigBuilder) Build(obj client.Object) error {
+	cm, ok := obj.(*v1.ConfigMap)
+	if !ok {
+		return errors.New("failed to cast to ConfigMap object")
+	}
+
+	if cm.ObjectMeta.Name == "" {
+		cm.ObjectMeta.Name = b.Name
+	}
+	cm.ObjectMeta.Namespace = b.GetNamespace()
+
 	cm.Labels = b.Labels
 
+	t, err := template.New("keyConfig").Parse(keyConfigTmpl)
+	if err != nil {
+		return fmt.Errorf("failed to parse keyConfig template: %w", err)
+	}
+
+	var buf bytes.Buffer
+	err = t.Execute(&buf, b.KeyConfig.Keys[0])
+	if err != nil {
+		return fmt.Errorf("failed to execute keyConfig template: %w", err)
+	}
+
+	cm.Data = map[string]string{api.DatabaseEncryptionKeyConfigFile: buf.String()}
+
 	return nil
 }
 
@@ -41,3 +94,12 @@ func (b *ConfigMapBuilder) Placeholder(cr client.Object) client.Object {
 		},
 	}
 }
+
+func (b *EncryptionConfigBuilder) Placeholder(cr client.Object) client.Object {
+	return &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      b.Name,
+			Namespace: cr.GetNamespace(),
+		},
+	}
+}