Merge pull request #938 from percona/K8SPS-421

K8SPS-421: Add keyring vault support

Author: hors

api/v1alpha1/perconaservermysql_types.go

@@ -128,6 +128,8 @@ type MySQLSpec struct {
 	SidecarVolumes []corev1.Volume    `json:"sidecarVolumes,omitempty"`
 	SidecarPVCs    []SidecarPVC       `json:"sidecarPVCs,omitempty"`
 
+	VaultSecretName string `json:"vaultSecretName,omitempty"`
+
 	PodSpec `json:",inline"`
 }
 
@@ -592,6 +594,16 @@ func (cr *PerconaServerMySQL) SetVersion() {
 	cr.Spec.CRVersion = version.Version()
 }
 
+func (cr *PerconaServerMySQL) Version() *v.Version {
+	return v.Must(v.NewVersion(cr.Spec.CRVersion))
+}
+
+// CompareVersion compares given version to current version.
+// Returns -1, 0, or 1 if given version is smaller, equal, or larger than the current version, respectively.
+func (cr *PerconaServerMySQL) CompareVersion(ver string) int {
+	return cr.Version().Compare(v.Must(v.NewVersion(ver)))
+}
+
 // CheckNSetDefaults validates and sets default values for the PerconaServerMySQL custom resource.
 func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion *platform.ServerVersion) error {
 	if len(cr.Spec.MySQL.ClusterType) == 0 {
@@ -882,6 +894,10 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(_ context.Context, serverVersion
 		cr.Spec.SSLSecretName = cr.Name + "-ssl"
 	}
 
+	if cr.Spec.MySQL.VaultSecretName == "" {
+		cr.Spec.MySQL.VaultSecretName = cr.Name + "-vault"
+	}
+
 	return nil
 }
 

build/ps-entrypoint.sh

@@ -167,6 +167,26 @@ create_default_cnf() {
 		sed -i "/\[mysqld\]/a ssl_key=${TLS_DIR}/tls.key" $CFG
 	fi
 
+	# if vault secret file exists we assume we need to turn on encryption
+	vault_secret="/etc/mysql/vault-keyring-secret/keyring_vault.conf"
+	if [[ -f "${vault_secret}" ]]; then
+		sed -i "/\[mysqld\]/a early-plugin-load=keyring_vault.so" $CFG
+		sed -i "/\[mysqld\]/a keyring_vault_config=${vault_secret}" $CFG
+
+		if [[ ${MYSQL_VERSION} =~ ^(8\.0|8\.4)$ ]]; then
+			sed -i "/\[mysqld\]/a default_table_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a table_encryption_privilege_check=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_undo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_redo_log_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_encryption=ON" $CFG
+			sed -i "/\[mysqld\]/a binlog_rotate_encryption_master_key_at_startup=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_temp_tablespace_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_parallel_dblwr_encrypt=ON" $CFG
+			sed -i "/\[mysqld\]/a innodb_encrypt_online_alter_logs=ON" $CFG
+			sed -i "/\[mysqld\]/a encrypt_tmp_files=ON" $CFG
+		fi
+	fi
+
 	for f in "${CUSTOM_CONFIG_FILES[@]}"; do
 		echo "${f}"
 		if [ -f "${f}" ]; then

build/run-restore.sh

@@ -41,7 +41,13 @@ main() {
 		"azure") run_azure | extract "${tmpdir}" ;;
 	esac
 
-	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}"
+	local keyring=""
+	if [[ -f ${KEYRING_VAULT_PATH} ]]; then
+		echo "Using keyring vault config: ${KEYRING_VAULT_PATH}"
+		keyring="--keyring-vault-config=${KEYRING_VAULT_PATH}"
+	fi
+
+	xtrabackup --prepare --rollback-prepared-trx --target-dir="${tmpdir}" ${keyring}
 	xtrabackup --datadir="${DATADIR}" --move-back --force-non-empty-directories --target-dir="${tmpdir}"
 
 	rm -rf "${tmpdir}"

cmd/sidecar/main.go

@@ -9,12 +9,15 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
+	"time"
 
 	"golang.org/x/sync/errgroup"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -69,10 +72,7 @@ func (s *Status) GetBackupConfig() *xb.BackupConfig {
 	return &cfg
 }
 
-func main() {
-	opts := zap.Options{Development: true}
-	logf.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
-
+func startServer() *http.Server {
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("/health/", func(w http.ResponseWriter, req *http.Request) {
@@ -81,8 +81,42 @@ func main() {
 	mux.HandleFunc("/backup/", backupHandler)
 	mux.HandleFunc("/logs/", logHandler)
 
-	log.Info("starting http server")
-	log.Error(http.ListenAndServe(":"+strconv.Itoa(mysql.SidecarHTTPPort), mux), "http server failed")
+	srv := &http.Server{Addr: ":" + strconv.Itoa(mysql.SidecarHTTPPort), Handler: mux}
+
+	go func() {
+		log.Info("starting http server")
+		// always returns error. ErrServerClosed on graceful close
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			log.Error(err, "http server failed")
+		}
+	}()
+
+	return srv
+}
+
+func main() {
+	opts := zap.Options{Development: true}
+	logf.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
+	srv := startServer()
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
+
+	<-stop
+
+	log.Info("received interrupt signal, shutting down http server")
+
+	// TODO: should this timeout use terminationGracePeriodSeconds?
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := srv.Shutdown(ctx); err != nil {
+		log.Error(err, "graceful shutdown failed")
+		os.Exit(1)
+	}
+
+	os.Exit(0)
 }
 
 func getSecret(username apiv1alpha1.SystemUser) (string, error) {

config/crd/bases/ps.percona.com_perconaservermysqls.yaml

@@ -5026,6 +5026,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/bundle.yaml

@@ -6949,6 +6949,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/cr.yaml

@@ -52,6 +52,7 @@ spec:
 #      - name: "my-secret-1"
 #      - name: "my-secret-2"
 #    initImage: perconalab/percona-server-mysql-operator:main
+#    vaultSecretName: cluster1-vault
     size: 3
 
 #    env:

deploy/crd.yaml

@@ -6949,6 +6949,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/cw-bundle.yaml

@@ -6949,6 +6949,8 @@ spec:
                       - whenUnsatisfiable
                       type: object
                     type: array
+                  vaultSecretName:
+                    type: string
                   volumeSpec:
                     properties:
                       emptyDir:

deploy/vault-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster1-vault
+type: Opaque
+stringData:
+  keyring_vault.conf: |-
+    token = <secret>
+    vault_url = http://vault-service.vault-service.svc.cluster.local:8200
+    secret_mount_point = secret