WIP: Re-enable Secret cache for VSO managed secrets.

Author: benashz

controllers/hcpvaultsecretsapp_controller.go

@@ -88,6 +88,7 @@ type HCPVaultSecretsAppReconciler struct {
 	referenceCache              ResourceReferenceCache
 	GlobalTransformationOptions *helpers.GlobalTransformationOptions
 	BackOffRegistry             *BackOffRegistry
+	SecretsClient               client.Client
 }
 
 // +kubebuilder:rbac:groups=secrets.hashicorp.com,resources=hcpvaultsecretsapps,verbs=get;list;watch;create;update;patch;delete
@@ -239,7 +240,7 @@ func (r *HCPVaultSecretsAppReconciler) Reconcile(ctx context.Context, req ctrl.R
 	doSync := true
 	// doRolloutRestart only if this is not the first time this secret has been synced
 	doRolloutRestart := o.Status.SecretMAC != ""
-	macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.Client, r.HMACValidator, o, data)
+	macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.SecretsClient, r.HMACValidator, o, data)
 	if err != nil {
 		return ctrl.Result{
 			RequeueAfter: computeHorizonWithJitter(requeueDurationOnError),
@@ -431,7 +432,7 @@ func (r *HCPVaultSecretsAppReconciler) getShadowSecretData(ctx context.Context,
 		return nil, fmt.Errorf("failed to marshal shadow secret data %s/%s: %w",
 			o.Namespace, o.Name, err)
 	}
-	valid, _, err := r.HMACValidator.Validate(ctx, r.Client, dataBytes, lastHMAC)
+	valid, _, err := r.HMACValidator.Validate(ctx, r.SecretsClient, dataBytes, lastHMAC)
 	if err != nil {
 		return nil, fmt.Errorf("failed to validate HMAC of HVS shadow secret data for %s/%s: %w",
 			o.Namespace, o.Name, err)
@@ -504,7 +505,7 @@ func (r *HCPVaultSecretsAppReconciler) storeShadowSecretData(ctx context.Context
 		return fmt.Errorf("failed to marshal shadow secret data %s/%s: %w",
 			o.Namespace, o.Name, err)
 	}
-	h, err := r.HMACValidator.HMAC(ctx, r.Client, b)
+	h, err := r.HMACValidator.HMAC(ctx, r.SecretsClient, b)
 	if err != nil {
 		return fmt.Errorf("failed to HMAC shadow secret data %s/%s: %w",
 			o.Namespace, o.Name, err)

controllers/vaultdynamicsecret_controller.go

@@ -72,6 +72,7 @@ type VaultDynamicSecretReconciler struct {
 	// This is done via the downwardAPI. We get the current Pod's UID from either the
 	// OPERATOR_POD_UID environment variable, or the /var/run/podinfo/uid file; in that order.
 	runtimePodUID types.UID
+	SecretsClient client.Client
 }
 
 // +kubebuilder:rbac:groups=secrets.hashicorp.com,resources=vaultdynamicsecrets,verbs=get;list;watch;create;update;patch;delete
@@ -423,7 +424,7 @@ func (r *VaultDynamicSecretReconciler) syncSecret(ctx context.Context, c vault.C
 			delete(dataToMAC, k)
 		}
 
-		macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.Client, r.HMACValidator, o, dataToMAC)
+		macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.SecretsClient, r.HMACValidator, o, dataToMAC)
 		if err != nil {
 			return nil, false, err
 		}

controllers/vaultpkisecret_controller.go

@@ -49,6 +49,7 @@ type VaultPKISecretReconciler struct {
 	BackOffRegistry             *BackOffRegistry
 	referenceCache              ResourceReferenceCache
 	GlobalTransformationOptions *helpers.GlobalTransformationOptions
+	SecretsClient               client.Client
 }
 
 // +kubebuilder:rbac:groups=secrets.hashicorp.com,resources=vaultpkisecrets,verbs=get;list;watch;create;update;patch;delete
@@ -134,7 +135,7 @@ func (r *VaultPKISecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		syncReason = consts.ReasonInexistentDestination
 	case destinationExists:
 		if schemaEpoch > 0 {
-			if matched, err := helpers.HMACDestinationSecret(ctx, r.Client,
+			if matched, err := helpers.HMACDestinationSecret(ctx, r.SecretsClient,
 				r.HMACValidator, o); err == nil && !matched {
 				syncReason = consts.ReasonSecretDataDrift
 			} else if err != nil {
@@ -252,7 +253,7 @@ func (r *VaultPKISecretReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	if b, err := json.Marshal(data); err == nil {
-		newMAC, err := r.HMACValidator.HMAC(ctx, r.Client, b)
+		newMAC, err := r.HMACValidator.HMAC(ctx, r.SecretsClient, b)
 		if err != nil {
 			logger.Error(err, "HMAC data")
 			o.Status.Error = consts.ReasonHMACDataError

controllers/vaultstaticsecret_controller.go

@@ -48,6 +48,7 @@ type VaultStaticSecretReconciler struct {
 	Recorder                    record.EventRecorder
 	ClientFactory               vault.ClientFactory
 	SecretDataBuilder           *helpers.SecretDataBuilder
+	SecretsClient               client.Client
 	HMACValidator               helpers.HMACValidator
 	referenceCache              ResourceReferenceCache
 	GlobalTransformationOptions *helpers.GlobalTransformationOptions
@@ -160,7 +161,7 @@ func (r *VaultStaticSecretReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		// doRolloutRestart only if this is not the first time this secret has been synced
 		doRolloutRestart = o.Status.SecretMAC != ""
 
-		macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.Client, r.HMACValidator, o, data)
+		macsEqual, messageMAC, err := helpers.HandleSecretHMAC(ctx, r.SecretsClient, r.HMACValidator, o, data)
 		if err != nil {
 			return ctrl.Result{RequeueAfter: computeHorizonWithJitter(requeueDurationOnError)}, nil
 		}

helpers/client.go

@@ -0,0 +1,67 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package helpers
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// NewSecretsClientForManager creates a new client for interacting with secrets that match
+// the given selector. This client is useful to avoid caching all secrets in a
+// cluster. The client will cache only secrets that match the selector.
+// The ctrl.Manager needs to be started before client can be used.
+func NewSecretsClientForManager(ctx context.Context, mgr ctrl.Manager, selector labels.Selector) (ctrlclient.Client, error) {
+	if selector.Empty() {
+		return nil, fmt.Errorf("selector cannot be empty")
+	}
+
+	scheme := mgr.GetScheme()
+	httpClient := mgr.GetHTTPClient()
+	restMapper := mgr.GetRESTMapper()
+	cacheOpts := cache.Options{
+		ReaderFailOnMissingInformer: true,
+		HTTPClient:                  httpClient,
+		Scheme:                      scheme,
+		Mapper:                      restMapper,
+		ByObject: map[ctrlclient.Object]cache.ByObject{
+			&corev1.Secret{}: {
+				Label: selector,
+			},
+		},
+	}
+
+	newCache, err := cache.New(mgr.GetConfig(), cacheOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := newCache.GetInformer(ctx, &corev1.Secret{}); err != nil {
+		return nil, err
+	}
+
+	if err := mgr.Add(newCache); err != nil {
+		return nil, err
+	}
+
+	client, err := ctrlclient.New(mgr.GetConfig(), ctrlclient.Options{
+		HTTPClient: httpClient,
+		Scheme:     scheme,
+		Mapper:     restMapper,
+		Cache: &ctrlclient.CacheOptions{
+			Reader: newCache,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}

helpers/secrets.go

@@ -34,6 +34,10 @@ const (
 	HVSSecretTypeKV       = "kv"
 	HVSSecretTypeRotating = "rotating"
 	HVSSecretTypeDynamic  = "dynamic"
+
+	ManagedByLabel = "app.kubernetes.io/managed-by"
+	AppNameLabel   = "app.kubernetes.io/name"
+	ComponentLabel = "app.kubernetes.io/component"
 )
 
 var SecretDataErrorContainsRaw = fmt.Errorf("key '%s' not permitted in Secret data", SecretDataKeyRaw)
@@ -48,10 +52,11 @@ var labelOwnerRefUID = fmt.Sprintf("%s/vso-ownerRefUID", secretsv1beta1.GroupVer
 // intersects with that of other components of the system, since this could lead to data loss.
 //
 // Make OwnerLabels public so that they can be accessed from tests.
+
 var OwnerLabels = map[string]string{
-	"app.kubernetes.io/name":       "vault-secrets-operator",
-	"app.kubernetes.io/managed-by": "hashicorp-vso",
-	"app.kubernetes.io/component":  "secret-sync",
+	ManagedByLabel: "hashicorp-vso",
+	AppNameLabel:   "vault-secrets-operator",
+	ComponentLabel: "secret-sync",
 }
 
 // OwnerLabelsForObj returns the canonical set of labels that should be set on

main.go

@@ -20,7 +20,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/selection"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -435,6 +437,35 @@ func main() {
 	}
 	ctx := ctrl.SetupSignalHandler()
 
+	var requirements []labels.Requirement
+	for _, k := range []string{helpers.ManagedByLabel, helpers.AppNameLabel} {
+		val, ok := helpers.OwnerLabels[k]
+		if !ok || val == "" {
+			setupLog.Error(errors.New("invalid option"),
+				fmt.Sprintf("Expected Owner label %q is not present, this is a bug", k))
+			os.Exit(1)
+		}
+
+		if r, err := labels.NewRequirement(
+			k, selection.Equals, []string{
+				val,
+			}); err != nil {
+			setupLog.Error(err, "Failed to create label requirement")
+			os.Exit(1)
+		} else {
+			requirements = append(requirements, *r)
+		}
+	}
+
+	// secretsClient is used to interact with secrets that match the selector. This
+	// client is useful to avoid caching all secrets in a cluster. The client will
+	// cache only secrets that match the selector.
+	secretsClient, err := helpers.NewSecretsClientForManager(ctx, mgr, labels.NewSelector().Add(requirements...))
+	if err != nil {
+		setupLog.Error(err, "Failed to create a Secrets client")
+		os.Exit(1)
+	}
+
 	var clientFactory vclient.CachingClientFactory
 	{
 		switch clientCachePersistenceModel {
@@ -467,6 +498,7 @@ func main() {
 		Scheme:                      mgr.GetScheme(),
 		Recorder:                    mgr.GetEventRecorderFor("VaultStaticSecret"),
 		SecretDataBuilder:           secretDataBuilder,
+		SecretsClient:               secretsClient,
 		HMACValidator:               hmacValidator,
 		ClientFactory:               clientFactory,
 		BackOffRegistry:             controllers.NewBackOffRegistry(backoffOpts...),
@@ -479,6 +511,7 @@ func main() {
 		Client:                      mgr.GetClient(),
 		Scheme:                      mgr.GetScheme(),
 		ClientFactory:               clientFactory,
+		SecretsClient:               secretsClient,
 		HMACValidator:               hmacValidator,
 		SyncRegistry:                controllers.NewSyncRegistry(),
 		Recorder:                    mgr.GetEventRecorderFor("VaultPKISecret"),
@@ -524,6 +557,7 @@ func main() {
 		Scheme:                      mgr.GetScheme(),
 		Recorder:                    mgr.GetEventRecorderFor("VaultDynamicSecret"),
 		ClientFactory:               clientFactory,
+		SecretsClient:               secretsClient,
 		HMACValidator:               hmacValidator,
 		SyncRegistry:                controllers.NewSyncRegistry(),
 		BackOffRegistry:             controllers.NewBackOffRegistry(backoffOpts...),
@@ -551,6 +585,7 @@ func main() {
 		Scheme:                      mgr.GetScheme(),
 		Recorder:                    mgr.GetEventRecorderFor("HCPVaultSecretsApp"),
 		SecretDataBuilder:           secretDataBuilder,
+		SecretsClient:               secretsClient,
 		HMACValidator:               hmacValidator,
 		MinRefreshAfter:             minRefreshAfterHVSA,
 		BackOffRegistry:             controllers.NewBackOffRegistry(backoffOpts...),