VSS: Add vault client callback handler (#867)

* VSS: add client callback to reconcile when reauth needed

* Adding VaultClientMeta

And sync reasons similar to VaultDynamicSecret

* Update vaultstaticsecret_controller.go

* Added integration test that confirms that no EOF error occurs after 3 ttl cycles

* comment description

---------

Co-authored-by: Jaired Jawed <jaired.jawed@hashicorp.com>

Author: tvoran

api/v1beta1/vaultstaticsecret_types.go

@@ -85,6 +85,9 @@ type VaultStaticSecretStatus struct {
 	// The SecretMac is also used to detect drift in the Destination Secret's Data.
 	// If drift is detected the data will be synced to the Destination.
 	SecretMAC string `json:"secretMAC,omitempty"`
+	// VaultClientMeta contains the status of the Vault client and is used during
+	// resource reconciliation.
+	VaultClientMeta VaultClientMeta `json:"vaultClientMeta,omitempty"`
 	// Conditions hold information that can be used by other apps to determine the
 	// health of the resource instance.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`

api/v1beta1/zz_generated.deepcopy.go

@@ -379,6 +379,21 @@ spec:
                   The SecretMac is also used to detect drift in the Destination Secret's Data.
                   If drift is detected the data will be synced to the Destination.
                 type: string
+              vaultClientMeta:
+                description: |-
+                  VaultClientMeta contains the status of the Vault client and is used during
+                  resource reconciliation.
+                properties:
+                  cacheKey:
+                    description: CacheKey is the unique key used to identify the client
+                      cache.
+                    type: string
+                  id:
+                    description: |-
+                      ID is the Vault ID of the authenticated client. The ID should never contain
+                      any sensitive information.
+                    type: string
+                type: object
             required:
             - lastGeneration
             type: object

chart/crds/secrets.hashicorp.com_vaultstaticsecrets.yaml

@@ -379,6 +379,21 @@ spec:
                   The SecretMac is also used to detect drift in the Destination Secret's Data.
                   If drift is detected the data will be synced to the Destination.
                 type: string
+              vaultClientMeta:
+                description: |-
+                  VaultClientMeta contains the status of the Vault client and is used during
+                  resource reconciliation.
+                properties:
+                  cacheKey:
+                    description: CacheKey is the unique key used to identify the client
+                      cache.
+                    type: string
+                  id:
+                    description: |-
+                      ID is the Vault ID of the authenticated client. The ID should never contain
+                      any sensitive information.
+                    type: string
+                type: object
             required:
             - lastGeneration
             type: object

config/crd/bases/secrets.hashicorp.com_vaultstaticsecrets.yaml

@@ -107,6 +107,20 @@ func (r *VaultStaticSecretReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		}, nil
 	}
 
+	destExists, _ := helpers.CheckSecretExists(ctx, r.Client, o)
+	if !o.Spec.Destination.Create && !destExists {
+		logger.Info("Destination secret does not exist, either create it or "+
+			"set .spec.destination.create=true", "destination", o.Spec.Destination)
+		return ctrl.Result{RequeueAfter: requeueDurationOnError}, nil
+	}
+
+	// we can ignore the error here, since it was handled above in the Get() call.
+	clientCacheKey, _ := c.GetCacheKey()
+
+	// update the VaultClientMeta in the resource's status.
+	o.Status.VaultClientMeta.CacheKey = clientCacheKey.String()
+	o.Status.VaultClientMeta.ID = c.ID()
+
 	var requeueAfter time.Duration
 	if o.Spec.RefreshAfter != "" {
 		d, err := parseDurationString(o.Spec.RefreshAfter, ".spec.refreshAfter", 0)
@@ -240,22 +254,15 @@ func (r *VaultStaticSecretReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		if err := helpers.SyncSecret(ctx, r.Client, o, data); err != nil {
 			r.Recorder.Eventf(o, corev1.EventTypeWarning, consts.ReasonSecretSyncError,
 				"Failed to update k8s secret: %s", err)
-
-			horizon := computeHorizonWithJitter(requeueDurationOnError)
-			if err := r.updateStatus(ctx, o, false, newSyncCondition(o, metav1.ConditionFalse, "Failed to sync the secret, horizon=%s, err=%s", horizon, err)); err != nil {
-				return ctrl.Result{}, err
-			}
-
-			return ctrl.Result{
-				RequeueAfter: horizon,
-			}, nil
+			return ctrl.Result{RequeueAfter: computeHorizonWithJitter(requeueDurationOnError)}, nil
 		}
 
 		conditions = append(conditions,
 			newSyncCondition(o, metav1.ConditionTrue,
 				"Secret synced, horizon=%s", requeueAfter),
 		)
 		r.Recorder.Event(o, corev1.EventTypeNormal, reason, "Secret synced")
+
 		if doRolloutRestart && len(o.Spec.RolloutRestartTargets) > 0 {
 			reason = consts.ReasonSecretRotated
 			// rollout-restart errors are not retryable
@@ -462,6 +469,8 @@ eventLoop:
 				r.Recorder.Eventf(&o, corev1.EventTypeWarning, consts.ReasonEventWatcherError,
 					"Error while watching events: %s", err)
 
+				logger.Error(err, "Error while watching events", "namespace", o.Namespace, "name", o.Name)
+
 				if errorCount >= errorThreshold {
 					logger.Error(err, "Too many errors while watching events, requeuing")
 					break eventLoop
@@ -596,6 +605,14 @@ func (r *VaultStaticSecretReconciler) SetupWithManager(mgr ctrl.Manager, opts co
 	if r.BackOffRegistry == nil {
 		r.BackOffRegistry = NewBackOffRegistry()
 	}
+
+	r.ClientFactory.RegisterClientCallbackHandler(
+		vault.ClientCallbackHandler{
+			On:       vault.ClientCallbackOnLifetimeWatcherDone | vault.ClientCallbackOnCacheRemoval,
+			Callback: r.vaultClientCallback,
+		},
+	)
+
 	r.SourceCh = make(chan event.GenericEvent)
 	r.eventWatcherRegistry = newEventWatcherRegistry()
 
@@ -629,6 +646,67 @@ func (r *VaultStaticSecretReconciler) SetupWithManager(mgr ctrl.Manager, opts co
 		Complete(r)
 }
 
+// vaultClientCallback requests reconciliation of all VaultStaticSecret
+// instances that were synced with Client
+func (r *VaultStaticSecretReconciler) vaultClientCallback(ctx context.Context, c vault.Client) {
+	logger := log.FromContext(ctx).WithName("vaultClientCallback")
+
+	cacheKey, err := c.GetCacheKey()
+	if err != nil {
+		// should never get here
+		logger.Error(err, "Invalid cache key, skipping syncing of VaultStaticSecret instances")
+		return
+	}
+
+	logger = logger.WithValues("cacheKey", cacheKey, "controller", "vss")
+	var l secretsv1beta1.VaultStaticSecretList
+	if err := r.Client.List(ctx, &l, client.InNamespace(
+		c.GetCredentialProvider().GetNamespace()),
+	); err != nil {
+		logger.Error(err, "Failed to list VaultStaticSecret instances")
+		return
+	}
+
+	if len(l.Items) == 0 {
+		return
+	}
+
+	reqs := map[client.ObjectKey]empty{}
+	for _, o := range l.Items {
+		if o.Status.VaultClientMeta.CacheKey == "" {
+			logger.V(consts.LogLevelWarning).Info("Skipping, cacheKey is empty",
+				"object", client.ObjectKeyFromObject(&o))
+			continue
+		}
+
+		curCacheKey := vault.ClientCacheKey(o.Status.VaultClientMeta.CacheKey)
+		if ok, err := curCacheKey.SameParent(cacheKey); ok {
+			evt := event.GenericEvent{
+				Object: &secretsv1beta1.VaultStaticSecret{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: o.GetNamespace(),
+						Name:      o.GetName(),
+					},
+				},
+			}
+
+			objKey := client.ObjectKeyFromObject(evt.Object)
+			if _, ok := reqs[objKey]; !ok {
+				// deduplicating is probably not necessary, but we do it just in case.
+				reqs[objKey] = empty{}
+				logger.V(consts.LogLevelDebug).Info("Enqueuing VaultStaticSecret instance",
+					"objKey", objKey)
+				logger.V(consts.LogLevelDebug).Info(
+					"Sending GenericEvent to the SourceCh", "evt", evt)
+				r.SourceCh <- evt
+			}
+		} else if err != nil {
+			logger.V(consts.LogLevelWarning).Info(
+				"Skipping, cacheKey error", "error", err)
+		}
+	}
+}
+
 func newKVRequest(s secretsv1beta1.VaultStaticSecretSpec) (vault.ReadRequest, error) {
 	var kvReq vault.ReadRequest
 	switch s.Type {

controllers/vaultstaticsecret_controller.go

@@ -1056,6 +1056,7 @@ sync the secret. This status is used during resource reconciliation.
 
 _Appears in:_
 - [VaultDynamicSecretStatus](#vaultdynamicsecretstatus)
+- [VaultStaticSecretStatus](#vaultstaticsecretstatus)
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |