feat: Enhance reloader with dynamic secret tracking and version change handling

- Introduced dynamic secret lease management in the Controller, allowing for tracking of secret metadata and restart times.
- Added methods to handle KV version changes and dynamic secret TTL thresholds for workload restarts.
- Implemented concurrency-safe access to workload tracking using sync.RWMutex.
- Updated the Vault client initialization to support dynamic secret renewal and version retrieval.
- Enhanced tests to cover scenarios for KV version changes, dynamic secret TTL thresholds, and mixed workload decisions.
- Refactored secret retrieval logic to return detailed secret information, including dynamic lease data.

Author: thiago-boche

deploy/charts/vault-secrets-reloader/templates/rbac.yaml

@@ -29,6 +29,14 @@ rules:
       - "list"
       - "update"
       - "watch"
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - "get"
+      - "list"
+      - "watch"
   - apiGroups:
       - ""
     resources:

deploy/charts/vault-secrets-reloader/values.yaml

@@ -98,6 +98,7 @@ env: {}
   # VAULT_PATH: "kubernetes"
   # VAULT_CLIENT_TIMEOUT: "10s"
   # VAULT_IGNORE_MISSING_SECRETS: "false"
+  # VAULT_DYNAMIC_SECRET_RESTART_THRESHOLD: "0.70"
 
 # -- Extra volume definitions for Reloader deployment
 volumes: []

pkg/reloader/collector.go

@@ -19,18 +19,24 @@ import (
 	"log/slog"
 	"regexp"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/bank-vaults/secrets-webhook/pkg/common"
 	corev1 "k8s.io/api/core/v1"
 )
 
+var (
+	vaultSecretRegex = regexp.MustCompile(`vault:([^#]+)#`)
+)
+
 type workloadSecretsStore interface {
-	Store(workload workload, secrets []string)
+	Store(workload workload, secrets []secretMetadata)
 	Delete(workload workload)
-	GetWorkloadSecretsMap() map[workload][]string
-	GetSecretWorkloadsMap() map[string][]workload
+	GetWorkloadSecretsMap() map[workload][]secretMetadata
+	GetSecretWorkloadsMap() map[secretMetadata][]workload
 }
 
 type workload struct {
@@ -39,18 +45,31 @@ type workload struct {
 	kind      string
 }
 
+type secretMetadata struct {
+	Path            string
+	IsKV            bool
+	KVVersion       int
+	IsDynamic       bool
+	DynamicLeaseTTL int // in seconds
+}
+
+type workloadTrackingInfo struct {
+	LastRestartTime    time.Time
+	ShortestDynamicTTL int // in seconds, 0 if no dynamic secrets
+}
+
 type workloadSecrets struct {
 	sync.RWMutex
-	workloadSecretsMap map[workload][]string
+	workloadSecretsMap map[workload][]secretMetadata
 }
 
 func newWorkloadSecrets() workloadSecretsStore {
 	return &workloadSecrets{
-		workloadSecretsMap: make(map[workload][]string),
+		workloadSecretsMap: make(map[workload][]secretMetadata),
 	}
 }
 
-func (w *workloadSecrets) Store(workload workload, secrets []string) {
+func (w *workloadSecrets) Store(workload workload, secrets []secretMetadata) {
 	w.Lock()
 	defer w.Unlock()
 	w.workloadSecretsMap[workload] = secrets
@@ -62,17 +81,17 @@ func (w *workloadSecrets) Delete(workload workload) {
 	delete(w.workloadSecretsMap, workload)
 }
 
-func (w *workloadSecrets) GetWorkloadSecretsMap() map[workload][]string {
+func (w *workloadSecrets) GetWorkloadSecretsMap() map[workload][]secretMetadata {
 	return w.workloadSecretsMap
 }
 
-func (w *workloadSecrets) GetSecretWorkloadsMap() map[string][]workload {
+func (w *workloadSecrets) GetSecretWorkloadsMap() map[secretMetadata][]workload {
 	w.Lock()
 	defer w.Unlock()
-	secretWorkloads := make(map[string][]workload)
-	for workload, secretPaths := range w.workloadSecretsMap {
-		for _, secretPath := range secretPaths {
-			secretWorkloads[secretPath] = append(secretWorkloads[secretPath], workload)
+	secretWorkloads := make(map[secretMetadata][]workload)
+	for workload, secretsMetadata := range w.workloadSecretsMap {
+		for _, secretMetadata := range secretsMetadata {
+			secretWorkloads[secretMetadata] = append(secretWorkloads[secretMetadata], workload)
 		}
 	}
 	return secretWorkloads
@@ -86,15 +105,128 @@ func (c *Controller) collectWorkloadSecrets(workload workload, template corev1.P
 
 	if len(vaultSecretPaths) == 0 {
 		collectorLogger.Debug("No Vault secret paths found in container env vars")
+
+		// Clear stale deployment state to prevent restarts based on old metadata
+		c.trackingMutex.Lock()
+		delete(c.workloadTracking, workload)
+		c.trackingMutex.Unlock()
+
 		return
 	}
 	collectorLogger.Debug(fmt.Sprintf("Vault secret paths found: %v", vaultSecretPaths))
 
+	// Query Vault for metadata about each secret
+	err := c.initVaultClientFn()
+	if err != nil {
+		collectorLogger.Error(fmt.Sprintf("Failed to initialize Vault client: %v", err))
+		return
+	}
+
+	// Build a lookup of already-tracked dynamic secrets to avoid re-reading them
+	c.trackingMutex.RLock()
+	existingMetadata, hasExisting := c.workloadSecrets.GetWorkloadSecretsMap()[workload]
+	c.trackingMutex.RUnlock()
+
+	dynamicMetadataByPath := make(map[string]secretMetadata)
+	if hasExisting {
+		for _, metadata := range existingMetadata {
+			if metadata.IsDynamic {
+				dynamicMetadataByPath[metadata.Path] = metadata
+			}
+		}
+	}
+
+	secretsMetadata := make([]secretMetadata, 0, len(vaultSecretPaths))
+	for _, secretPath := range vaultSecretPaths {
+		if metadata, exists := dynamicMetadataByPath[secretPath]; exists {
+			secretsMetadata = append(secretsMetadata, metadata)
+			collectorLogger.Debug(fmt.Sprintf("Secret %s is dynamic, using tracked TTL: %d seconds", secretPath, metadata.DynamicLeaseTTL))
+			continue
+		}
+
+		secretInfo, err := c.getSecretInfoFn(c.getVaultLogicalFn(), secretPath)
+		if err != nil {
+			collectorLogger.Error(fmt.Sprintf("Failed to get secret info for %s: %v", secretPath, err))
+			continue
+		}
+
+		metadata := secretMetadata{
+			Path: secretPath,
+		}
+
+		if secretInfo.IsKV {
+			metadata.IsKV = true
+			metadata.KVVersion = secretInfo.Version
+			collectorLogger.Debug(fmt.Sprintf("Secret %s is KV v2, version: %d", secretPath, secretInfo.Version))
+		} else if secretInfo.IsDynamic {
+			metadata.IsDynamic = true
+			metadata.DynamicLeaseTTL = secretInfo.LeaseInfo.LeaseDuration
+			collectorLogger.Debug(fmt.Sprintf("Secret %s is dynamic, TTL: %d seconds", secretPath, secretInfo.LeaseInfo.LeaseDuration))
+		}
+
+		secretsMetadata = append(secretsMetadata, metadata)
+	}
+
 	// Add workload and secrets to workloadSecrets map
-	c.workloadSecrets.Store(workload, vaultSecretPaths)
+	c.workloadSecrets.Store(workload, secretsMetadata)
 	collectorLogger.Info(fmt.Sprintf("Collected secrets from %s %s/%s", workload.kind, workload.namespace, workload.name))
 }
 
+func (c *Controller) trackWorkloadRestartTime(workload workload, pods []corev1.Pod) {
+	c.trackingMutex.Lock()
+	defer c.trackingMutex.Unlock()
+
+	// Get deployment secrets to calculate shortest dynamic TTL
+	workloadSecretsMeta, exists := c.workloadSecrets.GetWorkloadSecretsMap()[workload]
+	if !exists {
+		return
+	}
+
+	shortestTTL := 0
+	for _, secret := range workloadSecretsMeta {
+		if secret.IsDynamic {
+			if shortestTTL == 0 || secret.DynamicLeaseTTL < shortestTTL {
+				shortestTTL = secret.DynamicLeaseTTL
+			}
+		}
+	}
+
+	// Find the oldest running pod to determine deployment's effective start time
+	var oldestStartTime time.Time
+	for _, pod := range pods {
+		if pod.Status.StartTime == nil {
+			continue // Pod hasn't started yet
+		}
+
+		if pod.Status.Phase != corev1.PodRunning {
+			continue // Only consider running pods
+		}
+
+		if pod.ObjectMeta.DeletionTimestamp != nil {
+			continue // Skip pods that are terminating
+		}
+
+		if oldestStartTime.IsZero() || pod.Status.StartTime.Time.Before(oldestStartTime) {
+			oldestStartTime = pod.Status.StartTime.Time
+		}
+	}
+
+	// Only track if we found at least one running pod
+	if !oldestStartTime.IsZero() {
+		if existing, exists := c.workloadTracking[workload]; exists {
+			// Always update to reflect current pod state (oldest time may be earlier now,
+			// and TTL may have changed)
+			existing.LastRestartTime = oldestStartTime
+			existing.ShortestDynamicTTL = shortestTTL
+		} else {
+			c.workloadTracking[workload] = &workloadTrackingInfo{
+				LastRestartTime:    oldestStartTime,
+				ShortestDynamicTTL: shortestTTL,
+			}
+		}
+	}
+}
+
 func collectSecrets(template corev1.PodTemplateSpec) []string {
 	containers := []corev1.Container{}
 	containers = append(containers, template.Spec.Containers...)
@@ -114,12 +246,17 @@ func collectSecretsFromContainerEnvVars(containers []corev1.Container) []string
 	// iterate through all environment variables and extract secrets
 	for _, container := range containers {
 		for _, env := range container.Env {
-			// Skip if env var does not contain a vault secret or is a secret with pinned version
-			if isValidPrefix(env.Value) && unversionedSecretValue(env.Value) {
-				secret := regexp.MustCompile(`vault:(.*?)#`).FindStringSubmatch(env.Value)[1]
-				if secret != "" {
-					vaultSecretPaths = append(vaultSecretPaths, secret)
+			// Skip if env var does not contain a vault secret
+			if !isValidVaultSubstring(env.Value) {
+				continue
+			}
+
+			segments := extractVaultSecretSegments(env.Value)
+			for _, segment := range segments {
+				if segment.IsVersioned {
+					continue
 				}
+				vaultSecretPaths = append(vaultSecretPaths, segment.Path)
 			}
 		}
 	}
@@ -133,8 +270,12 @@ func collectSecretsFromAnnotations(annotations map[string]string) []string {
 	secretPaths := annotations[common.VaultFromPathAnnotation]
 	if secretPaths != "" {
 		for _, secretPath := range strings.Split(secretPaths, ",") {
-			if unversionedAnnotationSecretValue(secretPath) {
-				vaultSecretPaths = append(vaultSecretPaths, secretPath)
+			segments := extractVaultSecretSegments(secretPath)
+			for _, segment := range segments {
+				if segment.IsVersioned {
+					continue
+				}
+				vaultSecretPaths = append(vaultSecretPaths, segment.Path)
 			}
 		}
 	}
@@ -144,8 +285,12 @@ func collectSecretsFromAnnotations(annotations map[string]string) []string {
 		deprecatedSecretPaths := annotations[common.VaultEnvFromPathAnnotationDeprecated]
 		if deprecatedSecretPaths != "" {
 			for _, secretPath := range strings.Split(deprecatedSecretPaths, ",") {
-				if unversionedAnnotationSecretValue(secretPath) {
-					vaultSecretPaths = append(vaultSecretPaths, secretPath)
+				segments := extractVaultSecretSegments(secretPath)
+				for _, segment := range segments {
+					if segment.IsVersioned {
+						continue
+					}
+					vaultSecretPaths = append(vaultSecretPaths, segment.Path)
 				}
 			}
 		}
@@ -155,14 +300,63 @@ func collectSecretsFromAnnotations(annotations map[string]string) []string {
 }
 
 // implementation based on bank-vaults/secrets-webhook/pkg/provider/vault/provider.go
-func isValidPrefix(value string) bool {
-	return strings.HasPrefix(value, "vault:") || strings.HasPrefix(value, ">>vault:")
+func isValidVaultSubstring(value string) bool {
+	return vaultSecretRegex.MatchString(value)
+}
+
+type vaultSecretSegment struct {
+	Path        string
+	IsVersioned bool
+}
+
+func extractVaultSecretSegments(value string) []vaultSecretSegment {
+	segments := []vaultSecretSegment{}
+	searchIndex := 0
+	for {
+		start := strings.Index(value[searchIndex:], "vault:")
+		if start == -1 {
+			break
+		}
+		start += searchIndex + len("vault:")
+		segmentEnd := len(value)
+		if next := strings.Index(value[start:], "vault:"); next != -1 {
+			segmentEnd = start + next
+		}
+		segment := value[start:segmentEnd]
+		firstHash := strings.Index(segment, "#")
+		if firstHash == -1 {
+			searchIndex = start
+			continue
+		}
+		path := segment[:firstHash]
+		if path == "" {
+			searchIndex = start
+			continue
+		}
+		remainder := segment[firstHash+1:]
+		isVersioned := false
+		if remainder != "" {
+			parts := strings.Split(remainder, "#")
+			if len(parts) >= 2 {
+				last := parts[len(parts)-1]
+				if last != "" && isAllDigits(last) {
+					isVersioned = true
+				}
+			}
+		}
+		segments = append(segments, vaultSecretSegment{Path: path, IsVersioned: isVersioned})
+		searchIndex = start
+	}
+
+	return segments
 }
 
-// implementation based on bank-vaults/internal/pkg/injector/vault/injector.go
-func unversionedSecretValue(value string) bool {
-	split := strings.SplitN(value, "#", 3)
-	return len(split) == 2
+func isAllDigits(value string) bool {
+	if value == "" {
+		return false
+	}
+	_, err := strconv.ParseUint(value, 10, 64)
+	return err == nil
 }
 
 func unversionedAnnotationSecretValue(value string) bool {