Merge pull request #600 from xek/fix-daily-pod-recreation-OSPRH-16545

Fix daily keystone pod recreation by mounting keys directly

Author: openshift-merge-bot[bot]

controllers/keystoneapi_controller.go

@@ -1524,7 +1524,7 @@ func (r *KeystoneAPIReconciler) ensureFernetKeys(
 		numberKeys = int(*instance.Spec.FernetMaxActiveKeys)
 	}
 
-	secret, hash, err := oko_secret.GetSecret(ctx, helper, secretName, instance.Namespace)
+	secret, _, err := oko_secret.GetSecret(ctx, helper, secretName, instance.Namespace)
 
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		return err
@@ -1556,8 +1556,9 @@ func (r *KeystoneAPIReconciler) ensureFernetKeys(
 			return err
 		}
 	} else {
-		// add hash to envVars
-		(*envVars)[secret.Name] = env.SetValue(hash)
+		// DON'T add hash to envVars to prevent pod restarts when keys rotate
+		// Keys are mounted directly to /etc/keystone/fernet-keys, so Kubernetes
+		// will propagate changes automatically without needing pod recreation
 
 		changedKeys := false
 

pkg/keystone/volumes.go

@@ -130,12 +130,12 @@ func getVolumeMounts(
 			ReadOnly:  true,
 		},
 		{
-			MountPath: "/var/lib/fernet-keys",
+			MountPath: "/etc/keystone/fernet-keys",
 			ReadOnly:  true,
 			Name:      "fernet-keys",
 		},
 		{
-			MountPath: "/var/lib/credential-keys",
+			MountPath: "/etc/keystone/credential-keys",
 			ReadOnly:  true,
 			Name:      "credential-keys",
 		},

templates/keystoneapi/config/keystone-api-config.json

@@ -57,18 +57,6 @@
             "optional": true,
             "merge": true
         },
-        {
-            "source": "/var/lib/credential-keys",
-            "dest": "/etc/keystone/",
-            "owner": "keystone:keystone",
-            "perm": "0700"
-        },
-        {
-            "source": "/var/lib/fernet-keys",
-            "dest": "/etc/keystone/",
-            "owner": "keystone:keystone",
-            "perm": "0700"
-        },
         {
             "source": "/var/lib/config-data/default/my.cnf",
             "dest": "/etc/my.cnf",

tests/functional/keystoneapi_controller_test.go

@@ -1524,7 +1524,8 @@ var _ = Describe("Keystone controller", func() {
 
 			Eventually(func(g Gomega) {
 				keystone = GetKeystoneAPI(keystoneAPIName)
-				g.Expect(keystone.Status.Hash["input"]).ToNot(Equal(currentHash))
+				// With the new direct mounting approach, the input hash should NOT change
+				g.Expect(keystone.Status.Hash["input"]).To(Equal(currentHash))
 
 				updatedSecret := th.GetSecret(types.NamespacedName{Namespace: keystoneAPIName.Namespace, Name: "keystone"})
 				g.Expect(updatedSecret).ToNot(BeNil())

tests/kuttl/common/scripts/rotate_token.sh

@@ -28,11 +28,7 @@ for rotation in {1..5}; do
 
     sleep 100
 
-    # Wait for rollout to complete
-    if ! oc rollout status deployment/keystone -n $NAMESPACE --timeout=60s; then
-        echo "Rollout status check failed for rotation $rotation."
-        continue
-    fi
+    # Note: keystone is not being restarted
 
     echo "Rotation $rotation completed successfully."
 done

tests/kuttl/common/scripts/validate_test_token.sh

@@ -11,9 +11,9 @@ while [ $seconds -le 30 ]; do
     seconds=$(( seconds + 1 ))
 done
 
-sleep 20 # make sure a rollout started
-
-oc rollout status deployment/keystone -n $NAMESPACE
+# Wait for secret propagation to pods instead of waiting for rollout
+# Since pods no longer restart during key rotation, keys are updated via Kubernetes secret propagation
+sleep 60
 
 export OS_TOKEN=$(cat /tmp/temporary_test_token)
 

tests/kuttl/tests/keystone_tls/01-assert.yaml

@@ -45,10 +45,10 @@ spec:
           name: config-data
           readOnly: true
           subPath: keystone-api-config.json
-        - mountPath: /var/lib/fernet-keys
+        - mountPath: /etc/keystone/fernet-keys
           name: fernet-keys
           readOnly: true
-        - mountPath: /var/lib/credential-keys
+        - mountPath: /etc/keystone/credential-keys
           name: credential-keys
           readOnly: true
         - mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem