fix: Fix ACL SAVE when using ACL from PVC (#1645)

naimadswdn · Damian Seredyn · web-flow · commit 9299c997b3ff · 2026-02-05T13:50:12.000+08:00
Redis always rewrites ACLs (and redis.conf, RDB, AOF) via the same pattern: write a temp file alongside the target, fsync, then rename(2) over the original.
Because user.acl is mounted via subPath, Kubernetes turns that single file into its own bind mount.
Bind mounts behave like a mini mount point, and Linux forbids rename(2) on a mount target—exactly what Redis tries to do during ACL SAVE (it writes tempfile + rename).
The kernel therefore returns EBUSY, which surfaces as “Resource busy”.

The PR is fixing this behavior, by mounting the PVC as directory under the /data/redis.

Signed-off-by: Damian Seredyn &lt;s-DSeredyn@aras.com&gt;
Co-authored-by: Damian Seredyn &lt;s-DSeredyn@aras.com&gt;
diff --git a/api/common/v1beta2/common_types.go b/api/common/v1beta2/common_types.go
@@ -288,7 +288,8 @@ type ACLConfig struct {
 	Secret *corev1.SecretVolumeSource `json:"secret,omitempty"`
 	// PersistentVolumeClaim-based ACL configuration
 	// Specify the PVC name to mount ACL file from persistent storage
-	// The operator will automatically mount /etc/redis/user.acl from the PVC
+	// The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+	// This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
 	PersistentVolumeClaim *string `json:"persistentVolumeClaim,omitempty"`
 }
 
diff --git a/charts/redis-operator/crds/crds.yaml b/charts/redis-operator/crds/crds.yaml
@@ -117,7 +117,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
@@ -5544,7 +5545,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
@@ -13393,7 +13395,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml
@@ -118,7 +118,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml
@@ -146,7 +146,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml
@@ -124,7 +124,8 @@ spec:
                     description: |-
                       PersistentVolumeClaim-based ACL configuration
                       Specify the PVC name to mount ACL file from persistent storage
-                      The operator will automatically mount /etc/redis/user.acl from the PVC
+                      The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl
+                      This feature requires the GenerateConfigInInitContainer feature gate to be enabled.
                     type: string
                   secret:
                     description: |-
diff --git a/docs/content/en/docs/CRD Reference/API Reference/_index.md b/docs/content/en/docs/CRD Reference/API Reference/_index.md
@@ -43,7 +43,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `secret` _[SecretVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#secretvolumesource-v1-core)_ | Secret-based ACL configuration.<br />Adapts a Secret into a volume containing ACL rules.<br />The contents of the target Secret's Data field will be presented in a volume<br />as files using the keys in the Data field as the file names.<br />Secret volumes support ownership management and SELinux relabeling. |  |  |
-| `persistentVolumeClaim` _string_ | PersistentVolumeClaim-based ACL configuration<br />Specify the PVC name to mount ACL file from persistent storage<br />The operator will automatically mount /etc/redis/user.acl from the PVC |  |  |
+| `persistentVolumeClaim` _string_ | PersistentVolumeClaim-based ACL configuration<br />Specify the PVC name to mount ACL file from persistent storage<br />The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl<br />This feature requires the GenerateConfigInInitContainer feature gate to be enabled. |  |  |
 
 
 #### AdditionalVolume
diff --git a/example/v1beta2/acl-pvc/cluster.yaml b/example/v1beta2/acl-pvc/cluster.yaml
@@ -10,7 +10,7 @@ spec:
     image: quay.io/opstree/redis:latest
     imagePullPolicy: IfNotPresent
   # ACL configuration from PVC
-  # The operator will mount /etc/redis/user.acl from the PVC
+  # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl
   # Make sure the PVC contains a file named "user.acl" with Redis ACL rules
   acl:
     persistentVolumeClaim: "redis-acl-pvc"
diff --git a/example/v1beta2/acl-pvc/replication.yaml b/example/v1beta2/acl-pvc/replication.yaml
@@ -10,7 +10,7 @@ spec:
     image: quay.io/opstree/redis:latest
     imagePullPolicy: IfNotPresent
   # ACL configuration from PVC
-  # The operator will mount /etc/redis/user.acl from the PVC
+  # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl
   # Make sure the PVC contains a file named "user.acl" with Redis ACL rules
   acl:
     persistentVolumeClaim: "redis-acl-pvc"
diff --git a/example/v1beta2/acl-pvc/standalone.yaml b/example/v1beta2/acl-pvc/standalone.yaml
@@ -9,7 +9,7 @@ spec:
     image: quay.io/opstree/redis:latest
     imagePullPolicy: IfNotPresent
   # ACL configuration from PVC
-  # The operator will mount /etc/redis/user.acl from the PVC
+  # The operator mounts the PVC at /data/redis, so Redis reads /data/redis/user.acl
   # Make sure the PVC contains a file named "user.acl" with Redis ACL rules
   acl:
     persistentVolumeClaim: "redis-acl-pvc"
diff --git a/internal/agent/bootstrap/redis/config.go b/internal/agent/bootstrap/redis/config.go
@@ -41,6 +41,8 @@ func GenerateConfig() error {
 		nodeport           = util.CoalesceEnv1("NODEPORT", "false")
 		tlsMode            = util.CoalesceEnv1("TLS_MODE", "false")
 		clusterMode        = util.CoalesceEnv1("SETUP_MODE", "standalone")
+		aclMode            = util.CoalesceEnv1("ACL_MODE", "")
+		aclFilePath        = util.CoalesceEnv1("ACL_FILE_PATH", "/etc/redis/user.acl")
 	)
 
 	if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" {
@@ -112,8 +114,9 @@ func GenerateConfig() error {
 		fmt.Println("Running without TLS mode")
 	}
 
-	if aclMode := util.CoalesceEnv1("ACL_MODE", ""); aclMode == "true" {
-		cfg.Append("aclfile", "/etc/redis/user.acl")
+	if aclMode == "true" {
+		fmt.Println("ACL_MODE is true, modifying ACL file path to", aclFilePath)
+		cfg.Append("aclfile", aclFilePath)
 	} else {
 		fmt.Println("ACL_MODE is not true, skipping ACL file modification")
 	}
diff --git a/internal/agent/bootstrap/sentinel/config.go b/internal/agent/bootstrap/sentinel/config.go
@@ -88,8 +88,11 @@ func GenerateConfig() error {
 
 	// acl_setup
 	{
-		if aclMode, ok := util.CoalesceEnv("ACL_MODE", ""); ok && aclMode == "true" {
-			cfg.Append("aclfile", "/etc/redis/user.acl")
+		aclMode, _ := util.CoalesceEnv("ACL_MODE", "")
+		aclFilePath, _ := util.CoalesceEnv("ACL_FILE_PATH", "/etc/redis/user.acl")
+		if aclMode == "true" {
+			fmt.Println("ACL_MODE is true, modifying ACL file path to", aclFilePath)
+			cfg.Append("aclfile", aclFilePath)
 		} else {
 			fmt.Println("ACL_MODE is not true, skipping ACL file modification")
 		}
diff --git a/internal/k8sutils/statefulset.go b/internal/k8sutils/statefulset.go
@@ -800,15 +800,18 @@ func getVolumeMount(name string, persistenceEnabled *bool, clusterMode bool, nod
 	}
 
 	if aclConfig != nil {
-		volumeName := "acl-secret"
 		if aclConfig.PersistentVolumeClaim != nil {
-			volumeName = "acl-pvc"
+			VolumeMounts = append(VolumeMounts, corev1.VolumeMount{
+				Name:      "acl-pvc",
+				MountPath: "/data/redis",
+			})
+		} else {
+			VolumeMounts = append(VolumeMounts, corev1.VolumeMount{
+				Name:      "acl-secret",
+				MountPath: "/etc/redis/user.acl",
+				SubPath:   "user.acl",
+			})
 		}
-		VolumeMounts = append(VolumeMounts, corev1.VolumeMount{
-			Name:      volumeName,
-			MountPath: "/etc/redis/user.acl",
-			SubPath:   "user.acl",
-		})
 	}
 
 	if externalConfig != nil {
@@ -917,6 +920,14 @@ func getEnvironmentVariables(role string, enabledPassword *bool, secretName *str
 			Name:  "ACL_MODE",
 			Value: "true",
 		})
+		aclFilePath := "/etc/redis/user.acl"
+		if aclConfig.PersistentVolumeClaim != nil {
+			aclFilePath = "/data/redis/user.acl"
+		}
+		envVars = append(envVars, corev1.EnvVar{
+			Name:  "ACL_FILE_PATH",
+			Value: aclFilePath,
+		})
 	}
 
 	envVars = append(envVars, corev1.EnvVar{
diff --git a/internal/k8sutils/statefulset_test.go b/internal/k8sutils/statefulset_test.go
@@ -191,15 +191,34 @@ func TestGetVolumeMount(t *testing.T) {
 			expectedMounts:     []corev1.VolumeMount{{Name: "tls-certs", MountPath: "/tls", ReadOnly: true}},
 		},
 		{
-			name:               "6. Only acl enabled",
+			name:               "6. Only acl enabled (secret)",
 			persistenceEnabled: nil,
 			clusterMode:        false,
 			nodeConfVolume:     false,
 			externalConfig:     nil,
 			mountpath:          []corev1.VolumeMount{},
 			tlsConfig:          nil,
-			aclConfig:          &common.ACLConfig{},
-			expectedMounts:     []corev1.VolumeMount{{Name: "acl-secret", MountPath: "/etc/redis/user.acl", SubPath: "user.acl"}},
+			aclConfig: &common.ACLConfig{
+				Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"},
+			},
+			expectedMounts: []corev1.VolumeMount{
+				{Name: "acl-secret", MountPath: "/etc/redis/user.acl", SubPath: "user.acl"},
+			},
+		},
+		{
+			name:               "6b. Only acl enabled (PVC)",
+			persistenceEnabled: nil,
+			clusterMode:        false,
+			nodeConfVolume:     false,
+			externalConfig:     nil,
+			mountpath:          []corev1.VolumeMount{},
+			tlsConfig:          nil,
+			aclConfig: &common.ACLConfig{
+				PersistentVolumeClaim: ptr.To("acl-pvc"),
+			},
+			expectedMounts: []corev1.VolumeMount{
+				{Name: "acl-pvc", MountPath: "/data/redis"},
+			},
 		},
 		{
 			name:               "7. Everything enabled except externalConfig",
@@ -214,7 +233,9 @@ func TestGetVolumeMount(t *testing.T) {
 				},
 			},
 			tlsConfig: &common.TLSConfig{},
-			aclConfig: &common.ACLConfig{},
+			aclConfig: &common.ACLConfig{
+				Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"},
+			},
 			expectedMounts: []corev1.VolumeMount{
 				{Name: "persistent-volume", MountPath: "/data"},
 				{Name: "node-conf", MountPath: "/node-conf"},
@@ -242,7 +263,9 @@ func TestGetVolumeMount(t *testing.T) {
 			externalConfig:     nil,
 			mountpath:          []corev1.VolumeMount{},
 			tlsConfig:          nil,
-			aclConfig:          &common.ACLConfig{},
+			aclConfig: &common.ACLConfig{
+				Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"},
+			},
 			expectedMounts: []corev1.VolumeMount{
 				{Name: "persistent-volume", MountPath: "/data"},
 				{Name: "node-conf", MountPath: "/node-conf"},
@@ -1499,6 +1522,7 @@ func TestGetEnvironmentVariables(t *testing.T) {
 			},
 			clusterVersion: ptr.To("v6"),
 			expectedEnvironment: []corev1.EnvVar{
+				{Name: "ACL_FILE_PATH", Value: "/etc/redis/user.acl"},
 				{Name: "ACL_MODE", Value: "true"},
 				{Name: "PERSISTENCE_ENABLED", Value: "true"},
 				{Name: "REDIS_ADDR", Value: "redis://localhost:26379"},
@@ -1562,12 +1586,15 @@ func TestGetEnvironmentVariables(t *testing.T) {
 			secretKey:          ptr.To("test-key"),
 			persistenceEnabled: ptr.To(true),
 			tlsConfig:          nil,
-			aclConfig:          &common.ACLConfig{},
+			aclConfig: &common.ACLConfig{
+				Secret: &corev1.SecretVolumeSource{SecretName: "acl-secret"},
+			},
 			envVar: &[]corev1.EnvVar{
 				{Name: "TEST_ENV", Value: "test-value"},
 			},
 			port: ptr.To(6380),
 			expectedEnvironment: []corev1.EnvVar{
+				{Name: "ACL_FILE_PATH", Value: "/etc/redis/user.acl"},
 				{Name: "ACL_MODE", Value: "true"},
 				{Name: "PERSISTENCE_ENABLED", Value: "true"},
 				{Name: "REDIS_ADDR", Value: "redis://localhost:6379"},
@@ -1585,6 +1612,37 @@ func TestGetEnvironmentVariables(t *testing.T) {
 				{Name: "REDIS_PORT", Value: "6380"},
 			},
 		},
+		{
+			name:               "Test with cluster role and acl pvc",
+			role:               "cluster",
+			enabledPassword:    ptr.To(true),
+			secretName:         ptr.To("test-secret"),
+			secretKey:          ptr.To("test-key"),
+			persistenceEnabled: ptr.To(true),
+			tlsConfig:          nil,
+			aclConfig: &common.ACLConfig{
+				PersistentVolumeClaim: ptr.To("acl-pvc"),
+			},
+			envVar: nil,
+			port:   ptr.To(6381),
+			expectedEnvironment: []corev1.EnvVar{
+				{Name: "ACL_FILE_PATH", Value: "/data/redis/user.acl"},
+				{Name: "ACL_MODE", Value: "true"},
+				{Name: "PERSISTENCE_ENABLED", Value: "true"},
+				{Name: "REDIS_ADDR", Value: "redis://localhost:6379"},
+				{Name: "REDIS_PASSWORD", ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: "test-secret",
+						},
+						Key: "test-key",
+					},
+				}},
+				{Name: "REDIS_PORT", Value: "6381"},
+				{Name: "SERVER_MODE", Value: "cluster"},
+				{Name: "SETUP_MODE", Value: "cluster"},
+			},
+		},
 		{
 			name:               "Test with cluster role and only metrics enabled",
 			role:               "cluster",
@@ -1756,6 +1814,10 @@ func TestGenerateStatefulSetsDef(t *testing.T) {
 									Name:  "test-sts",
 									Image: "redis:latest",
 									Env: []corev1.EnvVar{
+										{
+											Name:  "ACL_FILE_PATH",
+											Value: "/etc/redis/user.acl",
+										},
 										{
 											Name:  "ACL_MODE",
 											Value: "true",
diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-cluster/chainsaw-test.yaml
@@ -96,7 +96,7 @@ spec:
             timeout: 30s
             content: >
               kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 --
-              cat /etc/redis/user.acl 2>&1
+              cat /data/redis/user.acl 2>&1
             check:
               (contains($stdout, 'user pvcuser')): true
               (contains($stdout, 'user readonly')): true
@@ -113,7 +113,32 @@ spec:
               (contains($stdout, 'pvcuser')): true
               (contains($stdout, 'readonly')): true
 
-    # Step 11: Test cluster operations with ACL
+    # Step 11: Test ACL SAVE persists to PVC
+    - name: Verify ACL SAVE writes to PVC
+      try:
+        - script:
+            timeout: 30s
+            content: >
+              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 --
+              redis-cli -c -p 6379 --user admin --pass admin@secure456 ACL SETUSER saveduser on ~* &* +@all >saved@secure789 2>&1
+            check:
+              (contains($stdout, 'OK')): true
+        - script:
+            timeout: 30s
+            content: >
+              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 --
+              redis-cli -c -p 6379 --user admin --pass admin@secure456 ACL SAVE 2>&1
+            check:
+              (contains($stdout, 'OK')): true
+        - script:
+            timeout: 30s
+            content: >
+              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 --
+              cat /data/redis/user.acl 2>&1
+            check:
+              (contains($stdout, 'user saveduser')): true
+
+    # Step 12: Test cluster operations with ACL
     - name: Test cluster info with ACL user
       try:
         - script:
@@ -124,13 +149,13 @@ spec:
             check:
               (contains($stdout, 'cluster_state:ok')): true
 
-    # Step 12: Verify PVC data persistence
+    # Step 13: Verify PVC data persistence
     - name: Verify ACL PVC volume mount
       try:
         - script:
             timeout: 30s
             content: >
               kubectl exec --namespace ${NAMESPACE} --container redis-cluster-acl-pvc-leader redis-cluster-acl-pvc-leader-0 --
-              cat /proc/mounts | grep '/etc/redis/user.acl' 2>&1
+              cat /proc/mounts | grep '/data/redis' 2>&1
             check:
-              (contains($stdout, '/etc/redis/user.acl')): true
+              (contains($stdout, '/data/redis')): true
diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-replication/chainsaw-test.yaml
@@ -76,7 +76,7 @@ spec:
             timeout: 30s
             content: >
               kubectl exec --namespace ${NAMESPACE} redis-replication-acl-pvc-0 --
-              cat /etc/redis/user.acl 2>&1
+              cat /data/redis/user.acl 2>&1
             check:
               (contains($stdout, 'user repluser')): true
 
@@ -87,7 +87,7 @@ spec:
             timeout: 30s
             content: >
               kubectl exec --namespace ${NAMESPACE} redis-replication-acl-pvc-1 --
-              cat /etc/redis/user.acl 2>&1
+              cat /data/redis/user.acl 2>&1
             check:
               (contains($stdout, 'user repluser')): true
 
diff --git a/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-pvc/redis-standalone/chainsaw-test.yaml

Original file line number	Diff line number	Diff line change
`@@ -288,7 +288,8 @@ type ACLConfig struct {`
`288`	`288`	Secret *corev1.SecretVolumeSource `json:"secret,omitempty"`
`289`	`289`	`// PersistentVolumeClaim-based ACL configuration`
`290`	`290`	`// Specify the PVC name to mount ACL file from persistent storage`
`291`		`- // The operator will automatically mount /etc/redis/user.acl from the PVC`
	`291`	`+ // The operator mounts the PVC at /data/redis so Redis can read and update /data/redis/user.acl`
	`292`	`+ // This feature requires the GenerateConfigInInitContainer feature gate to be enabled.`
`292`	`293`	PersistentVolumeClaim *string `json:"persistentVolumeClaim,omitempty"`
`293`	`294`	`}`
`294`	`295`
Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,8 @@ func GenerateConfig() error {`
`41`	`41`	`nodeport = util.CoalesceEnv1("NODEPORT", "false")`
`42`	`42`	`tlsMode = util.CoalesceEnv1("TLS_MODE", "false")`
`43`	`43`	`clusterMode = util.CoalesceEnv1("SETUP_MODE", "standalone")`
	`44`	`+ aclMode = util.CoalesceEnv1("ACL_MODE", "")`
	`45`	`+ aclFilePath = util.CoalesceEnv1("ACL_FILE_PATH", "/etc/redis/user.acl")`
`44`	`46`	`)`
`45`	`47`
`46`	`48`	`if val, ok := util.CoalesceEnv("REDIS_PASSWORD", ""); ok && val != "" {`
`@@ -112,8 +114,9 @@ func GenerateConfig() error {`
`112`	`114`	`fmt.Println("Running without TLS mode")`
`113`	`115`	`}`
`114`	`116`
`115`		`- if aclMode := util.CoalesceEnv1("ACL_MODE", ""); aclMode == "true" {`
`116`		`- cfg.Append("aclfile", "/etc/redis/user.acl")`
	`117`	`+ if aclMode == "true" {`
	`118`	`+ fmt.Println("ACL_MODE is true, modifying ACL file path to", aclFilePath)`
	`119`	`+ cfg.Append("aclfile", aclFilePath)`
`117`	`120`	`} else {`
`118`	`121`	`fmt.Println("ACL_MODE is not true, skipping ACL file modification")`
`119`	`122`	`}`