Skip to content

fix: persist local kms keys and fence pod deletes #172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
GatewayJ merged 2 commits into from
Jul 10, 2026
Merged

fix: persist local kms keys and fence pod deletes #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de9af2f
fix(operator): persist local kms keys and fence pod deletes
GatewayJ Jul 10, 2026
cd4bbf5
fix: grant pod cleanup owner lookup RBAC
GatewayJ Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 79 additions & 6 deletions console-web/app/(dashboard)/tenants/[namespace]/[name]/tenant-detail-client.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,9 @@ export function TenantDetailClient({ namespace, name, initialTab, initialYamlEdi
})
const [encLocal, setEncLocal] = useState({
keyDirectory: "",
masterKeySecretName: "",
masterKeySecretKey: "local-master-key",
allowInsecureDevDefaults: false,
})
const [encDefaultKeyId, setEncDefaultKeyId] = useState("")
const [encKmsSecretName, setEncKmsSecretName] = useState("")
Expand Down Expand Up @@ -464,10 +467,20 @@ export function TenantDetailClient({ namespace, name, initialTab, initialYamlEdi
if (data.local) {
setEncLocal({
keyDirectory: data.local.keyDirectory || "",
masterKeySecretName: data.local.masterKeySecretRef?.name || "",
masterKeySecretKey: data.local.masterKeySecretRef?.key || "local-master-key",
allowInsecureDevDefaults: data.local.allowInsecureDevDefaults || false,
})
} else {
setEncLocal({
keyDirectory: "",
masterKeySecretName: "",
masterKeySecretKey: "local-master-key",
allowInsecureDevDefaults: false,
})
}
setEncDefaultKeyId(data.defaultKeyId || "")
setEncKmsSecretName(data.kmsSecretName || "")
setEncKmsSecretName(data.backend === "vault" ? data.kmsSecretName || "" : "")
} catch (e) {
const err = e as ApiError
toast.error(err.message || t("Failed to load encryption config"))
Expand All @@ -483,21 +496,40 @@ export function TenantDetailClient({ namespace, name, initialTab, initialYamlEdi
toast.warning(t("Vault endpoint is required"))
return
}
if (
encEnabled &&
encBackend === "local" &&
!encLocal.allowInsecureDevDefaults &&
(!encLocal.masterKeySecretName.trim() || !encLocal.masterKeySecretKey.trim())
) {
toast.warning(t("Local KMS master key Secret name and key are required"))
return
}
setEncSaving(true)
try {
const body: UpdateEncryptionRequest = {
enabled: encEnabled,
backend: encBackend,
kmsSecretName: encKmsSecretName || undefined,
kmsSecretName: encBackend === "vault" ? encKmsSecretName.trim() || undefined : undefined,
defaultKeyId: encDefaultKeyId.trim() || undefined,
}
if (encBackend === "vault") {
body.vault = {
endpoint: encVault.endpoint,
endpoint: encVault.endpoint.trim(),
}
} else {
body.local = {
keyDirectory: encLocal.keyDirectory || undefined,
keyDirectory: encLocal.keyDirectory.trim() || undefined,
masterKeySecretRef:
!encLocal.allowInsecureDevDefaults &&
encLocal.masterKeySecretName.trim() &&
encLocal.masterKeySecretKey.trim()
? {
name: encLocal.masterKeySecretName.trim(),
key: encLocal.masterKeySecretKey.trim(),
}
: undefined,
allowInsecureDevDefaults: encLocal.allowInsecureDevDefaults,
}
}
const res = await api.updateEncryption(namespace, name, body)
Expand Down Expand Up @@ -1117,12 +1149,52 @@ export function TenantDetailClient({ namespace, name, initialTab, initialYamlEdi
<div className="space-y-2">
<Label>{t("Key Directory")}</Label>
<Input
placeholder="/data/kms-keys"
placeholder="{persistence.path}/rustfs0/.kms-keys"
value={encLocal.keyDirectory}
onChange={(e) => setEncLocal((l) => ({ ...l, keyDirectory: e.target.value }))}
/>
</div>
<div className="space-y-2">
<Label>{t("Master Key Secret Name")}</Label>
<Input
placeholder="local-kms-master-key"
value={encLocal.masterKeySecretName}
onChange={(e) => setEncLocal((l) => ({ ...l, masterKeySecretName: e.target.value }))}
disabled={encLocal.allowInsecureDevDefaults}
/>
</div>
<div className="space-y-2">
<Label>{t("Master Key Secret Key")}</Label>
<Input
placeholder="local-master-key"
value={encLocal.masterKeySecretKey}
onChange={(e) => setEncLocal((l) => ({ ...l, masterKeySecretKey: e.target.value }))}
disabled={encLocal.allowInsecureDevDefaults}
/>
</div>
<div className="flex items-end gap-3 pb-2">
<label htmlFor="local-kms-insecure-dev" className="text-sm">
{t("Allow insecure development defaults")}
</label>
<input
id="local-kms-insecure-dev"
type="checkbox"
checked={encLocal.allowInsecureDevDefaults}
onChange={(e) =>
setEncLocal((l) => ({
...l,
allowInsecureDevDefaults: e.target.checked,
}))
}
className="h-4 w-4 rounded border-border"
/>
</div>
</div>
<p className="text-xs text-muted-foreground">
{t(
"Local KMS requires a master key Secret unless insecure development defaults are explicitly enabled.",
)}
</p>
</div>
)}

Expand All @@ -1142,11 +1214,12 @@ export function TenantDetailClient({ namespace, name, initialTab, initialYamlEdi
placeholder={`${t("Optional")} – ${t("Secret containing vault-token")}`}
value={encKmsSecretName}
onChange={(e) => setEncKmsSecretName(e.target.value)}
disabled={encBackend !== "vault"}
/>
<p className="text-xs text-muted-foreground">
{encBackend === "vault"
? t("Required for Vault: Secret must contain key 'vault-token'.")
: t("Not required for Local backend.")}
: t("Only used for Vault. Local KMS uses masterKeySecretRef.")}
</p>
</div>
</div>
Expand Down
7 changes: 6 additions & 1 deletion console-web/i18n/locales/en-US.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,12 @@
"KMS Secret Name": "KMS Secret Name",
"Secret containing vault-token": "Secret containing vault-token",
"Required for Vault: Secret must contain key 'vault-token'.": "Required for Vault: Secret must contain key 'vault-token'.",
"Not required for Local backend.": "Not required for Local backend.",
"Only used for Vault. Local KMS uses masterKeySecretRef.": "Only used for Vault. Local KMS uses masterKeySecretRef.",
"Master Key Secret Name": "Master Key Secret Name",
"Master Key Secret Key": "Master Key Secret Key",
"Allow insecure development defaults": "Allow insecure development defaults",
"Local KMS requires a master key Secret unless insecure development defaults are explicitly enabled.": "Local KMS requires a master key Secret unless insecure development defaults are explicitly enabled.",
"Local KMS master key Secret name and key are required": "Local KMS master key Secret name and key are required",
"Failed to load encryption config": "Failed to load encryption config",
"Encryption config updated": "Encryption config updated",
"Loading encryption config...": "Loading encryption config...",
Expand Down
7 changes: 6 additions & 1 deletion console-web/i18n/locales/zh-CN.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,12 @@
"KMS Secret Name": "KMS Secret 名称",
"Secret containing vault-token and TLS certs": "包含 vault-token 和 TLS 证书的 Secret",
"Secret must contain key 'vault-token'. Optional: 'vault-ca-cert', 'vault-client-cert', 'vault-client-key'.": "Secret 必须包含 'vault-token' 键。可选：'vault-ca-cert'、'vault-client-cert'、'vault-client-key'。",
"Not required for Local backend.": "本地后端不需要 Secret。",
"Only used for Vault. Local KMS uses masterKeySecretRef.": "仅用于 Vault。Local KMS 使用 masterKeySecretRef。",
"Master Key Secret Name": "Master Key Secret 名称",
"Master Key Secret Key": "Master Key Secret 键",
"Allow insecure development defaults": "允许开发环境不安全默认值",
"Local KMS requires a master key Secret unless insecure development defaults are explicitly enabled.": "Local KMS 需要 master key Secret，除非显式启用开发环境不安全默认值。",
"Local KMS master key Secret name and key are required": "Local KMS master key Secret 名称和键不能为空",
"Auth Type": "认证方式",
"Not yet implemented in backend": "后端尚未实现",
"Retry (Seconds)": "重试间隔（秒）",
Expand Down
10 changes: 10 additions & 0 deletions console-web/types/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,11 @@ export interface VaultInfo {

export interface LocalKmsInfo {
keyDirectory: string | null
masterKeySecretRef: {
name: string
key: string
} | null
allowInsecureDevDefaults: boolean
}

export interface SecurityContextInfo {
Expand Down Expand Up @@ -423,6 +428,11 @@ export interface UpdateEncryptionRequest {
}
local?: {
keyDirectory?: string
masterKeySecretRef?: {
name: string
key: string
}
allowInsecureDevDefaults?: boolean
}
kmsSecretName?: string
defaultKeyId?: string
Expand Down
2 changes: 1 addition & 1 deletion deploy/k8s-dev/operator-rbac.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ rules:
resources: ["tokenreviews"]
verbs: ["create"]
- apiGroups: ["apps"]
resources: ["statefulsets"]
resources: ["statefulsets", "replicasets", "deployments"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: ["cert-manager.io"]
resources: ["certificates"]
Expand Down
51 changes: 48 additions & 3 deletions deploy/rustfs-operator/crds/tenant-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,44 @@ spec:
description: 'Local: optional key directory override.'
nullable: true
properties:
allowInsecureDevDefaults:
default: false
description: |-
Explicitly allow RustFS development-only insecure KMS defaults.

When true, the operator sets `RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`, allowing
Local KMS to start without a master key and store key material as plaintext-dev-only.
Do not use this in production.
type: boolean
keyDirectory:
description: 'Absolute directory for KMS key files (default: `/data/kms-keys`).'
description: |-
Absolute directory for KMS key files.

Must be in a subdirectory under a RustFS data PVC mount. Defaults to the first data PVC mount
under `persistence.path`, for example `/data/rustfs0/.kms-keys`.
nullable: true
type: string
masterKeySecretRef:
description: |-
Secret key selector for `RUSTFS_KMS_LOCAL_MASTER_KEY`.

Required for Local KMS unless `allowInsecureDevDefaults` is explicitly set to `true`.
The referenced Secret key should contain the local master key string used to encrypt
Local KMS key files at rest.
nullable: true
properties:
key:
description: Secret data key containing the local master key string.
minLength: 1
type: string
name:
description: Secret name in the Tenant namespace.
minLength: 1
type: string
required:
- key
- name
type: object
type: object
vault:
description: 'Vault: HTTP(S) endpoint (required when `backend: vault`).'
Expand All @@ -127,7 +161,14 @@ spec:
- endpoint
type: object
type: object
x-kubernetes-validations:
- message: Local KMS requires local.masterKeySecretRef unless local.allowInsecureDevDefaults is true
rule: '!self.enabled || self.backend != ''local'' || (has(self.local) && (has(self.local.masterKeySecretRef) || (has(self.local.allowInsecureDevDefaults) && self.local.allowInsecureDevDefaults == true)))'
env:
description: |-
Additional RustFS container environment variables.

`RUSTFS_KMS_*` is reserved; configure KMS through `spec.encryption`.
items:
description: EnvVar represents an environment variable present in a Container.
properties:
Expand Down Expand Up @@ -426,8 +467,10 @@ spec:
Typical use-case: a StatefulSet Pod gets stuck in Terminating when the node goes down.
Setting this to `ForceDelete` allows the operator to force delete the Pod object so the
StatefulSet controller can recreate it elsewhere.
Force deletion requires the Node object to be deleted or marked with an effective
`node.kubernetes.io/out-of-service` taint that the target Pod does not tolerate.

Values: DoNothing | Delete | ForceDelete
Values: DoNothing | Delete | ForceDelete | DeleteStatefulSetPod | DeleteDeploymentPod | DeleteBothStatefulSetAndDeploymentPod
enum:
- DoNothing
- Delete
Expand Down Expand Up @@ -1725,7 +1768,9 @@ spec:
description: Status of the condition (True, False, Unknown)
type: string
type:
description: Type of condition (Ready, Progressing, Degraded)
description: |-
Type of condition (Ready, Reconciling, Degraded, SpecValid, CredentialsReady, KmsReady,
TlsReady, PoolsReady, WorkloadsReady, ProvisioningReady)
type: string
required:
- message
Expand Down
51 changes: 48 additions & 3 deletions deploy/rustfs-operator/crds/tenant.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,44 @@ spec:
description: 'Local: optional key directory override.'
nullable: true
properties:
allowInsecureDevDefaults:
default: false
description: |-
Explicitly allow RustFS development-only insecure KMS defaults.

When true, the operator sets `RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS=true`, allowing
Local KMS to start without a master key and store key material as plaintext-dev-only.
Do not use this in production.
type: boolean
keyDirectory:
description: 'Absolute directory for KMS key files (default: `/data/kms-keys`).'
description: |-
Absolute directory for KMS key files.

Must be in a subdirectory under a RustFS data PVC mount. Defaults to the first data PVC mount
under `persistence.path`, for example `/data/rustfs0/.kms-keys`.
nullable: true
type: string
masterKeySecretRef:
description: |-
Secret key selector for `RUSTFS_KMS_LOCAL_MASTER_KEY`.

Required for Local KMS unless `allowInsecureDevDefaults` is explicitly set to `true`.
The referenced Secret key should contain the local master key string used to encrypt
Local KMS key files at rest.
nullable: true
properties:
key:
description: Secret data key containing the local master key string.
minLength: 1
type: string
name:
description: Secret name in the Tenant namespace.
minLength: 1
type: string
required:
- key
- name
type: object
type: object
vault:
description: 'Vault: HTTP(S) endpoint (required when `backend: vault`).'
Expand All @@ -127,7 +161,14 @@ spec:
- endpoint
type: object
type: object
x-kubernetes-validations:
- message: Local KMS requires local.masterKeySecretRef unless local.allowInsecureDevDefaults is true
rule: '!self.enabled || self.backend != ''local'' || (has(self.local) && (has(self.local.masterKeySecretRef) || (has(self.local.allowInsecureDevDefaults) && self.local.allowInsecureDevDefaults == true)))'
env:
description: |-
Additional RustFS container environment variables.

`RUSTFS_KMS_*` is reserved; configure KMS through `spec.encryption`.
items:
description: EnvVar represents an environment variable present in a Container.
properties:
Expand Down Expand Up @@ -426,8 +467,10 @@ spec:
Typical use-case: a StatefulSet Pod gets stuck in Terminating when the node goes down.
Setting this to `ForceDelete` allows the operator to force delete the Pod object so the
StatefulSet controller can recreate it elsewhere.
Force deletion requires the Node object to be deleted or marked with an effective
`node.kubernetes.io/out-of-service` taint that the target Pod does not tolerate.

Values: DoNothing | Delete | ForceDelete
Values: DoNothing | Delete | ForceDelete | DeleteStatefulSetPod | DeleteDeploymentPod | DeleteBothStatefulSetAndDeploymentPod
enum:
- DoNothing
- Delete
Expand Down Expand Up @@ -1725,7 +1768,9 @@ spec:
description: Status of the condition (True, False, Unknown)
type: string
type:
description: Type of condition (Ready, Progressing, Degraded)
description: |-
Type of condition (Ready, Reconciling, Degraded, SpecValid, CredentialsReady, KmsReady,
TlsReady, PoolsReady, WorkloadsReady, ProvisioningReady)
type: string
required:
- message
Expand Down
4 changes: 2 additions & 2 deletions deploy/rustfs-operator/templates/clusterrole.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ rules:
resources: ["tokenreviews"]
verbs: ["create"]

# StatefulSets for tenant pools
# StatefulSets for tenant pools and owner lookups for pod cleanup
- apiGroups: ["apps"]
resources: ["statefulsets"]
resources: ["statefulsets", "replicasets", "deployments"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

# cert-manager Certificate orchestration and readiness watches
Expand Down
Loading
Loading