Secrets manager crashes when trying to load non-String values from Vault #94
Description
When SecretDefinition object in Kubernetes cluster refers to key in Vault with non-String type (tested Boolean and Integer), it crashes with error:
{"level":"info","ts":1716805335.1330655,"logger":"backend.vault","msg":"successfully logged into vault cluster","vault_url":"https://<CENSORED>:8200","vault_engine":"kv2","vault_cluster_name":"<CENSORED>","vault_cluster_id":"<CENSORED>","vault_version":"1.16.1","vault_sealed":"false","vault_server_time_utc":1716805335}
{"level":"info","ts":1716805335.915437,"logger":"controller-runtime.metrics","msg":"metrics server is starting to listen","addr":":8080"}
{"level":"info","ts":1716805335.9160109,"logger":"setup","msg":"starting manager"}
{"level":"info","ts":1716805335.9166248,"logger":"controller-runtime.manager","msg":"starting metrics server","path":"/metrics"}
{"level":"info","ts":1716805335.9175177,"logger":"controller-runtime.manager.controller.SecretDefinition","msg":"Starting EventSource","reconciler group":"secrets-manager.tuenti.io","reconciler kind":"SecretDefinition","source":"kind source: /, Kind="}
{"level":"info","ts":1716805336.019119,"logger":"controller-runtime.manager.controller.SecretDefinition","msg":"Starting Controller","reconciler group":"secrets-manager.tuenti.io","reconciler kind":"SecretDefinition"}
{"level":"info","ts":1716805336.0191934,"logger":"controller-runtime.manager.controller.SecretDefinition","msg":"Starting workers","reconciler group":"secrets-manager.tuenti.io","reconciler kind":"SecretDefinition","worker count":1}
E0527 10:22:34.885121 1 runtime.go:78] Observed a panic: &runtime.TypeAssertionError{_interface:(*runtime._type)(0x174eb40), concrete:(*runtime._type)(0x179e5e0), asserted:(*runtime._type)(0x17159a0), missingMethod:""} (interface conversion: interface {} is json.Number, not string)
goroutine 447 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic(0x178d620, 0xc0000e76b0)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:74 +0x95
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:48 +0x86
panic(0x178d620, 0xc0000e76b0)
/usr/local/go/src/runtime/panic.go:965 +0x1b9
github.com/tuenti/secrets-manager/backend.(*client).ReadSecret(0xc0001b00b0, 0xc000a8be40, 0x3c, 0xc00049eb20, 0x7, 0x20, 0x0, 0x0, 0x0)
/workspace/backend/vault.go:262 +0x697
github.com/tuenti/secrets-manager/controllers.(*SecretDefinitionReconciler).getDesiredState(0xc000aae840, 0xc000d89410, 0xc000d89350, 0xc0007b3a20, 0x199c5cd)
/workspace/controllers/secretdefinition_controller.go:126 +0x158
github.com/tuenti/secrets-manager/controllers.(*SecretDefinitionReconciler).Reconcile(0xc000aae840, 0x1ba7448, 0xc000d89350, 0xc0007a4ff0, 0x12, 0xc00049ea10, 0x10, 0xc000d89350, 0xc000030000, 0x181b140, ...)
/workspace/controllers/secretdefinition_controller.go:264 +0x6eb
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc000c84780, 0x1ba73a0, 0xc000d86000, 0x17d8ea0, 0xc00089a760)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x30d
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc000c84780, 0x1ba73a0, 0xc000d86000, 0xc000ba0600)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.2(0x1ba73a0, 0xc000d86000)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x4a
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x37
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000ba0750)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x5f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00001df50, 0x1b74500, 0xc000a5e9c0, 0xc000d86001, 0xc000b90660)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0x9b
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000ba0750, 0x3b9aca00, 0x0, 0x3b9aca01, 0xc000b90660)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x98
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x1ba73a0, 0xc000d86000, 0xc000231090, 0x3b9aca00, 0x0, 0x10000c0005b6401)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0xa6
k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x1ba73a0, 0xc000d86000, 0xc000231090, 0x3b9aca00)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x57
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x40d
panic: interface conversion: interface {} is json.Number, not string [recovered]
panic: interface conversion: interface {} is json.Number, not string
goroutine 447 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:55 +0x109
panic(0x178d620, 0xc0000e76b0)
/usr/local/go/src/runtime/panic.go:965 +0x1b9
github.com/tuenti/secrets-manager/backend.(*client).ReadSecret(0xc0001b00b0, 0xc000a8be40, 0x3c, 0xc00049eb20, 0x7, 0x20, 0x0, 0x0, 0x0)
/workspace/backend/vault.go:262 +0x697
github.com/tuenti/secrets-manager/controllers.(*SecretDefinitionReconciler).getDesiredState(0xc000aae840, 0xc000d89410, 0xc000d89350, 0xc0007b3a20, 0x199c5cd)
/workspace/controllers/secretdefinition_controller.go:126 +0x158
github.com/tuenti/secrets-manager/controllers.(*SecretDefinitionReconciler).Reconcile(0xc000aae840, 0x1ba7448, 0xc000d89350, 0xc0007a4ff0, 0x12, 0xc00049ea10, 0x10, 0xc000d89350, 0xc000030000, 0x181b140, ...)
/workspace/controllers/secretdefinition_controller.go:264 +0x6eb
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc000c84780, 0x1ba73a0, 0xc000d86000, 0x17d8ea0, 0xc00089a760)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x30d
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc000c84780, 0x1ba73a0, 0xc000d86000, 0xc000ba0600)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.2(0x1ba73a0, 0xc000d86000)
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x4a
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1()
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x37
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000ba0750)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x5f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00001df50, 0x1b74500, 0xc000a5e9c0, 0xc000d86001, 0xc000b90660)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0x9b
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000ba0750, 0x3b9aca00, 0x0, 0x3b9aca01, 0xc000b90660)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x98
k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x1ba73a0, 0xc000d86000, 0xc000231090, 0x3b9aca00, 0x0, 0x10000c0005b6401)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0xa6
k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x1ba73a0, 0xc000d86000, 0xc000231090, 0x3b9aca00)
/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x57
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1
/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x40d
Steps to reproduce:
- Create secret in Vault with Integer value
echo '{"foo": 1}' | vault kv put secrets/testing/secrets-manager-crash
- Create simple SecretDefinition
apiVersion: secrets-manager.tuenti.io/v1alpha1
kind: SecretDefinition
metadata:
name: crashtest
spec:
keysMap:
foo:
key: foo
path: secrets/data/testing/secrets-manager-crash
name: crashtest
type: Opaque
status: {}
- observe Secret manager in crash loop