From eef41f0f8a0b52b8e0a3837b17c7f4afaf25f622 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Wed, 16 Jul 2025 14:50:30 +0200 Subject: [PATCH 01/10] feat(config): add UseConfigAsSecret --- config/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.go b/config/config.go index 4cbdaa67e9..33c993330c 100644 --- a/config/config.go +++ b/config/config.go @@ -2096,6 +2096,9 @@ type ControlPlaneAdvanced struct { // GlobalMetadata is metadata that will be added to all resources deployed by Helm. GlobalMetadata ControlPlaneGlobalMetadata `json:"globalMetadata,omitempty"` + + // UseConfigAsSecret defines if a Secret should be used instead of a ConfigMap for storing the vCluster configuration. + UseConfigAsSecret bool `json:"useConfigAsSecret,omitempty"` } type Registry struct { From 45332595e2c4b8d4921127b5ae7493e57045b532 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Wed, 4 Jun 2025 11:17:45 +0200 Subject: [PATCH 02/10] fix(chart): add support for using ConfigMap instead of Secret for vCluster configuration --- chart/templates/config-configmap.yaml | 18 ++++++++++++ chart/templates/config-secret.yaml | 2 ++ chart/templates/statefulset.yaml | 5 ++++ chart/tests/config-configmap_test.yaml | 38 ++++++++++++++++++++++++++ chart/values.schema.json | 4 +++ chart/values.yaml | 2 ++ 6 files changed, 69 insertions(+) create mode 100644 chart/templates/config-configmap.yaml create mode 100644 chart/tests/config-configmap_test.yaml diff --git a/chart/templates/config-configmap.yaml b/chart/templates/config-configmap.yaml new file mode 100644 index 0000000000..122f090a06 --- /dev/null +++ b/chart/templates/config-configmap.yaml @@ -0,0 +1,18 @@ +{{- if .Values.controlPlane.advanced.useConfigMap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: "vc-config-{{ .Release.Name }}" + namespace: {{ .Release.Namespace }} + labels: + app: vcluster + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + {{- if .Values.controlPlane.advanced.globalMetadata.annotations }} + annotations: +{{ toYaml .Values.controlPlane.advanced.globalMetadata.annotations | indent 4 }} + {{- end }} +data: + config.yaml: {{ .Values | toYaml | quote }} +{{- end }} diff --git a/chart/templates/config-secret.yaml b/chart/templates/config-secret.yaml index eb8170e5ed..4cfb065264 100644 --- a/chart/templates/config-secret.yaml +++ b/chart/templates/config-secret.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.controlPlane.advanced.useConfigMap }} apiVersion: v1 kind: Secret metadata: @@ -15,3 +16,4 @@ metadata: type: Opaque data: config.yaml: {{ .Values | toYaml | b64enc | quote }} +{{- end }} diff --git a/chart/templates/statefulset.yaml b/chart/templates/statefulset.yaml index daa55dd3c0..72f03c9336 100644 --- a/chart/templates/statefulset.yaml +++ b/chart/templates/statefulset.yaml @@ -110,8 +110,13 @@ spec: emptyDir: {} {{- end }} - name: vcluster-config + {{- if .Values.controlPlane.advanced.useConfigMap }} + configMap: + name: vc-config-{{ .Release.Name }} + {{- else }} secret: secretName: vc-config-{{ .Release.Name }} + {{- end }} {{- if .Values.controlPlane.statefulSet.persistence.dataVolume }} {{ toYaml .Values.controlPlane.statefulSet.persistence.dataVolume | indent 8 }} {{- else if not (include "vcluster.persistence.volumeClaim.enabled" .) }} diff --git a/chart/tests/config-configmap_test.yaml b/chart/tests/config-configmap_test.yaml new file mode 100644 index 0000000000..24759d0a26 --- /dev/null +++ b/chart/tests/config-configmap_test.yaml @@ -0,0 +1,38 @@ +suite: test config-configmap +templates: + - config-configmap.yaml + - config-secret.yaml + - statefulset.yaml +release: + name: release-name + namespace: test-namespace +tests: + - it: should create a ConfigMap and not a Secret when useConfigMap is true + set: + controlPlane.advanced.useConfigMap: true + asserts: + - hasDocuments: + count: 1 + template: config-configmap.yaml + - hasDocuments: + count: 0 + template: config-secret.yaml + - matchRegex: + path: spec.template.spec.volumes[?(@.name=="vcluster-config")].configMap.name + pattern: vc-config-release-name + template: statefulset.yaml + + - it: should create a Secret and not a ConfigMap when useConfigMap is false + set: + controlPlane.advanced.useConfigMap: false + asserts: + - hasDocuments: + count: 0 + template: config-configmap.yaml + - hasDocuments: + count: 1 + template: config-secret.yaml + - matchRegex: + path: spec.template.spec.volumes[?(@.name=="vcluster-config")].secret.secretName + pattern: vc-config-release-name + template: statefulset.yaml diff --git a/chart/values.schema.json b/chart/values.schema.json index 7e3f2aa984..c359449b72 100755 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -404,6 +404,10 @@ "type": "string", "description": "DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to\nupload all required vCluster images to a single private repository and set this value. Workload images are not affected by this." }, + "useConfigMap": { + "type": "boolean", + "description": "UseConfigMap defines if a ConfigMap should be used instead of a Secret for storing the vCluster configuration. This is useful in environments where Secret syncing is restricted, like in certain ArgoCD setups." + }, "virtualScheduler": { "$ref": "#/$defs/EnableSwitch", "description": "VirtualScheduler defines if a scheduler should be used within the virtual cluster or the scheduling decision for workloads will be made by the host cluster.\nDeprecated: Use ControlPlane.Distro.K8S.Scheduler instead." diff --git a/chart/values.yaml b/chart/values.yaml index c62f1d73b4..cd4a3358cd 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -713,6 +713,8 @@ controlPlane: # DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to # upload all required vCluster images to a single private repository and set this value. Workload images are not affected by this. defaultImageRegistry: "" + # UseConfigMap defines if a ConfigMap should be used instead of a Secret for storing the vCluster configuration. + useConfigMap: false # VirtualScheduler defines if a scheduler should be used within the virtual cluster or the scheduling decision for workloads will be made by the host cluster. # Deprecated: Use ControlPlane.Distro.K8S.Scheduler instead. virtualScheduler: From 56f262c1f6442940fb6f5dbcefb20821b1be2341 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:02:47 +0200 Subject: [PATCH 03/10] refactor(charts): useConfigAsSecret --- chart/templates/config-configmap.yaml | 2 +- chart/templates/config-secret.yaml | 2 +- chart/values.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chart/templates/config-configmap.yaml b/chart/templates/config-configmap.yaml index 122f090a06..e846036176 100644 --- a/chart/templates/config-configmap.yaml +++ b/chart/templates/config-configmap.yaml @@ -1,4 +1,4 @@ -{{- if .Values.controlPlane.advanced.useConfigMap }} +{{- if not .Values.controlPlane.advanced.useConfigAsSecret }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/chart/templates/config-secret.yaml b/chart/templates/config-secret.yaml index 4cfb065264..e745c6f080 100644 --- a/chart/templates/config-secret.yaml +++ b/chart/templates/config-secret.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.controlPlane.advanced.useConfigMap }} +{{- if .Values.controlPlane.advanced.useConfigAsSecret }} apiVersion: v1 kind: Secret metadata: diff --git a/chart/values.yaml b/chart/values.yaml index cd4a3358cd..c23fd16042 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -713,8 +713,8 @@ controlPlane: # DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to # upload all required vCluster images to a single private repository and set this value. Workload images are not affected by this. defaultImageRegistry: "" - # UseConfigMap defines if a ConfigMap should be used instead of a Secret for storing the vCluster configuration. - useConfigMap: false + # UseConfigAsSecret defines if a Secret should be used instead of a ConfigMap for storing the vCluster configuration. + useConfigAsSecret: false # VirtualScheduler defines if a scheduler should be used within the virtual cluster or the scheduling decision for workloads will be made by the host cluster. # Deprecated: Use ControlPlane.Distro.K8S.Scheduler instead. virtualScheduler: From 59110a140c40dd6da5f5dfe86e45b3b682584215 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:52:57 +0200 Subject: [PATCH 04/10] refactor(config): vCluster configuration handling for Secret and ConfigMap to confighelper --- pkg/setup/config.go | 186 +++++++++++++++++--------- pkg/util/confighelper/confighelper.go | 93 +++++++++++++ 2 files changed, 216 insertions(+), 63 deletions(-) create mode 100644 pkg/util/confighelper/confighelper.go diff --git a/pkg/setup/config.go b/pkg/setup/config.go index e558fd2c53..4f62a48afc 100644 --- a/pkg/setup/config.go +++ b/pkg/setup/config.go @@ -3,26 +3,26 @@ package setup import ( "context" "fmt" + "k8s.io/client-go/tools/clientcmd" "os" "k8s.io/client-go/util/retry" + "github.com/ghodss/yaml" vclusterconfig "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/config" "github.com/loft-sh/vcluster/pkg/k3s" "github.com/loft-sh/vcluster/pkg/pro" + "github.com/loft-sh/vcluster/pkg/util/confighelper" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) -const ( - AnnotationDistro = "vcluster.loft.sh/distro" - AnnotationStore = "vcluster.loft.sh/store" -) - func InitClients(vConfig *config.VirtualClusterConfig) error { var err error @@ -96,62 +96,44 @@ func InitAndValidateConfig(ctx context.Context, vConfig *config.VirtualClusterCo return nil } -// EnsureBackingStoreChanges ensures that only a certain set of allowed changes to the backing store and distro occur. -func EnsureBackingStoreChanges(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) error { - if ok, err := CheckUsingSecretAnnotation(ctx, client, name, namespace, distro, backingStoreType); err != nil { - return fmt.Errorf("using secret annotations: %w", err) - } else if ok { - if err := updateSecretAnnotations(ctx, client, name, namespace, distro, backingStoreType); err != nil { - return fmt.Errorf("update secret annotations: %w", err) - } - - return nil +// GetVClusterConfig retrieves and parses the vCluster configuration from either Secret or ConfigMap. +func GetVClusterConfig(ctx context.Context, kConf clientcmd.ClientConfig, name, namespace string) (*vclusterconfig.Config, error) { + clientConfig, err := kConf.ClientConfig() + if err != nil { + return nil, err } - - if ok, err := CheckUsingHeuristic(distro); err != nil { - return fmt.Errorf("using heuristic: %w", err) - } else if ok { - if err := updateSecretAnnotations(ctx, client, name, namespace, distro, backingStoreType); err != nil { - return fmt.Errorf("update secret annotations: %w", err) - } - - return nil + clientset, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, err } - return nil -} - -// CheckUsingHeuristic checks for known file path indicating the existence of a previous distro. -// -// It checks for the existence of the default K3s token path. -func CheckUsingHeuristic(distro string) (bool, error) { - // check if previously we were using k3s as a default and now have switched to a different distro - if distro != vclusterconfig.K3SDistro && distro != vclusterconfig.K8SDistro { - _, err := os.Stat(k3s.TokenPath) - if err == nil { - return false, fmt.Errorf("seems like you were using k3s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", distro) - } + configBytes, err := confighelper.GetVClusterConfigResource(ctx, clientset, name, namespace) + if err != nil { + return nil, err } - return true, nil + return unmarshalConfig(configBytes) } -// CheckUsingSecretAnnotation checks for backend store and distro changes using annotations on the vCluster's secret annotations. -// Returns true, if both annotations are set and the check was successful, otherwise false. -func CheckUsingSecretAnnotation(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) (bool, error) { - secret, err := client.CoreV1().Secrets(namespace).Get(ctx, "vc-config-"+name, metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("get secret: %w", err) +// unmarshalConfig parses YAML config bytes into a Config object +func unmarshalConfig(configBytes []byte) (*vclusterconfig.Config, error) { + vclusterConfig := &vclusterconfig.Config{} + if err := yaml.Unmarshal(configBytes, vclusterConfig); err != nil { + return nil, fmt.Errorf("failed to parse vCluster configuration: %w", err) } + return vclusterConfig, nil +} - if secret.Annotations == nil { - secret.Annotations = map[string]string{} +// CheckAnnotations validates the distro and store type annotations from either a Secret or ConfigMap +func CheckAnnotations(annotations map[string]string, distro string, backingStoreType vclusterconfig.StoreType) (bool, error) { + if annotations == nil { + annotations = map[string]string{} } - // (ThomasK33): If we already have an annotation set, we're dealing with an upgrade. + // If we already have an annotation set, we're dealing with an upgrade. // Thus we can check if the distro has changed. okCounter := 0 - if annotatedDistro, ok := secret.Annotations[AnnotationDistro]; ok { + if annotatedDistro, ok := annotations[confighelper.AnnotationDistro]; ok { if err := vclusterconfig.ValidateDistroChanges(distro, annotatedDistro); err != nil { return false, err } @@ -159,7 +141,7 @@ func CheckUsingSecretAnnotation(ctx context.Context, client kubernetes.Interface okCounter++ } - if annotatedStore, ok := secret.Annotations[AnnotationStore]; ok { + if annotatedStore, ok := annotations[confighelper.AnnotationStore]; ok { if err := vclusterconfig.ValidateStoreChanges(backingStoreType, vclusterconfig.StoreType(annotatedStore)); err != nil { return false, err } @@ -170,32 +152,85 @@ func CheckUsingSecretAnnotation(ctx context.Context, client kubernetes.Interface return okCounter == 2, nil } -// updateSecretAnnotations udates the vCluster's config secret with the currently used distro and backing store type. -func updateSecretAnnotations(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - secret, err := client.CoreV1().Secrets(namespace).Get(ctx, "vc-config-"+name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("get secret: %w", err) +// UpdateConfigAnnotations checks which resource (Secret or ConfigMap) exists and updates its annotations +func UpdateConfigAnnotations(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) error { + configName := confighelper.ConfigNamePrefix + name + + // Try Secret first + secret, secretErr := client.CoreV1().Secrets(namespace).Get(ctx, configName, metav1.GetOptions{}) + if secretErr == nil { + return UpdateSecretAnnotations(ctx, client, secret, distro, backingStoreType) + } + + // If Secret not found, try ConfigMap + if kerrors.IsNotFound(secretErr) { + configMap, cmErr := client.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) + if cmErr != nil { + return fmt.Errorf("failed to get configuration from either Secret or ConfigMap: Secret error: %v, ConfigMap error: %v", secretErr, cmErr) } - if secret.Annotations == nil { - secret.Annotations = map[string]string{} + return UpdateConfigMapAnnotations(ctx, client, configMap, distro, backingStoreType) + } + + return secretErr +} + +// UpdateSecretAnnotations updates a Secret's annotations with the vCluster distro and backing store type. +func UpdateSecretAnnotations(ctx context.Context, client kubernetes.Interface, secret *corev1.Secret, distro string, backingStoreType vclusterconfig.StoreType) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + // Apply annotations and check if changes were made + if !confighelper.UpdateAnnotations(&secret.Annotations, distro, string(backingStoreType)) { + return nil // No changes needed } - if secret.Annotations[AnnotationDistro] == distro && secret.Annotations[AnnotationStore] == string(backingStoreType) { - return nil + + // Update the Secret if changes were made + if _, err := client.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update secret: %w", err) } - secret.Annotations[AnnotationDistro] = distro - secret.Annotations[AnnotationStore] = string(backingStoreType) + return nil + }) +} - if _, err := client.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("update secret: %w", err) +// UpdateConfigMapAnnotations updates a ConfigMap's annotations with the vCluster distro and backing store type. +func UpdateConfigMapAnnotations(ctx context.Context, client kubernetes.Interface, configMap *corev1.ConfigMap, distro string, backingStoreType vclusterconfig.StoreType) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + // Apply annotations and check if changes were made + if !confighelper.UpdateAnnotations(&configMap.Annotations, distro, string(backingStoreType)) { + return nil // No changes needed + } + + // Update the ConfigMap if changes were made + if _, err := client.CoreV1().ConfigMaps(configMap.Namespace).Update(ctx, configMap, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update configmap: %w", err) } return nil }) } +// EnsureBackingStoreChanges ensures that only a certain set of allowed changes to the backing store and distro occur. +// Then updates the annotations on either Secret or ConfigMap based on what exists. +func EnsureBackingStoreChanges(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) error { + // First, check using existing config annotations + if ok, err := CheckUsingConfigAnnotation(ctx, client, name, namespace, distro, backingStoreType); err != nil { + return fmt.Errorf("using config annotations: %w", err) + } else if ok { + // If validation successful, update the annotations + return UpdateConfigAnnotations(ctx, client, name, namespace, distro, backingStoreType) + } + + // If no config annotations or validation failed, try heuristic check + if ok, err := CheckUsingHeuristic(distro); err != nil { + return fmt.Errorf("using heuristic: %w", err) + } else if ok { + // If validation successful, update the annotations + return UpdateConfigAnnotations(ctx, client, name, namespace, distro, backingStoreType) + } + + return nil +} + // SetGlobalOwner fetches the owning service and populates in translate.Owner if: the vcluster is configured to setOwner is, // and if the currentNamespace == targetNamespace (because cross namespace owner refs don't work). func SetGlobalOwner(ctx context.Context, vConfig *config.VirtualClusterConfig) error { @@ -233,3 +268,28 @@ func SetGlobalOwner(ctx context.Context, vConfig *config.VirtualClusterConfig) e return nil } + +// CheckUsingHeuristic checks for known file path indicating the existence of a previous distro. +// It checks for the existence of the default K3s token path. +func CheckUsingHeuristic(distro string) (bool, error) { + // check if previously we were using k3s as a default and now have switched to a different distro + if distro != vclusterconfig.K3SDistro && distro != vclusterconfig.K8SDistro { + _, err := os.Stat(k3s.TokenPath) + if err == nil { + return false, fmt.Errorf("seems like you were using k3s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", distro) + } + } + + return true, nil +} + +// CheckUsingConfigAnnotation checks for backend store and distro changes using annotations on the vCluster's configuration resource (Secret or ConfigMap). +// Returns true, if both annotations are set and the check was successful, otherwise false. +func CheckUsingConfigAnnotation(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) (bool, error) { + annotations, err := confighelper.GetResourceAnnotations(ctx, client, name, namespace) + if err != nil { + return false, err + } + + return CheckAnnotations(annotations, distro, backingStoreType) +} diff --git a/pkg/util/confighelper/confighelper.go b/pkg/util/confighelper/confighelper.go new file mode 100644 index 0000000000..7ac873bed7 --- /dev/null +++ b/pkg/util/confighelper/confighelper.go @@ -0,0 +1,93 @@ +package confighelper + +import ( + "context" + "fmt" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + // ConfigFileName is the name of the file within the Secret or ConfigMap containing the vCluster configuration + ConfigFileName = "config.yaml" + // ConfigNamePrefix is the prefix for vCluster configuration resources + ConfigNamePrefix = "vc-config-" + // AnnotationDistro is the annotation key for the vCluster distro type + AnnotationDistro = "vcluster.loft.sh/distro" + // AnnotationStore is the annotation key for the vCluster store type + AnnotationStore = "vcluster.loft.sh/store" +) + +// GetResourceAnnotations retrieves annotations from either Secret or ConfigMap +func GetResourceAnnotations(ctx context.Context, client kubernetes.Interface, name, namespace string) (map[string]string, error) { + configName := ConfigNamePrefix + name + + // Try to get annotations from Secret first + secret, secretErr := client.CoreV1().Secrets(namespace).Get(ctx, configName, metav1.GetOptions{}) + if secretErr == nil { + return secret.Annotations, nil + } + + // If Secret not found, try ConfigMap + if kerrors.IsNotFound(secretErr) { + configMap, cmErr := client.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) + if cmErr != nil { + return nil, fmt.Errorf("failed to get configuration from either Secret or ConfigMap: Secret error: %v, ConfigMap error: %v", secretErr, cmErr) + } + + return configMap.Annotations, nil + } + + return nil, fmt.Errorf("get secret: %w", secretErr) +} + +// UpdateAnnotations is a generic helper that sets the distro and store type annotations +func UpdateAnnotations(annotations *map[string]string, distro string, backingStoreType string) bool { + if *annotations == nil { + *annotations = map[string]string{} + } + + // Check if updates are needed + if (*annotations)[AnnotationDistro] == distro && (*annotations)[AnnotationStore] == backingStoreType { + return false // No changes needed + } + + // Update the annotations + (*annotations)[AnnotationDistro] = distro + (*annotations)[AnnotationStore] = backingStoreType + return true // Changes were made +} + +// GetVClusterConfigResource retrieves the data content from either the vCluster config Secret or ConfigMap +func GetVClusterConfigResource(ctx context.Context, clientset kubernetes.Interface, name, namespace string) ([]byte, error) { + configName := ConfigNamePrefix + name + + // Try Secret first + secret, secretErr := clientset.CoreV1().Secrets(namespace).Get(ctx, configName, metav1.GetOptions{}) + if secretErr == nil { + configBytes, ok := secret.Data[ConfigFileName] + if !ok { + return nil, fmt.Errorf("secret %s in namespace %s does not contain the expected %s field", configName, namespace, ConfigFileName) + } + return configBytes, nil + } + + if kerrors.IsNotFound(secretErr) { + // Try ConfigMap if Secret not found + configMap, cmErr := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{}) + if cmErr != nil { + return nil, fmt.Errorf("failed to get configuration from either Secret or ConfigMap: Secret error: %v, ConfigMap error: %v", secretErr, cmErr) + } + + configYaml, ok := configMap.Data[ConfigFileName] + if !ok { + return nil, fmt.Errorf("configMap %s in namespace %s does not contain the expected %s field", configName, namespace, ConfigFileName) + } + + return []byte(configYaml), nil + } + + return nil, secretErr +} From 7288c1afdd6880cd912f1400198ecd8df0614126 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:55:28 +0200 Subject: [PATCH 05/10] fix(schema): rename useConfigMap to useConfigAsSecret for clarity --- chart/values.schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index c359449b72..044f1c1ba2 100755 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -404,9 +404,9 @@ "type": "string", "description": "DefaultImageRegistry will be used as a prefix for all internal images deployed by vCluster or Helm. This makes it easy to\nupload all required vCluster images to a single private repository and set this value. Workload images are not affected by this." }, - "useConfigMap": { + "useConfigAsSecret": { "type": "boolean", - "description": "UseConfigMap defines if a ConfigMap should be used instead of a Secret for storing the vCluster configuration. This is useful in environments where Secret syncing is restricted, like in certain ArgoCD setups." + "description": "useConfigAsSecret defines if a Secret should be used instead of a ConfigMap for storing the vCluster configuration." }, "virtualScheduler": { "$ref": "#/$defs/EnableSwitch", @@ -4886,4 +4886,4 @@ "additionalProperties": false, "type": "object", "description": "Config is the vCluster config." -} \ No newline at end of file +} From 4f1d81cac9db21eb9265d86376e923b270fe301c Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:56:02 +0200 Subject: [PATCH 06/10] test(config): add flag to use ConfigMap for vCluster configuration --- pkg/util/confighelper/confighelper_test.go | 261 +++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 pkg/util/confighelper/confighelper_test.go diff --git a/pkg/util/confighelper/confighelper_test.go b/pkg/util/confighelper/confighelper_test.go new file mode 100644 index 0000000000..3da9874927 --- /dev/null +++ b/pkg/util/confighelper/confighelper_test.go @@ -0,0 +1,261 @@ +package confighelper + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestUpdateAnnotations(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + distro string + backingStoreType string + expected map[string]string + returnedChanged bool + }{ + { + name: "nil annotations", + annotations: nil, + distro: "k3s", + backingStoreType: "etcd", + expected: map[string]string{ + AnnotationDistro: "k3s", + AnnotationStore: "etcd", + }, + returnedChanged: true, + }, + { + name: "empty annotations", + annotations: map[string]string{}, + distro: "k8s", + backingStoreType: "etcd", + expected: map[string]string{ + AnnotationDistro: "k8s", + AnnotationStore: "etcd", + }, + returnedChanged: true, + }, + { + name: "existing annotations no change", + annotations: map[string]string{ + AnnotationDistro: "k3s", + AnnotationStore: "etcd", + }, + distro: "k3s", + backingStoreType: "etcd", + expected: map[string]string{ + AnnotationDistro: "k3s", + AnnotationStore: "etcd", + }, + returnedChanged: false, + }, + { + name: "existing annotations with change", + annotations: map[string]string{ + AnnotationDistro: "k3s", + AnnotationStore: "etcd", + "other": "value", + }, + distro: "k8s", + backingStoreType: "etcd", + expected: map[string]string{ + AnnotationDistro: "k8s", + AnnotationStore: "etcd", + "other": "value", + }, + returnedChanged: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + annotations := test.annotations + changed := UpdateAnnotations(&annotations, test.distro, test.backingStoreType) + + if changed != test.returnedChanged { + t.Errorf("Expected change: %v, got: %v", test.returnedChanged, changed) + } + + if !reflect.DeepEqual(annotations, test.expected) { + t.Errorf("Expected annotations: %v, got: %v", test.expected, annotations) + } + }) + } +} + +func TestGetVClusterConfigResource(t *testing.T) { + tests := []struct { + name string + setupClientset func() *fake.Clientset + expectedError bool + expectedContains string + }{ + { + name: "secret exists", + setupClientset: func() *fake.Clientset { + client := fake.NewSimpleClientset() + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-config-test", + Namespace: "test-namespace", + }, + Data: map[string][]byte{ + "config.yaml": []byte("distro: k3s"), + }, + } + _, err := client.CoreV1().Secrets("test-namespace").Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + panic(err) + } + return client + }, + expectedError: false, + expectedContains: "distro: k3s", + }, + { + name: "configmap exists", + setupClientset: func() *fake.Clientset { + client := fake.NewSimpleClientset() + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-config-test", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "config.yaml": "distro: k3s", + }, + } + _, err := client.CoreV1().ConfigMaps("test-namespace").Create(context.Background(), configMap, metav1.CreateOptions{}) + if err != nil { + panic(err) + } + return client + }, + expectedError: false, + expectedContains: "distro: k3s", + }, + { + name: "neither secret nor configmap exists", + setupClientset: func() *fake.Clientset { + return fake.NewSimpleClientset() + }, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := test.setupClientset() + result, err := GetVClusterConfigResource(context.Background(), client, "test", "test-namespace") + + if test.expectedError && err == nil { + t.Error("Expected error but got nil") + } + + if !test.expectedError && err != nil { + t.Errorf("Expected no error but got: %v", err) + } + + if !test.expectedError && err == nil { + if string(result) != test.expectedContains { + t.Errorf("Expected result to contain: %s, got: %s", test.expectedContains, string(result)) + } + } + }) + } +} + +func TestGetResourceAnnotations(t *testing.T) { + tests := []struct { + name string + setupClientset func() *fake.Clientset + expectedError bool + expectedAnnotations map[string]string + }{ + { + name: "secret exists with annotations", + setupClientset: func() *fake.Clientset { + client := fake.NewSimpleClientset() + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-config-test", + Namespace: "test-namespace", + Annotations: map[string]string{ + "vcluster.loft.sh/distro": "k3s", + "vcluster.loft.sh/store": "etcd", + }, + }, + } + _, err := client.CoreV1().Secrets("test-namespace").Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + panic(err) + } + return client + }, + expectedError: false, + expectedAnnotations: map[string]string{ + "vcluster.loft.sh/distro": "k3s", + "vcluster.loft.sh/store": "etcd", + }, + }, + { + name: "configmap exists with annotations", + setupClientset: func() *fake.Clientset { + client := fake.NewSimpleClientset() + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vc-config-test", + Namespace: "test-namespace", + Annotations: map[string]string{ + "vcluster.loft.sh/distro": "k8s", + "vcluster.loft.sh/store": "etcd", + }, + }, + } + _, err := client.CoreV1().ConfigMaps("test-namespace").Create(context.Background(), configMap, metav1.CreateOptions{}) + if err != nil { + panic(err) + } + return client + }, + expectedError: false, + expectedAnnotations: map[string]string{ + "vcluster.loft.sh/distro": "k8s", + "vcluster.loft.sh/store": "etcd", + }, + }, + { + name: "neither secret nor configmap exists", + setupClientset: func() *fake.Clientset { + return fake.NewSimpleClientset() + }, + expectedError: true, + expectedAnnotations: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := test.setupClientset() + annotations, err := GetResourceAnnotations(context.Background(), client, "test", "test-namespace") + + if test.expectedError && err == nil { + t.Error("Expected error but got nil") + } + + if !test.expectedError && err != nil { + t.Errorf("Expected no error but got: %v", err) + } + + if !test.expectedError && !reflect.DeepEqual(annotations, test.expectedAnnotations) { + t.Errorf("Expected annotations: %v, got: %v", test.expectedAnnotations, annotations) + } + }) + } +} From c1265148f2177b8242df1d30e6b48a29a9feae47 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:58:40 +0200 Subject: [PATCH 07/10] refactor(config): streamline vCluster configuration retrieval by replacing secret access with config helper --- pkg/cli/create_helm.go | 33 --------------------------------- pkg/cli/describe_helm.go | 29 ++++++----------------------- pkg/cli/find/find.go | 31 +++++++++++++++++-------------- 3 files changed, 23 insertions(+), 70 deletions(-) diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 37714d3ac7..89b511b9a8 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -962,36 +962,3 @@ func (cmd *createHelm) getVClusterConfigFromSnapshot(ctx context.Context) (strin return "", nil } - -func getConfigfileFromSecret(ctx context.Context, name, namespace string) (*config.Config, error) { - secretName := "vc-config-" + name - - kConf := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) - clientConfig, err := kConf.ClientConfig() - if err != nil { - return nil, err - } - - clientset, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, err - } - - secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - configBytes, ok := secret.Data["config.yaml"] - if !ok { - return nil, fmt.Errorf("secret %s in namespace %s does not contain the expected 'config.yaml' field", secretName, namespace) - } - - config := config.Config{} - err = yaml.Unmarshal(configBytes, &config) - if err != nil { - return nil, err - } - - return &config, nil -} diff --git a/pkg/cli/describe_helm.go b/pkg/cli/describe_helm.go index e1637e031e..99bc126ca5 100644 --- a/pkg/cli/describe_helm.go +++ b/pkg/cli/describe_helm.go @@ -3,7 +3,6 @@ package cli import ( "context" "encoding/json" - "fmt" "io" "strings" @@ -12,8 +11,7 @@ import ( "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/cli/find" "github.com/loft-sh/vcluster/pkg/cli/flags" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" + "github.com/loft-sh/vcluster/pkg/setup" "k8s.io/client-go/tools/clientcmd" ) @@ -35,33 +33,23 @@ func DescribeHelm(ctx context.Context, flags *flags.GlobalFlags, output io.Write namespace = flags.Namespace } - secretName := "vc-config-" + name - kConf := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) rawConfig, err := kConf.RawConfig() if err != nil { return err } - clientConfig, err := kConf.ClientConfig() - if err != nil { - return err - } - clientset, err := kubernetes.NewForConfig(clientConfig) + vclusterConfig, err := setup.GetVClusterConfig(ctx, kConf, name, namespace) if err != nil { return err } - secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, v1.GetOptions{}) + // Convert config to bytes for YAML/JSON output formats + configBytes, err := yaml.Marshal(vclusterConfig) if err != nil { return err } - configBytes, ok := secret.Data["config.yaml"] - if !ok { - return fmt.Errorf("secret %s in namespace %s does not contain the expected 'config.yaml' field", secretName, namespace) - } - switch format { case "yaml": _, err = output.Write(configBytes) @@ -80,7 +68,7 @@ func DescribeHelm(ctx context.Context, flags *flags.GlobalFlags, output io.Write } describeOutput := &DescribeOutput{} - err = extractFromValues(describeOutput, configBytes, format, fVcluster.Version, output) + err = extractFromValues(describeOutput, configBytes, vclusterConfig, format, fVcluster.Version, output) if err != nil { return err } @@ -96,12 +84,7 @@ func DescribeHelm(ctx context.Context, flags *flags.GlobalFlags, output io.Write return err } -func extractFromValues(d *DescribeOutput, configBytes []byte, format, version string, output io.Writer) error { - conf := &config.Config{} - err := yaml.Unmarshal(configBytes, conf) - if err != nil { - return err - } +func extractFromValues(d *DescribeOutput, configBytes []byte, conf *config.Config, format, version string, output io.Writer) error { switch format { case "yaml": diff --git a/pkg/cli/find/find.go b/pkg/cli/find/find.go index fc6bbc628c..1a4f10da7a 100644 --- a/pkg/cli/find/find.go +++ b/pkg/cli/find/find.go @@ -15,6 +15,7 @@ import ( "github.com/loft-sh/vcluster/pkg/platform" "github.com/loft-sh/vcluster/pkg/platform/kube" "github.com/loft-sh/vcluster/pkg/platform/sleepmode" + "github.com/loft-sh/vcluster/pkg/util/confighelper" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" @@ -469,10 +470,10 @@ func getVCluster(ctx context.Context, object client.Object, context, release str if status == "" { // Workload sleepmode cannot modify/annotate the VirtualClusterInstance, StatefulSet, or Deployment so it - // sets a sleep type on the config secret. Check that here. - sec, err := getConfigSecret(ctx, client, kubeClientConfig, namespace, release) + // sets a sleep type on the config secret or configmap. Check that here. + annotations, err := getConfigResource(ctx, client, kubeClientConfig, namespace, release) if err == nil { - if _, ok := sec.Annotations[clusterv1.SleepModeSleepTypeAnnotation]; ok { + if _, ok := annotations[clusterv1.SleepModeSleepTypeAnnotation]; ok { status = string(StatusWorkloadSleeping) } } @@ -548,22 +549,24 @@ func getPods(ctx context.Context, client kube.Interface, kubeClientConfig client return podList, nil } -func getConfigSecret(ctx context.Context, client kube.Interface, kubeClientConfig clientcmd.ClientConfig, namespace, releaseName string) (*corev1.Secret, error) { +func getConfigResource(ctx context.Context, client kube.Interface, kubeClientConfig clientcmd.ClientConfig, namespace, releaseName string) (map[string]string, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() - secret, err := client.CoreV1().Secrets(namespace).Get(ctx, "vc-config-"+releaseName, metav1.GetOptions{}) - if err != nil { - if kerrors.IsForbidden(err) { - // try the current namespace instead - if namespace, err = getAccessibleNS(kubeClientConfig); err != nil { - return nil, err - } - return client.CoreV1().Secrets(namespace).Get(ctx, releaseName, metav1.GetOptions{}) + // Try to get annotations using the shared function + ann, err := confighelper.GetResourceAnnotations(ctx, client, releaseName, namespace) + if err == nil { + return ann, nil + } + + if kerrors.IsForbidden(err) { + // try the current namespace instead + if namespace, err = getAccessibleNS(kubeClientConfig); err == nil { + // Try in the accessible namespace with the original name pattern + return confighelper.GetResourceAnnotations(ctx, client, releaseName, namespace) } - return nil, err } - return secret, nil + return nil, err } func getDeployments(ctx context.Context, client kube.Interface, namespace string, kubeClientConfig clientcmd.ClientConfig, timeout time.Duration) (*appsv1.DeploymentList, error) { From cf3aca7e379177a13143da2c340f4457734f7e0b Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 14:58:57 +0200 Subject: [PATCH 08/10] fix(helm): update vCluster config retrieval to support ConfigMap in addition to Secret --- pkg/cli/create_helm.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/cli/create_helm.go b/pkg/cli/create_helm.go index 89b511b9a8..5ef21ebb66 100644 --- a/pkg/cli/create_helm.go +++ b/pkg/cli/create_helm.go @@ -34,6 +34,7 @@ import ( "github.com/loft-sh/vcluster/pkg/helm" "github.com/loft-sh/vcluster/pkg/platform" platformclihelper "github.com/loft-sh/vcluster/pkg/platform/clihelper" + "github.com/loft-sh/vcluster/pkg/setup" "github.com/loft-sh/vcluster/pkg/snapshot" "github.com/loft-sh/vcluster/pkg/snapshot/pod" "github.com/loft-sh/vcluster/pkg/telemetry" @@ -240,8 +241,9 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags. return err } } else { - // When a vCluster is not legacy, there should be a config secret and we will fetch the values from the secret - currentVClusterConfig, err = getConfigfileFromSecret(ctx, vClusterName, cmd.Namespace) + // When a vCluster is not legacy, there should be a config secret or configmap and we will fetch the values from the secret or configmap + kConf := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + currentVClusterConfig, err = setup.GetVClusterConfig(ctx, kConf, vClusterName, cmd.Namespace) if err != nil { return err } From d9cd1cfad73067c6ff644f658279383947b8ab6b Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 15:28:19 +0200 Subject: [PATCH 09/10] fix(cart): update test descriptions and conditions for useConfigAsSecret flag --- chart/templates/statefulset.yaml | 8 ++++---- chart/tests/config-configmap_test.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/chart/templates/statefulset.yaml b/chart/templates/statefulset.yaml index 72f03c9336..138f4741d4 100644 --- a/chart/templates/statefulset.yaml +++ b/chart/templates/statefulset.yaml @@ -110,12 +110,12 @@ spec: emptyDir: {} {{- end }} - name: vcluster-config - {{- if .Values.controlPlane.advanced.useConfigMap }} - configMap: - name: vc-config-{{ .Release.Name }} - {{- else }} + {{- if .Values.controlPlane.advanced.useConfigAsSecret }} secret: secretName: vc-config-{{ .Release.Name }} + {{- else }} + configMap: + name: vc-config-{{ .Release.Name }} {{- end }} {{- if .Values.controlPlane.statefulSet.persistence.dataVolume }} {{ toYaml .Values.controlPlane.statefulSet.persistence.dataVolume | indent 8 }} diff --git a/chart/tests/config-configmap_test.yaml b/chart/tests/config-configmap_test.yaml index 24759d0a26..f08af692e4 100644 --- a/chart/tests/config-configmap_test.yaml +++ b/chart/tests/config-configmap_test.yaml @@ -7,9 +7,9 @@ release: name: release-name namespace: test-namespace tests: - - it: should create a ConfigMap and not a Secret when useConfigMap is true + - it: should create a ConfigMap and not a Secret when useConfigAsSecret is false set: - controlPlane.advanced.useConfigMap: true + controlPlane.advanced.useConfigAsSecret: false asserts: - hasDocuments: count: 1 @@ -22,9 +22,9 @@ tests: pattern: vc-config-release-name template: statefulset.yaml - - it: should create a Secret and not a ConfigMap when useConfigMap is false + - it: should create a Secret and not a ConfigMap when useConfigAsSecret is true set: - controlPlane.advanced.useConfigMap: false + controlPlane.advanced.useConfigAsSecret: true asserts: - hasDocuments: count: 0 From ee959d0da91e4c5645e09c7be42b92ee9a4fcfc6 Mon Sep 17 00:00:00 2001 From: luka-kroeger Date: Fri, 6 Jun 2025 15:28:43 +0200 Subject: [PATCH 10/10] fix(config): rename unmarshalConfig to UnmarshalConfig and improve config retrieval logic --- pkg/cli/describe_platform.go | 10 ++++- pkg/cli/find/find.go | 2 - pkg/setup/config.go | 60 +++++++++++++-------------- pkg/util/confighelper/confighelper.go | 8 +--- 4 files changed, 39 insertions(+), 41 deletions(-) diff --git a/pkg/cli/describe_platform.go b/pkg/cli/describe_platform.go index eda43b8a4b..3e060b2dca 100644 --- a/pkg/cli/describe_platform.go +++ b/pkg/cli/describe_platform.go @@ -7,6 +7,7 @@ import ( "github.com/ghodss/yaml" "github.com/loft-sh/log" + "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/cli/flags" "github.com/loft-sh/vcluster/pkg/platform" ) @@ -43,7 +44,14 @@ func DescribePlatform(ctx context.Context, globalFlags *flags.GlobalFlags, outpu describeOutput.Version = version - err = extractFromValues(describeOutput, []byte(values), format, version, output) + // Parse config first + vclusterConfig := &config.Config{} + err = yaml.Unmarshal([]byte(values), vclusterConfig) + if err != nil { + return err + } + + err = extractFromValues(describeOutput, []byte(values), vclusterConfig, format, version, output) if err != nil { return err } diff --git a/pkg/cli/find/find.go b/pkg/cli/find/find.go index 1a4f10da7a..821b59030c 100644 --- a/pkg/cli/find/find.go +++ b/pkg/cli/find/find.go @@ -553,7 +553,6 @@ func getConfigResource(ctx context.Context, client kube.Interface, kubeClientCon ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() - // Try to get annotations using the shared function ann, err := confighelper.GetResourceAnnotations(ctx, client, releaseName, namespace) if err == nil { return ann, nil @@ -562,7 +561,6 @@ func getConfigResource(ctx context.Context, client kube.Interface, kubeClientCon if kerrors.IsForbidden(err) { // try the current namespace instead if namespace, err = getAccessibleNS(kubeClientConfig); err == nil { - // Try in the accessible namespace with the original name pattern return confighelper.GetResourceAnnotations(ctx, client, releaseName, namespace) } } diff --git a/pkg/setup/config.go b/pkg/setup/config.go index 4f62a48afc..6dfe3ebd3d 100644 --- a/pkg/setup/config.go +++ b/pkg/setup/config.go @@ -112,11 +112,11 @@ func GetVClusterConfig(ctx context.Context, kConf clientcmd.ClientConfig, name, return nil, err } - return unmarshalConfig(configBytes) + return UnmarshalConfig(configBytes) } -// unmarshalConfig parses YAML config bytes into a Config object -func unmarshalConfig(configBytes []byte) (*vclusterconfig.Config, error) { +// UnmarshalConfig parses YAML config bytes into a Config object +func UnmarshalConfig(configBytes []byte) (*vclusterconfig.Config, error) { vclusterConfig := &vclusterconfig.Config{} if err := yaml.Unmarshal(configBytes, vclusterConfig); err != nil { return nil, fmt.Errorf("failed to parse vCluster configuration: %w", err) @@ -130,7 +130,7 @@ func CheckAnnotations(annotations map[string]string, distro string, backingStore annotations = map[string]string{} } - // If we already have an annotation set, we're dealing with an upgrade. + // (ThomasK33) If we already have an annotation set, we're dealing with an upgrade. // Thus we can check if the distro has changed. okCounter := 0 if annotatedDistro, ok := annotations[confighelper.AnnotationDistro]; ok { @@ -210,13 +210,11 @@ func UpdateConfigMapAnnotations(ctx context.Context, client kubernetes.Interface } // EnsureBackingStoreChanges ensures that only a certain set of allowed changes to the backing store and distro occur. -// Then updates the annotations on either Secret or ConfigMap based on what exists. func EnsureBackingStoreChanges(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) error { // First, check using existing config annotations if ok, err := CheckUsingConfigAnnotation(ctx, client, name, namespace, distro, backingStoreType); err != nil { return fmt.Errorf("using config annotations: %w", err) } else if ok { - // If validation successful, update the annotations return UpdateConfigAnnotations(ctx, client, name, namespace, distro, backingStoreType) } @@ -224,13 +222,36 @@ func EnsureBackingStoreChanges(ctx context.Context, client kubernetes.Interface, if ok, err := CheckUsingHeuristic(distro); err != nil { return fmt.Errorf("using heuristic: %w", err) } else if ok { - // If validation successful, update the annotations return UpdateConfigAnnotations(ctx, client, name, namespace, distro, backingStoreType) } return nil } +// CheckUsingHeuristic checks for known file path indicating the existence of a previous distro. +// It checks for the existence of the default K3s token path. +func CheckUsingHeuristic(distro string) (bool, error) { + // check if previously we were using k3s as a default and now have switched to a different distro + if distro != vclusterconfig.K3SDistro && distro != vclusterconfig.K8SDistro { + _, err := os.Stat(k3s.TokenPath) + if err == nil { + return false, fmt.Errorf("seems like you were using k3s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", distro) + } + } + + return true, nil +} + +// CheckUsingConfigAnnotation checks for backend store and distro changes using annotations on the vCluster's configuration resource (Secret or ConfigMap). +func CheckUsingConfigAnnotation(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) (bool, error) { + annotations, err := confighelper.GetResourceAnnotations(ctx, client, name, namespace) + if err != nil { + return false, err + } + + return CheckAnnotations(annotations, distro, backingStoreType) +} + // SetGlobalOwner fetches the owning service and populates in translate.Owner if: the vcluster is configured to setOwner is, // and if the currentNamespace == targetNamespace (because cross namespace owner refs don't work). func SetGlobalOwner(ctx context.Context, vConfig *config.VirtualClusterConfig) error { @@ -268,28 +289,3 @@ func SetGlobalOwner(ctx context.Context, vConfig *config.VirtualClusterConfig) e return nil } - -// CheckUsingHeuristic checks for known file path indicating the existence of a previous distro. -// It checks for the existence of the default K3s token path. -func CheckUsingHeuristic(distro string) (bool, error) { - // check if previously we were using k3s as a default and now have switched to a different distro - if distro != vclusterconfig.K3SDistro && distro != vclusterconfig.K8SDistro { - _, err := os.Stat(k3s.TokenPath) - if err == nil { - return false, fmt.Errorf("seems like you were using k3s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", distro) - } - } - - return true, nil -} - -// CheckUsingConfigAnnotation checks for backend store and distro changes using annotations on the vCluster's configuration resource (Secret or ConfigMap). -// Returns true, if both annotations are set and the check was successful, otherwise false. -func CheckUsingConfigAnnotation(ctx context.Context, client kubernetes.Interface, name, namespace, distro string, backingStoreType vclusterconfig.StoreType) (bool, error) { - annotations, err := confighelper.GetResourceAnnotations(ctx, client, name, namespace) - if err != nil { - return false, err - } - - return CheckAnnotations(annotations, distro, backingStoreType) -} diff --git a/pkg/util/confighelper/confighelper.go b/pkg/util/confighelper/confighelper.go index 7ac873bed7..0160f330d8 100644 --- a/pkg/util/confighelper/confighelper.go +++ b/pkg/util/confighelper/confighelper.go @@ -10,14 +10,10 @@ import ( ) const ( - // ConfigFileName is the name of the file within the Secret or ConfigMap containing the vCluster configuration - ConfigFileName = "config.yaml" - // ConfigNamePrefix is the prefix for vCluster configuration resources + ConfigFileName = "config.yaml" ConfigNamePrefix = "vc-config-" - // AnnotationDistro is the annotation key for the vCluster distro type AnnotationDistro = "vcluster.loft.sh/distro" - // AnnotationStore is the annotation key for the vCluster store type - AnnotationStore = "vcluster.loft.sh/store" + AnnotationStore = "vcluster.loft.sh/store" ) // GetResourceAnnotations retrieves annotations from either Secret or ConfigMap