plugin_controller_flux_test.go

// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
// SPDX-License-Identifier: Apache-2.0

package plugin

import (
	"encoding/json"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"

	helmv2 "github.com/fluxcd/helm-controller/api/v2"
	fluxmeta "github.com/fluxcd/pkg/apis/meta"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/envtest"

	greenhouseapis "github.com/cloudoperators/greenhouse/api"
	greenhousemetav1alpha1 "github.com/cloudoperators/greenhouse/api/meta/v1alpha1"
	greenhousev1alpha1 "github.com/cloudoperators/greenhouse/api/v1alpha1"
	"github.com/cloudoperators/greenhouse/internal/flux"
	"github.com/cloudoperators/greenhouse/internal/helm"
	"github.com/cloudoperators/greenhouse/internal/test"
	"github.com/cloudoperators/greenhouse/pkg/lifecycle"
)

var (
	remoteKubeConfig []byte
	remoteEnvTest    *envtest.Environment
	remoteK8sClient  client.Client
)

var (
	testPluginTeam = test.NewTeam(test.Ctx, "test-remote-cluster-team", test.TestNamespace, test.WithTeamLabel(greenhouseapis.LabelKeySupportGroup, "true"))

	testOrganization = test.NewOrganization(test.Ctx, test.TestNamespace)

	testCluster = test.NewCluster(test.Ctx, "test-flux-cluster", test.TestNamespace,
		test.WithAccessMode(greenhousev1alpha1.ClusterAccessModeDirect),
		test.WithClusterLabel(greenhouseapis.LabelKeyOwnedBy, testPluginTeam.Name))

	testClusterK8sSecret = corev1.Secret{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Secret",
			APIVersion: corev1.GroupName,
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-flux-cluster",
			Namespace: test.TestNamespace,
			Labels:    map[string]string{greenhouseapis.LabelKeyOwnedBy: testPluginTeam.Name},
		},
		Type: greenhouseapis.SecretTypeKubeConfig,
	}

	testPlugin = test.NewPlugin(test.Ctx, "test-flux-plugindefinition", test.TestNamespace,
		test.WithCluster("test-flux-cluster"),
		test.WithClusterPluginDefinition("test-flux-plugindefinition"),
		test.WithReleaseName("release-test-flux"),
		test.WithReleaseNamespace(test.TestNamespace),
		test.WithPluginLabel(greenhouseapis.LabelKeyOwnedBy, testPluginTeam.Name),
		test.WithPluginOptionValue("flatOption", test.MustReturnJSONFor("flatValue")),
		test.WithPluginOptionValue("nested.option", test.MustReturnJSONFor("nestedValue")),
		test.WithPluginOptionValueFrom("nested.secretOption", &greenhousev1alpha1.PluginValueFromSource{
			Secret: &greenhousev1alpha1.SecretKeyReference{
				Name: "test-cluster",
				Key:  greenhouseapis.GreenHouseKubeConfigKey,
			},
		}),
		test.WithPluginWaitFor([]greenhousev1alpha1.WaitForItem{
			{
				PluginRef: greenhousev1alpha1.PluginRef{
					Name: "", PluginPreset: "dependent-preset-1",
				},
			},
			{
				PluginRef: greenhousev1alpha1.PluginRef{
					Name: "dependent-plugin-1", PluginPreset: "",
				},
			},
		}),
	)

	testPluginDefinition = test.NewClusterPluginDefinition(
		test.Ctx,
		"test-flux-plugindefinition",
		test.WithHelmChart(&greenhousev1alpha1.HelmChartReference{
			Name:       "dummy",
			Repository: "oci://greenhouse/helm-charts",
			Version:    "1.0.0",
		}),
		test.AppendPluginOption(
			greenhousev1alpha1.PluginOption{
				Name:    "flatOptionDefault",
				Type:    greenhousev1alpha1.PluginOptionTypeString,
				Default: test.MustReturnJSONFor("flatDefault"),
			}),
		test.AppendPluginOption(
			greenhousev1alpha1.PluginOption{
				Name:    "nested.optionDefault",
				Type:    greenhousev1alpha1.PluginOptionTypeString,
				Default: test.MustReturnJSONFor("nestedDefault"),
			},
		),
	)

	uiPluginDefinition = test.NewClusterPluginDefinition(
		test.Ctx, "test-flux-ui-plugindefinition",
		test.WithVersion("1.0.0"),
		test.WithoutHelmChart(),
		test.WithUIApplication(&greenhousev1alpha1.UIApplicationReference{
			Name:    "test-ui-app",
			Version: "0.0.1",
		}),
	)

	uiPlugin = test.NewPlugin(test.Ctx, "test-flux-ui-plugin", test.TestNamespace,
		test.WithClusterPluginDefinition(uiPluginDefinition.Name),
		test.WithReleaseName("release-test-flux"),
		test.WithPluginLabel(greenhouseapis.LabelKeyOwnedBy, testPluginTeam.Name),
	)
)

var _ = Describe("Flux Plugin Controller", Ordered, func() {
	BeforeAll(func() {
		By("creating ClusterPluginDefinitions for the Flux Plugin tests")
		err := test.K8sClient.Create(test.Ctx, testPluginDefinition)
		Expect(err).ToNot(HaveOccurred(), "there should be no error creating the pluginDefinition")

		test.MockHelmChartReady(test.Ctx, test.K8sClient, testPluginDefinition, flux.HelmRepositoryDefaultNamespace)

		err = test.K8sClient.Create(test.Ctx, uiPluginDefinition)
		Expect(err).ToNot(HaveOccurred(), "there should be no error creating the UI pluginDefinition")

		By("bootstrapping remote cluster")
		_, remoteK8sClient, remoteEnvTest, remoteKubeConfig = test.StartControlPlane("6885", false, false)

		By("creating an Organization")
		Expect(client.IgnoreAlreadyExists(test.K8sClient.Create(test.Ctx, testOrganization))).Should(Succeed(), "there should be no error creating the Organization")

		By("creating a Team")
		Expect(test.K8sClient.Create(test.Ctx, testPluginTeam)).Should(Succeed(), "there should be no error creating the Team")

		By("creating a cluster")
		Expect(test.K8sClient.Create(test.Ctx, testCluster)).Should(Succeed(), "there should be no error creating the cluster resource")

		By("creating a secret with a valid kubeconfig for a remote cluster")
		testClusterK8sSecret.Data = map[string][]byte{
			greenhouseapis.KubeConfigKey: remoteKubeConfig,
		}
		Expect(test.K8sClient.Create(test.Ctx, &testClusterK8sSecret)).Should(Succeed())
	})

	AfterAll(func() {
		By("stopping the test environment")
		err := remoteEnvTest.Stop()
		Expect(err).
			NotTo(HaveOccurred(), "there must be no error stopping the remote environment")
	})

	It("should compute the HelmRelease values for a Plugin", func() {
		expected := map[string]any{
			"flatOption":        "flatValue",
			"flatOptionDefault": "flatDefault",
			"nested": map[string]any{
				"option":        "nestedValue",
				"optionDefault": "nestedDefault",
			},
		}

		// compute the expected global.greenhouse values
		greenhouseValues, err := helm.GetGreenhouseValues(test.Ctx, test.K8sClient, *testPlugin)
		Expect(err).ToNot(HaveOccurred(), "there should be no error getting the greenhouse values")
		greenhouseValueMap, err := helm.ConvertFlatValuesToHelmValues(greenhouseValues)
		Expect(err).ToNot(HaveOccurred(), "there should be no error converting the greenhouse values to Helm values")
		expected["global"] = greenhouseValueMap["global"]
		expectedRaw, err := json.Marshal(expected)
		Expect(err).ToNot(HaveOccurred(), "the expected HelmRelease values should be valid JSON")

		By("computing the Values for a Plugin")
		actualOptionValues, err := computeReleaseValues(test.Ctx, test.K8sClient, testPlugin, false, false)
		Expect(err).ToNot(HaveOccurred(), "there should be no error computing the HelmRelease values for the Plugin")

		actualRaw, err := generateHelmValues(test.Ctx, actualOptionValues)
		Expect(err).ToNot(HaveOccurred(), "there should be no error generating the Helm values JSON")
		Expect(actualOptionValues).Should(ContainElement(
			MatchFields(IgnoreExtras, Fields{
				"Name": Equal("nested.secretOption"),
			}),
		))

		By("checking the computed Values")
		Expect(actualRaw).To(Equal(expectedRaw), "the computed HelmRelease values should match the expected values")
	})

	It("should return error when generating Helm values with conflicting option paths", func() {
		By("creating conflicting option values where a key is both a string and a parent path")
		conflictingOptionValues := []greenhousev1alpha1.PluginOptionValue{
			{Name: "credentials.type", Value: test.MustReturnJSONFor("S3")},
			{Name: "credentials.type.config", Value: test.MustReturnJSONFor("bucket-name")},
		}

		By("attempting to generate Helm values")
		_, err := generateHelmValues(test.Ctx, conflictingOptionValues)

		By("verifying error is returned")
		Expect(err).To(HaveOccurred(), "generateHelmValues should fail with conflicting option value paths")
		Expect(err.Error()).To(ContainSubstring("unable to parse key"),
			"error message should indicate the parsing failure")
	})

	It("should create HelmRelease for Plugin", func() {
		By("creating test Plugin")
		Expect(test.K8sClient.Create(test.Ctx, testPlugin)).To(Succeed(), "failed to create Plugin")

		By("ensuring the Plugin Status reflects the HelmReleaseCreated condition")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")
			helmReleaseCreatedCondition := testPlugin.Status.GetConditionByType(greenhousev1alpha1.HelmReleaseCreatedCondition)
			g.Expect(helmReleaseCreatedCondition).ToNot(BeNil(), "HelmReleaseCreated condition should be set")
			g.Expect(helmReleaseCreatedCondition.IsTrue()).To(BeTrue(), "HelmReleaseCreated condition should be true")
		}).Should(Succeed(), "Plugin should have HelmReleaseCreated condition set to true")

		By("ensuring HelmRelease has been created")
		release := &helmv2.HelmRelease{}
		releaseKey := types.NamespacedName{Name: testPlugin.Name, Namespace: testPlugin.Namespace}
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, releaseKey, release)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get HelmRelease")
		}).Should(Succeed())

		By("ensuring Plugin dependencies has been resolved and set on the HelmRelease")
		Expect(release.Spec.DependsOn).To(ContainElement(helmv2.DependencyReference{Name: "dependent-plugin-1"}),
			"Flux HelmRelease should have the dependency for global plugin's release set")
		Expect(release.Spec.DependsOn).To(ContainElement(helmv2.DependencyReference{Name: "dependent-preset-1-" + testPlugin.Spec.ClusterName}),
			"Flux HelmRelease should have the dependency for resolved plugin's release set")

		By("ensuring the Plugin Status is updated")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred())

			helmReleaseDeployedCondition := testPlugin.Status.GetConditionByType(greenhousev1alpha1.HelmReleaseDeployedCondition)
			g.Expect(helmReleaseDeployedCondition).ToNot(BeNil())
			g.Expect(helmReleaseDeployedCondition.Status).To(Equal(metav1.ConditionUnknown), "HelmReleaseDeployed condition should be unknown")
			readyCondition := testPlugin.Status.GetConditionByType(greenhousemetav1alpha1.ReadyCondition)
			g.Expect(readyCondition).ToNot(BeNil())
			g.Expect(readyCondition.IsFalse()).To(BeTrue(), "Ready condition should be set to false")
			// The status won't change further, because Flux HelmController can't be registered here. See E2E tests.
		}).Should(Succeed())

		By("ensuring the Flux HelmRelease is suspended")
		test.MustSetAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.SuspendAnnotation, "true")

		By("checking if Plugin has Suspended condition set")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")
			// Check that Suspended condition is set
			suspendedCond := testPlugin.Status.GetConditionByType(greenhousemetav1alpha1.SuspendedCondition)
			g.Expect(suspendedCond).ToNot(BeNil(), "Suspended condition should be present")
			g.Expect(suspendedCond.IsTrue()).To(BeTrue(), "Suspended condition should be true")
		}).Should(Succeed())

		By("verifying HelmRelease is suspended")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, releaseKey, release)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get HelmRelease")
			g.Expect(release.Spec.Suspend).To(BeTrue(), "HelmRelease should be suspended")
		}).Should(Succeed())

		By("ensuring the Flux HelmRelease is resumed")
		test.MustRemoveAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.SuspendAnnotation)

		By("checking if Plugin Suspended condition is removed")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")
			// Check that Suspended condition is removed
			suspendedCond := testPlugin.Status.GetConditionByType(greenhousemetav1alpha1.SuspendedCondition)
			g.Expect(suspendedCond).To(BeNil(), "Suspended condition should be removed after resume")
		}).Should(Succeed())

		By("verifying HelmRelease is no longer suspended")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, releaseKey, release)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get HelmRelease")
			g.Expect(release.Spec.Suspend).To(BeFalse(), "HelmRelease should not be suspended")
		}).Should(Succeed())

		By("ensuring the HelmRelease has the last reconcile annotation updated")
		test.MustSetAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.ReconcileAnnotation, "foobar")

		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, releaseKey, release)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get HelmRelease")
			g.Expect(release.GetAnnotations()).Should(HaveKeyWithValue(fluxmeta.ReconcileRequestAnnotation, "foobar"), "HelmRelease should have the reconcile annotation updated")
			g.Expect(release.GetAnnotations()).Should(HaveKeyWithValue(helmv2.ResetRequestAnnotation, "foobar"), "HelmRelease should have the reset annotation updated to allow retry reset")

			err = test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")
			g.Expect(testPlugin.Status.LastReconciledAt).To(Equal("foobar"), "Plugin status LastReconcile should be updated")
		}).Should(Succeed())

		test.MustRemoveAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.ReconcileAnnotation)

		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, releaseKey, release)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get HelmRelease")
			g.Expect(release.GetAnnotations()).ShouldNot(HaveKey(fluxmeta.ReconcileRequestAnnotation), "HelmRelease should have the reconcile annotation removed")
			g.Expect(release.GetAnnotations()).ShouldNot(HaveKey(helmv2.ResetRequestAnnotation), "HelmRelease should have the reset annotation removed")

			err = test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")
			g.Expect(testPlugin.Status.LastReconciledAt).To(BeEmpty(), "Plugin status LastReconcile should be empty")
		}).Should(Succeed())
	})

	It("should skip exposed services when cluster has service-proxy-disabled annotation", func() {
		By("annotating the cluster with service-proxy-disabled=true")
		test.MustSetAnnotation(test.Ctx, test.K8sClient, testCluster, greenhousev1alpha1.ServiceProxyDisabledKey, "true")

		// By("triggering a reconcile on the Plugin")
		// test.MustSetAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.ReconcileAnnotation, "service-proxy-test")

		By("verifying exposed services are nil and condition reflects disabled state")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")

			g.Expect(testPlugin.Status.ExposedServices).To(BeNil(), "ExposedServices should be nil when service-proxy is disabled")

			exposedServicesSyncedCondition := testPlugin.Status.GetConditionByType(greenhousev1alpha1.ExposedServicesSyncedCondition)
			g.Expect(exposedServicesSyncedCondition).ToNot(BeNil(), "ExposedServicesSynced condition should be set")
			g.Expect(exposedServicesSyncedCondition.IsTrue()).To(BeTrue(), "ExposedServicesSynced condition should be true")
			g.Expect(exposedServicesSyncedCondition.Reason).To(Equal(greenhousev1alpha1.ExposedServicesDisabledReason),
				"ExposedServicesSynced condition should have ExposedServicesDisabled reason")
		}).Should(Succeed())

		By("removing the service-proxy-disabled annotation from the cluster")
		test.MustRemoveAnnotation(test.Ctx, test.K8sClient, testCluster, greenhousev1alpha1.ServiceProxyDisabledKey)

		// By("triggering another reconcile on the Plugin")
		// test.MustSetAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.ReconcileAnnotation, "service-proxy-test-2")

		By("verifying exposed services are fetched again after annotation removal")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred(), "failed to get Plugin")

			exposedServicesSyncedCondition := testPlugin.Status.GetConditionByType(greenhousev1alpha1.ExposedServicesSyncedCondition)
			g.Expect(exposedServicesSyncedCondition).ToNot(BeNil(), "ExposedServicesSynced condition should be set")
			g.Expect(exposedServicesSyncedCondition.Reason).ToNot(Equal(greenhousev1alpha1.ExposedServicesDisabledReason),
				"ExposedServicesSynced condition should no longer have ExposedServicesDisabled reason after annotation removal")
		}).Should(Succeed())

		// By("cleaning up reconcile annotation")
		// test.MustRemoveAnnotation(test.Ctx, test.K8sClient, testPlugin, lifecycle.ReconcileAnnotation)
	})

	It("should reconcile a UI-only Plugin", func() {
		By("creating UI-only Plugin")
		Expect(test.K8sClient.Create(test.Ctx, uiPlugin)).To(Succeed(), "failed to create UI-only Plugin")

		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(uiPlugin), uiPlugin)
			g.Expect(err).ToNot(HaveOccurred())
			readyCondition := uiPlugin.Status.GetConditionByType(greenhousemetav1alpha1.ReadyCondition)
			g.Expect(readyCondition).ToNot(BeNil())
			g.Expect(readyCondition.IsTrue()).To(BeTrue())
			g.Expect(uiPlugin.Status.UIApplication).To(Equal(uiPluginDefinition.Spec.UIApplication))
		}).Should(Succeed())

		By("ensuring HelmRelease has not been created")
		Eventually(func(g Gomega) {
			release := &helmv2.HelmRelease{}
			err := test.K8sClient.Get(test.Ctx, types.NamespacedName{Name: uiPlugin.Name, Namespace: uiPlugin.Namespace}, release)
			g.Expect(err).To(HaveOccurred(), "there should be an error getting the HelmRelease")
			g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
		}).Should(Succeed())

		By("checking that the UI plugin has the ui-plugin label")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(uiPlugin), uiPlugin)
			g.Expect(err).ToNot(HaveOccurred())
			g.Expect(uiPlugin.GetLabels()).To(HaveKeyWithValue(greenhouseapis.LabelKeyUIPlugin, "true"),
				"Plugin with UIApplication should have ui-plugin label")
			g.Expect(uiPlugin.GetLabels()).ToNot(HaveKey(greenhouseapis.LabelKeyPluginExposedServices),
				"Plugin without ExposedServices should not have plugin-exposed-services label")
		}).Should(Succeed())

		By("checking that the non-UI plugin does not have the ui-plugin label")
		Eventually(func(g Gomega) {
			err := test.K8sClient.Get(test.Ctx, client.ObjectKeyFromObject(testPlugin), testPlugin)
			g.Expect(err).ToNot(HaveOccurred())
			g.Expect(testPlugin.GetLabels()).ToNot(HaveKey(greenhouseapis.LabelKeyUIPlugin),
				"Plugin without UIApplication should not have ui-plugin label")
			g.Expect(uiPlugin.GetLabels()).ToNot(HaveKey(greenhouseapis.LabelKeyPluginExposedServices),
				"Plugin without ExposedServices should not have plugin-exposed-services label")
		}).Should(Succeed())
	})
})