From 313a440a37d69e11a861963c7ba7bf0cf69bcc71 Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Tue, 19 Aug 2025 23:36:15 +0000 Subject: [PATCH 1/7] test: Add client side install method for restoring Config Sync To guarantee the e2e client claims ownership of all fields for objects that might have drifted, use client-side apply when reinstalling Config Sync and Webhook fter ConfigManagement was previously installed and removed. --- e2e/nomostest/config_sync.go | 37 ++++++++++++++++++++++++++++++++---- e2e/testcases/cli_test.go | 24 ++++++++++++----------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 58918a7803..6927dc67ef 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -83,10 +83,11 @@ var ( // // All paths must be relative to the test file that is running. There is probably // a more elegant way to do this. - baseDir = filepath.FromSlash("../..") - outputManifestsDir = filepath.Join(baseDir, ".output", "staging", "oss") - configSyncManifest = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml") - multiConfigMaps = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName) + baseDir = filepath.FromSlash("../..") + outputManifestsDir = filepath.Join(baseDir, ".output", "staging", "oss") + configSyncManifest = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml") + admissionWebhookManifest = filepath.Join(outputManifestsDir, "admission-webhook.yaml") + multiConfigMaps = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName) ) var ( @@ -245,6 +246,34 @@ func InstallConfigSync(nt *NT) error { return nil } +// InstallConfigSyncFromManifest installs ConfigSync on the test cluster by directly +// applying the manifest file using kubectl client-side apply +func InstallConfigSyncFromManifest(nt *NT) error { + nt.T.Log("[SETUP] Installing Config Sync directly from manifest file") + + nt.T.Logf("Applying Config Sync manifest directly from %s", configSyncManifest) + + out, err := nt.Shell.Kubectl("apply", "-f", configSyncManifest) + if err != nil { + return fmt.Errorf("failed to apply Config Sync manifest: %v\n%s", err, out) + } + + nt.T.Logf("Applying multi-repo configmaps from %s", multiConfigMaps) + out, err = nt.Shell.Kubectl("apply", "-f", multiConfigMaps) + if err != nil { + return fmt.Errorf("failed to apply multi-repo configmaps: %v\n%s", err, out) + } + + // Apply the admission webhook manifest + nt.T.Logf("Applying admission webhook manifest from %s", admissionWebhookManifest) + out, err = nt.Shell.Kubectl("apply", "-f", admissionWebhookManifest) + if err != nil { + return fmt.Errorf("failed to apply admission webhook manifest: %v\n%s", err, out) + } + + return nil +} + // uninstallConfigSync uninstalls ConfigSync on the test cluster func uninstallConfigSync(nt *NT) error { nt.T.Log("[CLEANUP] Uninstalling Config Sync") diff --git a/e2e/testcases/cli_test.go b/e2e/testcases/cli_test.go index 854051e3e5..70ae1d1b0c 100644 --- a/e2e/testcases/cli_test.go +++ b/e2e/testcases/cli_test.go @@ -1297,13 +1297,14 @@ func TestApiResourceFormatting(t *testing.T) { } func TestNomosMigrate(t *testing.T) { - nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall) + nt := nomostest.New(t, nomostesting.NomosCLI) nt.T.Cleanup(func() { // Restore state of Config Sync installation after test - if err := nomostest.InstallConfigSync(nt); err != nil { + if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { nt.T.Fatal(err) } + nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { cmObj := &unstructured.Unstructured{ @@ -1451,11 +1452,11 @@ func TestNomosMigrate(t *testing.T) { configmanagement.RGControllerName, configmanagement.RGControllerNamespace) }) tg.Go(func() error { - return nt.Watcher.WatchForNotFound(kinds.Deployment(), + return nt.Watcher.WatchForCurrentStatus(kinds.Deployment(), core.RootReconcilerName(configsync.RootSyncName), configsync.ControllerNamespace) }) tg.Go(func() error { - return nt.Watcher.WatchForNotFound(kinds.RootSyncV1Beta1(), + return nt.Watcher.WatchForCurrentStatus(kinds.RootSyncV1Beta1(), configsync.RootSyncName, configsync.ControllerNamespace) }) if err := tg.Wait(); err != nil { @@ -1464,14 +1465,14 @@ func TestNomosMigrate(t *testing.T) { } func TestNomosMigrateMonoRepo(t *testing.T) { - nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall) + nt := nomostest.New(t, nomostesting.NomosCLI) nt.T.Cleanup(func() { // Restore state of Config Sync installation after test. - // This also emulates upgrading to the current version after migrating - if err := nomostest.InstallConfigSync(nt); err != nil { + if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { nt.T.Fatal(err) } + nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { crds := []string{ @@ -1707,13 +1708,14 @@ func TestNomosMigrateMonoRepo(t *testing.T) { // This test case validates the behavior of the uninstall script defined // at installation/uninstall_configmanagement.sh func TestACMUninstallScript(t *testing.T) { - nt := nomostest.New(t, nomostesting.NomosCLI, ntopts.SkipConfigSyncInstall) + nt := nomostest.New(t, nomostesting.NomosCLI) nt.T.Cleanup(func() { // Restore state of Config Sync installation after test - if err := nomostest.InstallConfigSync(nt); err != nil { + if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { nt.T.Fatal(err) } + nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { cmObj := &unstructured.Unstructured{ @@ -1861,11 +1863,11 @@ func TestACMUninstallScript(t *testing.T) { configmanagement.RGControllerName, configmanagement.RGControllerNamespace) }) tg.Go(func() error { - return nt.Watcher.WatchForNotFound(kinds.Deployment(), + return nt.Watcher.WatchForCurrentStatus(kinds.Deployment(), core.RootReconcilerName(configsync.RootSyncName), configsync.ControllerNamespace) }) tg.Go(func() error { - return nt.Watcher.WatchForNotFound(kinds.RootSyncV1Beta1(), + return nt.Watcher.WatchForCurrentStatus(kinds.RootSyncV1Beta1(), configsync.RootSyncName, configsync.ControllerNamespace) }) if err := tg.Wait(); err != nil { From 77a9062d208681afffe0bc63165bf82505525329 Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Fri, 5 Sep 2025 04:14:38 +0000 Subject: [PATCH 2/7] Add server-side false flag --- e2e/nomostest/config_sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 6927dc67ef..293083995f 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -259,14 +259,14 @@ func InstallConfigSyncFromManifest(nt *NT) error { } nt.T.Logf("Applying multi-repo configmaps from %s", multiConfigMaps) - out, err = nt.Shell.Kubectl("apply", "-f", multiConfigMaps) + out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", multiConfigMaps) if err != nil { return fmt.Errorf("failed to apply multi-repo configmaps: %v\n%s", err, out) } // Apply the admission webhook manifest nt.T.Logf("Applying admission webhook manifest from %s", admissionWebhookManifest) - out, err = nt.Shell.Kubectl("apply", "-f", admissionWebhookManifest) + out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", admissionWebhookManifest) if err != nil { return fmt.Errorf("failed to apply admission webhook manifest: %v\n%s", err, out) } From 9e508cc366ac718302436c30ce32e95435cd8761 Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Fri, 5 Sep 2025 04:14:38 +0000 Subject: [PATCH 3/7] Add server-side false flag --- e2e/nomostest/config_sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 293083995f..2f69cb548e 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -253,7 +253,7 @@ func InstallConfigSyncFromManifest(nt *NT) error { nt.T.Logf("Applying Config Sync manifest directly from %s", configSyncManifest) - out, err := nt.Shell.Kubectl("apply", "-f", configSyncManifest) + out, err := nt.Shell.Kubectl("apply", "--server-side=false", "-f", configSyncManifest) if err != nil { return fmt.Errorf("failed to apply Config Sync manifest: %v\n%s", err, out) } From 7f771eef8a2d7dc775947a546a2520ff11917204 Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Tue, 9 Sep 2025 22:51:30 +0000 Subject: [PATCH 4/7] Add go-client update as re-install option --- e2e/nomostest/config_sync.go | 54 +++++++++++++++--------------------- e2e/nomostest/new.go | 2 +- e2e/testcases/cli_test.go | 6 ++-- 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 2f69cb548e..db2b489b87 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -78,6 +78,18 @@ const ( shortSyncPollingPeriod = 5 * time.Second ) +// InstallMethod defines how Config Sync should be installed +type InstallMethod int + +const ( + // InstallMethodApply uses server-side apply (default) + InstallMethodApply InstallMethod = iota + // InstallMethodUpdate uses client-side update + InstallMethodUpdate + // InstallMethodManifest uses kubectl apply with manifest files + InstallMethodManifest +) + var ( // baseDir is the path to the Nomos repository root from test case files. // @@ -231,49 +243,27 @@ func parseConfigSyncManifests(nt *NT) ([]client.Object, error) { } // InstallConfigSync installs ConfigSync on the test cluster -func InstallConfigSync(nt *NT) error { - nt.T.Log("[SETUP] Installing Config Sync") +func InstallConfigSync(nt *NT, method InstallMethod) error { + nt.T.Log("[SETUP] Installing Config Sync using method: ", method) objs, err := parseConfigSyncManifests(nt) if err != nil { return err } for _, o := range objs { nt.T.Logf("installConfigSync obj: %v", core.GKNN(o)) - if err := nt.KubeClient.Apply(o); err != nil { - return err + if method == InstallMethodApply { + if err := nt.KubeClient.Apply(o); err != nil { + return err + } + } else if method == InstallMethodUpdate { + if err := nt.KubeClient.Update(o); err != nil { + return err + } } } return nil } -// InstallConfigSyncFromManifest installs ConfigSync on the test cluster by directly -// applying the manifest file using kubectl client-side apply -func InstallConfigSyncFromManifest(nt *NT) error { - nt.T.Log("[SETUP] Installing Config Sync directly from manifest file") - - nt.T.Logf("Applying Config Sync manifest directly from %s", configSyncManifest) - - out, err := nt.Shell.Kubectl("apply", "--server-side=false", "-f", configSyncManifest) - if err != nil { - return fmt.Errorf("failed to apply Config Sync manifest: %v\n%s", err, out) - } - - nt.T.Logf("Applying multi-repo configmaps from %s", multiConfigMaps) - out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", multiConfigMaps) - if err != nil { - return fmt.Errorf("failed to apply multi-repo configmaps: %v\n%s", err, out) - } - - // Apply the admission webhook manifest - nt.T.Logf("Applying admission webhook manifest from %s", admissionWebhookManifest) - out, err = nt.Shell.Kubectl("apply", "--server-side=false", "-f", admissionWebhookManifest) - if err != nil { - return fmt.Errorf("failed to apply admission webhook manifest: %v\n%s", err, out) - } - - return nil -} - // uninstallConfigSync uninstalls ConfigSync on the test cluster func uninstallConfigSync(nt *NT) error { nt.T.Log("[CLEANUP] Uninstalling Config Sync") diff --git a/e2e/nomostest/new.go b/e2e/nomostest/new.go index 316f06b0ca..47d67da80f 100644 --- a/e2e/nomostest/new.go +++ b/e2e/nomostest/new.go @@ -385,7 +385,7 @@ func FreshTestEnv(t nomostesting.NTB, opts *ntopts.New) *NT { return setupRegistry(nt) }) tg.Go(func() error { - return InstallConfigSync(nt) + return InstallConfigSync(nt, InstallMethodApply) }) tg.Go(func() error { return installPrometheus(nt) diff --git a/e2e/testcases/cli_test.go b/e2e/testcases/cli_test.go index 70ae1d1b0c..e19b700825 100644 --- a/e2e/testcases/cli_test.go +++ b/e2e/testcases/cli_test.go @@ -1301,7 +1301,7 @@ func TestNomosMigrate(t *testing.T) { nt.T.Cleanup(func() { // Restore state of Config Sync installation after test - if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { + if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } nt.Must(nt.WatchForAllSyncs()) @@ -1469,7 +1469,7 @@ func TestNomosMigrateMonoRepo(t *testing.T) { nt.T.Cleanup(func() { // Restore state of Config Sync installation after test. - if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { + if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } nt.Must(nt.WatchForAllSyncs()) @@ -1712,7 +1712,7 @@ func TestACMUninstallScript(t *testing.T) { nt.T.Cleanup(func() { // Restore state of Config Sync installation after test - if err := nomostest.InstallConfigSyncFromManifest(nt); err != nil { + if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } nt.Must(nt.WatchForAllSyncs()) From 0c6dcc9475837199d5cf1a1cdbedb46c1e2c70a7 Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Tue, 9 Sep 2025 23:54:48 +0000 Subject: [PATCH 5/7] Also re-install webhook after test done --- e2e/nomostest/config_sync.go | 17 +++++++---------- e2e/testcases/cli_test.go | 9 +++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index db2b489b87..b77b5fdca9 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -79,15 +79,13 @@ const ( ) // InstallMethod defines how Config Sync should be installed -type InstallMethod int +type InstallMethod string const ( // InstallMethodApply uses server-side apply (default) - InstallMethodApply InstallMethod = iota + InstallMethodApply InstallMethod = "apply" // InstallMethodUpdate uses client-side update - InstallMethodUpdate - // InstallMethodManifest uses kubectl apply with manifest files - InstallMethodManifest + InstallMethodUpdate InstallMethod = "update" ) var ( @@ -95,11 +93,10 @@ var ( // // All paths must be relative to the test file that is running. There is probably // a more elegant way to do this. - baseDir = filepath.FromSlash("../..") - outputManifestsDir = filepath.Join(baseDir, ".output", "staging", "oss") - configSyncManifest = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml") - admissionWebhookManifest = filepath.Join(outputManifestsDir, "admission-webhook.yaml") - multiConfigMaps = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName) + baseDir = filepath.FromSlash("../..") + outputManifestsDir = filepath.Join(baseDir, ".output", "staging", "oss") + configSyncManifest = filepath.Join(outputManifestsDir, "config-sync-manifest.yaml") + multiConfigMaps = filepath.Join(baseDir, "e2e", "raw-nomos", configSyncManifests, multiConfigMapsName) ) var ( diff --git a/e2e/testcases/cli_test.go b/e2e/testcases/cli_test.go index e19b700825..697f97da4e 100644 --- a/e2e/testcases/cli_test.go +++ b/e2e/testcases/cli_test.go @@ -1304,6 +1304,9 @@ func TestNomosMigrate(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } + if err := installwebhook(nt); err != nil { + nt.T.Fatal(err) + } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { @@ -1472,6 +1475,9 @@ func TestNomosMigrateMonoRepo(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } + if err := installwebhook(nt); err != nil { + nt.T.Fatal(err) + } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { @@ -1715,6 +1721,9 @@ func TestACMUninstallScript(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } + if err := installwebhook(nt); err != nil { + nt.T.Fatal(err) + } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { From 9b56f3b852bc8d40ce107c5ca71af80a52b89b8e Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Wed, 10 Sep 2025 05:59:18 +0000 Subject: [PATCH 6/7] Create object if not found when updating --- e2e/nomostest/config_sync.go | 13 ++++++++++--- e2e/testcases/cli_test.go | 9 --------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index b77b5fdca9..2c0cc8ef18 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -248,13 +248,20 @@ func InstallConfigSync(nt *NT, method InstallMethod) error { } for _, o := range objs { nt.T.Logf("installConfigSync obj: %v", core.GKNN(o)) - if method == InstallMethodApply { + switch method { + case InstallMethodApply: if err := nt.KubeClient.Apply(o); err != nil { return err } - } else if method == InstallMethodUpdate { + case InstallMethodUpdate: if err := nt.KubeClient.Update(o); err != nil { - return err + if apierrors.IsNotFound(err) { + if err := nt.KubeClient.Create(o); err != nil { + return err + } + } else { + return err + } } } } diff --git a/e2e/testcases/cli_test.go b/e2e/testcases/cli_test.go index 697f97da4e..e19b700825 100644 --- a/e2e/testcases/cli_test.go +++ b/e2e/testcases/cli_test.go @@ -1304,9 +1304,6 @@ func TestNomosMigrate(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } - if err := installwebhook(nt); err != nil { - nt.T.Fatal(err) - } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { @@ -1475,9 +1472,6 @@ func TestNomosMigrateMonoRepo(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } - if err := installwebhook(nt); err != nil { - nt.T.Fatal(err) - } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { @@ -1721,9 +1715,6 @@ func TestACMUninstallScript(t *testing.T) { if err := nomostest.InstallConfigSync(nt, nomostest.InstallMethodUpdate); err != nil { nt.T.Fatal(err) } - if err := installwebhook(nt); err != nil { - nt.T.Fatal(err) - } nt.Must(nt.WatchForAllSyncs()) }) nt.T.Cleanup(func() { From e8cc29495ac42f284734c59a052caa6c9ddd203e Mon Sep 17 00:00:00 2001 From: tiffanny29631 Date: Wed, 10 Sep 2025 06:21:39 +0000 Subject: [PATCH 7/7] Attach resourceVersion for updating existing objects --- e2e/nomostest/config_sync.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e/nomostest/config_sync.go b/e2e/nomostest/config_sync.go index 2c0cc8ef18..cdd4200af6 100644 --- a/e2e/nomostest/config_sync.go +++ b/e2e/nomostest/config_sync.go @@ -254,7 +254,8 @@ func InstallConfigSync(nt *NT, method InstallMethod) error { return err } case InstallMethodUpdate: - if err := nt.KubeClient.Update(o); err != nil { + currentObj := o.DeepCopyObject().(client.Object) + if err := nt.KubeClient.Get(currentObj.GetName(), currentObj.GetNamespace(), currentObj); err != nil { if apierrors.IsNotFound(err) { if err := nt.KubeClient.Create(o); err != nil { return err @@ -262,6 +263,12 @@ func InstallConfigSync(nt *NT, method InstallMethod) error { } else { return err } + } else { + // Attach existing resourceVersion to the object + o.SetResourceVersion(currentObj.GetResourceVersion()) + if err := nt.KubeClient.Update(o); err != nil { + return err + } } } }