🌱 Fix flaky unit-tests 🎉🎉🎉

controllers/controllers_suite_test.go

@@ -38,7 +38,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	infrav1 "github.com/syself/cluster-api-provider-hetzner/api/v1beta1"
+	"github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/client/mocks"
+	robotmock "github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/client/mocks/robot"
+	sshmock "github.com/syself/cluster-api-provider-hetzner/pkg/services/baremetal/client/mocks/ssh"
 	hcloudclient "github.com/syself/cluster-api-provider-hetzner/pkg/services/hcloud/client"
+	fakehcloudclient "github.com/syself/cluster-api-provider-hetzner/pkg/services/hcloud/client/fake"
+	"github.com/syself/cluster-api-provider-hetzner/pkg/services/hcloud/mockedsshclient"
 	"github.com/syself/cluster-api-provider-hetzner/test/helpers"
 )
 
@@ -63,6 +68,74 @@ func TestControllers(t *testing.T) {
 	RunSpecs(t, "Controller Suite")
 }
 
+type ControllerResetter struct {
+	HetznerClusterReconciler              *HetznerClusterReconciler
+	HCloudMachineReconciler               *HCloudMachineReconciler
+	HCloudMachineTemplateReconciler       *HCloudMachineTemplateReconciler
+	HetznerBareMetalHostReconciler        *HetznerBareMetalHostReconciler
+	HetznerBareMetalMachineReconciler     *HetznerBareMetalMachineReconciler
+	HCloudRemediationReconciler           *HCloudRemediationReconciler
+	HetznerBareMetalRemediationReconciler *HetznerBareMetalRemediationReconciler
+}
+
+var _ helpers.Resetter = &ControllerResetter{}
+
+func (r *ControllerResetter) Reset(namespace string, testEnv *helpers.TestEnvironment, t FullGinkgoTInterface) {
+	rescueSSHClient := &sshmock.Client{}
+	rescueSSHClient.Test(t)
+
+	osSSHClientAfterInstallImage := &sshmock.Client{}
+	osSSHClientAfterInstallImage.Test(t)
+
+	osSSHClientAfterCloudInit := &sshmock.Client{}
+	osSSHClientAfterCloudInit.Test(t)
+
+	robotClient := &robotmock.Client{}
+	robotClient.Test(t)
+
+	hcloudSSHClient := &sshmock.Client{}
+	hcloudSSHClient.Test(t)
+
+	hcloudClientFactory := fakehcloudclient.NewHCloudClientFactory()
+
+	robotClientFactory := mocks.NewRobotFactory(robotClient)
+	baremetalSSHClientFactory := mocks.NewSSHFactory(rescueSSHClient,
+		osSSHClientAfterInstallImage, osSSHClientAfterCloudInit)
+
+	// Reset clients used by the test code
+	testEnv.BaremetalSSHClientFactory = mocks.NewSSHFactory(rescueSSHClient,
+		osSSHClientAfterInstallImage, osSSHClientAfterCloudInit)
+	testEnv.HCloudSSHClientFactory = mockedsshclient.NewSSHFactory(hcloudSSHClient)
+	testEnv.RescueSSHClient = rescueSSHClient
+	testEnv.OSSSHClientAfterInstallImage = osSSHClientAfterInstallImage
+	testEnv.OSSSHClientAfterCloudInit = osSSHClientAfterCloudInit
+	testEnv.RobotClientFactory = robotClientFactory
+	testEnv.RobotClient = robotClient
+
+	// Reset clients used by Reconcile() and the namespace
+	r.HetznerClusterReconciler.HCloudClientFactory = hcloudClientFactory
+	r.HetznerClusterReconciler.Namespace = namespace
+
+	r.HCloudMachineReconciler.HCloudClientFactory = hcloudClientFactory
+	r.HCloudMachineReconciler.SSHClientFactory = baremetalSSHClientFactory
+	r.HCloudMachineReconciler.Namespace = namespace
+
+	r.HCloudMachineTemplateReconciler.HCloudClientFactory = hcloudClientFactory
+	r.HCloudMachineTemplateReconciler.Namespace = namespace
+
+	r.HetznerBareMetalHostReconciler.RobotClientFactory = robotClientFactory
+	r.HetznerBareMetalHostReconciler.SSHClientFactory = baremetalSSHClientFactory
+	r.HetznerBareMetalHostReconciler.Namespace = namespace
+
+	r.HCloudRemediationReconciler.HCloudClientFactory = hcloudClientFactory
+	r.HCloudRemediationReconciler.Namespace = namespace
+
+	r.HetznerBareMetalMachineReconciler.HCloudClientFactory = hcloudClientFactory
+	r.HetznerBareMetalMachineReconciler.Namespace = namespace
+
+	r.HetznerBareMetalRemediationReconciler.Namespace = namespace
+}
+
 var _ = BeforeSuite(func() {
 	utilruntime.Must(infrav1.AddToScheme(scheme.Scheme))
 	utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme))
@@ -71,51 +144,62 @@ var _ = BeforeSuite(func() {
 	hcloudClient = testEnv.HCloudClientFactory.NewClient("")
 	wg.Add(1)
 
-	Expect((&HetznerClusterReconciler{
+	resetter := ControllerResetter{}
+
+	hetznerClusterReconciler := &HetznerClusterReconciler{
 		Client:                         testEnv.Manager.GetClient(),
 		APIReader:                      testEnv.Manager.GetAPIReader(),
 		RateLimitWaitTime:              5 * time.Minute,
-		HCloudClientFactory:            testEnv.HCloudClientFactory,
 		TargetClusterManagersWaitGroup: &wg,
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HCloudMachineReconciler{
-		Client:              testEnv.Manager.GetClient(),
-		APIReader:           testEnv.Manager.GetAPIReader(),
-		HCloudClientFactory: testEnv.HCloudClientFactory,
-		SSHClientFactory:    testEnv.BaremetalSSHClientFactory,
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HCloudMachineTemplateReconciler{
-		Client:              testEnv.Manager.GetClient(),
-		APIReader:           testEnv.Manager.GetAPIReader(),
-		HCloudClientFactory: testEnv.HCloudClientFactory,
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HetznerBareMetalHostReconciler{
-		Client:              testEnv.Manager.GetClient(),
-		APIReader:           testEnv.Manager.GetAPIReader(),
-		RobotClientFactory:  testEnv.RobotClientFactory,
-		SSHClientFactory:    testEnv.BaremetalSSHClientFactory,
-		PreProvisionCommand: "dummy-pre-provision-command",
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HetznerBareMetalMachineReconciler{
-		Client:              testEnv.Manager.GetClient(),
-		APIReader:           testEnv.Manager.GetAPIReader(),
-		HCloudClientFactory: testEnv.HCloudClientFactory,
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HCloudRemediationReconciler{
-		Client:              testEnv.Manager.GetClient(),
-		APIReader:           testEnv.Manager.GetAPIReader(),
-		RateLimitWaitTime:   5 * time.Minute,
-		HCloudClientFactory: testEnv.HCloudClientFactory,
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
-
-	Expect((&HetznerBareMetalRemediationReconciler{
+	}
+	Expect(hetznerClusterReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HetznerClusterReconciler = hetznerClusterReconciler
+
+	hcloudMachineReconciler := &HCloudMachineReconciler{
+		Client:    testEnv.Manager.GetClient(),
+		APIReader: testEnv.Manager.GetAPIReader(),
+	}
+	Expect(hcloudMachineReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HCloudMachineReconciler = hcloudMachineReconciler
+
+	hcloudMachineTemplateReconciler := &HCloudMachineTemplateReconciler{
+		Client:    testEnv.Manager.GetClient(),
+		APIReader: testEnv.Manager.GetAPIReader(),
+	}
+	Expect(hcloudMachineTemplateReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HCloudMachineTemplateReconciler = hcloudMachineTemplateReconciler
+
+	hetznerBareMetalHostReconciler := &HetznerBareMetalHostReconciler{
+		Client:               testEnv.Manager.GetClient(),
+		APIReader:            testEnv.Manager.GetAPIReader(),
+		PreProvisionCommand:  "dummy-pre-provision-command",
+		SSHAfterInstallImage: true,
+	}
+	Expect(hetznerBareMetalHostReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HetznerBareMetalHostReconciler = hetznerBareMetalHostReconciler
+
+	hetznerBareMetalMachineReconciler := &HetznerBareMetalMachineReconciler{
+		Client:    testEnv.Manager.GetClient(),
+		APIReader: testEnv.Manager.GetAPIReader(),
+	}
+	Expect(hetznerBareMetalMachineReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HetznerBareMetalMachineReconciler = hetznerBareMetalMachineReconciler
+
+	hcloudRemediationReconciler := &HCloudRemediationReconciler{
+		Client:            testEnv.Manager.GetClient(),
+		APIReader:         testEnv.Manager.GetAPIReader(),
+		RateLimitWaitTime: 5 * time.Minute,
+	}
+	Expect(hcloudRemediationReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HCloudRemediationReconciler = hcloudRemediationReconciler
+
+	hetznerBareMetalRemediationReconciler := &HetznerBareMetalRemediationReconciler{
 		Client: testEnv.Manager.GetClient(),
-	}).SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	}
+	Expect(hetznerBareMetalRemediationReconciler.SetupWithManager(ctx, testEnv.Manager, controller.Options{})).To(Succeed())
+	resetter.HetznerBareMetalRemediationReconciler = hetznerBareMetalRemediationReconciler
+
+	testEnv.Resetter = &resetter
 
 	go func() {
 		defer GinkgoRecover()

controllers/hcloudmachine_controller.go

@@ -62,6 +62,9 @@ type HCloudMachineReconciler struct {
 	SSHClientFactory    sshclient.Factory
 	WatchFilterValue    string
 	ImageURLCommand     string
+
+	// Reconcile only this namespace. Only needed for testing
+	Namespace string
 }
 
 //+kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch
@@ -75,6 +78,11 @@ type HCloudMachineReconciler struct {
 func (r *HCloudMachineReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
 
+	if r.Namespace != "" && req.Namespace != r.Namespace {
+		// Just for testing, skip reconciling objects from finished tests.
+		return ctrl.Result{}, nil
+	}
+
 	// Fetch the HCloudMachine instance.
 	hcloudMachine := &infrav1.HCloudMachine{}
 	err := r.Get(ctx, req.NamespacedName, hcloudMachine)

controllers/hcloudmachine_controller_test.go

@@ -243,7 +243,7 @@ var _ = Describe("HCloudMachineReconciler", func() {
 		hcloudClient.Reset()
 
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachine-reconciler")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachine-reconciler")
 		Expect(err).NotTo(HaveOccurred())
 
 		capiCluster = &clusterv1.Cluster{
@@ -365,7 +365,7 @@ var _ = Describe("HCloudMachineReconciler", func() {
 				}, timeout).Should(BeTrue())
 			})
 
-			It("creates the HCloud machine in Hetzner 1 (flaky)", func() {
+			It("creates the HCloud machine in Hetzner 1", func() {
 				By("checking that no servers exist")
 
 				Eventually(func() bool {
@@ -655,7 +655,7 @@ var _ = Describe("Hetzner secret", func() {
 	BeforeEach(func() {
 		hcloudClient.Reset()
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachine-validation")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachine-validation")
 		Expect(err).NotTo(HaveOccurred())
 
 		hetznerClusterName = utils.GenerateName(nil, "hetzner-cluster-test")
@@ -807,7 +807,7 @@ var _ = Describe("HCloudMachine validation", func() {
 		hcloudClient.Reset()
 
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachine-validation")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachine-validation")
 		Expect(err).NotTo(HaveOccurred())
 
 		hcloudMachine = &infrav1.HCloudMachine{

controllers/hcloudmachinetemplate_controller.go

@@ -49,6 +49,9 @@ type HCloudMachineTemplateReconciler struct {
 	APIReader           client.Reader
 	HCloudClientFactory hcloudclient.Factory
 	WatchFilterValue    string
+
+	// Reconcile only this namespace. Only needed for testing
+	Namespace string
 }
 
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=hcloudmachinetemplates,verbs=get;list;watch;create;update;patch;delete
@@ -58,6 +61,11 @@ type HCloudMachineTemplateReconciler struct {
 func (r *HCloudMachineTemplateReconciler) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
 
+	if r.Namespace != "" && req.Namespace != r.Namespace {
+		// Just for testing, skip reconciling objects from finished tests.
+		return ctrl.Result{}, nil
+	}
+
 	machineTemplate := &infrav1.HCloudMachineTemplate{}
 	if err := r.Get(ctx, req.NamespacedName, machineTemplate); err != nil {
 		return reconcile.Result{}, client.IgnoreNotFound(err)

controllers/hcloudmachinetemplate_controller_test.go

@@ -40,7 +40,7 @@ var _ = Describe("HCloudMachineTemplateReconciler", func() {
 	BeforeEach(func() {
 		hcloudClient.Reset()
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachinetemplate-reconciler")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachinetemplate-reconciler")
 		Expect(err).NotTo(HaveOccurred())
 
 		hetznerSecret = getDefaultHetznerSecret(testNs.Name)
@@ -253,7 +253,7 @@ var _ = Describe("HCloudMachineTemplateReconciler", func() {
 			)
 			BeforeEach(func() {
 				var err error
-				testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachine-validation")
+				testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachine-validation")
 				Expect(err).NotTo(HaveOccurred())
 
 				hcloudMachineTemplate = &infrav1.HCloudMachineTemplate{

controllers/hcloudremediation_controller.go

@@ -49,6 +49,9 @@ type HCloudRemediationReconciler struct {
 	APIReader           client.Reader
 	HCloudClientFactory hcloudclient.Factory
 	WatchFilterValue    string
+
+	// Reconcile only this namespace. Only needed for testing
+	Namespace string
 }
 
 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=hcloudremediations,verbs=get;list;watch;create;update;patch;delete
@@ -60,6 +63,11 @@ type HCloudRemediationReconciler struct {
 func (r *HCloudRemediationReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
 
+	if r.Namespace != "" && req.Namespace != r.Namespace {
+		// Just for testing, skip reconciling objects from finished tests.
+		return ctrl.Result{}, nil
+	}
+
 	hcloudRemediation := &infrav1.HCloudRemediation{}
 	err := r.Get(ctx, req.NamespacedName, hcloudRemediation)
 	if err != nil {

controllers/hcloudremediation_controller_test.go

@@ -57,7 +57,7 @@ var _ = Describe("HCloudRemediationReconciler", func() {
 	BeforeEach(func() {
 		hcloudClient.Reset()
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "hcloudmachinetemplate-reconciler")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "hcloudmachinetemplate-reconciler")
 		Expect(err).NotTo(HaveOccurred())
 
 		hetznerSecret = getDefaultHetznerSecret(testNs.Name)
@@ -204,15 +204,15 @@ var _ = Describe("HCloudRemediationReconciler", func() {
 			})
 		})
 
-		It("checks that no remediation is tried if HCloud server does not exist anymore (flaky)", func() {
+		It("checks that no remediation is tried if HCloud server does not exist anymore", func() {
 			By("ensuring if hcloudMachine is provisioned")
 			Eventually(func() error {
 				if err := testEnv.Get(ctx, hcloudMachineKey, hcloudMachine); err != nil {
 					return err
 				}
 
 				if !hcloudMachine.Status.Ready {
-					return fmt.Errorf("hcloudMachine.Status.Ready is not true (yet) (flaky)")
+					return fmt.Errorf("hcloudMachine.Status.Ready is not true (yet)")
 				}
 				return nil
 			}, timeout).ShouldNot(HaveOccurred())

controllers/hetznerbaremetalhost_controller.go

@@ -63,6 +63,9 @@ type HetznerBareMetalHostReconciler struct {
 	PreProvisionCommand  string
 	SSHAfterInstallImage bool
 	ImageURLCommand      string
+
+	// Reconcile only this namespace. Only needed for testing
+	Namespace string
 }
 
 //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=hetznerbaremetalhosts,verbs=get;list;watch;create;update;patch;delete
@@ -73,6 +76,11 @@ type HetznerBareMetalHostReconciler struct {
 func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) {
 	log := ctrl.LoggerFrom(ctx)
 
+	if r.Namespace != "" && req.Namespace != r.Namespace {
+		// Just for testing, skip reconciling objects from finished tests.
+		return ctrl.Result{}, nil
+	}
+
 	start := time.Now()
 	defer func() {
 		// check duration of reconcile. Warn if it took too long.
@@ -116,6 +124,12 @@ func (r *HetznerBareMetalHostReconciler) Reconcile(ctx context.Context, req ctrl
 		// Use uncached APIReader
 		err := r.APIReader.Get(ctx, client.ObjectKeyFromObject(bmHost), apiserverHost)
 		if err != nil {
+			if apierrors.IsNotFound(err) {
+				// resource was deleted. No need to reconcile again.
+				reterr = nil
+				res = reconcile.Result{}
+				return
+			}
 			reterr = errors.Join(reterr,
 				fmt.Errorf("failed get HetznerBareMetalHost via uncached APIReader: %w", err))
 			return

controllers/hetznerbaremetalhost_controller_test.go

@@ -86,7 +86,7 @@ var _ = Describe("HetznerBareMetalHostReconciler", func() {
 
 	BeforeEach(func() {
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "baremetalhost-reconciler")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "baremetalhost-reconciler")
 		Expect(err).NotTo(HaveOccurred())
 
 		hetznerClusterName = utils.GenerateName(nil, "hetzner-cluster-test")
@@ -615,7 +615,7 @@ var _ = Describe("HetznerBareMetalHostReconciler - missing secrets", func() {
 
 	BeforeEach(func() {
 		var err error
-		testNs, err = testEnv.CreateNamespace(ctx, "baremetalmachine-reconciler")
+		testNs, err = testEnv.ResetAndCreateNamespace(ctx, "baremetalmachine-reconciler")
 		Expect(err).NotTo(HaveOccurred())
 
 		hetznerClusterName = utils.GenerateName(nil, "hetzner-cluster-test")