integration: start informer and scheduler outside of InitTestScheduler

In case two or more controllers share the informers created through InitTestScheduler,
it's not safe to start the informers until all controllers set their informer
indexers. Otherwise, some controller might fail to register their indexers
in time. Thus, it's responsibility of each consumer to make sure all informers
are started after all controllers had time to get initiliazed.

Author: ingvagabund

test/integration/node/lifecycle_test.go

@@ -161,13 +161,14 @@ func TestTaintBasedEvictions(t *testing.T) {
 				return
 			}
 
-			go nc.Run(testCtx.Ctx.Done())
-
-			// Waiting for all controller sync.
+			// Waiting for all controllers to sync
 			externalInformers.Start(testCtx.Ctx.Done())
 			externalInformers.WaitForCacheSync(testCtx.Ctx.Done())
-			testCtx.InformerFactory.Start(testCtx.Ctx.Done())
-			testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done())
+			testutils.SyncInformerFactory(testCtx)
+
+			// Run all controllers
+			go nc.Run(testCtx.Ctx.Done())
+			go testCtx.Scheduler.Run(testCtx.Ctx)
 
 			nodeRes := v1.ResourceList{
 				v1.ResourceCPU:    resource.MustParse("4000m"),

test/integration/scheduler/extender_test.go

@@ -351,6 +351,8 @@ func TestSchedulerExtender(t *testing.T) {
 	policy.APIVersion = "v1"
 
 	testCtx = testutils.InitTestScheduler(t, testCtx, false, &policy)
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
 	defer testutils.CleanupTest(t, testCtx)
 
 	DoTestPodScheduling(testCtx.NS, t, clientSet)

test/integration/scheduler/framework_test.go

@@ -892,10 +892,12 @@ func TestBindPlugin(t *testing.T) {
 		},
 	}
 
-	// Create the master and the scheduler with the test plugin set.
+	// Create the scheduler with the test plugin set.
 	testCtx := testutils.InitTestSchedulerWithOptions(t, testContext, false, nil, time.Second,
 		scheduler.WithProfiles(prof),
 		scheduler.WithFrameworkOutOfTreeRegistry(registry))
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
 	defer testutils.CleanupTest(t, testCtx)
 
 	// Add a few nodes.
@@ -1552,14 +1554,17 @@ func TestPreemptWithPermitPlugin(t *testing.T) {
 }
 
 func initTestSchedulerForFrameworkTest(t *testing.T, testCtx *testutils.TestContext, nodeCount int, opts ...scheduler.Option) *testutils.TestContext {
-	c := testutils.InitTestSchedulerWithOptions(t, testCtx, false, nil, time.Second, opts...)
+	testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, false, nil, time.Second, opts...)
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
+
 	if nodeCount > 0 {
-		_, err := createNodes(c.ClientSet, "test-node", nil, nodeCount)
+		_, err := createNodes(testCtx.ClientSet, "test-node", nil, nodeCount)
 		if err != nil {
 			t.Fatalf("Cannot create nodes: %v", err)
 		}
 	}
-	return c
+	return testCtx
 }
 
 // initRegistryAndConfig returns registry and plugins config based on give plugins.

test/integration/scheduler/preemption_test.go

@@ -148,6 +148,8 @@ func TestPreemption(t *testing.T) {
 		false, nil, time.Second,
 		scheduler.WithProfiles(prof),
 		scheduler.WithFrameworkOutOfTreeRegistry(registry))
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
 
 	defer testutils.CleanupTest(t, testCtx)
 	cs := testCtx.ClientSet
@@ -527,9 +529,15 @@ func TestPodPriorityResolution(t *testing.T) {
 	externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second)
 	admission.SetExternalKubeClientSet(externalClientset)
 	admission.SetExternalKubeInformerFactory(externalInformers)
+
+	// Waiting for all controllers to sync
+	testutils.SyncInformerFactory(testCtx)
 	externalInformers.Start(testCtx.Ctx.Done())
 	externalInformers.WaitForCacheSync(testCtx.Ctx.Done())
 
+	// Run all controllers
+	go testCtx.Scheduler.Run(testCtx.Ctx)
+
 	tests := []struct {
 		Name             string
 		PriorityClass    string

test/integration/scheduler/scheduler_test.go

@@ -539,6 +539,8 @@ func TestMultipleSchedulers(t *testing.T) {
 	// 5. create and start a scheduler with name "foo-scheduler"
 	fooProf := kubeschedulerconfig.KubeSchedulerProfile{SchedulerName: fooScheduler}
 	testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, true, nil, time.Second, scheduler.WithProfiles(fooProf))
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
 
 	//	6. **check point-2**:
 	//		- testPodWithAnnotationFitsFoo should be scheduled

test/integration/scheduler/taint_test.go

@@ -102,13 +102,15 @@ func TestTaintNodeByCondition(t *testing.T) {
 		t.Errorf("Failed to create node controller: %v", err)
 		return
 	}
-	go nc.Run(testCtx.Ctx.Done())
 
-	// Waiting for all controller sync.
+	// Waiting for all controllers to sync
 	externalInformers.Start(testCtx.Ctx.Done())
 	externalInformers.WaitForCacheSync(testCtx.Ctx.Done())
-	testCtx.InformerFactory.Start(testCtx.Ctx.Done())
-	testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done())
+	testutils.SyncInformerFactory(testCtx)
+
+	// Run all controllers
+	go nc.Run(testCtx.Ctx.Done())
+	go testCtx.Scheduler.Run(testCtx.Ctx)
 
 	// -------------------------------------------
 	// Test TaintNodeByCondition feature.

test/integration/scheduler/util.go

@@ -79,15 +79,21 @@ func initDisruptionController(t *testing.T, testCtx *testutils.TestContext) *dis
 // initTest initializes a test environment and creates master and scheduler with default
 // configuration.
 func initTest(t *testing.T, nsPrefix string, opts ...scheduler.Option) *testutils.TestContext {
-	return testutils.InitTestSchedulerWithOptions(t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil, time.Second, opts...)
+	testCtx := testutils.InitTestSchedulerWithOptions(t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil, time.Second, opts...)
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
+	return testCtx
 }
 
 // initTestDisablePreemption initializes a test environment and creates master and scheduler with default
 // configuration but with pod preemption disabled.
 func initTestDisablePreemption(t *testing.T, nsPrefix string) *testutils.TestContext {
-	return testutils.InitTestSchedulerWithOptions(
+	testCtx := testutils.InitTestSchedulerWithOptions(
 		t, testutils.InitTestMaster(t, nsPrefix, nil), true, nil,
 		time.Second, scheduler.WithPreemptionDisabled(true))
+	testutils.SyncInformerFactory(testCtx)
+	go testCtx.Scheduler.Run(testCtx.Ctx)
+	return testCtx
 }
 
 // waitForReflection waits till the passFunc confirms that the object it expects

test/integration/util/util.go

@@ -181,6 +181,12 @@ func PodDeleted(c clientset.Interface, podNamespace, podName string) wait.Condit
 	}
 }
 
+// SyncInformerFactory starts informer and waits for caches to be synced
+func SyncInformerFactory(testCtx *TestContext) {
+	testCtx.InformerFactory.Start(testCtx.Ctx.Done())
+	testCtx.InformerFactory.WaitForCacheSync(testCtx.Ctx.Done())
+}
+
 // CleanupTest cleans related resources which were created during integration test
 func CleanupTest(t *testing.T, testCtx *TestContext) {
 	// Kill the scheduler.
@@ -408,11 +414,6 @@ func InitTestSchedulerWithOptions(
 	stopCh := make(chan struct{})
 	eventBroadcaster.StartRecordingToSink(stopCh)
 
-	testCtx.InformerFactory.Start(testCtx.Scheduler.StopEverything)
-	testCtx.InformerFactory.WaitForCacheSync(testCtx.Scheduler.StopEverything)
-
-	go testCtx.Scheduler.Run(testCtx.Ctx)
-
 	return testCtx
 }