Fix TestCreateVM_Isolated stability by creating fresh ctx/fcClient per subtest

Signed-off-by: Arjun Raja Yogidas <arjunry@amazon.com>

Author: coderbirju

.buildkite/pipeline.yml

@@ -75,7 +75,7 @@ steps:
     artifact_paths:
       - "runtime/logs/*"
     command:
-      - make -C runtime integ-test FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime
+      - make -C runtime integ-test-TestCreateVM_Isolated FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime
     retry:
       automatic:
         - exit_status: "*"

runtime/service_integ_test.go

@@ -2250,17 +2250,12 @@ func TestCreateVM_Isolated(t *testing.T) {
 	require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", integtest.ContainerdSockPath)
 	defer client.Close()
 
-	ctx := namespaces.WithNamespace(context.Background(), "default")
-
-	fcClient, err := integtest.NewFCControlClient(integtest.ContainerdSockPath)
-	require.NoError(t, err, "failed to create ttrpc client")
-
 	kernelArgs := integtest.DefaultRuntimeConfig.KernelArgs
 
 	type subtest struct {
 		name                    string
 		request                 proto.CreateVMRequest
-		validate                func(*testing.T, error)
+		validate                func(*testing.T, context.Context, error)
 		validateUsesFindProcess bool
 		stopVM                  bool
 	}
@@ -2269,7 +2264,7 @@ func TestCreateVM_Isolated(t *testing.T) {
 		{
 			name:    "Happy Case",
 			request: proto.CreateVMRequest{},
-			validate: func(t *testing.T, err error) {
+			validate: func(t *testing.T, ctx context.Context, err error) {
 				require.NoError(t, err)
 			},
 			stopVM: true,
@@ -2283,7 +2278,7 @@ func TestCreateVM_Isolated(t *testing.T) {
 					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
 			},
-			validate: func(t *testing.T, err error) {
+			validate: func(t *testing.T, ctx context.Context, err error) {
 				require.NotNil(t, err, "expected an error but did not receive any")
 				time.Sleep(5 * time.Second)
 				firecrackerProcesses, err := findProcess(ctx, findFirecracker)
@@ -2300,11 +2295,12 @@ func TestCreateVM_Isolated(t *testing.T) {
 				RootDrive: &proto.FirecrackerRootDrive{
 					HostPath: "/var/lib/firecracker-containerd/runtime/rootfs-debug.img",
 				},
+				TimeoutSeconds: 5,
 			},
-			validate: func(t *testing.T, err error) {
+			validate: func(t *testing.T, ctx context.Context, err error) {
 				require.Error(t, err)
 				assert.Equal(t, codes.DeadlineExceeded, status.Code(err))
-				assert.Contains(t, err.Error(), "didn't start within 20s")
+				assert.Contains(t, err.Error(), "didn't start within 5s")
 			},
 			stopVM: true,
 		},
@@ -2317,7 +2313,7 @@ func TestCreateVM_Isolated(t *testing.T) {
 				},
 				TimeoutSeconds: 60,
 			},
-			validate: func(t *testing.T, err error) {
+			validate: func(t *testing.T, ctx context.Context, err error) {
 				require.NoError(t, err)
 			},
 			stopVM: true,
@@ -2331,6 +2327,15 @@ func TestCreateVM_Isolated(t *testing.T) {
 		if !s.validateUsesFindProcess {
 			t.Parallel()
 		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		ctx = namespaces.WithNamespace(ctx, "default")
+
+		fcClient, err := integtest.NewFCControlClient(integtest.ContainerdSockPath)
+		require.NoError(t, err, "failed to create ttrpc client")
+		defer fcClient.Close()
+
 		vmID := testNameToVMID(t.Name())
 
 		tempDir := t.TempDir()
@@ -2349,7 +2354,7 @@ func TestCreateVM_Isolated(t *testing.T) {
 		requireNonEmptyFifo(t, metricsFile)
 
 		// Some test cases are expected to have an error, some are not.
-		s.validate(t, createVMErr)
+		s.validate(t, ctx, createVMErr)
 
 		if createVMErr == nil && s.stopVM {
 			// Ensure the response fields are populated correctly
@@ -2360,6 +2365,8 @@ func TestCreateVM_Isolated(t *testing.T) {
 				TimeoutSeconds: 30,
 			})
 			require.Equal(t, status.Code(err), codes.OK)
+
+			time.Sleep(500 * time.Millisecond)
 		}
 	}