Measure only CreateVM

Previously the benchmark was measuring both CreateVM and StopVM due to
Go's benchmark's limitation.

Since we mostly care CreateVM, this change removes StopVM from
the measurement.

Signed-off-by: Kazuyoshi Kato <[email protected]>

Author: kzys

runtime/Makefile

@@ -92,7 +92,7 @@ benchmark-%: logs
 		--workdir="/src/runtime" \
 		--init \
 		$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
-		"go test $(EXTRAGOARGS) -run IGNORE -bench '$(subst benchmark-,,$@)'"
+		"go test $(EXTRAGOARGS) -run '$(subst benchmark-,,$@)'"
 
 logs:
 	mkdir logs

runtime/benchmark_test.go

@@ -27,42 +27,44 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const benchmarkCount = 10
+
 func createAndStopVM(
 	ctx context.Context,
 	fcClient fccontrol.FirecrackerService,
 	request proto.CreateVMRequest,
-) error {
+) (time.Duration, error) {
 	uuid, err := uuid.NewV4()
 	if err != nil {
-		return err
+		return 0, err
 	}
 
 	request.VMID = uuid.String()
 
+	t0 := time.Now()
 	_, err = fcClient.CreateVM(ctx, &request)
 	if err != nil {
-		return err
+		return 0, err
 	}
+	elapsed := time.Since(t0)
 
 	_, err = fcClient.StopVM(ctx, &proto.StopVMRequest{VMID: request.VMID})
 	if err != nil {
-		return err
+		return 0, err
 	}
 
-	return nil
+	return elapsed, nil
 }
 
-func benchmarkCreateAndStopVM(b *testing.B, vcpuCount uint32, kernelArgs string, parallel int) {
-	require := require.New(b)
-
+func benchmarkCreateAndStopVM(t *testing.T, vcpuCount uint32, kernelArgs string) {
 	client, err := containerd.New(containerdSockPath, containerd.WithDefaultRuntime(firecrackerRuntime))
-	require.NoError(err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
+	require.NoError(t, err, "unable to create client to containerd service at %s, is containerd running?", containerdSockPath)
 	defer client.Close()
 
 	ctx := namespaces.WithNamespace(context.Background(), "default")
 
 	pluginClient, err := ttrpcutil.NewClient(containerdSockPath + ".ttrpc")
-	require.NoError(err, "failed to create ttrpc client")
+	require.NoError(t, err, "failed to create ttrpc client")
 
 	fcClient := fccontrol.NewFirecrackerClient(pluginClient.Client())
 	request := proto.CreateVMRequest{
@@ -72,52 +74,42 @@ func benchmarkCreateAndStopVM(b *testing.B, vcpuCount uint32, kernelArgs string,
 		},
 	}
 
-	ch := make(chan error, parallel)
-	for i := 0; i < parallel; i++ {
-		go func() {
-			var err error
-			defer func() {
-				ch <- err
-			}()
-
-			for i := 0; i < b.N; i++ {
-				ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-				defer cancel()
-
-				err = createAndStopVM(ctx, fcClient, request)
-				if err != nil {
-					return
-				}
-			}
-		}()
+	var samples []time.Duration
+	for i := 0; i < benchmarkCount; i++ {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+
+		elapsed, err := createAndStopVM(ctx, fcClient, request)
+		if err != nil {
+			return
+		}
+		samples = append(samples, elapsed)
 	}
 
-	// Make sure all goroutines are finished successfully.
-	for i := 0; i < parallel; i++ {
-		require.NoError(<-ch)
+	var average time.Duration
+	for _, x := range samples {
+		average += x
 	}
-}
+	average = time.Duration(int64(average) / int64(len(samples)))
 
-func BenchmarkCreateAndStopVM_Vcpu1_Isolated(b *testing.B) {
-	prepareIntegTest(b)
-	benchmarkCreateAndStopVM(b, 1, defaultRuntimeConfig.KernelArgs, 1)
-}
-func BenchmarkCreateAndStopVM_Vcpu5_Isolated(b *testing.B) {
-	prepareIntegTest(b)
-	benchmarkCreateAndStopVM(b, 5, defaultRuntimeConfig.KernelArgs, 1)
+	t.Logf("%s: avg:%v samples:%v", t.Name(), average, samples)
 }
 
-func BenchmarkCreateAndStopVM_Quiet_Isolated(b *testing.B) {
-	prepareIntegTest(b)
-	benchmarkCreateAndStopVM(
-		b,
-		1,
-		// Same as https://github.com/firecracker-microvm/firecracker-demo/blob/c22499567b63b4edd85e19ca9b0e9fa398b3300b/start-firecracker.sh#L9
-		"ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1",
-		1,
-	)
-}
-func BenchmarkCreateAndStopVM_Parallel10_Isolated(b *testing.B) {
-	prepareIntegTest(b)
-	benchmarkCreateAndStopVM(b, 1, defaultRuntimeConfig.KernelArgs, 10)
+func TestPerf_CreateVM(t *testing.T) {
+	prepareIntegTest(t)
+
+	t.Run("vcpu=1", func(subtest *testing.T) {
+		benchmarkCreateAndStopVM(subtest, 1, defaultRuntimeConfig.KernelArgs)
+	})
+	t.Run("vcpu=5", func(subtest *testing.T) {
+		benchmarkCreateAndStopVM(subtest, 1, defaultRuntimeConfig.KernelArgs)
+	})
+	t.Run("firecracker-demo", func(subtest *testing.T) {
+		benchmarkCreateAndStopVM(
+			subtest,
+			1,
+			// Same as https://github.com/firecracker-microvm/firecracker-demo/blob/c22499567b63b4edd85e19ca9b0e9fa398b3300b/start-firecracker.sh#L9
+			"ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1",
+		)
+	})
 }