Merge pull request kubernetes#129441 from serathius/watchcache-benchmark

Improve benchmark to handle multiple dimensions

Author: k8s-ci-robot

staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_test.go

@@ -547,23 +547,102 @@ func (c *createWrapper) Create(ctx context.Context, key string, obj, out runtime
 	})
 }
 
-func BenchmarkStoreListCreate(b *testing.B) {
+func BenchmarkStoreCreateList(b *testing.B) {
 	klog.SetLogger(logr.Discard())
-	b.Run("RV=NotOlderThan", func(b *testing.B) {
-		ctx, cacher, _, terminate := testSetupWithEtcdServer(b)
-		b.Cleanup(terminate)
-		storagetesting.RunBenchmarkStoreListCreate(ctx, b, cacher, metav1.ResourceVersionMatchNotOlderThan)
-	})
-	b.Run("RV=ExactMatch", func(b *testing.B) {
-		ctx, cacher, _, terminate := testSetupWithEtcdServer(b)
-		b.Cleanup(terminate)
-		storagetesting.RunBenchmarkStoreListCreate(ctx, b, cacher, metav1.ResourceVersionMatchExact)
-	})
+	storeOptions := []struct {
+		name         string
+		btreeEnabled bool
+	}{
+		{
+			name:         "Btree",
+			btreeEnabled: true,
+		},
+		{
+			name:         "Map",
+			btreeEnabled: false,
+		},
+	}
+	for _, store := range storeOptions {
+		b.Run(fmt.Sprintf("Store=%s", store.name), func(b *testing.B) {
+			featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.BtreeWatchCache, store.btreeEnabled)
+			for _, rvm := range []metav1.ResourceVersionMatch{metav1.ResourceVersionMatchNotOlderThan, metav1.ResourceVersionMatchExact} {
+				b.Run(fmt.Sprintf("RV=%s", rvm), func(b *testing.B) {
+					for _, useIndex := range []bool{true, false} {
+						b.Run(fmt.Sprintf("Indexed=%v", useIndex), func(b *testing.B) {
+							opts := []setupOption{}
+							if useIndex {
+								opts = append(opts, withSpecNodeNameIndexerFuncs)
+							}
+							ctx, cacher, _, terminate := testSetupWithEtcdServer(b, opts...)
+							b.Cleanup(terminate)
+							storagetesting.RunBenchmarkStoreListCreate(ctx, b, cacher, rvm)
+						})
+					}
+				})
+			}
+		})
+	}
 }
 
 func BenchmarkStoreList(b *testing.B) {
 	klog.SetLogger(logr.Discard())
-	ctx, cacher, _, terminate := testSetupWithEtcdServer(b, withSpecNodeNameIndexerFuncs)
-	b.Cleanup(terminate)
-	storagetesting.RunBenchmarkStoreList(ctx, b, cacher)
+	// Based on https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md
+	dimensions := []struct {
+		namespaceCount       int
+		podPerNamespaceCount int
+		nodeCount            int
+	}{
+		{
+			namespaceCount:       10_000,
+			podPerNamespaceCount: 15,
+			nodeCount:            5_000,
+		},
+		{
+			namespaceCount:       50,
+			podPerNamespaceCount: 3_000,
+			nodeCount:            5_000,
+		},
+		{
+			namespaceCount:       100,
+			podPerNamespaceCount: 1_100,
+			nodeCount:            1000,
+		},
+	}
+	for _, dims := range dimensions {
+		b.Run(fmt.Sprintf("Namespaces=%d/Pods=%d/Nodes=%d", dims.namespaceCount, dims.namespaceCount*dims.podPerNamespaceCount, dims.nodeCount), func(b *testing.B) {
+			data := storagetesting.PrepareBenchchmarkData(dims.namespaceCount, dims.podPerNamespaceCount, dims.nodeCount)
+			storeOptions := []struct {
+				name         string
+				btreeEnabled bool
+			}{
+				{
+					name:         "Btree",
+					btreeEnabled: true,
+				},
+				{
+					name:         "Map",
+					btreeEnabled: false,
+				},
+			}
+			for _, store := range storeOptions {
+				b.Run(fmt.Sprintf("Store=%s", store.name), func(b *testing.B) {
+					featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.BtreeWatchCache, store.btreeEnabled)
+					ctx, cacher, _, terminate := testSetupWithEtcdServer(b, withSpecNodeNameIndexerFuncs)
+					b.Cleanup(terminate)
+					var out example.Pod
+					for _, pod := range data.Pods {
+						err := cacher.Create(ctx, computePodKey(pod), pod, &out, 0)
+						if err != nil {
+							b.Fatal(err)
+						}
+					}
+					for _, useIndex := range []bool{true, false} {
+						b.Run(fmt.Sprintf("Indexed=%v", useIndex), func(b *testing.B) {
+							storagetesting.RunBenchmarkStoreList(ctx, b, cacher, data, useIndex)
+						})
+					}
+				})
+			}
+		})
+	}
 }

staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go

@@ -952,6 +952,45 @@ func BenchmarkStoreListCreate(b *testing.B) {
 }
 
 func BenchmarkStoreList(b *testing.B) {
-	ctx, store, _ := testSetup(b)
-	storagetesting.RunBenchmarkStoreList(ctx, b, store)
+	klog.SetLogger(logr.Discard())
+	// Based on https://github.com/kubernetes/community/blob/master/sig-scalability/configs-and-limits/thresholds.md
+	dimensions := []struct {
+		namespaceCount       int
+		podPerNamespaceCount int
+		nodeCount            int
+	}{
+		{
+			namespaceCount:       10_000,
+			podPerNamespaceCount: 15,
+			nodeCount:            5_000,
+		},
+		{
+			namespaceCount:       50,
+			podPerNamespaceCount: 3_000,
+			nodeCount:            5_000,
+		},
+		{
+			namespaceCount:       100,
+			podPerNamespaceCount: 1_100,
+			nodeCount:            1000,
+		},
+	}
+	for _, dims := range dimensions {
+		b.Run(fmt.Sprintf("Namespaces=%d/Pods=%d/Nodes=%d", dims.namespaceCount, dims.namespaceCount*dims.podPerNamespaceCount, dims.nodeCount), func(b *testing.B) {
+			data := storagetesting.PrepareBenchchmarkData(dims.namespaceCount, dims.podPerNamespaceCount, dims.nodeCount)
+			ctx, store, _ := testSetup(b)
+			var out example.Pod
+			for _, pod := range data.Pods {
+				err := store.Create(ctx, computePodKey(pod), pod, &out, 0)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+			storagetesting.RunBenchmarkStoreList(ctx, b, store, data, false)
+		})
+	}
+}
+
+func computePodKey(obj *example.Pod) string {
+	return fmt.Sprintf("/pods/%s/%s", obj.Namespace, obj.Name)
 }