Add support for repetitions to benchmarkset

Matt Pryor · Matt Pryor · commit dab6e181597a · 2022-10-04T12:20:31.000+01:00
diff --git a/README.md b/README.md
@@ -85,8 +85,12 @@ spec:
     kind: IPerf
     spec:
       duration: 30
+  # The number of repetitions to run for each permutation
+  # Defaults to 1 if not given
+  repetitions: 1
   # Defines the permutations for the set
   # Each permutation is merged into the spec of the template
+  # If not given, a single permutation consisting of the given template is used
   permutations:
     # Permutations are calculated as a cross-product of the specified names and values
     product:
@@ -138,6 +142,10 @@ metadata:
 spec:
   # The MPI transport to use - one of TCP, RDMA
   transport: TCP
+  # Controls the maximum message length
+  # Selected lengths, in bytes, will be 0, 1, 2, 4, 8, 16, ..., 2^maxlog
+  # Defaults to 22 if not given, meaning the maximum message size will be 4MB
+  maxlog: 22
 ```
 
 ### OpenFOAM
diff --git a/examples/accelerated/iperf-set.yaml b/examples/accelerated/iperf-set.yaml
@@ -14,6 +14,7 @@ spec:
         limits:
           mellanox.com/mlnxnet: 1
       duration: 30
+  repetitions: 5
   permutations:
     product:
       streams: [1, 2, 4, 8, 16, 32, 64, 80, 128]
diff --git a/examples/accelerated/mpi-pingpong-set.yaml b/examples/accelerated/mpi-pingpong-set.yaml
@@ -13,6 +13,8 @@ spec:
       resources:
         limits:
           mellanox.com/mlnxnet: 1
+      maxlog: 25
+  repetitions: 5
   permutations:
     product:
       transport: [TCP, RDMA]
diff --git a/examples/accelerated/rdma-bw-set.yaml b/examples/accelerated/rdma-bw-set.yaml
@@ -17,6 +17,7 @@ spec:
       iterations: 25
       extraArgs:
         - --tclass=96
+  repetitions: 5
   permutations:
     product:
       mode: [read, write]
diff --git a/examples/accelerated/rdma-lat-set.yaml b/examples/accelerated/rdma-lat-set.yaml
@@ -0,0 +1,22 @@
+apiVersion: perftest.stackhpc.com/v1alpha1
+kind: BenchmarkSet
+metadata:
+  name: rdma-latency-accelerated
+spec:
+  template:
+    apiVersion: perftest.stackhpc.com/v1alpha1
+    kind: RDMALatency
+    spec:
+      imagePullPolicy: Always
+      hostNetwork: false
+      networkName: mlnx-net
+      resources:
+        limits:
+          mellanox.com/mlnxnet: 1
+      iterations: 1000
+      extraArgs:
+        - --tclass=96
+  repetitions: 5
+  permutations:
+    product:
+      mode: [read, write]
diff --git a/examples/accelerated/rdma-lat.yaml b/examples/accelerated/rdma-lat.yaml
diff --git a/examples/cni/iperf-set.yaml b/examples/cni/iperf-set.yaml
@@ -9,8 +9,9 @@ spec:
     spec:
       imagePullPolicy: Always
       hostNetwork: false
-      mtu: 9000
+      # mtu: 9000
       duration: 30
+  repetitions: 5
   permutations:
     product:
       streams: [1, 2, 4, 8, 16, 32, 64, 128]
diff --git a/examples/cni/mpi-pingpong-set.yaml b/examples/cni/mpi-pingpong-set.yaml
@@ -0,0 +1,15 @@
+apiVersion: perftest.stackhpc.com/v1alpha1
+kind: BenchmarkSet
+metadata:
+  name: mpi-pingpong-cni
+spec:
+  template:
+    apiVersion: perftest.stackhpc.com/v1alpha1
+    kind: MPIPingPong
+    spec:
+      imagePullPolicy: Always
+      hostNetwork: false
+      # mtu: 9000
+      transport: TCP
+      maxlog: 25
+  repetitions: 5
diff --git a/examples/cni/mpi-pingpong.yaml b/examples/cni/mpi-pingpong.yaml
diff --git a/examples/cni/openfoam-set.yaml b/examples/cni/openfoam-set.yaml
@@ -9,7 +9,7 @@ spec:
     spec:
       imagePullPolicy: Always
       hostNetwork: false
-      mtu: 9000
+      # mtu: 9000
       transport: TCP
       problemSize: S
       numProcs: 48
diff --git a/examples/hostnet/iperf-set.yaml b/examples/hostnet/iperf-set.yaml
@@ -10,6 +10,7 @@ spec:
       imagePullPolicy: Always
       hostNetwork: true
       duration: 30
+  repetitions: 5
   permutations:
     product:
       streams: [1, 2, 4, 8, 16, 32, 64, 128]
diff --git a/examples/hostnet/mpi-pingpong-set.yaml b/examples/hostnet/mpi-pingpong-set.yaml
@@ -0,0 +1,14 @@
+apiVersion: perftest.stackhpc.com/v1alpha1
+kind: BenchmarkSet
+metadata:
+  name: mpi-pingpong-hostnet
+spec:
+  template:
+    apiVersion: perftest.stackhpc.com/v1alpha1
+    kind: MPIPingPong
+    spec:
+      imagePullPolicy: Always
+      hostNetwork: true
+      transport: TCP
+      maxlog: 25
+  repetitions: 5
diff --git a/examples/hostnet/mpi-pingpong.yaml b/examples/hostnet/mpi-pingpong.yaml
diff --git a/examples/macvlan/iperf-set.yaml b/examples/macvlan/iperf-set.yaml
@@ -14,6 +14,7 @@ spec:
         limits:
           rdma/rdma_shared_device_a: 1
       duration: 30
+  repetitions: 5
   permutations:
     product:
       streams: [1, 2, 4, 8, 16, 32, 64, 80, 128]
diff --git a/examples/macvlan/mpi-pingpong-set.yaml b/examples/macvlan/mpi-pingpong-set.yaml
@@ -13,6 +13,8 @@ spec:
       resources:
         limits:
           rdma/rdma_shared_device_a: 1
+      maxlog: 25
+  repetitions: 5
   permutations:
     product:
       transport: [TCP, RDMA]
diff --git a/examples/macvlan/rdma-bw-set.yaml b/examples/macvlan/rdma-bw-set.yaml
@@ -17,6 +17,7 @@ spec:
       iterations: 25
       extraArgs:
         - --tclass=96
+  repetitions: 5
   permutations:
     product:
       mode: [read, write]
diff --git a/examples/macvlan/rdma-bw-write.yaml b/examples/macvlan/rdma-bw-write.yaml
diff --git a/examples/macvlan/rdma-lat-set.yaml b/examples/macvlan/rdma-lat-set.yaml
@@ -16,6 +16,7 @@ spec:
       iterations: 1000
       extraArgs:
         - --tclass=96
+  repetitions: 5
   permutations:
     product:
       mode: [read, write]
diff --git a/examples/sriov/iperf-set.yaml b/examples/sriov/iperf-set.yaml
@@ -14,6 +14,7 @@ spec:
         limits:
           nvidia.com/mlnxnet: 1
       duration: 30
+  repetitions: 5
   permutations:
     product:
       streams: [1, 2, 4, 8, 16, 32, 64, 80, 128]
diff --git a/examples/sriov/mpi-pingpong-set.yaml b/examples/sriov/mpi-pingpong-set.yaml
@@ -13,6 +13,8 @@ spec:
       resources:
         limits:
           nvidia.com/mlnxnet: 1
+      maxlog: 25
+  repetitions: 5
   permutations:
     product:
       transport: [TCP, RDMA]
diff --git a/examples/sriov/rdma-bw-set.yaml b/examples/sriov/rdma-bw-set.yaml
@@ -17,6 +17,7 @@ spec:
       iterations: 25
       extraArgs:
         - --tclass=96
+  repetitions: 5
   permutations:
     product:
       mode: [read, write]
diff --git a/examples/sriov/rdma-lat-set.yaml b/examples/sriov/rdma-lat-set.yaml
@@ -16,6 +16,7 @@ spec:
       iterations: 1000
       extraArgs:
         - --tclass=96
+  repetitions: 5
   permutations:
     product:
       mode: [read, write]
diff --git a/python/perftest/models/v1alpha1/pingpong.py b/python/perftest/models/v1alpha1/pingpong.py
@@ -56,6 +56,13 @@ class MPIPingPongSpec(base.BenchmarkSpec):
         MPITransport.TCP,
         description = "The transport to use for the benchmark."
     )
+    maxlog: schema.conint(gt = 0) = Field(
+        22,  # This means up to 4MB messages
+        description = (
+            "Controls the maximum message length for the benchmark. "
+            "Selected lengths, in bytes, will be 0, 1, 2, 4, 8, 16, ..., 2^maxlog."
+        )
+    )
 
 
 class MPIPingPongResult(schema.BaseModel):
@@ -149,6 +156,11 @@ class MPIPingPong(
             "type": "string",
             "jsonPath": ".spec.transport",
         },
+        {
+            "name": "Maxlog",
+            "type": "integer",
+            "jsonPath": ".spec.maxlog",
+        },
         {
             "name": "Status",
             "type": "string",
diff --git a/python/perftest/models/v1alpha1/set.py b/python/perftest/models/v1alpha1/set.py
@@ -31,6 +31,8 @@ class BenchmarkSetTemplate(schema.BaseModel):
 class BenchmarkSetPermutations(schema.BaseModel):
     """
     Defines the permutations to use for the benchmarks in the set.
+
+    If no permutations are defined, a single empty permutation is reported.
     """
     product: schema.Dict[str, t.List[schema.Any]] = Field(
         default_factory = dict,
@@ -52,21 +54,23 @@ def get_count(self) -> int:
             if self.product
             else 0
         )
-        return product_count + len(self.explicit)
+        return max(product_count + len(self.explicit), 1)
 
     def get_permutations(self) -> t.Iterable[t.Dict[str, t.Any]]:
         """
         Returns all the permutations for the benchmark set.
         """
-        if self.product:
+        if self.product or self.explicit:
             yield from (
                 dict(permutation)
                 for permutation in itertools.product(*(
                     [(k, v) for v in vs]
                     for k, vs in self.product.items()
                 ))
             )
-        yield from self.explicit
+            yield from self.explicit
+        else:
+            yield dict()
 
 
 class BenchmarkSetSpec(schema.BaseModel):
@@ -77,8 +81,12 @@ class BenchmarkSetSpec(schema.BaseModel):
         ...,
         description = "The template to use for the benchmarks."
     )
+    repetitions: schema.conint(gt = 0) = Field(
+        1,
+        description = "The number of repetitions to do for each permutation."
+    )
     permutations: BenchmarkSetPermutations = Field(
-        default_factory = dict,
+        default_factory = BenchmarkSetPermutations,
         description = "The permutations to use for the benchmarks."
     )
 
@@ -87,6 +95,10 @@ class BenchmarkSetStatus(schema.BaseModel):
     """
     Represents the status of a benchmark set.
     """
+    permutation_count: t.Optional[schema.conint(ge = 0)] = Field(
+        None,
+        description = "The number of permutations in the set."
+    )
     count: t.Optional[schema.conint(ge = 0)] = Field(
         None,
         description = "The number of benchmarks in the set."
@@ -116,19 +128,29 @@ class BenchmarkSet(
     CustomResource,
     subresources = {"status": {}},
     printer_columns = [
+        {
+            "name": "Permutations",
+            "type": "integer",
+            "jsonPath": ".status.permutationCount",
+        },
+        {
+            "name": "Repetitions",
+            "type": "integer",
+            "jsonPath": ".spec.repetitions",
+        },
         {
             "name": "Count",
-            "type": "string",
+            "type": "integer",
             "jsonPath": ".status.count",
         },
         {
             "name": "Succeeded",
-            "type": "string",
+            "type": "integer",
             "jsonPath": ".status.succeeded",
         },
         {
             "name": "Failed",
-            "type": "string",
+            "type": "integer",
             "jsonPath": ".status.failed",
         },
         {
diff --git a/python/perftest/operator.py b/python/perftest/operator.py
diff --git a/python/perftest/templates/mpipingpong.yaml.j2 b/python/perftest/templates/mpipingpong.yaml.j2
