Merge pull request #1271 from input-output-hk/jpraynaud/1220-benchmark-aggregator-performances

Benchmark aggregator performances

Author: jpraynaud

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-end-to-end"
-version = "0.2.16"
+version = "0.2.17"
 authors = { workspace = true }
 edition = { workspace = true }
 documentation = { workspace = true }

mithril-test-lab/mithril-end-to-end/Cargo.toml

@@ -34,7 +34,7 @@ make build
 ./mithril-end-to-end --db-directory db/ --bin-directory ../../target/release
 ```
 
-## Build and run the stress tester
+## Build and run an aggregator stress test
 
 ```bash
 # Build
@@ -50,6 +50,16 @@ make build
 ./load-aggregator -vvv --cardano-cli-path script/mock-cardano-cli --aggregator-dir ../../target/release --num-signers=100 --num-clients=200
 ```
 
+## Benchmark aggregator performances
+
+```bash
+# Build the load aggregator tool
+make build
+
+# Run a benchmark for [10, 20, 30 40, 50] signers x [100, 200, 300] clients
+MIN_SIGNERS=10 MAX_SIGNERS=50 STEP_SIGNERS=10 MIN_CLIENTS=100 MAX_CLIENTS=300 STEP_CLIENTS=100 ./benchmark-aggregator.sh
+```
+
 ## Launch a monitor
 
 ```bash

mithril-test-lab/mithril-end-to-end/README.md

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+set -e
+
+# Debug mode
+if [ -v DEBUG ]; then
+    set -x
+fi
+
+# Check if all env vars are set 
+if [ -z "${MIN_SIGNERS}" ]; then
+    MIN_SIGNERS=1
+fi
+
+if [ -z "${MAX_SIGNERS}" ]; then
+    echo "Missing environment variable: MAX_SIGNERS" >/dev/stderr
+    exit 1
+fi
+
+if [ -z "${STEP_SIGNERS}" ]; then
+    echo "Missing environment variable: STEP_SIGNERS" >/dev/stderr
+    exit 1
+fi
+
+if [ -z "${MIN_CLIENTS}" ]; then
+    MIN_CLIENTS=0
+fi
+
+if [ -z "${MAX_CLIENTS}" ]; then
+    echo "Missing environment variable: MAX_CLIENTS" >/dev/stderr
+    exit 1
+fi
+
+if [ -z "${STEP_CLIENTS}" ]; then
+    echo "Missing environment variable: STEP_CLIENTS" >/dev/stderr
+    exit 1
+fi
+
+if [ -z "${AGGREGATOR_DIR}" ]; then
+    AGGREGATOR_DIR=../../target/release
+    echo "Using the default AGGREGATOR_DIR: $AGGREGATOR_DIR"
+fi
+
+if [ -z "${OUT_FILE}" ]; then
+    OUT_FILE="benchmark/benchmark-data-[$MIN_SIGNERS,$MAX_SIGNERS;$STEP_SIGNERS]Sx[$MIN_CLIENTS,$MAX_CLIENTS;$STEP_CLIENTS]C-$(date +%Y-%m-%d-%H-%M-%S).tsv"
+    echo "Using the default OUT_FILE: $OUT_FILE"
+fi
+mkdir -p $(dirname "$OUT_FILE")
+OUT_FILE_TMP=${OUT_FILE}.tmp
+
+# Run stress test
+RUN_STRESS_TEST() {
+    NUM_SIGNERS=$1
+    NUM_SIGNERS=$(( NUM_SIGNERS > 0 ? NUM_SIGNERS : 1 ))
+    NUM_CLIENTS=$2
+    OUT_FILE=$3
+    INDEX_RUN=$4
+    TOTAL_RUN=$5
+    echo ">> [#$INDEX_RUN/$TOTAL_RUN] Running stress test with $NUM_SIGNERS signers and $NUM_CLIENTS clients"
+    COMMAND="./load-aggregator --cardano-cli-path ./script/mock-cardano-cli --aggregator-dir $AGGREGATOR_DIR --num-signers=$NUM_SIGNERS --num-clients=$NUM_CLIENTS"
+    if $($COMMAND >> $OUT_FILE_TMP) ; then
+        echo ">>>> Success"
+    else
+        echo "signers	clients	phase	duration/ms" >> $OUT_FILE_TMP
+        echo "$NUM_SIGNERS	$NUM_CLIENTS	failure	10" >> $OUT_FILE_TMP
+        echo ">>>> Failure"
+    fi
+    if [[ $INDEX_RUN -gt 1 ]] ; then
+        tail -n +2 $OUT_FILE_TMP >> $OUT_FILE
+    else
+        cat $OUT_FILE_TMP >> $OUT_FILE
+    fi
+    rm -f $OUT_FILE_TMP
+    echo ""
+}
+
+# Run aggregator benchmark over a range of signers and clients
+SIGNERS_RANGE=$(seq -s ' ' $MIN_SIGNERS $STEP_SIGNERS $MAX_SIGNERS)
+CLIENTS_RANGE=$(seq -s ' ' $MIN_CLIENTS $STEP_CLIENTS $MAX_CLIENTS)
+SIGNERS_RANGE_LENGTH=$(echo $SIGNERS_RANGE | grep -o " " | wc -l)
+CLIENTS_RANGE_LENGTH=$(echo $CLIENTS_RANGE | grep -o " " | wc -l)
+echo ""
+echo "Run aggregator benchmark over the range:"
+echo ">> Signers: [$SIGNERS_RANGE]"
+echo ">> Clients: [$CLIENTS_RANGE]"
+echo ""
+
+INDEX_RUN=1
+TOTAL_RUN=$(( $SIGNERS_RANGE_LENGTH * CLIENTS_RANGE_LENGTH ))
+for NUM_SIGNERS in $SIGNERS_RANGE; do 
+    for NUM_CLIENTS in $CLIENTS_RANGE; do
+        RUN_STRESS_TEST $NUM_SIGNERS $NUM_CLIENTS $OUT_FILE $INDEX_RUN $TOTAL_RUN
+        INDEX_RUN=$(( ${INDEX_RUN} + 1))
+    done 
+done 
+
+# Clean directory
+rm -f $OUT_FILE_TMP
+

mithril-test-lab/mithril-end-to-end/benchmark-aggregator.sh

@@ -157,6 +157,11 @@ impl Reporter {
     pub fn stop(&mut self) {
         match &self.current_timing {
             Some((phase, instant)) => {
+                let phase = if self.number_of_clients == 0 {
+                    format!("{phase} - without clients")
+                } else {
+                    format!("{phase} - with clients")
+                };
                 let timing = Timing {
                     phase: phase.clone(),
                     duration: instant.elapsed(),
@@ -170,11 +175,15 @@ impl Reporter {
     }
 
     pub fn print_report(&self) {
-        println!("number_of_signers\t{}", self.number_of_signers);
-        println!("number_of_clients\t{}", self.number_of_clients);
-        println!("phase\tduration/ms");
+        println!("signers\tclients\tphase\tduration/ms");
         for t in &self.timings {
-            println!("{}\t{}", t.phase, t.duration.as_millis());
+            println!(
+                "{}\t{}\t{}\t{}",
+                self.number_of_signers,
+                self.number_of_clients,
+                t.phase,
+                t.duration.as_millis()
+            );
         }
     }
 }