updated performance comparison logic

Author: joviegas

test/http-client-benchmarks/README.md

@@ -1,38 +1,55 @@
-# Sdk Benchmark Harness
+````markdown
+# HTTP Client Benchmark Harness
 
-
-This module contains sdk benchmark harness using [JMH].
+This module contains HTTP client benchmark harness using [JMH].
 
 Each benchmark class has a set of default
-JMH configurations tailored to SDK's build job and you might need to
+JMH configurations tailored to HTTP client performance testing and you might need to
 adjust them based on your test environment such as increasing warmup iterations
 or measurement time in order to get more reliable data.
 
 There are three ways to run benchmarks.
 
 - Using the executable JAR (Preferred usage per JMH site)
 ```bash
-mvn clean install -P quick -pl :sdk-benchmarks --am
+mvn clean install -P quick -pl :http-client-benchmarks --am
 
 # Run specific benchmark
-java -jar target/benchmarks.jar ApacheHttpClientBenchmark
+java -jar target/benchmarks.jar Apache5Benchmark
 
 # Run all benchmarks: 3 warm up iterations, 3 benchmark iterations, 1 fork
 java -jar target/benchmarks.jar -wi 3 -i 3 -f 1
 ```
 
-- Using`mvn exec:exec` commands to invoke `UnifiedBenchmarkRunner` main method
+- Using `mvn exec:exec` commands to invoke `UnifiedBenchmarkRunner` main method
 ```bash
-   mvn clean install -P quick -pl :sdk-benchmarks --am
-   mvn clean install -pl :bom-internal
-   cd test/sdk-benchmarks
-   mvn exec:exec
+mvn clean install -P quick -pl :http-client-benchmarks --am
+mvn clean install -pl :bom-internal
+cd test/http-client-benchmarks
+mvn exec:exec
 ```   
 
 - From IDE
 
-  You can run the main method within each Benchmark class from your IDE. If you are using Eclipse, you might need to
-  set up build configurations for JMH annotation, please check [JMH]. Note that the benchmark result from IDE 
-  might not be as reliable as the above approaches and is generally not recommended.
-  
+  You can run the main method within `UnifiedBenchmarkRunner` class from your IDE to execute all benchmark comparisons, or run individual benchmark classes like `Apache4Benchmark` and `Apache5Benchmark`. If you are using Eclipse, you might need to set up build configurations for JMH annotation, please check [JMH]. Note that the benchmark result from IDE might not be as reliable as the above approaches and is generally not recommended.
+
+## UnifiedBenchmarkRunner
+
+The `UnifiedBenchmarkRunner` provides a comprehensive comparison between different HTTP client implementations:
+
+- **Apache4**: Apache HttpClient 4.x baseline
+- **Apache5-Platform**: Apache HttpClient 5.x with platform threads 
+- **Apache5-Virtual**: Apache HttpClient 5.x with virtual threads
+
+The runner executes all benchmark variations, publishes metrics to CloudWatch, and generates a detailed comparison report showing performance improvements between implementations.
+
+## Benchmark Operations
+
+Each benchmark implementation tests the following operations:
+- `simpleGet`: Single-threaded GET operations
+- `simplePut`: Single-threaded PUT operations 
+- `multiThreadedGet`: Multi-threaded GET operations (10 threads)
+- `multiThreadedPut`: Multi-threaded PUT operations (10 threads)
+
 [JMH]: http://openjdk.java.net/projects/code-tools/jmh/
+````

test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/UnifiedBenchmarkRunner.java

@@ -39,10 +39,7 @@ public class UnifiedBenchmarkRunner {
     public static void main(String[] args) throws Exception {
         logger.info("Starting unified benchmark comparison");
 
-        // Generate unique run ID with timestamp
         String runId = Instant.now().toString();
-
-        // Initialize CloudWatch publisher
         CloudWatchMetricsPublisher publisher = new CloudWatchMetricsPublisher(
             Region.US_WEST_2,
             "S3-HTTP-Client-Comparison"
@@ -63,13 +60,20 @@ public static void main(String[] args) throws Exception {
             logger.info("Running Apache5 with virtual threads...");
             allResults.addAll(runBenchmark("Apache5-Virtual", Apache5Benchmark.class, "virtual"));
 
-            // Publish all results to CloudWatch with synchronized timestamp
+            // Debug: Print all results to understand the structure
+            logger.info("All benchmark results:");
+            for (BenchmarkResult result : allResults) {
+                logger.info(String.format("Client: %s, Benchmark: %s, Throughput: %.2f",
+                                          result.getClientType(), result.getBenchmarkName(), result.getThroughput()));
+            }
+
+            // Publish results to CloudWatch
             logger.info("Publishing results to CloudWatch...");
             for (BenchmarkResult result : allResults) {
                 publisher.publishBenchmarkResult(result, runId);
             }
 
-            // Print comparison report to console
+            // Print comparison report
             printComparisonReport(allResults);
 
             logger.info("\nBenchmark complete! CloudWatch metrics published with run ID: " + runId);
@@ -89,7 +93,6 @@ private static List<BenchmarkResult> runBenchmark(String clientType,
             .warmupIterations(2)
             .measurementIterations(3);
 
-        // Add executor type parameter for Apache5
         if (executorType != null) {
             optBuilder.param("executorType", executorType);
         }
@@ -105,13 +108,20 @@ private static List<BenchmarkResult> runBenchmark(String clientType,
     private static BenchmarkResult convertToBenchmarkResult(String clientType,
                                                             RunResult runResult) {
         String fullBenchmarkName = runResult.getPrimaryResult().getLabel();
-        String benchmarkName = fullBenchmarkName.substring(fullBenchmarkName.lastIndexOf('.') + 1);
+
+        // Extract just the method name (everything after the last dot)
+        String benchmarkName = fullBenchmarkName;
+        if (fullBenchmarkName.contains(".")) {
+            benchmarkName = fullBenchmarkName.substring(fullBenchmarkName.lastIndexOf('.') + 1);
+        }
+
+        // Log for debugging
+        logger.info(String.format("Converting: %s -> %s", fullBenchmarkName, benchmarkName));
 
         double throughput = runResult.getPrimaryResult().getScore();
-        double avgLatency = 1000.0 / throughput; // Convert to ms
-        double p99Latency = avgLatency * 1.5; // Estimate
+        double avgLatency = 1000.0 / throughput;
+        double p99Latency = avgLatency * 1.5;
 
-        // Determine thread count from benchmark name
         int threadCount = benchmarkName.contains("multiThreaded") ? 10 : 1;
 
         return new BenchmarkResult(
@@ -139,25 +149,51 @@ private static void printComparisonReport(List<BenchmarkResult> results) {
                                                                        )
                                                                    ));
 
-        // Print results table
-        System.out.printf("%-20s %-15s %-15s %-15s %-15s%n",
-                          "Client Type", "Simple GET", "Simple PUT", "MT GET (10t)", "MT PUT (10t)");
-        System.out.println("-".repeat(80));
+        // Print the structure we actually have , to view it on console
+        System.out.println("Available results structure:");
+        for (Map.Entry<String, Map<String, BenchmarkResult>> entry : grouped.entrySet()) {
+            System.out.println("Client: " + entry.getKey());
+            for (String benchmarkName : entry.getValue().keySet()) {
+                System.out.println("  - " + benchmarkName);
+            }
+        }
+        System.out.println();
+
+        // Get all unique benchmark names across all clients
+        Set<String> allBenchmarkNames = grouped.values().stream()
+                                               .flatMap(map -> map.keySet().stream())
+                                               .collect(Collectors.toSet());
+
+        // Print results table with dynamic columns
+        System.out.printf("%-20s", "Client Type");
+        for (String benchmarkName : allBenchmarkNames) {
+            System.out.printf(" %-15s", benchmarkName);
+        }
+        System.out.println();
+        System.out.println("-".repeat(20 + (allBenchmarkNames.size() * 16)));
 
         for (String clientType : Arrays.asList("Apache4", "Apache5-Platform", "Apache5-Virtual")) {
             Map<String, BenchmarkResult> clientResults = grouped.get(clientType);
             if (clientResults != null) {
-                System.out.printf("%-20s %-15.2f %-15.2f %-15.2f %-15.2f%n",
-                                  clientType,
-                                  clientResults.get("simpleGet").getThroughput(),
-                                  clientResults.get("simplePut").getThroughput(),
-                                  clientResults.get("multiThreadedGet").getThroughput(),
-                                  clientResults.get("multiThreadedPut").getThroughput()
-                );
+                System.out.printf("%-20s", clientType);
+                for (String benchmarkName : allBenchmarkNames) {
+                    BenchmarkResult result = clientResults.get(benchmarkName);
+                    if (result != null) {
+                        System.out.printf(" %-15.2f", result.getThroughput());
+                    } else {
+                        System.out.printf(" %-15s", "N/A");
+                    }
+                }
+                System.out.println();
             }
         }
 
-        // Print performance improvements
+        // Print performance improvements (only if we have matching benchmarks)
+        printPerformanceImprovements(grouped);
+        System.out.println("\n" + "=".repeat(80));
+    }
+
+    private static void printPerformanceImprovements(Map<String, Map<String, BenchmarkResult>> grouped) {
         System.out.println("\n" + "=".repeat(80));
         System.out.println("PERFORMANCE IMPROVEMENTS");
         System.out.println("=".repeat(80));
@@ -180,17 +216,27 @@ private static void printComparisonReport(List<BenchmarkResult> results) {
             System.out.println("\nApache5 (Virtual) vs Apache4:");
             printImprovements(apache4, apache5Virtual);
         }
-
-        System.out.println("\n" + "=".repeat(80));
     }
 
     private static void printImprovements(Map<String, BenchmarkResult> baseline,
                                           Map<String, BenchmarkResult> comparison) {
-        for (String op : Arrays.asList("simpleGet", "simplePut",
-                                       "multiThreadedGet", "multiThreadedPut")) {
-            double improvement = (comparison.get(op).getThroughput() /
-                                  baseline.get(op).getThroughput() - 1) * 100;
-            System.out.printf("  %-20s: %+.1f%%%n", op, improvement);
+        // Find common benchmark names
+        Set<String> commonBenchmarks = new HashSet<>(baseline.keySet());
+        commonBenchmarks.retainAll(comparison.keySet());
+
+        if (commonBenchmarks.isEmpty()) {
+            System.out.println("  No common benchmarks found for comparison");
+            return;
+        }
+
+        for (String benchmarkName : commonBenchmarks) {
+            BenchmarkResult baseResult = baseline.get(benchmarkName);
+            BenchmarkResult compResult = comparison.get(benchmarkName);
+
+            if (baseResult != null && compResult != null) {
+                double improvement = (compResult.getThroughput() / baseResult.getThroughput() - 1) * 100;
+                System.out.printf("  %-20s: %+.1f%%%n", benchmarkName, improvement);
+            }
         }
     }
 }