feat: add caddy submodule and test

Author: not-matthias

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "go-runner/testdata/projects/cli-runtime"]
 	path = go-runner/testdata/projects/cli-runtime
 	url = https://github.com/kubernetes/cli-runtime.git
+[submodule "go-runner/testdata/projects/caddy"]
+	path = go-runner/testdata/projects/caddy
+	url = https://github.com/caddyserver/caddy

go-runner/src/integration_tests.rs

@@ -63,7 +63,9 @@ fn assert_results_snapshots(profile_dir: &Path, project_name: &str) {
 }
 
 #[rstest]
-// // #[case::caddy("caddy")]
+// The 'BenchmarkMatchExpressionMatch/boolean_matches_succeed_for_placeholder_http.request.tls.client.subject-16' benchmark currently
+// panics, which causes the binary (which contains more benchmarks) to exit. Has to be fixed within Caddy.
+#[case::caddy("caddy")]
 #[case::fzf("fzf")]
 #[case::opentelemetry_go("opentelemetry-go")]
 #[case::golang_benchmarks("golang-benchmarks")]

go-runner/src/lib.rs

@@ -47,10 +47,13 @@ pub fn run_benchmarks<P: AsRef<Path>>(
             }
         };
 
-        runner::run(
+        if let Err(error) = runner::run(
             &binary_path,
             &["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime],
-        )?;
+        ) {
+            error!("Failed to run benchmarks for {}: {error}", package.name);
+            continue;
+        }
     }
 
     // 3. Collect the results

go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@caddy.snap

@@ -0,0 +1,107 @@
+---
+source: go-runner/src/integration_tests.rs
+expression: results
+---
+[
+  {
+    "creator": {
+      "name": "codspeed-go",
+      "version": "[version]",
+      "pid": "[pid]"
+    },
+    "instrument": {
+      "type": "walltime"
+    },
+    "benchmarks": [
+      {
+        "name": "BenchmarkLoad",
+        "uri": "admin_test.go::BenchmarkLoad",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkReplacer::escaped_placeholder",
+        "uri": "replacer_test.go::BenchmarkReplacer::escaped_placeholder",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkReplacer::int_replacement",
+        "uri": "replacer_test.go::BenchmarkReplacer::int_replacement",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkReplacer::no_placeholder",
+        "uri": "replacer_test.go::BenchmarkReplacer::no_placeholder",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkReplacer::placeholder",
+        "uri": "replacer_test.go::BenchmarkReplacer::placeholder",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      },
+      {
+        "name": "BenchmarkReplacer::string_replacement",
+        "uri": "replacer_test.go::BenchmarkReplacer::string_replacement",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      }
+    ]
+  },
+  {
+    "creator": {
+      "name": "codspeed-go",
+      "version": "[version]",
+      "pid": "[pid]"
+    },
+    "instrument": {
+      "type": "walltime"
+    },
+    "benchmarks": [
+      {
+        "name": "BenchmarkOpenResponseWriter",
+        "uri": "modules/caddyhttp/encode/encode_test.go::BenchmarkOpenResponseWriter",
+        "config": {
+          "warmup_time_ns": null,
+          "min_round_time_ns": null,
+          "max_time_ns": null,
+          "max_rounds": null
+        },
+        "stats": "[stats]"
+      }
+    ]
+  }
+]