Assign an artifact type (binary/library) to each compile benchmark

Kobzol · Kobzol · commit 1ffe4ac8a612 · 2023-07-26T08:37:40.000+02:00
diff --git a/collector/compile-benchmarks/await-call-tree/perf-config.json b/collector/compile-benchmarks/await-call-tree/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/bitmaps-3.1.0/perf-config.json b/collector/compile-benchmarks/bitmaps-3.1.0/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/cargo-0.60.0/perf-config.json b/collector/compile-benchmarks/cargo-0.60.0/perf-config.json
@@ -2,5 +2,6 @@
     "cargo_opts": "--lib",
     "runs": 1,
     "touch_file": "src/cargo/lib.rs",
-    "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/clap-3.1.6/perf-config.json b/collector/compile-benchmarks/clap-3.1.6/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/coercions/perf-config.json b/collector/compile-benchmarks/coercions/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/cranelift-codegen-0.82.1/perf-config.json b/collector/compile-benchmarks/cranelift-codegen-0.82.1/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/ctfe-stress-5/perf-config.json b/collector/compile-benchmarks/ctfe-stress-5/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/deep-vector/perf-config.json b/collector/compile-benchmarks/deep-vector/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/deeply-nested-multi/perf-config.json b/collector/compile-benchmarks/deeply-nested-multi/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/derive/perf-config.json b/collector/compile-benchmarks/derive/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/diesel-1.4.8/perf-config.json b/collector/compile-benchmarks/diesel-1.4.8/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/encoding/perf-config.json b/collector/compile-benchmarks/encoding/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/exa-0.10.1/perf-config.json b/collector/compile-benchmarks/exa-0.10.1/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/externs/perf-config.json b/collector/compile-benchmarks/externs/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/futures/perf-config.json b/collector/compile-benchmarks/futures/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/helloworld-tiny/perf-config.json b/collector/compile-benchmarks/helloworld-tiny/perf-config.json
@@ -1,5 +1,6 @@
 {
     "category": "secondary",
     "excluded_profiles": ["Doc", "Check", "Debug"],
-    "excluded_scenarios": ["IncrFull", "IncrPatched", "IncrUnchanged"]
+    "excluded_scenarios": ["IncrFull", "IncrPatched", "IncrUnchanged"],
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/helloworld/perf-config.json b/collector/compile-benchmarks/helloworld/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "primary"
+    "category": "primary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/html5ever-0.26.0/perf-config.json b/collector/compile-benchmarks/html5ever-0.26.0/perf-config.json
@@ -1,4 +1,5 @@
 {
-  "touch_file": "src/lib.rs",
-  "category": "primary"
+    "touch_file": "src/lib.rs",
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/html5ever/perf-config.json b/collector/compile-benchmarks/html5ever/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/hyper-0.14.18/perf-config.json b/collector/compile-benchmarks/hyper-0.14.18/perf-config.json
@@ -1,4 +1,5 @@
 {
-  "cargo_opts": "--features=client,http1,http2,server,stream",
-  "category": "primary"
+    "cargo_opts": "--features=client,http1,http2,server,stream",
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/image-0.24.1/perf-config.json b/collector/compile-benchmarks/image-0.24.1/perf-config.json
@@ -1,4 +1,5 @@
 {
     "runs": 1,
-    "category": "primary"
-}
+    "category": "primary",
+    "artifact": "library"
+}
diff --git a/collector/compile-benchmarks/inflate/perf-config.json b/collector/compile-benchmarks/inflate/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/issue-46449/perf-config.json b/collector/compile-benchmarks/issue-46449/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/issue-58319/perf-config.json b/collector/compile-benchmarks/issue-58319/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/issue-88862/perf-config.json b/collector/compile-benchmarks/issue-88862/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/libc-0.2.124/perf-config.json b/collector/compile-benchmarks/libc-0.2.124/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/many-assoc-items/perf-config.json b/collector/compile-benchmarks/many-assoc-items/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/match-stress/perf-config.json b/collector/compile-benchmarks/match-stress/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/piston-image/perf-config.json b/collector/compile-benchmarks/piston-image/perf-config.json
@@ -1,4 +1,5 @@
 {
     "runs": 1,
-    "category": "stable"
-}
+    "category": "stable",
+    "artifact": "library"
+}
diff --git a/collector/compile-benchmarks/projection-caching/perf-config.json b/collector/compile-benchmarks/projection-caching/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/regex-1.5.5/perf-config.json b/collector/compile-benchmarks/regex-1.5.5/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "primary"
-}
+    "category": "primary",
+    "artifact": "library"
+}
diff --git a/collector/compile-benchmarks/regex/perf-config.json b/collector/compile-benchmarks/regex/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/regression-31157/perf-config.json b/collector/compile-benchmarks/regression-31157/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/ripgrep-13.0.0-tiny/perf-config.json b/collector/compile-benchmarks/ripgrep-13.0.0-tiny/perf-config.json
@@ -2,5 +2,6 @@
   "touch_file": "crates/core/main.rs",
   "category": "secondary",
   "excluded_profiles": ["Doc", "Check", "Debug"],
-  "excluded_scenarios": ["IncrFull", "IncrPatched", "IncrUnchanged"]
+  "excluded_scenarios": ["IncrFull", "IncrPatched", "IncrUnchanged"],
+  "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/ripgrep-13.0.0/perf-config.json b/collector/compile-benchmarks/ripgrep-13.0.0/perf-config.json
@@ -1,4 +1,5 @@
 {
-  "touch_file": "crates/core/main.rs",
-  "category": "primary"
+    "touch_file": "crates/core/main.rs",
+    "category": "primary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/serde-1.0.136/perf-config.json b/collector/compile-benchmarks/serde-1.0.136/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/serde_derive-1.0.136/perf-config.json b/collector/compile-benchmarks/serde_derive-1.0.136/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/stm32f4-0.14.0/perf-config.json b/collector/compile-benchmarks/stm32f4-0.14.0/perf-config.json
@@ -1,4 +1,5 @@
 {
-  "cargo_opts": "--features=stm32f410",
-  "category": "primary"
+    "cargo_opts": "--features=stm32f410",
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/style-servo/perf-config.json b/collector/compile-benchmarks/style-servo/perf-config.json
@@ -4,5 +4,6 @@
     "cargo_toml": "components/style/Cargo.toml",
     "runs": 1,
     "touch_file": "components/style/lib.rs",
-    "category": "stable"
+    "category": "stable",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/syn-1.0.89/perf-config.json b/collector/compile-benchmarks/syn-1.0.89/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/syn/perf-config.json b/collector/compile-benchmarks/syn/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "stable"
-}
+    "category": "stable",
+    "artifact": "library"
+}
diff --git a/collector/compile-benchmarks/token-stream-stress/perf-config.json b/collector/compile-benchmarks/token-stream-stress/perf-config.json
@@ -1,5 +1,6 @@
 {
     "cargo_opts": "--bin token-stream-stress-bin",
     "touch_file": "src/main.rs",
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/tokio-webpush-simple/perf-config.json b/collector/compile-benchmarks/tokio-webpush-simple/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/main.rs",
-    "category": "stable"
+    "category": "stable",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/tt-muncher/perf-config.json b/collector/compile-benchmarks/tt-muncher/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/main.rs",
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/tuple-stress/perf-config.json b/collector/compile-benchmarks/tuple-stress/perf-config.json
@@ -1,4 +1,5 @@
 {
     "runs": 1,
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/ucd/perf-config.json b/collector/compile-benchmarks/ucd/perf-config.json
@@ -1,4 +1,5 @@
 {
     "runs": 1,
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/unicode-normalization-0.1.19/perf-config.json b/collector/compile-benchmarks/unicode-normalization-0.1.19/perf-config.json
@@ -1,3 +1,4 @@
 {
-  "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/unify-linearly/perf-config.json b/collector/compile-benchmarks/unify-linearly/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/unused-warnings/perf-config.json b/collector/compile-benchmarks/unused-warnings/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "binary"
 }
diff --git a/collector/compile-benchmarks/webrender-2022/perf-config.json b/collector/compile-benchmarks/webrender-2022/perf-config.json
@@ -2,5 +2,6 @@
     "cargo_toml": "webrender/Cargo.toml",
     "runs": 1,
     "touch_file": "webrender/src/lib.rs",
-    "category": "primary"
+    "category": "primary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/wf-projection-stress-65510/perf-config.json b/collector/compile-benchmarks/wf-projection-stress-65510/perf-config.json
@@ -1,3 +1,4 @@
 {
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/compile-benchmarks/wg-grammar/perf-config.json b/collector/compile-benchmarks/wg-grammar/perf-config.json
@@ -1,4 +1,5 @@
 {
     "touch_file": "src/lib.rs",
-    "category": "secondary"
+    "category": "secondary",
+    "artifact": "library"
 }
diff --git a/collector/src/compile/benchmark/mod.rs b/collector/src/compile/benchmark/mod.rs
@@ -22,6 +22,14 @@ fn default_runs() -> usize {
     3
 }
 
+#[derive(Debug, Default, Clone, serde::Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ArtifactType {
+    Binary,
+    #[default]
+    Library,
+}
+
 /// This is the internal representation of an individual benchmark's
 /// perf-config.json file.
 #[derive(Debug, Clone, serde::Deserialize)]
@@ -52,6 +60,8 @@ pub struct BenchmarkConfig {
     /// They will be ignored during benchmarking.
     #[serde(default)]
     excluded_scenarios: HashSet<Scenario>,
+
+    artifact: ArtifactType,
 }
 
 impl BenchmarkConfig {

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
`1`	`1`	`{`
`2`		`- "category": "secondary"`
	`2`	`+ "category": "secondary",`
	`3`	`+ "artifact": "library"`
`3`	`4`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
`1`	`1`	`{`
`2`		`- "category": "primary"`
	`2`	`+ "category": "primary",`
	`3`	`+ "artifact": "library"`
`3`	`4`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,5 +2,6 @@`
`2`	`2`	`"cargo_opts": "--lib",`
`3`	`3`	`"runs": 1,`
`4`	`4`	`"touch_file": "src/cargo/lib.rs",`
`5`		`- "category": "primary"`
	`5`	`+ "category": "primary",`
	`6`	`+ "artifact": "library"`
`6`	`7`	`}`