Statistic testing mode (build-and-stat-test) performing one build and count test runs for each crate

Author: azhogin

src/actions/experiments/create.rs

@@ -16,6 +16,7 @@ pub struct CreateExperiment {
     pub ignore_blacklist: bool,
     pub assign: Option<Assignee>,
     pub requirement: Option<String>,
+    pub stat_run: Option<i32>,
 }
 
 impl CreateExperiment {
@@ -34,6 +35,7 @@ impl CreateExperiment {
             ignore_blacklist: false,
             assign: None,
             requirement: None,
+            stat_run: None,
         }
     }
 }
@@ -57,8 +59,8 @@ impl Action for CreateExperiment {
                 "INSERT INTO experiments \
                  (name, mode, cap_lints, toolchain_start, toolchain_end, priority, created_at, \
                  status, github_issue, github_issue_url, github_issue_number, ignore_blacklist, \
-                 assigned_to, requirement) \
-                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14);",
+                 assigned_to, requirement, stat_run) \
+                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15);",
                 &[
                     &self.name,
                     &self.mode.to_str(),
@@ -74,6 +76,7 @@ impl Action for CreateExperiment {
                     &self.ignore_blacklist,
                     &self.assign.map(|a| a.to_string()),
                     &self.requirement,
+                    &self.stat_run,
                 ],
             )?;
 
@@ -130,6 +133,7 @@ mod tests {
             ignore_blacklist: true,
             assign: None,
             requirement: Some("linux".to_string()),
+            stat_run: None,
         }
         .apply(&ctx)
         .unwrap();
@@ -253,6 +257,7 @@ mod tests {
             ignore_blacklist: false,
             assign: None,
             requirement: None,
+            stat_run: None,
         }
         .apply(&ctx)
         .unwrap_err();
@@ -283,6 +288,7 @@ mod tests {
             ignore_blacklist: false,
             assign: None,
             requirement: None,
+            stat_run: None,
         }
         .apply(&ctx)
         .unwrap();
@@ -299,6 +305,7 @@ mod tests {
             ignore_blacklist: false,
             assign: None,
             requirement: None,
+            stat_run: None,
         }
         .apply(&ctx)
         .unwrap_err();

src/actions/experiments/edit.rs

@@ -205,6 +205,7 @@ mod tests {
             ignore_blacklist: false,
             assign: None,
             requirement: None,
+            stat_run: None,
         }
         .apply(&ctx)
         .unwrap();

src/cli.rs

@@ -116,6 +116,12 @@ pub enum Crater {
         assign: Option<Assignee>,
         #[clap(name = "requirement", long = "requirement")]
         requirement: Option<String>,
+        #[clap(
+            name = "stat-run",
+            long = "stat-run",
+            help = "For statistic testing mode (build-and-stat-test), amount of test runs."
+        )]
+        stat_run: Option<i32>,
     },
 
     #[clap(name = "edit", about = "edit an experiment configuration")]
@@ -305,6 +311,7 @@ impl Crater {
                 ref ignore_blacklist,
                 ref assign,
                 ref requirement,
+                ref stat_run,
             } => {
                 let config = Config::load()?;
                 let db = Database::open()?;
@@ -321,6 +328,7 @@ impl Crater {
                     ignore_blacklist: *ignore_blacklist,
                     assign: assign.clone(),
                     requirement: requirement.clone(),
+                    stat_run: *stat_run,
                 }
                 .apply(&ctx)?;
             }

src/db/migrations.rs

@@ -363,6 +363,18 @@ fn migrations() -> Vec<(&'static str, MigrationKind)> {
         MigrationKind::SQL("alter table agents add column latest_work_for text;"),
     ));
 
+    // `results` table analog for statistic runs.
+    // statistic run performs one build and several test runs.
+    // run_num == 0 keeps build log.
+    migrations.push((
+        "add_statistic_results",
+        MigrationKind::SQL(
+            "
+            ALTER TABLE experiments ADD COLUMN stat_run INTEGER DEFAULT 0;
+            ",
+        ),
+    ));
+
     migrations
 }
 

src/experiments.rs

@@ -25,6 +25,7 @@ string_enum!(pub enum Status {
 
 string_enum!(pub enum Mode {
     BuildAndTest => "build-and-test",
+    BuildAndStatTest => "build-and-stat-test",
     BuildOnly => "build-only",
     CheckOnly => "check-only",
     Clippy => "clippy",
@@ -259,6 +260,7 @@ pub struct Experiment {
     pub report_url: Option<String>,
     pub ignore_blacklist: bool,
     pub requirement: Option<String>,
+    pub stat_run: Option<i32>,
 }
 
 impl Experiment {
@@ -678,6 +680,7 @@ pub struct ExperimentDBRecord {
     report_url: Option<String>,
     ignore_blacklist: bool,
     requirement: Option<String>,
+    stat_run: Option<i32>,
 }
 
 impl ExperimentDBRecord {
@@ -700,6 +703,7 @@ impl ExperimentDBRecord {
             report_url: row.get("report_url")?,
             ignore_blacklist: row.get("ignore_blacklist")?,
             requirement: row.get("requirement")?,
+            stat_run: row.get("stat_run")?,
         })
     }
 
@@ -735,6 +739,7 @@ impl ExperimentDBRecord {
             report_url: self.report_url,
             ignore_blacklist: self.ignore_blacklist,
             requirement: self.requirement,
+            stat_run: self.stat_run,
         })
     }
 }

src/report/analyzer.rs

@@ -166,6 +166,7 @@ mod tests {
             report_url: None,
             ignore_blacklist: false,
             requirement: None,
+            stat_run: None,
         };
 
         let crates = record_crates! {db, ex,

src/report/display.rs

@@ -1,6 +1,7 @@
 use crate::prelude::*;
 use crate::report::Comparison;
-use crate::results::{BrokenReason, FailureReason, TestResult};
+use crate::results::{BrokenReason, FailureReason, StatFailureReasons, TestResult};
+use std::collections::HashMap;
 
 pub trait ResultName {
     fn short_name(&self) -> String;
@@ -38,6 +39,41 @@ impl ResultName for FailureReason {
     }
 }
 
+impl ResultName for StatFailureReasons {
+    fn short_name(&self) -> String {
+        let StatFailureReasons::Reasons(vec) = self else {
+            return String::new();
+        };
+        // ex: "failed (unknown) x2 | OOM x4 | ICE x1"
+        let unique_counts: HashMap<String, usize> =
+            vec.iter().fold(HashMap::new(), |mut map, val| {
+                *map.entry(val.short_name()).or_insert(0) += 1;
+                map
+            });
+        unique_counts
+            .iter()
+            .map(|v| format!("{} x{}", v.0, v.1))
+            .collect::<Vec<_>>()
+            .join(" | ")
+    }
+
+    fn long_name(&self) -> String {
+        let StatFailureReasons::Reasons(vec) = self else {
+            return String::new();
+        };
+        let unique_counts: HashMap<String, usize> =
+            vec.iter().fold(HashMap::new(), |mut map, val| {
+                *map.entry(val.long_name()).or_insert(0) += 1;
+                map
+            });
+        unique_counts
+            .iter()
+            .map(|v| format!("{} x{}", v.0, v.1))
+            .collect::<Vec<_>>()
+            .join(" | ")
+    }
+}
+
 impl ResultName for BrokenReason {
     fn short_name(&self) -> String {
         match self {
@@ -61,6 +97,7 @@ impl ResultName for TestResult {
             TestResult::PrepareFail(reason) => format!("prepare {}", reason.short_name()),
             TestResult::BuildFail(reason) => format!("build {}", reason.short_name()),
             TestResult::TestFail(reason) => format!("test {}", reason.short_name()),
+            TestResult::TestsFail(reason) => format!("tests: {}", reason.short_name()),
             TestResult::TestSkipped => "test skipped".into(),
             TestResult::TestPass => "test passed".into(),
             TestResult::Error => "error".into(),
@@ -73,6 +110,7 @@ impl ResultName for TestResult {
             TestResult::PrepareFail(reason) => format!("prepare {}", reason.long_name()),
             TestResult::BuildFail(reason) => format!("build {}", reason.long_name()),
             TestResult::TestFail(reason) => format!("test {}", reason.long_name()),
+            TestResult::TestsFail(reason) => format!("tests: {}", reason.long_name()),
             TestResult::BrokenCrate(reason) => reason.long_name(),
             TestResult::TestSkipped
             | TestResult::TestPass
@@ -118,6 +156,7 @@ impl ResultColor for TestResult {
             TestResult::BrokenCrate(_) => Color::Single("#44176e"),
             TestResult::BuildFail(_) => Color::Single("#db3026"),
             TestResult::TestFail(_) => Color::Single("#65461e"),
+            TestResult::TestsFail(_) => Color::Single("#65461e"),
             TestResult::TestSkipped | TestResult::TestPass => Color::Single("#62a156"),
             TestResult::Error => Color::Single("#d77026"),
             TestResult::PrepareFail(_) => Color::Striped("#44176e", "#d77026"),

src/report/mod.rs

@@ -4,7 +4,9 @@ use crate::dirs::WORK_DIR;
 use crate::experiments::Experiment;
 use crate::prelude::*;
 use crate::report::analyzer::{analyze_report, ReportConfig, ToolchainSelect};
-use crate::results::{EncodedLog, EncodingType, FailureReason, ReadResults, TestResult};
+use crate::results::{
+    EncodedLog, EncodingType, FailureReason, ReadResults, StatFailureReasons, TestResult,
+};
 use crate::toolchain::Toolchain;
 use crate::utils;
 use crates_index::SparseIndex;
@@ -514,32 +516,61 @@ fn compare(
             (TestPass, TestPass) => Comparison::SameTestPass,
 
             // (spurious) fixed
-            (BuildFail(reason), TestSkipped | TestFail(_) | TestPass)
+            (BuildFail(reason), TestSkipped | TestFail(_) | TestsFail(_) | TestPass)
             | (TestFail(reason), TestPass) => {
                 if reason.is_spurious() {
                     Comparison::SpuriousFixed
                 } else {
                     Comparison::Fixed
                 }
             }
+            (TestsFail(reason), TestPass) => {
+                if reason.is_spurious() {
+                    Comparison::SpuriousFixed
+                } else {
+                    Comparison::Fixed
+                }
+            }
 
             // (spurious) regressed
-            (TestSkipped | TestFail(_) | TestPass, BuildFail(reason))
+            (TestSkipped | TestFail(_) | TestsFail(_) | TestPass, BuildFail(reason))
             | (TestPass, TestFail(reason)) => {
                 if reason.is_spurious() {
                     Comparison::SpuriousRegressed
                 } else {
                     Comparison::Regressed
                 }
             }
+            (TestPass, TestsFail(reason)) => {
+                if reason.is_spurious() {
+                    Comparison::SpuriousRegressed
+                } else {
+                    Comparison::Regressed
+                }
+            }
 
             (Skipped, _) | (_, Skipped) => Comparison::Skipped,
             (BrokenCrate(_), _) | (_, BrokenCrate(_)) => Comparison::Broken,
             (PrepareFail(_), _) | (_, PrepareFail(_)) => Comparison::PrepareFail,
             (Error, _) | (_, Error) => Comparison::Error,
-            (TestFail(_) | TestPass, TestSkipped) | (TestSkipped, TestFail(_) | TestPass) => {
+            (TestFail(_) | TestsFail(_) | TestPass, TestSkipped)
+            | (TestSkipped, TestFail(_) | TestsFail(_) | TestPass) => {
                 panic!("can't compare {res1} and {res2}");
             }
+
+            // Stat tests comparison
+            (TestsFail(v1), TestsFail(v2)) => {
+                let (StatFailureReasons::Reasons(v1), StatFailureReasons::Reasons(v2)) = (v1, v2)
+                else {
+                    panic!("can't compare {res1} and {res2}");
+                };
+                if v2.len() > v1.len() {
+                    Comparison::Regressed
+                } else {
+                    Comparison::SameTestFail
+                }
+            }
+            (TestsFail(_), TestFail(_)) | (TestFail(_), TestsFail(_)) => Comparison::SameTestFail,
         },
         _ if config.should_skip(krate) => Comparison::Skipped,
         _ => Comparison::Unknown,
@@ -940,6 +971,7 @@ mod tests {
             report_url: None,
             ignore_blacklist: false,
             requirement: None,
+            stat_run: None,
         };
 
         let mut db = DummyDB::default();

src/results/db.rs

@@ -266,10 +266,13 @@ impl crate::runner::RecordProgress for DatabaseDB<'_> {
         result: &TestResult,
         version: Option<(&Crate, &Crate)>,
     ) -> Fallible<()> {
-        self.store_result(ex, krate, toolchain, result, log, EncodingType::Plain)?;
-        if let Some((old, new)) = version {
+        let krate = if let Some((old, new)) = version {
             self.update_crate_version(ex, old, new)?;
-        }
+            new
+        } else {
+            krate
+        };
+        self.store_result(ex, krate, toolchain, result, log, EncodingType::Plain)?;
         Ok(())
     }
 }