Auto merge of #108148 - parthopdas:master, r=oli-obk

Implementing "<test_binary> --list --format json" for use by IDE test explorers / runners

Fixes #107307

PR 1 of 2 - wiring up just the new information + implement the command line changes i.e. --format json + tests

upcoming:
PR 2 of 2 - clean up "#[cfg(not(bootstrap))]" from PR 1

As per the discussions on
- MCP: https://rust-lang.zulipchat.com/#narrow/stream/233931-t-compiler.2Fmajor-changes/topic/Implementing.20.22.3Ctest_binary.3E.20--list.20--form.E2.80.A6.20compiler-team.23592/near/328747548
- preRFC: https://internals.rust-lang.org/t/pre-rfc-implementing-test-binary-list-format-json-for-use-by-ide-test-explorers-runners/18308
- FYI on Discord: https://discord.com/channels/442252698964721669/459149169546887178/1075581549409484820

Author: bors

test/src/console.rs

@@ -41,6 +41,46 @@ impl<T: Write> Write for OutputLocation<T> {
     }
 }
 
+pub struct ConsoleTestDiscoveryState {
+    pub log_out: Option<File>,
+    pub tests: usize,
+    pub benchmarks: usize,
+    pub ignored: usize,
+    pub options: Options,
+}
+
+impl ConsoleTestDiscoveryState {
+    pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestDiscoveryState> {
+        let log_out = match opts.logfile {
+            Some(ref path) => Some(File::create(path)?),
+            None => None,
+        };
+
+        Ok(ConsoleTestDiscoveryState {
+            log_out,
+            tests: 0,
+            benchmarks: 0,
+            ignored: 0,
+            options: opts.options,
+        })
+    }
+
+    pub fn write_log<F, S>(&mut self, msg: F) -> io::Result<()>
+    where
+        S: AsRef<str>,
+        F: FnOnce() -> S,
+    {
+        match self.log_out {
+            None => Ok(()),
+            Some(ref mut o) => {
+                let msg = msg();
+                let msg = msg.as_ref();
+                o.write_all(msg.as_bytes())
+            }
+        }
+    }
+}
+
 pub struct ConsoleTestState {
     pub log_out: Option<File>,
     pub total: usize,
@@ -138,53 +178,44 @@ impl ConsoleTestState {
 
 // List the tests to console, and optionally to logfile. Filters are honored.
 pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
-    let mut output = match term::stdout() {
+    let output = match term::stdout() {
         None => OutputLocation::Raw(io::stdout().lock()),
         Some(t) => OutputLocation::Pretty(t),
     };
 
-    let quiet = opts.format == OutputFormat::Terse;
-    let mut st = ConsoleTestState::new(opts)?;
-
-    let mut ntest = 0;
-    let mut nbench = 0;
+    let mut out: Box<dyn OutputFormatter> = match opts.format {
+        OutputFormat::Pretty | OutputFormat::Junit => {
+            Box::new(PrettyFormatter::new(output, false, 0, false, None))
+        }
+        OutputFormat::Terse => Box::new(TerseFormatter::new(output, false, 0, false)),
+        OutputFormat::Json => Box::new(JsonFormatter::new(output)),
+    };
+    let mut st = ConsoleTestDiscoveryState::new(opts)?;
 
+    out.write_discovery_start()?;
     for test in filter_tests(opts, tests).into_iter() {
         use crate::TestFn::*;
 
-        let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test;
+        let TestDescAndFn { desc, testfn } = test;
 
         let fntype = match testfn {
             StaticTestFn(..) | DynTestFn(..) => {
-                ntest += 1;
+                st.tests += 1;
                 "test"
             }
             StaticBenchFn(..) | DynBenchFn(..) => {
-                nbench += 1;
+                st.benchmarks += 1;
                 "benchmark"
             }
         };
 
-        writeln!(output, "{name}: {fntype}")?;
-        st.write_log(|| format!("{fntype} {name}\n"))?;
-    }
+        st.ignored += if desc.ignore { 1 } else { 0 };
 
-    fn plural(count: u32, s: &str) -> String {
-        match count {
-            1 => format!("1 {s}"),
-            n => format!("{n} {s}s"),
-        }
+        out.write_test_discovered(&desc, fntype)?;
+        st.write_log(|| format!("{fntype} {}\n", desc.name))?;
     }
 
-    if !quiet {
-        if ntest != 0 || nbench != 0 {
-            writeln!(output)?;
-        }
-
-        writeln!(output, "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark"))?;
-    }
-
-    Ok(())
+    out.write_discovery_finish(&st)
 }
 
 // Updates `ConsoleTestState` depending on result of the test execution.

test/src/formatters/json.rs

@@ -2,7 +2,7 @@ use std::{borrow::Cow, io, io::prelude::Write};
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::TestDesc,
@@ -60,6 +60,56 @@ impl<T: Write> JsonFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JsonFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        self.writeln_message(&format!(r#"{{ "type": "suite", "event": "discovery" }}"#))
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        let TestDesc {
+            name,
+            ignore,
+            ignore_message,
+            #[cfg(not(bootstrap))]
+            source_file,
+            #[cfg(not(bootstrap))]
+            start_line,
+            #[cfg(not(bootstrap))]
+            start_col,
+            #[cfg(not(bootstrap))]
+            end_line,
+            #[cfg(not(bootstrap))]
+            end_col,
+            ..
+        } = desc;
+
+        #[cfg(bootstrap)]
+        let source_file = "";
+        #[cfg(bootstrap)]
+        let start_line = 0;
+        #[cfg(bootstrap)]
+        let start_col = 0;
+        #[cfg(bootstrap)]
+        let end_line = 0;
+        #[cfg(bootstrap)]
+        let end_col = 0;
+
+        self.writeln_message(&format!(
+            r#"{{ "type": "{test_type}", "event": "discovered", "name": "{}", "ignore": {ignore}, "ignore_message": "{}", "source_path": "{}", "start_line": {start_line}, "start_col": {start_col}, "end_line": {end_line}, "end_col": {end_col} }}"#,
+            EscapedString(name.as_slice()),
+            ignore_message.unwrap_or(""),
+            EscapedString(source_file),
+        ))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        let ConsoleTestDiscoveryState { tests, benchmarks, ignored, .. } = state;
+
+        let total = tests + benchmarks;
+        self.writeln_message(&format!(
+            r#"{{ "type": "suite", "event": "completed", "tests": {tests}, "benchmarks": {benchmarks}, "total": {total}, "ignored": {ignored} }}"#
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let shuffle_seed_json = if let Some(shuffle_seed) = shuffle_seed {
             format!(r#", "shuffle_seed": {shuffle_seed}"#)

test/src/formatters/junit.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use super::OutputFormatter;
 use crate::{
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     test_result::TestResult,
     time,
     types::{TestDesc, TestType},
@@ -27,6 +27,18 @@ impl<T: Write> JunitFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for JunitFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_test_discovered(&mut self, _desc: &TestDesc, _test_type: &str) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
+    fn write_discovery_finish(&mut self, _state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
+    }
+
     fn write_run_start(
         &mut self,
         _test_count: usize,

test/src/formatters/mod.rs

@@ -1,7 +1,7 @@
 use std::{io, io::prelude::Write};
 
 use crate::{
-    console::ConsoleTestState,
+    console::{ConsoleTestDiscoveryState, ConsoleTestState},
     test_result::TestResult,
     time,
     types::{TestDesc, TestName},
@@ -18,6 +18,10 @@ pub(crate) use self::pretty::PrettyFormatter;
 pub(crate) use self::terse::TerseFormatter;
 
 pub(crate) trait OutputFormatter {
+    fn write_discovery_start(&mut self) -> io::Result<()>;
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()>;
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()>;
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()>;
     fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>;
     fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>;

test/src/formatters/pretty.rs

@@ -3,7 +3,7 @@ use std::{io, io::prelude::Write};
 use super::OutputFormatter;
 use crate::{
     bench::fmt_bench_samples,
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     term,
     test_result::TestResult,
     time,
@@ -181,6 +181,33 @@ impl<T: Write> PrettyFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for PrettyFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        self.write_plain(format!("{}: {test_type}\n", desc.name))
+    }
+
+    fn write_discovery_finish(&mut self, state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        fn plural(count: usize, s: &str) -> String {
+            match count {
+                1 => format!("1 {s}"),
+                n => format!("{n} {s}s"),
+            }
+        }
+
+        if state.tests != 0 || state.benchmarks != 0 {
+            self.write_plain("\n")?;
+        }
+
+        self.write_plain(format!(
+            "{}, {}\n",
+            plural(state.tests, "test"),
+            plural(state.benchmarks, "benchmark")
+        ))
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         let noun = if test_count != 1 { "tests" } else { "test" };
         let shuffle_seed_msg = if let Some(shuffle_seed) = shuffle_seed {

test/src/formatters/terse.rs

@@ -3,7 +3,7 @@ use std::{io, io::prelude::Write};
 use super::OutputFormatter;
 use crate::{
     bench::fmt_bench_samples,
-    console::{ConsoleTestState, OutputLocation},
+    console::{ConsoleTestDiscoveryState, ConsoleTestState, OutputLocation},
     term,
     test_result::TestResult,
     time,
@@ -167,6 +167,18 @@ impl<T: Write> TerseFormatter<T> {
 }
 
 impl<T: Write> OutputFormatter for TerseFormatter<T> {
+    fn write_discovery_start(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn write_test_discovered(&mut self, desc: &TestDesc, test_type: &str) -> io::Result<()> {
+        self.write_plain(format!("{}: {test_type}\n", desc.name))
+    }
+
+    fn write_discovery_finish(&mut self, _state: &ConsoleTestDiscoveryState) -> io::Result<()> {
+        Ok(())
+    }
+
     fn write_run_start(&mut self, test_count: usize, shuffle_seed: Option<u64>) -> io::Result<()> {
         self.total_test_count = test_count;
         let noun = if test_count != 1 { "tests" } else { "test" };