ptest: Allow all test runs to be logged

Check the environment variable `PTEST_LOG_ALL`, and if it is set, write
the output of all test runs to a log file instead of just the failures.
Failed test are still written to failure-nnnn.log, but successful runs
are written to success-nnnn.log.  If the environment variable is not
set, ptest behaves as before, only writing to log files when there is a
failure.

Signed-off-by: David Brown <[email protected]>

Author: d3zd3z

ptest/src/main.rs

@@ -14,9 +14,10 @@ use chrono::Local;
 use log::{debug, error, warn};
 use std::{
     collections::HashSet,
+    env,
     fs::{self, OpenOptions},
     io::{ErrorKind, stdout, Write},
-    process::{Command, Output},
+    process::Command,
     result,
     sync::{
         Arc,
@@ -82,6 +83,15 @@ struct State {
     total: usize,
 }
 
+/// Result of a test run.
+struct TestResult {
+    /// Was this run successful.
+    success: bool,
+
+    /// The captured output.
+    output: Vec<u8>,
+}
+
 impl State {
     fn new(total: usize) -> Arc<Mutex<State>> {
         Arc::new(Mutex::new(State {
@@ -101,46 +111,41 @@ impl State {
         self.status();
     }
 
-    fn done(&mut self, fs: &FeatureSet, output: Result<Option<Output>>) {
+    fn done(&mut self, fs: &FeatureSet, output: Result<TestResult>) {
         let key = fs.textual();
         self.running.remove(&key);
         self.done.insert(key.clone());
         match output {
-            Ok(None) => {
-                // println!("Success {} ({} running)", key, self.running.len());
-            }
-            Ok(Some(output)) => {
-                // Write the output into a file.
-                let mut count = 1;
-                let (mut fd, logname) = loop {
-                    let name = format!("./failure-{:04}.log", count);
-                    count += 1;
-                    match OpenOptions::new()
-                        .create_new(true)
-                        .write(true)
-                        .open(&name)
-                    {
-                        Ok(file) => break (file, name),
-                        Err(ref err) if err.kind() == ErrorKind::AlreadyExists => continue,
-                        Err(err) => {
-                            error!("Unable to write log file to current directory: {:?}", err);
-                            return;
+            Ok(output) => {
+                if !output.success || log_all() {
+                    // Write the output into a file.
+                    let mut count = 1;
+                    let (mut fd, logname) = loop {
+                        let base = if output.success { "success" } else { "failure" };
+                        let name = format!("./{}-{:04}.log", base, count);
+                        count += 1;
+                        match OpenOptions::new()
+                            .create_new(true)
+                            .write(true)
+                            .open(&name)
+                        {
+                            Ok(file) => break (file, name),
+                            Err(ref err) if err.kind() == ErrorKind::AlreadyExists => continue,
+                            Err(err) => {
+                                error!("Unable to write log file to current directory: {:?}", err);
+                                return;
+                            }
                         }
+                    };
+                    fd.write_all(&output.output).unwrap();
+                    if !output.success {
+                        error!("Failure {} log:{:?} ({} running)", key, logname,
+                        self.running.len());
                     }
-                };
-                writeln!(&mut fd, "Test failure {}", key).unwrap();
-                writeln!(&mut fd, "time: {}", Local::now().to_rfc3339()).unwrap();
-                writeln!(&mut fd, "----------------------------------------").unwrap();
-                writeln!(&mut fd, "stdout:").unwrap();
-                fd.write_all(&output.stdout).unwrap();
-                writeln!(&mut fd, "----------------------------------------").unwrap();
-                writeln!(&mut fd, "\nstderr:").unwrap();
-                fd.write_all(&output.stderr).unwrap();
-                error!("Failure {} log:{:?} ({} running)", key, logname,
-                    self.running.len());
+                }
             }
             Err(err) => {
-                error!("Unable to run test {:?} ({:?})", key, err);
+                error!("Unable to run test {:?} ({:?}", key, err);
             }
         }
         self.status();
@@ -239,18 +244,31 @@ impl FeatureSet {
     /// Run a test for this given feature set.  Output is captured and will be returned if there is
     /// an error.  Each will be run successively, and the first failure will be returned.
     /// Otherwise, it returns None, which means everything worked.
-    fn run(&self) -> Result<Option<Output>> {
+    fn run(&self) -> Result<TestResult> {
+        let mut output = vec![];
+        let mut success = true;
         for v in &self.values {
-            let output = Command::new("bash")
+            let cmdout = Command::new("bash")
                .arg("./ci/sim_run.sh")
                .current_dir("..")
                .env(&self.env, v)
                .output()?;
-            if !output.status.success() {
-                return Ok(Some(output));
+            // Grab the output for logging, etc.
+            writeln!(&mut output, "Test {} {}",
+                if cmdout.status.success() { "success" } else { "FAILURE" },
+                self.textual())?;
+            writeln!(&mut output, "time: {}", Local::now().to_rfc3339())?;
+            writeln!(&mut output, "----------------------------------------")?;
+            writeln!(&mut output, "stdout:")?;
+            output.extend(&cmdout.stdout);
+            writeln!(&mut output, "----------------------------------------")?;
+            writeln!(&mut output, "stderr:")?;
+            output.extend(&cmdout.stderr);
+            if !cmdout.status.success() {
+                success = false;
             }
         }
-        return Ok(None);
+        Ok(TestResult { success, output })
     }
 
     /// Convert this feature set into a textual representation
@@ -282,3 +300,8 @@ fn lookup_matrix(y: &Yaml) -> Option<&Vec<Yaml>> {
         .as_hash()?.get(&features)?
         .as_vec()
 }
+
+/// Query if we should be logging all tests and not only failures.
+fn log_all() -> bool {
+    env::var("PTEST_LOG_ALL").is_ok()
+}