Add integrated test runner for real inputs

Author: ictrobot

crates/aoc/Cargo.toml

@@ -16,9 +16,10 @@ year2017 = { path = "../year2017", optional = true }
 year2024 = { path = "../year2024", optional = true }
 
 [features]
-default = ["all-years", "all-simd", "unsafe"]
+default = ["all-years", "all-simd", "unsafe", "test-runner"]
 const_lut = ["year2024?/const_lut"]
 all-simd = ["utils/all-simd"]
+test-runner = []
 # xtask update features
 all-years = ["year2015", "year2016", "year2017", "year2024"]
 unsafe = ["year2015?/unsafe", "year2016?/unsafe", "year2017?/unsafe", "year2024?/unsafe", "utils/unsafe"]

crates/aoc/src/cli/options.rs renamed to crates/aoc/src/cli/arguments.rs

@@ -1,42 +1,54 @@
+use crate::cli::UsageError;
+use crate::cli::mode::{self, MainFn};
 use aoc::{PUZZLES, PuzzleFn};
 use std::collections::VecDeque;
 use std::error::Error;
 use std::num::NonZeroUsize;
+use std::path::PathBuf;
+use std::{fs, io};
 use utils::date::{Day, Year};
 use utils::multiversion::{VERSIONS, Version};
 
 #[derive(Debug, Default)]
-pub struct Options {
+pub struct Arguments {
     program_name: Option<String>,
     pub help: bool,
     pub version_override: Option<Version>,
     pub threads_override: Option<NonZeroUsize>,
+    pub inputs_dir: Option<PathBuf>,
+    mode: Option<MainFn>,
     pub year: Option<Year>,
     pub day: Option<Day>,
+    pub extra_args: VecDeque<String>,
 }
 
-impl Options {
-    pub fn parse() -> Result<Self, String> {
+impl Arguments {
+    pub fn parse() -> Result<Self, UsageError> {
         let mut result = Self::default();
 
         let mut args: VecDeque<String> = std::env::args().collect();
         result.program_name = args.pop_front();
 
         while let Some(option) = args.pop_front() {
             if option == "--" {
-                break;
+                result.extra_args = args;
+                return Ok(result);
             }
 
             if let Some(option) = option.strip_prefix("--") {
                 // Long form options
                 if let Some((before, after)) = option.split_once('=') {
                     result
                         .handle_long(before, ArgumentValue::Provided(after.to_string()))
-                        .map_err(|e| format!("option --{before}: {e}"))?;
+                        .map_err(|e| {
+                            UsageError::InvalidArguments(format!("option --{before}: {e}").into())
+                        })?;
                 } else {
                     result
                         .handle_long(option, ArgumentValue::Available(&mut args))
-                        .map_err(|e| format!("option --{option}: {e}"))?;
+                        .map_err(|e| {
+                            UsageError::InvalidArguments(format!("option --{option}: {e}").into())
+                        })?;
                 }
                 continue;
             }
@@ -50,13 +62,19 @@ impl Options {
                     for option in options {
                         result
                             .handle_short(option, ArgumentValue::None)
-                            .map_err(|e| format!("option -{option}: {e}"))?;
+                            .map_err(|e| {
+                                UsageError::InvalidArguments(
+                                    format!("option -{option}: {e}").into(),
+                                )
+                            })?;
                     }
 
-                    // Last short form option can consume a value
+                    // The last short form option can consume a value
                     result
                         .handle_short(last, ArgumentValue::Available(&mut args))
-                        .map_err(|e| format!("option -{last}: {e}"))?;
+                        .map_err(|e| {
+                            UsageError::InvalidArguments(format!("option -{last}: {e}").into())
+                        })?;
                     continue;
                 }
             }
@@ -65,46 +83,64 @@ impl Options {
             break;
         }
 
+        if let Some(i) = args.iter().position(|x| x == "--") {
+            result.extra_args = args.split_off(i + 1);
+            args.pop_back();
+        }
+
         if let Some(year) = args.pop_front() {
             result.year = match year.parse() {
                 Ok(y) => Some(y),
-                Err(err) => return Err(err.to_string()),
+                Err(err) => return Err(UsageError::InvalidArguments(err.into())),
             };
 
             if let Some(day) = args.pop_front() {
                 result.day = match day.parse() {
                     Ok(y) => Some(y),
-                    Err(err) => return Err(err.to_string()),
+                    Err(err) => return Err(UsageError::InvalidArguments(err.into())),
                 };
 
                 if !args.is_empty() {
-                    return Err("too many arguments".to_string());
+                    return Err(UsageError::TooManyArguments);
                 }
             }
         }
 
         Ok(result)
     }
 
-    pub fn help(&self) -> String {
+    pub fn help_string(&self) -> String {
         format!(
             r"Usage:
     {program_name}
         Run all solutions
 
     {program_name} $year
         Run all solutions for the provided year
-        
+
     {program_name} $year $day
         Run the solution for the provided date
 
 Options:
+    --stdin
+        Run a single solution, reading input from stdin. $year and $day must be provided.
+
+    --test
+        Runs all solutions against all inputs from the inputs directory, comparing the outputs to
+        the stored correct answers in the inputs directory. $year may be provided to only test the
+        provided year. A custom command template may be provided following a `--` argument to test
+        another binary. Requires the 'test-runner' feature to be enabled.
+
     --multiversion/-m $version
         Override which implementation of multiversioned functions should be used.
         Supported versions: {multiversion_options:?}
-        
+
     --threads/-t $threads
         Override the number of threads to use for multithreaded solutions.
+        In `--test` mode this controls the number of simultaneous tests.
+
+    --inputs $dir
+        Specify the directory storing inputs. Defaults to './inputs'.
 
     --help/-h
         Print this help
@@ -121,6 +157,10 @@ Options:
             "help" => self.option_help(value),
             "multiversion" => self.option_multiversion(value),
             "threads" => self.option_threads(value),
+            "inputs" => self.option_inputs(value),
+            "stdin" => self.option_mode(value, mode::stdin::main),
+            #[cfg(feature = "test-runner")]
+            "test" => self.option_mode(value, mode::test::main),
             _ => Err("unknown option".into()),
         }
     }
@@ -158,13 +198,54 @@ Options:
         Ok(())
     }
 
+    fn option_inputs(&mut self, value: ArgumentValue) -> Result<(), Box<dyn Error>> {
+        let value = value.required()?.into();
+        if self.inputs_dir.is_some() {
+            return Err("option provided more than once".into());
+        }
+        if !fs::metadata(&value).is_ok_and(|m| m.is_dir()) {
+            return Err("inputs path must be a directory".into());
+        }
+        self.inputs_dir = Some(value);
+        Ok(())
+    }
+
+    fn option_mode(&mut self, value: ArgumentValue, mode: MainFn) -> Result<(), Box<dyn Error>> {
+        value.none()?;
+        if self.mode.is_some() {
+            return Err("mode options are mutually exclusive".into());
+        }
+        self.mode = Some(mode);
+        Ok(())
+    }
+
+    pub fn main_fn(&self) -> MainFn {
+        self.mode.unwrap_or(mode::default::main)
+    }
+
     pub fn matching_puzzles(&self) -> Vec<(Year, Day, PuzzleFn)> {
         PUZZLES
             .iter()
             .copied()
             .filter(|&(y, d, ..)| self.year.unwrap_or(y) == y && self.day.unwrap_or(d) == d)
             .collect()
     }
+
+    pub fn inputs_dir(&self) -> PathBuf {
+        self.inputs_dir
+            .clone()
+            .unwrap_or_else(|| PathBuf::from("./inputs"))
+    }
+
+    pub fn read_input(&self, year: Year, day: Day) -> Result<String, (String, io::Error)> {
+        let mut path = self.inputs_dir();
+        path.push(format!("year{year:#}"));
+        path.push(format!("day{day:#}.txt"));
+        match fs::read_to_string(&path) {
+            Ok(s) => Ok(s.trim_ascii_end().replace("\r\n", "\n")),
+            Err(err) => Err((path.to_string_lossy().to_string(), err)),
+        }
+    }
 }
 
 #[must_use]

crates/aoc/src/cli/error.rs

@@ -0,0 +1,57 @@
+use std::error::Error;
+use std::fmt::{Debug, Display, Formatter};
+use std::process::ExitCode;
+use utils::date::{Day, Year};
+
+#[derive(Debug)]
+pub enum UsageError {
+    InvalidArguments(Box<dyn Error>),
+    MissingArguments(Box<dyn Error>),
+    TooManyArguments,
+    UnsupportedPuzzle(Year, Day),
+    NoSupportedPuzzles,
+}
+
+impl Display for UsageError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            UsageError::InvalidArguments(err) => write!(f, "invalid arguments: {err}"),
+            UsageError::MissingArguments(err) => write!(f, "missing required arguments: {err}"),
+            UsageError::TooManyArguments => write!(f, "too many arguments"),
+            UsageError::UnsupportedPuzzle(year, day) => {
+                write!(f, "unsupported puzzle: {year:#} day {day:#}")
+            }
+            UsageError::NoSupportedPuzzles => write!(f, "no matching supported puzzles"),
+        }
+    }
+}
+
+impl Error for UsageError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        match self {
+            UsageError::InvalidArguments(err) | UsageError::MissingArguments(err) => {
+                Some(err.as_ref())
+            }
+            _ => None,
+        }
+    }
+}
+
+impl UsageError {
+    pub fn exit_code() -> ExitCode {
+        ExitCode::from(2)
+    }
+}
+
+// Used by aoc::cli::mode::test to indicate that the process should exit with failure, but without
+// printing an error message as it has already printed the failures.
+#[derive(Debug)]
+pub struct FailedNoErrorMessage;
+
+impl Display for FailedNoErrorMessage {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "failed")
+    }
+}
+
+impl Error for FailedNoErrorMessage {}

crates/aoc/src/cli/mod.rs

@@ -1,3 +1,6 @@
-mod options;
+mod arguments;
+mod error;
+pub mod mode;
 
-pub use options::Options;
+pub use arguments::Arguments;
+pub use error::{FailedNoErrorMessage, UsageError};

crates/aoc/src/cli/mode/default.rs

@@ -0,0 +1,66 @@
+use crate::cli::{Arguments, UsageError};
+use std::error::Error;
+use std::time::{Duration, Instant};
+
+#[expect(clippy::print_stdout)]
+pub fn main(args: &Arguments) -> Result<(), Box<dyn Error>> {
+    if !args.extra_args.is_empty() {
+        return Err(UsageError::TooManyArguments.into());
+    }
+
+    let puzzles = args.matching_puzzles();
+    if puzzles.is_empty() {
+        return Err(UsageError::NoSupportedPuzzles.into());
+    }
+
+    // FIXME support 80 character wide output (without time?)
+    println!(
+        "Puzzle  │ Part 1               │ Part 2                                 │ Time      "
+    );
+    println!(
+        "────────┼──────────────────────┼────────────────────────────────────────┼───────────"
+    );
+    let mut total = Duration::default();
+    for (year, day, f) in puzzles {
+        let input = args
+            .read_input(year, day)
+            .map_err(|(path, err)| format!("{year:#} {day:#}: failed to read {path:?}: {err}"))?;
+
+        let start = Instant::now();
+        let (part1, part2) = f(&input).map_err(|err| format!("{year:#} {day:#}: {err}"))?;
+        let elapsed = start.elapsed();
+        total += elapsed;
+
+        // Hack to treat "🎄" as two characters wide
+        // ("🎄" is 1 wide in Unicode 8 but 2 wide in Unicode 9+)
+        let part1_width = if part1 == "🎄" { 19 } else { 20 };
+        let part2_width = if part2 == "🎄" { 37 } else { 38 };
+
+        println!(
+            "{year:#} {day:#} │ {part1:<part1_width$} │ {part2:<part2_width$} │ {}",
+            format_duration(elapsed)
+        );
+    }
+
+    println!(
+        "────────┼──────────────────────┼────────────────────────────────────────┼───────────"
+    );
+    println!(
+        "                                                                        │ {}",
+        format_duration(total),
+    );
+
+    Ok(())
+}
+
+fn format_duration(d: Duration) -> String {
+    let (unit, multiplier) = if d.as_micros() < 1000 {
+        ("µ", 1_000_000.)
+    } else {
+        ("m", 1_000.)
+    };
+
+    let float = d.as_secs_f64() * multiplier;
+    let precision = if float < 1000. { 3 } else { 0 };
+    format!("{float:7.precision$} {unit}s")
+}

crates/aoc/src/cli/mode/mod.rs

@@ -0,0 +1,9 @@
+use crate::cli::Arguments;
+use std::error::Error;
+
+pub mod default;
+pub mod stdin;
+#[cfg(feature = "test-runner")]
+pub mod test;
+
+pub type MainFn = fn(&Arguments) -> Result<(), Box<dyn Error>>;