Time Measure & Support for Multiple Measures (#285)

* Add simple time measurement

While imperfect, this is still useful data that anchors benchmark
iterations to a number that is more meaningful to humans sometimes.
Combined with the ability to capture multiple measures, this is
pretty useful to have available.

* Add support for capturing multiple measurements

Previously, only a single measure type could be requested.  Often,
we might want to capture multiple measures on a benchmark run as
different measurements might provide a more complete picture of
a given benchmark iteration.

Author: posborne

crates/cli/src/benchmark.rs

@@ -4,6 +4,7 @@ use rand::{rngs::SmallRng, Rng, SeedableRng};
 use sightglass_data::{Format, Measurement, Phase};
 use sightglass_recorder::bench_api::Engine;
 use sightglass_recorder::cpu_affinity::bind_to_single_core;
+use sightglass_recorder::measure::multi::MultiMeasure;
 use sightglass_recorder::measure::Measurements;
 use sightglass_recorder::{bench_api::BenchApi, benchmark, measure::MeasureType};
 use std::{
@@ -80,9 +81,11 @@ pub struct BenchmarkCommand {
     output_file: Option<String>,
 
     /// The type of measurement to use (cycles, insts-retired, perf-counters, noop, vtune)
-    /// when recording the benchmark performance.
-    #[structopt(long, short, default_value = "cycles")]
-    measure: MeasureType,
+    /// when recording the benchmark performance.  This option can be specified more than
+    /// once if to record multiple measurements.  If no measures are specified,
+    /// the "cycles" measure will be used.
+    #[structopt(long = "measure", short = "m", multiple = true)]
+    measures: Vec<MeasureType>,
 
     /// Pass this flag to only run benchmarks over "small" workloads (rather
     /// than the larger, default workloads).
@@ -194,7 +197,12 @@ impl BenchmarkCommand {
                 let stdin = None;
 
                 let mut measurements = Measurements::new(this_arch(), engine, wasm_file);
-                let mut measure = self.measure.build();
+                let mut measure = if self.measures.len() <= 1 {
+                    let measure = self.measures.first().unwrap_or(&MeasureType::Cycles);
+                    measure.build()
+                } else {
+                    Box::new(MultiMeasure::new(self.measures.iter().map(|m| m.build())))
+                };
 
                 // Create the bench API engine and cache it for reuse across all
                 // iterations of this benchmark.
@@ -354,8 +362,12 @@ impl BenchmarkCommand {
                 .arg(self.iterations_per_process.to_string())
                 .arg("--engine")
                 .arg(&engine)
-                .arg("--measure")
-                .arg(self.measure.to_string())
+                .args(
+                    self.measures
+                        .iter()
+                        .map(|m| ["--measure".to_string(), m.to_string()])
+                        .flatten(),
+                )
                 .arg("--raw")
                 .arg("--output-format")
                 // Always use JSON when privately communicating with a

crates/recorder/src/measure/mod.rs

@@ -83,7 +83,9 @@ pub mod counters;
 pub mod insts;
 
 pub mod cycles;
+pub mod multi;
 pub mod noop;
+pub mod time;
 pub mod vtune;
 
 /// [MeasureType] enumerates the implementations of [Measure] and allows us to `build` an instance
@@ -98,6 +100,9 @@ pub enum MeasureType {
     /// No measurement.
     Noop,
 
+    /// Simple time measurement.
+    Time,
+
     /// Measure cycles using, e.g., `RDTSC`.
     Cycles,
 
@@ -117,6 +122,7 @@ impl fmt::Display for MeasureType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             MeasureType::Noop => write!(f, "noop"),
+            MeasureType::Time => write!(f, "time"),
             MeasureType::Cycles => write!(f, "cycles"),
             MeasureType::VTune => write!(f, "vtune"),
             #[cfg(target_os = "linux")]
@@ -132,6 +138,7 @@ impl FromStr for MeasureType {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "noop" => Ok(Self::Noop),
+            "time" => Ok(Self::Time),
             "cycles" => Ok(Self::Cycles),
             "vtune" => Ok(Self::VTune),
             #[cfg(target_os = "linux")]
@@ -150,6 +157,7 @@ impl MeasureType {
     pub fn build(&self) -> Box<dyn Measure> {
         match self {
             Self::Noop => Box::new(noop::NoopMeasure::new()),
+            Self::Time => Box::new(time::TimeMeasure::new()),
             Self::Cycles => Box::new(cycles::CycleMeasure::new()),
             Self::VTune => Box::new(vtune::VTuneMeasure::new()),
             #[cfg(target_os = "linux")]

crates/recorder/src/measure/multi.rs

@@ -0,0 +1,35 @@
+//! Multiplexing measurement in order to take multiple measures
+//! from a single entry point.
+
+use sightglass_data::Phase;
+
+use super::{Measure, Measurements};
+
+pub struct MultiMeasure {
+    measures: Vec<Box<dyn Measure>>,
+}
+
+impl MultiMeasure {
+    pub fn new<I>(measures: I) -> Self
+    where
+        I: IntoIterator<Item = Box<dyn Measure>>,
+    {
+        Self {
+            measures: measures.into_iter().collect(),
+        }
+    }
+}
+
+impl Measure for MultiMeasure {
+    fn start(&mut self, phase: Phase) {
+        for measure in self.measures.iter_mut() {
+            measure.start(phase)
+        }
+    }
+
+    fn end(&mut self, phase: Phase, measurements: &mut Measurements) {
+        for measure in self.measures.iter_mut() {
+            measure.end(phase, measurements)
+        }
+    }
+}

crates/recorder/src/measure/time.rs

@@ -0,0 +1,29 @@
+//! Measure time elapsed time for each benchmark; note that this is
+//! often probably not the best measurement as, unlike cycle counts,
+//! there are likely additional external factors that could skew
+//! results (such as cpu frequency scaling, etc.).
+
+use std::time::Instant;
+
+use super::{Measure, Measurements};
+use sightglass_data::Phase;
+
+pub struct TimeMeasure(Option<Instant>);
+
+impl TimeMeasure {
+    pub fn new() -> Self {
+        Self(None)
+    }
+}
+
+impl Measure for TimeMeasure {
+    fn start(&mut self, _phase: Phase) {
+        let start = Instant::now();
+        self.0 = Some(start);
+    }
+
+    fn end(&mut self, phase: Phase, measurements: &mut Measurements) {
+        let elapsed = self.0.take().unwrap().elapsed();
+        measurements.add(phase, "nanoseconds".into(), elapsed.as_nanos() as u64);
+    }
+}