Add test suite coverage-map to test coverage mappings emitted by LLVM

We compile each test file to LLVM IR assembly, and then pass that IR to a
dedicated program that can decode LLVM coverage maps and print them in a more
human-readable format. We can then check that output against known-good
snapshots.

This test suite has some advantages over the existing `run-coverage` tests:

- We can test coverage instrumentation without needing to run target binaries.

- We can observe subtle improvements/regressions in the underlying coverage
mappings that don't make a visible difference to coverage reports.

Author: Zalathar

src/bootstrap/builder.rs

@@ -726,6 +726,7 @@ impl<'a> Builder<'a> {
                 test::Tidy,
                 test::Ui,
                 test::RunPassValgrind,
+                test::CoverageMap,
                 test::RunCoverage,
                 test::MirOpt,
                 test::Codegen,

src/bootstrap/test.rs

@@ -1340,6 +1340,12 @@ host_test!(RunMakeFullDeps {
 
 default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" });
 
+default_test!(CoverageMap {
+    path: "tests/coverage-map",
+    mode: "coverage-map",
+    suite: "coverage-map"
+});
+
 host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" });
 host_test!(RunCoverageRustdoc {
     path: "tests/run-coverage-rustdoc",
@@ -1545,6 +1551,14 @@ note: if you're sure you want to do this, please open an issue as to why. In the
                 .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
         }
 
+        if mode == "coverage-map" {
+            let coverage_dump = builder.ensure(tool::CoverageDump {
+                compiler: compiler.with_stage(0),
+                target: compiler.host,
+            });
+            cmd.arg("--coverage-dump-path").arg(coverage_dump);
+        }
+
         if mode == "run-make" || mode == "run-coverage" {
             let rust_demangler = builder
                 .ensure(tool::RustDemangler {

src/tools/compiletest/src/common.rs

@@ -66,6 +66,7 @@ string_enum! {
         JsDocTest => "js-doc-test",
         MirOpt => "mir-opt",
         Assembly => "assembly",
+        CoverageMap => "coverage-map",
         RunCoverage => "run-coverage",
     }
 }
@@ -161,6 +162,9 @@ pub struct Config {
     /// The rust-demangler executable.
     pub rust_demangler_path: Option<PathBuf>,
 
+    /// The coverage-dump executable.
+    pub coverage_dump_path: Option<PathBuf>,
+
     /// The Python executable to use for LLDB and htmldocck.
     pub python: String,
 
@@ -639,6 +643,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
     UI_STDERR_32,
     UI_STDERR_16,
     UI_COVERAGE,
+    UI_COVERAGE_MAP,
 ];
 pub const UI_STDERR: &str = "stderr";
 pub const UI_STDOUT: &str = "stdout";
@@ -649,6 +654,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
 pub const UI_STDERR_32: &str = "32bit.stderr";
 pub const UI_STDERR_16: &str = "16bit.stderr";
 pub const UI_COVERAGE: &str = "coverage";
+pub const UI_COVERAGE_MAP: &str = "cov-map";
 
 /// Absolute path to the directory where all output for all tests in the given
 /// `relative_dir` group should reside. Example:

src/tools/compiletest/src/lib.rs

@@ -48,6 +48,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
         .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
         .optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
+        .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
         .reqopt("", "python", "path to python to use for doc tests", "PATH")
         .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
         .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
@@ -218,6 +219,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         rustc_path: opt_path(matches, "rustc-path"),
         rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
         rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
+        coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
         python: matches.opt_str("python").unwrap(),
         jsondocck_path: matches.opt_str("jsondocck-path"),
         jsondoclint_path: matches.opt_str("jsondoclint-path"),

src/tools/compiletest/src/runtest.rs

@@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
-use crate::common::{Pretty, RunCoverage, RunPassValgrind};
-use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT};
+use crate::common::{CoverageMap, Pretty, RunCoverage, RunPassValgrind};
+use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
 use crate::compute_diff::{write_diff, write_filtered_diff};
 use crate::errors::{self, Error, ErrorKind};
 use crate::header::TestProps;
@@ -254,6 +254,7 @@ impl<'test> TestCx<'test> {
             MirOpt => self.run_mir_opt_test(),
             Assembly => self.run_assembly_test(),
             JsDocTest => self.run_js_doc_test(),
+            CoverageMap => self.run_coverage_map_test(),
             RunCoverage => self.run_coverage_test(),
         }
     }
@@ -467,6 +468,46 @@ impl<'test> TestCx<'test> {
         }
     }
 
+    fn run_coverage_map_test(&self) {
+        let Some(coverage_dump_path) = &self.config.coverage_dump_path else {
+            self.fatal("missing --coverage-dump");
+        };
+
+        let proc_res = self.compile_test_and_save_ir();
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+        drop(proc_res);
+
+        let llvm_ir_path = self.output_base_name().with_extension("ll");
+
+        let mut dump_command = Command::new(coverage_dump_path);
+        dump_command.arg(llvm_ir_path);
+        let proc_res = self.run_command_to_procres(&mut dump_command);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("coverage-dump failed!", &proc_res);
+        }
+
+        let kind = UI_COVERAGE_MAP;
+
+        let expected_coverage_dump = self.load_expected_output(kind);
+        let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
+
+        let coverage_dump_errors = self.compare_output(
+            kind,
+            &actual_coverage_dump,
+            &expected_coverage_dump,
+            self.props.compare_output_lines_by_subset,
+        );
+
+        if coverage_dump_errors > 0 {
+            self.fatal_proc_rec(
+                &format!("{coverage_dump_errors} errors occurred comparing coverage output."),
+                &proc_res,
+            );
+        }
+    }
+
     fn run_coverage_test(&self) {
         let should_run = self.run_if_enabled();
         let proc_res = self.compile_test(should_run, Emit::None);
@@ -650,6 +691,10 @@ impl<'test> TestCx<'test> {
         let mut cmd = Command::new(tool_path);
         configure_cmd_fn(&mut cmd);
 
+        self.run_command_to_procres(&mut cmd)
+    }
+
+    fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes {
         let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`"));
 
         let proc_res = ProcRes {
@@ -2321,9 +2366,11 @@ impl<'test> TestCx<'test> {
                     }
                 }
                 DebugInfo => { /* debuginfo tests must be unoptimized */ }
-                RunCoverage => {
-                    // Coverage reports are affected by optimization level, and
-                    // the current snapshots assume no optimization by default.
+                CoverageMap | RunCoverage => {
+                    // Coverage mappings and coverage reports are affected by
+                    // optimization level, so they ignore the optimize-tests
+                    // setting and set an optimization level in their mode's
+                    // compile flags (below) or in per-test `compile-flags`.
                 }
                 _ => {
                     rustc.arg("-O");
@@ -2392,8 +2439,22 @@ impl<'test> TestCx<'test> {
 
                 rustc.arg(dir_opt);
             }
+            CoverageMap => {
+                rustc.arg("-Cinstrument-coverage");
+                // These tests only compile to MIR, so they don't need the
+                // profiler runtime to be present.
+                rustc.arg("-Zno-profiler-runtime");
+                // Coverage mappings are sensitive to MIR optimizations, and
+                // the current snapshots assume `opt-level=2` unless overridden
+                // by `compile-flags`.
+                rustc.arg("-Copt-level=2");
+            }
             RunCoverage => {
                 rustc.arg("-Cinstrument-coverage");
+                // Coverage reports are sometimes sensitive to optimizations,
+                // and the current snapshots assume no optimization unless
+                // overridden by `compile-flags`.
+                rustc.arg("-Copt-level=0");
             }
             RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake
             | CodegenUnits | JsDocTest | Assembly => {

tests/coverage-map/if.cov-map

@@ -0,0 +1,15 @@
+Function name: if::main
+Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 03, 01, 02, 0c, 05, 02, 0d, 02, 06, 02, 02, 06, 00, 07, 07, 01, 05, 01, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 2
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+Number of file 0 mappings: 4
+- Code(Counter(0)) at (prev + 3, 1) to (start + 2, 12)
+- Code(Counter(1)) at (prev + 2, 13) to (start + 2, 6)
+- Code(Expression(0, Sub)) at (prev + 2, 6) to (start + 0, 7)
+    = (c0 - c1)
+- Code(Expression(1, Add)) at (prev + 1, 5) to (start + 1, 2)
+    = (c1 + (c0 - c1))
+

tests/coverage-map/if.rs

@@ -0,0 +1,9 @@
+// compile-flags: --edition=2021
+
+fn main() {
+    let cond = std::env::args().len() == 1;
+    if cond {
+        println!("true");
+    }
+    println!("done");
+}

tests/coverage-map/long_and_wide.cov-map

@@ -0,0 +1,32 @@
+Function name: long_and_wide::far_function
+Raw bytes (10): 0x[01, 01, 00, 01, 01, 96, 01, 01, 00, 15]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 150, 1) to (start + 0, 21)
+
+Function name: long_and_wide::long_function
+Raw bytes (10): 0x[01, 01, 00, 01, 01, 10, 01, 84, 01, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 16, 1) to (start + 132, 2)
+
+Function name: long_and_wide::main
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 07, 01, 04, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 7, 1) to (start + 4, 2)
+
+Function name: long_and_wide::wide_function
+Raw bytes (10): 0x[01, 01, 00, 01, 01, 0e, 01, 00, 8b, 01]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 139)
+