coverage: Eliminate more counters by giving them to unreachable nodes

When preparing a function's coverage counters and metadata during codegen, any
part of the original coverage graph that was removed by MIR optimizations can
be treated as having an execution count of zero.

Somewhat counter-intuitively, if we give those unreachable nodes a _higher_
priority for receiving physical counters (instead of counter expressions), that
ends up reducing the total number of physical counters needed.

This works because if a node is unreachable, we don't actually create a
physical counter for it. Instead that node gets a fixed zero counter, and any
other node that would have relied on that physical counter in its counter
expression can just ignore that term completely.

Author: Zalathar

compiler/rustc_mir_transform/src/coverage/counters.rs

@@ -114,12 +114,8 @@ pub(crate) fn transcribe_counters(
         let mut new_counters_for_sites = |sites: Vec<BasicCoverageBlock>| {
             sites.into_iter().map(|node| new.ensure_phys_counter(node)).collect::<Vec<_>>()
         };
-        let mut pos = new_counters_for_sites(pos);
-        let mut neg = new_counters_for_sites(neg);
-
-        // These sorts are also not strictly necessary; see above.
-        pos.sort();
-        neg.sort();
+        let pos = new_counters_for_sites(pos);
+        let neg = new_counters_for_sites(neg);
 
         let pos_counter = new.make_sum(&pos).unwrap_or(CovTerm::Zero);
         let new_counter = new.make_subtracted_sum(pos_counter, &neg);

compiler/rustc_mir_transform/src/coverage/query.rs

@@ -123,11 +123,20 @@ fn coverage_ids_info<'tcx>(
         }
     }
 
-    // FIXME(Zalathar): It should be possible to sort `priority_list[1..]` by
-    // `!bcbs_seen.contains(bcb)` to simplify the mappings even further, at the
-    // expense of some churn in the tests. When doing so, also consider removing
-    // the sorts in `transcribe_counters`.
-    let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &fn_cov_info.priority_list);
+    // Clone the priority list so that we can re-sort it.
+    let mut priority_list = fn_cov_info.priority_list.clone();
+    // The first ID in the priority list represents the synthetic "sink" node,
+    // and must remain first so that it _never_ gets a physical counter.
+    debug_assert_eq!(priority_list[0], priority_list.iter().copied().max().unwrap());
+    assert!(!bcbs_seen.contains(priority_list[0]));
+    // Partition the priority list, so that unreachable nodes (removed by MIR opts)
+    // are sorted later and therefore are _more_ likely to get a physical counter.
+    // This is counter-intuitive, but it means that `transcribe_counters` can
+    // easily skip those unused physical counters and replace them with zero.
+    // (The original ordering remains in effect within both partitions.)
+    priority_list[1..].sort_by_key(|&bcb| !bcbs_seen.contains(bcb));
+
+    let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &priority_list);
     let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen);
 
     let CoverageCounters {

tests/coverage/assert_not.cov-map

@@ -1,13 +1,13 @@
 Function name: assert_not::main
-Raw bytes (29): 0x[01, 01, 00, 05, 01, 06, 01, 01, 12, 05, 02, 05, 00, 14, 09, 01, 05, 00, 14, 0d, 01, 05, 00, 16, 0d, 01, 01, 00, 02]
+Raw bytes (29): 0x[01, 01, 00, 05, 01, 06, 01, 01, 12, 01, 02, 05, 00, 14, 01, 01, 05, 00, 14, 01, 01, 05, 00, 16, 01, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 5
 - Code(Counter(0)) at (prev + 6, 1) to (start + 1, 18)
-- Code(Counter(1)) at (prev + 2, 5) to (start + 0, 20)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 20)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 22)
-- Code(Counter(3)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c3
+- Code(Counter(0)) at (prev + 2, 5) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 22)
+- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
+Highest counter ID seen: c0
 

tests/coverage/async2.cov-map

@@ -8,16 +8,16 @@ Number of file 0 mappings: 1
 Highest counter ID seen: c0
 
 Function name: async2::async_func::{closure#0}
-Raw bytes (24): 0x[01, 01, 00, 04, 01, 0f, 17, 03, 09, 05, 03, 0a, 02, 06, 00, 02, 05, 00, 06, 01, 01, 01, 00, 02]
+Raw bytes (24): 0x[01, 01, 00, 04, 01, 0f, 17, 03, 09, 01, 03, 0a, 02, 06, 00, 02, 05, 00, 06, 01, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 15, 23) to (start + 3, 9)
-- Code(Counter(1)) at (prev + 3, 10) to (start + 2, 6)
+- Code(Counter(0)) at (prev + 3, 10) to (start + 2, 6)
 - Code(Zero) at (prev + 2, 5) to (start + 0, 6)
 - Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c1
+Highest counter ID seen: c0
 
 Function name: async2::async_func_just_println
 Raw bytes (9): 0x[01, 01, 00, 01, 01, 17, 01, 00, 24]
@@ -47,14 +47,14 @@ Number of file 0 mappings: 1
 Highest counter ID seen: c0
 
 Function name: async2::non_async_func
-Raw bytes (24): 0x[01, 01, 00, 04, 01, 07, 01, 03, 09, 05, 03, 0a, 02, 06, 00, 02, 05, 00, 06, 01, 01, 01, 00, 02]
+Raw bytes (24): 0x[01, 01, 00, 04, 01, 07, 01, 03, 09, 01, 03, 0a, 02, 06, 00, 02, 05, 00, 06, 01, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 7, 1) to (start + 3, 9)
-- Code(Counter(1)) at (prev + 3, 10) to (start + 2, 6)
+- Code(Counter(0)) at (prev + 3, 10) to (start + 2, 6)
 - Code(Zero) at (prev + 2, 5) to (start + 0, 6)
 - Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c1
+Highest counter ID seen: c0
 

tests/coverage/bad_counter_ids.cov-map

@@ -20,25 +20,25 @@ Number of file 0 mappings: 3
 Highest counter ID seen: c0
 
 Function name: bad_counter_ids::eq_good
-Raw bytes (14): 0x[01, 01, 00, 02, 01, 10, 01, 02, 1f, 05, 03, 01, 00, 02]
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 10, 01, 02, 1f, 01, 03, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 2
 - Code(Counter(0)) at (prev + 16, 1) to (start + 2, 31)
-- Code(Counter(1)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c1
+- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
+Highest counter ID seen: c0
 
 Function name: bad_counter_ids::eq_good_message
-Raw bytes (19): 0x[01, 01, 00, 03, 01, 15, 01, 02, 0f, 00, 02, 20, 00, 2b, 05, 01, 01, 00, 02]
+Raw bytes (19): 0x[01, 01, 00, 03, 01, 15, 01, 02, 0f, 00, 02, 20, 00, 2b, 01, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 21, 1) to (start + 2, 15)
 - Code(Zero) at (prev + 2, 32) to (start + 0, 43)
-- Code(Counter(1)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c1
+- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
+Highest counter ID seen: c0
 
 Function name: bad_counter_ids::ne_bad
 Raw bytes (14): 0x[01, 01, 00, 02, 01, 2e, 01, 02, 1f, 00, 03, 01, 00, 02]
@@ -51,15 +51,15 @@ Number of file 0 mappings: 2
 Highest counter ID seen: c0
 
 Function name: bad_counter_ids::ne_bad_message
-Raw bytes (19): 0x[01, 01, 00, 03, 01, 33, 01, 02, 0f, 05, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
+Raw bytes (19): 0x[01, 01, 00, 03, 01, 33, 01, 02, 0f, 01, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 51, 1) to (start + 2, 15)
-- Code(Counter(1)) at (prev + 2, 32) to (start + 0, 43)
+- Code(Counter(0)) at (prev + 2, 32) to (start + 0, 43)
 - Code(Zero) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c1
+Highest counter ID seen: c0
 
 Function name: bad_counter_ids::ne_good
 Raw bytes (14): 0x[01, 01, 00, 02, 01, 1a, 01, 02, 1f, 01, 03, 01, 00, 02]