Rollup merge of rust-lang#87404 - rylev:artifact-size-profiling, r=wesleywiser

Add support for artifact size profiling

This adds support for profiling artifact file sizes (incremental compilation artifacts and query cache to begin with).

Eventually we want to track this in perf.rlo so we can ensure that file sizes do not change dramatically on each pull request.

This relies on support in measureme: rust-lang/measureme#169. Once that lands we can update this PR to not point to a git dependency.

This was worked on together with `@michaelwoerister.`

r? `@wesleywiser`

Author: JohnTitor

Cargo.lock

@@ -2143,6 +2143,20 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "measureme"
+version = "10.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd460fad6e55ca82fa0cd9dab0d315294188fd9ec6efbf4105e5635d4872ef9c"
+dependencies = [
+ "log",
+ "memmap2",
+ "parking_lot",
+ "perf-event-open-sys",
+ "rustc-hash",
+ "smallvec",
+]
+
 [[package]]
 name = "memchr"
 version = "2.4.1"
@@ -2247,7 +2261,7 @@ dependencies = [
  "hex 0.4.2",
  "libc",
  "log",
- "measureme",
+ "measureme 9.1.2",
  "rand 0.8.4",
  "rustc-workspace-hack",
  "rustc_version 0.4.0",
@@ -3235,7 +3249,7 @@ dependencies = [
  "indexmap",
  "jobserver",
  "libc",
- "measureme",
+ "measureme 9.1.2",
  "memmap2",
  "parking_lot",
  "rustc-ap-rustc_graphviz",
@@ -3674,7 +3688,7 @@ dependencies = [
  "bitflags",
  "cstr",
  "libc",
- "measureme",
+ "measureme 10.0.0",
  "rustc-demangle",
  "rustc_arena",
  "rustc_ast",
@@ -3767,7 +3781,7 @@ dependencies = [
  "indexmap",
  "jobserver",
  "libc",
- "measureme",
+ "measureme 10.0.0",
  "memmap2",
  "parking_lot",
  "rustc-hash",
@@ -4292,7 +4306,7 @@ dependencies = [
 name = "rustc_query_impl"
 version = "0.0.0"
 dependencies = [
- "measureme",
+ "measureme 10.0.0",
  "rustc-rayon-core",
  "rustc_ast",
  "rustc_data_structures",

compiler/rustc_codegen_llvm/Cargo.toml

@@ -11,7 +11,7 @@ doctest = false
 bitflags = "1.0"
 cstr = "0.2"
 libc = "0.2"
-measureme = "9.1.0"
+measureme = "10.0.0"
 snap = "1"
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }

compiler/rustc_data_structures/Cargo.toml

@@ -23,7 +23,7 @@ rustc-hash = "1.1.0"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_index = { path = "../rustc_index", package = "rustc_index" }
 bitflags = "1.2.1"
-measureme = "9.1.0"
+measureme = "10.0.0"
 libc = "0.2"
 stacker = "0.1.14"
 tempfile = "3.2"

compiler/rustc_data_structures/src/profiling.rs

@@ -110,12 +110,14 @@ bitflags::bitflags! {
         const FUNCTION_ARGS       = 1 << 6;
         const LLVM                = 1 << 7;
         const INCR_RESULT_HASHING = 1 << 8;
+        const ARTIFACT_SIZES = 1 << 9;
 
         const DEFAULT = Self::GENERIC_ACTIVITIES.bits |
                         Self::QUERY_PROVIDERS.bits |
                         Self::QUERY_BLOCKED.bits |
                         Self::INCR_CACHE_LOADS.bits |
-                        Self::INCR_RESULT_HASHING.bits;
+                        Self::INCR_RESULT_HASHING.bits |
+                        Self::ARTIFACT_SIZES.bits;
 
         const ARGS = Self::QUERY_KEYS.bits | Self::FUNCTION_ARGS.bits;
     }
@@ -136,6 +138,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
     ("args", EventFilter::ARGS),
     ("llvm", EventFilter::LLVM),
     ("incr-result-hashing", EventFilter::INCR_RESULT_HASHING),
+    ("artifact-sizes", EventFilter::ARTIFACT_SIZES),
 ];
 
 /// Something that uniquely identifies a query invocation.
@@ -285,6 +288,33 @@ impl SelfProfilerRef {
         })
     }
 
+    /// Record the size of an artifact that the compiler produces
+    ///
+    /// `artifact_kind` is the class of artifact (e.g., query_cache, object_file, etc.)
+    /// `artifact_name` is an identifier to the specific artifact being stored (usually a filename)
+    #[inline(always)]
+    pub fn artifact_size<A>(&self, artifact_kind: &str, artifact_name: A, size: u64)
+    where
+        A: Borrow<str> + Into<String>,
+    {
+        drop(self.exec(EventFilter::ARTIFACT_SIZES, |profiler| {
+            let builder = EventIdBuilder::new(&profiler.profiler);
+            let event_label = profiler.get_or_alloc_cached_string(artifact_kind);
+            let event_arg = profiler.get_or_alloc_cached_string(artifact_name);
+            let event_id = builder.from_label_and_arg(event_label, event_arg);
+            let thread_id = get_thread_id();
+
+            profiler.profiler.record_integer_event(
+                profiler.artifact_size_event_kind,
+                event_id,
+                thread_id,
+                size,
+            );
+
+            TimingGuard::none()
+        }))
+    }
+
     #[inline(always)]
     pub fn generic_activity_with_args(
         &self,
@@ -372,7 +402,7 @@ impl SelfProfilerRef {
     ) {
         drop(self.exec(event_filter, |profiler| {
             let event_id = StringId::new_virtual(query_invocation_id.0);
-            let thread_id = std::thread::current().id().as_u64().get() as u32;
+            let thread_id = get_thread_id();
 
             profiler.profiler.record_instant_event(
                 event_kind(profiler),
@@ -425,6 +455,7 @@ pub struct SelfProfiler {
     incremental_result_hashing_event_kind: StringId,
     query_blocked_event_kind: StringId,
     query_cache_hit_event_kind: StringId,
+    artifact_size_event_kind: StringId,
 }
 
 impl SelfProfiler {
@@ -447,6 +478,7 @@ impl SelfProfiler {
             profiler.alloc_string("IncrementalResultHashing");
         let query_blocked_event_kind = profiler.alloc_string("QueryBlocked");
         let query_cache_hit_event_kind = profiler.alloc_string("QueryCacheHit");
+        let artifact_size_event_kind = profiler.alloc_string("ArtifactSize");
 
         let mut event_filter_mask = EventFilter::empty();
 
@@ -491,6 +523,7 @@ impl SelfProfiler {
             incremental_result_hashing_event_kind,
             query_blocked_event_kind,
             query_cache_hit_event_kind,
+            artifact_size_event_kind,
         })
     }
 
@@ -561,7 +594,7 @@ impl<'a> TimingGuard<'a> {
         event_kind: StringId,
         event_id: EventId,
     ) -> TimingGuard<'a> {
-        let thread_id = std::thread::current().id().as_u64().get() as u32;
+        let thread_id = get_thread_id();
         let raw_profiler = &profiler.profiler;
         let timing_guard =
             raw_profiler.start_recording_interval_event(event_kind, event_id, thread_id);
@@ -655,6 +688,10 @@ pub fn duration_to_secs_str(dur: std::time::Duration) -> String {
     format!("{:.3}", dur.as_secs_f64())
 }
 
+fn get_thread_id() -> u32 {
+    std::thread::current().id().as_u64().get() as u32
+}
+
 // Memory reporting
 cfg_if! {
     if #[cfg(windows)] {

compiler/rustc_incremental/src/persist/file_format.rs

@@ -95,6 +95,12 @@ where
         return;
     }
 
+    sess.prof.artifact_size(
+        &name.replace(' ', "_"),
+        path_buf.file_name().unwrap().to_string_lossy(),
+        encoder.position() as u64,
+    );
+
     debug!("save: data written to disk successfully");
 }
 

compiler/rustc_query_impl/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 doctest = false
 
 [dependencies]
-measureme = "9.0.0"
+measureme = "10.0.0"
 rustc-rayon-core = "0.3.1"
 tracing = "0.1"
 rustc_ast = { path = "../rustc_ast" }

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -222,7 +222,7 @@ impl<K: DepKind> EncoderState<K> {
         index
     }
 
-    fn finish(self) -> FileEncodeResult {
+    fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let Self { mut encoder, total_node_count, total_edge_count, result, stats: _ } = self;
         let () = result?;
 
@@ -235,7 +235,11 @@ impl<K: DepKind> EncoderState<K> {
         IntEncodedWithFixedSize(edge_count).encode(&mut encoder)?;
         debug!("position: {:?}", encoder.position());
         // Drop the encoder so that nothing is written after the counts.
-        encoder.flush()
+        let result = encoder.flush();
+        // FIXME(rylev): we hardcode the dep graph file name so we don't need a dependency on
+        // rustc_incremental just for that.
+        profiler.artifact_size("dep_graph", "dep-graph.bin", encoder.position() as u64);
+        result
     }
 }
 
@@ -332,6 +336,6 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
 
     pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
-        self.status.into_inner().finish()
+        self.status.into_inner().finish(profiler)
     }
 }

compiler/rustc_session/src/options.rs

@@ -1283,7 +1283,7 @@ options! {
         "specify the events recorded by the self profiler;
         for example: `-Z self-profile-events=default,query-keys`
         all options: none, all, default, generic-activity, query-provider, query-cache-hit
-                     query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm"),
+                     query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
     share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "make the current crate share its generic instantiations"),
     show_span: Option<String> = (None, parse_opt_string, [TRACKED],