Add support for capturing self-profile data during xperf

Also fake the cpu-clock counter because it's necessary to get the
self-profiling data to display in the site.

Author: wesleywiser

collector/src/etw_parser.rs

@@ -287,10 +287,11 @@ fn parse_events(r: &mut dyn BufRead, headers: Vec<EventHeader>) -> anyhow::Resul
     Ok(events)
 }
 
-#[derive(Debug, Eq, PartialEq)]
+#[derive(Debug, PartialEq)]
 pub struct Counters {
     pub instructions_retired: u64,
     pub total_cycles: u64,
+    pub cpu_clock: f64,
 }
 
 impl std::ops::Add for Counters {
@@ -300,6 +301,7 @@ impl std::ops::Add for Counters {
         Self {
             instructions_retired: self.instructions_retired + rhs.instructions_retired,
             total_cycles: self.total_cycles + rhs.total_cycles,
+            cpu_clock: self.cpu_clock + rhs.cpu_clock,
         }
     }
 }
@@ -314,6 +316,7 @@ impl std::ops::Sub for Counters {
         Self {
             instructions_retired: self.instructions_retired - rhs.instructions_retired,
             total_cycles: self.total_cycles - rhs.total_cycles,
+            cpu_clock: self.cpu_clock - rhs.cpu_clock,
         }
     }
 }
@@ -323,6 +326,11 @@ impl Default for Counters {
         Self {
             instructions_retired: 0,
             total_cycles: 0,
+            // FIXME(wesleywiser): We should be properly calculating this value by taking the total time
+            // each rustc thread runs per core and adding them togther. This placeholder value is here
+            // so that we can still render the detailed query statistics page (although the "Time (%)"
+            // column will show the wrong value).
+            cpu_clock: 1.0,
         }
     }
 }
@@ -332,6 +340,8 @@ impl From<&Pmc> for Counters {
         Self {
             instructions_retired: pmc.instructions_retired,
             total_cycles: pmc.total_cycles,
+            // FIXME(wesleywiser): see comment in `<Counters as Default>::default()`.
+            cpu_clock: 0.0,
         }
     }
 }
@@ -593,9 +603,10 @@ FirstReliableCSwitchEventTimeStamp, 6016
             })
         ];
 
-        let expected= Counters {
+        let expected = Counters {
             instructions_retired: 1561453,
             total_cycles: 366237,
+            cpu_clock: 1.0,
         };
 
         assert_eq!(expected, super::process_events(&events)?);

collector/src/execute.rs

@@ -1418,6 +1418,7 @@ fn process_stat_output(
 
             stats.insert("cycles".into(), counters.total_cycles as f64);
             stats.insert("instructions:u".into(), counters.instructions_retired as f64);
+            stats.insert("cpu-clock".into(), counters.cpu_clock);
             continue;
         }
 

collector/src/rustc-fake.rs

@@ -3,6 +3,7 @@ use std::env;
 use std::ffi::OsString;
 use std::fs;
 use std::path::Path;
+use std::path::PathBuf;
 use std::process::Command;
 use std::time::{Duration, Instant};
 
@@ -77,67 +78,11 @@ fn main() {
                 print_memory();
                 print_time(dur);
                 if wrapper == "perf-stat-self-profile" {
-                    let crate_name = args
-                        .windows(2)
-                        .find(|args| args[0] == "--crate-name")
-                        .and_then(|args| args[1].to_str())
-                        .expect("rustc to be invoked with crate name");
-                    let mut prefix = None;
-                    let mut full_path = None;
-                    // We don't know the pid of rustc, and can't easily get it -- we only know the
-                    // `perf` pid. So just blindly look in the directory to hopefully find it.
-                    for entry in fs::read_dir(&prof_out_dir).unwrap() {
-                        let entry = entry.unwrap();
-                        if entry
-                            .file_name()
-                            .to_str()
-                            .map_or(false, |s| s.starts_with(crate_name))
-                        {
-                            if entry.file_name().to_str().unwrap().ends_with("mm_profdata") {
-                                full_path = Some(entry.path());
-                                break;
-                            }
-                            let file = entry.file_name().to_str().unwrap().to_owned();
-                            let new_prefix = Some(file[..file.find('.').unwrap()].to_owned());
-                            assert!(
-                                prefix.is_none() || prefix == new_prefix,
-                                "prefix={:?}, new_prefix={:?}",
-                                prefix,
-                                new_prefix
-                            );
-                            prefix = new_prefix;
-                        }
-                    }
-                    if let Some(profile_data) = full_path {
-                        // measureme 0.8 has a single file
-                        println!("!self-profile-file:{}", profile_data.to_str().unwrap());
-                        let filename = profile_data.file_name().unwrap().to_str().unwrap();
-                        let json = match run_summarize("summarize", &prof_out_dir, filename) {
-                            Ok(s) => s,
-                            Err(e1) => {
-                                match run_summarize("summarize-9.0", &prof_out_dir, filename) {
-                                    Ok(s) => s,
-                                    Err(e2) => {
-                                        panic!("failed to run summarize and summarize-9.0. Errors:\nsummarize: {:?}\nsummarize-9.0: {:?}", e1, e2);
-                                    }
-                                }
-                            }
-                        };
-                        println!("!self-profile-output:{}", json);
-                    } else {
-                        let prefix = prefix.expect(&format!("found prefix {:?}", prof_out_dir));
-                        let json = run_summarize("summarize", &prof_out_dir, &prefix)
-                            .or_else(|_| run_summarize("summarize-0.7", &prof_out_dir, &prefix))
-                            .expect("able to run summarize or summarize-0.7");
-                        println!("!self-profile-dir:{}", prof_out_dir.to_str().unwrap());
-                        println!("!self-profile-prefix:{}", prefix);
-                        println!("!self-profile-output:{}", json);
-                    }
+                    process_self_profile_output(prof_out_dir, &args[..]);
                 }
             }
 
             "xperf-stat" | "xperf-stat-self-profile" => {
-                // TODO handle self-profiling
                 // Read the path to tracelog.exe from either an environment variable, falling back to assuming it's on the PATH.
                 let tracelog = std::env::var("TRACELOG").unwrap_or("tracelog.exe".to_string());
                 let mut cmd = Command::new(tracelog);
@@ -148,7 +93,17 @@ fn main() {
                 assert!(status.success(), "tracelog did not complete successfully");
 
                 let mut tool = Command::new(tool);
-                tool.args(args);
+                tool.args(&args);
+
+                let prof_out_dir = std::env::current_dir().unwrap().join("self-profile-output");
+                if wrapper == "xperf-stat-self-profile" {
+                    tool.arg(&format!(
+                        "-Zself-profile={}",
+                        prof_out_dir.to_str().unwrap()
+                    ));
+                    let _ = fs::remove_dir_all(&prof_out_dir);
+                    let _ = fs::create_dir_all(&prof_out_dir);
+                }
 
                 let start = Instant::now();
                 let status = tool.status().expect("tool failed to start");
@@ -172,6 +127,10 @@ fn main() {
 
                 let counters_file = std::env::current_dir().unwrap().join("pmc_counters.txt");
                 println!("!counters-file:{}", counters_file.to_str().unwrap());
+
+                if wrapper == "xperf-stat-self-profile" {
+                    process_self_profile_output(prof_out_dir, &args[..]);
+                }
             }
 
             "self-profile" => {
@@ -325,6 +284,65 @@ fn main() {
     }
 }
 
+fn process_self_profile_output(prof_out_dir: PathBuf, args: &[OsString]) {
+    let crate_name = args
+        .windows(2)
+        .find(|args| args[0] == "--crate-name")
+        .and_then(|args| args[1].to_str())
+        .expect("rustc to be invoked with crate name");
+    let mut prefix = None;
+    let mut full_path = None;
+    // We don't know the pid of rustc, and can't easily get it -- we only know the
+    // `perf` pid. So just blindly look in the directory to hopefully find it.
+    for entry in fs::read_dir(&prof_out_dir).unwrap() {
+        let entry = entry.unwrap();
+        if entry
+            .file_name()
+            .to_str()
+            .map_or(false, |s| s.starts_with(crate_name))
+        {
+            if entry.file_name().to_str().unwrap().ends_with("mm_profdata") {
+                full_path = Some(entry.path());
+                break;
+            }
+            let file = entry.file_name().to_str().unwrap().to_owned();
+            let new_prefix = Some(file[..file.find('.').unwrap()].to_owned());
+            assert!(
+                prefix.is_none() || prefix == new_prefix,
+                "prefix={:?}, new_prefix={:?}",
+                prefix,
+                new_prefix
+            );
+            prefix = new_prefix;
+        }
+    }
+    if let Some(profile_data) = full_path {
+        // measureme 0.8 has a single file
+        println!("!self-profile-file:{}", profile_data.to_str().unwrap());
+        let filename = profile_data.file_name().unwrap().to_str().unwrap();
+        let json = match run_summarize("summarize", &prof_out_dir, filename) {
+            Ok(s) => s,
+            Err(e1) => {
+                match run_summarize("summarize-9.0", &prof_out_dir, filename) {
+                    Ok(s) => s,
+                    Err(e2) => {
+                        panic!("failed to run summarize and summarize-9.0. Errors:\nsummarize: {:?}\nsummarize-9.0: {:?}", e1, e2);
+                    }
+                }
+            }
+        };
+        println!("!self-profile-output:{}", json);
+    } else {
+        let prefix = prefix.expect(&format!("found prefix {:?}", prof_out_dir));
+        let json = run_summarize("summarize", &prof_out_dir, &prefix)
+            .or_else(|_| run_summarize("summarize-0.7", &prof_out_dir, &prefix))
+            .expect("able to run summarize or summarize-0.7");
+        println!("!self-profile-dir:{}", prof_out_dir.to_str().unwrap());
+        println!("!self-profile-prefix:{}", prefix);
+        println!("!self-profile-output:{}", json);
+    }
+}
+
 /// Run a command via bash, in order to redirect its output to a file.
 /// `redirect` should be something like "> out" or "2> out".
 fn bash_command(tool: OsString, args: Vec<OsString>, redirect: &str) -> Command {