diff --git a/Cargo.lock b/Cargo.lock index ddefc3f08..3d66c9e23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" dependencies = [ "cfg-if", - "getrandom 0.2.8", + "getrandom 0.2.16", "once_cell", "version_check", "zerocopy", @@ -39,6 +39,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -265,9 +274,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "camino" @@ -318,10 +327,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -407,7 +417,7 @@ dependencies = [ "env_logger", "flate2", "futures", - "hashbrown", + "hashbrown 0.14.5", "humansize", "jobserver", "libc", @@ -480,9 +490,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" @@ -645,7 +655,7 @@ dependencies = [ "csv", "env_logger", "futures-util", - "hashbrown", + "hashbrown 0.14.5", "intern", "log", "native-tls", @@ -711,9 +721,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -832,6 +842,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" + [[package]] name = "flagset" version = "0.4.6" @@ -854,6 +870,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -979,9 +1001,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", @@ -1012,7 +1034,7 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.20", "bstr", "fnv", "log", @@ -1060,13 +1082,24 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashlink" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -1150,9 +1183,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -1372,7 +1405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -1430,16 +1463,16 @@ version = "0.1.0" dependencies = [ "arc-swap", "bumpalo", - "hashbrown", + "hashbrown 0.14.5", "parking_lot", "serde", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" @@ -1534,9 +1567,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.179" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" [[package]] name = "libm" @@ -1602,11 +1635,11 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.0" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown", + "hashbrown 0.15.5", ] [[package]] @@ -1977,9 +2010,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2147,7 +2180,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.16", ] [[package]] @@ -2206,27 +2239,39 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ - "getrandom 0.2.8", + "getrandom 0.2.16", "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ - "aho-corasick", + "aho-corasick 1.1.4", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick 1.1.4", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" @@ -2452,9 +2497,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -2615,7 +2660,7 @@ dependencies = [ "database", "env_logger", "futures", - "hashbrown", + "hashbrown 0.14.5", "headers", "hex", "hmac", @@ -2755,9 +2800,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -2906,18 +2951,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", @@ -3097,26 +3142,25 @@ dependencies = [ [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", ] diff --git a/collector/Cargo.toml b/collector/Cargo.toml index 4e35ce576..bb079112e 100644 --- a/collector/Cargo.toml +++ b/collector/Cargo.toml @@ -16,7 +16,7 @@ log = { workspace = true } reqwest = { workspace = true, features = ["blocking", "json"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } -tokio = { workspace = true, features = ["rt", "process"] } +tokio = { workspace = true, features = ["rt", "process", "fs"] } cfg-if = "1" thiserror = "2" diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index 19f1e00ee..44b48c00c 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -1,5 +1,13 @@ #![recursion_limit = "1024"] +use anyhow::Context; +use chrono::Utc; +use clap::builder::TypedValueParser; +use clap::{Arg, Parser}; +use collector::compare::compare_artifacts; +use hashbrown::HashSet; +use humansize::{format_size, BINARY}; +use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use std::cmp::{Ordering, Reverse}; use std::collections::HashMap; use std::ffi::OsStr; @@ -15,15 +23,6 @@ use std::process::Command; use std::str::FromStr; use std::time::Duration; use std::{str, time::Instant}; - -use anyhow::Context; -use chrono::Utc; -use clap::builder::TypedValueParser; -use clap::{Arg, Parser}; -use collector::compare::compare_artifacts; -use hashbrown::HashSet; -use humansize::{format_size, BINARY}; -use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator}; use tabled::builder::Builder; use tabled::settings::object::{Columns, Rows}; use tabled::settings::style::Border; @@ -58,7 +57,10 @@ use collector::toolchain::{ }; use collector::utils::cachegrind::cachegrind_diff; use collector::utils::{is_installed, wait_for_future}; -use collector::{command_output, utils, CollectorCtx, CollectorStepBuilder}; +use collector::{ + command_output, utils, CollectorCtx, CollectorStepBuilder, LocalSelfProfileStorage, + S3SelfProfileStorage, SelfProfileStorage, +}; use database::{ ArtifactId, ArtifactIdNumber, BenchmarkJob, BenchmarkJobConclusion, CollectorConfig, Commit, CommitType, Connection, Pool, @@ -111,7 +113,7 @@ struct CompileBenchmarkConfig { scenarios: Vec, backends: Vec, iterations: Option, - is_self_profile: bool, + self_profile_storage: Option>, bench_rustc: bool, targets: Vec, } @@ -1038,7 +1040,13 @@ fn main_result() -> anyhow::Result { scenarios, backends, iterations: Some(iterations), - is_self_profile: self_profile.self_profile, + self_profile_storage: if self_profile.self_profile { + Some(Box::new(LocalSelfProfileStorage::new(Path::new( + "self-profile-storage", + )))) + } else { + None + }, bench_rustc: bench_rustc.bench_rustc, targets: vec![Target::default()], }; @@ -1657,7 +1665,11 @@ async fn create_benchmark_configs( scenarios: Scenario::all(), backends: vec![job.backend().into()], iterations: if is_release { Some(3) } else { None }, - is_self_profile: !is_release, + self_profile_storage: if !is_release { + Some(Box::new(S3SelfProfileStorage::new())) + } else { + None + }, bench_rustc, targets: vec![job.target().into()], }) @@ -2165,7 +2177,7 @@ async fn bench_published_artifact( scenarios, backends: vec![CodegenBackend::Llvm], iterations: Some(3), - is_self_profile: false, + self_profile_storage: None, bench_rustc: false, targets: vec![Target::default()], }), @@ -2233,7 +2245,7 @@ async fn bench_compile( benchmark_name, &shared.artifact_id, collector, - config.is_self_profile, + config.self_profile_storage.as_deref(), ); let result = measure(&mut processor).await; if let Err(s) = result { diff --git a/collector/src/compile/execute/bencher.rs b/collector/src/compile/execute/bencher.rs index 2832ea69e..d65489cfc 100644 --- a/collector/src/compile/execute/bencher.rs +++ b/collector/src/compile/execute/bencher.rs @@ -8,20 +8,19 @@ use crate::compile::execute::{ rustc, DeserializeStatError, PerfTool, ProcessOutputData, Processor, Retry, SelfProfileFiles, Stats, }; +use crate::self_profile::SelfProfileId; use crate::toolchain::Toolchain; use crate::utils::git::get_rustc_perf_commit; -use crate::CollectorCtx; -use anyhow::Context; +use crate::{CollectorCtx, SelfProfileStorage}; use database::CollectionId; use futures::stream::FuturesUnordered; use futures::{future, StreamExt}; -use std::collections::VecDeque; use std::future::Future; -use std::io::Read; -use std::path::PathBuf; use std::pin::Pin; use std::process::Command; +use std::time::Instant; use std::{env, process}; +use tokio::task::JoinSet; pub struct RecordedSelfProfile { collection: CollectionId, @@ -45,7 +44,7 @@ pub struct BenchProcessor<'a> { artifact: &'a database::ArtifactId, collector_ctx: &'a CollectorCtx, is_first_collection: bool, - is_self_profile: bool, + self_profile_storage: Option<&'a dyn SelfProfileStorage>, tries: u8, self_profiles: Vec, } @@ -56,7 +55,7 @@ impl<'a> BenchProcessor<'a> { benchmark: &'a BenchmarkName, artifact: &'a database::ArtifactId, collector_ctx: &'a CollectorCtx, - is_self_profile: bool, + self_profile_storage: Option<&'a dyn SelfProfileStorage>, ) -> Self { // Check we have `perf` or (`xperf.exe` and `tracelog.exe`) available. if cfg!(unix) { @@ -81,7 +80,7 @@ impl<'a> BenchProcessor<'a> { artifact, collector_ctx, is_first_collection: true, - is_self_profile, + self_profile_storage, tries: 0, self_profiles: vec![], } @@ -136,7 +135,7 @@ impl<'a> BenchProcessor<'a> { impl Processor for BenchProcessor<'_> { fn perf_tool(&self) -> PerfTool { - if self.is_first_collection && self.is_self_profile { + if self.is_first_collection && self.self_profile_storage.is_some() { if cfg!(unix) { PerfTool::BenchTool(Bencher::PerfStatSelfProfile) } else { @@ -252,7 +251,7 @@ impl Processor for BenchProcessor<'_> { fn postprocess_results<'b>(&'b mut self) -> Pin + 'b>> { Box::pin(async move { - if env::var_os("RUSTC_PERF_UPLOAD_TO_S3").is_some() { + if let Some(self_profile_storage) = self.self_profile_storage { let futs = self .self_profiles .iter() @@ -268,86 +267,37 @@ impl Processor for BenchProcessor<'_> { .collect::>(); future::join_all(futs).await; - // Upload profiles to S3. Buffer up to 10 uploads at a time. - let mut uploads: VecDeque = VecDeque::new(); + let start = Instant::now(); + let profile_count = self.self_profiles.len(); + + // Buffer up to 10 self-profile stores at a time. + let mut futures = JoinSet::new(); for profile in self.self_profiles.drain(..) { - if uploads.len() == 10 { - uploads.pop_front().unwrap().wait(); + if futures.len() == 10 { + if let Err(error) = futures.join_next().await.unwrap().unwrap() { + log::error!("Failed to store self-profile result: {error:?}"); + } } - // FIXME: Record codegen backend in the self profile name - let prefix = PathBuf::from("self-profile") - .join(self.collector_ctx.artifact_row_id.0.to_string()) - .join(self.benchmark.0.as_str()) - .join(profile.profile.to_string()) - .join(profile.scenario.to_id()); - let upload = - SelfProfileS3Upload::new(prefix, profile.collection, profile.files); - uploads.push_back(upload); + let id = SelfProfileId { + artifact_id_number: self.collector_ctx.artifact_row_id, + collection: profile.collection, + benchmark: self.benchmark.clone(), + profile: profile.profile, + scenario: profile.scenario, + }; + futures.spawn(self_profile_storage.store(id, profile.files)); } - for upload in uploads { - upload.wait(); + for result in futures.join_all().await { + if let Err(error) = result { + log::error!("Failed to store self-profile result: {error:?}"); + } } + log::trace!( + "Stored {profile_count} self-profile(s) in {}s", + start.elapsed().as_secs_f64() + ); } }) } } - -/// Uploads self-profile results to S3 -struct SelfProfileS3Upload( - std::process::Child, - // This field is used only for its Drop impl - #[allow(unused)] tempfile::NamedTempFile, -); - -impl SelfProfileS3Upload { - fn new( - prefix: PathBuf, - collection: database::CollectionId, - files: SelfProfileFiles, - ) -> SelfProfileS3Upload { - // Files are placed at - // * self-profile//// - // /self-profile-.{extension} - let upload = tempfile::NamedTempFile::new() - .context("create temporary file") - .unwrap(); - let filename = match files { - SelfProfileFiles::Eight { file } => { - let data = std::fs::read(file).expect("read profile data"); - let mut data = snap::read::FrameEncoder::new(&data[..]); - let mut compressed = Vec::new(); - data.read_to_end(&mut compressed).expect("compressed"); - std::fs::write(upload.path(), &compressed).expect("write compressed profile data"); - - format!("self-profile-{collection}.mm_profdata.sz") - } - }; - - let child = Command::new("aws") - .arg("s3") - .arg("cp") - .arg("--storage-class") - .arg("INTELLIGENT_TIERING") - .arg("--only-show-errors") - .arg(upload.path()) - .arg(format!( - "s3://rustc-perf/{}", - &prefix.join(filename).to_str().unwrap() - )) - .spawn() - .expect("spawn aws"); - - SelfProfileS3Upload(child, upload) - } - - fn wait(mut self) { - let start = std::time::Instant::now(); - let status = self.0.wait().expect("waiting for child"); - if !status.success() { - panic!("S3 upload failed: {status:?}"); - } - - log::trace!("uploaded to S3, additional wait: {:?}", start.elapsed()); - } -} diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs index 1792af944..f54ae246b 100644 --- a/collector/src/compile/execute/mod.rs +++ b/collector/src/compile/execute/mod.rs @@ -671,7 +671,7 @@ enum DeserializeStatError { IOError(#[from] std::io::Error), } -enum SelfProfileFiles { +pub enum SelfProfileFiles { Eight { file: PathBuf }, } diff --git a/collector/src/lib.rs b/collector/src/lib.rs index 2769e2573..2af7fbc8e 100644 --- a/collector/src/lib.rs +++ b/collector/src/lib.rs @@ -11,11 +11,13 @@ pub mod codegen; pub mod compare; pub mod compile; pub mod runtime; +mod self_profile; pub mod toolchain; pub mod utils; use crate::compile::benchmark::{Benchmark, BenchmarkName}; use crate::runtime::{BenchmarkGroup, BenchmarkSuite}; +pub use crate::self_profile::{LocalSelfProfileStorage, S3SelfProfileStorage, SelfProfileStorage}; use database::selector::CompileTestCase; use database::{ArtifactId, ArtifactIdNumber, Connection}; use hashbrown::HashSet; diff --git a/collector/src/self_profile.rs b/collector/src/self_profile.rs new file mode 100644 index 000000000..14f06e8ec --- /dev/null +++ b/collector/src/self_profile.rs @@ -0,0 +1,141 @@ +use crate::compile::benchmark::BenchmarkName; +use crate::compile::execute::SelfProfileFiles; +use anyhow::Context; +use database::{ArtifactIdNumber, CollectionId, Profile, Scenario}; +use std::future::Future; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +// TODO: Record codegen backend in the self profile name +// TODO: remove collection ID from self-profile ID +/// Uniquely identifies a self-profile archive. +#[derive(Debug)] +pub struct SelfProfileId { + pub artifact_id_number: ArtifactIdNumber, + pub collection: CollectionId, + pub benchmark: BenchmarkName, + pub profile: Profile, + pub scenario: Scenario, +} + +impl SelfProfileId { + fn file_prefix(&self) -> PathBuf { + PathBuf::from("self-profile") + .join(self.artifact_id_number.0.to_string()) + .join(self.benchmark.0.as_str()) + .join(self.profile.to_string()) + .join(self.scenario.to_id()) + } +} + +pub trait SelfProfileStorage { + /// Store a self-profile with the given ID. + fn store( + &self, + id: SelfProfileId, + files: SelfProfileFiles, + ) -> Pin> + Send>>; +} + +pub struct LocalSelfProfileStorage { + directory: PathBuf, +} + +impl LocalSelfProfileStorage { + pub fn new(dir: &Path) -> Self { + Self { + directory: dir.to_owned(), + } + } +} + +impl SelfProfileStorage for LocalSelfProfileStorage { + fn store( + &self, + id: SelfProfileId, + files: SelfProfileFiles, + ) -> Pin> + Send>> { + let prefix = id.file_prefix(); + let path = self + .directory + .join(prefix) + .join(format!("self-profile-{}.mm_profdata.sz", id.collection)); + Box::pin(async move { + tokio::fs::create_dir_all(path.parent().unwrap()).await?; + match files { + SelfProfileFiles::Eight { file } => { + tokio::fs::copy(&file, &path).await?; + } + } + + Ok(()) + }) + } +} + +#[derive(Default)] +pub struct S3SelfProfileStorage; + +impl S3SelfProfileStorage { + pub fn new() -> Self { + Self + } +} + +impl SelfProfileStorage for S3SelfProfileStorage { + fn store( + &self, + id: SelfProfileId, + files: SelfProfileFiles, + ) -> Pin> + Send>> { + Box::pin(async move { + // Files are placed at + // * self-profile//// + // /self-profile-.{extension} + let prefix = id.file_prefix(); + let upload = tempfile::NamedTempFile::new().context("cannot create temporary file")?; + let filename = match files { + SelfProfileFiles::Eight { file } => { + let data = tokio::fs::read(file) + .await + .context("cannot read self-profile data")?; + let mut data = snap::read::FrameEncoder::new(&data[..]); + let mut compressed = Vec::new(); + data.read_to_end(&mut compressed) + .context("cannot compress self-profile data")?; + tokio::fs::write(upload.path(), &compressed) + .await + .context("cannot write compressed self-profile data")?; + + format!("self-profile-{}.mm_profdata.sz", id.collection) + } + }; + + let output = tokio::process::Command::new("aws") + .arg("s3") + .arg("cp") + .arg("--storage-class") + .arg("INTELLIGENT_TIERING") + .arg("--only-show-errors") + .arg(upload.path()) + .arg(format!( + "s3://rustc-perf/{}", + &prefix.join(filename).to_str().unwrap() + )) + .spawn() + .context("cannot spawn aws binary")? + .wait_with_output() + .await + .context("cannot run aws binary")?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "Could not upload self-profile to S3\nStdout:\n{}\nStderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(()) + }) + } +} diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index 8872f08eb..292bf3257 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -792,13 +792,6 @@ impl Connection for SqliteConnection { _profile: Profile, _scenario: crate::Scenario, ) { - #![allow(clippy::diverging_sub_expression)] - - // FIXME: this is left for the future, if we ever need to support it. It - // shouldn't be too hard, but we may also want to just intern the raw - // self profile files into sqlite database or something like that, not - // yet clear. - unimplemented!("recording raw self profile files is not implemented for sqlite") } async fn record_error(