diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index ea025270f..ed985ff21 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - flamegraphs permissions: # deployments permission to deploy GitHub pages website @@ -12,8 +13,13 @@ permissions: contents: write jobs: - test: + benchmarks: runs-on: ${{ matrix.os }} + env: + BENCHER_PROJECT: htsget-rs + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + BENCHER_ADAPTER: rust-criterion + BENCHER_TESTBED: localhost strategy: matrix: rust: [stable] @@ -45,15 +51,47 @@ jobs: uses: baptiste0928/cargo-install@v1 with: crate: cargo-criterion - - name: Run search benchmarks - run: cargo criterion --bench search-benchmarks --message-format=json -- LIGHT 1> search-benchmarks-output.json - - name: Store search benchmark result - uses: brainstorm/github-action-benchmark@cargo-criterion-v3 - with: - name: Search benchmark - tool: 'cargo-criterion' - output-file-path: search-benchmarks-output.json - native-benchmark-data-dir-path: target/criterion - fail-on-alert: false - github-token: ${{ secrets.HTSGET_RS_BENCHMARKS_TOKEN }} - auto-push: true + - uses: bencherdev/bencher@v0.2.43 + - name: Run bencher.dev + run: | + bencher run \ + --if-branch "$GITHUB_REF_NAME" \ + --else-if-branch "$GITHUB_BASE_REF" \ + --else-if-branch main \ + --err \ + "cargo bench" + # - name: Install cargo flamegraphs + # uses: baptiste0928/cargo-install@v1 + # with: + # crate: flamegraph + # - name: Run search benchmarks + # run: cargo criterion --bench search-benchmarks --message-format=json -- LIGHT 1> search-benchmarks-output.json + # - name: Generate flamegraphs and flamecharts for profiling + # run: | + # if [ "$RUNNER_OS" == "Linux" ]; then + # sudo cat /proc/sys/kernel/kptr_restrict + # sudo cat /proc/sys/kernel/perf_event_paranoid + # echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid + # # Chart == X axis is time + # # cargo run record -P ... TODO: not-perf + # #cargo flamegraph --deterministic --flamechart -o flamechart.svg --bench search-benchmarks -- --bench >& flamechart.bin + # # Graph == X axis is stack profile population + # cargo flamegraph --deterministic --bench search-benchmarks -- --bench >& flamegraph.bin + # else # PERFILE binary contents spewed on stdout on Linux?: https://github.com/flamegraph-rs/flamegraph/issues/248 + # cargo flamegraph --deterministic --flamechart -o flamechart.svg --bench search-benchmarks -- --bench + # # Graph == X axis is stack profile population + # cargo flamegraph --deterministic --bench search-benchmarks -- --bench + # fi + # shell: bash + # - name: Batch flame* results under criterion-rs output + # run: mkdir -p target/criterion/flameplots && cp flame*.svg flame*.bin target/criterion/flameplots + # - name: Store search benchmark result + # uses: brainstorm/github-action-benchmark@cargo-criterion-v3 + # with: + # name: Search benchmark + # tool: 'cargo-criterion' + # output-file-path: search-benchmarks-output.json + # native-benchmark-data-dir-path: target/criterion + # fail-on-alert: false + # github-token: ${{ secrets.HTSGET_RS_BENCHMARKS_TOKEN }} + # auto-push: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3572194b..abf4d016a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,10 +4,9 @@ on: push: branches: - main - pull_request: jobs: - test: + tests: runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/.gitignore b/.gitignore index 460ee7ef4..0eb230021 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ target *.code-workspace *.vcf.gz +*.svg .vscode .idea deploy/.build diff --git a/Cargo.lock b/Cargo.lock index dd66e6e57..4298cde42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,6 +218,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -676,7 +685,7 @@ dependencies = [ "hyper-rustls", "lazy_static", "pin-project-lite", - "rustls", + "rustls 0.20.8", "tokio", "tower", "tracing", @@ -885,6 +894,21 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide 0.6.2", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -955,6 +979,12 @@ version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +[[package]] +name = "bytemuck" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" + [[package]] name = "byteorder" version = "1.4.3" @@ -1104,6 +1134,7 @@ dependencies = [ "anstyle", "bitflags", "clap_lex 0.4.1", + "once_cell", "strsim", ] @@ -1183,6 +1214,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpp_demangle" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c76f98bdfc7f66172e6c7065f981ebb576ffc903fe4c0561d9f0c2509226dc6" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.7" @@ -1386,6 +1426,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d7439c3735f405729d52c3fbbe4de140eaf938a1fe47d227c27f8254d4302a5" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1471,6 +1520,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "flate2" version = "1.0.26" @@ -1478,7 +1539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.7.1", ] [[package]] @@ -1606,6 +1667,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" + [[package]] name = "h2" version = "0.3.18" @@ -1706,7 +1773,6 @@ dependencies = [ "htsget-http", "htsget-search", "htsget-test", - "reqwest", "rustls 0.21.1", "rustls-pemfile", "serde", @@ -1717,6 +1783,24 @@ dependencies = [ "tracing-actix-web", ] +[[package]] +name = "htsget-benchmarks" +version = "0.1.0" +dependencies = [ + "async-trait", + "criterion", + "htsget-config", + "htsget-http", + "htsget-search", + "htsget-test", + "http", + "pprof", + "reqwest", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "htsget-config" version = "0.2.0" @@ -1789,7 +1873,6 @@ dependencies = [ "axum-extra", "base64 0.21.0", "bytes", - "criterion", "data-url", "futures", "futures-util", @@ -1973,6 +2056,24 @@ dependencies = [ "serde", ] +[[package]] +name = "inferno" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fb7c1b80a1dfa604bb4a649a5c5aeef3d913f7c520cb42b40e534e8a61bcdfc" +dependencies = [ + "ahash 0.8.3", + "indexmap", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml 0.26.0", + "rgb", + "str_stack", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -2279,6 +2380,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.8.0" @@ -2310,6 +2420,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -2331,6 +2450,18 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "static_assertions", +] + [[package]] name = "nom" version = "7.1.3" @@ -2536,6 +2667,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -2571,6 +2712,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" +[[package]] +name = "object" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -2758,6 +2908,28 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "pprof" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" +dependencies = [ + "backtrace", + "cfg-if", + "criterion", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "parking_lot", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2797,6 +2969,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.28.2" @@ -2969,6 +3150,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "rgb" +version = "0.8.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.16.20" @@ -2984,6 +3174,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -3099,7 +3295,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "quick-xml", + "quick-xml 0.28.2", "serde", "serde_urlencoded", "sha1", @@ -3401,12 +3597,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "strsim" version = "0.10.0" @@ -3419,6 +3627,29 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "symbolic-common" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "10.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 0319c37c4..4b86b08a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,5 +5,10 @@ members = [ "htsget-http", "htsget-lambda", "htsget-search", - "htsget-test" + "htsget-test", + "htsget-benchmarks", ] + +# Flamegraphs require debuginfo for symbol names. +[profile.bench] +debug = true diff --git a/htsget-actix/Cargo.toml b/htsget-actix/Cargo.toml index 573ad00a6..a69708cfa 100644 --- a/htsget-actix/Cargo.toml +++ b/htsget-actix/Cargo.toml @@ -36,10 +36,5 @@ tracing = "0.1" async-trait = "0.1" criterion = { version = "0.4", features = ["async_tokio"] } -reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "rustls-tls"] } tempfile = "3.3" -[[bench]] -name = "request-benchmarks" -harness = false -path = "benches/request_benchmarks.rs" diff --git a/htsget-actix/benches/request_benchmarks.rs b/htsget-actix/benches/request_benchmarks.rs deleted file mode 100644 index cfe215f89..000000000 --- a/htsget-actix/benches/request_benchmarks.rs +++ /dev/null @@ -1,338 +0,0 @@ -use std::path::PathBuf; -use std::process::{Child, Command}; -use std::thread::sleep; -use std::{convert::TryInto, fs, time::Duration}; - -use criterion::measurement::WallTime; -use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; -use reqwest::blocking::Client; -use reqwest::blocking::ClientBuilder; -use reqwest::Result; -use serde::{Deserialize, Serialize}; - -use htsget_config::types::{Headers, JsonResponse}; -use htsget_http::{PostRequest, Region}; -use htsget_test::http_tests::{default_config_fixed_port, default_dir, default_dir_data}; - -const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0"; -const BENCHMARK_DURATION_SECONDS: u64 = 30; -const NUMBER_OF_SAMPLES: usize = 50; - -#[derive(Serialize)] -struct Empty; - -#[derive(Deserialize)] -struct RefserverConfig { - #[serde(rename = "htsgetConfig")] - htsget_config: RefserverProps, -} - -#[derive(Deserialize)] -struct RefserverProps { - props: RefserverAddr, -} - -#[derive(Deserialize)] -struct RefserverAddr { - port: u64, - host: String, -} - -struct DropGuard(Child); - -impl Drop for DropGuard { - fn drop(&mut self) { - drop(self.0.kill()); - } -} - -fn request(url: reqwest::Url, json_content: &impl Serialize, client: &Client) -> usize { - let response: JsonResponse = client - .post(url) - .json(json_content) - .send() - .unwrap() - .json() - .unwrap(); - - response - .htsget - .urls - .iter() - .map(|json_url| { - Ok( - client - .get(&json_url.url) - .headers( - json_url - .headers - .as_ref() - .unwrap_or(&Headers::default()) - .as_ref_inner() - .try_into() - .unwrap(), - ) - .send()? - .bytes()? - .len(), - ) - }) - .fold(0, |acc, x: Result| acc + x.unwrap_or(0)) -} - -fn format_url(url: &str, path: &str) -> reqwest::Url { - reqwest::Url::parse(url) - .expect("invalid url") - .join(path) - .expect("invalid url") -} - -fn bench_pair( - group: &mut BenchmarkGroup, - name: &str, - htsget_url: reqwest::Url, - refserver_url: reqwest::Url, - json_content: &impl Serialize, -) { - let client = ClientBuilder::new() - .danger_accept_invalid_certs(true) - .use_rustls_tls() - .build() - .unwrap(); - group.bench_with_input( - format!("{} {}", name, "htsget-rs"), - &htsget_url, - |b, input| b.iter(|| request(input.clone(), json_content, &client)), - ); - group.bench_with_input( - format!("{} {}", name, "htsget-refserver"), - &refserver_url, - |b, input| b.iter(|| request(input.clone(), json_content, &client)), - ); -} - -#[cfg(target_os = "windows")] -pub fn new_command(cmd: &str) -> Command { - let mut command = Command::new("cmd.exe"); - command.arg("/c"); - command.arg(cmd); - command -} - -#[cfg(not(target_os = "windows"))] -pub fn new_command(cmd: &str) -> Command { - Command::new(cmd) -} - -fn query_server_until_response(url: &reqwest::Url) { - let client = Client::new(); - for _ in 0..120 { - sleep(Duration::from_secs(1)); - if let Err(err) = client.get(url.clone()).send() { - if err.is_connect() { - continue; - } - } - break; - } -} - -fn start_htsget_rs() -> (DropGuard, String) { - let config = default_config_fixed_port(); - - let child = new_command("cargo") - .current_dir(default_dir()) - .arg("run") - .arg("-p") - .arg("htsget-actix") - .arg("--no-default-features") - .env("HTSGET_PATH", default_dir_data()) - .env("RUST_LOG", "warn") - .spawn() - .unwrap(); - - let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); - query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); - let htsget_rs_ticket_url = format!("http://{}", config.data_server().addr()); - query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); - - (DropGuard(child), htsget_rs_url) -} - -fn start_htsget_refserver() -> (DropGuard, String) { - let config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("benches") - .join("htsget-refserver-config.json"); - let refserver_config: RefserverConfig = - serde_json::from_str(&fs::read_to_string(&config_path).unwrap()).unwrap(); - - new_command("docker") - .arg("image") - .arg("pull") - .arg(REFSERVER_DOCKER_IMAGE) - .spawn() - .unwrap() - .wait() - .unwrap(); - - let child = new_command("docker") - .current_dir(default_dir()) - .arg("container") - .arg("run") - .arg("-p") - .arg(format!( - "{}:3000", - &refserver_config.htsget_config.props.port - )) - .arg("-v") - .arg(format!( - "{}:/data", - default_dir() - .join("data") - .canonicalize() - .unwrap() - .to_string_lossy() - )) - .arg("-v") - .arg(format!( - "{}:/config", - &config_path - .canonicalize() - .unwrap() - .parent() - .unwrap() - .to_string_lossy() - )) - .arg(REFSERVER_DOCKER_IMAGE) - .arg("./htsget-refserver") - .arg("-config") - .arg("/config/htsget-refserver-config.json") - .spawn() - .unwrap(); - - let refserver_url = refserver_config.htsget_config.props.host; - query_server_until_response(&format_url(&refserver_url, "reads/service-info")); - - (DropGuard(child), refserver_url) -} - -fn criterion_benchmark(c: &mut Criterion) { - let mut group = c.benchmark_group("Requests"); - group - .sample_size(NUMBER_OF_SAMPLES) - .measurement_time(Duration::from_secs(BENCHMARK_DURATION_SECONDS)); - - let (_htsget_rs_server, htsget_rs_url) = start_htsget_rs(); - let (_htsget_refserver_server, htsget_refserver_url) = start_htsget_refserver(); - - let json_content = PostRequest { - format: None, - class: None, - fields: None, - tags: None, - notags: None, - regions: None, - }; - bench_pair( - &mut group, - "[LIGHT] simple request", - format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), - format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"), - &json_content, - ); - - let json_content = PostRequest { - format: None, - class: None, - fields: None, - tags: None, - notags: None, - regions: Some(vec![Region { - reference_name: "20".to_string(), - start: None, - end: None, - }]), - }; - bench_pair( - &mut group, - "[LIGHT] with region", - format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), - format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"), - &json_content, - ); - - let json_content = PostRequest { - format: None, - class: None, - fields: None, - tags: None, - notags: None, - regions: Some(vec![ - Region { - reference_name: "20".to_string(), - start: None, - end: None, - }, - Region { - reference_name: "11".to_string(), - start: Some(4999977), - end: Some(5008321), - }, - ]), - }; - bench_pair( - &mut group, - "[LIGHT] with two regions", - format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), - format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"), - &json_content, - ); - - let json_content = PostRequest { - format: None, - class: None, - fields: None, - tags: None, - notags: None, - regions: Some(vec![Region { - reference_name: "chrM".to_string(), - start: Some(1), - end: Some(153), - }]), - }; - bench_pair( - &mut group, - "[LIGHT] with VCF", - format_url(&htsget_rs_url, "variants/data/vcf/sample1-bcbio-cancer"), - format_url(&htsget_refserver_url, "variants/sample1-bcbio-cancer"), - &json_content, - ); - - let json_content = PostRequest { - format: None, - class: None, - fields: None, - tags: None, - notags: None, - regions: Some(vec![Region { - reference_name: "14".to_string(), - start: None, - end: None, - }]), - }; - bench_pair( - &mut group, - "[HEAVY] with big VCF", - format_url( - &htsget_rs_url, - "variants/data/vcf/internationalgenomesample", - ), - format_url(&htsget_refserver_url, "variants/internationalgenomesample"), - &json_content, - ); - - group.finish(); -} - -criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); diff --git a/htsget-benchmarks/Cargo.toml b/htsget-benchmarks/Cargo.toml new file mode 100644 index 000000000..780327a42 --- /dev/null +++ b/htsget-benchmarks/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "htsget-benchmarks" +version = "0.1.0" +edition = "2021" + +[features] +# Enable the correct features of htsget dependencies based on the feature flag of this crate. +s3-storage = ["htsget-config/s3-storage", "htsget-search/s3-storage", "htsget-http/s3-storage"] +default = ["s3-storage"] + +[dependencies] +http = "0.2" +async-trait = "0.1" + +# htsget-rs +# The path is not so important here, it's more about which feature flags are enabled. +htsget-config = { version = "0.2.0", path = "../htsget-config", default-features = false } +htsget-search = { version = "0.2.0", path = "../htsget-search", default-features = false } +htsget-http = { version = "0.2.0", path = "../htsget-http", default-features = false } +htsget-test = { version = "0.2.0", path = "../htsget-test", default-features = false } +tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +# Benchmarking +criterion = { version = "0.4", features = ["async_tokio"] } +pprof = { version = "0.11", features = ["criterion", "flamegraph"]} +reqwest = { version = "0.11", default-features = false, features = ["json", "blocking", "rustls-tls"] } + +[[bench]] +name = "search-benchmarks" +harness = false +path = "benches/search_benchmarks.rs" + +# [[bench]] +# name = "request-benchmarks" +# harness = false +# path = "benches/request_benchmarks.rs" \ No newline at end of file diff --git a/htsget-benchmarks/README.md b/htsget-benchmarks/README.md new file mode 100644 index 000000000..a02dd84ac --- /dev/null +++ b/htsget-benchmarks/README.md @@ -0,0 +1,24 @@ +# Quickstart + +## Profiling + +With cargo-instruments (OSX-only profiling, WIP): + +``` +time cargo instruments -t time --all-features --bench search-benchmarks +``` + +With not-perf (with criterion-rs external profiling integration, WIP): + +``` +cargo run record -P `cargo bench` -w -o datafile +``` + +## Benchmarking + +``` +bencher run \ + --branch main \ + --project htsget-rs \ + "cargo bench" +``` diff --git a/htsget-actix/benches/htsget-refserver-config.json b/htsget-benchmarks/benches/htsget-refserver-config.json similarity index 100% rename from htsget-actix/benches/htsget-refserver-config.json rename to htsget-benchmarks/benches/htsget-refserver-config.json diff --git a/htsget-benchmarks/benches/refserver_benchmarks.rs b/htsget-benchmarks/benches/refserver_benchmarks.rs new file mode 100644 index 000000000..4254ec48a --- /dev/null +++ b/htsget-benchmarks/benches/refserver_benchmarks.rs @@ -0,0 +1,137 @@ +// use std::process::{Child, Command}; +// use serde::{Deserialize, Serialize}; +// use std::path::PathBuf; +// use std::time::Duration; +// use std::thread::sleep; +// use std::fs; + +// use htsget_test::http_tests::default_dir; +// use reqwest::blocking::Client; + +// use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; + +// const REFSERVER_DOCKER_IMAGE: &str = "ga4gh/htsget-refserver:1.5.0"; + +// #[derive(Serialize)] +// struct Empty; + +// #[derive(Deserialize)] +// struct RefserverConfig { +// #[serde(rename = "htsgetConfig")] +// htsget_config: RefserverProps, +// } + +// #[derive(Deserialize)] +// struct RefserverProps { +// props: RefserverAddr, +// } + +// #[derive(Deserialize)] +// struct RefserverAddr { +// port: u64, +// host: String, +// } +// struct DropGuard(Child); + +// impl Drop for DropGuard { +// fn drop(&mut self) { +// drop(self.0.kill()); +// } +// } + +// #[cfg(target_os = "windows")] +// pub fn new_command(cmd: &str) -> Command { +// let mut command = Command::new("cmd.exe"); +// command.arg("/c"); +// command.arg(cmd); +// command +// } + +// #[cfg(not(target_os = "windows"))] +// pub fn new_command(cmd: &str) -> Command { +// Command::new(cmd) +// } + +// fn format_url(url: &str, path: &str) -> reqwest::Url { +// reqwest::Url::parse(url) +// .expect("invalid url") +// .join(path) +// .expect("invalid url") +// } + +// fn query_server_until_response(url: &reqwest::Url) { +// let client = Client::new(); +// for _ in 0..120 { +// sleep(Duration::from_secs(1)); +// if let Err(err) = client.get(url.clone()).send() { +// if err.is_connect() { +// continue; +// } +// } +// break; +// } +// } + +// fn start_htsget_refserver() -> (DropGuard, String) { +// let config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) +// .join("benches") +// .join("htsget-refserver-config.json"); +// let refserver_config: RefserverConfig = +// serde_json::from_str(&fs::read_to_string(&config_path).unwrap()).unwrap(); + +// new_command("docker") +// .arg("image") +// .arg("pull") +// .arg(REFSERVER_DOCKER_IMAGE) +// .spawn() +// .unwrap() +// .wait() +// .unwrap(); + +// let child = new_command("docker") +// .current_dir(default_dir()) +// .arg("container") +// .arg("run") +// .arg("-p") +// .arg(format!( +// "{}:3000", +// &refserver_config.htsget_config.props.port +// )) +// .arg("-v") +// .arg(format!( +// "{}:/data", +// default_dir() +// .join("data") +// .canonicalize() +// .unwrap() +// .to_string_lossy() +// )) +// .arg("-v") +// .arg(format!( +// "{}:/config", +// &config_path +// .canonicalize() +// .unwrap() +// .parent() +// .unwrap() +// .to_string_lossy() +// )) +// .arg(REFSERVER_DOCKER_IMAGE) +// .arg("./htsget-refserver") +// .arg("-config") +// .arg("/config/htsget-refserver-config.json") +// .spawn() +// .unwrap(); + +// let refserver_url = refserver_config.htsget_config.props.host; +// query_server_until_response(&format_url(&refserver_url, "reads/service-info")); + +// (DropGuard(child), refserver_url) +// } + + +// fn criterion_benchmark(_c: &mut Criterion) { +// let (_htsget_refserver_server, htsget_refserver_url) = start_htsget_refserver(); +// //format_url(&htsget_refserver_url, "reads/htsnexus_test_NA12878"), +// todo!() +// } \ No newline at end of file diff --git a/htsget-benchmarks/benches/request_benchmarks.rs b/htsget-benchmarks/benches/request_benchmarks.rs new file mode 100644 index 000000000..8cdf7e922 --- /dev/null +++ b/htsget-benchmarks/benches/request_benchmarks.rs @@ -0,0 +1,169 @@ +// use std::process::{Child, Command}; +// use std::thread::sleep; +// use std::{convert::TryInto, time::Duration}; + +// use criterion::measurement::WallTime; +// use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion}; +// use reqwest::blocking::Client; +// use serde::{Serialize}; + +// use htsget_config::types::{Headers, JsonResponse}; +// use htsget_http::{PostRequest, Region}; +// use htsget_test::http_tests::{default_config_fixed_port, default_dir, default_dir_data}; + +// const BENCHMARK_DURATION_SECONDS: u64 = 30; +// const NUMBER_OF_SAMPLES: usize = 50; + +// struct DropGuard(Child); + +// impl Drop for DropGuard { +// fn drop(&mut self) { +// drop(self.0.kill()); +// } +// } + +// fn request(url: reqwest::Url, json_content: &impl Serialize, client: &Client) -> usize { +// let response: JsonResponse = client +// .post(url) +// .json(json_content) +// .send() +// .unwrap() +// .json() +// .unwrap(); + +// response +// .htsget +// .urls +// .iter() +// .map(|json_url| { +// Ok( +// client +// .get(&json_url.url) +// .headers( +// json_url +// .headers +// .as_ref() +// .unwrap_or(&Headers::default()) +// .as_ref_inner() +// .try_into() +// .unwrap(), +// ) +// .send()? +// .bytes()? +// .len(), +// ) +// }) +// .fold(0, |acc, x: Result| acc + x.unwrap_or(0)) +// } + +// fn format_url(url: &str, path: &str) -> reqwest::Url { +// reqwest::Url::parse(url) +// .expect("invalid url") +// .join(path) +// .expect("invalid url") +// } + +// fn bench_query(group: &mut BenchmarkGroup, name: &str, query: Query) { +// group.bench_with_input(name, &query, |b, input| { +// b.to_async(Runtime::new().unwrap()) +// .iter(|| perform_query(input.clone())) +// }); +// } + +// fn query_server_until_response(url: &reqwest::Url) { +// let client = Client::new(); +// for _ in 0..120 { +// sleep(Duration::from_secs(1)); +// if let Err(err) = client.get(url.clone()).send() { +// if err.is_connect() { +// continue; +// } +// } +// break; +// } +// } + +// #[cfg(target_os = "windows")] +// pub fn new_command(cmd: &str) -> Command { +// let mut command = Command::new("cmd.exe"); +// command.arg("/c"); +// command.arg(cmd); +// command +// } + +// #[cfg(not(target_os = "windows"))] +// pub fn new_command(cmd: &str) -> Command { +// Command::new(cmd) +// } + +// fn start_htsget_rs() -> (DropGuard, String) { +// let config = default_config_fixed_port(); + +// let child = new_command("cargo") +// .current_dir(default_dir()) +// .arg("run") +// .arg("-p") +// .arg("htsget-actix") +// .arg("--no-default-features") +// .env("HTSGET_PATH", default_dir_data()) +// .env("RUST_LOG", "warn") +// .spawn() +// .unwrap(); + +// let htsget_rs_url = format!("http://{}", config.ticket_server().addr()); +// query_server_until_response(&format_url(&htsget_rs_url, "reads/service-info")); +// let htsget_rs_ticket_url = format!("http://{}", config.data_server().addr()); +// query_server_until_response(&format_url(&htsget_rs_ticket_url, "")); + +// (DropGuard(child), htsget_rs_url) +// } + + +// fn criterion_benchmark(c: &mut Criterion) { +// let mut group = c.benchmark_group("Requests"); +// group +// .sample_size(NUMBER_OF_SAMPLES) +// .measurement_time(Duration::from_secs(BENCHMARK_DURATION_SECONDS)); + +// let (_htsget_rs_server, htsget_rs_url) = start_htsget_rs(); + +// bench_query( +// &mut group, +// "[LIGHT] simple request", +// format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), +// ); + +// bench_query( +// &mut group, +// "[LIGHT] with region", +// format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), +// ); + +// bench_query( +// &mut group, +// "[LIGHT] with two regions", +// format_url(&htsget_rs_url, "reads/data/bam/htsnexus_test_NA12878"), +// ); + +// bench_query( +// &mut group, +// "[LIGHT] with VCF", +// format_url(&htsget_rs_url, "variants/data/vcf/sample1-bcbio-cancer"), +// ); + +// // bench_pair( +// // &mut group, +// // "[HEAVY] with big VCF", +// // format_url( +// // &htsget_rs_url, +// // "variants/data/vcf/internationalgenomesample", +// // ), +// // format_url(&htsget_refserver_url, "variants/internationalgenomesample"), +// // &json_content, +// // ); + +// group.finish(); +// } + +// criterion_group!(benches, criterion_benchmark); +// criterion_main!(benches); diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-benchmarks/benches/search_benchmarks.rs similarity index 55% rename from htsget-search/benches/search_benchmarks.rs rename to htsget-benchmarks/benches/search_benchmarks.rs index daa3bb91a..0ff88aad6 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-benchmarks/benches/search_benchmarks.rs @@ -62,65 +62,65 @@ fn criterion_benchmark(c: &mut Criterion) { Query::new("bam/htsnexus_test_NA12878", Bam).with_class(Header), ); - bench_query( - &mut group, - "[LIGHT] Cram query all", - Query::new("cram/htsnexus_test_NA12878", Cram), - ); - bench_query( - &mut group, - "[LIGHT] Cram query specific", - Query::new("cram/htsnexus_test_NA12878", Cram) - .with_reference_name("11") - .with_start(4999977) - .with_end(5008321), - ); - bench_query( - &mut group, - "[LIGHT] Cram query header", - Query::new("cram/htsnexus_test_NA12878", Cram).with_class(Header), - ); + // bench_query( + // &mut group, + // "[LIGHT] Cram query all", + // Query::new("cram/htsnexus_test_NA12878", Cram), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Cram query specific", + // Query::new("cram/htsnexus_test_NA12878", Cram) + // .with_reference_name("11") + // .with_start(4999977) + // .with_end(5008321), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Cram query header", + // Query::new("cram/htsnexus_test_NA12878", Cram).with_class(Header), + // ); - bench_query( - &mut group, - "[LIGHT] Vcf query all", - Query::new("vcf/sample1-bcbio-cancer", Vcf), - ); - bench_query( - &mut group, - "[LIGHT] Vcf query specific", - Query::new("vcf/sample1-bcbio-cancer", Vcf) - .with_reference_name("chrM") - .with_start(151) - .with_end(153), - ); - bench_query( - &mut group, - "[LIGHT] Vcf query header", - Query::new("vcf/sample1-bcbio-cancer", Vcf).with_class(Header), - ); + // bench_query( + // &mut group, + // "[LIGHT] Vcf query all", + // Query::new("vcf/sample1-bcbio-cancer", Vcf), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Vcf query specific", + // Query::new("vcf/sample1-bcbio-cancer", Vcf) + // .with_reference_name("chrM") + // .with_start(151) + // .with_end(153), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Vcf query header", + // Query::new("vcf/sample1-bcbio-cancer", Vcf).with_class(Header), + // ); - bench_query( - &mut group, - "[LIGHT] Bcf query all", - Query::new("bcf/sample1-bcbio-cancer", Bcf), - ); - bench_query( - &mut group, - "[LIGHT] Bcf query specific", - Query::new("bcf/sample1-bcbio-cancer", Bcf) - .with_reference_name("chrM") - .with_start(151) - .with_end(153), - ); - bench_query( - &mut group, - "[LIGHT] Bcf query header", - Query::new("bcf/sample1-bcbio-cancer", Bcf).with_class(Header), - ); + // bench_query( + // &mut group, + // "[LIGHT] Bcf query all", + // Query::new("bcf/sample1-bcbio-cancer", Bcf), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Bcf query specific", + // Query::new("bcf/sample1-bcbio-cancer", Bcf) + // .with_reference_name("chrM") + // .with_start(151) + // .with_end(153), + // ); + // bench_query( + // &mut group, + // "[LIGHT] Bcf query header", + // Query::new("bcf/sample1-bcbio-cancer", Bcf).with_class(Header), + // ); group.finish(); } criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); +criterion_main!(benches); \ No newline at end of file diff --git a/htsget-benchmarks/lib.rs b/htsget-benchmarks/lib.rs new file mode 100644 index 000000000..47f070824 --- /dev/null +++ b/htsget-benchmarks/lib.rs @@ -0,0 +1 @@ +pub mod not_perf; \ No newline at end of file diff --git a/htsget-benchmarks/src/main.rs b/htsget-benchmarks/src/main.rs new file mode 100644 index 000000000..4673b0867 --- /dev/null +++ b/htsget-benchmarks/src/main.rs @@ -0,0 +1,8 @@ +use not_perf::profiler::ProfilingController; +use not_perf::args::GenericProfilerArgs; + +fn main() { + let pid = std::os::unix::process::parent_id(); // TODO: Locate benchmark pids and/or binaries? + let prof_args = GenericProfilerArgs::new(pid.try_into().unwrap()); + let _prof_ctrl = ProfilingController::new(&prof_args); +} \ No newline at end of file diff --git a/htsget-benchmarks/src/not_perf.rs b/htsget-benchmarks/src/not_perf.rs new file mode 100644 index 000000000..58ac50ea1 --- /dev/null +++ b/htsget-benchmarks/src/not_perf.rs @@ -0,0 +1,3 @@ +use not_perf; +asdflkjasdlkfjasl;djfla;ksjdflkajsdf +nperf_core::moo; \ No newline at end of file diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index ede3312c8..40cb3d878 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -61,9 +61,4 @@ s3s-aws = { version = "0.5" } # Axum server reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } -criterion = { version = "0.4", features = ["async_tokio"] } -[[bench]] -name = "search-benchmarks" -harness = false -path = "benches/search_benchmarks.rs"