From 18d3f89fd8499927006686934e410813dfac9795 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 09:36:04 +0100 Subject: [PATCH 01/12] Add criterion PR --- .github/workflows/pr_criterion.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/pr_criterion.yaml diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml new file mode 100644 index 0000000000..bca1f8c834 --- /dev/null +++ b/.github/workflows/pr_criterion.yaml @@ -0,0 +1,13 @@ +on: [pull_request] +name: benchmark pull requests +jobs: + runBenchmark: + name: run benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: boa-dev/criterion-compare-action@v3 + with: + branchName: ${{ github.base_ref }} + # Optional. Default is `${{ github.token }}`. + # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From c577a64a875785f0d7eebe91e639619ab6129a3d Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 09:42:38 +0100 Subject: [PATCH 02/12] more protoc --- .github/workflows/pr_criterion.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index bca1f8c834..007333d41e 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -6,6 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: arduino/setup-protoc@v3 - uses: boa-dev/criterion-compare-action@v3 with: branchName: ${{ github.base_ref }} From 5e4cd10055db7c73cc532d37a7a17f2cae7951ef Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 10:00:29 +0100 Subject: [PATCH 03/12] Try fix bench issue --- .github/workflows/pr_criterion.yaml | 8 +++++++- opentelemetry-appender-tracing/Cargo.toml | 3 +++ opentelemetry-sdk/Cargo.toml | 3 +++ opentelemetry/Cargo.toml | 3 +++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 007333d41e..2a4b95dad4 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -7,8 +7,14 @@ jobs: steps: - uses: actions/checkout@v3 - uses: arduino/setup-protoc@v3 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable - uses: boa-dev/criterion-compare-action@v3 with: +# benchName: "anyvalue" +# branchName: main +# token: bloop branchName: ${{ github.base_ref }} # Optional. Default is `${{ github.token }}`. - # token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/opentelemetry-appender-tracing/Cargo.toml b/opentelemetry-appender-tracing/Cargo.toml index dee03e5199..cd83d14791 100644 --- a/opentelemetry-appender-tracing/Cargo.toml +++ b/opentelemetry-appender-tracing/Cargo.toml @@ -44,5 +44,8 @@ name = "logs" harness = false required-features = ["spec_unstable_logs_enabled"] +[lib] +bench = false + [lints] workspace = true diff --git a/opentelemetry-sdk/Cargo.toml b/opentelemetry-sdk/Cargo.toml index 819e0b4ca2..10e138542a 100644 --- a/opentelemetry-sdk/Cargo.toml +++ b/opentelemetry-sdk/Cargo.toml @@ -108,5 +108,8 @@ name = "log" harness = false required-features = ["logs"] +[lib] +bench = false + [lints] workspace = true diff --git a/opentelemetry/Cargo.toml b/opentelemetry/Cargo.toml index 6dd6dcc863..453fb40312 100644 --- a/opentelemetry/Cargo.toml +++ b/opentelemetry/Cargo.toml @@ -57,5 +57,8 @@ harness = false name = "anyvalue" harness = false +[lib] +bench = false + [lints] workspace = true From ca1724babc00f1e716e520a45c00394a93f612f1 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 10:12:28 +0100 Subject: [PATCH 04/12] context attach bench only --- .github/workflows/pr_criterion.yaml | 3 +- opentelemetry/Cargo.toml | 5 + opentelemetry/benches/context_attach.rs | 122 ++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 opentelemetry/benches/context_attach.rs diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 2a4b95dad4..66241f4656 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -12,7 +12,8 @@ jobs: toolchain: stable - uses: boa-dev/criterion-compare-action@v3 with: -# benchName: "anyvalue" + features: ["rt-tokio", "testing", "metrics", "logs"] + benchName: "context_attach" # branchName: main # token: bloop branchName: ${{ github.base_ref }} diff --git a/opentelemetry/Cargo.toml b/opentelemetry/Cargo.toml index 453fb40312..6902a527f8 100644 --- a/opentelemetry/Cargo.toml +++ b/opentelemetry/Cargo.toml @@ -57,6 +57,11 @@ harness = false name = "anyvalue" harness = false +[[bench]] +name = "context_attach" +harness = false +required-features = ["tracing"] + [lib] bench = false diff --git a/opentelemetry/benches/context_attach.rs b/opentelemetry/benches/context_attach.rs new file mode 100644 index 0000000000..748ec546e3 --- /dev/null +++ b/opentelemetry/benches/context_attach.rs @@ -0,0 +1,122 @@ +use criterion::{ + black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, BenchmarkId, + Criterion, Throughput, +}; +use opentelemetry::{ + trace::{SpanContext, TraceContextExt}, + Context, +}; + +// Run this benchmark with: +// cargo bench --bench context_attach + +fn criterion_benchmark(c: &mut Criterion) { + let span_context = Context::new().with_remote_span_context(SpanContext::empty_context()); + let contexts = vec![ + ("empty_cx", Context::new()), + ("single_value_cx", Context::new().with_value(Value(4711))), + ("span_cx", span_context), + ]; + for (name, cx) in contexts { + single_cx_scope(&mut group(c), name, &cx); + nested_cx_scope(&mut group(c), name, &cx); + overlapping_cx_scope(&mut group(c), name, &cx); + } +} + +fn single_cx_scope( + group: &mut BenchmarkGroup<'_, WallTime>, + context_type: &str, + context: &Context, +) { + let _restore = Context::current().attach(); + group.throughput(Throughput::Elements(1)).bench_function( + BenchmarkId::new("single_cx_scope", context_type), + |b| { + b.iter_batched( + || context.clone(), + |cx| { + single_cx(cx); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); +} + +#[inline(never)] +fn single_cx(cx: Context) { + let cx = black_box(cx.attach()); + let _ = black_box(dummy_work()); + drop(cx); +} + +fn nested_cx_scope(group: &mut BenchmarkGroup<'_, WallTime>, cx_type: &str, context: &Context) { + let _restore = Context::current().attach(); + group.throughput(Throughput::Elements(1)).bench_function( + BenchmarkId::new("nested_cx_scope", cx_type), + |b| { + b.iter_batched( + || (context.clone(), context.clone()), + |(cx1, cx2)| { + nested_cx(cx1, cx2); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); +} + +#[inline(never)] +fn nested_cx(cx1: Context, cx2: Context) { + let outer = black_box(cx1.attach()); + let inner = black_box(cx2.attach()); + let _ = black_box(dummy_work()); + drop(inner); + drop(outer); +} + +fn overlapping_cx_scope( + group: &mut BenchmarkGroup<'_, WallTime>, + cx_type: &str, + context: &Context, +) { + let _restore = Context::current().attach(); + group.throughput(Throughput::Elements(1)).bench_function( + BenchmarkId::new("overlapping_cx_scope", cx_type), + |b| { + b.iter_batched( + || (context.clone(), context.clone()), + |(cx1, cx2)| { + overlapping_cx(cx1, cx2); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); +} + +#[inline(never)] +fn overlapping_cx(cx1: Context, cx2: Context) { + let outer = cx1.attach(); + let inner = cx2.attach(); + let _ = black_box(dummy_work()); + drop(outer); + drop(inner); +} + +#[inline(never)] +fn dummy_work() -> i32 { + black_box(1 + 1) +} + +fn group(c: &mut Criterion) -> BenchmarkGroup { + c.benchmark_group("context_attach") +} + +#[derive(Debug, PartialEq)] +struct Value(i32); + +criterion_group!(benches, criterion_benchmark); + +criterion_main!(benches); \ No newline at end of file From 10f77a74bbfe0473aee336e1c1ebdb8d11122d0a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 10:14:36 +0100 Subject: [PATCH 05/12] fix --- .github/workflows/pr_criterion.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 66241f4656..5830d34a7a 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -12,7 +12,6 @@ jobs: toolchain: stable - uses: boa-dev/criterion-compare-action@v3 with: - features: ["rt-tokio", "testing", "metrics", "logs"] benchName: "context_attach" # branchName: main # token: bloop From a43ab8aaa78f35ae0f7d60310276654545ff0812 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 10:27:38 +0100 Subject: [PATCH 06/12] Bench opentelemetry only --- .github/workflows/pr_criterion.yaml | 5 +++-- opentelemetry-otlp/Cargo.toml | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 5830d34a7a..961b5fc9db 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -12,9 +12,10 @@ jobs: toolchain: stable - uses: boa-dev/criterion-compare-action@v3 with: - benchName: "context_attach" + cwd: opentelemetry +# benchName: "context_attach" # branchName: main # token: bloop branchName: ${{ github.base_ref }} # Optional. Default is `${{ github.token }}`. - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file +# token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/opentelemetry-otlp/Cargo.toml b/opentelemetry-otlp/Cargo.toml index 2e0fd8cf4c..4e45843312 100644 --- a/opentelemetry-otlp/Cargo.toml +++ b/opentelemetry-otlp/Cargo.toml @@ -86,3 +86,6 @@ integration-testing = ["tonic", "prost", "tokio/full", "trace", "logs"] [lints] workspace = true + +[lib] +bench = false \ No newline at end of file From 59b982683abf4e38cf557db3afb0ed288d5db9f0 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 10:40:14 +0100 Subject: [PATCH 07/12] Run for the other dirs too --- .github/workflows/pr_criterion.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 961b5fc9db..5f3ce5bdbd 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -13,9 +13,14 @@ jobs: - uses: boa-dev/criterion-compare-action@v3 with: cwd: opentelemetry -# benchName: "context_attach" -# branchName: main -# token: bloop branchName: ${{ github.base_ref }} - # Optional. Default is `${{ github.token }}`. -# token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-appender-tracing + features: spec_unstable_logs_enabled + branchName: ${{ github.base_ref }} + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-sdk + features: rt-tokio,testing,metrics,logs,spec_unstable_metrics_views + branchName: ${{ github.base_ref }} \ No newline at end of file From a3be162d548c9d12acf2e54175a7034c54d32253 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 11:37:02 +0100 Subject: [PATCH 08/12] chore: Stop libtest being added --- opentelemetry-appender-tracing/Cargo.toml | 3 +++ opentelemetry-sdk/Cargo.toml | 3 +++ opentelemetry/Cargo.toml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/opentelemetry-appender-tracing/Cargo.toml b/opentelemetry-appender-tracing/Cargo.toml index dee03e5199..cd83d14791 100644 --- a/opentelemetry-appender-tracing/Cargo.toml +++ b/opentelemetry-appender-tracing/Cargo.toml @@ -44,5 +44,8 @@ name = "logs" harness = false required-features = ["spec_unstable_logs_enabled"] +[lib] +bench = false + [lints] workspace = true diff --git a/opentelemetry-sdk/Cargo.toml b/opentelemetry-sdk/Cargo.toml index 819e0b4ca2..10e138542a 100644 --- a/opentelemetry-sdk/Cargo.toml +++ b/opentelemetry-sdk/Cargo.toml @@ -108,5 +108,8 @@ name = "log" harness = false required-features = ["logs"] +[lib] +bench = false + [lints] workspace = true diff --git a/opentelemetry/Cargo.toml b/opentelemetry/Cargo.toml index 6dd6dcc863..453fb40312 100644 --- a/opentelemetry/Cargo.toml +++ b/opentelemetry/Cargo.toml @@ -57,5 +57,8 @@ harness = false name = "anyvalue" harness = false +[lib] +bench = false + [lints] workspace = true From 95343a1429505ccdaa100866f2882f58082e7f91 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 11:38:39 +0100 Subject: [PATCH 09/12] otel-otlp shouldn't have the lib disable in it --- opentelemetry-otlp/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/opentelemetry-otlp/Cargo.toml b/opentelemetry-otlp/Cargo.toml index 4e45843312..2e0fd8cf4c 100644 --- a/opentelemetry-otlp/Cargo.toml +++ b/opentelemetry-otlp/Cargo.toml @@ -86,6 +86,3 @@ integration-testing = ["tonic", "prost", "tokio/full", "trace", "logs"] [lints] workspace = true - -[lib] -bench = false \ No newline at end of file From 64ea26596bd5d6f79f56df20096c0da8aeac5723 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 11:43:47 +0100 Subject: [PATCH 10/12] formatting --- opentelemetry/benches/context_attach.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/benches/context_attach.rs b/opentelemetry/benches/context_attach.rs index 748ec546e3..4d416f244d 100644 --- a/opentelemetry/benches/context_attach.rs +++ b/opentelemetry/benches/context_attach.rs @@ -119,4 +119,4 @@ struct Value(i32); criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); From 9311459f420e42a1928672ecaf8efecf1498118e Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Feb 2025 12:00:19 +0100 Subject: [PATCH 11/12] Allow commenting --- .github/workflows/pr_criterion.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml index 5f3ce5bdbd..47f4267097 100644 --- a/.github/workflows/pr_criterion.yaml +++ b/.github/workflows/pr_criterion.yaml @@ -3,6 +3,8 @@ name: benchmark pull requests jobs: runBenchmark: name: run benchmark + permissions: + pull-requests: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From e9b77ca5028b1704a76d048af71c90b163a4db72 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Feb 2025 08:00:02 +0100 Subject: [PATCH 12/12] =?UTF-8?q?Remove=20bench=20(because=20Bj=C3=B6rn=20?= =?UTF-8?q?is=20adding=20it)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- opentelemetry/Cargo.toml | 5 - opentelemetry/benches/context_attach.rs | 122 ------------------------ 2 files changed, 127 deletions(-) delete mode 100644 opentelemetry/benches/context_attach.rs diff --git a/opentelemetry/Cargo.toml b/opentelemetry/Cargo.toml index 4a7205110b..064d5004b3 100644 --- a/opentelemetry/Cargo.toml +++ b/opentelemetry/Cargo.toml @@ -57,11 +57,6 @@ harness = false name = "anyvalue" harness = false -[[bench]] -name = "context_attach" -harness = false -required-features = ["tracing"] - [lib] bench = false diff --git a/opentelemetry/benches/context_attach.rs b/opentelemetry/benches/context_attach.rs deleted file mode 100644 index 4d416f244d..0000000000 --- a/opentelemetry/benches/context_attach.rs +++ /dev/null @@ -1,122 +0,0 @@ -use criterion::{ - black_box, criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, BenchmarkId, - Criterion, Throughput, -}; -use opentelemetry::{ - trace::{SpanContext, TraceContextExt}, - Context, -}; - -// Run this benchmark with: -// cargo bench --bench context_attach - -fn criterion_benchmark(c: &mut Criterion) { - let span_context = Context::new().with_remote_span_context(SpanContext::empty_context()); - let contexts = vec![ - ("empty_cx", Context::new()), - ("single_value_cx", Context::new().with_value(Value(4711))), - ("span_cx", span_context), - ]; - for (name, cx) in contexts { - single_cx_scope(&mut group(c), name, &cx); - nested_cx_scope(&mut group(c), name, &cx); - overlapping_cx_scope(&mut group(c), name, &cx); - } -} - -fn single_cx_scope( - group: &mut BenchmarkGroup<'_, WallTime>, - context_type: &str, - context: &Context, -) { - let _restore = Context::current().attach(); - group.throughput(Throughput::Elements(1)).bench_function( - BenchmarkId::new("single_cx_scope", context_type), - |b| { - b.iter_batched( - || context.clone(), - |cx| { - single_cx(cx); - }, - criterion::BatchSize::SmallInput, - ); - }, - ); -} - -#[inline(never)] -fn single_cx(cx: Context) { - let cx = black_box(cx.attach()); - let _ = black_box(dummy_work()); - drop(cx); -} - -fn nested_cx_scope(group: &mut BenchmarkGroup<'_, WallTime>, cx_type: &str, context: &Context) { - let _restore = Context::current().attach(); - group.throughput(Throughput::Elements(1)).bench_function( - BenchmarkId::new("nested_cx_scope", cx_type), - |b| { - b.iter_batched( - || (context.clone(), context.clone()), - |(cx1, cx2)| { - nested_cx(cx1, cx2); - }, - criterion::BatchSize::SmallInput, - ); - }, - ); -} - -#[inline(never)] -fn nested_cx(cx1: Context, cx2: Context) { - let outer = black_box(cx1.attach()); - let inner = black_box(cx2.attach()); - let _ = black_box(dummy_work()); - drop(inner); - drop(outer); -} - -fn overlapping_cx_scope( - group: &mut BenchmarkGroup<'_, WallTime>, - cx_type: &str, - context: &Context, -) { - let _restore = Context::current().attach(); - group.throughput(Throughput::Elements(1)).bench_function( - BenchmarkId::new("overlapping_cx_scope", cx_type), - |b| { - b.iter_batched( - || (context.clone(), context.clone()), - |(cx1, cx2)| { - overlapping_cx(cx1, cx2); - }, - criterion::BatchSize::SmallInput, - ); - }, - ); -} - -#[inline(never)] -fn overlapping_cx(cx1: Context, cx2: Context) { - let outer = cx1.attach(); - let inner = cx2.attach(); - let _ = black_box(dummy_work()); - drop(outer); - drop(inner); -} - -#[inline(never)] -fn dummy_work() -> i32 { - black_box(1 + 1) -} - -fn group(c: &mut Criterion) -> BenchmarkGroup { - c.benchmark_group("context_attach") -} - -#[derive(Debug, PartialEq)] -struct Value(i32); - -criterion_group!(benches, criterion_benchmark); - -criterion_main!(benches);