benches: move to criterion-rs and GitHub actions (#366)

This moves all benches to criterion-rs, in order to allow running
benchmarks on a stable toolchain.
This also moves benchmarking and linting jobs to a pinned toolchain,
under GitHub actions.

Signed-off-by: Luca BRUNO <[email protected]>

Author: lucab

.github/workflows/rust.yml

@@ -0,0 +1,45 @@
+---
+name: Rust
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+  # Pinned toolchain for linting and benchmarks
+  ACTIONS_LINTS_TOOLCHAIN: 1.47.0
+
+jobs:
+  linting:
+    name: "Lints, pinned toolchain"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Install toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env['ACTIONS_LINTS_TOOLCHAIN']  }}
+          default: true
+          components: rustfmt, clippy
+      - name: cargo fmt (check)
+        run: cargo fmt --all -- --check -l
+      - name: cargo clippy
+        run: cargo clippy --all -- -D clippy
+  criterion:
+    name: "Benchmarks (criterion)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Install toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env['ACTIONS_LINTS_TOOLCHAIN']  }}
+          default: true
+      - name: cargo bench (prometheus)
+        run: cargo bench -p prometheus
+      - name: cargo bench (prometheus-static-metric)
+        run: cargo bench -p prometheus-static-metric

.travis.yml

@@ -29,12 +29,6 @@ matrix:
     env: FEATURES="nightly protobuf push process"
   - rust: beta
   - rust: stable
-    install:
-      - rustup component add clippy-preview
-      - rustup component add rustfmt-preview
-    before_script:
-      - cargo fmt --all -- --check
-      - cargo clippy --all -- -D clippy
   allow_failures:
   - rust: nightly
 
@@ -44,8 +38,4 @@ script:
     if [ $TRAVIS_RUST_VERSION = 'nightly' ]; then
         cargo build -p prometheus-static-metric --examples --no-default-features --features="$FEATURES"
         cargo test -p prometheus-static-metric --no-default-features --features="$FEATURES"
-
-        # Run benchmarks.
-        cargo bench --no-default-features --features="$FEATURES"
-        cargo bench -p prometheus-static-metric --no-default-features --features="$FEATURES"
     fi

Cargo.toml

@@ -39,6 +39,7 @@ thiserror = "^1.0"
 procfs = { version = "^0.8", optional = true, default-features = false }
 
 [dev-dependencies]
+criterion = "0.3"
 getopts = "^0.2"
 hyper = "^0.13"
 tokio = { version = "^0.3", features = ["macros", "rt-multi-thread"] }
@@ -48,3 +49,27 @@ protobuf-codegen-pure = { version = "^2.0", optional = true }
 
 [workspace]
 members = ["static-metric"]
+
+[[bench]]
+name = "atomic"
+harness = false
+
+[[bench]]
+name = "counter"
+harness = false
+
+[[bench]]
+name = "desc"
+harness = false
+
+[[bench]]
+name = "gauge"
+harness = false
+
+[[bench]]
+name = "histogram"
+harness = false
+
+[[bench]]
+name = "text_encoder"
+harness = false

benches/atomic.rs

@@ -11,28 +11,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(test)]
+use criterion::{criterion_group, criterion_main, Criterion};
+use prometheus::core::*;
 
-extern crate test;
-
-#[path = "../src/atomic64.rs"]
-mod atomic64;
-
-use crate::atomic64::*;
-use test::Bencher;
-
-#[bench]
-fn bench_atomic_f64(b: &mut Bencher) {
+fn bench_atomic_f64(c: &mut Criterion) {
     let val = AtomicF64::new(0.0);
-    b.iter(|| {
-        val.inc_by(12.0);
+    c.bench_function("atomic_f64", |b| {
+        b.iter(|| {
+            val.inc_by(12.0);
+        })
     });
 }
 
-#[bench]
-fn bench_atomic_i64(b: &mut Bencher) {
+fn bench_atomic_i64(c: &mut Criterion) {
     let val = AtomicI64::new(0);
-    b.iter(|| {
-        val.inc_by(12);
+    c.bench_function("atomic_i64", |b| {
+        b.iter(|| {
+            val.inc_by(12);
+        })
     });
 }
+
+criterion_group!(benches, bench_atomic_f64, bench_atomic_i64);
+criterion_main!(benches);

benches/counter.rs

@@ -11,46 +11,42 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(test)]
-
-extern crate test;
-
+use criterion::{criterion_group, criterion_main, Criterion};
+use prometheus::{Counter, CounterVec, IntCounter, Opts};
 use std::collections::HashMap;
 use std::sync::{atomic, Arc};
 use std::thread;
 
-use prometheus::{Counter, CounterVec, IntCounter, Opts};
-use test::Bencher;
-
-#[bench]
-fn bench_counter_with_label_values(b: &mut Bencher) {
+fn bench_counter_with_label_values(c: &mut Criterion) {
     let counter = CounterVec::new(
         Opts::new("benchmark_counter", "A counter to benchmark it."),
         &["one", "two", "three"],
     )
     .unwrap();
-    b.iter(|| counter.with_label_values(&["eins", "zwei", "drei"]).inc())
+    c.bench_function("counter_with_label_values", |b| {
+        b.iter(|| counter.with_label_values(&["eins", "zwei", "drei"]).inc())
+    });
 }
 
-#[bench]
-fn bench_counter_with_mapped_labels(b: &mut Bencher) {
+fn bench_counter_with_mapped_labels(c: &mut Criterion) {
     let counter = CounterVec::new(
         Opts::new("benchmark_counter", "A counter to benchmark it."),
         &["one", "two", "three"],
     )
     .unwrap();
 
-    b.iter(|| {
-        let mut labels = HashMap::with_capacity(3);
-        labels.insert("two", "zwei");
-        labels.insert("one", "eins");
-        labels.insert("three", "drei");
-        counter.with(&labels).inc();
-    })
+    c.bench_function("counter_with_mapped_labels", |b| {
+        b.iter(|| {
+            let mut labels = HashMap::with_capacity(3);
+            labels.insert("two", "zwei");
+            labels.insert("one", "eins");
+            labels.insert("three", "drei");
+            counter.with(&labels).inc();
+        })
+    });
 }
 
-#[bench]
-fn bench_counter_with_prepared_mapped_labels(b: &mut Bencher) {
+fn bench_counter_with_prepared_mapped_labels(c: &mut Criterion) {
     let counter = CounterVec::new(
         Opts::new("benchmark_counter", "A counter to benchmark it."),
         &["one", "two", "three"],
@@ -62,25 +58,24 @@ fn bench_counter_with_prepared_mapped_labels(b: &mut Bencher) {
     labels.insert("one", "eins");
     labels.insert("three", "drei");
 
-    b.iter(|| {
-        counter.with(&labels).inc();
-    })
+    c.bench_function("counter_with_prepared_mapped_labels", |b| {
+        b.iter(|| {
+            counter.with(&labels).inc();
+        })
+    });
 }
 
-#[bench]
-fn bench_counter_no_labels(b: &mut Bencher) {
+fn bench_counter_no_labels(c: &mut Criterion) {
     let counter = Counter::new("benchmark_counter", "A counter to benchmark.").unwrap();
-    b.iter(|| counter.inc())
+    c.bench_function("counter_no_labels", |b| b.iter(|| counter.inc()));
 }
 
-#[bench]
-fn bench_int_counter_no_labels(b: &mut Bencher) {
+fn bench_int_counter_no_labels(c: &mut Criterion) {
     let counter = IntCounter::new("benchmark_int_counter", "A int_counter to benchmark.").unwrap();
-    b.iter(|| counter.inc())
+    c.bench_function("int_counter_no_labels", |b| b.iter(|| counter.inc()));
 }
 
-#[bench]
-fn bench_counter_no_labels_concurrent_nop(b: &mut Bencher) {
+fn bench_counter_no_labels_concurrent_nop(c: &mut Criterion) {
     let signal_exit = Arc::new(atomic::AtomicBool::new(false));
     let counter = Counter::new("foo", "bar").unwrap();
 
@@ -95,7 +90,9 @@ fn bench_counter_no_labels_concurrent_nop(b: &mut Bencher) {
         })
         .collect();
 
-    b.iter(|| counter.inc());
+    c.bench_function("counter_no_labels_concurrent_nop", |b| {
+        b.iter(|| counter.inc());
+    });
 
     // Wait for accompanying thread to exit.
     signal_exit.store(true, atomic::Ordering::Relaxed);
@@ -104,8 +101,7 @@ fn bench_counter_no_labels_concurrent_nop(b: &mut Bencher) {
     }
 }
 
-#[bench]
-fn bench_counter_no_labels_concurrent_write(b: &mut Bencher) {
+fn bench_counter_no_labels_concurrent_write(c: &mut Criterion) {
     let signal_exit = Arc::new(atomic::AtomicBool::new(false));
     let counter = Counter::new("foo", "bar").unwrap();
 
@@ -122,7 +118,9 @@ fn bench_counter_no_labels_concurrent_write(b: &mut Bencher) {
         })
         .collect();
 
-    b.iter(|| counter.inc());
+    c.bench_function("counter_no_labels_concurrent_write", |b| {
+        b.iter(|| counter.inc());
+    });
 
     // Wait for accompanying thread to exit.
     signal_exit.store(true, atomic::Ordering::Relaxed);
@@ -131,8 +129,7 @@ fn bench_counter_no_labels_concurrent_write(b: &mut Bencher) {
     }
 }
 
-#[bench]
-fn bench_int_counter_no_labels_concurrent_write(b: &mut Bencher) {
+fn bench_int_counter_no_labels_concurrent_write(c: &mut Criterion) {
     let signal_exit = Arc::new(atomic::AtomicBool::new(false));
     let counter = IntCounter::new("foo", "bar").unwrap();
 
@@ -149,7 +146,9 @@ fn bench_int_counter_no_labels_concurrent_write(b: &mut Bencher) {
         })
         .collect();
 
-    b.iter(|| counter.inc());
+    c.bench_function("int_counter_no_labels_concurrent_write", |b| {
+        b.iter(|| counter.inc());
+    });
 
     // Wait for accompanying thread to exit.
     signal_exit.store(true, atomic::Ordering::Relaxed);
@@ -158,8 +157,7 @@ fn bench_int_counter_no_labels_concurrent_write(b: &mut Bencher) {
     }
 }
 
-#[bench]
-fn bench_counter_with_label_values_concurrent_write(b: &mut Bencher) {
+fn bench_counter_with_label_values_concurrent_write(c: &mut Criterion) {
     let signal_exit = Arc::new(atomic::AtomicBool::new(false));
     let counter = CounterVec::new(Opts::new("foo", "bar"), &["one", "two", "three"]).unwrap();
 
@@ -175,11 +173,27 @@ fn bench_counter_with_label_values_concurrent_write(b: &mut Bencher) {
         })
         .collect();
 
-    b.iter(|| counter.with_label_values(&["eins", "zwei", "drei"]).inc());
+    c.bench_function("counter_with_label_values_concurrent_write", |b| {
+        b.iter(|| counter.with_label_values(&["eins", "zwei", "drei"]).inc());
+    });
 
     // Wait for accompanying thread to exit.
     signal_exit.store(true, atomic::Ordering::Relaxed);
     for h in thread_handles {
         h.join().unwrap();
     }
 }
+
+criterion_group!(
+    benches,
+    bench_counter_no_labels,
+    bench_counter_no_labels_concurrent_nop,
+    bench_counter_no_labels_concurrent_write,
+    bench_counter_with_label_values,
+    bench_counter_with_label_values_concurrent_write,
+    bench_counter_with_mapped_labels,
+    bench_counter_with_prepared_mapped_labels,
+    bench_int_counter_no_labels,
+    bench_int_counter_no_labels_concurrent_write,
+);
+criterion_main!(benches);

benches/desc.rs

@@ -11,22 +11,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(test)]
-
-extern crate test;
-
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use prometheus::core::Desc;
 
-use test::{black_box, Bencher};
-
-#[bench]
-fn description_validation(b: &mut Bencher) {
-    b.iter(|| {
-        black_box(Desc::new(
-            "api_http_requests_total".to_string(),
-            "not empty help".to_string(),
-            vec!["method".to_string(), "handler".to_string()],
-            Default::default(),
-        ))
+fn description_validation(c: &mut Criterion) {
+    c.bench_function("description_validation", |b| {
+        b.iter(|| {
+            black_box(Desc::new(
+                "api_http_requests_total".to_string(),
+                "not empty help".to_string(),
+                vec!["method".to_string(), "handler".to_string()],
+                Default::default(),
+            ))
+        });
     });
 }
+
+criterion_group!(benches, description_validation);
+criterion_main!(benches);