RUST-626 Run benchmarks in Evergreen (#428)

Author: patrickfreed

.evergreen/benchmarks.yml

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -o errexit
+
+. ~/.cargo/env
+
+cd benchmarks
+cargo run \
+      --release \
+      -- --output="../benchmark-results.json" --bson
+
+cat ../benchmark-results.json

.evergreen/run-bson-benchmarks.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -o errexit
+
+. ~/.cargo/env
+
+FEATURES=""
+
+if [ "$ASYNC_RUNTIME" = "tokio" ]; then
+    FEATURES="tokio-runtime"
+elif [ "$ASYNC_RUNTIME" = "async-std" ]; then
+    FEATURES="async-std-runtime"
+else
+    echo "invalid async runtime: ${ASYNC_RUNTIME}" >&2
+    exit 1
+fi
+
+rustc --version
+
+SECONDS=0
+cargo build --release
+DURATION_IN_SECONDS="$SECONDS"
+
+cat > benchmark-results.json <<-EOF
+[
+  {
+    "info": {
+      "test_name": "Compile Time",
+      "args": {}
+    },
+    "metrics": [
+      {
+        "name": "compile_time_seconds",
+        "value": $DURATION_IN_SECONDS
+      }
+    ]
+  }
+]
+EOF
+
+cat benchmark-results.json

.evergreen/run-compile-benchmarks.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -o errexit
+
+. ~/.cargo/env
+
+FEATURES=""
+
+if [ "$ASYNC_RUNTIME" = "tokio" ]; then
+    FEATURES="tokio-runtime"
+elif [ "$ASYNC_RUNTIME" = "async-std" ]; then
+    FEATURES="async-std-runtime"
+else
+    echo "invalid async runtime: ${ASYNC_RUNTIME}" >&2
+    exit 1
+fi
+
+cd benchmarks
+cargo run \
+      --release \
+      --no-default-features \
+      --features ${FEATURES} \
+      -- --output="../benchmark-results.json" --single --multi --parallel
+
+cat ../benchmark-results.json

.evergreen/run-driver-benchmarks.sh

@@ -6,11 +6,18 @@ edition = "2018"
 
 [features]
 default = ["tokio-runtime"]
-tokio-runtime = ["tokio/fs", "tokio/macros", "tokio/rt", "tokio/rt-multi-thread", "tokio-stream"]
-async-std-runtime = ["async-std"]
+tokio-runtime = [
+  "tokio/fs",
+  "tokio/macros",
+  "tokio/rt",
+  "tokio/rt-multi-thread",
+  "tokio-stream",
+  "mongodb/tokio-runtime"
+]
+async-std-runtime = ["async-std", "mongodb/async-std-runtime"]
 
 [dependencies]
-mongodb = { path = ".." }
+mongodb = { path = "..", default-features = false }
 serde_json = "1.0.59"
 lazy_static = "1.4.0"
 clap = "2.33.3"
@@ -22,3 +29,4 @@ tokio-stream = { version = "0.1.6", features = ["io-util"], optional = true }
 async-std = { version = "1.9.0", optional = true, features = ["attributes", "unstable"] }
 futures = "0.3.8"
 anyhow = "1.0.34"
+serde = "1"

benchmarks/Cargo.toml

@@ -2,17 +2,23 @@ use std::{convert::TryInto, path::PathBuf};
 
 use anyhow::{bail, Result};
 use futures::stream::StreamExt;
-use mongodb::{Client, Collection, Database, bson::{Bson, Document}};
+use mongodb::{
+    bson::{Bson, Document},
+    Client,
+    Collection,
+    Database,
+};
 use serde_json::Value;
 
 use crate::{
-    bench::{Benchmark, COLL_NAME, DATABASE_NAME},
+    bench::{drop_database, Benchmark, COLL_NAME, DATABASE_NAME},
     fs::read_to_string,
 };
 
 pub struct FindManyBenchmark {
     db: Database,
     coll: Collection<Document>,
+    uri: String,
 }
 
 // Specifies the options to `FindManyBenchmark::setup` operation.
@@ -29,7 +35,7 @@ impl Benchmark for FindManyBenchmark {
     async fn setup(options: Self::Options) -> Result<Self> {
         let client = Client::with_uri_str(&options.uri).await?;
         let db = client.database(&DATABASE_NAME);
-        db.drop(None).await?;
+        drop_database(options.uri.as_str(), DATABASE_NAME.as_str()).await?;
 
         let num_iter = options.num_iter;
 
@@ -45,7 +51,11 @@ impl Benchmark for FindManyBenchmark {
         let docs = vec![doc.clone(); num_iter];
         coll.insert_many(docs, None).await?;
 
-        Ok(FindManyBenchmark { db, coll })
+        Ok(FindManyBenchmark {
+            db,
+            coll,
+            uri: options.uri,
+        })
     }
 
     async fn do_task(&self) -> Result<()> {
@@ -58,7 +68,7 @@ impl Benchmark for FindManyBenchmark {
     }
 
     async fn teardown(&self) -> Result<()> {
-        self.db.drop(None).await?;
+        drop_database(self.uri.as_str(), self.db.name()).await?;
 
         Ok(())
     }

benchmarks/src/bench/find_many.rs

@@ -1,18 +1,24 @@
 use std::{convert::TryInto, path::PathBuf};
 
 use anyhow::{bail, Result};
-use mongodb::{Client, Collection, Database, bson::{Bson, Document, doc}};
+use mongodb::{
+    bson::{doc, Bson, Document},
+    Client,
+    Collection,
+    Database,
+};
 use serde_json::Value;
 
 use crate::{
-    bench::{Benchmark, COLL_NAME, DATABASE_NAME},
+    bench::{drop_database, Benchmark, COLL_NAME, DATABASE_NAME},
     fs::read_to_string,
 };
 
 pub struct FindOneBenchmark {
     db: Database,
     num_iter: usize,
     coll: Collection<Document>,
+    uri: String,
 }
 
 // Specifies the options to a `FindOneBenchmark::setup` operation.
@@ -29,7 +35,7 @@ impl Benchmark for FindOneBenchmark {
     async fn setup(options: Self::Options) -> Result<Self> {
         let client = Client::with_uri_str(&options.uri).await?;
         let db = client.database(&DATABASE_NAME);
-        db.drop(None).await?;
+        drop_database(options.uri.as_str(), DATABASE_NAME.as_str()).await?;
 
         let num_iter = options.num_iter;
 
@@ -47,7 +53,12 @@ impl Benchmark for FindOneBenchmark {
             coll.insert_one(doc.clone(), None).await?;
         }
 
-        Ok(FindOneBenchmark { db, num_iter, coll })
+        Ok(FindOneBenchmark {
+            db,
+            num_iter,
+            coll,
+            uri: options.uri,
+        })
     }
 
     async fn do_task(&self) -> Result<()> {
@@ -61,7 +72,7 @@ impl Benchmark for FindOneBenchmark {
     }
 
     async fn teardown(&self) -> Result<()> {
-        self.db.drop(None).await?;
+        drop_database(self.uri.as_str(), self.db.name()).await?;
 
         Ok(())
     }

benchmarks/src/bench/find_one.rs

@@ -1,6 +1,6 @@
 use std::{convert::TryInto, fs::File, path::PathBuf};
 
-use anyhow::{bail, Result};
+use anyhow::{bail, Context, Result};
 use mongodb::{
     bson::{Bson, Document},
     Client,
@@ -9,16 +9,17 @@ use mongodb::{
 };
 use serde_json::Value;
 
-use crate::bench::{Benchmark, COLL_NAME, DATABASE_NAME};
+use crate::bench::{drop_database, Benchmark, COLL_NAME, DATABASE_NAME};
 
 pub struct InsertManyBenchmark {
     db: Database,
     num_copies: usize,
     coll: Collection<Document>,
     doc: Document,
+    uri: String,
 }
 
-// Specifies the options to a `InsertManyBenchmark::setup` operation.
+/// Specifies the options to a `InsertManyBenchmark::setup` operation.
 pub struct Options {
     pub num_copies: usize,
     pub path: PathBuf,
@@ -32,9 +33,10 @@ impl Benchmark for InsertManyBenchmark {
     async fn setup(options: Self::Options) -> Result<Self> {
         let client = Client::with_uri_str(&options.uri).await?;
         let db = client.database(&DATABASE_NAME);
-        db.drop(None).await?;
+        drop_database(options.uri.as_str(), DATABASE_NAME.as_str()).await?;
 
         let num_copies = options.num_copies;
+        let uri = options.uri.clone();
 
         // This benchmark uses a file that's quite large, and unfortunately `serde_json` has no
         // async version of `from_reader`, so rather than read the whole file into memory at once,
@@ -55,25 +57,34 @@ impl Benchmark for InsertManyBenchmark {
                 Bson::Document(doc) => doc,
                 _ => bail!("invalid json test file"),
             },
+            uri,
         })
     }
 
     async fn before_task(&mut self) -> Result<()> {
         self.coll.drop(None).await?;
-        self.db.create_collection(COLL_NAME.as_str(), None).await?;
+        self.db
+            .create_collection(COLL_NAME.as_str(), None)
+            .await
+            .context("create in before")?;
 
         Ok(())
     }
 
     async fn do_task(&self) -> Result<()> {
         let insertions = vec![&self.doc; self.num_copies];
-        self.coll.insert_many(insertions, None).await?;
+        self.coll
+            .insert_many(insertions, None)
+            .await
+            .context("insert many")?;
 
         Ok(())
     }
 
     async fn teardown(&self) -> Result<()> {
-        self.db.drop(None).await?;
+        drop_database(self.uri.as_str(), self.db.name())
+            .await
+            .context("teardown")?;
 
         Ok(())
     }

benchmarks/src/bench/insert_many.rs

@@ -1,6 +1,6 @@
 use std::{convert::TryInto, fs::File, path::PathBuf};
 
-use anyhow::{bail, Result};
+use anyhow::{bail, Context, Result};
 use mongodb::{
     bson::{Bson, Document},
     Client,
@@ -9,16 +9,17 @@ use mongodb::{
 };
 use serde_json::Value;
 
-use crate::bench::{Benchmark, COLL_NAME, DATABASE_NAME};
+use crate::bench::{drop_database, Benchmark, COLL_NAME, DATABASE_NAME};
 
 pub struct InsertOneBenchmark {
     db: Database,
     num_iter: usize,
     coll: Collection<Document>,
     doc: Document,
+    uri: String,
 }
 
-// Specifies the options to a `InsertOneBenchmark::setup` operation.
+/// Specifies the options to a `InsertOneBenchmark::setup` operation.
 pub struct Options {
     pub num_iter: usize,
     pub path: PathBuf,
@@ -32,9 +33,10 @@ impl Benchmark for InsertOneBenchmark {
     async fn setup(options: Self::Options) -> Result<Self> {
         let client = Client::with_uri_str(&options.uri).await?;
         let db = client.database(&DATABASE_NAME);
-        db.drop(None).await?;
+        drop_database(&options.uri, &DATABASE_NAME).await?;
 
         let num_iter = options.num_iter;
+        let uri = options.uri.clone();
 
         // This benchmark uses a file that's quite large, and unfortunately `serde_json` has no
         // async version of `from_reader`, so rather than read the whole file into memory at once,
@@ -55,26 +57,35 @@ impl Benchmark for InsertOneBenchmark {
                 Bson::Document(doc) => doc,
                 _ => bail!("invalid json test file"),
             },
+            uri,
         })
     }
 
     async fn before_task(&mut self) -> Result<()> {
         self.coll.drop(None).await?;
-        self.db.create_collection(COLL_NAME.as_str(), None).await?;
+        self.db
+            .create_collection(COLL_NAME.as_str(), None)
+            .await
+            .context("create collection")?;
 
         Ok(())
     }
 
     async fn do_task(&self) -> Result<()> {
         for _ in 0..self.num_iter {
-            self.coll.insert_one(&self.doc, None).await?;
+            self.coll
+                .insert_one(&self.doc, None)
+                .await
+                .context("insert one")?;
         }
 
         Ok(())
     }
 
     async fn teardown(&self) -> Result<()> {
-        self.db.drop(None).await?;
+        drop_database(&self.uri, &DATABASE_NAME)
+            .await
+            .context("drop database teardown")?;
 
         Ok(())
     }