RUST-1044 Update evergreen config to test compressors. (#486)

Update evergreen config to use compressors.

Author: Andrew Witten

.evergreen/check-clippy.sh

@@ -4,5 +4,7 @@ set -o errexit
 
 . ~/.cargo/env
 cargo clippy --all-targets -p mongodb -- -D warnings
+# check clippy with compressors separately
+cargo clippy --all-targets -p mongodb --features zstd-compression,snappy-compression,zlib-compression -- -D warnings
 cargo clippy --all-targets --no-default-features --features async-std-runtime -p mongodb -- -D warnings
 cargo clippy --all-targets --no-default-features --features sync -p mongodb -- -D warnings

.evergreen/compile-only-tokio.sh

@@ -4,4 +4,12 @@ set -o errexit
 
 . ~/.cargo/env
 
-rustup run $RUST_VERSION cargo build
+# Enable snappy, zlib unconditionally
+FEATURE_FLAGS=snappy-compression,zlib-compression
+
+# Zstd requires Rust version 1.54
+if [[ $RUST_VERSION == "nightly" ]]; then
+    FEATURE_FLAGS=$FEATURE_FLAGS,zstd-compression
+fi
+
+rustup run $RUST_VERSION cargo build --features $FEATURE_FLAGS

.evergreen/config.yml

@@ -259,7 +259,7 @@ functions:
           ECS_SRC_DIR=$AUTH_AWS_DIR/src
 
           mkdir -p $ECS_SRC_DIR/.evergreen
-          
+
           # fix issue with `TestData` in SERVER-46340
           sed -i '1s+^+TestData = {};\n+' $AUTH_AWS_DIR/lib/ecs_hosted_test.js
 
@@ -294,7 +294,7 @@ functions:
           export MONGODB_URI="${MONGODB_URI}"
           export SSL="${SSL}"
           . .evergreen/generate-uri.sh
-          
+
           export CERT_PATH=$DRIVERS_TOOLS/.evergreen/x509gen/client.pem
 
           .evergreen/run-x509-tests.sh
@@ -320,7 +320,7 @@ functions:
           ${PREPARE_SHELL}
 
           .evergreen/run-connection-string-tests.sh
-            
+
   "prepare resources":
     - command: shell.exec
       params:
@@ -369,6 +369,9 @@ functions:
           export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
           . .evergreen/generate-uri.sh
 
+          SNAPPY_COMPRESSION_ENABLED=${SNAPPY_COMPRESSION_ENABLED} \
+          ZLIB_COMPRESSION_ENABLED=${ZLIB_COMPRESSION_ENABLED} \
+          ZSTD_COMPRESSION_ENABLED=${ZSTD_COMPRESSION_ENABLED} \
           SINGLE_THREAD=${SINGLE_THREAD} \
             ASYNC_RUNTIME=${ASYNC_RUNTIME} \
             MONGODB_API_VERSION=${MONGODB_API_VERSION} \
@@ -546,7 +549,7 @@ functions:
       params:
         script: |
           ${PREPARE_SHELL}
-          
+
           cd "$MONGO_ORCHESTRATION_HOME"
           # source the mongo-orchestration virtualenv if it exists
           if [ -f venv/bin/activate ]; then
@@ -615,7 +618,7 @@ functions:
     - command: expansions.update
       params:
         file: lb-expansion.yml
-  
+
   "stop load balancer":
     - command: shell.exec
       params:
@@ -836,7 +839,7 @@ tasks:
           MONGODB_VERSION: "latest"
           TOPOLOGY: "sharded_cluster"
       - func: "run tests"
-  
+
   - name: "test-latest-load_balancer"
     tags: ["latest", "load_balancer"]
     commands:
@@ -1314,6 +1317,24 @@ axes:
            AUTH: "noauth"
            SSL: "nossl"
 
+  - id: "compressor"
+    display_name: "Compressor"
+    values:
+      - id: "no-compression"
+        display_name: No Compressor
+      - id: "zstd-compression"
+        display_name: Zstd
+        variables:
+          ZSTD_COMPRESSION_ENABLED: "true"
+      - id: "zlib-compression"
+        display_name: Zlib
+        variables:
+          ZLIB_COMPRESSION_ENABLED: "true"
+      - id: "snappy-compression"
+        display_name: Snappy
+        variables:
+          SNAPPY_COMPRESSION_ENABLED: "true"
+
   - id: "os"
     display_name: OS
     values:
@@ -1370,7 +1391,18 @@ buildvariants:
       - windows-64-vs2017
     auth-and-tls: "*"
     async-runtime: "*"
-  display_name: "! ${os} ${auth-and-tls} with ${async-runtime}"
+    compressor: "*"
+  exclude_spec:
+    - compressor: ["zlib-compression", "zstd-compression", "snappy-compression"]
+      auth-and-tls: "noauth-and-notls"
+      os: "*"
+      async-runtime: "*"
+    - compressor: ["zlib-compression", "zstd-compression", "snappy-compression"]
+      async-runtime: "async-std"
+      os: "*"
+      auth-and-tls: "*"
+
+  display_name: "! ${os} ${auth-and-tls} with ${async-runtime} and ${compressor}"
   tasks:
      - ".latest"
      - ".5.0"
@@ -1386,26 +1418,30 @@ buildvariants:
         os: "ubuntu-20.04"
         auth-and-tls: "*"
         async-runtime: "*"
+        compressor: "*"
       then:
         remove_tasks: [".3.6", ".4.0", ".4.2"]
     - if:
         os: "ubuntu-18.04"
         auth-and-tls: "auth-and-tls"
         async-runtime: "*"
+        compressor: "*"
       then:
         remove_tasks: ".3.6"
     # ubuntu 1804 ARM64 only has downloads for 4.2+
     - if:
         os: ["ubuntu-18.04-arm64"]
         auth-and-tls: "*"
         async-runtime: "*"
+        compressor: "*"
       then:
         remove_tasks: [".3.6", ".4.0"]
     # haproxy isn't installed on windows / ubuntu-arm
     - if:
         os: ["ubuntu-18.04-arm64", "windows-64-vs2017"]
         auth-and-tls: "*"
         async-runtime: "*"
+        compressor: "*"
       then:
         remove_tasks: ".load_balancer"
 - matrix_name: "x509-auth"
@@ -1432,7 +1468,7 @@ buildvariants:
   display_name: "${os} PLAIN auth with ${async-runtime}"
   tasks:
     - "test-plain-auth"
-    
+
 - matrix_name: "atlas-connect"
   matrix_spec:
     os:

.evergreen/run-tokio-tests.sh

@@ -8,6 +8,8 @@ if [ "$SINGLE_THREAD" = true ]; then
 	OPTIONS="-- --test-threads=1"
 fi
 
-echo "cargo test options: ${OPTIONS}"
+FEATURE_FLAGS="zstd-compression,snappy-compression,zlib-compression"
 
-RUST_BACKTRACE=1 cargo test $OPTIONS
+echo "cargo test options: --features $FEATURE_FLAGS ${OPTIONS}"
+
+RUST_BACKTRACE=1 cargo test --features $FEATURE_FLAGS $OPTIONS

src/client/options/mod.rs

@@ -976,7 +976,7 @@ impl ClientOptions {
     ///   * `authSource`: maps to the `source` field of the `credential` field
     ///   * `authMechanismProperties`: maps to the `mechanism_properties` field of the `credential`
     ///     field
-    ///   * `compressors`: not yet implemented
+    ///   * `compressors`: maps to the `compressors` field
     ///   * `connectTimeoutMS`: maps to the `connect_timeout` field
     ///   * `direct`: maps to the `direct` field
     ///   * `heartbeatFrequencyMS`: maps to the `heartbeat_frequency` field
@@ -1010,7 +1010,8 @@ impl ClientOptions {
     ///   * `w`: maps to the `w` field of the `write_concern` field
     ///   * `waitQueueTimeoutMS`: unsupported, does not map to any field
     ///   * `wTimeoutMS`: maps to the `w_timeout` field of the `write_concern` field
-    ///   * `zlibCompressionLevel`: not yet implemented
+    ///   * `zlibCompressionLevel`: maps to the `level` field of the `Compressor::Zlib` variant
+    ///     (which requires the `zlib-compression` feature flag) of the [`Compressor`] enum
     ///
     /// Note: if the `sync` feature is enabled, then this method will be replaced with [the sync
     /// version](#method.parse-1).

src/compression/mod.rs

@@ -14,8 +14,13 @@ use flate2::{
 
 #[cfg(feature = "zlib-compression")]
 use std::convert::TryInto;
-#[cfg(feature = "zstd-compression")]
-use std::io::prelude::*;
+
+#[cfg(any(
+    feature = "zstd-compression",
+    feature = "zlib-compression",
+    feature = "snappy-compression"
+))]
+use std::io::Write;
 
 use crate::error::{Error, ErrorKind, Result};
 

src/test/mod.rs

@@ -37,7 +37,7 @@ use lazy_static::lazy_static;
 use self::util::TestLock;
 use crate::{
     client::options::{ServerApi, ServerApiVersion},
-    options::ClientOptions,
+    options::{ClientOptions, Compressor},
 };
 use std::{fs::read_to_string, str::FromStr};
 
@@ -62,6 +62,12 @@ lazy_static! {
         std::env::var("SINGLE_MONGOS_LB_URI").ok();
     pub(crate) static ref LOAD_BALANCED_MULTIPLE_URI: Option<String> =
         std::env::var("MULTI_MONGOS_LB_URI").ok();
+    pub(crate) static ref ZSTD_COMPRESSION_ENABLED: bool =
+        matches!(std::env::var("ZSTD_COMPRESSION_ENABLED"), Ok(s) if s == "true");
+    pub(crate) static ref ZLIB_COMPRESSION_ENABLED: bool =
+        matches!(std::env::var("ZLIB_COMPRESSION_ENABLED"), Ok(s) if s == "true");
+    pub(crate) static ref SNAPPY_COMPRESSION_ENABLED: bool =
+        matches!(std::env::var("SNAPPY_COMPRESSION_ENABLED"), Ok(s) if s == "true");
 }
 
 pub(crate) fn client_options_for_uri(uri: &str) -> ClientOptions {
@@ -74,10 +80,39 @@ pub(crate) fn client_options_for_uri(uri: &str) -> ClientOptions {
     {
         options.test_options_mut().mock_service_id = true;
     }
-
+    options.compressors = get_compressors();
     options
 }
 
+fn get_compressors() -> Option<Vec<Compressor>> {
+    #[allow(unused_mut)]
+    let mut compressors = vec![];
+
+    if *SNAPPY_COMPRESSION_ENABLED {
+        #[cfg(feature = "snappy-compression")]
+        compressors.push(Compressor::Snappy);
+        #[cfg(not(feature = "snappy-compression"))]
+        panic!("To use snappy compression, the \"snappy-compression\" feature flag must be set.");
+    }
+    if *ZLIB_COMPRESSION_ENABLED {
+        #[cfg(feature = "zlib-compression")]
+        compressors.push(Compressor::Zlib { level: None });
+        #[cfg(not(feature = "zlib-compression"))]
+        panic!("To use zlib compression, the \"zlib-compression\" feature flag must be set.");
+    }
+    if *ZSTD_COMPRESSION_ENABLED {
+        #[cfg(feature = "zstd-compression")]
+        compressors.push(Compressor::Zstd { level: None });
+        #[cfg(not(feature = "zstd-compression"))]
+        panic!("To use zstd compression, the \"zstd-compression\" feature flag must be set.");
+    }
+    if compressors.is_empty() {
+        None
+    } else {
+        Some(compressors)
+    }
+}
+
 fn get_default_uri() -> String {
     if let Some(uri) = LOAD_BALANCED_SINGLE_URI.clone() {
         if !uri.is_empty() {