Update tests & benchmarks to the non-generic AVX2Encoder

Author: dequbed

Cargo.toml

@@ -27,6 +27,7 @@ structopt = "0.3.21"
 # test fixtures for engine tests
 rstest = "0.11.0"
 rstest_reuse = "0.1.3"
+lazy_static = "1.4.0"
 
 [features]
 default = ["std"]

benches/avx2benchmarks.rs

@@ -3,18 +3,25 @@ extern crate base64;
 extern crate criterion;
 extern crate rand;
 
+#[macro_use]
+extern crate lazy_static;
+
+use std::ops::Deref;
+
 use base64::display;
 use base64::{
     decode_engine, decode_engine_slice, decode_engine_vec, encode_engine, encode_engine_slice,
     encode_engine_string, write, engine::DEFAULT_ENGINE,
 };
 
-use base64::engine::avx2::{AVX2Encoder, AVX2Config, Standard};
+use base64::engine::avx2::{AVX2Encoder, AVX2Config};
 use criterion::{black_box, Bencher, BenchmarkId, Criterion, Throughput};
 use rand::{FromEntropy, Rng};
 use std::io::{self, Read, Write};
 
-const AVX2_ENGINE: AVX2Encoder<Standard> = AVX2Encoder::from(AVX2Config::new());
+lazy_static! {
+    static ref AVX2_ENGINE: AVX2Encoder = AVX2Encoder::from_standard(AVX2Config::new());
+}
 
 fn do_decode_bench(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
@@ -29,10 +36,10 @@ fn do_decode_bench(b: &mut Bencher, &size: &usize) {
 fn do_decode_bench_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
-    let encoded = encode_engine(&v, &AVX2_ENGINE);
+    let encoded = encode_engine(&v, AVX2_ENGINE.deref());
 
     b.iter(|| {
-        let orig = decode_engine(&encoded, &AVX2_ENGINE);
+        let orig = decode_engine(&encoded, AVX2_ENGINE.deref());
         black_box(&orig);
     });
 }
@@ -53,11 +60,11 @@ fn do_decode_bench_reuse_buf(b: &mut Bencher, &size: &usize) {
 fn do_decode_bench_reuse_buf_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
-    let encoded = encode_engine(&v, &AVX2_ENGINE);
+    let encoded = encode_engine(&v, AVX2_ENGINE.deref());
 
     let mut buf = Vec::new();
     b.iter(|| {
-        decode_engine_vec(&encoded, &mut buf, &AVX2_ENGINE).unwrap();
+        decode_engine_vec(&encoded, &mut buf, AVX2_ENGINE.deref()).unwrap();
         black_box(&buf);
         buf.clear();
     });
@@ -79,12 +86,12 @@ fn do_decode_bench_slice(b: &mut Bencher, &size: &usize) {
 fn do_decode_bench_slice_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
-    let encoded = encode_engine(&v, &AVX2_ENGINE);
+    let encoded = encode_engine(&v, AVX2_ENGINE.deref());
 
     let mut buf = Vec::new();
     buf.resize(size, 0);
     b.iter(|| {
-        decode_engine_slice(&encoded, &mut buf, &AVX2_ENGINE).unwrap();
+        decode_engine_slice(&encoded, &mut buf, AVX2_ENGINE.deref()).unwrap();
         black_box(&buf);
     });
 }
@@ -110,15 +117,15 @@ fn do_decode_bench_stream(b: &mut Bencher, &size: &usize) {
 fn do_decode_bench_stream_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
-    let encoded = encode_engine(&v, &AVX2_ENGINE);
+    let encoded = encode_engine(&v, AVX2_ENGINE.deref());
 
     let mut buf = Vec::new();
     buf.resize(size, 0);
     buf.truncate(0);
 
     b.iter(|| {
         let mut cursor = io::Cursor::new(&encoded[..]);
-        let mut decoder = base64::read::DecoderReader::from(&mut cursor, &AVX2_ENGINE);
+        let mut decoder = base64::read::DecoderReader::from(&mut cursor, AVX2_ENGINE.deref());
         decoder.read_to_end(&mut buf).unwrap();
         buf.clear();
         black_box(&buf);
@@ -138,7 +145,7 @@ fn do_encode_bench_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size);
     fill(&mut v);
     b.iter(|| {
-        let e = encode_engine(&v, &AVX2_ENGINE);
+        let e = encode_engine(&v, AVX2_ENGINE.deref());
         black_box(&e);
     });
 }
@@ -156,7 +163,7 @@ fn do_encode_bench_display_avx(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size);
     fill(&mut v);
     b.iter(|| {
-        let e = format!("{}", display::Base64Display::from(&v, &AVX2_ENGINE));
+        let e = format!("{}", display::Base64Display::from(&v, AVX2_ENGINE.deref()));
         black_box(&e);
     });
 }
@@ -176,7 +183,7 @@ fn do_encode_bench_reuse_buf_avx(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
     let mut buf = String::new();
     b.iter(|| {
-        encode_engine_string(&v, &mut buf, &AVX2_ENGINE);
+        encode_engine_string(&v, &mut buf, AVX2_ENGINE.deref());
         buf.clear();
     });
 }
@@ -199,7 +206,7 @@ fn do_encode_bench_slice_avx(b: &mut Bencher, &size: &usize) {
     // conservative estimate of encoded size
     buf.resize(v.len() * 2, 0);
     b.iter(|| {
-        encode_engine_slice(&v, &mut buf, &AVX2_ENGINE);
+        encode_engine_slice(&v, &mut buf, AVX2_ENGINE.deref());
     });
 }
 
@@ -225,7 +232,7 @@ fn do_encode_bench_stream_avx(b: &mut Bencher, &size: &usize) {
     buf.reserve(size * 2);
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderWriter::from(&mut buf, &AVX2_ENGINE);
+        let mut stream_enc = write::EncoderWriter::from(&mut buf, AVX2_ENGINE.deref());
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
     });
@@ -248,7 +255,7 @@ fn do_encode_bench_string_stream_avx(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
 
     b.iter(|| {
-        let mut stream_enc = write::EncoderStringWriter::from(&AVX2_ENGINE);
+        let mut stream_enc = write::EncoderStringWriter::from(AVX2_ENGINE.deref());
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();
@@ -276,7 +283,7 @@ fn do_encode_bench_string_reuse_buf_stream_avx(b: &mut Bencher, &size: &usize) {
     let mut buf = String::new();
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderStringWriter::from_consumer(&mut buf, &AVX2_ENGINE);
+        let mut stream_enc = write::EncoderStringWriter::from_consumer(&mut buf, AVX2_ENGINE.deref());
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();
@@ -547,11 +554,63 @@ fn decode_benchmarks(c: &mut Criterion, label: &str, byte_sizes: &[usize]) {
 
 }
 
+fn do_align_bench(b: &mut Bencher, &size: &usize) {
+    let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4 + 32);
+    fill(&mut v);
+
+    let (pre, aligned_u32, post) = unsafe {
+        v.align_to_mut::<u32>()
+    };
+    let aligned: &[u8] = unsafe {
+        core::mem::transmute(aligned_u32)
+    };
+    assert!(pre.len() == 0);
+    assert!(post.len() == 0);
+
+
+    let encoded = encode_engine(&v, AVX2_ENGINE.deref());
+
+    let mut buf = Vec::new();
+    buf.resize(size, 0);
+    b.iter(|| {
+        decode_engine_slice(&encoded, &mut buf, AVX2_ENGINE.deref()).unwrap();
+        black_box(&buf);
+    });
+}
+fn do_unalign_bench(b: &mut Bencher, &size: &usize) {
+    let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4 + 32);
+    fill(&mut v);
+
+    let encoded = encode_engine(&v[5..], AVX2_ENGINE.deref());
+
+    let mut buf = Vec::new();
+    buf.resize(size, 0);
+    b.iter(|| {
+        decode_engine_slice(&encoded, &mut buf, AVX2_ENGINE.deref()).unwrap();
+        black_box(&buf);
+    });
+}
+
+fn align_benchmarks(c: &mut Criterion, label: &str, byte_sizes: &[usize]) {
+    let mut group = c.benchmark_group(label);
+    for size in byte_sizes {
+        group
+            .warm_up_time(std::time::Duration::from_millis(500))
+            .measurement_time(std::time::Duration::from_secs(3))
+            .throughput(Throughput::Bytes(*size as u64))
+            .bench_with_input(BenchmarkId::new("aligned", size), size, do_align_bench)
+            .bench_with_input(BenchmarkId::new("unaligned", size), size, do_unalign_bench);
+    }
+    group.finish();
+}
+
 fn bench(c: &mut Criterion) {
     encode_benchmarks(c, "encode_small_input", &BYTE_SIZES[..]);
     encode_benchmarks(c, "encode_large_input", &LARGE_BYTE_SIZES[..]);
     decode_benchmarks(c, "decode_small_input", &BYTE_SIZES[..]);
     decode_benchmarks(c, "decode_large_input", &LARGE_BYTE_SIZES[..]);
+
+    align_benchmarks(c, "align_benchmark", &LARGE_BYTE_SIZES[..]);
 }
 
 criterion_group!(benches, bench);

src/engine/tests.rs

@@ -962,20 +962,20 @@ mod avx2_tests {
     pub(super) struct AVX2Wrapper {}
 
     impl EngineWrapper for AVX2Wrapper {
-        type Engine = avx2::AVX2Encoder<avx2::Standard>;
+        type Engine = avx2::AVX2Encoder;
 
         fn standard() -> Self::Engine {
-            avx2::AVX2Encoder::from(avx2::AVX2Config::default())
+            avx2::AVX2Encoder::from_standard(avx2::AVX2Config::default())
         }
 
         fn standard_forgiving() -> Self::Engine {
-            avx2::AVX2Encoder::from(avx2::AVX2Config::default()
+            avx2::AVX2Encoder::from_standard(avx2::AVX2Config::default()
                 .with_decode_allow_trailing_bits(true))
         }
 
         fn random<R: Rng>(_rng: &mut R) -> Self::Engine {
             // The avx alg can't handle custom alphabets yet
-            avx2::AVX2Encoder::from(avx2::AVX2Config::default())
+            avx2::AVX2Encoder::from_standard(avx2::AVX2Config::default())
         }
 
         fn random_alphabet<R: Rng>(rng: &mut R, alphabet: &Alphabet) -> Self::Engine {

tests/decode.rs

@@ -89,17 +89,17 @@ fn decode_imap() {
 mod avx2test {
     use super::*;
 
-    use base64::engine::avx2::{AVX2Encoder, AVX2Config, Standard};
-    const AVX_ENGINE: AVX2Encoder<Standard> = AVX2Encoder::from(AVX2Config::new());
+    use base64::engine::avx2::{AVX2Encoder, AVX2Config};
 
     #[test]
     fn decode_long() {
+        let engine = AVX2Encoder::from_standard(AVX2Config::new());
         let out = decode_engine(
             "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0\
             BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+Ag\
             YKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHC\
             w8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==",
-            &AVX_ENGINE
+            &engine
         ).unwrap();
         println!("{:?}", out);
         for (a,b) in out.iter().enumerate() {
@@ -109,22 +109,24 @@ mod avx2test {
 
     #[test]
     fn decode_long_err() {
+        let engine = AVX2Encoder::from_standard(AVX2Config::new());
         let out = decode_engine(
             "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0.P0\
             BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+Ag\
             YKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHC\
             w8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==",
-            &AVX_ENGINE
+            &engine
         ).unwrap_err();
 
         assert_eq!(DecodeError::InvalidByte(83, '.' as u8), out);
     }
 
     #[test]
     fn decode_reject_null() {
+        let engine = AVX2Encoder::from_standard(AVX2Config::new());
         assert_eq!(
             DecodeError::InvalidByte(3, 0x0),
-            decode_engine("YWx\0pY2U==", &AVX_ENGINE).unwrap_err()
+            decode_engine("YWx\0pY2U==", &engine).unwrap_err()
         );
     }
 

tests/encode.rs

@@ -13,10 +13,10 @@ fn compare_encode(expected: &str, target: &[u8]) {
 fn compare_encode(expected: &str, target: &[u8]) {
     assert_eq!(expected, encode(target));
 
-    use base64::engine::avx2::{AVX2Encoder, AVX2Config, Standard};
-    const AVX_ENGINE: AVX2Encoder<Standard> = AVX2Encoder::from(AVX2Config::new());
+    use base64::engine::avx2::{AVX2Encoder, AVX2Config};
+    let engine: AVX2Encoder = AVX2Encoder::from_standard(AVX2Config::new());
 
-    assert_eq!(expected, encode_engine(target, &AVX_ENGINE));
+    assert_eq!(expected, encode_engine(target, &engine));
 }
 
 #[test]
@@ -127,11 +127,11 @@ fn encode_url_safe_without_padding() {
 mod avx2tests {
     use super::*;
 
-    use base64::engine::avx2::{AVX2Encoder, AVX2Config, Urlsafe};
-    const AVX_URL_ENGINE: AVX2Encoder<Urlsafe> = AVX2Encoder::from(AVX2Config::new());
+    use base64::engine::avx2::{AVX2Encoder, AVX2Config};
 
     #[test]
     fn encode_all_bytes_url() {
+        let engine: AVX2Encoder = AVX2Encoder::from_url_safe(AVX2Config::new());
         let mut bytes = Vec::<u8>::with_capacity(256);
 
         for i in 0..255 {
@@ -147,7 +147,7 @@ mod avx2tests {
              8_T19vf4-fr7_P3-_w==",
             encode_engine(
                 &bytes,
-                &AVX_URL_ENGINE
+                &engine
             )
         );
     }

tests/helpers.rs

@@ -26,16 +26,15 @@ pub fn compare_decode(expected: &str, target: &str) {
         String::from_utf8(decode_engine(target.as_bytes(), engine).unwrap()).unwrap()
     );
 
-    use base64::engine::avx2::{AVX2Encoder, AVX2Config, Standard};
-    const AVX_ENGINE: AVX2Encoder<Standard> = AVX2Encoder::from(AVX2Config::new());
-    let engine = &AVX_ENGINE;
+    use base64::engine::avx2::{AVX2Encoder, AVX2Config};
+    let engine = AVX2Encoder::from_standard(AVX2Config::new());
 
     assert_eq!(
         expected,
-        String::from_utf8(decode_engine(target, engine).unwrap()).unwrap()
+        String::from_utf8(decode_engine(target, &engine).unwrap()).unwrap()
     );
     assert_eq!(
         expected,
-        String::from_utf8(decode_engine(target.as_bytes(), engine).unwrap()).unwrap()
+        String::from_utf8(decode_engine(target.as_bytes(), &engine).unwrap()).unwrap()
     );
 }