Introduce the Engine abstraction.

This will allow users to plug different implementations in to the rest of the API.

While I'm touching almost everything and breaking backwards compatibility, a bunch of renaming and cleanup was also done.

Author: marshallpierce

.travis.yml

@@ -5,9 +5,7 @@ sudo: required
 
 matrix:
   include:
-    - rust: 1.36.0
-    # cfg(doctest) is experimental in 1.39 but ignored with 1.34.0, and that snuck in when 1.39.0 wasn't tested
-    - rust: 1.39.0
+    - rust: 1.47.0
     - rust: stable
     - rust: beta
     - rust: nightly
@@ -37,7 +35,8 @@ script:
   - cargo build --all-targets
   - cargo build --verbose $TARGET --no-default-features 
   - cargo build --verbose $TARGET $FEATURES
-  - 'if [[ -z "$TARGET" ]]; then cargo test --verbose; fi'
+  # tests are cpu intensive enough that running them in release mode is much faster
+  - 'if [[ -z "$TARGET" ]]; then cargo test --release --verbose; fi'
   - 'if [[ -z "$TARGET" ]]; then cargo doc --verbose; fi'
   # run for just a second to confirm that it can build and run ok
   - 'if [[ "$TRAVIS_RUST_VERSION" = nightly ]]; then cargo fuzz list | xargs -L 1 -I FUZZER cargo fuzz run FUZZER -- -max_total_time=1; fi'

Cargo.toml

@@ -16,10 +16,12 @@ name = "benchmarks"
 harness = false
 
 [dev-dependencies]
-# 0.3.3 requires rust 1.36.0 for stable copied()
 criterion = "0.3.4"
 rand = "0.6.1"
 structopt = "0.3.21"
+# test fixtures for engine tests
+rstest = "0.6.4"
+rstest_reuse = "0.1.1"
 
 [features]
 default = ["std"]

README.md

@@ -9,7 +9,7 @@ Made with CLion. Thanks to JetBrains for supporting open source!
 
 It's base64. What more could anyone want?
 
-This library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at multiple levels of abstraction so you can choose the level of convenience vs performance that you want, e.g. `decode_config_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input), whereas `decode_config` allocates a new `Vec<u8>` and returns it, which might be more convenient in some cases, but is slower (although still fast enough for almost any purpose) at 2.1 GiB/s.
+This library's goals are to be *correct* and *fast*. It's thoroughly tested and widely used. It exposes functionality at multiple levels of abstraction so you can choose the level of convenience vs performance that you want, e.g. `decode_engine_slice` decodes into an existing `&mut [u8]` and is pretty fast (2.6GiB/s for a 3 KiB input), whereas `decode_engine` allocates a new `Vec<u8>` and returns it, which might be more convenient in some cases, but is slower (although still fast enough for almost any purpose) at 2.1 GiB/s.
 
 Example
 ---
@@ -33,7 +33,7 @@ See the [docs](https://docs.rs/base64) for all the details.
 Rust version compatibility
 ---
 
-The minimum required Rust version is 1.36.0.
+The minimum required Rust version is 1.47.0.
 
 # Contributing
 
@@ -50,12 +50,6 @@ Benchmarks are in `benches/`. Running them requires nightly rust, but `rustup` m
 rustup run nightly cargo bench
 ```
 
-Decoding is aided by some pre-calculated tables, which are generated by:
-
-```bash
-cargo run --example make_tables > src/tables.rs.tmp && mv src/tables.rs.tmp src/tables.rs
-```
-
 no_std
 ---
 

RELEASE-NOTES.md

@@ -1,6 +1,13 @@
-# 0.14.0
-
-- MSRV is now 1.36.0
+# 0.20.0
+
+- Extended the `Config` concept into the `Engine` abstraction, allowing the user to pick different encoding / decoding implementations.
+  - What was formerly the only algorithm is now the `FastPortable` engine, so named because it's portable (works on any CPU) and relatively fast.
+  - This opens the door to a portable constant-time implementation ([#153](https://github.com/marshallpierce/rust-base64/pull/153), presumably `ConstantTimePortable`?) for security-sensitive applications that need side-channel resistance, and CPU-specific SIMD implementations for  more speed.
+  - Standard base64 per the RFC is available via `DEFAULT_ENGINE`. To use different alphabets or other settings (padding, etc), create your own engine instance.
+- `CharacterSet` is now `Alphabet` (per the RFC), and allows creating custom alphabets. The corresponding tables that were previously code-generated are now built dynamically.
+- Since there are already multiple breaking changes, various functions are renamed to be more consistent and discoverable
+- DecoderReader now owns its delegate reader
+- MSRV is now 1.47.0
 
 # 0.13.0
 

benches/benchmarks.rs

@@ -5,16 +5,15 @@ extern crate rand;
 
 use base64::display;
 use base64::{
-    decode, decode_config_buf, decode_config_slice, encode, encode_config_buf, encode_config_slice,
-    write, Config,
+    decode, decode_engine_slice, decode_engine_vec, encode, encode_engine_slice,
+    encode_engine_string, write,
 };
 
-use criterion::{black_box, Bencher, Criterion, Throughput, BenchmarkId};
+use base64::engine::DEFAULT_ENGINE;
+use criterion::{black_box, Bencher, BenchmarkId, Criterion, Throughput};
 use rand::{FromEntropy, Rng};
 use std::io::{self, Read, Write};
 
-const TEST_CONFIG: Config = base64::STANDARD;
-
 fn do_decode_bench(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size * 3 / 4);
     fill(&mut v);
@@ -33,7 +32,7 @@ fn do_decode_bench_reuse_buf(b: &mut Bencher, &size: &usize) {
 
     let mut buf = Vec::new();
     b.iter(|| {
-        decode_config_buf(&encoded, TEST_CONFIG, &mut buf).unwrap();
+        decode_engine_vec(&encoded, &mut buf, &DEFAULT_ENGINE).unwrap();
         black_box(&buf);
         buf.clear();
     });
@@ -47,7 +46,7 @@ fn do_decode_bench_slice(b: &mut Bencher, &size: &usize) {
     let mut buf = Vec::new();
     buf.resize(size, 0);
     b.iter(|| {
-        decode_config_slice(&encoded, TEST_CONFIG, &mut buf).unwrap();
+        decode_engine_slice(&encoded, &mut buf, &DEFAULT_ENGINE).unwrap();
         black_box(&buf);
     });
 }
@@ -63,7 +62,7 @@ fn do_decode_bench_stream(b: &mut Bencher, &size: &usize) {
 
     b.iter(|| {
         let mut cursor = io::Cursor::new(&encoded[..]);
-        let mut decoder = base64::read::DecoderReader::new(&mut cursor, TEST_CONFIG);
+        let mut decoder = base64::read::DecoderReader::from(&mut cursor, &DEFAULT_ENGINE);
         decoder.read_to_end(&mut buf).unwrap();
         buf.clear();
         black_box(&buf);
@@ -83,7 +82,7 @@ fn do_encode_bench_display(b: &mut Bencher, &size: &usize) {
     let mut v: Vec<u8> = Vec::with_capacity(size);
     fill(&mut v);
     b.iter(|| {
-        let e = format!("{}", display::Base64Display::with_config(&v, TEST_CONFIG));
+        let e = format!("{}", display::Base64Display::from(&v, &DEFAULT_ENGINE));
         black_box(&e);
     });
 }
@@ -93,7 +92,7 @@ fn do_encode_bench_reuse_buf(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
     let mut buf = String::new();
     b.iter(|| {
-        encode_config_buf(&v, TEST_CONFIG, &mut buf);
+        encode_engine_string(&v, &mut buf, &DEFAULT_ENGINE);
         buf.clear();
     });
 }
@@ -105,7 +104,7 @@ fn do_encode_bench_slice(b: &mut Bencher, &size: &usize) {
     // conservative estimate of encoded size
     buf.resize(v.len() * 2, 0);
     b.iter(|| {
-        encode_config_slice(&v, TEST_CONFIG, &mut buf);
+        encode_engine_slice(&v, &mut buf, &DEFAULT_ENGINE);
     });
 }
 
@@ -117,7 +116,7 @@ fn do_encode_bench_stream(b: &mut Bencher, &size: &usize) {
     buf.reserve(size * 2);
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderWriter::new(&mut buf, TEST_CONFIG);
+        let mut stream_enc = write::EncoderWriter::from(&mut buf, &DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
     });
@@ -128,7 +127,7 @@ fn do_encode_bench_string_stream(b: &mut Bencher, &size: &usize) {
     fill(&mut v);
 
     b.iter(|| {
-        let mut stream_enc = write::EncoderStringWriter::new(TEST_CONFIG);
+        let mut stream_enc = write::EncoderStringWriter::from(&DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();
@@ -142,7 +141,7 @@ fn do_encode_bench_string_reuse_buf_stream(b: &mut Bencher, &size: &usize) {
     let mut buf = String::new();
     b.iter(|| {
         buf.clear();
-        let mut stream_enc = write::EncoderStringWriter::from(&mut buf, TEST_CONFIG);
+        let mut stream_enc = write::EncoderStringWriter::from_consumer(&mut buf, &DEFAULT_ENGINE);
         stream_enc.write_all(&v).unwrap();
         stream_enc.flush().unwrap();
         let _ = stream_enc.into_inner();
@@ -174,11 +173,31 @@ fn encode_benchmarks(c: &mut Criterion, label: &str, byte_sizes: &[usize]) {
         group
             .throughput(Throughput::Bytes(*size as u64))
             .bench_with_input(BenchmarkId::new("encode", size), size, do_encode_bench)
-            .bench_with_input(BenchmarkId::new("encode_display", size), size, do_encode_bench_display)
-            .bench_with_input(BenchmarkId::new("encode_reuse_buf", size), size, do_encode_bench_reuse_buf)
-            .bench_with_input(BenchmarkId::new("encode_slice", size), size, do_encode_bench_slice)
-            .bench_with_input(BenchmarkId::new("encode_reuse_buf_stream", size), size, do_encode_bench_stream)
-            .bench_with_input(BenchmarkId::new("encode_string_stream", size), size, do_encode_bench_string_stream)
+            .bench_with_input(
+                BenchmarkId::new("encode_display", size),
+                size,
+                do_encode_bench_display,
+            )
+            .bench_with_input(
+                BenchmarkId::new("encode_reuse_buf", size),
+                size,
+                do_encode_bench_reuse_buf,
+            )
+            .bench_with_input(
+                BenchmarkId::new("encode_slice", size),
+                size,
+                do_encode_bench_slice,
+            )
+            .bench_with_input(
+                BenchmarkId::new("encode_reuse_buf_stream", size),
+                size,
+                do_encode_bench_stream,
+            )
+            .bench_with_input(
+                BenchmarkId::new("encode_string_stream", size),
+                size,
+                do_encode_bench_string_stream,
+            )
             .bench_with_input(
                 BenchmarkId::new("encode_string_reuse_buf_stream", size),
                 size,
@@ -198,9 +217,21 @@ fn decode_benchmarks(c: &mut Criterion, label: &str, byte_sizes: &[usize]) {
             .measurement_time(std::time::Duration::from_secs(3))
             .throughput(Throughput::Bytes(*size as u64))
             .bench_with_input(BenchmarkId::new("decode", size), size, do_decode_bench)
-            .bench_with_input(BenchmarkId::new("decode_reuse_buf", size), size, do_decode_bench_reuse_buf)
-            .bench_with_input(BenchmarkId::new("decode_slice", size), size, do_decode_bench_slice)
-            .bench_with_input(BenchmarkId::new("decode_stream", size), size, do_decode_bench_stream);
+            .bench_with_input(
+                BenchmarkId::new("decode_reuse_buf", size),
+                size,
+                do_decode_bench_reuse_buf,
+            )
+            .bench_with_input(
+                BenchmarkId::new("decode_slice", size),
+                size,
+                do_decode_bench_slice,
+            )
+            .bench_with_input(
+                BenchmarkId::new("decode_stream", size),
+                size,
+                do_decode_bench_stream,
+            );
     }
 
     group.finish();

examples/base64.rs

@@ -4,37 +4,28 @@ use std::path::PathBuf;
 use std::process;
 use std::str::FromStr;
 
-use base64::{read, write};
+use base64::{alphabet, engine, read, write};
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
-enum CharacterSet {
+enum Alphabet {
     Standard,
     UrlSafe,
 }
 
-impl Default for CharacterSet {
+impl Default for Alphabet {
     fn default() -> Self {
-        CharacterSet::Standard
+        Alphabet::Standard
     }
 }
 
-impl Into<base64::Config> for CharacterSet {
-    fn into(self) -> base64::Config {
-        match self {
-            CharacterSet::Standard => base64::STANDARD,
-            CharacterSet::UrlSafe => base64::URL_SAFE,
-        }
-    }
-}
-
-impl FromStr for CharacterSet {
+impl FromStr for Alphabet {
     type Err = String;
-    fn from_str(s: &str) -> Result<CharacterSet, String> {
+    fn from_str(s: &str) -> Result<Alphabet, String> {
         match s {
-            "standard" => Ok(CharacterSet::Standard),
-            "urlsafe" => Ok(CharacterSet::UrlSafe),
-            _ => Err(format!("charset '{}' unrecognized", s)),
+            "standard" => Ok(Alphabet::Standard),
+            "urlsafe" => Ok(Alphabet::UrlSafe),
+            _ => Err(format!("alphabet '{}' unrecognized", s)),
         }
     }
 }
@@ -45,10 +36,10 @@ struct Opt {
     /// decode data
     #[structopt(short = "d", long = "decode")]
     decode: bool,
-    /// The character set to choose. Defaults to the standard base64 character set.
-    /// Supported character sets include "standard" and "urlsafe".
-    #[structopt(long = "charset")]
-    charset: Option<CharacterSet>,
+    /// The alphabet to choose. Defaults to the standard base64 alphabet.
+    /// Supported alphabets include "standard" and "urlsafe".
+    #[structopt(long = "alphabet")]
+    alphabet: Option<Alphabet>,
     /// The file to encode/decode.
     #[structopt(parse(from_os_str))]
     file: Option<PathBuf>,
@@ -68,14 +59,23 @@ fn main() {
         }
         Some(f) => Box::new(File::open(f).unwrap()),
     };
-    let config = opt.charset.unwrap_or_default().into();
+
+    let alphabet = opt.alphabet.unwrap_or_default();
+    let engine = engine::fast_portable::FastPortable::from(
+        &match alphabet {
+            Alphabet::Standard => alphabet::STANDARD,
+            Alphabet::UrlSafe => alphabet::URL_SAFE,
+        },
+        engine::fast_portable::PAD,
+    );
+
     let stdout = io::stdout();
     let mut stdout = stdout.lock();
     let r = if opt.decode {
-        let mut decoder = read::DecoderReader::new(&mut input, config);
+        let mut decoder = read::DecoderReader::from(&mut input, &engine);
         io::copy(&mut decoder, &mut stdout)
     } else {
-        let mut encoder = write::EncoderWriter::new(&mut stdout, config);
+        let mut encoder = write::EncoderWriter::from(&mut stdout, &engine);
         io::copy(&mut input, &mut encoder)
     };
     if let Err(e) = r {