Feature gate object_store and reqwest (#71)

* pulled `coalesce_ranges` conditionally in-tree, feature-gated object_store and reqwest

* undid coalesce_ranges addition

* Update src/error.rs

Co-authored-by: Kyle Barron <[email protected]>

* Update Cargo.toml

Co-authored-by: Kyle Barron <[email protected]>

* added CI metadata to toml

* fixed errors/warnings for all features: Added std::fs::File AsyncFileReader impl

* added CI

* removed AsyncFileReader impl for std::fs::File, added impls for tokio::fs::File

* added required tokio features

* revert util test changes

* Add tokio wrapper

* Only run `check` for all feature flag combinations, not `test`

* remove wasm32 target

* Minimal tokio features

* add line to doc

---------

Co-authored-by: Kyle Barron <[email protected]>
Co-authored-by: Kyle Barron <[email protected]>

Author: feefladder

.github/workflows/test.yml

@@ -26,8 +26,10 @@ jobs:
       - name: "clippy --all"
         run: cargo clippy --all --all-features --tests -- -D warnings
 
-      - name: "cargo check"
-        run: cargo check --all --all-features
+      - run: cargo install cargo-all-features
+
+      - name: Check all combinations of features can build
+        run: cargo check-all-features
 
       - name: "cargo test"
         run: |

.gitignore

@@ -1,3 +1,4 @@
+.vscode
 *.tif
 *.buf
 

Cargo.toml

@@ -15,14 +15,29 @@ flate2 = "1.0.20"
 futures = "0.3.31"
 jpeg = { package = "jpeg-decoder", version = "0.3.0", default-features = false }
 num_enum = "0.7.3"
-object_store = "0.12"
-# In the future we could make this feature-flagged, but for now we depend on
-# object_store which uses reqwest.
-reqwest = { version = "0.12", default-features = false }
+object_store = { version = "0.12", optional = true }
+reqwest = { version = "0.12", default-features = false, optional = true }
 thiserror = "1"
-tokio = { version = "1.43.0", optional = true }
+tokio = { version = "1.43.0", optional = true, default-features = false, features = [
+    "io-util",
+    "sync",
+] }
 weezl = "0.1.0"
 
 [dev-dependencies]
+object_store = "0.12"
 tiff = "0.9.1"
-tokio = { version = "1.9", features = ["macros", "fs", "rt-multi-thread"] }
+tokio = { version = "1.9", features = [
+    "macros",
+    "fs",
+    "rt-multi-thread",
+    "io-util",
+] }
+
+[features]
+default = ["object_store", "reqwest"]
+tokio = ["dep:tokio"]
+reqwest = ["dep:reqwest"]
+object_store = ["dep:object_store"]
+
+[package.metadata.cargo-all-features]

src/error.rs

@@ -9,7 +9,7 @@ use thiserror::Error;
 pub enum AsyncTiffError {
     /// End of file error.
     #[error("End of File: expected to read {0} bytes, got {1}")]
-    EndOfFile(usize, usize),
+    EndOfFile(u64, u64),
 
     /// General error.
     #[error("General error: {0}")]
@@ -24,6 +24,7 @@ pub enum AsyncTiffError {
     JPEGDecodingError(#[from] jpeg::Error),
 
     /// Error while fetching data using object store.
+    #[cfg(feature = "object_store")]
     #[error(transparent)]
     ObjectStore(#[from] object_store::Error),
 
@@ -32,6 +33,7 @@ pub enum AsyncTiffError {
     InternalTIFFError(#[from] crate::tiff::TiffError),
 
     /// Reqwest error
+    #[cfg(feature = "reqwest")]
     #[error(transparent)]
     ReqwestError(#[from] reqwest::Error),
 

src/reader.rs

@@ -8,8 +8,8 @@ use std::sync::Arc;
 use byteorder::{BigEndian, LittleEndian, ReadBytesExt};
 use bytes::buf::Reader;
 use bytes::{Buf, Bytes};
-use futures::future::{BoxFuture, FutureExt, TryFutureExt};
-use object_store::ObjectStore;
+use futures::future::{BoxFuture, FutureExt};
+use futures::TryFutureExt;
 
 use crate::error::{AsyncTiffError, AsyncTiffResult};
 
@@ -67,45 +67,75 @@ impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
     }
 }
 
-// #[cfg(feature = "tokio")]
-// impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Debug + Send + Sync> AsyncFileReader
-//     for T
-// {
-//     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
-//         use tokio::io::{AsyncReadExt, AsyncSeekExt};
-
-//         async move {
-//             self.seek(std::io::SeekFrom::Start(range.start)).await?;
-
-//             let to_read = (range.end - range.start).try_into().unwrap();
-//             let mut buffer = Vec::with_capacity(to_read);
-//             let read = self.take(to_read as u64).read_to_end(&mut buffer).await?;
-//             if read != to_read {
-//                 return Err(AsyncTiffError::EndOfFile(to_read, read));
-//             }
-
-//             Ok(buffer.into())
-//         }
-//         .boxed()
-//     }
-// }
+/// A wrapper for things that implement [AsyncRead] and [AsyncSeek] to also implement
+/// [AsyncFileReader].
+///
+/// This wrapper is needed because `AsyncRead` and `AsyncSeek` require mutable access to seek and
+/// read data, while the `AsyncFileReader` trait requires immutable access to read data.
+///
+/// This wrapper stores the inner reader in a `Mutex`.
+///
+/// [AsyncRead]: tokio::io::AsyncRead
+/// [AsyncSeek]: tokio::io::AsyncSeek
+#[cfg(feature = "tokio")]
+#[derive(Debug)]
+pub struct TokioReader<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug>(
+    tokio::sync::Mutex<T>,
+);
+
+#[cfg(feature = "tokio")]
+impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> TokioReader<T> {
+    /// Create a new TokioReader from a reader.
+    pub fn new(inner: T) -> Self {
+        Self(tokio::sync::Mutex::new(inner))
+    }
+}
+
+#[cfg(feature = "tokio")]
+impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> AsyncFileReader
+    for TokioReader<T>
+{
+    fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
+        use std::io::SeekFrom;
+        use tokio::io::{AsyncReadExt, AsyncSeekExt};
+
+        async move {
+            let mut file = self.0.lock().await;
+
+            file.seek(SeekFrom::Start(range.start)).await?;
+
+            let to_read = range.end - range.start;
+            let mut buffer = Vec::with_capacity(to_read as usize);
+            let read = file.read(&mut buffer).await? as u64;
+            if read != to_read {
+                return Err(AsyncTiffError::EndOfFile(to_read, read));
+            }
+
+            Ok(buffer.into())
+        }
+        .boxed()
+    }
+}
 
 /// An AsyncFileReader that reads from an [`ObjectStore`] instance.
+#[cfg(feature = "object_store")]
 #[derive(Clone, Debug)]
 pub struct ObjectReader {
-    store: Arc<dyn ObjectStore>,
+    store: Arc<dyn object_store::ObjectStore>,
     path: object_store::path::Path,
 }
 
+#[cfg(feature = "object_store")]
 impl ObjectReader {
     /// Creates a new [`ObjectReader`] for the provided [`ObjectStore`] and path
     ///
     /// [`ObjectMeta`] can be obtained using [`ObjectStore::list`] or [`ObjectStore::head`]
-    pub fn new(store: Arc<dyn ObjectStore>, path: object_store::path::Path) -> Self {
+    pub fn new(store: Arc<dyn object_store::ObjectStore>, path: object_store::path::Path) -> Self {
         Self { store, path }
     }
 }
 
+#[cfg(feature = "object_store")]
 impl AsyncFileReader for ObjectReader {
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         let range = range.start as _..range.end as _;
@@ -134,19 +164,22 @@ impl AsyncFileReader for ObjectReader {
 }
 
 /// An AsyncFileReader that reads from a URL using reqwest.
+#[cfg(feature = "reqwest")]
 #[derive(Debug, Clone)]
 pub struct ReqwestReader {
     client: reqwest::Client,
     url: reqwest::Url,
 }
 
+#[cfg(feature = "reqwest")]
 impl ReqwestReader {
     /// Construct a new ReqwestReader from a reqwest client and URL.
     pub fn new(client: reqwest::Client, url: reqwest::Url) -> Self {
         Self { client, url }
     }
 }
 
+#[cfg(feature = "reqwest")]
 impl AsyncFileReader for ReqwestReader {
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         let url = self.url.clone();