Remove get_metadata from AsyncFileReader

Use file APIs directly

Author: kylebarron

src/cog.rs

@@ -1,8 +1,4 @@
-use std::sync::Arc;
-
-use crate::error::AsyncTiffResult;
 use crate::ifd::ImageFileDirectory;
-use crate::reader::AsyncFileReader;
 
 /// A TIFF file.
 #[derive(Debug, Clone)]
@@ -16,14 +12,6 @@ impl TIFF {
         Self { ifds }
     }
 
-    /// Open a new TIFF file.
-    ///
-    /// This will read all the Image File Directories (IFDs) in the file.
-    pub async fn try_open(reader: Arc<dyn AsyncFileReader>) -> AsyncTiffResult<Self> {
-        let ifds = reader.get_metadata().await?;
-        Ok(Self { ifds })
-    }
-
     /// Access the underlying Image File Directories.
     pub fn ifds(&self) -> &[ImageFileDirectory] {
         &self.ifds
@@ -36,7 +24,8 @@ mod test {
     use std::sync::Arc;
 
     use crate::decoder::DecoderRegistry;
-    use crate::reader::ObjectReader;
+    use crate::metadata::{PrefetchMetadataFetch, TiffMetadataReader};
+    use crate::reader::{AsyncFileReader, ObjectReader};
 
     use super::*;
     use object_store::local::LocalFileSystem;
@@ -48,11 +37,20 @@ mod test {
         let folder = "/Users/kyle/github/developmentseed/async-tiff/";
         let path = object_store::path::Path::parse("m_4007307_sw_18_060_20220803.tif").unwrap();
         let store = Arc::new(LocalFileSystem::new_with_prefix(folder).unwrap());
-        let reader = Arc::new(ObjectReader::new(store, path));
-
-        let cog_reader = TIFF::try_open(reader.clone()).await.unwrap();
+        let reader = Arc::new(ObjectReader::new(store, path)) as Arc<dyn AsyncFileReader>;
+        let prefetch_reader = PrefetchMetadataFetch::new(reader.clone(), 32 * 1024)
+            .await
+            .unwrap();
+        let mut metadata_reader = TiffMetadataReader::try_open(&prefetch_reader)
+            .await
+            .unwrap();
+        let ifds = metadata_reader
+            .read_all_ifds(&prefetch_reader)
+            .await
+            .unwrap();
+        let tiff = TIFF::new(ifds);
 
-        let ifd = &cog_reader.ifds[1];
+        let ifd = &tiff.ifds[1];
         let decoder_registry = DecoderRegistry::default();
         let tile = ifd.fetch_tile(0, 0, reader.as_ref()).await.unwrap();
         let tile = tile.decode(&decoder_registry).unwrap();

src/metadata/mod.rs

@@ -1,4 +1,60 @@
-//! TIFF metadata API
+//! API for reading metadata out of a TIFF file.
+//!
+//! ### Reading all TIFF metadata
+//!
+//! We can use [`TiffMetadataReader::read_all_ifds`] to read all IFDs up front:
+//!
+//! ```
+//! use std::env::current_dir;
+//! use std::sync::Arc;
+//!
+//! use object_store::local::LocalFileSystem;
+//!
+//! use async_tiff::metadata::{PrefetchMetadataFetch, TiffMetadataReader};
+//! use async_tiff::reader::ObjectReader;
+//!
+//! // Create new Arc<dyn ObjectStore>
+//! let store = Arc::new(LocalFileSystem::new_with_prefix(current_dir().unwrap()).unwrap());
+//!
+//! // Create new ObjectReader to map the ObjectStore to the AsyncFileReader trait
+//! let reader = ObjectReader::new(
+//!     store,
+//!     "tests/image_tiff/images/tiled-jpeg-rgb-u8.tif".into(),
+//! );
+//!
+//! // Use PrefetchMetadataFetch to ensure that a given number of bytes at the start of the
+//! // file are prefetched.
+//! //
+//! // This or a similar caching layer should **always** be used and ensures that the
+//! // underlying read calls that the TiffMetadataReader makes don't translate to actual
+//! // network fetches.
+//! let prefetch_reader = PrefetchMetadataFetch::new(reader.clone(), 32 * 1024)
+//!     .await
+//!     .unwrap();
+//!
+//! // Create a TiffMetadataReader wrapping some MetadataFetch
+//! let mut metadata_reader = TiffMetadataReader::try_open(&prefetch_reader)
+//!     .await
+//!     .unwrap();
+//!
+//! // Read all IFDs out of the source.
+//! let ifds = metadata_reader
+//!     .read_all_ifds(&prefetch_reader)
+//!     .await
+//!     .unwrap();
+//! ```
+//!
+//!
+//! ### Caching/prefetching/buffering
+//!
+//! The underlying [`ImageFileDirectoryReader`] used to read tags out of the TIFF file reads each
+//! tag individually. This means that it will make many small byte range requests to the
+//! [`MetadataFetch`] implementation.
+//!
+//! Thus, it is **imperative to always supply some sort of caching, prefetching, or buffering**
+//! middleware when reading metadata. [`PrefetchMetadataFetch`] is an example of this, which
+//! fetches the first `N` bytes out of a file.
+//!
 
 mod fetch;
 mod reader;

src/metadata/reader.rs

@@ -11,7 +11,15 @@ use crate::tiff::tags::{Tag, Type};
 use crate::tiff::{TiffError, TiffFormatError, Value};
 use crate::ImageFileDirectory;
 
-/// Read TIFF metadata from an async source.
+/// Entry point to reading TIFF metadata.
+///
+/// This is a stateful reader because we don't know how many IFDs will be encountered.
+///
+/// ```notest
+/// // fetch implements MetadataFetch
+/// let mut metadata_reader = TiffMetadataReader::try_open(&fetch).await?;
+/// let ifds = metadata_reader.read_all_ifds(&fetch).await?;
+/// ```
 pub struct TiffMetadataReader {
     endianness: Endianness,
     bigtiff: bool,
@@ -189,9 +197,9 @@ impl ImageFileDirectoryReader {
         })
     }
 
-    /// Manually read the tag with the specified idx.
+    /// Manually read the tag with the specified index.
     ///
-    /// If there are no more tags, returns `None`.
+    /// Panics if the tag index is out of range of the tag count.
     ///
     /// This can be useful if you need to access tags at a low level. You'll need to call
     /// [`ImageFileDirectory::from_tags`] on the resulting collection of tags.

src/reader.rs

@@ -12,8 +12,6 @@ use futures::future::{BoxFuture, FutureExt};
 use futures::TryFutureExt;
 
 use crate::error::AsyncTiffResult;
-use crate::metadata::TiffMetadataReader;
-use crate::ImageFileDirectory;
 
 /// The asynchronous interface used to read COG files
 ///
@@ -32,9 +30,6 @@ use crate::ImageFileDirectory;
 ///
 /// [`tokio::fs::File`]: https://docs.rs/tokio/latest/tokio/fs/struct.File.html
 pub trait AsyncFileReader: Debug + Send + Sync {
-    /// Retrieve the bytes in `range` as part of a request for header metadata.
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>>;
-
     /// Retrieve the bytes in `range` as part of a request for image data, not header metadata.
     ///
     /// This is also used as the default implementation of [`MetadataFetch`] if not overridden.
@@ -62,10 +57,6 @@ pub trait AsyncFileReader: Debug + Send + Sync {
 
 /// This allows Box<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
 impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>> {
-        self.as_ref().get_metadata()
-    }
-
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         self.as_ref().get_bytes(range)
     }
@@ -80,10 +71,6 @@ impl AsyncFileReader for Box<dyn AsyncFileReader + '_> {
 
 /// This allows Arc<dyn AsyncFileReader + '_> to be used as an AsyncFileReader,
 impl AsyncFileReader for Arc<dyn AsyncFileReader + '_> {
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>> {
-        self.as_ref().get_metadata()
-    }
-
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         self.as_ref().get_bytes(range)
     }
@@ -144,15 +131,6 @@ impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> Toki
 impl<T: tokio::io::AsyncRead + tokio::io::AsyncSeek + Unpin + Send + Debug> AsyncFileReader
     for TokioReader<T>
 {
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>> {
-        async move {
-            let mut tiff_metadata_reader = TiffMetadataReader::try_open(self).await?;
-            let ifds = tiff_metadata_reader.read_all_ifds(self).await?;
-            Ok(ifds)
-        }
-        .boxed()
-    }
-
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         self.make_range_request(range).boxed()
     }
@@ -186,15 +164,6 @@ impl ObjectReader {
 
 #[cfg(feature = "object_store")]
 impl AsyncFileReader for ObjectReader {
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>> {
-        async move {
-            let mut tiff_metadata_reader = TiffMetadataReader::try_open(self).await?;
-            let ifds = tiff_metadata_reader.read_all_ifds(self).await?;
-            Ok(ifds)
-        }
-        .boxed()
-    }
-
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         self.make_range_request(range).boxed()
     }
@@ -253,15 +222,6 @@ impl ReqwestReader {
 
 #[cfg(feature = "reqwest")]
 impl AsyncFileReader for ReqwestReader {
-    fn get_metadata(&self) -> BoxFuture<'_, AsyncTiffResult<Vec<ImageFileDirectory>>> {
-        async move {
-            let mut tiff_metadata_reader = TiffMetadataReader::try_open(self).await?;
-            let ifds = tiff_metadata_reader.read_all_ifds(self).await?;
-            Ok(ifds)
-        }
-        .boxed()
-    }
-
     fn get_bytes(&self, range: Range<u64>) -> BoxFuture<'_, AsyncTiffResult<Bytes>> {
         self.make_range_request(range)
     }

tests/image_tiff/util.rs

@@ -1,7 +1,8 @@
 use std::env::current_dir;
 use std::sync::Arc;
 
-use async_tiff::reader::ObjectReader;
+use async_tiff::metadata::TiffMetadataReader;
+use async_tiff::reader::{AsyncFileReader, ObjectReader};
 use async_tiff::TIFF;
 use object_store::local::LocalFileSystem;
 
@@ -10,6 +11,9 @@ const TEST_IMAGE_DIR: &str = "tests/image_tiff/images/";
 pub(crate) async fn open_tiff(filename: &str) -> TIFF {
     let store = Arc::new(LocalFileSystem::new_with_prefix(current_dir().unwrap()).unwrap());
     let path = format!("{TEST_IMAGE_DIR}/{filename}");
-    let reader = Arc::new(ObjectReader::new(store.clone(), path.as_str().into()));
-    TIFF::try_open(reader).await.unwrap()
+    let reader = Arc::new(ObjectReader::new(store.clone(), path.as_str().into()))
+        as Arc<dyn AsyncFileReader>;
+    let mut metadata_reader = TiffMetadataReader::try_open(&reader).await.unwrap();
+    let ifds = metadata_reader.read_all_ifds(&reader).await.unwrap();
+    TIFF::new(ifds)
 }