Improved docs (#6)

Author: jorgecarleitao

src/error.rs

@@ -1,14 +1,22 @@
+//! Contains [`Error`]
 use crate::proto::stream::Kind;
 
+/// Possible errors from this crate.
 #[derive(Debug, Clone)]
 pub enum Error {
+    /// Generic error returned when the file is out of spec
     OutOfSpec,
-    RleLiteralTooLarge,
+    /// When a string column contains a value with invalid UTF8
     InvalidUtf8,
+    /// When the user requests a column that does not exist
     InvalidColumn(u32),
+    /// When the user requests a type that does not exist for the given column
     InvalidKind(u32, Kind),
+    /// When decoding a float fails
     DecodeFloat,
+    /// When decompression fails
     Decompression,
+    /// When decoding the proto files fail
     InvalidProto,
 }
 

src/lib.md

@@ -0,0 +1,56 @@
+Welcome to `orc-format` documentation. Thanks for checking it out!
+
+This Rust crate is a toolkit to read and deserialize ORC to your favourite in-memory format.
+
+Below is an example of how to read a column from ORC into memory:
+
+```rust
+use std::fs::File;
+
+use orc_format::{error::Error, read, read::Column};
+
+
+fn get_stripe(path: &str, column: u32) -> Result<Column, Error> {
+    // open the file, as expected. buffering this is not necessary - we
+    // are very careful about the number of `read`s we perform.
+    let mut f = File::open(path).expect("no file found");
+
+    // read the files' metadata
+    let metadata = read::read_metadata(&mut f)?;
+    // and copy the compression it is using
+    let compression = metadata.postscript.compression();
+
+    // the next step is to identify which stripe we want to read. Let's say it is the first one.
+    let stripe = &metadata.footer.stripes[0];
+
+    // Each stripe has a footer - we need to read it to extract the location of each column on it.
+    let stripe_footer = read::read_stripe_footer(&mut f, stripe, compression, &mut vec![])?;
+
+    // Finally, we read the column into `Column`
+    read::read_stripe_column(&mut f, stripe, stripe_footer, compression, column, vec![])
+}
+```
+
+To deserialize the values of a column, use things inside `read::decode`.
+For example, the below contains the deserialization of the "Present" to a `Vec<bool>`.
+
+```rust
+use orc_format::{error::Error, proto::stream::Kind, read::decode::BooleanIter, read::Column};
+
+fn deserialize_present(column: &Column, scratch: &mut Vec<u8>) -> Result<Vec<bool>, Error> {
+    let mut reader = column.get_stream(Kind::Present, std::mem::take(scratch))?;
+
+    let mut validity = Vec::with_capacity(column.number_of_rows());
+    BooleanIter::new(&mut reader, column.number_of_rows()).try_for_each(|item| {
+        validity.push(item?);
+        Result::<(), Error>::Ok(())
+    })?;
+
+    *scratch = std::mem::take(&mut reader.into_inner());
+
+    Ok(validity)
+}
+```
+
+Check out the integration tests of the crate to find deserialization of other types such
+as floats, integers, strings and dictionaries.

src/lib.rs

@@ -1,3 +1,5 @@
+#![doc = include_str!("lib.md")]
+#![forbid(unsafe_code)]
 pub mod error;
 pub mod proto;
 pub mod read;

src/read/decode/float.rs

@@ -38,6 +38,7 @@ pub struct FloatIter<'a, T: Float, R: std::io::Read> {
 }
 
 impl<'a, T: Float, R: std::io::Read> FloatIter<'a, T, R> {
+    /// Returns a new [`FloatIter`]
     #[inline]
     pub fn new(reader: &'a mut R, length: usize) -> Self {
         Self {
@@ -47,11 +48,13 @@ impl<'a, T: Float, R: std::io::Read> FloatIter<'a, T, R> {
         }
     }
 
+    /// The number of items remaining
     #[inline]
     pub fn len(&self) -> usize {
         self.remaining
     }
 
+    /// Whether the iterator is empty
     #[must_use]
     pub fn is_empty(&self) -> bool {
         self.len() == 0

src/read/decode/mod.rs

@@ -1,11 +1,12 @@
+//! Contains different iterators that receive a reader ([`std::io::Read`])
+//! and return values for each of ORC's physical types (e.g. boolean).
 mod boolean_rle;
 mod float;
 mod rle_v2;
 mod variable_length;
 
 pub use boolean_rle::{BooleanIter, BooleanRleRunIter, BooleanRun};
 pub use float::FloatIter;
-pub use rle_v2::IteratorEnum;
 pub use rle_v2::{SignedRleV2Iter, SignedRleV2Run, UnsignedRleV2Iter, UnsignedRleV2Run};
 pub use variable_length::Values;
 

src/read/decode/rle_v2.rs

@@ -413,12 +413,6 @@ impl Iterator for SignedDeltaRun {
     }
 }
 
-pub enum IteratorEnum<I, II, III> {
-    Direct(I),
-    Delta(II),
-    ShortRepeat(III),
-}
-
 #[inline]
 fn run_encoding(header: u8) -> EncodingTypeV2 {
     match (header & 128 == 128, header & 64 == 64) {
@@ -433,13 +427,18 @@ fn run_encoding(header: u8) -> EncodingTypeV2 {
     }
 }
 
+/// An enum describing one of the RLE v2 runs for unsigned integers
 pub enum UnsignedRleV2Run {
+    /// Direct
     Direct(UnsignedDirectRun),
+    /// Delta
     Delta(UnsignedDeltaRun),
+    /// Short repeat
     ShortRepeat(UnsignedShortRepeat),
 }
 
 impl UnsignedRleV2Run {
+    /// Returns a new [`UnsignedRleV2Run`] owning `scratch`.
     pub fn try_new<R: Read>(reader: &mut R, scratch: Vec<u8>) -> Result<Self, Error> {
         let mut header = [0u8];
         reader.read_exact(&mut header)?;
@@ -460,6 +459,7 @@ impl UnsignedRleV2Run {
         }
     }
 
+    /// The number of items remaining
     pub fn len(&self) -> usize {
         match self {
             Self::Direct(run) => run.len(),
@@ -468,19 +468,22 @@ impl UnsignedRleV2Run {
         }
     }
 
+    /// Whether the iterator is empty
     #[must_use]
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 }
 
+/// A fallible [`Iterator`] of [`UnsignedRleV2Run`].
 pub struct UnsignedRleV2Iter<'a, R: Read> {
     reader: &'a mut R,
     scratch: Vec<u8>,
     length: usize,
 }
 
 impl<'a, R: Read> UnsignedRleV2Iter<'a, R> {
+    /// Returns a new [`UnsignedRleV2Iter`].
     pub fn new(reader: &'a mut R, length: usize, scratch: Vec<u8>) -> Self {
         Self {
             reader,
@@ -514,6 +517,7 @@ impl SignedDirectRun {
         self.0.len()
     }
 
+    /// Whether the iterator is empty
     #[must_use]
     pub fn is_empty(&self) -> bool {
         self.len() == 0
@@ -540,10 +544,12 @@ impl SignedShortRepeat {
         UnsignedShortRepeat::try_new(header, reader, scratch).map(Self)
     }
 
+    /// The number of items remaining
     pub fn len(&self) -> usize {
         self.0.len()
     }
 
+    /// Whether the iterator is empty
     #[must_use]
     pub fn is_empty(&self) -> bool {
         self.len() == 0
@@ -562,14 +568,19 @@ impl Iterator for SignedShortRepeat {
     }
 }
 
+/// An enum describing one of the RLE v2 runs for signed integers
 #[derive(Debug)]
 pub enum SignedRleV2Run {
+    /// Direct
     Direct(SignedDirectRun),
+    /// Delta
     Delta(SignedDeltaRun),
+    /// Short repeat
     ShortRepeat(SignedShortRepeat),
 }
 
 impl SignedRleV2Run {
+    /// Returns a new [`SignedRleV2Run`], moving `scratch` to itself
     pub fn try_new<R: Read>(reader: &mut R, scratch: Vec<u8>) -> Result<Self, Error> {
         let mut header = [0u8];
         reader.read_exact(&mut header)?;
@@ -590,6 +601,7 @@ impl SignedRleV2Run {
         }
     }
 
+    /// The number of items remaining
     pub fn len(&self) -> usize {
         match self {
             Self::Direct(run) => run.len(),
@@ -598,19 +610,22 @@ impl SignedRleV2Run {
         }
     }
 
+    /// Whether the iterator is empty
     #[must_use]
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 }
 
+/// A fallible [`Iterator`] of [`SignedRleV2Run`].
 pub struct SignedRleV2Iter<R: Read> {
     reader: R,
     scratch: Vec<u8>,
     length: usize,
 }
 
 impl<R: Read> SignedRleV2Iter<R> {
+    /// Returns a new [`SignedRleV2Iter`].
     pub fn new(reader: R, length: usize, scratch: Vec<u8>) -> Self {
         Self {
             reader,

src/read/decompress/mod.rs

@@ -1,3 +1,4 @@
+//! Contains [`Decompressor`]
 use std::io::Read;
 
 use fallible_streaming_iterator::FallibleStreamingIterator;
@@ -91,13 +92,15 @@ impl<'a> FallibleStreamingIterator for DecompressorIter<'a> {
     }
 }
 
+/// A [`Read`]er fulfilling the ORC specification of reading compressed data.
 pub struct Decompressor<'a> {
     decompressor: DecompressorIter<'a>,
     offset: usize,
     is_first: bool,
 }
 
 impl<'a> Decompressor<'a> {
+    /// Creates a new [`Decompressor`] that will use `scratch` as a temporary region.
     pub fn new(stream: &'a [u8], compression: CompressionKind, scratch: Vec<u8>) -> Self {
         Self {
             decompressor: DecompressorIter::new(stream, compression, scratch),
@@ -106,6 +109,7 @@ impl<'a> Decompressor<'a> {
         }
     }
 
+    /// Returns the internal memory region, so it can be re-used
     pub fn into_inner(self) -> Vec<u8> {
         self.decompressor.into_inner()
     }

src/read/mod.rs

@@ -5,7 +5,6 @@
 //! 2. Read the stripe (or part of it in projection pushdown)
 //! 3. For each column, select the relevant region of the stripe
 //! 4. Attach an Iterator to the region
-#![forbid(unsafe_code)]
 
 use std::io::{Read, Seek, SeekFrom};
 

src/read/stripe.rs

@@ -54,10 +54,12 @@ impl Column {
             .ok_or(Error::InvalidKind(self.column, kind))
     }
 
+    /// Returns the encoding of the column
     pub fn encoding(&self) -> &ColumnEncoding {
         &self.footer.columns[self.column as usize]
     }
 
+    /// Returns the number of items in the dictionary, if any
     pub fn dictionary_size(&self) -> Option<usize> {
         self.footer.columns[self.column as usize]
             .dictionary_size