Improved API and fixed edge case in delta runs (#7)

Author: jorgecarleitao

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "orc-format"
-version = "0.2.0"
+version = "0.3.0"
 license = "MIT/Apache-2.0"
 description = "Unofficial implementation of Apache ORC spec in safe Rust"
 homepage = "https://github.com/DataEngineeringLabs/orc-format"

src/lib.md

@@ -10,24 +10,22 @@ use std::fs::File;
 use orc_format::{error::Error, read, read::Column};
 
 
-fn get_stripe(path: &str, column: u32) -> Result<Column, Error> {
+fn get_column(path: &str, column: u32) -> Result<Column, Error> {
     // open the file, as expected. buffering this is not necessary - we
     // are very careful about the number of `read`s we perform.
     let mut f = File::open(path).expect("no file found");
 
     // read the files' metadata
     let metadata = read::read_metadata(&mut f)?;
-    // and copy the compression it is using
-    let compression = metadata.postscript.compression();
 
     // the next step is to identify which stripe we want to read. Let's say it is the first one.
-    let stripe = &metadata.footer.stripes[0];
+    let stripe = 0;
 
     // Each stripe has a footer - we need to read it to extract the location of each column on it.
-    let stripe_footer = read::read_stripe_footer(&mut f, stripe, compression, &mut vec![])?;
+    let stripe_footer = read::read_stripe_footer(&mut f, &metadata, stripe, &mut vec![])?;
 
     // Finally, we read the column into `Column`
-    read::read_stripe_column(&mut f, stripe, stripe_footer, compression, column, vec![])
+    read::read_stripe_column(&mut f, &metadata, stripe, stripe_footer, column, vec![])
 }
 ```
 

src/lib.rs

@@ -3,5 +3,3 @@
 pub mod error;
 pub mod proto;
 pub mod read;
-
-pub use fallible_streaming_iterator;

src/read/stripe.rs renamed to src/read/column.rs

@@ -71,9 +71,9 @@ impl Column {
         self.number_of_rows as usize
     }
 
-    /// Returns the underlying scratch containing a pre-allocated memory region
+    /// Returns the underlying footer and the pre-allocated memory region
     /// containing all (compressed) streams of this column.
-    pub fn into_inner(self) -> Vec<u8> {
-        self.data
+    pub fn into_inner(self) -> (StripeFooter, Vec<u8>) {
+        (self.footer, self.data)
     }
 }

src/read/decode/boolean_rle.rs

@@ -11,14 +11,18 @@ pub enum BooleanRun {
     Literals([u8; 255]),
 }
 
-pub struct BooleanRleRunIter<'a, R: Read> {
-    reader: &'a mut R,
+pub struct BooleanRleRunIter<R: Read> {
+    reader: R,
 }
 
-impl<'a, R: Read> BooleanRleRunIter<'a, R> {
-    pub fn new(reader: &'a mut R) -> Self {
+impl<R: Read> BooleanRleRunIter<R> {
+    pub fn new(reader: R) -> Self {
         Self { reader }
     }
+
+    pub fn into_inner(self) -> R {
+        self.reader
+    }
 }
 
 fn read_literals<R: Read>(reader: &mut R, header: i8) -> Result<[u8; 255], Error> {
@@ -33,22 +37,22 @@ fn read_literals<R: Read>(reader: &mut R, header: i8) -> Result<[u8; 255], Error
     Ok(literals)
 }
 
-impl<'a, R: Read> Iterator for BooleanRleRunIter<'a, R> {
+impl<R: Read> Iterator for BooleanRleRunIter<R> {
     type Item = Result<BooleanRun, Error>;
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
-        let header = read_u8(self.reader);
+        let header = read_u8(&mut self.reader);
         let header = match header {
             Ok(header) => header as i8,
             Err(e) => return Some(Err(e.into())),
         };
         if header < 0 {
-            Some(read_literals(self.reader, header).map(BooleanRun::Literals))
+            Some(read_literals(&mut self.reader, header).map(BooleanRun::Literals))
         } else {
             let length = header as u16 + 3;
             // this is not ok - it may require more than one byte
-            let value = read_u8(self.reader);
+            let value = read_u8(&mut self.reader);
             let value = match value {
                 Ok(value) => value,
                 Err(e) => return Some(Err(e.into())),
@@ -58,16 +62,16 @@ impl<'a, R: Read> Iterator for BooleanRleRunIter<'a, R> {
     }
 }
 
-pub struct BooleanIter<'a, R: Read> {
-    iter: BooleanRleRunIter<'a, R>,
+pub struct BooleanIter<R: Read> {
+    iter: BooleanRleRunIter<R>,
     current: Option<BooleanRun>,
     position: u8,
     byte_position: usize,
     remaining: usize,
 }
 
-impl<'a, R: Read> BooleanIter<'a, R> {
-    pub fn new(reader: &'a mut R, length: usize) -> Self {
+impl<'a, R: Read> BooleanIter<R> {
+    pub fn new(reader: R, length: usize) -> Self {
         Self {
             iter: BooleanRleRunIter::new(reader),
             current: None,
@@ -76,9 +80,13 @@ impl<'a, R: Read> BooleanIter<'a, R> {
             remaining: length,
         }
     }
+
+    pub fn into_inner(self) -> R {
+        self.iter.into_inner()
+    }
 }
 
-impl<'a, R: Read> Iterator for BooleanIter<'a, R> {
+impl<R: Read> Iterator for BooleanIter<R> {
     type Item = Result<bool, Error>;
 
     #[inline]

src/read/decode/float.rs

@@ -31,16 +31,16 @@ impl Float for f64 {
 }
 
 /// An iterator
-pub struct FloatIter<'a, T: Float, R: std::io::Read> {
-    reader: &'a mut R,
+pub struct FloatIter<T: Float, R: std::io::Read> {
+    reader: R,
     remaining: usize,
     phantom: std::marker::PhantomData<T>,
 }
 
-impl<'a, T: Float, R: std::io::Read> FloatIter<'a, T, R> {
+impl<T: Float, R: std::io::Read> FloatIter<T, R> {
     /// Returns a new [`FloatIter`]
     #[inline]
-    pub fn new(reader: &'a mut R, length: usize) -> Self {
+    pub fn new(reader: R, length: usize) -> Self {
         Self {
             reader,
             remaining: length,
@@ -59,9 +59,14 @@ impl<'a, T: Float, R: std::io::Read> FloatIter<'a, T, R> {
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    /// Returns its internal reader
+    pub fn into_inner(self) -> R {
+        self.reader
+    }
 }
 
-impl<'a, T: Float, R: std::io::Read> Iterator for FloatIter<'a, T, R> {
+impl<T: Float, R: std::io::Read> Iterator for FloatIter<T, R> {
     type Item = Result<T, Error>;
 
     #[inline]

src/read/decode/mod.rs

@@ -6,8 +6,11 @@ mod rle_v2;
 mod variable_length;
 
 pub use boolean_rle::{BooleanIter, BooleanRleRunIter, BooleanRun};
-pub use float::FloatIter;
-pub use rle_v2::{SignedRleV2Iter, SignedRleV2Run, UnsignedRleV2Iter, UnsignedRleV2Run};
+pub use float::{Float, FloatIter};
+pub use rle_v2::{
+    SignedRleV2Iter, SignedRleV2Run, SignedRleV2RunIter, UnsignedRleV2Iter, UnsignedRleV2Run,
+    UnsignedRleV2RunIter,
+};
 pub use variable_length::Values;
 
 #[inline]