New work

Author: friendlymatthew

src/reader.rs

@@ -37,7 +37,6 @@ impl CsvReader {
             quotation_bitsets,
         }
     }
-
     pub fn read(&mut self) -> Result<Vec<Row>> {
         // todo! what happens when a csv is greater than 128 bytes?
         // probably would want something like:
@@ -46,93 +45,89 @@ impl CsvReader {
 
         let mut current_row = Vec::new();
 
-        let mut bit_cursor = 0;
+        let cursor = 0;
 
         for i in 0..self.quotation_bitsets.len() {
-            /*
-            Things we need to do:
-            1. find which commas are truly field separators and which are not.
-            2. BWO step 1, we'd need to identify quoted regions. CSV uses a double-quote convention
-             */
-
-            // removes all escaped quotations
-            let valid_quotations = {
-                let quotations = self.quotation_bitsets[i];
-
-                let escaped_quotations = quotations & (quotations << 1);
-                let escaped_quotations = escaped_quotations | (escaped_quotations >> 1);
-
-                quotations & !escaped_quotations
-            };
-
-            // masks all characters inside quotations
-            let inside_quotations = parallel_prefix_xor_64(valid_quotations);
-
-            let mut valid_commas = self.comma_bitsets[i] & !inside_quotations;
-            let mut valid_whitespace = self.whitespace_bitsets[i] & !inside_quotations;
+            let valid_quotations  = remove_escaped_quotations(self.quotation_bitsets[i]);
+            let outside_quotations = !mark_inside_quotations(valid_quotations);
 
-            dbg!(valid_commas, valid_whitespace);
+            let mut valid_commas = self.comma_bitsets[i] & outside_quotations;
+            let mut valid_whitespace = self.whitespace_bitsets[i] & outside_quotations;
 
-            let mut first_comma = valid_commas.leading_zeros() as usize;
-            let mut first_lf = valid_whitespace.leading_zeros() as usize;
-
-            if first_comma == 0 && first_lf == 0 {
-                // congratulations, we just skipped 64 bytes.
+            if valid_commas == 0 && valid_whitespace == 0 {
                 continue;
             }
 
-            let mut start_field_cursor = 0;
-
-            while first_comma != 0 || first_lf != 0 {
-                let end = first_comma.min(first_lf);
+            let mut bitset_cursor = 0;
 
-                current_row.push(Range {
-                    start: bit_cursor + start_field_cursor,
-                    end: bit_cursor + end,
-                });
+            loop {
+                let first_comma = valid_commas.leading_zeros() as usize;
+                let first_whitespace = valid_whitespace.leading_zeros() as usize;
 
-                start_field_cursor = end + 1;
+                let bits_traveled = first_comma.min(first_whitespace);
 
-                if first_comma < first_lf {
-                    valid_commas &= !(1u64 << (63 - first_comma));
-                    first_comma = valid_commas.leading_zeros() as usize;
-                } else {
-                    rows.push(Row::from(current_row.clone()));
+                if bits_traveled == 64 {
+                    break;
+                }
 
-                    valid_whitespace &= !(1u64 << (63 - first_lf));
-                    first_lf = valid_whitespace.leading_zeros() as usize;
+                current_row.push(
+                    Range {
+                        start: cursor + bitset_cursor,
+                        end: cursor + bitset_cursor + bits_traveled
+                    }
+                );
 
-                    return Ok(rows);
+                if first_whitespace < first_comma {
+                    rows.push(Row::from(current_row.clone()));
+                    current_row.clear();
+                    break;
                 }
+
+                bitset_cursor += bits_traveled + 1;
+                valid_commas <<= bits_traveled + 1;
+                valid_whitespace <<= bits_traveled + 1;
             }
         }
 
         Ok(rows)
     }
 }
 
-fn build_u64(chunk: &[u8x16], broadcast: u8x16) -> u64 {
-    let mut packed: u64 = 0;
-    for (i, &c) in chunk.iter().enumerate() {
-        let word = c.eq(broadcast).bitset() as u64;
-        packed |= word << (48 - i * 16);
-    }
+fn remove_escaped_quotations(q: u64) -> u64 {
+    let escaped = q & (q << 1);
+    let escaped = escaped | (escaped>> 1);
 
-    packed
+    q & !escaped
 }
 
+/// `mark_inside_quotations` does a parallel xor to mark all bits inbetween a quote pair.
+/// Note because of how xor works, the closing quote will be marked as 0. This is fine since
+/// we use this to mask commas and whitespace in between quote pairs.
 #[inline]
-fn parallel_prefix_xor_64(mut x: u64) -> u64 {
+fn mark_inside_quotations(mut x: u64) -> u64 {
     x ^= x << 1;
     x ^= x << 2;
     x ^= x << 4;
     x ^= x << 8;
     x ^= x << 16;
     x ^= x << 32;
 
-    x
+    x << 1
+}
+
+// todo, find a quicker way to do this
+#[inline]
+fn build_u64(chunk: &[u8x16], broadcast: u8x16) -> u64 {
+    let mut packed: u64 = 0;
+    for (i, &c) in chunk.iter().enumerate() {
+        let word = c.eq(broadcast).bitset() as u64;
+        packed |= word << (48 - i * 16);
+    }
+
+    packed
 }
 
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -261,28 +256,39 @@ mod tests {
         Ok(())
     }
 
+    // #[test]
+    // fn read_taxi_zone_lookup() -> Result<()> {
+    //     let data = std::fs::read("taxi_zone_lookup.csv")?;
+    //     assert_eq!(
+    //         b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\r\n",
+    //         &data[..46]
+    //     );
+    // 
+    //     let rows = CsvReader::new(&data).read()?;
+    //     dbg!(&rows);
+    // 
+    //     for row in CsvReader::new(&data).read()? {
+    //         let fields = row
+    //             .fields()
+    //             .into_iter()
+    //             .map(|field_range| String::from_utf8(data[field_range.clone()].to_vec()).unwrap())
+    //             .collect::<Vec<_>>();
+    // 
+    //         dbg!(fields);
+    //     }
+    // 
+    //     Ok(())
+    // }
+
     #[test]
-    fn read_taxi_zone_lookup() -> Result<()> {
-        let data = std::fs::read("taxi_zone_lookup.csv")?;
-        assert_eq!(
-            b"\"LocationID\",\"Borough\",\"Zone\",\"service_zone\"\r\n",
-            &data[..46]
-        );
-
-        let rows = CsvReader::new(&data).read()?;
-        assert_eq!(rows.len(), 1);
-        dbg!(&rows);
-
-        for row in CsvReader::new(&data).read()? {
-            let fields = row
-                .fields()
-                .into_iter()
-                .map(|field_range| String::from_utf8(data[field_range.clone()].to_vec()).unwrap())
-                .collect::<Vec<_>>();
-
-            dbg!(fields);
-        }
+    fn test_mark_inside_quotations() {
+        let res = mark_inside_quotations(0b10001000);
+        assert_eq!(res, 0b11110000);
 
-        Ok(())
+        let res2 = mark_inside_quotations(0b1001_1001);
+        assert_eq!(res2, 0b1110_1110);
+
+        let res4 = mark_inside_quotations(0b1100_1001);
+        assert_eq!(res4, 0b1000_1110);
     }
 }

src/u8x16.rs

@@ -41,6 +41,8 @@ impl u8x16 {
         unsafe { vceqq_u8(self.0, other.0) }.into()
     }
 
+    // figure out a better way to do this
+    // maybe just get the MSB
     pub fn bitset(self) -> u16 {
         let bs: [u8; 16] = self.into();