Refactoring xan reverse for better performance

Author: Yomguithereal

CHANGELOG.md

@@ -53,6 +53,7 @@ The **parallel** update.
 * Adding support for `xan search -l -p -t`.
 * Adding `rms` moonblade aggregation function.
 * Adding `xan scrape -E/--encoding`.
+* Adding CDX files support.
 
 *Fixes*
 
@@ -93,6 +94,7 @@ The **parallel** update.
 
 * Switching hashmaps to `ahash`.
 * Optimizing moonblade pipelines with more than a single underscore substitution.
+* Improving `xan reverse` performance.
 
 *Quality of Life*
 

src/cmd/reverse.rs

@@ -47,7 +47,7 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
 
     let rconfig = &mut Config::new(&args.arg_input)
         .delimiter(args.flag_delimiter)
-        .no_headers(true);
+        .no_headers(args.flag_no_headers);
 
     if args.flag_in_memory {
         run_without_memory_efficiency(rconfig, args)
@@ -57,49 +57,37 @@ pub fn run(argv: &[&str]) -> CliResult<()> {
 }
 
 fn run_with_memory_efficiency(rconfig: &mut Config, args: Args) -> CliResult<()> {
-    rconfig.no_headers = true;
+    let (headers, mut reverse_reader) = rconfig.reverse_reader().map_err(|_| {
+        "can't use provided input: needs to be loaded in the RAM using -m, --in-memory flag"
+    })?;
 
-    let mut config_csv_reader = rconfig.reader()?;
-    let headers_offset = if args.flag_no_headers {
-        0
-    } else {
-        config_csv_reader.byte_headers()?;
-        config_csv_reader.position().byte()
-    };
-
-    let reverse_reader = rconfig.io_reader_for_reverse_reading(headers_offset);
-
-    match reverse_reader {
-        Err(_) => Err(
-            "can't use provided input: needs to be loaded in the RAM using -m, --in-memory flag",
-        )?,
-        Ok(rr) => {
-            let mut wtr = Config::new(&args.flag_output).writer()?;
-            let mut reverse_csv_reader = rconfig.csv_reader_from_reader(rr);
-
-            if !args.flag_no_headers && headers_offset > 0 {
-                let headers = config_csv_reader.byte_headers()?;
-                wtr.write_byte_record(headers)?;
-            }
-
-            for record in reverse_csv_reader.byte_records().flatten() {
-                let new_record: Vec<Vec<u8>> = record
-                    .iter()
-                    .rev()
-                    .map(|b| b.iter().rev().copied().collect())
-                    .collect();
-
-                wtr.write_record(new_record)?;
-            }
-
-            Ok(wtr.flush()?)
+    let mut wtr = Config::new(&args.flag_output).writer()?;
+
+    if !args.flag_no_headers && !headers.is_empty() {
+        wtr.write_byte_record(&headers)?;
+    }
+
+    let mut record = csv::ByteRecord::new();
+    let mut reversed_record = csv::ByteRecord::new();
+    let mut reversed_bytes: Vec<u8> = Vec::new();
+
+    while reverse_reader.read_byte_record(&mut record)? {
+        reversed_record.clear();
+
+        for cell in record.iter().rev() {
+            reversed_bytes.clear();
+            reversed_bytes.extend(cell.iter().rev());
+
+            reversed_record.push_field(&reversed_bytes);
         }
+
+        wtr.write_byte_record(&reversed_record)?;
     }
+
+    Ok(wtr.flush()?)
 }
 
 fn run_without_memory_efficiency(rconfig: &mut Config, args: Args) -> CliResult<()> {
-    rconfig.no_headers = args.flag_no_headers;
-
     let mut reader = rconfig.reader()?;
     let all = reader.byte_records().collect::<Result<Vec<_>, _>>()?;
 

src/config.rs

@@ -488,24 +488,13 @@ impl Config {
         let reverse_reader = ReverseRead::new(forward_reader.into_inner(), filesize, offset);
         let mut reader_builder = self.csv_reader_builder();
         reader_builder.has_headers(false);
-
+        dbg!(offset, filesize);
         Ok((
             headers,
             reader_builder.from_reader(Box::new(reverse_reader)),
         ))
     }
 
-    pub fn io_reader_for_reverse_reading(
-        &self,
-        offset: u64,
-    ) -> CliResult<Box<dyn io::Read + 'static>> {
-        let mut reader = self.io_reader_for_random_access()?;
-
-        let filesize = reader.seek(SeekFrom::End(0))?;
-
-        Ok(Box::new(ReverseRead::new(reader, filesize, offset)))
-    }
-
     pub fn csv_reader_builder(&self) -> csv::ReaderBuilder {
         let mut builder = csv::ReaderBuilder::new();
 

tests/test_slice.rs

@@ -207,3 +207,41 @@ fn slice_conditions() {
     let expected = vec![svec!["n"], svec!["2"]];
     assert_eq!(got, expected);
 }
+
+#[test]
+fn slice_last() {
+    let wrk = Workdir::new("slice_last");
+    wrk.create(
+        "data.csv",
+        vec![
+            svec!["n"],
+            svec!["zero"],
+            svec!["one"],
+            svec!["two"],
+            svec!["three"],
+            svec!["four"],
+            svec!["five"],
+        ],
+    );
+    let mut cmd = wrk.command("slice");
+    cmd.args(["-L", "3"]).arg("data.csv");
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![svec!["n"], svec!["three"], svec!["four"], svec!["five"]];
+    assert_eq!(got, expected);
+
+    let mut cmd = wrk.command("slice");
+    cmd.args(["-L", "300"]).arg("data.csv");
+
+    let got: Vec<Vec<String>> = wrk.read_stdout(&mut cmd);
+    let expected = vec![
+        svec!["n"],
+        svec!["zero"],
+        svec!["one"],
+        svec!["two"],
+        svec!["three"],
+        svec!["four"],
+        svec!["five"],
+    ];
+    assert_eq!(got, expected);
+}