Merge pull request #64 from pangenome/lower_memory_usage_again

Lower memory usage during IMPG index creationg

Author: AndreaGuarracino

src/impg.rs

@@ -1,4 +1,4 @@
-use crate::paf::{PafRecord, ParseErr, Strand};
+use crate::paf::{PartialPafRecord, ParseErr, Strand};
 use crate::seqidx::SequenceIndex;
 use coitrees::{BasicCOITree, Interval, IntervalTree};
 use log::debug;
@@ -297,7 +297,7 @@ pub struct Impg {
 
 impl Impg {
     pub fn from_multi_paf_records(
-        records_by_file: &[(Vec<PafRecord>, String)],
+        records_by_file: &[(Vec<PartialPafRecord>, String)],
         seq_index: SequenceIndex
     ) -> Result<Self, ParseErr> {
         // Use par_iter to process the files in parallel and collect both pieces of information
@@ -1360,13 +1360,11 @@ mod tests {
         let target_id = seq_index.get_or_insert_id("t1", Some(200));
         let reader = BufReader::new(&paf_data[..]);
         let expected_records = vec![
-            PafRecord {
+            PartialPafRecord {
                 query_id: query_id,
-                query_length: 100,
                 query_start: 10,
                 query_end: 20,
                 target_id: target_id,
-                target_length: 200,
                 target_start: 30,
                 target_end: 40,
                 strand_and_cigar_offset: 45, // Forward strand

src/main.rs

@@ -12,7 +12,7 @@ use std::io::{self, BufReader, BufWriter};
 use std::num::NonZeroUsize;
 use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
-use crate::paf::PafRecord;
+use crate::paf::PartialPafRecord;
 use rayon::prelude::*;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
@@ -425,7 +425,7 @@ fn generate_multi_index(paf_files: &[String], num_threads: NonZeroUsize, custom_
     // Process PAF files in parallel using Rayon
     let records_by_file: Vec<_> = (0..paf_files.len())
         .into_par_iter()
-        .map(|file_index| -> io::Result<(Vec<PafRecord>, String)> {
+        .map(|file_index| -> io::Result<(Vec<PartialPafRecord>, String)> {
             let paf_file = &paf_files[file_index];
 
             // Increment the counter and get the new value atomically

src/paf.rs

@@ -4,13 +4,11 @@ use std::num::ParseIntError;
 use crate::seqidx::SequenceIndex;
 
 #[derive(Debug, PartialEq, Clone)]
-pub struct PafRecord {
+pub struct PartialPafRecord {
     pub query_id: u32,
-    pub query_length: usize,
     pub query_start: usize,
     pub query_end: usize,
     pub target_id: u32,
-    pub target_length: usize,
     pub target_start: usize,
     pub target_end: usize,
     pub strand_and_cigar_offset: u64, // Track strand and cigar offset
@@ -25,7 +23,7 @@ pub enum Strand {
     Reverse,
 }
 
-impl PafRecord {
+impl PartialPafRecord {
     const STRAND_BIT: u64 = 0x8000000000000000; // Most significant bit for u64
 
     pub fn strand(&self) -> Strand {
@@ -86,11 +84,9 @@ impl PafRecord {
         // Create the record and set strand
         let mut record = Self {
             query_id,
-            query_length,
             query_start,
             query_end,
             target_id,
-            target_length,
             target_start,
             target_end,
             strand_and_cigar_offset: cigar_offset,
@@ -113,12 +109,12 @@ pub enum ParseErr {
     InvalidFormat(String),
 }
 
-pub fn parse_paf<R: BufRead>(reader: R, seq_index: &mut SequenceIndex) -> Result<Vec<PafRecord>, ParseErr> {
+pub fn parse_paf<R: BufRead>(reader: R, seq_index: &mut SequenceIndex) -> Result<Vec<PartialPafRecord>, ParseErr> {
     let mut bytes_read: u64 = 0;
     let mut records = Vec::new();
     for line_result in reader.lines() {
         let line = line_result.map_err(ParseErr::IoError)?;
-        let record = PafRecord::parse(&line, bytes_read, seq_index)?;
+        let record = PartialPafRecord::parse(&line, bytes_read, seq_index)?;
         records.push(record);
 
         // Size of line plus newline
@@ -135,21 +131,19 @@ mod tests {
     fn test_parse_paf_valid() {
         let line = "seq1\t100\t0\t100\t+\tseq2\t100\t0\t100\t60\t100\t255";
         let mut seq_index = SequenceIndex::new();
-        let record = PafRecord::parse(line, 0, &mut seq_index).unwrap();
+        let record = PartialPafRecord::parse(line, 0, &mut seq_index).unwrap();
 
         // IDs should be 0 and 1 as they're the first entries in the SequenceIndex
         let query_id = seq_index.get_id("seq1").unwrap();
         let target_id = seq_index.get_id("seq2").unwrap();
-        
+
         assert_eq!(
             record,
-            PafRecord {
+            PartialPafRecord {
                 query_id,
-                query_length: 100,
                 query_start: 0,
                 query_end: 100,
                 target_id,
-                target_length: 100,
                 target_start: 0,
                 target_end: 100,
                 // If no cigar, then the offset is just the length of the line and cigar_bytes=0
@@ -164,22 +158,22 @@ mod tests {
     fn test_parse_paf_valid_2() {
         let line = "seq1\t100\t0\t100\t+\tseq2\t100\t0\t100\t60\t100\t255\tcg:Z:10=";
         let mut seq_index = SequenceIndex::new();
-        assert!(PafRecord::parse(line, 0, &mut seq_index).is_ok());
+        assert!(PartialPafRecord::parse(line, 0, &mut seq_index).is_ok());
     }
 
     #[test]
     fn test_parse_paf_invalid() {
         // it's got a character 'z' in the length field
         let line = "seq1\t100\t0\t100\t+\tseq2\t100\tz\t100\t60\t100\t255\tcg:Z:10M";
         let mut seq_index = SequenceIndex::new();
-        assert!(PafRecord::parse(line, 0, &mut seq_index).is_err());
+        assert!(PartialPafRecord::parse(line, 0, &mut seq_index).is_err());
     }
 
     #[test]
     fn test_parse_paf_cigar_invalid() {
         // it's got Q in the CIGAR string
         let line = "seq1\t100\t0\t100\t+\tseq2\t100\tz\t100\t60\t100\t255\tcg:Z:10Q";
         let mut seq_index = SequenceIndex::new();
-        assert!(PafRecord::parse(line, 0, &mut seq_index).is_err());
+        assert!(PartialPafRecord::parse(line, 0, &mut seq_index).is_err());
     }
 }