Adjust maxMergedLength/minMergedLength in mergeReadPairPartial if they are larger than lenF + lenR, to avoid underflow in size_t subtraction

Author: csoneson

NEWS.md

@@ -1,3 +1,7 @@
+# mutscan 0.3.3
+
+* Bugfix in mergeReadPairPartial for situation where specified minMergedLength/maxMergedLength is larger than the total length of the two merged sequences
+
 # mutscan 0.3.1
 
 * Adapt summarizeExperiment to include errorStatistics in metadata(se)

src/digestFastqs.cpp

@@ -829,12 +829,15 @@ bool mergeReadPairPartial(std::string &varSeqForward, std::vector<int> &varIntQu
   if (maxMergedLength == 0) {
     maxMergedLength = lenF + lenR;
   }
+  if (maxMergedLength > lenF + lenR) {
+    maxMergedLength = lenF + lenR;
+  }
   // if maxMergedLength is specified, adjust allowed minOverlap
   if (minOverlap < (lenF + lenR - maxMergedLength)) {
     minOverlap = (lenF + lenR - maxMergedLength);
   }
   if (minOverlap > lenF || minOverlap > lenR) {
-    return true; //  no valid overlap possible
+      return true; //  no valid overlap possible
   }
 
   if (maxOverlap == 0) {
@@ -851,6 +854,9 @@ bool mergeReadPairPartial(std::string &varSeqForward, std::vector<int> &varIntQu
       minMergedLength = lenR;
     }
   }
+  if (minMergedLength > lenF + lenR) {
+    return true; // no valid overlap possible
+  }
   // if minMergedLength is specified, adjust allowed maxOverlap
   if (maxOverlap > (lenF + lenR - minMergedLength)) {
     maxOverlap = (lenF + lenR - minMergedLength);

tests/testthat/test_digestFastqs_helpers.R

@@ -397,7 +397,22 @@ test_that("mergeReadPairsPartial works", {
   expect_identical(res0d$mergedQual, qF2)
   expect_identical(res0d$mergedLengths, 7L)
   expect_true(res0d$return)
-
+  
+  ## minMergedLength > lenF + lenR -> no valid overlap
+  res0e <- mutscan:::test_mergeReadPairPartial(sF2, qF2, sR2, qR2, lF2, lR2, 1, 7, 15, 9, 1, FALSE)
+  expect_type(res0e, "list")
+  expect_identical(res0e$mergedSeq, sF2)
+  expect_identical(res0e$mergedQual, qF2)
+  expect_identical(res0e$mergedLengths, 7L)
+  expect_true(res0e$return)
+  ## maxMergedLength > lenF + lenR -> maxMergedLength = lenF + lenR
+  res0f <- mutscan:::test_mergeReadPairPartial(sF0, qF0, sR0, qR0, lF0, lR0, 1, 0, 5, 19, 1, FALSE)
+  expect_type(res0f, "list")
+  expect_identical(res0f$mergedSeq, sF0)
+  expect_identical(res0f$mergedQual, rep(c(10L, 40L), c(2, 5)))
+  expect_identical(res0f$mergedLengths, 7L)
+  expect_false(res0f$return)
+  
   ## padded reads
   for (i in 1:10) {
     sF <- paste(rep(c("C","A"), c(i, 6)), collapse = "")