Merge pull request #534 from digital-preservation/DR2-2066_ignoreLineBreaksInCellsWhenCountingRows

Ignore line breaks in cells when counting rows

Author: techncl

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala

@@ -12,7 +12,7 @@ package uk.gov.nationalarchives.csv.validator
 import cats.data.{Chain, Validated, ValidatedNel}
 import cats.syntax.all._
 import com.univocity.parsers.common.TextParsingException
-import com.univocity.parsers.csv.{CsvParser, CsvParserSettings}
+import com.univocity.parsers.csv.{CsvParser, CsvParserSettings, CsvRoutines}
 import org.apache.commons.io.input.BOMInputStream
 import uk.gov.nationalarchives.csv.validator.api.TextFile
 import uk.gov.nationalarchives.csv.validator.metadata.{Cell, Row}
@@ -292,10 +292,10 @@ trait MetaDataValidator {
 
   protected def rulesForCell(columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): MetaDataValidation[Any]
 
-  protected def countRows(textFile: TextFile, schema: Schema): Int = {
+  protected def countRows(textFile: TextFile): Int = {
     withReader(textFile) {
-      reader =>
-        countRows(reader, schema)
+      // getInputDimension is more efficient and ignores new lines in cells but it closes the Reader after use; only use it when it's OK to discard the reader.
+      reader => Try(new CsvRoutines().getInputDimension(reader).rowCount().toInt).getOrElse(-1)
     }
   }
 

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala

@@ -99,7 +99,7 @@ trait CsvValidator extends SchemaParser {
 
     val csvValidation = withReader(csvFile) {
       reader =>
-        val totalRows = countRows(csvFile, csvSchema)
+        val totalRows = countRows(csvFile)
         validateKnownRows(reader, csvSchema, this.maxCharsPerCell, progress.map(p => {ProgressFor(totalRows, p)} ), rowCallback)
     }
     encodingValidationNel.isValid && csvValidation

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/api/metadataMultipleLineBreaksInCell.csv

@@ -0,0 +1,9 @@
+col1,"this
+     cell
+     has
+     multiple
+     line
+     breaks
+     in
+     it"
+col1, col2

csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala

@@ -60,6 +60,24 @@ class CsvValidatorSpec extends Specification with TestResources {
         case Validated.Valid(_) => ok
       }
     }
+
+    val callback = new ProgressCallback {
+      var processed = -1
+      var total = -2
+      override def update(complete: Percentage): Unit = throw new NotImplementedError()
+
+      override def update(_total: Int, _processed: Int): Unit = {
+        total = _total
+        processed = _processed
+      }
+    }
+
+    "have a total (of rows) equal to the actual number of rows in the metadata file even if there are multiple line breaks in a cell" in {
+      app.validate(TextFile(Paths.get(baseResourcePkgPath).resolve("metadataMultipleLineBreaksInCell.csv")), parse(baseResourcePkgPath + "/schema.csvs"), Some(callback)) must beLike {
+        case Validated.Valid(_) => ok
+      }
+      callback.total must beEqualTo(2)
+    }
   }
 }