added a cache to store compiled regex and stopped storing success messages to reduce memory usage

Author: JessFlan

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/AllErrorsMetaDataValidator.scala

@@ -28,7 +28,15 @@ trait AllErrorsMetaDataValidator extends MetaDataValidator {
       } else {
         val row = rows.next()
         val result = validateRow(row, schema, Some(rows.hasNext))
-        validateRows(result :: results)
+        /*
+        **  Only store the results if they contain a warning or a failure.  This means the validator is not limited by the available memory 
+        **  when processing large files.
+        */
+        if (containsErrors(result) || containsWarnings(result) ) {
+            validateRows(result :: results)
+         } else {
+            validateRows(results)
+        }
       }
     }
 

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/FailFastMetaDataValidator.scala

@@ -26,16 +26,22 @@ trait FailFastMetaDataValidator extends MetaDataValidator {
 
   override def validateRows(rows: Iterator[Row], schema: Schema): MetaDataValidation[Any] = {
 
-    def containsErrors(e: MetaDataValidation[Any]): Boolean = e.fold(_.list.collectFirst(FailMessage.isError).nonEmpty, _ => false)
-
     @tailrec
     def validateRows(results: List[MetaDataValidation[Any]] = List.empty[MetaDataValidation[Any]]) : List[MetaDataValidation[Any]] = {
       if(results.headOption.map(containsErrors(_)).getOrElse(false) || !rows.hasNext) {
         results.reverse
       } else {
         val row = rows.next()
         val result = validateRow(row, schema, Some(rows.hasNext))
-        validateRows(result :: results)
+        /*
+        **  Only store the results if they contain a warning or a failure.  This means the validator is not limited by the available memory 
+        **  when processing large files.
+        */
+        if (containsErrors(result) || containsWarnings(result) ) {           
+          validateRows(result :: results)
+         } else {
+            validateRows(results)
+        }
       }
     }
 
@@ -90,4 +96,4 @@ trait FailFastMetaDataValidator extends MetaDataValidator {
     else if(isWarningDirective) validateAllRulesForCell(columnDefinition.rules)
     else validateRulesForCell(columnDefinition.rules)
   }
-}
+}

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala

@@ -46,9 +46,15 @@ object FailMessage {
   }
 }
 
+
+
 case class ProgressFor(rowsToValidate: Int, progress: ProgressCallback)
 
 trait MetaDataValidator {
+  // Helper functions for checking if a result contains a warning or error.
+  def containsErrors(e: MetaDataValidation[Any]): Boolean = e.fold(_.list.collectFirst(FailMessage.isError).nonEmpty, _ => false)
+  def containsWarnings(e: MetaDataValidation[Any]): Boolean = e.fold(_.list.collectFirst(FailMessage.isWarning).nonEmpty, _ => false)
+
   type MetaDataValidation[S] = ValidationNel[FailMessage, S]
 
   def validate(csv: JReader, schema: Schema, progress: Option[ProgressCallback]): MetaDataValidation[Any] = {

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/Rule.scala

@@ -16,6 +16,7 @@ import scala.util.Try
 import scala.util.parsing.input.Positional
 import scalaz._
 import scalaz.Scalaz._
+import java.util.regex.{Pattern, Matcher}
 
 abstract class Rule(name: String, val argProviders: ArgProvider*) extends Positional {
 
@@ -75,7 +76,6 @@ abstract class Rule(name: String, val argProviders: ArgProvider*) extends Positi
 
   def explicitName: Option[String] = explicitColumn.map("$" + _.ref + "/")
 
-
   def ruleName: String = explicitName.getOrElse("") + name
 
   def columnIdentifierToIndex(schema: Schema, id: ColumnIdentifier): Int = {
@@ -89,7 +89,6 @@ abstract class Rule(name: String, val argProviders: ArgProvider*) extends Positi
   }
 
 
-
   def toValueError(row: Row, columnIndex:Int ) =
     s"""value: ${'"'}${row.cells(columnIndex).value}${'"'}"""
 
@@ -99,8 +98,36 @@ abstract class Rule(name: String, val argProviders: ArgProvider*) extends Positi
 
 }
 
+/** 
+ *  This object is a place to store the precompilled regexs
+ *  @author Jess Flanagan 
+ */
+object RegexCache
+{
+  val cache  = collection.mutable.Map[String, Pattern]()
+  
+/** 
+ *  This function returns compiled regexs. 
+ *  First we check to see if its already in the cache. Otherwise we compile it, add it to the cache and return the 
+ *  compiled version. This results in a significant speed up for processing large files.
+ *  @param pattern A regex pattern string.
+ *  @returns A compiled representation of a regular expression.
+ *  @author Jess Flanagan 
+ */
+  def getCompiledRegex(pattern: String): Pattern ={
+    if (!cache.contains(pattern)){
+       cache += (pattern -> Pattern.compile(pattern)) 
+    }
+    cache(pattern);
+  }
+}
+
 abstract class PatternRule(name: String, pattern: String) extends Rule(name) {
-  override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = cellValue matches pattern
+  // Uses the cache to retrieve a compiled regex representation for the pattern string.
+  override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean ={
+    RegexCache.getCompiledRegex(pattern).matcher(cellValue).matches()
+
+  }
 }
 
 trait DateParser {
@@ -114,4 +141,4 @@ abstract class DateRule(name: String, dateRegex: String, dateParser: DateParser)
       case _ => false
     }
   }
-}
+}

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/Rule.scala

@@ -17,7 +17,7 @@ import uk.gov.nationalarchives.csv.validator.Util.{FileSystem, TypedPath}
 import uk.gov.nationalarchives.csv.validator.api.CsvValidator._
 import uk.gov.nationalarchives.csv.validator.metadata.Row
 import uk.gov.nationalarchives.csv.validator.schema._
-
+import java.util.regex.{Pattern, Matcher}
 import scala.annotation.tailrec
 import scala.collection.mutable
 import scala.util.Try
@@ -92,12 +92,22 @@ case class IfRule(condition: Rule, rules: List[Rule], elseRules: Option[List[Rul
   }
 }
 
+abstract class PatternRule(name: String, pattern: String) extends Rule(name) {
+  // Uses the cache to retrieve a compiled regex representation for the pattern string.
+  override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean ={
+    RegexCache.getCompiledRegex(pattern).matcher(cellValue).matches()
+
+  }
+
+}
 
 case class RegExpRule(regex: String) extends Rule("regex") {
   override def valid(cellValue: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): Boolean = {
 
     val regexp = if (columnDefinition.directives.contains(IgnoreCase())) "(?i)" + regex else regex
-    cellValue matches regexp
+    
+    RegexCache.getCompiledRegex(regexp).matcher(cellValue).matches()
+
   }
 
   override def toError = {