Refactored Global Directives so that they can be expressed in any order

and also added some further tests around Global Directives

Author: adamretter

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala

@@ -64,9 +64,10 @@ trait SchemaParser extends RegexParsers {
   def versionDecl: Parser[String] = ("version" ~> Schema.version <~ eol).withFailureMessage(s"Schema version declaration 'version ${Schema.version}' missing or incorrect")
 
   /**
-   * [4] GlobalDirectives	::=	SeparatorDirective? QuotedDirective? TotalColumnsDirective? (NoHeaderDirective | IgnoreColumnNameCaseDirective)?
+   * [4] GlobalDirectives	::=	SeparatorDirective? QuotedDirective? TotalColumnsDirective? (NoHeaderDirective | IgnoreColumnNameCaseDirective)?    /* expr: unordered */
    */
-  def globalDirectives: Parser[List[GlobalDirective]] = rep(positioned((separatorDirective | quotedDirective | totalColumnsDirective | (noHeaderDirective | ignoreColumnNameCaseDirective)) <~ opt(eol)))
+  def globalDirectives: Parser[List[GlobalDirective]] = opt(mingle(List(separatorDirective, quotedDirective, totalColumnsDirective, noHeaderDirective | ignoreColumnNameCaseDirective).map(positioned(_) <~ opt(eol)))
+    .withFailureMessage("Invalid global directive")) ^^ { _.getOrElse(List.empty) }
 
   /**
    * [5] DirectivePrefix ::= "@"
@@ -96,17 +97,20 @@ trait SchemaParser extends RegexParsers {
   /**
    * [10]	TotalColumnsDirective	::=	DirectivePrefix "totalColumns" PositiveNonZeroIntegerLiteral
    */
+  //def totalColumnsDirective: Parser[TotalColumns] = (directivePrefix ~> "totalColumns" ~> positiveNonZeroIntegerLiteral ^^ { TotalColumns(_) }).withFailureMessage("@totalColumns invalid")
   def totalColumnsDirective: Parser[TotalColumns] = (directivePrefix ~> "totalColumns" ~> positiveNonZeroIntegerLiteral ^^ { TotalColumns(_) }).withFailureMessage("@totalColumns invalid")
 
   /**
    * [11]	NoHeaderDirective	::=	DirectivePrefix "noHeader"
    */
+//  def noHeaderDirective: Parser[NoHeader] = directivePrefix ~> "noHeader" ^^^ NoHeader()
   def noHeaderDirective: Parser[NoHeader] = directivePrefix ~> "noHeader" ^^^ NoHeader()
 
   /**
    * [12]	IgnoreColumnNameCaseDirective	::=	DirectivePrefix "ignoreColumnNameCase"
    */
   def ignoreColumnNameCaseDirective: Parser[IgnoreColumnNameCase] = directivePrefix ~> "ignoreColumnNameCase" ^^^ IgnoreColumnNameCase()
+  //def ignoreColumnNameCaseDirective: Parser[IgnoreColumnNameCase] = "ignoreColumnNameCase" ^^^ IgnoreColumnNameCase()
 
   /**
    * [13]	Body ::= BodyPart+
@@ -418,6 +422,58 @@ trait SchemaParser extends RegexParsers {
   val nonBreakingCharPattern = """[^\r\n\f]"""
   //</editor-fold>
 
+  /**
+   * Given 1 or more Parsers
+   * this function produces
+   * all permutations of
+   * all combinations.
+   *
+   * Put more simply if you have a List
+   * of Parsers, we create a Parser
+   * that matches n of those parsers
+   * in any order
+   *
+   * @param parsers A list of parsers to mingle
+   * @return A parser that represents all permutations of
+   *         all combinations of the parsers
+   */
+  private def mingle[T, U](parsers : List[Parser[T]]): Parser[List[T]] = {
+
+    /**
+     * All permutations of all combinations
+     * of a List
+     */
+    def mingle[T](data: List[T]): List[List[T]] = {
+      (for(i <- 1 to data.length) yield
+        data.combinations(i).flatMap(_.permutations)
+      ).toList.flatten
+    }
+
+    /**
+     * Combines n parsers together
+     * in the same manner as p1 ~ p2 ~ ... pN
+     */
+    def combine[T](parsers: List[Parser[T]]): Parser[List[T]] = {
+      parsers.foldRight(success(List.empty[T])) {
+        case (p, acc) => for {
+          pRes <- p
+          accRes <- acc
+        } yield pRes :: accRes
+      }
+    }
+
+    def longestFirst(l1: List[_], l2: List[_]) = l1.length > l2.length
+
+    val mingled = mingle[Parser[T]](parsers)
+      .sortWith(longestFirst)
+    //we sort longest first here,
+    //to make sure the parser that matches
+    //the most input will always be put first
+
+    val alternates = mingled.map(combine(_))
+    alternates.reduceLeft(_ | _)
+  }
+
 
   def parseAndValidate(reader: Reader): ValidationNel[FailMessage, Schema] = {
 

csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParserGlobalDirectivesSpec.scala

@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) 2013, The National Archives <[email protected]>
+ * http://www.nationalarchives.gov.uk
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package uk.gov.nationalarchives.csv.validator.schema
+
+import org.specs2.mutable._
+import scalaz.{Failure => FailureZ}
+import java.io.StringReader
+
+
+class SchemaParserGlobalDirectivesSpec extends Specification {
+
+  object TestSchemaParser extends SchemaParser { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false }
+
+  import TestSchemaParser._
+
+  "Schema" should {
+
+    "succeed for a @totalColumns global directive" in {
+      val schema =
+        """version 1.0
+           |@totalColumns 1
+           |column1: """.stripMargin
+
+      parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_)), List(ColumnDefinition("column1", Nil, Nil))),_) => ok }
+    }
+
+    "with @totalColumns and @noHeader global directives" should {
+      "succeed on seperate lines" in {
+        val schema =
+          """version 1.0
+            |@totalColumns 1
+            |@noHeader
+            |column1: """.stripMargin
+
+        parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_), NoHeader()), List(ColumnDefinition("column1", Nil, Nil))), _) => ok}
+      }
+
+      "succeed on same line" in {
+        val schema =
+          """version 1.0
+            |@totalColumns 1 @noHeader
+            |column1: """.stripMargin
+
+        parse(new StringReader(schema)) must beLike { case Success(Schema(List(TotalColumns(_), NoHeader()), List(ColumnDefinition("column1", Nil, Nil))), _) => ok}
+      }
+    }
+
+    "@noHeader and @ignoreColumnNameCase global directives (mutually exclusive)" should {
+      "fail for @noHeader followed by @ignoreColumnNameCase" >> pending("Need to improve error messages") {
+        val schema =
+          """version 1.0
+            |@noHeader
+            |@ignoreColumnNameCase
+            |column1: """.stripMargin
+
+        parse(new StringReader(schema)) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
+      }
+
+      "fail for @ignoreColumnNameCase followed by @noHeader" >> pending("Need to improve error messages") {
+        val schema =
+          """version 1.0
+            |@ignoreColumnNameCase
+            |@noHeader
+            |column1: """.stripMargin
+
+        parse(new StringReader(schema)) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
+      }
+    }
+
+    "succeed with no global directives" in {
+      val schema =
+        """version 1.0
+          |column1: """.stripMargin
+
+      parse(new StringReader(schema)) must beLike { case Success(Schema(Nil, List(ColumnDefinition("column1", Nil, Nil))), _) => ok}
+    }
+  }
+}

csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParserTotalColumnsSpec.scala

@@ -20,35 +20,35 @@ class SchemaParserTotalColumnsSpec extends Specification {
   "Schema" should {
 
     "fail for TotalColumns with missing value" in {
-      parse(new StringReader("version 1.0\n@totalColumns")) must beLike { case Failure(message, _) => message mustEqual "@totalColumns invalid" }
+      parse(new StringReader("version 1.0\n@totalColumns")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for incorrect TotalColumns field name" in {
-      parse(new StringReader("version 1.0\n@ToalColumns 23")) must beLike { case Failure(message, _) => message mustEqual "Invalid column definition" }
+      parse(new StringReader("version 1.0\n@ToalColumns 23")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for incorrect TotalColumns field name with no value" in {
-      parse(new StringReader("version 1.0\n@TtalColumn")) must beLike { case Failure(message, _) => message mustEqual "Invalid column definition" }
+      parse(new StringReader("version 1.0\n@TtalColumn")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for TotalColumns field name incorrect case" in {
-      parse(new StringReader("version 1.0\n@TotalColumns 65")) must beLike { case Failure(message, _) => message mustEqual "Invalid column definition" }
+      parse(new StringReader("version 1.0\n@TotalColumns 65")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for TotalColumns of zero" in {
-      parse(new StringReader("version 1.0\n@totalColumns 0")) must beLike { case Failure(message, _) => message mustEqual "@totalColumns invalid" }
+      parse(new StringReader("version 1.0\n@totalColumns 0")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for TotalColumns with negative integer" in {
-      parse(new StringReader("version 1.0\n@totalColumns -23")) must beLike { case Failure(message, _) => message mustEqual "@totalColumns invalid" }
+      parse(new StringReader("version 1.0\n@totalColumns -23")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
 
     "fail for TotalColumns with non integer" in {
       parse(new StringReader("version 1.0\n@totalColumns 132.45")) must beLike { case Failure(message, _) => message mustEqual "Invalid column definition" }
     }
 
     "fail for TotalColumns with non numeric" in {
-      parse(new StringReader("version 1.0\n@totalColumns blah")) must beLike { case Failure(message, _) => message mustEqual "@totalColumns invalid" }
+      parse(new StringReader("version 1.0\n@totalColumns blah")) must beLike { case Failure(message, _) => message mustEqual "Invalid global directive" }
     }
   }
 }