Merge pull request #117 from rhubner/url-decode

Url decode

Author: valydia

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala

@@ -12,8 +12,8 @@ import uk.gov.nationalarchives.csv.validator.schema.{Schema, SchemaParser}
 import scalaz._, Scalaz._
 import scalax.file.Path
 import uk.gov.nationalarchives.csv.validator._
-import java.io.{Reader => JReader, File}
-import java.nio.charset.{Charset => JCharset}
+import _root_.java.io.{Reader => JReader, File}
+import _root_.java.nio.charset.{Charset => JCharset}
 
 object CsvValidator {
 

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/Schema.scala

@@ -15,7 +15,7 @@ import util.parsing.input.Positional
 case class Schema(globalDirectives: List[GlobalDirective], columnDefinitions: List[ColumnDefinition], version: String = Schema.version)
 
 object Schema {
-  val version = "1.1"
+  val version = "1.2"
 }
 
 abstract class GlobalDirective(val name: String) extends Positional

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala

@@ -10,6 +10,7 @@ package uk.gov.nationalarchives.csv.validator.schema
 
 import uk.gov.nationalarchives.csv.validator.schema.v1_0.{SchemaParser => SchemaParser1_0}
 import uk.gov.nationalarchives.csv.validator.schema.v1_1.{SchemaParser => SchemaParser1_1, _}
+import uk.gov.nationalarchives.csv.validator.schema.v1_2.{SchemaParser => SchemaParser1_2, _}
 
 import scala.util.parsing.combinator._
 import scala.language.reflectiveCalls
@@ -147,6 +148,18 @@ with TraceableParsers {
 
     SchemaValidator.versionValid(version).map(Failure(_, next)).getOrElse {
       version match {
+        case "1.2" =>
+          val parser1_2 = new SchemaParser1_2 {override val enforceCaseSensitivePathChecks: Boolean = ecspc
+            override val pathSubstitutions: List[(String, String)] = ps
+            override val trace: Boolean = t
+          }
+
+          parser1_2.parseVersionAware(reader) match {
+            case parser1_2.Success(s, n) => Success(s, n)
+            case parser1_2.Failure(msg, n) => Failure(msg, n)
+            case parser1_2.Error(msg, n) => Error(msg, n)
+          }
+
         case "1.1" =>
           val parser1_1 = new SchemaParser1_1 {override val enforceCaseSensitivePathChecks: Boolean = ecspc
             override val pathSubstitutions: List[(String, String)] = ps

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_2/Schema.scala

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2013, The National Archives <[email protected]>
+ * http://www.nationalarchives.gov.uk
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package uk.gov.nationalarchives.csv.validator.schema.v1_2
+
+import uk.gov.nationalarchives.csv.validator.metadata.Row
+import uk.gov.nationalarchives.csv.validator.schema.{Schema, ArgProvider}
+import java.net.{URLDecoder => JURLDecoder}
+
+case class UriDecode(value: ArgProvider, charset: Option[ArgProvider]) extends ArgProvider {
+
+  val DefaultCharset = "UTF-8"
+
+  override def referenceValue(columnIndex: Int, row: Row, schema: Schema): Option[String] = value.referenceValue(columnIndex, row, schema).map( value => {
+
+    val codepage = charset.flatMap(x => x.referenceValue(columnIndex, row, schema)).getOrElse(DefaultCharset)
+
+    JURLDecoder.decode(value,codepage)
+
+  })
+
+  override def toError: String = "uriDecode(" + value.toError + ")"
+}

csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_2/SchemaParser.scala

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2013, The National Archives <[email protected]>
+ * http://www.nationalarchives.gov.uk
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package uk.gov.nationalarchives.csv.validator.schema.v1_2
+
+import scala.language.reflectiveCalls
+import uk.gov.nationalarchives.csv.validator.schema.{Literal, ArgProvider}
+import uk.gov.nationalarchives.csv.validator.schema.v1_1.{SchemaParser => SchemaParser1_1}
+
+trait SchemaParser extends SchemaParser1_1 {
+
+  /**
+   * [59] StringProvider ::= ColumnRef | StringLiteral
+   */
+  override lazy val stringProvider: PackratParser[ArgProvider] = "StringProvider" ::= noext | concat | urlDecode | columnRef | stringLiteral ^^ {
+    s => Literal(Some(s))
+  }
+
+  lazy val urlDecode: PackratParser[ArgProvider] = "UriDecode" ::= "uriDecode(" ~> stringProvider ~ opt("," ~> stringProvider)  <~ ")" ^^ {
+    case value ~ charset => UriDecode(value, charset)
+  }
+
+}

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecode.csvs

@@ -0,0 +1,4 @@
+version 1.2
+@totalColumns 2 @noHeader
+identifier:
+filename: in(uriDecode($identifier))

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeFail.csv

@@ -0,0 +1 @@
+file:/some/folder/some%21file.txt,some file.txt

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodePass.csv

@@ -0,0 +1 @@
+file:/some/folder/some%20file.txt,some file.txt

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeWithCharset.csvs

@@ -0,0 +1,5 @@
+version 1.2
+@totalColumns 3 @noHeader
+identifier:
+filename: in(uriDecode($identifier, $charset))
+charset:

csv-validator-core/src/test/resources/uk/gov/nationalarchives/csv/validator/acceptance/uriDecodeWithCharsetPass.csv

@@ -0,0 +1,2 @@
+file:/some/folder/some%20file.txt,some file.txt,UTF-8
+file:/some/folder/text%9Atext.txt,textštext.txt,windows-1252