Add Schema validation to transformJsonToXml developer tool (#1216)

* Add Schema validation to transform tool

* Add ErrorHandler for XML/XSD schema validation errors.
* Add Schema validator which retrieves the configured redactions context and validates against the corresponding schema.
* Add functionality to output the validation errors in a log file.
* Inject and use schema validation when running the transformation process.
* Update developer information.

* * Add sonar exclusions for transform tool

* * Add sonar exclusions for transform tool
* Update XmlSchemaValidator to use newDefaultInstance instead

* * Add sonar exclusions for transform tool
* Update XmlSchemaValidator to use newDefaultInstance instead

* * Add sonar exclusions for transform tool
* Update XmlSchemaValidator to use newDefaultInstance instead

* * Refactor package name to match naming conventions

* * Extracted constants for schema validation message templates.
* Changed logging to `Error` when the schema file could not be loaded.
* Renamed parameter `filename` to be more specific about its intent.

* * Rename package folder from `transformJsonToXmlTool` to `transformjsontoxmltool` to address checkstyle package name issues

Author: MartinWheelerMT

nhs-england-developer-information.md

@@ -191,7 +191,7 @@ We provide a mock MHS adaptor for local development and testing.
 
 ### How to transform arbitrary json ASR payload files
 
-This is an interoperability testing tool to transform arbitrary/ad-hoc json ASR payloads and access the outputs.
+This is an interoperability testing tool to transform arbitrary/ad-hoc json ASR payloads, access the outputs and validate the produced XML against the relevant schema.
 
 1. Navigate to the input folder and place all Json files to convert here.
    `integration-adaptor-gp2gp/transformJsonToXml/input/`
@@ -203,9 +203,11 @@ This is an interoperability testing tool to transform arbitrary/ad-hoc json ASR
    ./TransformJsonToXml.sh
     ```
 
-3. The Converted .Xml files will be located in the output folder.
+3. The converted XML files will be located in the output folder.
    `integration-adaptor-gp2gp/transformJsonToXml/output/`
 
+4. Any schema validation errors will be located with the extension `.validation-errors.log` and will be located in the same output folder as the converted XML files.
+
 ## Troubleshooting
 
 ### "Invalid source release 17" error

service/build.gradle

@@ -172,6 +172,7 @@ sonar {
 		property("sonar.host.url", "https://sonarcloud.io")
 		property("sonar.projectKey", "NHSDigital_integration-adaptor-gp2gp")
 		property("sonar.organization", "nhsdigital")
+		property("sonar.exclusions", "**/transformjsontoxmltool/*")
 	}
 }
 

…ormJsonToXmlTool/TransformJsonToXml.java …ormjsontoxmltool/TransformJsonToXml.javaservice/src/main/java/uk/nhs/adaptors/gp2gp/transformJsonToXmlTool/TransformJsonToXml.java renamed to service/src/main/java/uk/nhs/adaptors/gp2gp/transformjsontoxmltool/TransformJsonToXml.java service/src/main/java/uk/nhs/adaptors/gp2gp/transformJsonToXmlTool/TransformJsonToXml.java renamed to service/src/main/java/uk/nhs/adaptors/gp2gp/transformjsontoxmltool/TransformJsonToXml.java

@@ -48,7 +48,6 @@
     matchIfMissing = false)
 @Component
 public class TransformJsonToXml implements CommandLineRunner {
-
     private static final String JSON_FILE_INPUT_PATH =
         Paths.get("src/").toFile().getAbsoluteFile().getAbsolutePath() + "/../../transformJsonToXml/input/";
     private static final String XML_OUTPUT_PATH =
@@ -57,6 +56,7 @@ public class TransformJsonToXml implements CommandLineRunner {
     private final MessageContext messageContext;
     private final OutputMessageWrapperMapper outputMessageWrapperMapper;
     private final EhrExtractMapper ehrExtractMapper;
+    private final XmlSchemaValidator xmlSchemaValidator;
 
     public static void main(String[] args) {
         SpringApplication.run(TransformJsonToXml.class, args).close();
@@ -68,6 +68,7 @@ public void run(String... args) {
             getFiles().forEach(file -> {
                 String xmlResult = mapJsonToXml(file.getJsonFileInput());
                 writeToFile(xmlResult, file.getJsonFileName());
+                xmlSchemaValidator.validateOutputToXmlSchema(file.getJsonFileName(), xmlResult);
             });
         } catch (NHSNumberNotFound | UnreadableJsonFileException | NoJsonFileFound | Hl7TranslatedResponseError e) {
             LOGGER.error("error: " + e.getMessage());

service/src/main/java/uk/nhs/adaptors/gp2gp/transformjsontoxmltool/XmlSchemaValidator.java

@@ -0,0 +1,105 @@
+package uk.nhs.adaptors.gp2gp.transformjsontoxmltool;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.springframework.stereotype.Component;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import uk.nhs.adaptors.gp2gp.common.configuration.RedactionsContext;
+
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
+
+@Component
+@Slf4j
+@RequiredArgsConstructor()
+public class XmlSchemaValidator {
+    private final RedactionsContext redactionsContext;
+
+    private static final String SCHEMA_PATH = "../service/src/test/resources/mim/Schemas/";
+    private static final String RCMR_IN030000UK06_SCHEMA_PATH =  SCHEMA_PATH + RedactionsContext.NON_REDACTION_INTERACTION_ID + ".xsd";
+    private static final String RCMR_IN030000UK07_SCHEMA_PATH =  SCHEMA_PATH + RedactionsContext.REDACTION_INTERACTION_ID + ".xsd";
+    private static final String OUTPUT_PATH =
+        Paths.get("src/").toFile().getAbsoluteFile().getAbsolutePath() + "/../../transformJsonToXml/output/";
+
+    private static final String VALIDATING_AGAINST_SCHEMA_TEMPLATE = "Validating {} against {} schema";
+    private static final String COULD_NOT_LOAD_SCHEMA_FILE_TEMPLATE = "Could not load schema file for {} context.";
+    private static final String FAILED_TO_VALIDATE_SCHEMA_TEMPLATE = "Failed to validate {} against {} schema";
+    private static final String COULD_NOT_READ_FROM_STREAM_SOURCE_TEMPLATE = "Could not read from stream source for produced XML for {}";
+    private static final String SUCCESSFULLY_VALIDATED_SCHEMA_TEMPLATE = "Successfully validated {} against {} schema";
+    private static final String VALIDATION_ERRORS_WRITTEN_TEMPLATE = "Validation errors written to {}";
+    private static final String COULD_NOT_WRITE_VALIDATION_ERRORS_TEMPLATE = "Could not write validation errors to {}";
+
+    public void validateOutputToXmlSchema(String inputJsonFilename, String xmlResult) {
+        LOGGER.info(VALIDATING_AGAINST_SCHEMA_TEMPLATE, inputJsonFilename, RedactionsContext.REDACTION_INTERACTION_ID);
+
+        var xsdErrorHandler = new XsdErrorHandler();
+        Validator xmlValidator;
+
+        try {
+            xmlValidator = getXmlValidator(xsdErrorHandler);
+        } catch (SAXException e) {
+            LOGGER.error(COULD_NOT_LOAD_SCHEMA_FILE_TEMPLATE, RedactionsContext.REDACTION_INTERACTION_ID);
+            return;
+        }
+
+        try {
+            var xmlResultSource = new StreamSource(new StringReader(xmlResult));
+            xmlValidator.validate(xmlResultSource);
+        } catch (SAXParseException parseException) {
+            LOGGER.warn(FAILED_TO_VALIDATE_SCHEMA_TEMPLATE, inputJsonFilename, RedactionsContext.REDACTION_INTERACTION_ID);
+            writeValidationExceptionsToFile(xsdErrorHandler, inputJsonFilename);
+        } catch (IOException e) {
+            LOGGER.error(COULD_NOT_READ_FROM_STREAM_SOURCE_TEMPLATE, inputJsonFilename);
+            return;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        if (!xsdErrorHandler.isValid()) {
+            LOGGER.warn(FAILED_TO_VALIDATE_SCHEMA_TEMPLATE, inputJsonFilename, RedactionsContext.REDACTION_INTERACTION_ID);
+            writeValidationExceptionsToFile(xsdErrorHandler, inputJsonFilename);
+            return;
+        }
+
+        LOGGER.info(SUCCESSFULLY_VALIDATED_SCHEMA_TEMPLATE, inputJsonFilename, RedactionsContext.REDACTION_INTERACTION_ID);
+    }
+
+    private void writeValidationExceptionsToFile(XsdErrorHandler xsdErrorHandler, String fileName) {
+        String outputFileName = FilenameUtils.removeExtension(fileName) + ".validation-errors.log";
+        try (BufferedWriter writer = new BufferedWriter(new FileWriter(OUTPUT_PATH + outputFileName, StandardCharsets.UTF_8))) {
+            for (SAXParseException e : xsdErrorHandler.getExceptions()) {
+                var message = String.format("[%d:%d] %s", e.getLineNumber(), e.getColumnNumber(), e.getMessage());
+                writer.write(message);
+                writer.newLine();
+            }
+            LOGGER.info(VALIDATION_ERRORS_WRITTEN_TEMPLATE, outputFileName);
+        } catch (IOException e) {
+            LOGGER.error(COULD_NOT_WRITE_VALIDATION_ERRORS_TEMPLATE, outputFileName, e);
+        }
+    }
+
+    private Validator getXmlValidator(XsdErrorHandler xsdErrorHandler) throws SAXException {
+        var schemaPath = RedactionsContext.REDACTION_INTERACTION_ID.equals(redactionsContext.ehrExtractInteractionId())
+            ? RCMR_IN030000UK07_SCHEMA_PATH
+            : RCMR_IN030000UK06_SCHEMA_PATH;
+
+        var schemaFactory = SchemaFactory.newDefaultInstance();
+        var schemaFileStream = new StreamSource(new File(schemaPath));
+        var schema = schemaFactory.newSchema(schemaFileStream);
+        var validator = schema.newValidator();
+
+        validator.setErrorHandler(xsdErrorHandler);
+
+        return validator;
+    }
+}

service/src/main/java/uk/nhs/adaptors/gp2gp/transformjsontoxmltool/XsdErrorHandler.java

@@ -0,0 +1,36 @@
+package uk.nhs.adaptors.gp2gp.transformjsontoxmltool;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXParseException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+@Getter
+public class XsdErrorHandler implements ErrorHandler {
+
+    private final List<SAXParseException> exceptions = new ArrayList<>();
+
+    public boolean isValid() {
+        return exceptions.isEmpty();
+    }
+
+    @Override
+    public void warning(SAXParseException exception) {
+        exceptions.add(exception);
+    }
+
+    @Override
+    public void error(SAXParseException exception) {
+        exceptions.add(exception);
+    }
+
+    @Override
+    public void fatalError(SAXParseException exception) throws SAXParseException {
+        exceptions.add(exception);
+        throw exception;
+    }
+}