Merge pull request #14 from InformaticsMatters/issue-9

Now validates against used outputDescriptor definitions

Author: alanbchristie

.gitignore

@@ -10,6 +10,6 @@ dist
 work
 .nextflow
 .nextflow.log*
-/tmp
+**/tmp
 **/*.egg-info
 **/.DS_Store

README.md

@@ -26,6 +26,21 @@ installed as normal:
 >   The module utilities should support both Python 2 and 3 but we recommend
     any modules/pipelines you write support both flavours.
 
+## Adding this project as a submodule for a new pipelines project
+If you have started a new pipelines project you can add these utilities
+to it by running the following from the project root of your new project: -
+
+    $ git submodule add https://github.com/InformaticsMatters/pipelines-utils.git
+
+This will create a new directory (`pipelines-utils`) and a `.gitmodules` file.
+You must add the `.gitmodules` to your project. Read about the Git [Submodule]
+for more information.
+ 
+If you checkout a new working copy you will need to initialise the submodule
+with the command: -
+
+    $ git submodule update --init --remote
+
 ## Running the test framework
 >   We plan to release the Python utility modules to [PIP] at some stage. The
     distribution is based on `setup.py` is present and works and will be

src/groovy/Log.groovy

@@ -0,0 +1,65 @@
+#!/usr/bin/env groovy
+
+/**
+ * Copyright (c) 2018 Informatics Matters Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A simple log/printing utility module.
+ */
+
+// Number of warnings generated
+def warningCount = 7
+
+/**
+ * Print a simple separator (a bar) to stdout.
+ * Used to visually separate generated output into logical blocks.
+ */
+static separate() {
+
+    // To look 'pretty' this should align with the field width
+    // used in the info() function
+    println "+----------------+"
+
+}
+
+/**
+ * Print an 'info' message prefixed with `->` string. You can specify a
+ * tag and a message which is printed as "-> <tag> : <msg>"
+ */
+static info(String tag, String msg) {
+
+    println ":" + sprintf('%16s: %s', tag, msg)
+
+}
+
+/**
+ * Print a warning message (and counts it).
+ */
+static warning(String msg) {
+
+    println "WARNING: $msg"
+    Log.warningCount += 1
+
+}
+
+/**
+ * Print an error message.
+ */
+static err(String msg) {
+
+    println "ERROR: $msg"
+
+}

src/groovy/MediaChecker.groovy

@@ -0,0 +1,86 @@
+#!/usr/bin/env groovy
+
+/**
+ * Copyright (c) 2018 Informatics Matters Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The MediaTypes class. A class to provide validation of files produced
+ * by an executing pipeline.
+ */
+class MediaChecker {
+
+    final static String mediaTypesFile = 'MediaTypes.txt'
+
+    // A map of media type and expected file extension(s).
+    // This is populated by the constructor, which reads the content
+    // of 'mediaTypesFile'.
+    def mediaTypesLookup = null
+
+    /**
+     * Default constructor.
+     * Reads media types config file.
+     */
+    MediaChecker() {
+
+        mediaTypesLookup = new ConfigSlurper().parse(new File(mediaTypesFile).toURI().toURL())
+
+    }
+
+    /**
+     * Checks that a pipeline's expected files exist.
+     *
+     * @param serviceDescriptor The test service descriptor
+     * @param path The test path (where the pipeline's files are to be found)
+     * @return True on success, False otherwise.
+     */
+    boolean check(serviceDescriptor, File path) {
+
+        boolean retVal = true
+
+        // Check output descriptors for expected output file names...
+        // Do all, regardless of individual failure.
+        def opDescriptors = serviceDescriptor.serviceConfig.outputDescriptors
+        opDescriptors.each { desc ->
+
+            // There must be a matching media type in our txt/lookup file.
+            def typeExtList = mediaTypesLookup.media_types[desc.mediaType]
+            if (typeExtList == null || typeExtList.size() == 0) {
+                Log.err("Pipeline service descriptor output mediaType" +
+                        " '$desc.mediaType' is not listed in '$mediaTypesFile'")
+            } else {
+                // What are the expected file extensions for this media type?
+                // It's a list of 1 or more entries.
+                typeExtList.each { extension ->
+
+                    String opName = desc.name + extension
+                    Log.info('Media check', opName)
+                    String opPath = path.toString() + File.separator + opName
+                    if (!new File(opPath).exists()) {
+                        Log.err("The pipeline's 'outputDescriptor'" +
+                                " expected '$opName' but the file wasn't found")
+                        retVal = false
+                    }
+
+                }
+            }
+
+        }
+
+        return retVal
+
+    }
+
+}

src/groovy/MediaTypes.txt

@@ -0,0 +1,18 @@
+// Input/Output Descriptor dictionary for use with the PipelineTester.
+// This is a map of input/output descriptor `mediaType` identifiers,
+// with expected filename extension(s).
+//
+// Read by MediaChecker.groovy
+
+media_types = [
+
+    // Please keep the keys in alphabetical order...
+
+    'application/x-squonk-dataset-molecule+json': ['.data.gz', '.metadata'],
+    'application/x-squonk-dataset-basic+json': ['.data.gz', '.metadata'],
+
+    'image/gif': ['.gif'],
+    'image/jpeg': ['.jpg'],
+    'image/png': ['.png'],
+
+]

src/groovy/PipelineTester.groovy

@@ -31,7 +31,7 @@
 
 // Version
 // Update with every change/release
-String version = '2.0.0'
+String version = '2.1.0'
 
 println "+----------------+"
 println "| PipelineTester | v$version"