Now validates the job definition against the built-in schema

Author: Alan Christie

MANIFEST.in

@@ -1 +1,2 @@
 include LICENSE
+include jote/schema.yaml

jote/jote.py

@@ -8,6 +8,7 @@
 import shutil
 from typing import Any, Dict, List, Optional, Tuple
 
+import jsonschema
 from munch import DefaultMunch
 import yaml
 from yamllint import linter
@@ -27,6 +28,18 @@
 # It must exist in the root of the repo we're running in.
 _YAMLLINT_FILE: str = '.yamllint'
 
+# The (built-in) Job Definition schema...
+# from the same directory as ours.
+_SCHEMA_FILE: str = os.path.join(os.path.dirname(__file__), 'schema.yaml')
+
+# Load the schema YAML file now.
+# This must work as the file is installed along with this module.
+_JOB_SCHEMA: Dict[str, Any] = {}
+assert os.path.isfile(_SCHEMA_FILE)
+with open(_SCHEMA_FILE, 'r', encoding='utf8') as schema_file:
+    _JOB_SCHEMA = yaml.load(schema_file, Loader=yaml.FullLoader)
+assert _JOB_SCHEMA
+
 
 def _print_test_banner(collection: str,
                        job_name: str,
@@ -64,6 +77,29 @@ def _lint(definition_filename: str) -> bool:
     return True
 
 
+def _validate_schema(definition_filename: str) -> bool:
+    """Checks the Job Definition against the built-in schema.
+    """
+
+    with open(definition_filename, 'rt', encoding='UTF-8') as definition_file:
+        job_def: Optional[Dict[str, Any]] =\
+            yaml.load(definition_file, Loader=yaml.FullLoader)
+    assert job_def
+
+    # Validate the Job Definition against our schema
+    try:
+        jsonschema.validate(job_def, schema=_JOB_SCHEMA)
+    except jsonschema.ValidationError as ex:
+        print(f'! Job definition "{definition_filename}"'
+              ' does not comply with schema')
+        print(f'! Errors is "{ex.message}"')
+        print('! Full response follows:')
+        print(ex)
+        return False
+
+    return True
+
+
 def _check_cwd() -> bool:
     """Checks the execution directory for sanity (cwd). Here we must find
     a .yamllint file and a data-manager directory?
@@ -102,9 +138,15 @@ def _load(skip_lint: bool = False) -> Tuple[List[DefaultMunch], int]:
     jd_filenames: List[str] = glob.glob(f'{_DEFINITION_DIRECTORY}/*.yaml')
     for jd_filename in jd_filenames:
 
+        # Does the definition comply with the dschema,
+        # no options here - it must.
+        if not _validate_schema(jd_filename):
+            return [], -1
+
+        # YAML-lint the definition?
         if not skip_lint:
             if not _lint(jd_filename):
-                return [], -1
+                return [], -2
 
         with open(jd_filename, 'r', encoding='UTF-8') as jd_file:
             job_def: Dict[str, Any] = yaml.load(jd_file, Loader=yaml.FullLoader)

jote/schema.yaml

@@ -0,0 +1,180 @@
+---
+# The JSONSchema for the 'JobDefinition' (JD) YAML files.
+#
+# See https://json-schema.org/understanding-json-schema/index.html
+
+$schema: http://json-schema.org/draft-07/schema#
+
+title: Data Manager Job Definition
+description: >-
+  Defines one or more jobs that can be executed
+  by the Data manager Job Operator
+
+type: object
+properties:
+  kind:
+    const: DataManagerJobDefinition
+  kind-version:
+    enum:
+    - '2021.1'
+  name:
+    type: string
+    maxLength: 80
+  description:
+    tyep: string
+  collection:
+    type: string
+    minLength: 1
+    maxLength: 80
+    pattern: '^[a-z]{1}[a-z0-9-]*$'
+  repository-url:
+    type: string
+    maxlength: 2048
+    format: uri
+  repository-tag:
+    type: string
+    minLength: 1
+    maxLength: 24
+  jobs:
+    $ref: '#/definitions/job-identity'
+required:
+- kind
+- kind-version
+- collection
+- repository-url
+- repository-tag
+- jobs
+
+# Sub-object definitions ------------------------------------------------------
+# Things like the Job structure, Image structure etc.
+
+definitions:
+
+  # A Job.
+  # Consists of an identity (i.e. 'filter-molecules')
+  # followed by a Job object.
+  job-identity:
+    type: object
+    patternProperties:
+      '^[a-z]{1}[a-z0-9-]{0,79}$':
+        $ref: '#/definitions/job'
+    additionalProperties: false
+    minProperties: 1
+
+  # An individual Job
+  job:
+    type: object
+    properties:
+      name:
+        type: string
+        minLength: 1
+        maxLength: 80
+      description:
+        type: string
+      version:
+        type: string
+        minLength: 1
+        maxLength: 24
+      category:
+        type: string
+      doc-url:
+        type: string
+      keywords:
+        type: array
+        items:
+          type: string
+      image:
+        $ref: '#/definitions/image'
+      command-encoding:
+        const: JINJA2_3_0
+      command:
+        type: string
+        minLength: 1
+        maxLength: 4096
+    required:
+    - version
+    - image
+    - command
+    - name
+
+  # A Job container image
+  # The 'type' is optional and is used to indicate
+  # a single job _image_ (the default) or a workflow image like _nextflow_
+  image:
+    type: object
+    additionalProperties: false
+    properties:
+      name:
+        type: string
+        minLength: 1
+        maxLenght: 120
+      tag:
+        type: string
+        minLength: 1
+        maxLength: 24
+      project-directory:
+        type: string
+        minLength: 1
+        maxLength: 255
+        pattern: '^(/[a-zA-Z0-9_-]+)+$'
+      working-directory:
+        type: string
+        minLength: 1
+        maxLength: 255
+        pattern: '^(/[a-zA-Z0-9_-]+)+$'
+      type:
+        type: string
+        enum:
+        - simple
+        - nextflow
+        default: simple
+      environment:
+        $ref: '#/definitions/environment'
+    required:
+    - name
+    - tag
+    - project-directory
+
+  # Image environment definitions.
+  environment:
+    type: array
+    items:
+      anyOf:
+      - $ref: '#/definitions/environment-value-from-api-token'
+
+  # An Image environment from an 'api-token'.
+  # Roles is an optional list of API token realm roles where, for now,
+  # we limit the number in the list to a maximum 1.
+  environment-value-from-api-token:
+    type: object
+    additionalProperties: false
+    properties:
+      name:
+        $ref: '#/definitions/environment-name'
+      value-from:
+        type: object
+        properties:
+          api-token:
+            type: object
+            properties:
+              roles:
+                type: array
+                items:
+                  type: string
+                  pattern: '^[a-z]{1,}[a-z-_]{0,}$'
+                minItems: 0
+                maxItems: 1
+                uniqueItems: true
+        required:
+        - api-token
+    required:
+    - name
+    - value-from
+
+  # The pattern for Image environment names.
+  # Classic linux/shell,
+  # i.e. letters, digits and '_' and must begin letter or '_'
+  environment-name:
+    type: string
+    minLength: 1
+    pattern: '^[a-zA-Z_]{1,}[a-zA-Z0-9_]{0,}$'

requirements.txt

@@ -1,5 +1,6 @@
 docker-compose == 1.29.2
 im-data-manager-job-decoder == 1.0.0
+jsonschema == 3.2.0
 munch == 2.5.0
 pyyaml == 5.4.1
 yamllint == 1.26.3

setup.py

@@ -35,6 +35,7 @@ def get_long_description():
     platforms=['any'],
     # Our modules to package
     packages=find_packages(exclude=["*.test", "*.test.*", "test.*", "test"]),
+    include_package_data=True,
 
     # Project classification:
     # https://pypi.python.org/pypi?%3Aaction=list_classifiers