Now contains the schema (and a validation function)

Author: Alan Christie

MANIFEST.in

@@ -1 +1,2 @@
 include LICENSE
+include decoder/schema.yaml

README.rst

@@ -18,6 +18,10 @@ The following encoding/decoding formats are supported: -
 
 - jinja2 (3.0)
 
+The package also provides a ``validate_job_schema()`` function,
+which can (should) be used to validate the Job definition against the
+built-in schema.
+
 .. _jinja2: https://jinja.palletsprojects.com/en/3.0.x/
 
 Installation (Python)
@@ -28,9 +32,14 @@ there::
 
     pip install im-data-manager-job-decoder
 
-Once installed you can access the protocol buffers with:
+Once installed you can validate the definition (expected to be a dictionary
+formed from the definition YAML file) with:
 
 >>> from decoder import decoder
+>>> error = decoder.validate_job_schema(job_defintion)
+
+And run the decoder with:
+
 >>> decoded, success = decoder.decode(text, variables, 'command', decoder.TextEncoding.JINJA2_3_0)
 
 .. _PyPI: https://pypi.org/project/im-data-manager-job-decoder

decoder/decoder.py

@@ -5,19 +5,52 @@
 given a 'template' string and a 'dictionary' of parameters and values.
 """
 import enum
-from typing import Dict, Optional, Tuple
+import os
+from typing import Any, Dict, Optional, Tuple
+
+import jsonschema
+import yaml
 
 # The decoding engine implementations.
 # The modules are expected to be called 'decode_<TextEncoding.lower()>'
 from . import decode_jinja2_3_0
 
+# The (built-in) Job Definition schema...
+# from the same directory as us.
+_SCHEMA_FILE: str = os.path.join(os.path.dirname(__file__), 'schema.yaml')
+
+# Load the schema YAML file now.
+# This must work as the file is installed along with this module.
+_JOB_SCHEMA: Dict[str, Any] = {}
+assert os.path.isfile(_SCHEMA_FILE)
+with open(_SCHEMA_FILE, 'r', encoding='utf8') as schema_file:
+    _JOB_SCHEMA = yaml.load(schema_file, Loader=yaml.FullLoader)
+assert _JOB_SCHEMA
+
 
 class TextEncoding(enum.Enum):
     """A general text encoding format, used initially for Job text fields.
     """
     JINJA2_3_0 = 1      # Encoding that complies with Jinja2 v3.0.x
 
 
+def validate_job_schema(job_definition: Dict[str, Any]) -> Optional[str]:
+    """Checks the Job Definition (a preloaded job-definition dictionary)
+    against the built-in schema. If there's an error the error text is
+    returned, otherwise None.
+    """
+    assert job_definition
+
+    # Validate the Job Definition against our schema
+    try:
+        jsonschema.validate(job_definition, schema=_JOB_SCHEMA)
+    except jsonschema.ValidationError as ex:
+        return str(ex.message)
+
+    # OK if we get here
+    return None
+
+
 def decode(template_text: str,
            variable_map: Optional[Dict[str, str]],
            subject: str,

decoder/schema.yaml

@@ -0,0 +1,189 @@
+---
+# The JSONSchema for the 'JobDefinition' (JD) YAML files.
+#
+# See https://json-schema.org/understanding-json-schema/index.html
+
+$schema: http://json-schema.org/draft-07/schema#
+
+title: Data Manager Job Definition
+description: >-
+  Defines one or more jobs that can be executed
+  by the Data manager Job Operator
+
+type: object
+properties:
+  kind:
+    const: DataManagerJobDefinition
+  kind-version:
+    enum:
+    - '2021.1'
+  name:
+    type: string
+    maxLength: 80
+  description:
+    tyep: string
+  collection:
+    type: string
+    minLength: 1
+    maxLength: 80
+    pattern: '^[a-z]{1}[a-z0-9-]*$'
+  repository-url:
+    type: string
+    maxlength: 2048
+    format: uri
+  repository-tag:
+    type: string
+    minLength: 1
+    maxLength: 24
+  jobs:
+    $ref: '#/definitions/job-identity'
+required:
+- kind
+- kind-version
+- collection
+- repository-url
+- repository-tag
+- jobs
+
+# Sub-object definitions ------------------------------------------------------
+# Things like the Job structure, Image structure etc.
+
+definitions:
+
+  # A Job.
+  # Consists of an identity (i.e. 'filter-molecules')
+  # followed by a Job object.
+  job-identity:
+    type: object
+    patternProperties:
+      '^[a-z]{1}[a-z0-9-]{0,79}$':
+        $ref: '#/definitions/job'
+    additionalProperties: false
+    minProperties: 1
+
+  # An individual Job
+  job:
+    type: object
+    properties:
+      name:
+        type: string
+        minLength: 1
+        maxLength: 80
+      description:
+        type: string
+      version:
+        type: string
+        minLength: 1
+        maxLength: 24
+      category:
+        type: string
+      doc-url:
+        type: string
+      keywords:
+        type: array
+        items:
+          type: string
+      image:
+        $ref: '#/definitions/image'
+      command-encoding:
+        const: JINJA2_3_0
+      command:
+        type: string
+        minLength: 1
+        maxLength: 4096
+    required:
+    - version
+    - image
+    - command
+    - name
+
+  # A Job container image
+  # The 'type' is optional and is used to indicate
+  # a single job _image_ (the default) or a workflow image like _nextflow_
+  image:
+    type: object
+    additionalProperties: false
+    properties:
+      name:
+        type: string
+        minLength: 1
+        maxLenght: 120
+      tag:
+        type: string
+        minLength: 1
+        maxLength: 24
+      project-directory:
+        type: string
+        minLength: 1
+        maxLength: 255
+        pattern: '^(/[a-zA-Z0-9_-]+)+$'
+      working-directory:
+        type: string
+        minLength: 1
+        maxLength: 255
+        pattern: '^(/[a-zA-Z0-9_-]+)+$'
+      type:
+        type: string
+        enum:
+        - simple
+        - nextflow
+        default: simple
+      pull-secret:
+        $ref: '#/definitions/rfc-1035-name'
+      environment:
+        $ref: '#/definitions/env-var-name'
+    required:
+    - name
+    - tag
+    - project-directory
+
+  # Image environment definitions.
+  environment:
+    type: array
+    items:
+      anyOf:
+      - $ref: '#/definitions/environment-value-from-api-token'
+
+  # An Image environment from an 'api-token'.
+  # Roles is an optional list of API token realm roles where, for now,
+  # we limit the number in the list to a maximum 1.
+  environment-value-from-api-token:
+    type: object
+    additionalProperties: false
+    properties:
+      name:
+        $ref: '#/definitions/environment-name'
+      value-from:
+        type: object
+        properties:
+          api-token:
+            type: object
+            properties:
+              roles:
+                type: array
+                items:
+                  type: string
+                  pattern: '^[a-z]{1,}[a-z-_]{0,}$'
+                minItems: 0
+                maxItems: 1
+                uniqueItems: true
+        required:
+        - api-token
+    required:
+    - name
+    - value-from
+
+  # The pattern for Image environment names.
+  # Classic linux/shell,
+  # i.e. letters, digits and '_' and must begin letter or '_'
+  env-var-name:
+    type: string
+    minLength: 1
+    pattern: '^[a-zA-Z_]{1,}[a-zA-Z0-9_]{0,}$'
+
+  # The pattern for RFC 1035 label names.
+  rfc-1035-name:
+    type: string
+    minLength: 1
+    maxLength: 63
+    pattern: '^[a-z]([a-z0-9-]*[a-z0-9])?$'

requirements.txt

@@ -1 +1,3 @@
+jsonschema == 3.2.0
 jinja2 == 3.0.3
+pyyaml == 5.4.1