Add candidate schema for version 2 (#142)

* Add candidate schema for version 2

* Store PEtab extension config in petab.Problem

Author: dweindl

.github/workflows/ci_tests.yml

@@ -38,7 +38,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip wheel
         pip install -r .ci_pip_reqs.txt
         pip install .[reports,combine,tests]
 

petab/C.py

@@ -264,6 +264,8 @@
 OBSERVABLE_FILES = 'observable_files'
 #:
 VISUALIZATION_FILES = 'visualization_files'
+#:
+EXTENSIONS = 'extensions'
 
 
 # MORE

petab/lint.py

@@ -763,6 +763,13 @@ def lint_problem(problem: 'petab.Problem') -> bool:
     # pylint: disable=too-many-statements
     errors_occurred = False
 
+    if problem.extensions_config:
+        logger.warning(
+            "Validation of PEtab extensions is not yet implemented, "
+            "but the given problem uses the following extensions: "
+            f"{'', ''.join(problem.extensions_config.keys())}"
+        )
+
     # Run checks on individual files
     if problem.model is not None:
         logger.info("Checking model...")

petab/problem.py

@@ -46,19 +46,22 @@ class Problem:
         sbml_reader: Stored to keep object alive (deprecated).
         sbml_document: Stored to keep object alive (deprecated).
         sbml_model: PEtab SBML model (deprecated)
+        extensions_config: Information on the extensions used
     """
 
-    def __init__(self,
-                 sbml_model: libsbml.Model = None,
-                 sbml_reader: libsbml.SBMLReader = None,
-                 sbml_document: libsbml.SBMLDocument = None,
-                 model: Model = None,
-                 condition_df: pd.DataFrame = None,
-                 measurement_df: pd.DataFrame = None,
-                 parameter_df: pd.DataFrame = None,
-                 visualization_df: pd.DataFrame = None,
-                 observable_df: pd.DataFrame = None):
-
+    def __init__(
+            self,
+            sbml_model: libsbml.Model = None,
+            sbml_reader: libsbml.SBMLReader = None,
+            sbml_document: libsbml.SBMLDocument = None,
+            model: Model = None,
+            condition_df: pd.DataFrame = None,
+            measurement_df: pd.DataFrame = None,
+            parameter_df: pd.DataFrame = None,
+            visualization_df: pd.DataFrame = None,
+            observable_df: pd.DataFrame = None,
+            extensions_config: Dict = None,
+    ):
         self.condition_df: Optional[pd.DataFrame] = condition_df
         self.measurement_df: Optional[pd.DataFrame] = measurement_df
         self.parameter_df: Optional[pd.DataFrame] = parameter_df
@@ -80,6 +83,7 @@ def __init__(self,
                 sbml_document=sbml_document)
 
         self.model: Optional[Model] = model
+        self.extensions_config = extensions_config or {}
 
     def __getattr__(self, name):
         # For backward-compatibility, allow access to SBML model related
@@ -113,6 +117,7 @@ def from_files(
                                        Iterable[Union[str, Path]]] = None,
             observable_files: Union[str, Path,
                                     Iterable[Union[str, Path]]] = None,
+            extensions_config: Dict = None,
     ) -> 'Problem':
         """
         Factory method to load model and tables from files.
@@ -124,6 +129,7 @@ def from_files(
             parameter_file: PEtab parameter table
             visualization_files: PEtab visualization tables
             observable_files: PEtab observables tables
+            extensions_config: Information on the extensions used
         """
         warn("petab.Problem.from_files is deprecated and will be removed in a "
              "future version. Use `petab.Problem.from_yaml instead.",
@@ -154,12 +160,15 @@ def from_files(
             observable_files, observables.get_observable_df) \
             if observable_files else None
 
-        return Problem(model=model,
-                       condition_df=condition_df,
-                       measurement_df=measurement_df,
-                       parameter_df=parameter_df,
-                       observable_df=observable_df,
-                       visualization_df=visualization_df)
+        return Problem(
+            model=model,
+            condition_df=condition_df,
+            measurement_df=measurement_df,
+            parameter_df=parameter_df,
+            observable_df=observable_df,
+            visualization_df=visualization_df,
+            extensions_config=extensions_config,
+        )
 
     @staticmethod
     def from_yaml(yaml_config: Union[Dict, Path, str]) -> 'Problem':
@@ -258,12 +267,15 @@ def from_yaml(yaml_config: Union[Dict, Path, str]) -> 'Problem':
             observable_files, observables.get_observable_df) \
             if observable_files else None
 
-        return Problem(condition_df=condition_df,
-                       measurement_df=measurement_df,
-                       parameter_df=parameter_df,
-                       observable_df=observable_df,
-                       model=model,
-                       visualization_df=visualization_df)
+        return Problem(
+            condition_df=condition_df,
+            measurement_df=measurement_df,
+            parameter_df=parameter_df,
+            observable_df=observable_df,
+            model=model,
+            visualization_df=visualization_df,
+            extensions_config=yaml_config.get(EXTENSIONS, {})
+        )
 
     @staticmethod
     def from_combine(filename: Union[Path, str]) -> 'Problem':
@@ -281,10 +293,10 @@ def from_combine(filename: Union[Path, str]) -> 'Problem':
         # other SWIG interfaces
         try:
             import libcombine
-        except ImportError:
+        except ImportError as e:
             raise ImportError(
                 "To use PEtab's COMBINE functionality, libcombine "
-                "(python-libcombine) must be installed.")
+                "(python-libcombine) must be installed.") from e
 
         archive = libcombine.CombineArchive()
         if archive.initializeFromArchive(str(filename)) is None:

petab/schemas/petab_schema.v2.0.0.yaml

@@ -0,0 +1,111 @@
+# For syntax see: https://json-schema.org/understanding-json-schema/index.html
+#$schema: "https://json-schema.org/draft/2019-09/meta/core"
+$schema: "http://json-schema.org/draft-06/schema"
+description: PEtab parameter estimation problem config file schema
+
+properties:
+
+  format_version:
+    anyof:
+      - type: string
+        #  (corresponding to PEP 440).
+        pattern: ^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$
+      - type: integer
+
+    description: Version of the PEtab format
+
+  parameter_file:
+    oneOf:
+    - type: string
+    - type: array
+    description: |
+      File name (absolute or relative) or URL to PEtab parameter table
+      containing parameters of all models listed in `problems`. A single
+      table may be split into multiple files and described as an array here.
+  problems:
+    type: array
+    description: |
+      One or multiple PEtab problems (sets of model, condition, observable
+      and measurement files). If different model and data files are
+      independent, they can be specified as separate PEtab problems, which
+      may allow more efficient handling. Files in one problem cannot refer
+      to models entities or data specified inside another problem.
+    items:
+
+      type: object
+      description: |
+        A set of PEtab model, condition, observable and measurement
+        files and optional visualization files.
+      properties:
+
+        sbml_files:
+          type: array
+          description: List of PEtab SBML files.
+
+          items:
+            type: string
+            description: PEtab SBML file name or URL.
+
+        measurement_files:
+          type: array
+          description: List of PEtab measurement files.
+
+          items:
+            type: string
+            description: PEtab measurement file name or URL.
+
+        condition_files:
+          type: array
+          description: List of PEtab condition files.
+
+          items:
+            type: string
+            description: PEtab condition file name or URL.
+
+        observable_files:
+          type: array
+          description: List of PEtab observable files.
+
+          items:
+            type: string
+            description: PEtab observable file name or URL.
+
+        visualization_files:
+          type: array
+          description: List of PEtab visualization files.
+
+          items:
+            type: string
+            description: PEtab visualization file name or URL.
+
+      required:
+        - sbml_files
+        - observable_files
+        - measurement_files
+        - condition_files
+
+  extensions:
+    type: object
+    description: |
+      PEtab extensions being used.
+    patternProperties:
+      "^[a-zA-Z][\\-\\w]*$":
+
+        type: object
+        description: |
+          Information on a specific extension
+        properties:
+          version:
+            type: string
+            pattern: ^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$
+
+        required:
+          - version
+      additionalProperties: true
+
+    additionalProperties: false
+
+required:
+  - format_version
+  - parameter_file
+  - problems

petab/yaml.py

@@ -17,6 +17,7 @@
 SCHEMAS = {
     '1': SCHEMA_DIR / "petab_schema.v1.0.0.yaml",
     '1.0.0': SCHEMA_DIR / "petab_schema.v1.0.0.yaml",
+    '2.0.0': SCHEMA_DIR / "petab_schema.v2.0.0.yaml",
 }
 
 __all__ = ['validate', 'validate_yaml_syntax', 'validate_yaml_semantics',
@@ -66,8 +67,11 @@ def validate_yaml_syntax(
         #  but let's still use the latest PEtab schema for full validation
         version = yaml_config.get(FORMAT_VERSION, None) \
                       or list(SCHEMAS.values())[-1]
-        schema = SCHEMAS[str(version)]
-
+        try:
+            schema = SCHEMAS[str(version)]
+        except KeyError as e:
+            raise ValueError("Unknown PEtab version given in problem "
+                             f"specification: {version}") from e
     schema = load_yaml(schema)
     jsonschema.validate(instance=yaml_config, schema=schema)