feat: HiFa v1.1.0 schema development

.pre-commit-config.yaml

@@ -61,7 +61,7 @@ repos:
         name: mypy with Python 3.8
         files: src
         additional_dependencies:
-          ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging']
+          ['numpy', 'types-tqdm', 'click', 'types-jsonpatch', 'types-pyyaml', 'types-jsonschema', 'importlib_metadata', 'packaging', 'importlib_resources']
         args: ["--python-version=3.8"]
       - <<: *mypy
         name: mypy with Python 3.11

pyproject.toml

@@ -46,6 +46,7 @@ classifiers = [
 dependencies = [
     "click>=8.0.0",  # for console scripts
     "importlib_resources>=1.4.0; python_version < '3.9'",  # for resources in schema
+    "typing_extensions; python_version < '3.11'", # for typing
     "jsonpatch>=1.15",
     "jsonschema>=4.15.0",  # for utils
     "pyyaml>=5.1",  # for parsing CLI equal-delimited options
@@ -264,7 +265,6 @@ module = [
   'pyhf.modifiers.*',
   'pyhf.exceptions.*',
   'pyhf.parameters.*',
-  'pyhf.schema.*',
   'pyhf.writexml',
   'pyhf.workspace',
   'pyhf.patchset',

src/pyhf/cli/cli.py

@@ -4,7 +4,7 @@
 import click
 
 from pyhf import __version__
-from pyhf.cli import rootio, spec, infer, patchset, complete
+from pyhf.cli import rootio, spec, infer, patchset, complete, upgrade
 from pyhf.contrib import cli as contrib
 from pyhf import utils
 
@@ -56,3 +56,5 @@ def pyhf():
 pyhf.add_command(complete.cli)
 
 pyhf.add_command(contrib.cli)
+
+pyhf.add_command(upgrade.cli)

src/pyhf/cli/upgrade.py

@@ -0,0 +1,72 @@
+"""The pyhf upgrade CLI subcommand."""
+import logging
+
+import click
+import json
+
+from pyhf.schema.upgrader import upgrade
+
+log = logging.getLogger(__name__)
+
+
+@click.group(name='upgrade')
+def cli():
+    """Operations for upgrading specifications."""
+
+
+@cli.command()
+@click.argument('workspace', default='-')
+@click.option(
+    '--version',
+    help='The version to upgrade to',
+    default=None,
+)
+@click.option(
+    '--output-file',
+    help='The location of the output json file. If not specified, prints to screen.',
+    default=None,
+)
+def workspace(workspace, version, output_file):
+    """
+    Upgrade a HistFactory JSON workspace.
+    """
+    with click.open_file(workspace, 'r', encoding="utf-8") as specstream:
+        spec = json.load(specstream)
+
+    ws = upgrade(to_version=version).workspace(spec)
+
+    if output_file is None:
+        click.echo(json.dumps(ws, indent=4, sort_keys=True))
+    else:
+        with open(output_file, 'w+', encoding="utf-8") as out_file:
+            json.dump(ws, out_file, indent=4, sort_keys=True)
+        log.debug(f"Written to {output_file:s}")
+
+
+@cli.command()
+@click.argument('patchset', default='-')
+@click.option(
+    '--version',
+    help='The version to upgrade to',
+    default=None,
+)
+@click.option(
+    '--output-file',
+    help='The location of the output json file. If not specified, prints to screen.',
+    default=None,
+)
+def patchset(patchset, version, output_file):
+    """
+    Upgrade a pyhf JSON PatchSet.
+    """
+    with click.open_file(patchset, 'r', encoding="utf-8") as specstream:
+        spec = json.load(specstream)
+
+    ps = upgrade(to_version=version).patchset(spec)
+
+    if output_file is None:
+        click.echo(json.dumps(ps, indent=4, sort_keys=True))
+    else:
+        with open(output_file, 'w+', encoding="utf-8") as out_file:
+            json.dump(ps, out_file, indent=4, sort_keys=True)
+        log.debug(f"Written to {output_file:s}")

src/pyhf/readxml.py

@@ -473,7 +473,7 @@ def parse(
         'measurements': measurements,
         'channels': channels,
         'observations': observations,
-        'version': schema.version,  # type: ignore[typeddict-unknown-key]
+        'version': schema.versions['workspace.json'],  # type: ignore[attr-defined]
     }
     try:
         schema.validate(result, 'workspace.json')

src/pyhf/schema/__init__.py

@@ -1,25 +1,31 @@
 """
 See :class:`~pyhf.schema.Schema` for documentation.
 """
-import pathlib
+from __future__ import annotations
 import sys
+from typing import Any
 from pyhf.schema.loader import load_schema
 from pyhf.schema.validator import validate
 from pyhf.schema import variables
+from pyhf.typing import Self, SchemaVersion, PathOrStr, Traversable
+from pyhf.schema.upgrader import upgrade
+
+from pathlib import Path
 
 __all__ = [
     "load_schema",
     "validate",
     "path",
     "version",
+    "upgrade",
 ]
 
 
-def __dir__():
+def __dir__() -> list[str]:
     return __all__
 
 
-class Schema(sys.modules[__name__].__class__):
+class Schema(sys.modules[__name__].__class__):  # type: ignore[misc]
     """
     A module-level wrapper around :mod:`pyhf.schema` which will provide additional functionality for interacting with schemas.
 
@@ -61,7 +67,7 @@ class Schema(sys.modules[__name__].__class__):
 
     """
 
-    def __call__(self, new_path: pathlib.Path):
+    def __call__(self, new_path: PathOrStr) -> Self:
         """
         Change the local search path for finding schemas locally.
 
@@ -71,15 +77,15 @@ def __call__(self, new_path: pathlib.Path):
         Returns:
             self (pyhf.schema.Schema): Returns itself (for contextlib management)
         """
-        self.orig_path, variables.schemas = variables.schemas, new_path
+        self.orig_path, variables.schemas = variables.schemas, Path(new_path)
         self.orig_cache = dict(variables.SCHEMA_CACHE)
         variables.SCHEMA_CACHE.clear()
         return self
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         pass
 
-    def __exit__(self, *args, **kwargs):
+    def __exit__(self, *args: Any, **kwargs: Any) -> None:
         """
         Reset the local search path for finding schemas locally.
 
@@ -90,14 +96,14 @@ def __exit__(self, *args, **kwargs):
         variables.SCHEMA_CACHE = self.orig_cache
 
     @property
-    def path(self):
+    def path(self) -> Traversable | Path:
         """
         The local path for schemas.
         """
         return variables.schemas
 
     @property
-    def version(self):
+    def versions(self) -> dict[str, SchemaVersion]:
         """
         The default version used for finding schemas.
         """

src/pyhf/schema/loader.py

@@ -3,6 +3,7 @@
 import json
 import pyhf.exceptions
 from pyhf.schema import variables
+from pyhf.typing import Schema
 
 # importlib.resources.as_file wasn't added until Python 3.9
 # c.f. https://docs.python.org/3.9/library/importlib.html#importlib.resources.as_file
@@ -12,7 +13,7 @@
     import importlib_resources as resources
 
 
-def load_schema(schema_id: str):
+def load_schema(schema_id: str) -> Schema:
     """
     Get a schema by relative path from cache, or load it into the cache and return.
 
@@ -54,9 +55,3 @@ def load_schema(schema_id: str):
             schema = json.load(json_schema)
             variables.SCHEMA_CACHE[schema['$id']] = schema
         return variables.SCHEMA_CACHE[schema['$id']]
-
-
-# pre-populate the cache to avoid network access
-# on first validation in standard usage
-# (not in pyhf.schema.variables to avoid circular imports)
-load_schema(f'{variables.SCHEMA_VERSION}/defs.json')

src/pyhf/schema/upgrader.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from pyhf.typing import Workspace, PatchSet, SchemaVersion, UpgradeProtocol
+import copy
+
+
+class Upgrade_1_0_1:
+    """
+    Used for testing functionality of upgrade.
+    """
+
+    version: SchemaVersion = '1.0.1'
+
+    @classmethod
+    def workspace(cls, spec: Workspace) -> Workspace:
+        """
+        Upgrade the provided workspace specification.
+
+        Args:
+            spec (dict): The specification to validate.
+            schema_name (str): The name of the schema to upgrade.
+
+        Returns:
+            upgraded_spec (dict): Upgraded workspace specification.
+
+        Raises:
+            pyhf.exceptions.InvalidSpecification: the specification is invalid
+        """
+
+        new_spec = copy.deepcopy(spec)
+        if spec['version'] == '1.0.0':
+            new_spec['version'] = cls.version
+        return new_spec
+
+    @classmethod
+    def patchset(cls, spec: PatchSet) -> PatchSet:
+        """
+        Upgrade the provided patchset specification.
+
+        Args:
+            spec (dict): The specification to validate.
+            schema_name (str): The name of the schema to upgrade.
+
+        Returns:
+            upgraded_spec (dict): Upgraded patchset specification.
+
+        Raises:
+            pyhf.exceptions.InvalidSpecification: the specification is invalid
+        """
+
+        new_spec = copy.deepcopy(spec)
+        if spec['version'] == '1.0.0':
+            new_spec['version'] = cls.version
+        return new_spec
+
+
+def upgrade(*, to_version: SchemaVersion | None = None) -> type[UpgradeProtocol]:
+    if to_version is None or to_version == '1.0.1':
+        return Upgrade_1_0_1
+
+    raise ValueError(f'{to_version} is not a valid version to upgrade to.')