scripts: replace pykwalify with jsonschema

Converted `arch-schema.yml`, `board-schema.yml`, and `soc-schema.yml`
from PyYAML format to JSON Schema format, including baking in some of
the validation rules that were deferred to ad hoc Python right into the
schemas (e.g. mutual exclusivity of certain fields).

Signed-off-by: Benjamin Cabé <[email protected]>

Author: kartben

scripts/list_boards.py

@@ -11,21 +11,24 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 
+import jsonschema
 import list_hardware
-import pykwalify.core
 import yaml
+from jsonschema.exceptions import best_match
 from list_hardware import unique_paths
 
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
-BOARD_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'board-schema.yml')
+BOARD_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'board-schema.yaml')
 with open(BOARD_SCHEMA_PATH) as f:
     board_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-BOARD_VALIDATOR = pykwalify.core.Core(schema_data=board_schema, source_data={})
+validator_class = jsonschema.validators.validator_for(board_schema)
+validator_class.check_schema(board_schema)
+board_validator = validator_class(board_schema)
 
 BOARD_YML = 'board.yml'
 
@@ -231,23 +234,14 @@ def load_v2_boards(board_name, board_yml, systems):
         with board_yml.open('r', encoding='utf-8') as f:
             b = yaml.load(f.read(), Loader=SafeLoader)
 
-        try:
-            BOARD_VALIDATOR.source = b
-            BOARD_VALIDATOR.validate()
-        except pykwalify.errors.SchemaError as e:
-            sys.exit(f'ERROR: Malformed "build" section in file: {board_yml.as_posix()}\n{e}')
-
-        mutual_exclusive = {'board', 'boards'}
-        if len(mutual_exclusive - b.keys()) < 1:
-            sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n'
-                     f'{mutual_exclusive} are mutual exclusive at this level.')
+        errors = list(board_validator.iter_errors(b))
+        if errors:
+            sys.exit('ERROR: Malformed board YAML file: '
+                     f'{board_yml.as_posix()}\n'
+                     f'{best_match(errors).message} in {best_match(errors).json_path}')
 
         board_array = b.get('boards', [b.get('board', None)])
         for board in board_array:
-            mutual_exclusive = {'name', 'extend'}
-            if len(mutual_exclusive - board.keys()) < 1:
-                sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
 
             # This is a extending an existing board, place in array to allow later processing.
             if 'extend' in board:
@@ -260,19 +254,6 @@ def load_v2_boards(board_name, board_yml, systems):
                 # Not the board we're looking for, ignore.
                 continue
 
-            board_revision = board.get('revision')
-            if board_revision is not None and board_revision.get('format') != 'custom':
-                if board_revision.get('default') is None:
-                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                             "Cannot find required key 'default'. Path: '/board/revision.'")
-                if board_revision.get('revisions') is None:
-                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                             "Cannot find required key 'revisions'. Path: '/board/revision.'")
-
-            mutual_exclusive = {'socs', 'variants'}
-            if len(mutual_exclusive - board.keys()) < 1:
-                sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
             socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', []))
                     for s in board.get('socs', {})]
 

scripts/list_hardware.py

@@ -9,26 +9,31 @@
 from dataclasses import dataclass
 from pathlib import Path, PurePath
 
-import pykwalify.core
+import jsonschema
 import yaml
+from jsonschema.exceptions import best_match
 
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
 
-SOC_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'soc-schema.yml')
+SOC_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'soc-schema.yaml')
 with open(SOC_SCHEMA_PATH) as f:
     soc_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-SOC_VALIDATOR = pykwalify.core.Core(schema_data=soc_schema, source_data={})
-
-ARCH_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'arch-schema.yml')
+ARCH_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'arch-schema.yaml')
 with open(ARCH_SCHEMA_PATH) as f:
     arch_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-ARCH_VALIDATOR = pykwalify.core.Core(schema_data=arch_schema, source_data={})
+validator_class = jsonschema.validators.validator_for(soc_schema)
+validator_class.check_schema(soc_schema)
+soc_validator = validator_class(soc_schema)
+
+validator_class = jsonschema.validators.validator_for(arch_schema)
+validator_class.check_schema(arch_schema)
+arch_validator = validator_class(arch_schema)
 
 SOC_YML = 'soc.yml'
 ARCHS_YML_PATH = PurePath('arch/archs.yml')
@@ -44,12 +49,12 @@ def __init__(self, folder='', soc_yaml=None):
         if soc_yaml is None:
             return
 
-        try:
-            data = yaml.load(soc_yaml, Loader=SafeLoader)
-            SOC_VALIDATOR.source = data
-            SOC_VALIDATOR.validate()
-        except (yaml.YAMLError, pykwalify.errors.SchemaError) as e:
-            sys.exit(f'ERROR: Malformed yaml {soc_yaml.as_posix()}', e)
+        data = yaml.load(soc_yaml, Loader=SafeLoader)
+        errors = list(soc_validator.iter_errors(data))
+        if errors:
+            sys.exit('ERROR: Malformed soc YAML file: \n'
+                        f'{soc_yaml}\n'
+                        f'{best_match(errors).message} in {best_match(errors).json_path}')
 
         for f in data.get('family', []):
             family = Family(f['name'], [folder], [], [])
@@ -82,10 +87,6 @@ def __init__(self, folder='', soc_yaml=None):
             self._socs.extend(socs)
 
         for soc in data.get('socs', []):
-            mutual_exclusive = {'name', 'extend'}
-            if len(mutual_exclusive - soc.keys()) < 1:
-                sys.exit(f'ERROR: Malformed content in SoC file: {soc_yaml}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
             if soc.get('name') is not None:
                 self._socs.append(Soc(soc['name'], [c['name'] for c in soc.get('cpuclusters', [])],
                                   [folder], '', ''))
@@ -94,8 +95,9 @@ def __init__(self, folder='', soc_yaml=None):
                                            [c['name'] for c in soc.get('cpuclusters', [])],
                                            [folder], '', ''))
             else:
+                # This should not happen if schema validation passed
                 sys.exit(f'ERROR: Malformed "socs" section in SoC file: {soc_yaml}\n'
-                         f'Cannot find one of required keys {mutual_exclusive}.')
+                         f'SoC entry must have either "name" or "extend" property.')
 
         # Ensure that any runner configuration matches socs and cpuclusters declared in the same
         # soc.yml file
@@ -217,11 +219,11 @@ def find_v2_archs(args):
             with Path(archs_yml).open('r', encoding='utf-8') as f:
                 archs = yaml.load(f.read(), Loader=SafeLoader)
 
-            try:
-                ARCH_VALIDATOR.source = archs
-                ARCH_VALIDATOR.validate()
-            except pykwalify.errors.SchemaError as e:
-                sys.exit(f'ERROR: Malformed "build" section in file: {archs_yml.as_posix()}\n{e}')
+            errors = list(arch_validator.iter_errors(archs))
+            if errors:
+                sys.exit('ERROR: Malformed arch YAML file: '
+                         f'{archs_yml.as_posix()}\n'
+                         f'{best_match(errors).message} in {best_match(errors).json_path}')
 
             if args.arch is not None:
                 archs = {'archs': list(filter(

scripts/requirements-actions.in

@@ -10,6 +10,7 @@ gitlint-core>=0.19.1
 gitpython>=3.1.41
 ijson
 intelhex
+jsonschema
 junit2html
 junitparser>=4.0.1
 mypy