Add Buildkite schema and fix minor script issues (#217)

1. Add Buildkite to the catalog
2. Add a test-case from the Buildkite repo
3. Fix minor issues in the vendor-schemas script
   - bugfix KeyError when adding new schemas
   - strip trailing whitespace so that pre-commit run doesn't munge
     hash comparison
4. Update generate-hooks-config script to handle 'types_or'
5. Run vendor-schemas and generate-hooks-config

Resolves #198

Author: sirosen

.pre-commit-hooks.yaml

@@ -34,6 +34,20 @@
   files: ^bamboo-specs/.*\.(yml|yaml)$
   types: [yaml]
 
+- id: check-buildkite
+  name: Validate Buildkite Pipelines
+  description: 'Validate Buildkite Pipelines against the schema provided by Buildkite'
+  entry: check-jsonschema --builtin-schema vendor.buildkite
+  language: python
+  files: >
+    (?x)^(
+      buildkite\.(yml|yaml|json)|
+      buildkite\.(.+)\.(yml|yaml|json)|
+      (.*/)?\.buildkite/pipeline\.(yml|yaml|json)|
+      (.*/)?\.buildkite/pipeline\.(.+)\.(yml|yaml|json)
+    )$
+  types_or: [json,yaml]
+
 - id: check-dependabot
   name: Validate Dependabot Config (v2)
   description: 'Validate Dependabot Config (v2) against the schema provided by SchemaStore'

CHANGELOG.rst

@@ -14,6 +14,7 @@ Unreleased
 - Add ``--fill-defaults`` argument which eagerly populates ``"default"``
   values whenever they are encountered and a value is not already present
   (:issue:`200`)
+- Add Buildkite schema and pre-commit hook (:issue:`198`)
 
 0.19.2
 ------

docs/precommit_usage.rst

@@ -71,6 +71,20 @@ Validate Bamboo Specs against the schema provided by SchemaStore
         - id: check-bamboo-spec
 
 
+``check-buildkite``
+~~~~~~~~~~~~~~~~~~~
+
+Validate Buildkite Pipelines against the schema provided by Buildkite
+
+.. code-block:: yaml
+    :caption: example config
+
+    - repo: https://github.com/python-jsonschema/check-jsonschema
+      rev: 0.19.2
+      hooks:
+        - id: check-buildkite
+
+
 ``check-dependabot``
 ~~~~~~~~~~~~~~~~~~~~
 

docs/usage.rst

@@ -70,6 +70,7 @@ SchemaStore and other sources:
 
 - ``vendor.azure-pipelines``
 - ``vendor.bamboo-spec``
+- ``vendor.buildkite``
 - ``vendor.dependabot``
 - ``vendor.github-actions``
 - ``vendor.github-workflows``

scripts/generate-hooks-config.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import importlib.metadata
+import typing as t
 
 from check_jsonschema.catalog import SCHEMA_CATALOG
 
@@ -40,10 +41,21 @@ def update_hook_config(new_config: str) -> None:
         fp.write(content)
 
 
-def format_hook(config) -> str:
+def generate_hook_lines(config) -> t.Iterator[str]:
+    yield f"- id: {config['id']}"
+    yield f"  name: {config['name']}"
+    yield f"  description: '{config['description']}'"
+
     add_args = " ".join(config.get("add_args", []))
     if add_args:
         add_args = " " + add_args
+    yield (
+        "  entry: check-jsonschema --builtin-schema "
+        f"vendor.{config['schema_name']}{add_args}"
+    )
+
+    yield "  language: python"
+
     if isinstance(config["files"], list):
         config[
             "files"
@@ -54,23 +66,23 @@ def format_hook(config) -> str:
             "|\n      ".join(config["files"])
         )
 
-    config_str = f"""\
-- id: {config["id"]}
-  name: {config["name"]}
-  description: '{config["description"]}'
-  entry: check-jsonschema --builtin-schema vendor.{config["schema_name"]}{add_args}
-  language: python
-  files: {config["files"]}
-"""
+    yield f"  files: {config['files']}"
+
     if "types" in config:
-        config_str += f"""\
-  types: [{','.join(config['types'])}]
-"""
-    return config_str
+        yield f"  types: [{','.join(config['types'])}]"
+    if "types_or" in config:
+        yield f"  types_or: [{','.join(config['types_or'])}]"
+
+    yield ""
+
+
+def generate_all_hook_config_lines() -> t.Iterator[str]:
+    for hook_config in iter_catalog_hooks():
+        yield from generate_hook_lines(hook_config)
 
 
-def generate_hook_config() -> str:
-    return "\n".join(format_hook(h) for h in iter_catalog_hooks())
+def format_all_hook_config() -> str:
+    return "\n".join(generate_all_hook_config_lines())
 
 
 def update_usage_list_schemas() -> None:
@@ -139,7 +151,7 @@ def update_docs() -> None:
 
 
 def main() -> None:
-    update_hook_config(generate_hook_config())
+    update_hook_config(format_all_hook_config())
     update_docs()
 
 

scripts/vendor-schemas.py

@@ -50,6 +50,11 @@ def load_old_hashes() -> None:
                 OLD_HASHES[name] = fp.read().strip()
 
 
+def normalize_schema_contents(data: bytes) -> bytes:
+    lines = data.split(b"\n")
+    return b"\n".join(line.rstrip(b" ") for line in lines)
+
+
 def download_schemas() -> None:
     print("downloading schemas to check for updates")
     session = requests.Session()
@@ -59,17 +64,18 @@ def download_schemas() -> None:
 
         print(f"  {schema_name} ({schema_url})")
         res = session.get(schema_url)
+        new_content = normalize_schema_contents(res.content)
+
         sha = hashlib.sha256()
-        sha.update(res.content)
+        sha.update(new_content)
         new_digest = sha.hexdigest()
 
         # early abort if downloaded content matches old hash
         if new_digest == OLD_HASHES.get(schema_name):
             continue
 
-        print("    content changed")
         with open(schema2filename(schema_name), "wb") as fp:
-            fp.write(res.content)
+            fp.write(new_content)
         UPDATED_SCHEMAS.add(schema_name)
 
 
@@ -105,7 +111,7 @@ def _save_new_hash(name: str, digest: str) -> None:
             _save_new_hash(name, digest)
 
         # if the existing hash does not match the new hash, save the new one
-        if digest != OLD_HASHES[name]:
+        elif digest != OLD_HASHES[name]:
             _save_new_hash(name, digest)
 
 

src/check_jsonschema/builtin_schemas/vendor/LICENSE.buildkite

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Buildkite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/check_jsonschema/builtin_schemas/vendor/README.md

@@ -29,3 +29,8 @@ renovatebot
 The GitLab CI schema is provided by GitLab and licensed under the license for
 the public gitlab repo. In particular, it falls under the "MIT Expat" portion
 of the license for that repo.
+
+### Buildkite
+
+The Buildkite schema is provided by Buildkite and licensed under the license
+for their 'pipeline-schema' repo.