Add JSON Schema infrastructure, use for Derivation

For manual, and testing formats

Author: roberth

ci/gha/tests/default.nix

@@ -116,6 +116,7 @@ rec {
     ) nixComponentsInstrumented)
     // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
       "${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests;
+      "${componentTestsPrefix}nix-json-schema-checks" = nixComponentsInstrumented.nix-json-schema-checks;
     };
 
   codeCoverage =

doc/manual/meson.build

@@ -115,6 +115,7 @@ manual = custom_target(
     builtins_md,
     rl_next_generated,
     summary_rl_next,
+    json_schema_generated_files,
     nix_input,
   ],
   output : [

doc/manual/package.nix

@@ -12,6 +12,7 @@
   rsync,
   nix-cli,
   changelog-d,
+  json-schema-for-humans,
   officialRelease,
 
   # Configuration Options
@@ -55,6 +56,7 @@ mkMesonDerivation (finalAttrs: {
     jq
     python3
     rsync
+    json-schema-for-humans
     changelog-d
   ]
   ++ lib.optionals (!officialRelease) [

doc/manual/source/SUMMARY.md.in

@@ -117,6 +117,7 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
+    - [Hash](protocols/json/hash.md)
     - [Store Object Info](protocols/json/store-object-info.md)
     - [Derivation](protocols/json/derivation.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)

doc/manual/source/meson.build

@@ -1,3 +1,6 @@
+# Process JSON schema documentation
+subdir('protocols')
+
 summary_rl_next = custom_target(
   command : [
     bash,

doc/manual/source/protocols/json/derivation.md

@@ -1,120 +1,7 @@
-# Derivation JSON Format
+{{#include derivation-v3-fixed.md}}
 
-> **Warning**
->
-> This JSON format is currently
-> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
-> and subject to change.
+<!--
+## Raw Schema
 
-The JSON serialization of a
-[derivations](@docroot@/glossary.md#gloss-store-derivation)
-is a JSON object with the following fields:
-
-* `name`:
-  The name of the derivation.
-  This is used when calculating the store paths of the derivation's outputs.
-
-* `version`:
-  Must be `3`.
-  This is a guard that allows us to continue evolving this format.
-  The choice of `3` is fairly arbitrary, but corresponds to this informal version:
-
-  - Version 0: A-Term format
-
-  - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
-
-  - Version 2: Separate `method` and `hashAlgo` fields in output specs
-
-  - Version 3: Drop store dir from store paths, just include base name.
-
-  Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
-
-* `outputs`:
-  Information about the output paths of the derivation.
-  This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
-
-  * `path`:
-    The output path, if it is known in advanced.
-    Otherwise, `null`.
-
-
-  * `method`:
-    For an output which will be [content addressed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
-    Valid method strings are:
-
-    - [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
-    - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
-    - [`text`](@docroot@/store/store-object/content-address.md#method-text)
-    - [`git`](@docroot@/store/store-object/content-address.md#method-git)
-
-    Otherwise, `null`.
-
-  * `hashAlgo`:
-    For an output which will be [content addressed], the name of the hash algorithm used.
-    Valid algorithm strings are:
-
-    - `blake3`
-    - `md5`
-    - `sha1`
-    - `sha256`
-    - `sha512`
-
-  * `hash`:
-    For fixed-output derivations, the expected content hash in base-16.
-
-  > **Example**
-  >
-  > ```json
-  > "outputs": {
-  >   "out": {
-  >     "method": "nar",
-  >     "hashAlgo": "sha256",
-  >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
-  >   }
-  > }
-  > ```
-
-* `inputSrcs`:
-  A list of store paths on which this derivation depends.
-
-  > **Example**
-  >
-  > ```json
-  > "inputSrcs": [
-  >   "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
-  >   "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
-  > ]
-  > ```
-
-* `inputDrvs`:
-  A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
-
-  > **Example**
-  >
-  > ```json
-  > "inputDrvs": {
-  >   "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
-  >   "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
-  > }
-  > ```
-
-  specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
-
-* `system`:
-  The system type on which this derivation is to be built
-  (e.g. `x86_64-linux`).
-
-* `builder`:
-  The absolute path of the program to be executed to run the build.
-  Typically this is the `bash` shell
-  (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
-
-* `args`:
-  The command-line arguments passed to the `builder`.
-
-* `env`:
-  The environment passed to the `builder`.
-
-* `structuredAttrs`:
-  [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
-  Structured attributes are JSON, and thus embedded as-is.
+[JSON Schema for Derivation v3](schema/derivation-v3.json)
+-->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -0,0 +1,14 @@
+# For some reason, backticks in the JSON schema are being escaped rather
+# than being kept as intentional code spans. This removes all backtick
+# escaping, which is an ugly solution, but one that is fine, because we
+# are not using backticks for any other purpose.
+s/\\`/`/g
+
+# The way that semi-external references are rendered (i.e. ones to
+# sibling schema files, as opposed to separate website ones, is not nice
+# for humans. Replace it with a nice relative link within the manual
+# instead.
+#
+# As we have more such relative links, more replacements of this nature
+# should appear below.
+s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g

doc/manual/source/protocols/json/hash.md

@@ -0,0 +1,7 @@
+{{#include hash-v1-fixed.md}}
+
+<!--
+## Raw Schema
+
+[JSON Schema for Hash v1](schema/hash-v1.json)
+-->

doc/manual/source/protocols/json/json-schema-for-humans-config.yaml

@@ -0,0 +1,17 @@
+# Configuration file for json-schema-for-humans
+#
+# https://github.com/coveooss/json-schema-for-humans/blob/main/docs/examples/examples_md_default/Configuration.md
+
+template_name: md
+show_toc: true
+# impure timestamp and distracting
+with_footer: false
+recursive_detection_depth: 3
+show_breadcrumbs: false
+description_is_markdown: true
+template_md_options:
+  properties_table_columns:
+    - Property
+    - Type
+    - Pattern
+    - Title/Description

doc/manual/source/protocols/json/meson.build

@@ -0,0 +1,74 @@
+# Tests in: ../../../../src/json-schema-checks
+
+fs = import('fs')
+
+# Find json-schema-for-humans if available
+json_schema_for_humans = find_program('generate-schema-doc', required : false)
+
+# Configuration for json-schema-for-humans
+json_schema_config = files('json-schema-for-humans-config.yaml')
+
+schemas = [
+  'hash-v1',
+  'derivation-v3',
+]
+
+schema_files = files()
+foreach schema_name : schemas
+  schema_files += files('schema' / schema_name + '.yaml')
+endforeach
+
+
+schema_outputs = []
+foreach schema_name : schemas
+  schema_outputs += schema_name + '.md'
+endforeach
+
+json_schema_generated_files = []
+
+# Generate markdown documentation from JSON schema
+# Note: output must be just a filename, not a path
+gen_file = custom_target(
+  schema_name + '-schema-docs.tmp',
+  command : [
+    json_schema_for_humans,
+    '--config-file',
+    json_schema_config,
+    meson.current_source_dir() / 'schema',
+    meson.current_build_dir(),
+  ],
+  input : schema_files + [
+    json_schema_config,
+  ],
+  output : schema_outputs,
+  capture : false,
+  build_by_default : true,
+)
+
+idx = 0
+if json_schema_for_humans.found()
+  foreach schema_name : schemas
+    #schema_file = 'schema' / schema_name + '.yaml'
+
+    # There is one so-so hack, and one horrible hack being done here.
+    sedded_file = custom_target(
+      schema_name + '-schema-docs',
+      command : [
+        'sed',
+        '-f',
+        # Out of line to avoid https://github.com/mesonbuild/meson/issues/1564
+        files('fixup-json-schema-generated-doc.sed'),
+        '@INPUT@',
+      ],
+      capture : true,
+      input : gen_file[idx],
+      output : schema_name + '-fixed.md',
+    )
+    idx += 1
+    json_schema_generated_files += [ sedded_file ]
+  endforeach
+else
+  warning(
+    'json-schema-for-humans not found, skipping JSON schema documentation generation',
+  )
+endif