Merge pull request #13980 from obsidiansystems/drv-json-issue-13570

Make the JSON format for derivation use basename store paths

Author: roberth

doc/manual/rl-next/derivation-json.md

@@ -0,0 +1,17 @@
+---
+synopsis: Derivation JSON format now uses store path basenames (no store dir) only
+prs: [13980]
+issues: [13570]
+---
+
+Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde
+in Rust, and Aeson in Haskell), has show that the use of the store dir
+in JSON formats is an impediment to systematic JSON formats, because it
+requires the serializer/deserializer to take an extra paramater (the
+store dir).
+
+We ultimately want to rectify this issue with all (non-stable, able to
+be changed) JSON formats. To start with, we are changing the JSON format
+for derivations because the `nix derivation` commands are --- in
+addition to being formally unstable --- less widely used than other
+unstable commands.

doc/manual/source/protocols/json/derivation.md

@@ -14,6 +14,21 @@ is a JSON object with the following fields:
   The name of the derivation.
   This is used when calculating the store paths of the derivation's outputs.
 
+* `version`:
+  Must be `3`.
+  This is a guard that allows us to continue evolving this format.
+  The choice of `3` is fairly arbitrary, but corresponds to this informal version:
+
+  - Version 0: A-Term format
+
+  - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
+
+  - Version 2: Separate `method` and `hashAlgo` fields in output specs
+
+  - Verison 3: Drop store dir from store paths, just include base name.
+
+  Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
+
 * `outputs`:
   Information about the output paths of the derivation.
   This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
@@ -52,7 +67,6 @@ is a JSON object with the following fields:
   > ```json
   > "outputs": {
   >   "out": {
-  >     "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
   >     "method": "nar",
   >     "hashAlgo": "sha256",
   >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
@@ -63,15 +77,24 @@ is a JSON object with the following fields:
 * `inputSrcs`:
   A list of store paths on which this derivation depends.
 
+  > **Example**
+  >
+  > ```json
+  > "inputSrcs": [
+  >   "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
+  >   "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
+  > ]
+  > ```
+
 * `inputDrvs`:
   A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
 
   > **Example**
   >
   > ```json
   > "inputDrvs": {
-  >   "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
-  >   "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  >   "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+  >   "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
   > }
   > ```
 

src/libstore-c/nix_api_store.cc

@@ -181,7 +181,7 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto drv = nix::Derivation::fromJSON(*store->ptr, nlohmann::json::parse(json));
+        auto drv = static_cast<nix::Derivation>(nlohmann::json::parse(json));
 
         auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true);
 

src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json

@@ -21,5 +21,6 @@
       "method": "nar"
     }
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json

@@ -32,5 +32,6 @@
     ],
     "system": "my-system"
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -10,14 +10,14 @@
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
   },
   "inputDrvs": {
-    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+    "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
         "out"
       ]
     },
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
@@ -26,7 +26,7 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
   ],
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
@@ -100,5 +100,6 @@
     ],
     "system": "my-system"
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -26,14 +26,14 @@
     "system": "my-system"
   },
   "inputDrvs": {
-    "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
+    "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
         "out"
       ]
     },
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": {
       "dynamicOutputs": {},
       "outputs": [
         "dev",
@@ -42,7 +42,7 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+    "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
   ],
   "name": "advanced-attributes",
   "outputs": {
@@ -51,5 +51,6 @@
       "method": "nar"
     }
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }

src/libstore-tests/data/derivation/ca/self-contained.json

@@ -19,5 +19,6 @@
       "method": "nar"
     }
   },
-  "system": "x86_64-linux"
+  "system": "x86_64-linux",
+  "version": 3
 }

src/libstore-tests/data/derivation/dynDerivationDeps.json

@@ -8,7 +8,7 @@
     "BIG_BAD": "WOLF"
   },
   "inputDrvs": {
-    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
+    "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
       "dynamicOutputs": {
         "cat": {
           "dynamicOutputs": {},
@@ -30,9 +30,10 @@
     }
   },
   "inputSrcs": [
-    "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
+    "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
   ],
   "name": "dyn-dep-derivation",
   "outputs": {},
-  "system": "wasm-sel4"
+  "system": "wasm-sel4",
+  "version": 3
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json

@@ -15,8 +15,9 @@
   "name": "advanced-attributes-defaults",
   "outputs": {
     "out": {
-      "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"
+      "path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"
     }
   },
-  "system": "my-system"
+  "system": "my-system",
+  "version": 3
 }