refactor: greatly improves caching of rust packages (#144)

minikin · web-flow · commit 83195007f00f · 2022-12-13T23:24:51.000+01:00
# Problem

The problem is broken into two separate parts. The first is that the
current Nix structure in the repository, with all packages being placed
into the `packages` cell block, does not reflect the actual contents of
the files. For example, in many cases, these packages are just Rust
libraries that produce zero artifacts during the build process. This
makes qualifying them as packages ambiguous at best and potentially
harmful at worst.

The second relates to the overall repository structure and how it
contributes to a problematic packaging experience. The repository is set
up as a cargo workspace where individual workspace members share the
cargo lockfile located at the workspace root. This is problematic in the
Nix ecosystem, which relies on lockfiles for reproducibly building
projects. The existing build process (which was copied over before the
`std` conversion) relied on building workspace members using this
lockfile which inadvertently added a dependency to the whole repository
into every single build. In other words, a change in one workspace
member would cause **all other workspace members** input hashes to
change, thus causing Nix to rebuild them unnecessarily. This is
particularly problematic when it comes to having proper caching. In the
current state, we effectively cannot reuse cache artifacts across
commits due to this unwanted dependency.

# Solution

The solution to the first stated problem is to refactor packages that
are strictly libraries into a separate cell block entitled `libraries`.
This at least clearly delineates these as packages that won't produce an
artifact (as is the default case with cargo libraries). However, the
utility of packaging these workspace members is still questionable. The
only real benefit I can foresee is possibly running unit tests during
the build process (which we are not currently doing). We should have a
follow-up action to discuss if these should just be removed instead to
reduce the size of the Nix codebase.

The solution to the second problem is a bit more creative. As noted, the
general structure of the repository, unfortunately, makes avoiding
bringing in unnecessary dependencies into the derivation closure
difficult. To get around this, the `mkPackage` function has been
reworked to minimize the number of unnecessary inputs being fed into the
underlying derivation. This is accomplished by working out the local
dependencies of a given package and only including those dependencies in
the final closure. This ensures that rebuilds will only happen when the
package or one of its dependencies changes.

In addition to the above two changes, the substitutes in the `flake.nix`
were updated to point to their proper values. This was causing some
builds to take over six hours because Nix was trying to "build the
world." It's worth nothing that this is primarily due to `voting-tools`
and its Haskell dependencies which are known to be unique and thus
heavily rely on an existing cache.
diff --git a/flake.nix b/flake.nix
@@ -36,40 +36,50 @@
         (std.blockTypes.functions "constants")
         (std.blockTypes.functions "lib")
         (std.blockTypes.functions "toolchains")
+        (std.blockTypes.installables "artifacts")
+        (std.blockTypes.installables "libraries")
         (std.blockTypes.installables "packages")
         (std.blockTypes.nixago "configs")
         (std.blockTypes.runnables "operables")
       ];
     }
     {
       devShells = std.harvest inputs.self ["automation" "devshells"];
+      artifacts = std.harvest inputs.self [
+        ["artifacts" "artifacts"]
+      ];
+      containers = std.harvest inputs.self [
+        ["jormungandr" "containers"]
+        ["vit-servicing-station" "containers"]
+        ["vit-testing" "containers"]
+      ];
+      libraries = std.harvest inputs.self [
+        ["catalyst-toolbox" "libraries"]
+        ["chain-libs" "libraries"]
+        ["chain-wallet-libs" "libraries"]
+        ["jormungandr" "libraries"]
+        ["jortestkit" "libraries"]
+        ["vit-servicing-station" "libraries"]
+        ["vit-testing" "libraries"]
+      ];
       packages = std.harvest inputs.self [
-        ["artifacts" "packages"]
         ["catalyst-toolbox" "packages"]
-        ["chain-libs" "packages"]
-        ["chain-wallet-libs" "packages"]
         ["jormungandr" "packages"]
-        ["jortestkit" "packages"]
         ["vit-servicing-station" "packages"]
         ["vit-testing" "packages"]
         ["voting-tools" "packages"]
         ["voting-tools-rs" "packages"]
       ];
-      containers = std.harvest inputs.self [
-        ["jormungandr" "containers"]
-        ["vit-servicing-station" "containers"]
-        ["vit-testing" "containers"]
-      ];
     };
 
   nixConfig = {
     extra-substituters = [
-      #"https://hydra.iohk.io"
-      "https://iog-gov-nix.s3.eu-central-1.amazonaws.com"
+      "https://cache.iog.io"
+      "https://iog-catalyst-cache.s3.eu-central-1.amazonaws.com"
     ];
     extra-trusted-public-keys = [
       "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="
-      "gov:uG8+LG8RqFGScUmOrDkGb4VCbtNhChbnycVnxZxb8AY="
+      "catalyst:kNW0n7ijUJDvu4BrpqC3j54rgoHNccXx7ABuVzuL9WM="
     ];
     allow-import-from-derivation = "true";
   };
diff --git a/nix/artifacts/artifacts.nix b/nix/artifacts/artifacts.nix
diff --git a/nix/catalyst-toolbox/libraries.nix b/nix/catalyst-toolbox/libraries.nix
@@ -0,0 +1,13 @@
+{
+  inputs,
+  cell,
+}: let
+  inherit (inputs) nixpkgs std;
+  inherit (inputs.cells.lib) lib;
+  l = nixpkgs.lib // builtins;
+
+  name = "catalyst-toolbox";
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
+in {
+  snapshot-lib = mkSimplePkg "snapshot-lib";
+}
diff --git a/nix/catalyst-toolbox/packages.nix b/nix/catalyst-toolbox/packages.nix
@@ -7,15 +7,12 @@
   l = nixpkgs.lib // builtins;
 
   name = "catalyst-toolbox";
-  root = inputs.self + "/src/${name}";
-
-  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = root + "/${subPkg}";};
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
   catalyst-toolbox = lib.mkPackage {
-    pkgPath = root + "/catalyst-toolbox";
+    pkgPath = "${name}/catalyst-toolbox";
     nativeBuildInputs = with nixpkgs; [
       postgresql.lib
     ];
   };
-  snapshot-lib = mkSimplePkg "snapshot-lib";
 }
diff --git a/nix/chain-libs/libraries.nix b/nix/chain-libs/libraries.nix
@@ -7,9 +7,7 @@
   l = nixpkgs.lib // builtins;
 
   name = "chain-libs";
-  root = inputs.self + "/src/${name}";
-
-  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = root + "/${subPkg}";};
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
   chain-addr = mkSimplePkg "chain-addr";
   cardano-legacy-address = mkSimplePkg "cardano-legacy-address";
diff --git a/nix/chain-wallet-libs/libraries.nix b/nix/chain-wallet-libs/libraries.nix
@@ -7,16 +7,13 @@
   l = nixpkgs.lib // builtins;
 
   name = "chain-wallet-libs";
-  root = inputs.self + "/src/${name}";
-
-  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = root + "/${subPkg}";};
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
-  bip39 = mkSimplePkg "bip39";
   chain-path-derivation = mkSimplePkg "chain-path-derivation";
   hdkeygen = mkSimplePkg "hdkeygen";
   symmetric-cipher = mkSimplePkg "symmetric-cipher";
   wallet = mkSimplePkg "wallet";
   wallet-c = mkSimplePkg "bindings/wallet-c";
   wallet-core = mkSimplePkg "bindings/wallet-core";
-  wallet-js = mkSimplePkg "bindings/wallet-js";
+  wallet-wasm-js = mkSimplePkg "bindings/wallet-wasm-js";
 }
diff --git a/nix/jormungandr/libraries.nix b/nix/jormungandr/libraries.nix
@@ -0,0 +1,20 @@
+{
+  inputs,
+  cell,
+}: let
+  inherit (inputs) nixpkgs std;
+  inherit (inputs.cells.lib) lib;
+  l = nixpkgs.lib // builtins;
+
+  name = "jormungandr";
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
+in {
+  blockchain = mkSimplePkg "modules/blockchain";
+  explorer = mkSimplePkg "explorer";
+  hersir = mkSimplePkg "testing/hersir";
+  jormungandr-lib = mkSimplePkg "jormungandr-lib";
+  loki = mkSimplePkg "testing/loki";
+  mjolnir = mkSimplePkg "testing/mjolnir";
+  settings = mkSimplePkg "modules/settings";
+  thor = mkSimplePkg "testing/thor";
+}
diff --git a/nix/jormungandr/operables.nix b/nix/jormungandr/operables.nix
@@ -3,12 +3,12 @@
   cell,
 }: let
   inherit (inputs) nixpkgs std;
-  inherit (inputs.cells) artifacts;
+  inherit (inputs.cells.artifacts) artifacts;
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
   mkOperable = package: namespace: let
-    artifacts' = artifacts.packages."artifacts-${namespace}";
+    artifacts' = artifacts."artifacts-${namespace}";
   in
     std.lib.ops.mkOperable {
       inherit package;
diff --git a/nix/jormungandr/packages.nix b/nix/jormungandr/packages.nix
@@ -7,24 +7,14 @@
   l = nixpkgs.lib // builtins;
 
   name = "jormungandr";
-  root = inputs.self + "/src/${name}";
-
-  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = root + "/${subPkg}";};
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
   jormungandr = lib.mkPackage {
-    pkgPath = root + "/jormungandr";
+    pkgPath = "${name}/jormungandr";
     cargoOptions = [
       "--features"
       "prometheus-metrics"
     ];
   };
-  blockchain = mkSimplePkg "modules/blockchain";
-  explorer = mkSimplePkg "explorer";
-  hersir = mkSimplePkg "testing/hersir";
   jcli = mkSimplePkg "jcli";
-  jormungandr-lib = mkSimplePkg "jormungandr-lib";
-  loki = mkSimplePkg "testing/loki";
-  mjolnir = mkSimplePkg "testing/mjolnir";
-  settings = mkSimplePkg "modules/settings";
-  thor = mkSimplePkg "testing/thor";
 }
diff --git a/nix/jortestkit/libraries.nix b/nix/jortestkit/libraries.nix
@@ -7,9 +7,7 @@
   l = nixpkgs.lib // builtins;
 
   name = "jortestkit";
-  root = inputs.self + "/src/${name}";
-
-  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = root + "/${subPkg}";};
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
   jortestkit = mkSimplePkg "";
 }
diff --git a/nix/lib/lib.nix b/nix/lib/lib.nix
@@ -3,39 +3,109 @@
   cell,
 }: let
   inherit (inputs) nixpkgs std;
-  inherit (cell.toolchains) naersk;
+  inherit (cell.toolchains) naersk rustToolchain;
   l = nixpkgs.lib // builtins;
 in rec {
   readTOML = file: l.fromTOML (l.readFile file);
+
+  # Combines root + path1 + path2 and then resolves any relative references
+  # between them (i.e. root/foo/bar/../baz -> root/foo/baz)
+  reifyPath = root: path1: path2: let
+    absPath = l.toPath (root + path1 + "/${path2}");
+    relPath = l.replaceStrings [root] [""] absPath;
+  in
+    relPath;
+
+  # Given the path to the root of a cargo project, recursively fetches all local
+  # dependencies (those specified by `path = ...`) and returns a list of their
+  # deduplicated relative paths from the root.
+  recurseFetchDeps = path: let
+    toml = readTOML (inputs.self + "/src/" + path + "/Cargo.toml");
+    deps = l.filterAttrs (k: v: v ? path) (
+      if toml ? dependencies
+      then toml.dependencies
+      else {}
+    );
+    paths =
+      l.mapAttrsToList (
+        k: v:
+          reifyPath (inputs.self + "/src/") path v.path
+      )
+      deps;
+    allPaths = (l.map (p: recurseFetchDeps p) paths) ++ [path];
+  in
+    l.unique (l.flatten allPaths);
+
+  # Recreates a cargo workspace by accepting a partial workspace representation
+  # (src) and then initializing all workspace members that are not present in
+  # the partial workspace.
+  #
+  # This is required in order to limit the input when compiling workspace
+  # members. By only including required dependencies, and initializing the rest
+  # to a known state, changes to derivation inputs can be reduced.
+  mkDummySrc = members: src:
+    nixpkgs.runCommand "source" {}
+    ''
+      cp -r ${src} tmp
+      chmod -R 0744 tmp
+
+      for c in ${l.concatStringsSep " " members}
+      do
+        if [[ ! -f tmp/$c/Cargo.toml ]]; then
+          mkdir -p "tmp/$c"
+          ${rustToolchain}/bin/cargo init tmp/$c
+        fi
+      done
+
+      cp -r tmp $out
+    '';
+
+  # Given the path to a cargo project (relative from src/), returns a derivation
+  # for building the project that utilizes the workspace lockfile but also
+  # filters the inputs to only include required dependencies. This improves
+  # caching by reducing the amount the generated output hash changes.
   mkPackage = {
     pkgPath,
     cargoOptions ? [],
     nativeBuildInputs ? [],
   }: let
-    pkgCargo = readTOML (pkgPath + "/Cargo.toml");
+    rootPkgCargo = readTOML (inputs.self + "/Cargo.toml");
+    pkgCargo = readTOML (inputs.self + "/src/" + pkgPath + "/Cargo.toml");
     inherit (pkgCargo.package) name version;
 
-    cargoOptions' = ["--package" "${name}"] ++ cargoOptions;
+    deps = l.map (d: "src/${d}") (recurseFetchDeps pkgPath);
+    filteredSrc = std.incl inputs.self ([
+        "Cargo.toml"
+        "Cargo.lock"
+      ]
+      ++ deps);
+
     nativeBuildInputs' = with nixpkgs;
       [
         pkg-config
         protobuf
         rustfmt
+        postgresql.lib
       ]
-      # TODO: move this
-      ++ (pkgs.lib.optionals (name == "voting_tools_rs" || name == "vit-servicing-station-server") [
-        postgresql
-      ])
       ++ nativeBuildInputs;
   in
     naersk.buildPackage {
       inherit name version;
 
+      # We have to invoke cargo from within the member's directory
+      preBuild = "cd src/${pkgPath}";
+
+      # The output artifacts are stored in the workspace root, failing to change
+      # back will result in naesrk failing to find the artifacts.
+      postBuild = "cd -";
+
       root = inputs.self;
+      src = mkDummySrc rootPkgCargo.workspace.members filteredSrc;
+
       nativeBuildInputs = nativeBuildInputs';
 
-      cargoBuildOptions = x: x ++ cargoOptions';
-      cargoTestOptions = x: x ++ cargoOptions';
+      cargoBuildOptions = x: x ++ cargoOptions;
+      cargoTestOptions = x: x ++ cargoOptions;
 
       PROTOC = "${nixpkgs.protobuf}/bin/protoc";
       PROTOC_INCLUDE = "${nixpkgs.protobuf}/include";
diff --git a/nix/vit-servicing-station/libraries.nix b/nix/vit-servicing-station/libraries.nix
@@ -0,0 +1,14 @@
+{
+  inputs,
+  cell,
+}: let
+  inherit (inputs) nixpkgs std;
+  inherit (inputs.cells.lib) lib;
+  l = nixpkgs.lib // builtins;
+
+  name = "vit-servicing-station";
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
+in {
+  vit-servicing-station-lib = mkSimplePkg "vit-servicing-station-lib";
+  vit-servicing-station-tests = mkSimplePkg "vit-servicing-station-tests";
+}
diff --git a/nix/vit-servicing-station/operables.nix b/nix/vit-servicing-station/operables.nix
@@ -3,12 +3,12 @@
   cell,
 }: let
   inherit (inputs) nixpkgs std;
-  inherit (inputs.cells) artifacts;
+  inherit (inputs.cells.artifacts) artifacts;
   inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
   mkVitOperable = package: namespace: let
-    artifacts' = artifacts.packages."artifacts-${namespace}";
+    artifacts' = artifacts."artifacts-${namespace}";
   in
     std.lib.ops.mkOperable {
       inherit package;
diff --git a/nix/vit-servicing-station/packages.nix b/nix/vit-servicing-station/packages.nix
@@ -7,18 +7,8 @@
   l = nixpkgs.lib // builtins;
 
   name = "vit-servicing-station";
-  root = inputs.self + "/src/${name}";
-
-  mkVitPkg = subPkg:
-    lib.mkPackage {
-      pkgPath = root + "/${subPkg}";
-      nativeBuildInputs = with nixpkgs; [
-        postgresql.lib
-      ];
-    };
+  mkSimplePkg = subPkg: lib.mkPackage {pkgPath = "${name}/${subPkg}";};
 in {
-  vit-servicing-station-cli = mkVitPkg "vit-servicing-station-cli";
-  vit-servicing-station-lib = mkVitPkg "vit-servicing-station-lib";
-  vit-servicing-station-server = mkVitPkg "vit-servicing-station-server";
-  vit-servicing-station-tests = mkVitPkg "vit-servicing-station-tests";
+  vit-servicing-station-cli = mkSimplePkg "vit-servicing-station-cli";
+  vit-servicing-station-server = mkSimplePkg "vit-servicing-station-server";
 }
diff --git a/nix/vit-testing/containers.nix b/nix/vit-testing/containers.nix
@@ -3,16 +3,15 @@
   cell,
 }: let
   inherit (inputs) nixpkgs std;
+  inherit (inputs.cells.lib) constants;
   l = nixpkgs.lib // builtins;
 
-  name = "voting-center-backend";
-  operable = cell.operables.voting-center-backend;
-
   mkOCI = name: let
     operable = cell.operables.${name};
   in
     std.lib.ops.mkStandardOCI {
-      inherit name operable;
+      inherit operable;
+      name = "${constants.registry}/${name}";
     };
 in {
   snapshot-trigger-service = mkOCI "snapshot-trigger-service";
diff --git a/nix/vit-testing/libraries.nix b/nix/vit-testing/libraries.nix
diff --git a/nix/vit-testing/packages.nix b/nix/vit-testing/packages.nix
diff --git a/nix/voting-tools-rs/packages.nix b/nix/voting-tools-rs/packages.nix
