Fix "bash: Argument list too long" (input-output-hk#1937)

This change avoids duplicates in propagatedBuildInputs it uses `checkUnique` to enforce uniqueness at the top level and `uniqueWithName` fix places where duplicates might arise.

For instance:

* `packageInputs` in `shell-for.nix` gave rise to duplicates when the shell was for building more than one component (and they shared dependencies).

* `libDeps` in `make-config-files.nix` would include duplicates if a `build-dependency` was repeated in a `.cabal` files (Cabal 3.8.1.0 includes two references to `process`).

It should also remove duplicate `framework` `lib` and `pkgconfig-depends`.

Fixes input-output-hk#1933

Author: hamishmack

builder/comp-builder.nix

@@ -376,21 +376,21 @@ let
       mainProgram = exeName;
     };
 
-    propagatedBuildInputs =
-         frameworks # Frameworks will be needed at link time
+    propagatedBuildInputs = haskellLib.checkUnique "${ghc.targetPrefix}${fullName} propagatedBuildInputs" (
+         haskellLib.uniqueWithName frameworks # Frameworks will be needed at link time
       # Not sure why pkgconfig needs to be propagatedBuildInputs but
       # for gi-gtk-hs it seems to help.
-      ++ map pkgs.lib.getDev (builtins.concatLists pkgconfig)
+      ++ haskellLib.uniqueWithName (map pkgs.lib.getDev (builtins.concatLists pkgconfig))
       # These only need to be propagated for library components (otherwise they
       # will be in `buildInputs`)
-      ++ lib.optionals (haskellLib.isLibrary componentId) configFiles.libDeps
+      ++ lib.optionals (haskellLib.isLibrary componentId) configFiles.libDeps # libDeps is already deduplicated
       ++ lib.optionals (stdenv.hostPlatform.isWindows)
-        (lib.flatten component.libs);
+        (haskellLib.uniqueWithName (lib.flatten component.libs)));
 
-    buildInputs =
-      lib.optionals (!haskellLib.isLibrary componentId) configFiles.libDeps
+    buildInputs = haskellLib.checkUnique "${ghc.targetPrefix}${fullName} buildInputs" (
+      lib.optionals (!haskellLib.isLibrary componentId) configFiles.libDeps # libDeps is already deduplicated
       ++ lib.optionals (!stdenv.hostPlatform.isWindows)
-        (lib.flatten component.libs);
+        (haskellLib.uniqueWithName (lib.flatten component.libs)));
 
     nativeBuildInputs =
       [ghc buildPackages.removeReferencesTo]

builder/haddock-builder.nix

@@ -72,11 +72,11 @@ let
     outputs = ["out" "configFiles" "ghc"]
     ++ lib.optional doHaddock' "doc";
 
-    propagatedBuildInputs =
-         builtins.concatLists pkgconfig
-      ++ configFiles.libDeps;
+    propagatedBuildInputs = haskellLib.checkUnique "${fullName} propagatedBuildInputs" (
+         haskellLib.uniqueWithName (map lib.getDev (builtins.concatLists pkgconfig))
+      ++ configFiles.libDeps); # libDeps is already deduplicated
 
-    buildInputs = component.libs;
+    buildInputs = haskellLib.uniqueWithName (lib.flatten component.libs);
 
     nativeBuildInputs =
       [ ghc buildPackages.removeReferencesTo ]

builder/make-config-files.nix

@@ -51,10 +51,12 @@ let
   libDir         = ghc.libDir or "lib/${ghcCommand}-${ghc.version}";
   packageCfgDir  = "${libDir}/package.conf.d";
 
-  libDeps = map chooseDrv (
-    (if enableDWARF then (x: map (p: p.dwarf or p) x) else x: x)
-    ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
-    (map haskellLib.dependToLib component.depends))
+  libDeps = haskellLib.uniqueWithName (
+    map chooseDrv (
+      (if enableDWARF then (x: map (p: p.dwarf or p) x) else x: x)
+      ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
+      (map haskellLib.dependToLib component.depends))
+    )
   );
   script = ''
     ${target-pkg} init $configFiles/${packageCfgDir}

builder/shell-for.nix

@@ -88,8 +88,9 @@ let
   name = if (mkDrvArgs.name or null) == null then identifierName else mkDrvArgs.name;
 
   # We need to remove any dependencies which would bring in selected components (see above).
-  packageInputs = removeSelectedInputs (lib.concatMap (cfg: cfg.depends) selectedConfigs)
-    ++ additionalPackages;
+  packageInputs = haskellLib.uniqueWithName
+    (removeSelectedInputs (haskellLib.uniqueWithName (lib.concatMap (cfg: cfg.depends) selectedConfigs))
+      ++ additionalPackages);
 
   # Add the system libraries and build tools of the selected haskell packages to the shell.
   # We need to remove any inputs which are selected components (see above).
@@ -98,17 +99,11 @@ let
   #
   # Also, we take care to keep duplicates out of the list, otherwise we may see
   # "Argument list too long" errors from bash when entering a shell.
-  #
-  # Version of `lib.unique` that should be fast if the name attributes are unique
-  uniqueWithName = list:
-    lib.concatMap lib.unique (
-      builtins.attrValues (
-        builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") list));
   allSystemInputs = lib.concatMap (c: c.buildInputs ++ c.propagatedBuildInputs) selectedComponents;
-  systemInputs = removeSelectedInputs (uniqueWithName allSystemInputs);
+  systemInputs = removeSelectedInputs (haskellLib.uniqueWithName allSystemInputs);
 
   nativeBuildInputs = removeSelectedInputs
-    (uniqueWithName (lib.concatMap (c: c.executableToolDepends)
+    (haskellLib.uniqueWithName (lib.concatMap (c: c.executableToolDepends)
       # When not using `exactDeps` cabal may try to build arbitrary dependencies
       # so in this case we need to provide the build tools for all of `hsPkgs`.
       # In some cases those tools may be unwanted or broken so the `allToolDeps`

lib/default.nix

@@ -588,4 +588,19 @@ in {
 
   # How to run ldd when checking for static linking
   lddForTests = "${pkgs.pkgsBuildBuild.glibc.bin}/bin/ldd";
+
+  # Version of `lib.unique` that should be fast if the name attributes are unique
+  uniqueWithName = list:
+    lib.concatMap lib.unique (
+      builtins.attrValues (
+        builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") list));
+
+  # Assert that each item in the list is unique
+  checkUnique = msg: x:
+    if __length x == __length (uniqueWithName x)
+      then x
+      else builtins.throw "Duplicate items found in ${msg} ${
+        __toJSON (__attrNames (lib.filterAttrs (_: v: __length v > 1) (
+          builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") x)))
+      }";
 }