Merge branch 'master' of github.com:input-output-hk/haskell.nix into circuithub

ocharles · ocharles · commit 3e0e9cab2235 · 2023-05-09T15:25:56.000+01:00
diff --git a/build.nix b/build.nix
@@ -6,8 +6,11 @@
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
 , evalPackages ? import nixpkgs nixpkgsArgs
-, nixpkgsForHydra ? haskellNix.sources.nixpkgs-2105
-, pkgsForHydra ? import nixpkgsForHydra (nixpkgsArgs // { inherit (pkgs) system; })
+# This version is used to make our GitHub Action runners happy
+# Using `nixpkgs-unstable` currently results in:
+#   version `GLIBCXX_3.4.30' not found
+, nixpkgsForGitHubAction ? haskellNix.sources.nixpkgs-2211
+, pkgsForGitHubAction ? import nixpkgsForGitHubAction (nixpkgsArgs // { inherit (pkgs) system; })
 , ifdLevel ? 1000
 , compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
 , haskellNix ? (import ./default.nix {})
@@ -75,12 +78,7 @@ in rec {
     update-docs = pkgs.buildPackages.callPackage ./scripts/update-docs.nix {
       generatedOptions = pkgs.callPackage ./scripts/options-doc.nix { };
     };
-    # Because this is going to be used to test caching on hydra, it must not
-    # use the darcs package from the haskell.nix we are testing.  For that reason
-    # it uses `pkgs.buildPackages.callPackage` not `haskell.callPackage`
-    # (We could pull in darcs from a known good haskell.nix for hydra to
-    # use)
-    check-hydra = pkgsForHydra.buildPackages.callPackage ./scripts/check-hydra.nix {};
+    check-hydra = pkgs.buildPackages.callPackage ./scripts/check-hydra.nix {};
     check-closure-size = pkgs.buildPackages.callPackage ./scripts/check-closure-size.nix {
       # Includes cabal-install since this is commonly used.
       nix-tools = pkgs.linkFarm "common-tools" [
@@ -89,21 +87,15 @@ in rec {
       ];
     };
     check-materialization-concurrency = pkgs.buildPackages.callPackage ./scripts/check-materialization-concurrency/check.nix {};
-    # Forcing nixpkgs-unstable here because this test makes a script
-    # that when run will build `aeson` (used by `tests/cabal-simple`)
-    # and we currently do not build that on hydra for nixpkgs-2205 (used by `pkgs`).
-    # Using nixpkgs-unstable should allow buildkite to find what it needs
-    # in the hydra cache when it runs the script.
-    check-path-support = (import haskellNix.sources.nixpkgs-unstable nixpkgsArgs)
-        .buildPackages.callPackage ./scripts/check-path-support.nix {
+    check-path-support = pkgsForGitHubAction.buildPackages.callPackage ./scripts/check-path-support.nix {
       inherit compiler-nix-name;
     };
   };
 
   # These are pure parts of maintainer-script so they can be built by hydra
   # and added to the cache to speed up buildkite.
   maintainer-script-cache = pkgs.recurseIntoAttrs (
-      (pkgs.lib.optionalAttrs (pkgsForHydra.system == "x86_64-linux") {
+      (pkgs.lib.optionalAttrs (pkgs.system == "x86_64-linux") {
         inherit (maintainer-scripts) check-hydra;
       })
     // (pkgs.lib.optionalAttrs (ifdLevel > 2) {
diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix
@@ -201,7 +201,11 @@ let
       ++ lib.optional stdenv.hostPlatform.isLinux (enableFeature enableDeadCodeElimination "split-sections")
       ++ lib.optionals haskellLib.isCrossHost (
         map (arg: "--hsc2hs-option=" + arg) (["--cross-compile"] ++ lib.optionals (stdenv.hostPlatform.isWindows) ["--via-asm"])
-        ++ lib.optional (package.buildType == "Configure") "--configure-option=--host=${stdenv.hostPlatform.config}" )
+        ++ lib.optional (package.buildType == "Configure") "--configure-option=--host=${
+           # Older ghcjs patched config.sub to support "js-unknown-ghcjs" (not "javascript-unknown-ghcjs")
+           if stdenv.hostPlatform.isGhcjs && builtins.compareVersions defaults.ghc.version "9" < 0
+             then "js-unknown-ghcjs"
+             else stdenv.hostPlatform.config}" )
       ++ configureFlags
       ++ (ghc.extraConfigureFlags or [])
       ++ lib.optional enableDebugRTS "--ghc-option=-debug"
@@ -373,21 +377,21 @@ let
       mainProgram = exeName;
     };
 
-    propagatedBuildInputs =
-         frameworks # Frameworks will be needed at link time
+    propagatedBuildInputs = haskellLib.checkUnique "${ghc.targetPrefix}${fullName} propagatedBuildInputs" (
+         haskellLib.uniqueWithName frameworks # Frameworks will be needed at link time
       # Not sure why pkgconfig needs to be propagatedBuildInputs but
       # for gi-gtk-hs it seems to help.
-      ++ map pkgs.lib.getDev (builtins.concatLists pkgconfig)
+      ++ haskellLib.uniqueWithName (map pkgs.lib.getDev (builtins.concatLists pkgconfig))
       # These only need to be propagated for library components (otherwise they
       # will be in `buildInputs`)
-      ++ lib.optionals (haskellLib.isLibrary componentId) configFiles.libDeps
+      ++ lib.optionals (haskellLib.isLibrary componentId) configFiles.libDeps # libDeps is already deduplicated
       ++ lib.optionals (stdenv.hostPlatform.isWindows)
-        (lib.flatten component.libs);
+        (haskellLib.uniqueWithName (lib.flatten component.libs)));
 
-    buildInputs =
-      lib.optionals (!haskellLib.isLibrary componentId) configFiles.libDeps
+    buildInputs = haskellLib.checkUnique "${ghc.targetPrefix}${fullName} buildInputs" (
+      lib.optionals (!haskellLib.isLibrary componentId) configFiles.libDeps # libDeps is already deduplicated
       ++ lib.optionals (!stdenv.hostPlatform.isWindows)
-        (lib.flatten component.libs);
+        (haskellLib.uniqueWithName (lib.flatten component.libs)));
 
     nativeBuildInputs =
       [ghc buildPackages.removeReferencesTo]
diff --git a/builder/haddock-builder.nix b/builder/haddock-builder.nix
@@ -72,11 +72,11 @@ let
     outputs = ["out" "configFiles" "ghc"]
     ++ lib.optional doHaddock' "doc";
 
-    propagatedBuildInputs =
-         builtins.concatLists pkgconfig
-      ++ configFiles.libDeps;
+    propagatedBuildInputs = haskellLib.checkUnique "${fullName} propagatedBuildInputs" (
+         haskellLib.uniqueWithName (map lib.getDev (builtins.concatLists pkgconfig))
+      ++ configFiles.libDeps); # libDeps is already deduplicated
 
-    buildInputs = component.libs;
+    buildInputs = haskellLib.uniqueWithName (lib.flatten component.libs);
 
     nativeBuildInputs =
       [ ghc buildPackages.removeReferencesTo ]
diff --git a/builder/make-config-files.nix b/builder/make-config-files.nix
@@ -51,10 +51,12 @@ let
   libDir         = ghc.libDir or "lib/${ghcCommand}-${ghc.version}";
   packageCfgDir  = "${libDir}/package.conf.d";
 
-  libDeps = map chooseDrv (
-    (if enableDWARF then (x: map (p: p.dwarf or p) x) else x: x)
-    ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
-    (map haskellLib.dependToLib component.depends))
+  libDeps = haskellLib.uniqueWithName (
+    map chooseDrv (
+      (if enableDWARF then (x: map (p: p.dwarf or p) x) else x: x)
+      ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
+      (map haskellLib.dependToLib component.depends))
+    )
   );
   script = ''
     ${target-pkg} init $configFiles/${packageCfgDir}
diff --git a/builder/shell-for.nix b/builder/shell-for.nix
@@ -88,8 +88,9 @@ let
   name = if (mkDrvArgs.name or null) == null then identifierName else mkDrvArgs.name;
 
   # We need to remove any dependencies which would bring in selected components (see above).
-  packageInputs = removeSelectedInputs (lib.concatMap (cfg: cfg.depends) selectedConfigs)
-    ++ additionalPackages;
+  packageInputs = haskellLib.uniqueWithName
+    (removeSelectedInputs (haskellLib.uniqueWithName (lib.concatMap (cfg: cfg.depends) selectedConfigs))
+      ++ additionalPackages);
 
   # Add the system libraries and build tools of the selected haskell packages to the shell.
   # We need to remove any inputs which are selected components (see above).
@@ -98,17 +99,11 @@ let
   #
   # Also, we take care to keep duplicates out of the list, otherwise we may see
   # "Argument list too long" errors from bash when entering a shell.
-  #
-  # Version of `lib.unique` that should be fast if the name attributes are unique
-  uniqueWithName = list:
-    lib.concatMap lib.unique (
-      builtins.attrValues (
-        builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") list));
   allSystemInputs = lib.concatMap (c: c.buildInputs ++ c.propagatedBuildInputs) selectedComponents;
-  systemInputs = removeSelectedInputs (uniqueWithName allSystemInputs);
+  systemInputs = removeSelectedInputs (haskellLib.uniqueWithName allSystemInputs);
 
   nativeBuildInputs = removeSelectedInputs
-    (uniqueWithName (lib.concatMap (c: c.executableToolDepends)
+    (haskellLib.uniqueWithName (lib.concatMap (c: c.executableToolDepends)
       # When not using `exactDeps` cabal may try to build arbitrary dependencies
       # so in this case we need to provide the build tools for all of `hsPkgs`.
       # In some cases those tools may be unwanted or broken so the `allToolDeps`
diff --git a/compiler/ghc/default.nix b/compiler/ghc/default.nix
@@ -275,7 +275,20 @@ let
           + lib.optionalString useLLVM "+llvm"
           + lib.optionalString enableDWARF "+debug_info"
           + lib.optionalString targetPlatform.isGhcjs "+native_bignum+no_profiled_libs"
-      } --docs=no-sphinx -j --verbose";
+      } --docs=no-sphinx -j --verbose"
+      # This is needed to prevent $GCC from emitting out of line atomics.
+      # Those would then result in __aarch64_ldadd1_sync and others being referenced, which
+      # we don't handle in the RTS properly yet. Until we figure out how to _properly_ deal
+      # with the RTS_SYMBOLS in GHC, we are better off disableing the out of line atomics.
+      + lib.optionalString ( hostPlatform.isAarch64 && targetPlatform.isLinux && targetPlatform.isAarch64)
+        " '*.*.ghc.c.opts += -optc-mno-outline-atomics'"
+      # For cross compilers only the RTS should be built with -mno-outline-atomics
+      + lib.optionalString (!hostPlatform.isAarch64 && targetPlatform.isLinux && targetPlatform.isAarch64)
+        " '*.rts.ghc.c.opts += -optc-mno-outline-atomics'"
+      # The following is required if we build on aarch64-darwin for aarch64-iOS. Otherwise older 
+      # iPhones/iPads/... won't understand the compiled code, as the compiler will emit LDSETALH
+      # + lib.optionalString (targetPlatform.???) "'*.rts.ghc.c.opts += -optc-mcpu=apple-a7 -optc-march=armv8-a+norcpc'"
+      ;
 
   # When installation is done by copying the stage1 output the directory layout
   # is different.
@@ -319,7 +332,7 @@ stdenv.mkDerivation (rec {
 
   # configure was run by configured-src already.
   phases = [ "unpackPhase" "patchPhase" ]
-            ++ lib.optional (ghc-patches != []) "autoreconfPhase"
+            ++ lib.optional (ghc-patches != [] && !stdenv.targetPlatform.isGhcjs) "autoreconfPhase" # autoreconf can replace config.sub with one that is missing ghcjs
             ++ [ "configurePhase" "buildPhase"
              "checkPhase" "installPhase"
              "fixupPhase"
diff --git a/flake.lock b/flake.lock
diff --git a/lib/default.nix b/lib/default.nix
@@ -588,4 +588,19 @@ in {
 
   # How to run ldd when checking for static linking
   lddForTests = "${pkgs.pkgsBuildBuild.glibc.bin}/bin/ldd";
+
+  # Version of `lib.unique` that should be fast if the name attributes are unique
+  uniqueWithName = list:
+    lib.concatMap lib.unique (
+      builtins.attrValues (
+        builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") list));
+
+  # Assert that each item in the list is unique
+  checkUnique = msg: x:
+    if __length x == __length (uniqueWithName x)
+      then x
+      else builtins.throw "Duplicate items found in ${msg} ${
+        __toJSON (__attrNames (lib.filterAttrs (_: v: __length v > 1) (
+          builtins.groupBy (x: if __typeOf x == "set" then x.name or "noname" else "notset") x)))
+      }";
 }
diff --git a/lib/pkgconf-nixpkgs-map.nix b/lib/pkgconf-nixpkgs-map.nix
@@ -2552,10 +2552,10 @@ pkgs:
     "libavc1394" = [ "libavc1394" ];
     "libavif" = [ "libavif" ];
     "ayatana-appindicator3-0.1" = [ "libayatana-appindicator" ];
-    "ayatana-appindicator-0.1" = [ "libayatana-appindicator-gtk2" ];
+#    "ayatana-appindicator-0.1" = [ "libayatana-appindicator-gtk2" ];
 #    "ayatana-appindicator3-0.1" = [ "libayatana-appindicator-gtk3" ];
     "ayatana-indicator3-0.4" = [ "libayatana-indicator" ];
-    "ayatana-indicator-0.4" = [ "libayatana-indicator-gtk2" ];
+#    "ayatana-indicator-0.4" = [ "libayatana-indicator-gtk2" ];
 #    "ayatana-indicator3-0.4" = [ "libayatana-indicator-gtk3" ];
     "libb2" = [ "libb2" ];
     "baseencode" = [ "libbaseencode" ];
diff --git a/nix-tools/.buildkite/nix-tools-build.sh b/nix-tools/.buildkite/nix-tools-build.sh
@@ -1,6 +1,10 @@
 #! /usr/bin/env nix-shell
 #! nix-shell -I "nixpkgs=channel:nixos-22.11" -i bash -p nixUnstable cabal-install ghc git nix-prefetch-git cacert
 
+# This file uses nixpkgs 22.11 to make our GHA runners happy
+# Using `nixpkgs-unstable` currently results in:
+#   version `GLIBCXX_3.4.30' not found
+
 # The `nix-shell` is set to run without `--pure`.
 # It is possible to use `--pure` if we need to, but it requires setting these.
 #  export LANG=en_US.UTF-8
@@ -22,7 +26,7 @@ cabal new-update
 
 echo
 echo "+++ Run stable version of make-install-plan and plan-to-nix"
-nix build --impure --expr '(let haskellNix = import (builtins.fetchTarball "https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz") {}; in (import haskellNix.sources.nixpkgs haskellNix.nixpkgsArgs).haskell-nix.nix-tools.ghc8107)' -o nt
+nix build --impure --expr '(let haskellNix = import (builtins.fetchTarball "https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz") {}; in (import haskellNix.sources.nixpkgs-2211 haskellNix.nixpkgsArgs).haskell-nix.nix-tools.ghc8107)' -o nt
 ./nt/bin/make-install-plan
 rm -rf .buildkite/nix1
 ./nt/bin/plan-to-nix --output .buildkite/nix1 --plan-json dist-newstyle/cache/plan.json
diff --git a/overlays/bootstrap.nix b/overlays/bootstrap.nix
@@ -1033,7 +1033,10 @@ in {
     cabal-install-tool = {compiler-nix-name, ...}@args:
       (final.haskell-nix.tool compiler-nix-name "cabal" ({pkgs, ...}: {
         evalPackages = pkgs.buildPackages;
-        compilerSelection = p: p.haskell-nix.compiler // p.haskell.compiler;
+        compilerSelection = p: p.haskell-nix.compiler // p.haskell.compiler
+          # Avoid ghc961 from nixpkgs for now as the files in it are not
+          # where we expect them.
+          // { inherit (p.haskell-nix.compiler) ghc961; };
         version = "3.8.1.0";
         index-state = final.haskell-nix.internalHackageIndexState;
         materialized = ../materialized + "/${compiler-nix-name}/cabal-install";
diff --git a/overlays/ghcjs.nix b/overlays/ghcjs.nix
@@ -13,11 +13,17 @@ final: prev:
         # Apply the patches that came with `ghcjs`
         # Also add a "Keep alive" message to prevent hydra timeouts when hsc2hs runs
         packages = pkgs.lib.genAttrs (pkgs.lib.optionals (__elem config.compiler.nix-name ["ghc865" "ghc884" "ghc8107"]) ["base" "directory" "filepath" "ghc-prim" "integer-gmp" "process" "template-haskell" "time" "unix" "Win32" ])
-          (name: {
+          (name: let
+              ghcjs-src = pkgs.buildPackages.haskell-nix.compiler.${config.compiler.nix-name}.project.configured-src;
+              # This src derivation is needed for the unpatched config.sub file
+              # (the config.sub in the hackage is too old and the one created by autoreconf is too new for the patch).
+              ghcjs-src' = pkgs.buildPackages.haskell-nix.compiler.${config.compiler.nix-name}.configured-src;
+            in {
             components.library.preConfigure = ''
               tr -d '\r' < ${name}.cabal > ${name}.cabal-new
               mv ${name}.cabal-new ${name}.cabal
-              patch -p3 < ${pkgs.buildPackages.haskell-nix.compiler.${config.compiler.nix-name}.project.configured-src}/lib/patches/${name}.patch
+              if [[ -e config.sub ]]; then cp ${ghcjs-src'}/config.sub config.sub; fi
+              patch -p3 < ${ghcjs-src}/lib/patches/${name}.patch
             '';
             components.library.preBuild = ''
               # Avoid timeouts while unix package runs hsc2hs (it does not print anything
diff --git a/overlays/windows.nix b/overlays/windows.nix
@@ -4,17 +4,15 @@
 # and not end up with the expected changes we want.
 final: prev:
 {
-   # on windows we have this habit of putting libraries
-   # into `bin`, whereas on unix it's usually `lib`. For
-   # this confuses nix easily. So we'll just move the
-   # .dll's from `bin` into `$out/lib`. Such that they
-   # are trivially found.
-  #  openssl = prev.openssl.overrideAttrs (drv: {
-  #   #  postInstall = with prev.stdenv; drv.postInstall + lib.optionalString hostPlatform.isWindows ''
-  #   #    cp $bin/bin/*.dll $out/lib/
-  #   #  '';
-  #   postFixup = "";
-  #  });
+  # Work around for https://github.com/NixOS/nixpkgs/pull/229465
+  openssl = if !prev.stdenv.hostPlatform.isWindows then prev.openssl else prev.openssl.overrideAttrs (drv: {
+    nativeBuildInputs = final.lib.filter (x: x.name or "" != "make-shell-wrapper-hook") drv.nativeBuildInputs;
+    postInstall = ''
+      function makeWrapper () {
+        echo Skipping makeWrapper
+      }
+    '' + drv.postInstall;
+  });
 } // prev.lib.optionalAttrs (prev ? mfpr) {
    mfpr = if !prev.stdenv.hostPlatform.isWindows then prev.mpfr else prev.mfpr.overrideAttrs (drv: {
      configureFlags = (drv.configureFlags or []) ++ [ "--enable-static --disable-shared" ];
