Merge remote-tracking branch 'origin/master' into hkm/th-orphans-issue

Author: hamishmack

.github/workflows/pipeline.yml

@@ -139,49 +139,40 @@ jobs:
 
   hydra-ifdLevel-0-and-1:
     runs-on: [self-hosted, linux]
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
       - name: "Check that jobset will evaluate in Hydra at ifdLevel 0 and 1"
         run: |
           nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-          ./check-hydra.sh --arg ifdLevel 0
-          ./check-hydra.sh --arg ifdLevel 1
+          sed -i 's/runningHydraEvalTest = false;/runningHydraEvalTest = true;/' flake.nix
+          sed -i 's/ifdLevel = 3;/ifdLevel = 0;/' flake.nix
+          ./check-hydra.sh
+          sed -i 's/ifdLevel = 0;/ifdLevel = 1;/' flake.nix
+          ./check-hydra.sh
 
   hydra-ifdLevel-2:
     runs-on: [self-hosted, linux]
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
       - name: "Check that jobset will evaluate in Hydra at ifdLevel 2"
         run: |
           nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-          ./check-hydra.sh --arg ifdLevel 2
+          sed -i 's/runningHydraEvalTest = false;/runningHydraEvalTest = true;/' flake.nix
+          sed -i 's/ifdLevel = 3;/ifdLevel = 2;/' flake.nix
+          ./check-hydra.sh
 
-  hydra-ifdLevel-3-and-ghc-8-10:
+  hydra-ifdLevel-3:
     runs-on: [self-hosted, linux]
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
-      - name: "Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 8.10"
+      - name: "Check that jobset will evaluate in Hydra at ifdLevel 3"
         run: |
           nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-          ./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 6 x == "ghc810"'
-
-  hydra-ifdLevel-3-and-ghc-9-2:
-    runs-on: [self-hosted, linux]
-    steps:
-      - uses: actions/checkout@v4
-      - name: "Check that jobset will evaluate in Hydra at ifdLevel 3 and ghc 9.2"
-        run: |
-          nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-          ./check-hydra.sh --arg ifdLevel 3 --arg include 'x: __substring 0 5 x == "ghc92"'
-
-  hydra-ifdLevel-3-and-not-ghc-8-10-or-9-2:
-    runs-on: [self-hosted, linux]
-    steps:
-      - uses: actions/checkout@v4
-      - name: "Check that jobset will evaluate in Hydra at ifdLevel 3 and not (ghc 8.10 or ghc 9.2)"
-        run: |
-          nix-build build.nix -A maintainer-scripts.check-hydra -o check-hydra.sh
-          ./check-hydra.sh --arg ifdLevel 3 --arg include 'x: !(__substring 0 6 x == "ghc810" || __substring 0 5 x == "ghc92")'
+          sed -i 's/runningHydraEvalTest = false;/runningHydraEvalTest = true;/' flake.nix
+          ./check-hydra.sh
 
   closure-size:
     runs-on: [self-hosted, linux]

builder/comp-builder.nix

@@ -28,7 +28,7 @@ let self =
     # (not just the one we are building).
     # Enable for tests in packages that use cabal-doctest.
     ( haskellLib.isTest componentId &&
-      lib.any (x: x.identifier.name or "" == "cabal-doctest") package.setup-depends
+      lib.any (x: x.identifier.name or "" == "cabal-doctest") (package.setup-depends ++ setup.config.depends or [])
     )
 , allComponent # Used when `configureAllComponents` is set to get a suitable configuration.
 

builder/ghc-for-component-wrapper.nix

@@ -126,6 +126,7 @@ let
 
   ''
   + lib.optionalString (stdenv.targetPlatform.isMusl && !haskellLib.isNativeMusl && builtins.compareVersions ghc.version "9.9" >0) ''
+     ln -s $wrappedGhc/bin/${targetPrefix}unlit $wrappedGhc/bin/unlit
      ln -s $wrappedGhc/bin/${ghcCommand}-iserv $wrappedGhc/bin/ghc-iserv
      ln -s $wrappedGhc/bin/${ghcCommand}-iserv-prof $wrappedGhc/bin/ghc-iserv-prof
   ''

builder/make-config-files.nix

@@ -5,7 +5,7 @@
 let
   # Sort and remove duplicates from nonReinstallablePkgs.
   # That way changes to the order of nonReinstallablePkgs does not require rebuilds.
-  nonReinstallablePkgs' = __attrNames (lib.genAttrs nonReinstallablePkgs (x: x));
+  nonReinstallablePkgs' = __attrNames (lib.genAttrs (component.pre-existing or [] ++ nonReinstallablePkgs) (x: x));
 
   ghc = if enableDWARF then defaults.ghc.dwarf else defaults.ghc;
 

builder/shell-for.nix

@@ -68,7 +68,8 @@ let
   selectedComponents =
     lib.filter isSelectedComponent  (lib.attrValues transitiveDependenciesComponents);
 
-  allHsPkgsComponents = lib.concatMap haskellLib.getAllComponents (builtins.attrValues hsPkgs);
+  allHsPkgsComponents = lib.concatMap haskellLib.getAllComponents
+    (lib.filter (x: !(x.isRedirect or false)) (builtins.attrValues hsPkgs));
 
   # Given a list of `depends`, removes those which are selected components
   removeSelectedInputs =
@@ -114,9 +115,10 @@ let
   # Set up a "dummy" component to use with ghcForComponent.
   component = {
     depends = packageInputs;
-    libs = [];
-    pkgconfig = [];
-    frameworks = [];
+    pre-existing = lib.concatMap (x: (haskellLib.dependToLib x).config.pre-existing or []) packageInputs;
+    libs         = lib.concatMap (x: (haskellLib.dependToLib x).config.libs or []) packageInputs;
+    pkgconfig    = lib.concatMap (x: (haskellLib.dependToLib x).config.pkgconfig or []) packageInputs;
+    frameworks   = lib.concatMap (x: (haskellLib.dependToLib x).config.frameworks or []) packageInputs;
     doExactConfig = false;
   };
   configFiles = makeConfigFiles {

changelog.md

@@ -1,6 +1,82 @@
 This file contains a summary of changes to Haskell.nix and `nix-tools`
 that will impact users.
 
+## Sep 17, 2024
+
+Cabal projects now use the more granular Unit IDs from plan.json
+to identify packages.  This allows for different versions of a
+package to be used when building `built-tool-depends` and setup
+dependencies.
+
+Overrides in the `modules` argument apply to all versions of
+the package.  However to make this work we needed to make
+each `packages.somepackage` an option (instead of using an
+`attrsOf` the submodule type).
+
+It is now an error to override a package that is not in the
+plan.  This can be a problem if different GHC versions, target
+platforms, or cabal flag settings cause the package to be
+excluded from the plan.  Adding `package-keys` can tell
+haskell.nix to include the option anyway:
+
+```
+  modules = [{
+    # Tell haskell.nix that `somepackage` may exist.
+    package-keys = ["somepackage"];
+    # Now the following will not cause an error even
+    # if `somepackage` is not in the plan
+    packages.somepackage.flags.someflag = true;
+  }];
+```
+
+There is a helper function you can use to add `package-keys`
+for all of the `builtins.attrNames` of `packages`:
+
+```
+  modules = [(pkgs.haskell-nix.haskellLib.addPackageKeys {
+    packages.somepackage.flags.someflag = true;
+  })];
+```
+
+Do not use the module's `pkgs` arg to look `addPackageKeys` up
+though or it will result an `infinite recursion` error.
+
+Code that uses `options.packages` will also need to be updated.
+For instance the following code that uses `options.packages`
+to set `--Werror` for local packages:
+
+```
+  ({ lib, ... }: {
+    options.packages = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule (
+        { config, lib, ... }:
+        lib.mkIf config.package.isLocal
+        {
+          configureFlags = [ "--ghc-option=-Werror"];
+        }
+      ));
+    };
+  })
+```
+
+Now needs to do it for each of the entry in `config.package-keys`
+instead of using `attrsOf`:
+
+```
+  ({ config, lib, ... }: {
+    options.packages = lib.genAttrs config.package-keys (_:
+      lib.mkOption {
+        type = lib.types.submodule (
+          { config, lib, ... }:
+          lib.mkIf config.package.isLocal
+          {
+            configureFlags = [ "--ghc-option=-Werror"];
+          }
+        );
+      });
+  })
+```
+
 ## Jun 5, 2024
 
 Haskell.nix now respects the `pre-existing` packages selected

ci.nix

@@ -62,19 +62,17 @@
       # from here (so that is no longer cached) also remove ./materialized/ghcXXX.
       # Update supported-ghc-versions.md to reflect any changes made here.
       nixpkgs.lib.optionalAttrs (nixpkgsName == "R2405") {
-        ghc94 = false;
         ghc96 = false;
         ghc98 = false;
       } // nixpkgs.lib.optionalAttrs (nixpkgsName == "unstable") {
         ghc810 = true;
-        ghc90 = false;
-        ghc92 = true;
-        ghc94 = true;
+        ghc92 = false;
+        ghc94 = false;
         ghc96 = true;
-        ghc96llvm = true;
         ghc98 = true;
-        ghc98llvm = true;
+        ghc98llvm = false;
         ghc910 = true;
+        ghc910llvm = true;
         ghc911 = true;
       })));
   crossSystems = nixpkgsName: nixpkgs: compiler-nix-name:
@@ -85,21 +83,21 @@
       && (__match ".*llvm" compiler-nix-name == null)
       && ((system == "x86_64-linux"  && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948"])
        || (system == "aarch64-linux" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948"])
-       || (system == "x86_64-darwin" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948"])
-       || (system == "aarch64-darwin" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948"])
+       || (system == "x86_64-darwin" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948" "ghc966" "ghc982"])
+       || (system == "aarch64-darwin" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948" "ghc966" "ghc982"])
        )) {
     inherit (lib.systems.examples) ghcjs;
   } // lib.optionalAttrs (
          (__match ".*llvm" compiler-nix-name == null)
-      && ((system == "x86_64-linux"  && !builtins.elem compiler-nix-name ["ghc884" "ghc91120240620"]) # Including GHC HEAD here because the patches for rts/RtsSymbols.c no longer apply and mingwW64 GHC build fails without them
+      && ((system == "x86_64-linux"  && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc91120240918"]) # Including GHC HEAD here because the patches for rts/RtsSymbols.c no longer apply and mingwW64 GHC build fails without them
        || (system == "x86_64-darwin" && builtins.elem compiler-nix-name []))) { # TODO add ghc versions when we have more darwin build capacity
     inherit (lib.systems.examples) mingwW64;
   } // lib.optionalAttrs (nixpkgsName == "unstable"
       && (__match ".*llvm" compiler-nix-name == null)
-      && ((system == "x86_64-linux"  && !builtins.elem compiler-nix-name ["ghc884" "ghc8107" "ghc902" "ghc928"])
+      && ((system == "x86_64-linux"  && !builtins.elem compiler-nix-name ["ghc884" "ghc8107" "ghc902" "ghc928" "ghc948"])
        || (system == "x86_64-darwin" && builtins.elem compiler-nix-name []))) { # TODO add ghc versions when we have more darwin build capacity
     inherit (lib.systems.examples) ucrt64;
-  } // lib.optionalAttrs (system == "x86_64-linux" && nixpkgsName == "unstable" && !builtins.elem compiler-nix-name ["ghc884"]) {
+  } // lib.optionalAttrs (system == "x86_64-linux" && nixpkgsName == "unstable" && !builtins.elem compiler-nix-name ["ghc884" "ghc902" "ghc928" "ghc948"]) {
     # Musl cross only works on linux
     # aarch64 cross only works on linux
     inherit (lib.systems.examples) musl64 aarch64-multiplatform;

compiler/ghc/default.nix

@@ -232,7 +232,7 @@ let
     if targetPlatform.isGhcjs
       then [ buildPackages.emscripten ]
     else if hostPlatform == buildPlatform
-      then [ targetPackages.stdenv.cc ] ++ lib.optional useLLVM llvmPackages.llvm
+      then [ targetPackages.stdenv.cc ] ++ lib.optionals useLLVM [llvmPackages.llvm llvmPackages.clang]
     else assert targetPlatform == hostPlatform; # build != host == target
       [ stdenv.cc ] ++ lib.optional useLLVM buildLlvmPackages.llvm;
 
@@ -362,14 +362,8 @@ let
   # Sometimes we have to dispatch between the bintools wrapper and the unwrapped
   # derivation for certain tools depending on the platform.
   bintoolsFor = {
-    # GHC needs install_name_tool on all darwin platforms. On aarch64-darwin it is
-    # part of the bintools wrapper (due to codesigning requirements), but not on
-    # x86_64-darwin.
-    install_name_tool =
-      if stdenv.targetPlatform.isAarch64
-      then targetCC.bintools
-      else targetCC.bintools.bintools;
-    # Same goes for strip.
+    # GHC needs install_name_tool on all darwin platforms.
+    install_name_tool = targetCC.bintools.bintools;
     strip =
       # TODO(@sternenseemann): also use wrapper if linker == "bfd" or "gold"
       if stdenv.targetPlatform.isAarch64
@@ -403,14 +397,26 @@ stdenv.mkDerivation (rec {
         done
     ''
     # Use emscripten and the `config.sub` saved by `postPatch`
-    + lib.optionalString (targetPlatform.isGhcjs) ''
+    + lib.optionalString (targetPlatform.isGhcjs) (''
         export CC="${targetCC}/bin/emcc"
         export CXX="${targetCC}/bin/em++"
         export LD="${targetCC}/bin/emcc"
-        export NM="${targetCC}/share/emscripten/emnm"
+    '' + (
+      # Including AR and RANLIB here breaks tests.js-template-haskell for GHC 9.6
+      # `LLVM ERROR: malformed uleb128, extends past end`
+      if builtins.compareVersions ghc-version "9.8" >= 0
+        then ''
+          export AR="${targetCC}/bin/emar"
+          export NM="${targetCC}/share/emscripten/emnm"
+          export RANLIB="${targetCC}/bin/emranlib"
+        ''
+        else ''
+          export NM="${targetCC}/share/emscripten/emnm"
+        ''
+    ) + ''
         export EM_CACHE=$(mktemp -d)
         mv config.sub.ghcjs config.sub
-    ''
+    '')
     # GHC is a bit confused on its cross terminology, as these would normally be
     # the *host* tools.
     + lib.optionalString (!targetPlatform.isGhcjs) (''
@@ -744,7 +750,7 @@ stdenv.mkDerivation (rec {
   } // extra-passthru;
 
   meta = {
-    homepage = http://haskell.org/ghc;
+    homepage = "https://haskell.org/ghc";
     description = "The Glasgow Haskell Compiler";
     maintainers = [];
     inherit (ghc.meta) license platforms;