Merge remote-tracking branch 'origin/master' into hkm/embed-file

Author: hamishmack

BUGLOG

@@ -0,0 +1,86 @@
+This file contains bugs we find while working on haskell.nix.  The format is as
+follow:
+<separator: 80 * '-'>
+YYYY-MM-DD: nix-job name
+
+<error>
+
+<discussion>
+
+--------------------------------------------------------------------------------
+2024-04-09 x86_64-linux.R2305.ghc8107.mingwW64.ghc
+
+/build/ghc62733_0/ghc_1.s:50:0: error:
+     Error: CFI instruction used without previous .cfi_startproc
+   |
+50 |         .cfi_escape 0x16, 0x07, 0x04, 0x77, 152, 65
+   | ^
+`x86_64-w64-mingw32-cc' failed in phase `Assembler'. (Exit code: 1)
+make[1]: *** [rts/ghc.mk:325: rts/dist/build/StgCRun.o] Error 1
+
+The source for this is
+> https://github.com/ghc/ghc/blob/1f02b7430b2fbab403d7ffdde9cfd006e884678e/rts/StgCRun.c#L433
+
+It appears that GCC C17 12.2.0 does _not_ emit .cfi_startproc / .cfi_endprocs
+whereas GCC C17 13.2.0 _does_. Specificall x86_64-w64-mingw32-cc. So this might
+be a cross compilation issue.
+
+The -g is hardcoded in
+https://github.com/ghc/ghc/blob/1f02b7430b2fbab403d7ffdde9cfd006e884678e/mk/config.mk.in#L361
+
+Turns out, this was disabled for anything but linux in https://github.com/ghc/ghc/commit/5b08e0c06e038448a63aa9bd7f163b23d824ba4b,
+hence we backport that patch to GHC-8.10 when targeting windows (to prevent mass rebuilds for
+other archs).
+
+--------------------------------------------------------------------------------
+2024-04-10 x86_64-linux.R2305.ghc902.mingwW64.ghc
+
+make[1]: *** [utils/hsc2hs/ghc.mk:22: utils/hsc2hs/dist-install/build/tmp/hsc2hs.exe] Error 1
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x2a): relocation truncated to fit: R_X86_64_32S against `.text'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x46): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.data'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x8b): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x95): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0xe3): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0xed): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x13b): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x145): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x193): relocation truncated to fit: R_X86_64_32S against symbol `stg_bh_upd_frame_info' defined in .text section in /build/ghc-9.0.2/rts/dist/build/libHSrts.a(Updates.o)
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x19d): relocation truncated to fit: IMAGE_REL_AMD64_ADDR32 against `.rdata'
+utils/runghc/dist-install/build/Main.o:fake:(.text+0x1eb): additional relocation overflows omitted from the output
+
+We notice `fake`, which is GHC failing to provide .file identifier in the source.
+We also see lots of R_X64_64_32S relocations, which are signed 32bit relocations.
+These fall with ASLR and high entropy base images from later binutils.
+
+The underlying issue is that GHC emits _absolute_ label loads (mov $... reg), instead
+of %rpi or other relative loads. This then leads to the linker emitting 32bit
+absolute relocation. With the final image being potentially loaded into high memory
+(e.g. dynamic base, and the base image being set to some high address), the linker
+starts falling over itself, because it simply can't resolve those absolute addresses
+in the 32bit slots.
+
+This was fixed in GHC upstream in https://gitlab.haskell.org/ghc/ghc/-/merge_requests/7449,
+while the patch in haskell.nix is a bit more pedestrian and just sets PIC on windows to
+always be on, and then uses the PIC pipeline.
+
+--------------------------------------------------------------------------------
+2024-06-18 x86_64-linux.unstable.ghc9101.ucrt64.tests.th-dlls-minimal.build
+
+0024:err:seh:call_stack_handlers invalid frame 00007FFFFF68EF18 (0000000000022000-0000000000220000)
+0024:err:seh:NtRaiseException Exception frame is not in stack limits => unable to dispatch exception.
+iserv-proxy: {handle: <socket: 11>}: GHCi.Message.remoteCall: end of file
+
+This is due to GHC mislinking GNU import libraries (dll.a). What happens is that
+GHC ends up creating GOT entries for function calls instead of PLT entries. The
+loader/linker in GHC for Windows has logic to lazy load .dll's as referenced. For
+this symbols get a dependency symbol attached, this could be a symbol indicating
+the DLL that needs to be loaded. While walking the dependencies to find the dll to
+load (or in some cases just the dependent symbol, not the dll), we override the
+symbol type with the one of the dependent symbol.  This however means we'll
+override the type of a symbol with the DATA type each time the symbol leads to a
+dllInstance to be loaded. Subsequently we end up creating a GOT entry instead of
+a PLT entry for the symbol, irrepsective of the original symbol being a code or
+data symbol. If code symbols end up getting GOT stubs, we see the above crash as
+the control flow jumps to the location of the stub, and instead of a PLT/jump
+island just lands in the address of the target symbol, which is in most cases
+non-sensical machine code.

ci.nix

@@ -22,17 +22,22 @@
     "R2211" = inputs.nixpkgs-2211;
     "R2305" = inputs.nixpkgs-2305;
     "R2311" = inputs.nixpkgs-2311;
+    "R2405" = inputs.nixpkgs-2405;
     "unstable" = inputs.nixpkgs-unstable;
   };
 
   nixpkgsArgs = {
     # set checkMaterialization as per top-level argument
     overlays = [
       haskellNix.overlay
-      (_final: prev: {
+      (final: prev: {
         haskell-nix = prev.haskell-nix // {
           inherit checkMaterialization;
+          extraPkgconfigMappings = prev.haskell-nix.extraPkgconfigMappings or {} // {
+            "libsodium" = [ "libsodium-18" ];
+          };
         };
+        libsodium-18 = (final.callPackage (inputs.nixpkgs-2311 + "/pkgs/development/libraries/libsodium") {}).overrideAttrs (_: { dontDisableStatic = true; });
       })
     ];
     # Needed for dwarf tests
@@ -56,7 +61,7 @@
       # cabal-install and nix-tools plans.  When removing a ghc version
       # from here (so that is no longer cached) also remove ./materialized/ghcXXX.
       # Update supported-ghc-versions.md to reflect any changes made here.
-      nixpkgs.lib.optionalAttrs (nixpkgsName == "R2311") {
+      nixpkgs.lib.optionalAttrs (nixpkgsName == "R2405") {
         ghc94 = false;
         ghc96 = false;
         ghc98 = false;
@@ -120,8 +125,6 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: pinnedNixpkgsSrc:
           ghc = pkgs.buildPackages.haskell-nix.compiler.${compiler-nix-name};
         } // pkgs.lib.optionalAttrs runTests {
           inherit (build) tests tools maintainer-scripts maintainer-script-cache;
-        } // pkgs.lib.optionalAttrs (ifdLevel >= 2) {
-          inherit (pkgs.haskell-nix.iserv-proxy-exes.${compiler-nix-name}) iserv-proxy;
         } // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
           hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit evalPackages compiler-nix-name; }).getComponent "exe:hello";
         });

compiler/ghc/default.nix

@@ -251,7 +251,9 @@ let
       compiler-nix-name =
         if builtins.compareVersions ghc-version "9.4.7" < 0
           then "ghc928"
-          else "ghc962";
+        else if buildPackages.haskell.compiler ? ghc964
+          then "ghc964"
+        else "ghc962";
     in
     buildPackages.haskell-nix.tool compiler-nix-name "hadrian" {
       compilerSelection = p: p.haskell.compiler;
@@ -272,7 +274,9 @@ let
           then ../../materialized/${compiler-nix-name}/hadrian-ghc981
         else if builtins.compareVersions ghc-version "9.9" < 0
           then ../../materialized/${compiler-nix-name}/hadrian-ghc98
-        else ../../materialized/${compiler-nix-name}/hadrian-ghc99;
+        else if builtins.compareVersions ghc-version "9.11" < 0
+          then ../../materialized/${compiler-nix-name}/hadrian-ghc910
+        else null;
       modules = [{
         reinstallableLibGhc = false;
         # Apply the patches in a way that does not require using something
@@ -701,11 +705,16 @@ stdenv.mkDerivation (rec {
           ${hadrian}/bin/hadrian ${hadrianArgs} _build/stage0/compiler/build/$a
           cp _build/stage0/compiler/build/$a compiler/GHC/Builtin/$a
         done
-      '' + lib.optionalString stdenv.isDarwin ''
+      '' + lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
         substituteInPlace mk/system-cxx-std-lib-1.0.conf \
           --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
         find . -name 'system*.conf*'
         cat mk/system-cxx-std-lib-1.0.conf
+      '' + lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+        substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+          --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+        find . -name 'system*.conf*'
+        cat mk/system-cxx-std-lib-1.0.conf
       '' + lib.optionalString (installStage1 && stdenv.targetPlatform.isMusl) ''
         substituteInPlace hadrian/cfg/system.config \
           --replace 'cross-compiling       = YES' \
@@ -786,11 +795,16 @@ stdenv.mkDerivation (rec {
     export XATTR=$(mktemp -d)/nothing
   '';
 } // lib.optionalAttrs useHadrian {
-  postConfigure = lib.optionalString stdenv.isDarwin ''
+  postConfigure = lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
     substituteInPlace mk/system-cxx-std-lib-1.0.conf \
       --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
     find . -name 'system*.conf*'
     cat mk/system-cxx-std-lib-1.0.conf
+  '' + lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+    substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+      --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+    find . -name 'system*.conf*'
+    cat mk/system-cxx-std-lib-1.0.conf
   '' + lib.optionalString (installStage1 && !haskell-nix.haskellLib.isCrossTarget && stdenv.targetPlatform.isMusl) ''
     substituteInPlace hadrian/cfg/system.config \
       --replace 'cross-compiling       = YES' \
@@ -850,12 +864,18 @@ stdenv.mkDerivation (rec {
         ${hadrian}/bin/hadrian ${hadrianArgs} binary-dist-dir
         cd _build/bindist/ghc-*
         ./configure --prefix=$out ${lib.concatStringsSep " " configureFlags}
-        ${lib.optionalString stdenv.isDarwin ''
+        ${lib.optionalString (stdenv.isDarwin && (__tryEval libcxxabi).success) ''
           substituteInPlace mk/system-cxx-std-lib-1.0.conf \
             --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
           substituteInPlace lib/package.conf.d/system-cxx-std-lib-1.0.conf \
             --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib ${libcxxabi}/lib'
         ''}
+        ${lib.optionalString (stdenv.isDarwin && !(__tryEval libcxxabi).success) ''
+          substituteInPlace mk/system-cxx-std-lib-1.0.conf \
+            --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+          substituteInPlace lib/package.conf.d/system-cxx-std-lib-1.0.conf \
+            --replace 'dynamic-library-dirs:' 'dynamic-library-dirs: ${libcxx}/lib'
+        ''}
         mkdir -p utils
         cp -r ../../../utils/completion utils
         make install

default.nix

@@ -88,6 +88,12 @@ self // {
   pkgs-2305 = import self.inputs.nixpkgs-2305 (nixpkgsArgs // {
     localSystem = { inherit system; };
   });
+  pkgs-2311 = import self.inputs.nixpkgs-2311 (nixpkgsArgs // {
+    localSystem = { inherit system; };
+  });
+  pkgs-2405 = import self.inputs.nixpkgs-2405 (nixpkgsArgs // {
+    localSystem = { inherit system; };
+  });
   pkgs-unstable = import self.inputs.nixpkgs-unstable (nixpkgsArgs // {
     localSystem = { inherit system; };
   });

flake.lock

@@ -10,10 +10,8 @@
     nixpkgs-2211 = { url = "github:NixOS/nixpkgs/nixpkgs-22.11-darwin"; };
     nixpkgs-2305 = { url = "github:NixOS/nixpkgs/nixpkgs-23.05-darwin"; };
     nixpkgs-2311 = { url = "github:NixOS/nixpkgs/nixpkgs-23.11-darwin"; };
-    # The libsodium bump in 85c6e70b555fe892a049fa3d9dce000dc23a9562 breaks th-dll tests.
-    # And later it breaks in th-dll due to some change in the windows libs. We should probably
-    # drop unsable.
-    nixpkgs-unstable = { url = "github:NixOS/nixpkgs?rev=47585496bcb13fb72e4a90daeea2f434e2501998"; }; # nixpkgs-unstable };
+    nixpkgs-2405 = { url = "github:NixOS/nixpkgs/nixpkgs-24.05-darwin"; };
+    nixpkgs-unstable = { url = "github:NixOS/nixpkgs/nixpkgs-unstable"; };
     flake-compat = { url = "github:input-output-hk/flake-compat/hkm/gitlab-fix"; flake = false; };
     "hls-1.10" = { url = "github:haskell/haskell-language-server/1.10.0.0"; flake = false; };
     "hls-2.0" = { url = "github:haskell/haskell-language-server/2.0.0.1"; flake = false; };
@@ -85,6 +83,7 @@
     , nixpkgs-2211
     , nixpkgs-2305
     , nixpkgs-2311
+    , nixpkgs-2405
     , flake-compat
     , ...
     }@inputs:

flake.nix

@@ -268,9 +268,14 @@ let
       sourceRepos = sourceReposBuild;
       inherit (repoResult) repos extra-hackages;
       makeFixedProjectFile = ''
+        HOME=$(mktemp -d)
         cp -f ${evalPackages.writeText "cabal.project" sourceRepoFixedProjectFile} ./cabal.project
         chmod +w -R ./cabal.project
-      '';
+      '' + pkgs.lib.strings.concatStrings (
+            map (f: ''
+              git config --global --add safe.directory ${f.location}/.git
+            '') sourceReposEval
+          );
       # This will be used to replace refernces to the minimal git repos with just the index
       # of the repo.  The index will be used in lib/import-and-filter-project.nix to
       # lookup the correct repository in `sourceReposBuild`.  This avoids having

lib/call-cabal-project-to-nix.nix

@@ -6,15 +6,15 @@ pkgs:
 
     # Only include derivations that exist in the current pkgs.
     # This allows us to use this mapping to be used in allPkgConfigWrapper.
-    # See ./overlas
+    # See ./overlays
     lookupAttrsIn = x: __mapAttrs (_pname: names:
-        # The first entry is should be used for the version by allPkgConfigWrapper
-        # so we need it to be present.
-        if __length names != 0 && x ? ${__head names}
-          then
-            pkgs.lib.concatMap (
-              name: if x ? ${name} then [ x.${name} ] else []) names
-          else []);
+      # The first entry is should be used for the version by allPkgConfigWrapper
+      # so we need it to be present.
+      with lib; optionals (__length names != 0 && x ? ${__head names})
+        (concatMap
+          (name: optionals (x ? ${name})
+            (let p = __tryEval (x.${name}); in optional p.success p.value))
+          names));
   in lookupAttrsIn pkgs ({
     # Based on https://github.com/NixOS/cabal2nix/blob/11c68fdc79461fb74fa1dfe2217c3709168ad752/src/Distribution/Nixpkgs/Haskell/FromCabal/Name.hs#L23