Skip to content

Commit 64efc98

Browse files
authored
Support githash package and cross package refs (#843)
* Adds support for cross package refs (with a project). Relative directory references between packages within a project should now work. * Adds `includeSiblings` to `cleanSourceWith`. When `true` it prevents the `subDir` arg from causing filtering of other directories. * Adds `keepGitDir` to `cleanGit` to allow `.git` directory to be kept (useful for components that use the `githash` package).
1 parent faaca67 commit 64efc98

16 files changed

+383
-105
lines changed

builder/comp-builder.nix

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,27 @@ let self =
7777
let
7878
# TODO fix cabal wildcard support so hpack wildcards can be mapped to cabal wildcards
7979
canCleanSource = !(cabal-generator == "hpack" && !(package.cleanHpack or false));
80-
cleanSrc = if canCleanSource then haskellLib.cleanCabalComponent package component src else src;
80+
# In order to support relative references to other packages we need to use
81+
# the `origSrc` diretory as the root `src` for the derivation.
82+
# We use `rootAndSubDir` here to split the cleaned source into a `cleanSrc.root`
83+
# path (that respects the filtering) and a `cleanSrc.subDir` that
84+
# is the sub directory in that root path that contains the package.
85+
# `cleanSrc.subDir` is used in `prePatch` and `lib/cover.nix`.
86+
cleanSrc = haskellLib.rootAndSubDir (if canCleanSource
87+
then haskellLib.cleanCabalComponent package component "${componentId.ctype}-${componentId.cname}" src
88+
else
89+
# We can clean out the siblings though to at least avoid changes to other packages
90+
# from triggering a rebuild of this one.
91+
# Passing `subDir` but not `includeSiblings = true;` will exclude anything not
92+
# in the `subDir`.
93+
if src ? origSrc && src ? filter && src.origSubDir or "" != ""
94+
then haskellLib.cleanSourceWith {
95+
name = src.name or "source";
96+
src = src.origSrc;
97+
subDir = lib.removePrefix "/" src.origSubDir;
98+
inherit (src) filter;
99+
}
100+
else src);
81101

82102
nameOnly = "${package.identifier.name}-${componentId.ctype}-${componentId.cname}";
83103

@@ -189,7 +209,7 @@ let
189209

190210
# Attributes that are common to both the build and haddock derivations
191211
commonAttrs = {
192-
src = cleanSrc;
212+
src = cleanSrc.root;
193213

194214
LANG = "en_US.UTF-8"; # GHC needs the locale configured during the Haddock phase.
195215
LC_ALL = "en_US.UTF-8";
@@ -198,14 +218,27 @@ let
198218

199219
SETUP_HS = setup + /bin/Setup;
200220

201-
prePatch = if (cabalFile != null)
202-
then ''cat ${cabalFile} > ${package.identifier.name}.cabal''
203-
else
204-
# When building hpack package we use the internal nix-tools
205-
# (compiled with a fixed GHC version)
206-
lib.optionalString (cabal-generator == "hpack") ''
207-
${buildPackages.haskell-nix.internal-nix-tools}/bin/hpack
208-
'';
221+
prePatch =
222+
# If the package is in a sub directory `cd` there first.
223+
# In some cases the `cleanSrc.subDir` will be empty and the `.cabal`
224+
# file will be in the root of `src` (`cleanSrc.root`). This
225+
# will happen when:
226+
# * the .cabal file is in the projects `src.origSrc or src`
227+
# * the package src was overridden with a value that does not
228+
# include an `origSubDir`
229+
(lib.optionalString (cleanSrc.subDir != "") ''
230+
cd ${lib.removePrefix "/" cleanSrc.subDir}
231+
''
232+
) +
233+
(if cabalFile != null
234+
then ''cat ${cabalFile} > ${package.identifier.name}.cabal''
235+
else
236+
# When building hpack package we use the internal nix-tools
237+
# (compiled with a fixed GHC version)
238+
lib.optionalString (cabal-generator == "hpack") ''
239+
${buildPackages.haskell-nix.internal-nix-tools}/bin/hpack
240+
''
241+
);
209242
}
210243
# patches can (if they like) depend on the version and revision of the package.
211244
// lib.optionalAttrs (patches != []) {
@@ -244,7 +277,9 @@ let
244277
passthru = {
245278
inherit (package) identifier;
246279
config = component;
247-
inherit configFiles executableToolDepends cleanSrc exeName;
280+
srcSubDir = cleanSrc.subDir;
281+
srcSubDirPath = cleanSrc.root + cleanSrc.subDir;
282+
inherit configFiles executableToolDepends exeName;
248283
exePath = drv + "/bin/${exeName}";
249284
env = shellWrappers;
250285
profiled = self (drvArgs // { enableLibraryProfiling = true; });

builder/hspkg-builder.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ in rec {
118118
checks = pkgs.recurseIntoAttrs (builtins.mapAttrs
119119
(_: d: haskellLib.check d)
120120
(lib.filterAttrs (_: d: d.config.doCheck) components.tests));
121-
inherit (package) identifier detailLevel isLocal;
121+
inherit (package) identifier detailLevel isLocal isProject;
122122
inherit setup cabalFile;
123123
isHaskell = true;
124124
inherit src;

builder/setup-builder.nix

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
, prePatch ? null, postPatch ? null
66
, preBuild ? component.preBuild , postBuild ? component.postBuild
77
, preInstall ? component.preInstall , postInstall ? component.postInstall
8-
, cleanSrc ? haskellLib.cleanCabalComponent package component src
8+
, cleanSrc ? haskellLib.cleanCabalComponent package component "setup" src
99
}:
1010

1111
let
12+
cleanSrc' = haskellLib.rootAndSubDir cleanSrc;
13+
1214
fullName = "${name}-setup";
1315

1416
includeGhcPackage = lib.any (p: p.identifier.name == "ghc") component.depends;
@@ -35,7 +37,7 @@ let
3537
drv =
3638
stdenv.mkDerivation ({
3739
name = "${ghc.targetPrefix}${fullName}";
38-
src = cleanSrc;
40+
src = cleanSrc'.root;
3941
buildInputs = component.libs
4042
++ component.frameworks
4143
++ builtins.concatLists component.pkgconfig;
@@ -44,7 +46,10 @@ let
4446
passthru = {
4547
inherit (package) identifier;
4648
config = component;
47-
inherit configFiles cleanSrc;
49+
srcSubDir = cleanSrc'.subDir;
50+
srcSubDirPath = cleanSrc'.root + cleanSrc'.subDir;
51+
cleanSrc = cleanSrc';
52+
inherit configFiles;
4853
};
4954

5055
meta = {
@@ -83,6 +88,13 @@ let
8388
runHook postInstall
8489
'';
8590
}
91+
// (lib.optionalAttrs (cleanSrc'.subDir != "") {
92+
prePatch =
93+
# If the package is in a sub directory `cd` there first
94+
''
95+
cd ${lib.removePrefix "/" cleanSrc'.subDir}
96+
'';
97+
})
8698
// (lib.optionalAttrs (patches != []) { patches = map (p: if builtins.isFunction p then p { inherit (package.identifier) version; inherit revision; } else p) patches; })
8799
// hooks
88100
);

changelog.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
This file contains a summary of changes to Haskell.nix and `nix-tools`
22
that will impact users.
33

4+
## Jan 14, 2021
5+
* Added support for cross package refs (with a project). Relative
6+
directory references between packages within a project should now
7+
work.
8+
* Added `includeSiblings` to `cleanSourceWith`. When `true` it
9+
prevents the `subDir` arg from causing filtering of other directories.
10+
* Added `keepGitDir` to `cleanGit` to allow `.git` directory to be kept
11+
(useful for components that use the `githash` package).
12+
413
## Nov 26, 2020
514
* Renamed `otherShells` arg for `shellFor` to `inputsFrom
615

lib/clean-cabal-component.nix

Lines changed: 95 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,127 @@
11
# Use cleanSourceWith to filter just the files needed for a particular
22
# component of the package
3-
{ lib, cleanSourceWith }: package: component: src:
3+
{ lib, cleanSourceWith, canCleanSource }: package: component: componentName: src:
44
let
5-
srcStr' = src.origSrcSubDir or src.origSrc or null;
5+
srcStr' = src.origSrc or null;
6+
subDir = if src.origSubDir or "" == ""
7+
then ""
8+
else lib.removePrefix "/" src.origSubDir + "/";
9+
# Remove a directory for each .. part of a path.
10+
removeDotDots = parts: lib.reverseList (
11+
builtins.foldl' (a: b:
12+
if b == ".."
13+
then builtins.tail a
14+
else builtins.concatLists [ [b] a ]) [] parts);
615
# Transform
7-
# "." -> ""
8-
# "./." -> ""
9-
# "./xyz" -> "xyz"
10-
normalizeRelativePath = rel:
11-
if rel == "." || rel == "./."
12-
then ""
13-
else lib.strings.removePrefix "./" rel;
14-
# Like normalizeRelativePath but with a trailing / when needed
15-
normalizeRelativeDir = dir:
16-
let p = normalizeRelativePath dir;
16+
# "." -> ""
17+
# "./." -> ""
18+
# "./xyz" -> "xyz"
19+
# "../abc" -> ERROR
20+
# "abc/.." -> ""
21+
# "abc/../xyz" -> "xyz"
22+
# "abc/./xyz" -> "abc/xyz"
23+
# "abc/./../xyz" -> "xyz"
24+
# "abc/.././xyz" -> "xyz"
25+
# "abc/" -> "abc/"
26+
normalizeRelativePath = path:
27+
let
28+
# Split the path into component parts and remove the empty ones and single dots.
29+
nonEmptyParts = lib.filter (x: x != "" && x != ".") (lib.splitString "/" path);
30+
in lib.concatStringsSep "/" (removeDotDots nonEmptyParts)
31+
# Keep the trailing slash if there was one.
32+
+ (if lib.hasSuffix "/" path then "/" else "");
33+
isAbsolutePath = path: lib.hasPrefix "/" path;
34+
isRelativePath = path: !(isAbsolutePath path);
35+
normalizePath = path:
36+
(if isAbsolutePath path
37+
then "/"
38+
else ""
39+
) + normalizeRelativePath path;
40+
combinePaths = a: b: if isAbsolutePath b
41+
then b
42+
else normalizePath (a + "/" + b);
43+
# Like normalizePath but with a trailing / when needed
44+
normalizeDir = dir:
45+
let p = normalizePath dir;
1746
in if p == "" || p == "/"
1847
then ""
1948
else if lib.hasSuffix "/" p
2049
then p
2150
else p + "/";
2251
in
23-
if srcStr' == null || package.detailLevel != "FullDetails"
52+
if srcStr' == null || package.detailLevel != "FullDetails" || !canCleanSource src
2453
then src
2554
else
2655
let
2756
srcStr = toString srcStr';
28-
dataDir = normalizeRelativeDir package.dataDir;
29-
hsSourceDirs = builtins.map normalizeRelativeDir component.hsSourceDirs
30-
++ (if component.hsSourceDirs == [] then [""] else []);
31-
includeDirs = builtins.map normalizeRelativeDir component.includeDirs;
32-
dirsNeeded = [dataDir]
57+
dataDir = combinePaths subDir package.dataDir;
58+
hsSourceDirs = builtins.map (d: combinePaths subDir d) component.hsSourceDirs
59+
++ (if component.hsSourceDirs == [] then [subDir] else []);
60+
includeDirs = builtins.map (d: combinePaths subDir d) component.includeDirs;
61+
dirsNeeded = builtins.map (d: combinePaths subDir d) (
62+
[dataDir]
3363
++ hsSourceDirs
34-
++ includeDirs;
64+
++ includeDirs
65+
++ package.licenseFiles
66+
++ package.extraSrcFiles
67+
++ component.extraSrcFiles
68+
++ package.extraDocFiles
69+
++ builtins.map (f: dataDir + f) package.dataFiles
70+
++ otherSourceFiles);
3571
fileMatch = dir: list:
3672
let
37-
prefixes = builtins.map (f: dir + f) (
73+
prefixes = builtins.map (f: combinePaths dir f) (
3874
lib.lists.remove null (lib.lists.flatten (
3975
builtins.map (f: builtins.match "([^*]*)[*].*" f) list)));
40-
exactMatches = builtins.map (f: dataDir + f) (
76+
exactMatches = builtins.map (f: combinePaths dir f) (
4177
lib.lists.remove null (lib.lists.flatten (
4278
builtins.map (f: builtins.match "([^*]*)" f) list)));
4379
in rPath: lib.any (d: lib.strings.hasPrefix d rPath) prefixes
4480
|| lib.any (d: d == rPath) exactMatches;
4581
dataFileMatch = fileMatch dataDir package.dataFiles;
46-
licenseMatch = fileMatch "" package.licenseFiles;
47-
extraSrcMatch = fileMatch "" (
82+
licenseMatch = fileMatch subDir package.licenseFiles;
83+
extraSrcMatch = fileMatch subDir (
4884
package.extraSrcFiles
4985
++ component.extraSrcFiles);
50-
extraDocMatch = fileMatch "" package.extraDocFiles;
51-
otherSourceFiles =
86+
extraDocMatch = fileMatch subDir package.extraDocFiles;
87+
otherSourceFiles = builtins.map (f: combinePaths subDir f) (
5288
component.asmSources
5389
++ component.cmmSources
5490
++ component.cSources
5591
++ component.cxxSources
56-
++ component.jsSources;
92+
++ component.jsSources);
5793
in cleanSourceWith {
58-
inherit src;
59-
filter = path: type:
60-
assert (if !lib.strings.hasPrefix (srcStr + "/") (path + "/")
61-
then throw ("Unexpected path " + path + " (expected something in " + srcStr + "/)")
62-
else true);
63-
let
64-
srcStrLen = lib.strings.stringLength srcStr;
65-
rPath = lib.strings.substring (srcStrLen + 1) (lib.strings.stringLength path - srcStrLen - 1) path;
66-
# This is a handy way to find out why different files are included
67-
# traceReason = reason: v: if v then builtins.trace (rPath + " : " + reason) true else false;
68-
traceReason = reason: v: v;
69-
in
70-
traceReason "directory is needed" (
71-
lib.any (d: lib.strings.hasPrefix (rPath + "/") d) (
72-
dirsNeeded
73-
++ package.licenseFiles
74-
++ package.extraSrcFiles
75-
++ component.extraSrcFiles
76-
++ package.extraDocFiles
77-
++ builtins.map (f: dataDir + f) package.dataFiles
78-
++ otherSourceFiles))
79-
|| traceReason "cabal package definition" (lib.strings.hasSuffix ".cabal" rPath)
80-
|| traceReason "hpack package defintion" (rPath == "package.yaml")
81-
|| traceReason "data file" (lib.strings.hasPrefix dataDir rPath
82-
&& dataFileMatch rPath)
83-
|| traceReason "haskell source dir" (lib.any (d: lib.strings.hasPrefix d rPath) hsSourceDirs)
84-
|| traceReason "include dir" (lib.any (d: lib.strings.hasPrefix d rPath) includeDirs)
85-
|| traceReason "license file" (licenseMatch rPath)
86-
|| traceReason "extra source file" (extraSrcMatch rPath)
87-
|| traceReason "extra doc file" (extraDocMatch rPath)
88-
|| traceReason "other source file" (lib.any (f: f == rPath) otherSourceFiles);
94+
name = src.name or "source" + "-${componentName}";
95+
subDir = lib.removePrefix "/" (src.origSubDir or "");
96+
includeSiblings = true;
97+
src = cleanSourceWith {
98+
src = src.origSrc or src;
99+
filter = path: type:
100+
(!(src ? filter) || src.filter path type) && (
101+
assert (if !lib.strings.hasPrefix (srcStr + "/") (path + "/")
102+
then throw ("Unexpected path " + path + " (expected something in " + srcStr + "/)")
103+
else true);
104+
let
105+
srcStrLen = lib.strings.stringLength srcStr;
106+
rPath = lib.strings.substring (srcStrLen + 1) (lib.strings.stringLength path - srcStrLen - 1) path;
107+
# This is a handy way to find out why different files are included
108+
# traceReason = reason: v: if v then builtins.trace (rPath + " : " + reason) true else false;
109+
traceReason = reason: v: v;
110+
in
111+
traceReason "directory is needed" (
112+
lib.any (d: lib.strings.hasPrefix (rPath + "/") d) dirsNeeded)
113+
|| traceReason "cabal package definition" (lib.strings.hasPrefix subDir rPath
114+
&& lib.strings.hasSuffix ".cabal" rPath)
115+
|| traceReason "hpack package defintion" (lib.strings.hasPrefix subDir rPath
116+
&& rPath == "package.yaml")
117+
|| traceReason "data file" (lib.strings.hasPrefix dataDir rPath
118+
&& dataFileMatch rPath)
119+
|| traceReason "haskell source dir" (lib.any (d: lib.strings.hasPrefix d rPath) hsSourceDirs)
120+
|| traceReason "include dir" (lib.any (d: lib.strings.hasPrefix d rPath) includeDirs)
121+
|| traceReason "license file" (licenseMatch rPath)
122+
|| traceReason "extra source file" (extraSrcMatch rPath)
123+
|| traceReason "extra doc file" (extraDocMatch rPath)
124+
|| traceReason "other source file" (lib.any (f: f == rPath) otherSourceFiles)
125+
);
126+
};
89127
}

0 commit comments

Comments
 (0)