Backward compatibility hack for Git inputs depending on Nix < 2.20 behaviour

Before Nix 2.20, we used git, which applies Git filters (in particular
doing end-of-line conversion based on .gitattributes), `export-ignore`
and `export-subst`. In 2.20, we switched to libgit2 and stopped
applying those, which is probably better for reproducibility
(e.g. `export-subst` can do pretty crazy things). However, that breaks
existing lock files / fetchTree calls for Git inputs that rely on
them, since it invalidates the NAR hash.

So as a backward compatibility hack, we now check the NAR hash
computed over the Git tree without them. If there is a hash mismatch,
we try again *with* them (by calling `git archive`, just like Nix <
2.20). If that succeeds, we print a warning and return the filtered
tree. This is inefficient, but should be fine for legacy flakes.

Author: edolstra

src/libfetchers/git.cc

@@ -16,6 +16,7 @@
 #include "nix/util/json-utils.hh"
 #include "nix/util/archive.hh"
 #include "nix/util/mounted-source-accessor.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 #include <regex>
 #include <string.h>
@@ -643,6 +644,30 @@ struct GitInputScheme : InputScheme
                + (getLfsAttr(input) ? ";l" : "");
     }
 
+    /**
+     * Get a `SourceAccessor` for the given Git revision by creating a git archive and unpacking it to the Nix store.
+     * This is used for Nix < 2.20 compatibility.
+     */
+    ref<SourceAccessor> getGitArchiveAccessor(
+        Store & store, RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
+    {
+        auto tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+
+        auto source = sinkToSource([&](Sink & sink) {
+            runProgram2(
+                {.program = "git",
+                 .args = {"-C", repoDir, "--git-dir", repoInfo.gitDir, "archive", rev.gitRev()},
+                 .standardOut = &sink});
+        });
+
+        unpackTarfile(*source, tmpDir);
+
+        auto storePath = store.addToStore("source", {getFSSourceAccessor(), CanonPath(tmpDir)});
+
+        return ref{store.getFSAccessor(storePath)};
+    }
+
     std::pair<ref<SourceAccessor>, Input>
     getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
@@ -791,6 +816,34 @@ struct GitInputScheme : InputScheme
         };
         auto accessor = repo->getAccessor(rev, options, "«" + input.to_string(true) + "»");
 
+        /* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Nix applying Git filters,
+         * `export-ignore` or `export-subst`. Nix >= 2.20 doesn't do those, so we may get a NAR hash mismatch. If that
+         * happens, try again using `git archive`. */
+        if (auto expectedNarHash = input.getNarHash()) {
+            if (accessor->pathExists(CanonPath(".gitattributes"))) {
+                auto narHashNew =
+                    fetchToStore2(settings, *store, {accessor}, FetchMode::DryRun, input.getName()).second;
+                if (expectedNarHash != narHashNew) {
+                    GitAccessorOptions options2{.exportIgnore = true};
+                    auto accessor2 = getGitArchiveAccessor(*store, repoInfo, repoDir, rev);
+                    accessor2->fingerprint = options2.makeFingerprint(rev) + ";f";
+                    auto narHashOld =
+                        fetchToStore2(settings, *store, {accessor2}, FetchMode::DryRun, input.getName()).second;
+                    if (expectedNarHash == narHashOld) {
+                        warn(
+                            "Git input '%s' specifies a NAR hash '%s' that was created by Nix < 2.20.\n"
+                            "Nix >= 2.20 does not apply Git filters, `export-ignore` and `export-subst` by default, which changes the NAR hash.\n"
+                            "Please update the NAR hash to '%s'.",
+                            input.to_string(),
+                            expectedNarHash->to_string(HashFormat::SRI, true),
+                            narHashNew.to_string(HashFormat::SRI, true));
+                        accessor = accessor2;
+                        options = options2;
+                    }
+                }
+            }
+        }
+
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */

tests/functional/fetchGit.sh

@@ -310,3 +310,36 @@ git -C "$empty" config user.name "Foobar"
 git -C "$empty" commit --allow-empty --allow-empty-message --message ""
 
 nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.lastModified != 0; assert attrs.rev != \"0000000000000000000000000000000000000000\"; assert attrs.revCount == 1; true"
+
+# Test backward compatibility hack for Nix < 2.20 locks / fetchTree calls that expect Git filters to be applied.
+eol="$TEST_ROOT/git-eol"
+mkdir -p "$eol"
+git init "$eol"
+git -C "$eol" config user.email "foobar@example.com"
+git -C "$eol" config user.name "Foobar"
+printf "Hello\nWorld\n" > "$eol/crlf"
+printf "ignore me" > "$eol/ignored"
+git -C "$eol" add crlf ignored
+git -C "$eol" commit -a -m Initial
+echo "Version: \$Format:%s\$" > "$eol/version"
+printf "crlf text eol=crlf\nignored export-ignore\nversion export-subst\n" > "$eol/.gitattributes"
+git -C "$eol" add .gitattributes version
+git -C "$eol" commit -a -m 'Apply gitattributes'
+
+rev="$(git -C "$eol" rev-parse HEAD)"
+
+export _NIX_TEST_BARF_ON_UNCACHEABLE=1
+
+oldHash="sha256-cOuYSqDjvOBmKCuH5nXEfHRIAUVJZlictW0raF+3ynk="
+newHash="sha256-WZ5VePvmUcbRbkWLlNtCywWrAcr7EvVeJP8xKdZR7pc="
+
+expectStderr 0 nix eval --expr \
+    "let tree = builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"$oldHash\"; }; in assert builtins.readFile \"\${tree}/crlf\" == \"Hello\r\nWorld\r\n\"; assert !builtins.pathExists \"\${tree}/ignored\"; assert builtins.readFile \"\${tree}/version\" == \"Version: Apply gitattributes\n\"; true" \
+    | grepQuiet "Please update the NAR hash to '$newHash'"
+
+nix eval --expr \
+    "let tree = builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"$newHash\"; }; in assert builtins.readFile \"\${tree}/crlf\" == \"Hello\nWorld\n\"; assert builtins.pathExists \"\${tree}/ignored\"; assert builtins.readFile \"\${tree}/version\" == \"Version: \$Format:%s\$\n\"; true"
+
+expectStderr 102 nix eval --expr \
+    "builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"sha256-DLDvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA=\"; }" \
+    | grepQuiet "NAR hash mismatch"