[Do not merge yet] Support SHA-256 git repos with libgit2 experimental API

packaging/dependencies.nix

@@ -74,4 +74,14 @@ scope: {
         buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase;
         installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase;
       });
+
+  libgit2 = pkgs.libgit2.overrideAttrs (attrs: {
+    cmakeFlags = (attrs.cmakeFlags or [ ]) ++ [
+      (lib.mesonBool "EXPERIMENTAL_SHA256" true)
+    ];
+
+    postInstall = (attrs.postInstall or "") + ''
+      substituteInPlace $(find $dev/include -type f) --replace-quiet '#include "git2/' '#include "git2-experimental/'
+    '';
+  });
 }

src/libfetchers-tests/git-utils.cc

@@ -1,19 +1,19 @@
 #include "nix/fetchers/git-utils.hh"
 #include "nix/util/file-system.hh"
 #include <gmock/gmock.h>
-#include <git2/global.h>
-#include <git2/repository.h>
-#include <git2/signature.h>
-#include <git2/types.h>
-#include <git2/object.h>
-#include <git2/tag.h>
+#include <git2-experimental/global.h>
+#include <git2-experimental/repository.h>
+#include <git2-experimental/signature.h>
+#include <git2-experimental/types.h>
+#include <git2-experimental/object.h>
+#include <git2-experimental/tag.h>
 #include <gtest/gtest.h>
 #include "nix/util/fs-sink.hh"
 #include "nix/util/serialise.hh"
 #include "nix/fetchers/git-lfs-fetch.hh"
 
-#include <git2/blob.h>
-#include <git2/tree.h>
+#include <git2-experimental/blob.h>
+#include <git2-experimental/tree.h>
 
 namespace nix {
 

src/libfetchers-tests/git.cc

@@ -5,7 +5,7 @@
 #include "nix/fetchers/fetchers.hh"
 #include "nix/fetchers/git-utils.hh"
 
-#include <git2.h>
+#include <git2-experimental.h>
 #include <gtest/gtest.h>
 
 #include <filesystem>

src/libfetchers-tests/meson.build

@@ -33,7 +33,7 @@ deps_private += rapidcheck
 gtest = dependency('gtest', main : true)
 deps_private += gtest
 
-libgit2 = dependency('libgit2')
+libgit2 = dependency('libgit2-experimental')
 deps_private += libgit2
 
 subdir('nix-meson-build-support/common')

src/libfetchers/attrs.cc

@@ -1,5 +1,6 @@
 #include "nix/fetchers/attrs.hh"
 #include "nix/fetchers/fetchers.hh"
+#include "nix/fetchers/git-utils.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -111,7 +112,7 @@ StringMap attrsToQuery(const Attrs & attrs)
 
 Hash getRevAttr(const Attrs & attrs, const std::string & name)
 {
-    return Hash::parseAny(getStrAttr(attrs, name), HashAlgorithm::SHA1);
+    return parseGitHash(getStrAttr(attrs, name));
 }
 
 } // namespace nix::fetchers

src/libfetchers/fetchers.cc

@@ -447,6 +447,9 @@ std::optional<Hash> Input::getRev() const
         } catch (BadHash & e) {
             // Default to sha1 for backwards compatibility with existing
             // usages (e.g. `builtins.fetchTree` calls or flake inputs).
+            //
+            // Note that means that for SHA-256 git repos, prefixing
+            // must be used.
             hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
         }
     }

src/libfetchers/git-lfs-fetch.cc

@@ -7,10 +7,10 @@
 #include "nix/util/hash.hh"
 #include "nix/store/ssh.hh"
 
-#include <git2/attr.h>
-#include <git2/config.h>
-#include <git2/errors.h>
-#include <git2/remote.h>
+#include <git2-experimental/attr.h>
+#include <git2-experimental/config.h>
+#include <git2-experimental/errors.h>
+#include <git2-experimental/remote.h>
 
 #include <nlohmann/json.hpp>
 

src/libfetchers/git-utils.cc

@@ -13,27 +13,27 @@
 #include "nix/util/thread-pool.hh"
 #include "nix/util/pool.hh"
 
-#include <git2/attr.h>
-#include <git2/blob.h>
-#include <git2/branch.h>
-#include <git2/commit.h>
-#include <git2/config.h>
-#include <git2/describe.h>
-#include <git2/errors.h>
-#include <git2/global.h>
-#include <git2/indexer.h>
-#include <git2/object.h>
-#include <git2/odb.h>
-#include <git2/refs.h>
-#include <git2/remote.h>
-#include <git2/repository.h>
-#include <git2/revparse.h>
-#include <git2/status.h>
-#include <git2/submodule.h>
-#include <git2/sys/odb_backend.h>
-#include <git2/sys/mempack.h>
-#include <git2/tag.h>
-#include <git2/tree.h>
+#include <git2-experimental/attr.h>
+#include <git2-experimental/blob.h>
+#include <git2-experimental/branch.h>
+#include <git2-experimental/commit.h>
+#include <git2-experimental/config.h>
+#include <git2-experimental/describe.h>
+#include <git2-experimental/errors.h>
+#include <git2-experimental/global.h>
+#include <git2-experimental/indexer.h>
+#include <git2-experimental/object.h>
+#include <git2-experimental/odb.h>
+#include <git2-experimental/refs.h>
+#include <git2-experimental/remote.h>
+#include <git2-experimental/repository.h>
+#include <git2-experimental/revparse.h>
+#include <git2-experimental/status.h>
+#include <git2-experimental/submodule.h>
+#include <git2-experimental/sys/odb_backend.h>
+#include <git2-experimental/sys/mempack.h>
+#include <git2-experimental/tag.h>
+#include <git2-experimental/tree.h>
 
 #include <boost/unordered/concurrent_flat_set.hpp>
 #include <boost/unordered/unordered_flat_map.hpp>
@@ -91,10 +91,21 @@ typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
 
 Hash toHash(const git_oid & oid)
 {
-#ifdef GIT_EXPERIMENTAL_SHA256
-    assert(oid.type == GIT_OID_SHA1);
-#endif
-    Hash hash(HashAlgorithm::SHA1);
+    HashAlgorithm algo;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wswitch-enum"
+    switch (oid.type) {
+    case GIT_OID_SHA1:
+        algo = HashAlgorithm::SHA1;
+        break;
+    case GIT_OID_SHA256:
+        algo = HashAlgorithm::SHA256;
+        break;
+    default:
+        unreachable();
+    }
+#pragma GCC diagnostic pop
+    Hash hash(algo);
     memcpy(hash.hash, oid.id, hash.hashSize);
     return hash;
 }
@@ -111,7 +122,21 @@ static void initLibGit2()
 git_oid hashToOID(const Hash & hash)
 {
     git_oid oid;
-    if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
+    git_oid_t t;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wswitch-enum"
+    switch (hash.algo) {
+    case HashAlgorithm::SHA1:
+        t = GIT_OID_SHA1;
+        break;
+    case HashAlgorithm::SHA256:
+        t = GIT_OID_SHA256;
+        break;
+    default:
+        throw Error("unsupported hash algorithm for Git: %s", printHashAlgo(hash.algo));
+    }
+#pragma GCC diagnostic pop
+    if (git_oid_fromstr(&oid, hash.gitRev().c_str(), t))
         throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
     return oid;
 }
@@ -304,7 +329,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         //                     (synchronously on the git_packbuilder_write_buf thread)
         Indexer indexer;
         git_indexer_progress stats;
-        if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr))
+        git_indexer_options indexer_opts = GIT_INDEXER_OPTIONS_INIT;
+        if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), &indexer_opts))
             throw Error("creating git packfile indexer: %s", git_error_last()->message);
 
         // TODO: provide index callback for checkInterrupt() termination
@@ -1378,4 +1404,21 @@ bool isLegalRefName(const std::string & refName)
     return false;
 }
 
+Hash parseGitHash(std::string_view hashStr)
+{
+    HashAlgorithm algo;
+    switch (hashStr.size()) {
+    case 40:
+        algo = HashAlgorithm::SHA1;
+        break;
+    case 64:
+        algo = HashAlgorithm::SHA256;
+        break;
+    default:
+        throw Error(
+            "invalid git hash '%s': expected 40 (SHA1) or 64 (SHA256) hex characters, got %d", hashStr, hashStr.size());
+    }
+    return Hash::parseNonSRIUnprefixed(hashStr, algo);
+}
+
 } // namespace nix

src/libfetchers/github.cc

@@ -48,7 +48,7 @@ struct GitArchiveInputScheme : InputScheme
         auto size = path.size();
         if (size == 3) {
             if (std::regex_match(path[2], revRegex))
-                rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
+                rev = parseGitHash(path[2]);
             else if (isLegalRefName(path[2]))
                 ref = path[2];
             else
@@ -74,7 +74,7 @@ struct GitArchiveInputScheme : InputScheme
             if (name == "rev") {
                 if (rev)
                     throw BadURL("URL '%s' contains multiple commit hashes", url);
-                rev = Hash::parseAny(value, HashAlgorithm::SHA1);
+                rev = parseGitHash(value);
             } else if (name == "ref") {
                 if (!isLegalRefName(value))
                     throw BadURL("URL '%s' contains an invalid branch/tag name", url);
@@ -403,8 +403,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
             store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         return RefInfo{
-            .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
-            .treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)};
+            .rev = parseGitHash(std::string{json["sha"]}),
+            .treeHash = parseGitHash(std::string{json["commit"]["tree"]["sha"]})};
     }
 
     DownloadUrl getDownloadUrl(const Input & input) const override
@@ -478,7 +478,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
-            return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
+            return RefInfo{.rev = parseGitHash(std::string(json[0]["id"]))};
         }
         if (json.is_array() && json.size() == 0) {
             throw Error("No commits returned by GitLab API -- does the git ref really exist?");
@@ -579,7 +579,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         if (!id)
             throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
 
-        return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)};
+        return RefInfo{.rev = parseGitHash(*id)};
     }
 
     DownloadUrl getDownloadUrl(const Input & input) const override

src/libfetchers/include/nix/fetchers/git-lfs-fetch.hh

@@ -5,7 +5,7 @@
 #include "nix/util/serialise.hh"
 #include "nix/util/url.hh"
 
-#include <git2/repository.h>
+#include <git2-experimental/repository.h>
 
 #include <nlohmann/json_fwd.hpp>
 

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -167,4 +167,14 @@ struct Setter
  */
 bool isLegalRefName(const std::string & refName);
 
+/**
+ * Parse a base16-encoded git hash string and determine the hash
+ * algorithm based on the length (40 chars = SHA1, 64 chars = SHA256).
+ *
+ * @note For Nix-native information we should *not* do length tricks,
+ * but instead always rely on an explicit algorithm. This hack should be
+ * only for foreign hash literals.
+ */
+Hash parseGitHash(std::string_view hashStr);
+
 } // namespace nix

src/libfetchers/indirect.cc

@@ -23,7 +23,7 @@ struct IndirectInputScheme : InputScheme
         if (path.size() == 1) {
         } else if (path.size() == 2) {
             if (std::regex_match(path[1], revRegex))
-                rev = Hash::parseAny(path[1], HashAlgorithm::SHA1);
+                rev = parseGitHash(path[1]);
             else if (isLegalRefName(path[1]))
                 ref = path[1];
             else
@@ -34,7 +34,7 @@ struct IndirectInputScheme : InputScheme
             ref = path[1];
             if (!std::regex_match(path[2], revRegex))
                 throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]);
-            rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
+            rev = parseGitHash(path[2]);
         } else
             throw BadURL("GitHub URL '%s' is invalid", url);
 

src/libfetchers/meson.build

@@ -28,7 +28,7 @@ subdir('nix-meson-build-support/subprojects')
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
-libgit2 = dependency('libgit2', version : '>= 1.9')
+libgit2 = dependency('libgit2-experimental', version : '>= 1.9')
 deps_private += libgit2
 
 subdir('nix-meson-build-support/common')