Merge pull request #14598 from NixOS/nar-listing-dedup

Deduplicate `listNar` and `MemorySourceAccessor::File`

Author: Ericson2314

src/libstore-tests/references.cc

@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
     // Create an in-memory file system with various reference patterns
     auto accessor = make_ref<MemorySourceAccessor>();
     accessor->root = File::Directory{
-        .contents{
+        .entries{
             {
                 // file1.txt: contains hash1
                 "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
                 // subdir: a subdirectory
                 "subdir",
                 File::Directory{
-                    .contents{
+                    .entries{
                         {
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",

src/libstore/binary-cache-store.cc

@@ -8,7 +8,7 @@
 #include "nix/util/sync.hh"
 #include "nix/store/remote-fs-accessor.hh"
 #include "nix/store/nar-info-disk-cache.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
+            {"root", listNarDeep(*narAccessor, CanonPath::root)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/include/nix/store/meson.build

@@ -55,7 +55,6 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
-  'nar-accessor.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-accessor.hh

@@ -300,7 +300,6 @@ sources = files(
   'make-content-addressed.cc',
   'misc.cc',
   'names.cc',
-  'nar-accessor.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/meson.build

@@ -1,6 +1,6 @@
 #include <nlohmann/json.hpp>
 #include "nix/store/remote-fs-accessor.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libstore/remote-fs-accessor.cc

@@ -230,7 +230,7 @@ TEST_F(GitTest, both_roundrip)
 
     auto files = make_ref<MemorySourceAccessor>();
     files->root = File::Directory{
-        .contents{
+        .entries{
             {
                 "foo",
                 File::Regular{
@@ -240,7 +240,7 @@ TEST_F(GitTest, both_roundrip)
             {
                 "bar",
                 File::Directory{
-                    .contents =
+                    .entries =
                         {
                             {
                                 "baz",

src/libutil-tests/git.cc

@@ -4,59 +4,111 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
 /**
- * An source accessor for an in-memory file system.
+ * File System Object definitions
+ *
+ * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
  */
-struct MemorySourceAccessor : virtual SourceAccessor
+namespace fso {
+
+template<typename RegularContents>
+struct Regular
+{
+    bool executable = false;
+    RegularContents contents;
+
+    auto operator<=>(const Regular &) const = default;
+};
+
+/**
+ * Child parameter because sometimes we want "shallow" directories without
+ * full file children.
+ */
+template<typename Child>
+struct DirectoryT
+{
+    using Name = std::string;
+
+    std::map<Name, Child, std::less<>> entries;
+
+    inline bool operator==(const DirectoryT &) const noexcept;
+    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
+};
+
+struct Symlink
+{
+    std::string target;
+
+    auto operator<=>(const Symlink &) const = default;
+};
+
+/**
+ * For when we know there is child, but don't know anything about it.
+ *
+ * This is not part of the core File System Object data model --- this
+ * represents not knowing, not an additional type of file.
+ */
+struct Opaque
+{
+    auto operator<=>(const Opaque &) const = default;
+};
+
+/**
+ * `File<std::string>` nicely defining what a "file system object"
+ * is in Nix.
+ *
+ * With a different type arugment, it is also can be a "skeletal"
+ * version is that abstract syntax for a "NAR listing".
+ */
+template<typename RegularContents, bool recur>
+struct VariantT
 {
+    bool operator==(const VariantT &) const noexcept;
+    std::strong_ordering operator<=>(const VariantT &) const noexcept;
+
+    using Regular = nix::fso::Regular<RegularContents>;
+
     /**
-     * In addition to being part of the implementation of
-     * `MemorySourceAccessor`, this has a side benefit of nicely
-     * defining what a "file system object" is in Nix.
+     * In the default case, we do want full file children for our directory.
      */
-    struct File
-    {
-        bool operator==(const File &) const noexcept;
-        std::strong_ordering operator<=>(const File &) const noexcept;
+    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
 
-        struct Regular
-        {
-            bool executable = false;
-            std::string contents;
+    using Symlink = nix::fso::Symlink;
 
-            bool operator==(const Regular &) const = default;
-            auto operator<=>(const Regular &) const = default;
-        };
+    using Raw = std::variant<Regular, Directory, Symlink>;
+    Raw raw;
 
-        struct Directory
-        {
-            using Name = std::string;
+    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
 
-            std::map<Name, File, std::less<>> contents;
+    SourceAccessor::Stat lstat() const;
+};
 
-            bool operator==(const Directory &) const noexcept;
-            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
-            bool operator<(const Directory &) const noexcept;
-        };
+template<typename Child>
+inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
 
-        struct Symlink
-        {
-            std::string target;
+template<typename Child>
+inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
 
-            bool operator==(const Symlink &) const = default;
-            auto operator<=>(const Symlink &) const = default;
-        };
+template<typename RegularContents, bool recur>
+inline bool
+VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
 
-        using Raw = std::variant<Regular, Directory, Symlink>;
-        Raw raw;
+template<typename RegularContents, bool recur>
+inline std::strong_ordering
+VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
 
-        MAKE_WRAPPER_CONSTRUCTOR(File);
+} // namespace fso
 
-        Stat lstat() const;
-    };
+/**
+ * An source accessor for an in-memory file system.
+ */
+struct MemorySourceAccessor : virtual SourceAccessor
+{
+    using File = fso::VariantT<std::string, true>;
 
     std::optional<File> root;
 
@@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
-inline bool MemorySourceAccessor::File::Directory::operator==(
-    const MemorySourceAccessor::File::Directory &) const noexcept = default;
-
-inline bool
-MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
-{
-    return contents < other.contents;
-}
-
-inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
-inline std::strong_ordering
-MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
-
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -50,6 +50,7 @@ headers = files(
   'memory-source-accessor.hh',
   'mounted-source-accessor.hh',
   'muxable-pipe.hh',
+  'nar-accessor.hh',
   'os-string.hh',
   'pool.hh',
   'pos-idx.hh',

src/libutil/include/nix/util/meson.build

@@ -0,0 +1,88 @@
+#pragma once
+///@file
+
+#include "nix/util/memory-source-accessor.hh"
+
+#include <functional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+struct Source;
+
+/**
+ * Return an object that provides access to the contents of a NAR
+ * file.
+ */
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
+
+ref<SourceAccessor> makeNarAccessor(Source & source);
+
+/**
+ * Create a NAR accessor from a NAR listing (in the format produced by
+ * listNar()). The callback getNarBytes(offset, length) is used by the
+ * readFile() method of the accessor to get the contents of files
+ * inside the NAR.
+ */
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
+
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
+
+struct NarListingRegularFile
+{
+    /**
+     * @see `SourceAccessor::Stat::fileSize`
+     */
+    std::optional<uint64_t> fileSize;
+
+    /**
+     * @see `SourceAccessor::Stat::narOffset`
+     *
+     * We only set to non-`std::nullopt` if it is also non-zero.
+     */
+    std::optional<uint64_t> narOffset;
+
+    auto operator<=>(const NarListingRegularFile &) const = default;
+};
+
+/**
+ * Abstract syntax for a "NAR listing".
+ */
+using NarListing = fso::VariantT<NarListingRegularFile, true>;
+
+/**
+ * Shallow NAR listing where directory children are not recursively expanded.
+ * Uses a variant that can hold Regular/Symlink fully, but Directory children
+ * are just unit types indicating presence without content.
+ */
+using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
+
+/**
+ * Return a deep structured representation of the contents of a NAR (except file
+ * contents), recursively listing all children.
+ */
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Return a shallow structured representation of the contents of a NAR (except file
+ * contents), only listing immediate children without recursing.
+ */
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Serialize a NarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const NarListing & listing);
+
+/**
+ * Serialize a ShallowNarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const ShallowNarListing & listing);
+
+} // namespace nix