Deduplicate listNar and MemorySourceAccessor::File

`listNar` did the not-so-pretty thing of going straight to JSON. Now it
uses `MemorySourceAccessor::File`, or rather variations of it, to go to
a C++ data type first, and only JSON second.

To accomplish this we add some type parameters to the `File` data type.
Actually, we need to do two rounds of this, because shallow NAR
listings. There is `FileT` and `DirectoryT` accordingly.

Author: Ericson2314

src/libstore/binary-cache-store.cc

@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(*narAccessor, CanonPath::root, true)},
+            {"root", listNarDeep(*narAccessor, CanonPath::root)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/remote-fs-accessor.cc

@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(*narAccessor, CanonPath::root, true);
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -4,59 +4,111 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
 /**
- * An source accessor for an in-memory file system.
+ * File System Object definitions
+ *
+ * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
  */
-struct MemorySourceAccessor : virtual SourceAccessor
+namespace fso {
+
+template<typename RegularContents>
+struct Regular
+{
+    bool executable = false;
+    RegularContents contents;
+
+    auto operator<=>(const Regular &) const = default;
+};
+
+/**
+ * Child parameter because sometimes we want "shallow" directories without
+ * full file children.
+ */
+template<typename Child>
+struct DirectoryT
+{
+    using Name = std::string;
+
+    std::map<Name, Child, std::less<>> entries;
+
+    inline bool operator==(const DirectoryT &) const noexcept;
+    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
+};
+
+struct Symlink
+{
+    std::string target;
+
+    auto operator<=>(const Symlink &) const = default;
+};
+
+/**
+ * For when we know there is child, but don't know anything about it.
+ *
+ * This is not part of the core File System Object data model --- this
+ * represents not knowing, not an additional type of file.
+ */
+struct Opaque
+{
+    auto operator<=>(const Opaque &) const = default;
+};
+
+/**
+ * `File<std::string>` nicely defining what a "file system object"
+ * is in Nix.
+ *
+ * With a different type arugment, it is also can be a "skeletal"
+ * version is that abstract syntax for a "NAR listing".
+ */
+template<typename RegularContents, bool recur>
+struct VariantT
 {
+    bool operator==(const VariantT &) const noexcept;
+    std::strong_ordering operator<=>(const VariantT &) const noexcept;
+
+    using Regular = nix::fso::Regular<RegularContents>;
+
     /**
-     * In addition to being part of the implementation of
-     * `MemorySourceAccessor`, this has a side benefit of nicely
-     * defining what a "file system object" is in Nix.
+     * In the default case, we do want full file children for our directory.
      */
-    struct File
-    {
-        bool operator==(const File &) const noexcept;
-        std::strong_ordering operator<=>(const File &) const noexcept;
+    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
 
-        struct Regular
-        {
-            bool executable = false;
-            std::string contents;
+    using Symlink = nix::fso::Symlink;
 
-            bool operator==(const Regular &) const = default;
-            auto operator<=>(const Regular &) const = default;
-        };
+    using Raw = std::variant<Regular, Directory, Symlink>;
+    Raw raw;
 
-        struct Directory
-        {
-            using Name = std::string;
+    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
 
-            std::map<Name, File, std::less<>> entries;
+    SourceAccessor::Stat lstat() const;
+};
 
-            bool operator==(const Directory &) const noexcept;
-            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
-            bool operator<(const Directory &) const noexcept;
-        };
+template<typename Child>
+inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
 
-        struct Symlink
-        {
-            std::string target;
+template<typename Child>
+inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
 
-            bool operator==(const Symlink &) const = default;
-            auto operator<=>(const Symlink &) const = default;
-        };
+template<typename RegularContents, bool recur>
+inline bool
+VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
 
-        using Raw = std::variant<Regular, Directory, Symlink>;
-        Raw raw;
+template<typename RegularContents, bool recur>
+inline std::strong_ordering
+VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
 
-        MAKE_WRAPPER_CONSTRUCTOR(File);
+} // namespace fso
 
-        Stat lstat() const;
-    };
+/**
+ * An source accessor for an in-memory file system.
+ */
+struct MemorySourceAccessor : virtual SourceAccessor
+{
+    using File = fso::VariantT<std::string, true>;
 
     std::optional<File> root;
 
@@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
-inline bool MemorySourceAccessor::File::Directory::operator==(
-    const MemorySourceAccessor::File::Directory &) const noexcept = default;
-
-inline bool
-MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
-{
-    return entries < other.entries;
-}
-
-inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
-inline std::strong_ordering
-MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
-
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */

src/libutil/include/nix/util/nar-accessor.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "nix/util/source-accessor.hh"
+#include "nix/util/memory-source-accessor.hh"
 
 #include <functional>
 
@@ -34,10 +34,55 @@ GetNarBytes seekableGetNarBytes(const Path & path);
 
 ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
 
+struct NarListingRegularFile
+{
+    /**
+     * @see `SourceAccessor::Stat::fileSize`
+     */
+    std::optional<uint64_t> fileSize;
+
+    /**
+     * @see `SourceAccessor::Stat::narOffset`
+     *
+     * We only set to non-`std::nullopt` if it is also non-zero.
+     */
+    std::optional<uint64_t> narOffset;
+
+    auto operator<=>(const NarListingRegularFile &) const = default;
+};
+
+/**
+ * Abstract syntax for a "NAR listing".
+ */
+using NarListing = fso::VariantT<NarListingRegularFile, true>;
+
+/**
+ * Shallow NAR listing where directory children are not recursively expanded.
+ * Uses a variant that can hold Regular/Symlink fully, but Directory children
+ * are just unit types indicating presence without content.
+ */
+using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
+
+/**
+ * Return a deep structured representation of the contents of a NAR (except file
+ * contents), recursively listing all children.
+ */
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Return a shallow structured representation of the contents of a NAR (except file
+ * contents), only listing immediate children without recursing.
+ */
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Serialize a NarListing to JSON.
+ */
+void to_json(nlohmann::json & j, const NarListing & listing);
+
 /**
- * Write a JSON representation of the contents of a NAR (except file
- * contents).
+ * Serialize a ShallowNarListing to JSON.
  */
-nlohmann::json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse);
+void to_json(nlohmann::json & j, const ShallowNarListing & listing);
 
 } // namespace nix

src/libutil/memory-source-accessor.cc

@@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path)
     return open(path, std::nullopt);
 }
 
-MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+template<>
+SourceAccessor::Stat MemorySourceAccessor::File::lstat() const
 {
     return std::visit(
         overloaded{
             [](const Regular & r) {
-                return Stat{
-                    .type = tRegular,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tRegular,
                     .fileSize = r.contents.size(),
                     .isExecutable = r.executable,
                 };
             },
             [](const Directory &) {
-                return Stat{
-                    .type = tDirectory,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tDirectory,
                 };
             },
             [](const Symlink &) {
-                return Stat{
-                    .type = tSymlink,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tSymlink,
                 };
             },
         },