Merge pull request #14291 from NixOS/skip-source

Add skip() method to Source interface to allow efficient seeks

Author: edolstra

src/libutil/archive.cc

@@ -132,6 +132,11 @@ static void parseContents(CreateRegularFileSink & sink, Source & source)
 
     sink.preallocateContents(size);
 
+    if (sink.skipContents) {
+        source.skip(size + (size % 8 ? 8 - (size % 8) : 0));
+        return;
+    }
+
     uint64_t left = size;
     std::array<char, 65536> buf;
 
@@ -166,7 +171,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
     auto expectTag = [&](std::string_view expected) {
         auto tag = getString();
         if (tag != expected)
-            throw badArchive("expected tag '%s', got '%s'", expected, tag);
+            throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024));
     };
 
     expectTag("(");

src/libutil/fs-sink.cc

@@ -196,6 +196,8 @@ void NullFileSystemObjectSink::createRegularFile(
         void isExecutable() override {}
     } crf;
 
+    crf.skipContents = true;
+
     // Even though `NullFileSystemObjectSink` doesn't do anything, it's important
     // that we call the function, to e.g. advance the parser using this
     // sink.

src/libutil/include/nix/util/fs-sink.hh

@@ -14,6 +14,14 @@ namespace nix {
  */
 struct CreateRegularFileSink : Sink
 {
+    /**
+     * If set to true, the sink will not be called with the contents
+     * of the file. `preallocateContents()` will still be called to
+     * convey the file size. Useful for sinks that want to efficiently
+     * discard the contents of the file.
+     */
+    bool skipContents = false;
+
     virtual void isExecutable() = 0;
 
     /**

src/libutil/include/nix/util/serialise.hh

@@ -97,6 +97,8 @@ struct Source
     void drainInto(Sink & sink);
 
     std::string drain();
+
+    virtual void skip(size_t len);
 };
 
 /**
@@ -177,6 +179,7 @@ struct FdSource : BufferedSource
     Descriptor fd;
     size_t read = 0;
     BackedStringView endOfFileError{"unexpected end-of-file"};
+    bool isSeekable = true;
 
     FdSource()
         : fd(INVALID_DESCRIPTOR)
@@ -200,6 +203,8 @@ struct FdSource : BufferedSource
      */
     bool hasData();
 
+    void skip(size_t len) override;
+
 protected:
     size_t readUnbuffered(char * data, size_t len) override;
 private:

src/libutil/serialise.cc

@@ -94,9 +94,8 @@ void Source::drainInto(Sink & sink)
 {
     std::array<char, 8192> buf;
     while (true) {
-        size_t n;
         try {
-            n = read(buf.data(), buf.size());
+            auto n = read(buf.data(), buf.size());
             sink({buf.data(), n});
         } catch (EndOfFile &) {
             break;
@@ -111,6 +110,16 @@ std::string Source::drain()
     return std::move(s.s);
 }
 
+void Source::skip(size_t len)
+{
+    std::array<char, 8192> buf;
+    while (len) {
+        auto n = read(buf.data(), std::min(len, buf.size()));
+        assert(n <= len);
+        len -= n;
+    }
+}
+
 size_t BufferedSource::read(char * data, size_t len)
 {
     if (!buffer)
@@ -120,7 +129,7 @@ size_t BufferedSource::read(char * data, size_t len)
         bufPosIn = readUnbuffered(buffer.get(), bufSize);
 
     /* Copy out the data in the buffer. */
-    size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
+    auto n = std::min(len, bufPosIn - bufPosOut);
     memcpy(data, buffer.get() + bufPosOut, n);
     bufPosOut += n;
     if (bufPosIn == bufPosOut)
@@ -191,6 +200,39 @@ bool FdSource::hasData()
     }
 }
 
+void FdSource::skip(size_t len)
+{
+    /* Discard data in the buffer. */
+    if (len && buffer && bufPosIn - bufPosOut) {
+        if (len >= bufPosIn - bufPosOut) {
+            len -= bufPosIn - bufPosOut;
+            bufPosIn = bufPosOut = 0;
+        } else {
+            bufPosOut += len;
+            len = 0;
+        }
+    }
+
+#ifndef _WIN32
+    /* If we can, seek forward in the file to skip the rest. */
+    if (isSeekable && len) {
+        if (lseek(fd, len, SEEK_CUR) == -1) {
+            if (errno == ESPIPE)
+                isSeekable = false;
+            else
+                throw SysError("seeking forward in file");
+        } else {
+            read += len;
+            return;
+        }
+    }
+#endif
+
+    /* Otherwise, skip by reading. */
+    if (len)
+        BufferedSource::skip(len);
+}
+
 size_t StringSource::read(char * data, size_t len)
 {
     if (pos == s.size())