Merge branch 'master' of https://github.com/NixOS/nix into upstream/master

Author: Anton-Latukha

doc/manual/release-notes/rl-1.12.xml

@@ -393,6 +393,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
     package repository.</para>
   </listitem>
 
+  <listitem>
+    <para>Automatic garbage collection.</para>
+  </listitem>
+
 </itemizedlist>
 
 <para>This release has contributions from TBD.</para>

src/libstore/build.cc

@@ -3957,6 +3957,8 @@ void Worker::run(const Goals & _topGoals)
 
         checkInterrupt();
 
+        store.autoGC(false);
+
         /* Call every wake goal (in the ordering established by
            CompareGoalPtrs). */
         while (!awake.empty() && !topGoals.empty()) {
@@ -4014,6 +4016,9 @@ void Worker::waitForInput()
        is a build timeout, then wait for input until the first
        deadline for any child. */
     auto nearest = steady_time_point::max(); // nearest deadline
+    if (settings.minFree.get() != 0)
+        // Periodicallty wake up to see if we need to run the garbage collector.
+        nearest = before + std::chrono::seconds(10);
     for (auto & i : children) {
         if (!i.respectTimeouts) continue;
         if (0 != settings.maxSilentTime)

src/libstore/gc.cc

@@ -1,6 +1,7 @@
 #include "derivations.hh"
 #include "globals.hh"
 #include "local-store.hh"
+#include "finally.hh"
 
 #include <functional>
 #include <queue>
@@ -9,6 +10,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/statvfs.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
@@ -18,7 +20,6 @@ namespace nix {
 
 
 static string gcLockName = "gc.lock";
-static string tempRootsDir = "temproots";
 static string gcRootsDir = "gcroots";
 
 
@@ -153,30 +154,25 @@ void LocalStore::addTempRoot(const Path & path)
     if (!state->fdTempRoots) {
 
         while (1) {
-            Path dir = (format("%1%/%2%") % stateDir % tempRootsDir).str();
-            createDirs(dir);
-
-            state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str();
-
             AutoCloseFD fdGCLock = openGCLock(ltRead);
 
-            if (pathExists(state->fnTempRoots))
+            if (pathExists(fnTempRoots))
                 /* It *must* be stale, since there can be no two
                    processes with the same pid. */
-                unlink(state->fnTempRoots.c_str());
+                unlink(fnTempRoots.c_str());
 
-            state->fdTempRoots = openLockFile(state->fnTempRoots, true);
+            state->fdTempRoots = openLockFile(fnTempRoots, true);
 
             fdGCLock = -1;
 
-            debug(format("acquiring read lock on '%1%'") % state->fnTempRoots);
+            debug(format("acquiring read lock on '%1%'") % fnTempRoots);
             lockFile(state->fdTempRoots.get(), ltRead, true);
 
             /* Check whether the garbage collector didn't get in our
                way. */
             struct stat st;
             if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError(format("statting '%1%'") % state->fnTempRoots);
+                throw SysError(format("statting '%1%'") % fnTempRoots);
             if (st.st_size == 0) break;
 
             /* The garbage collector deleted this file before we could
@@ -188,14 +184,14 @@ void LocalStore::addTempRoot(const Path & path)
 
     /* Upgrade the lock to a write lock.  This will cause us to block
        if the garbage collector is holding our lock. */
-    debug(format("acquiring write lock on '%1%'") % state->fnTempRoots);
+    debug(format("acquiring write lock on '%1%'") % fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltWrite, true);
 
     string s = path + '\0';
     writeFull(state->fdTempRoots.get(), s);
 
     /* Downgrade to a read lock. */
-    debug(format("downgrading to read lock on '%1%'") % state->fnTempRoots);
+    debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
     lockFile(state->fdTempRoots.get(), ltRead, true);
 }
 
@@ -204,11 +200,10 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
-    DirEntries tempRootFiles = readDirectory(
-        (format("%1%/%2%") % stateDir % tempRootsDir).str());
+    DirEntries tempRootFiles = readDirectory(tempRootsDir);
 
     for (auto & i : tempRootFiles) {
-        Path path = (format("%1%/%2%/%3%") % stateDir % tempRootsDir % i.name).str();
+        Path path = tempRootsDir + "/" + i.name;
 
         debug(format("reading temporary root file '%1%'") % path);
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
@@ -222,21 +217,25 @@ void LocalStore::readTempRoots(PathSet & tempRoots, FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-        /* Try to acquire a write lock without blocking.  This can
-           only succeed if the owning process has died.  In that case
-           we don't care about its temporary roots. */
-        if (lockFile(fd->get(), ltWrite, false)) {
-            printError(format("removing stale temporary roots file '%1%'") % path);
-            unlink(path.c_str());
-            writeFull(fd->get(), "d");
-            continue;
-        }
+        if (path != fnTempRoots) {
+
+            /* Try to acquire a write lock without blocking.  This can
+               only succeed if the owning process has died.  In that case
+               we don't care about its temporary roots. */
+            if (lockFile(fd->get(), ltWrite, false)) {
+                printError(format("removing stale temporary roots file '%1%'") % path);
+                unlink(path.c_str());
+                writeFull(fd->get(), "d");
+                continue;
+            }
+
+            /* Acquire a read lock.  This will prevent the owning process
+               from upgrading to a write lock, therefore it will block in
+               addTempRoot(). */
+            debug(format("waiting for read lock on '%1%'") % path);
+            lockFile(fd->get(), ltRead, true);
 
-        /* Acquire a read lock.  This will prevent the owning process
-           from upgrading to a write lock, therefore it will block in
-           addTempRoot(). */
-        debug(format("waiting for read lock on '%1%'") % path);
-        lockFile(fd->get(), ltRead, true);
+        }
 
         /* Read the entire file. */
         string contents = readFile(fd->get());
@@ -848,4 +847,80 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 }
 
 
+void LocalStore::autoGC(bool sync)
+{
+    auto getAvail = [this]() {
+        struct statvfs st;
+        if (statvfs(realStoreDir.c_str(), &st))
+            throw SysError("getting filesystem info about '%s'", realStoreDir);
+
+        return (uint64_t) st.f_bavail * st.f_bsize;
+    };
+
+    std::shared_future<void> future;
+
+    {
+        auto state(_state.lock());
+
+        if (state->gcRunning) {
+            future = state->gcFuture;
+            debug("waiting for auto-GC to finish");
+            goto sync;
+        }
+
+        auto now = std::chrono::steady_clock::now();
+
+        if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
+
+        auto avail = getAvail();
+
+        state->lastGCCheck = now;
+
+        if (avail >= settings.minFree || avail >= settings.maxFree) return;
+
+        if (avail > state->availAfterGC * 0.97) return;
+
+        state->gcRunning = true;
+
+        std::promise<void> promise;
+        future = state->gcFuture = promise.get_future().share();
+
+        std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
+
+            try {
+
+                /* Wake up any threads waiting for the auto-GC to finish. */
+                Finally wakeup([&]() {
+                    auto state(_state.lock());
+                    state->gcRunning = false;
+                    state->lastGCCheck = std::chrono::steady_clock::now();
+                    promise.set_value();
+                });
+
+                printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
+
+                GCOptions options;
+                options.maxFreed = settings.maxFree - avail;
+
+                GCResults results;
+
+                collectGarbage(options, results);
+
+                _state.lock()->availAfterGC = getAvail();
+
+            } catch (...) {
+                // FIXME: we could propagate the exception to the
+                // future, but we don't really care.
+                ignoreException();
+            }
+
+        }).detach();
+    }
+
+ sync:
+    // Wait for the future outside of the state lock.
+    if (sync) future.get();
+}
+
+
 }

src/libstore/globals.hh

@@ -4,8 +4,9 @@
 #include "config.hh"
 
 #include <map>
-#include <sys/types.h>
+#include <limits>
 
+#include <sys/types.h>
 
 namespace nix {
 
@@ -342,6 +343,13 @@ public:
 
     Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
         "A list of servers used by builtins.fetchurl to fetch files by hash."};
+
+    Setting<uint64_t> minFree{this, 0, "min-free",
+        "Automatically run the garbage collector when free disk space drops below the specified amount."};
+
+    Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
+        "Stop deleting garbage when free disk space is above the specified amount."};
+
 };
 
 

src/libstore/local-store.cc

@@ -51,6 +51,8 @@ LocalStore::LocalStore(const Params & params)
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
     , trashDir(realStoreDir + "/trash")
+    , tempRootsDir(stateDir + "/temproots")
+    , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
     , publicKeys(getDefaultPublicKeys())
 {
     auto state(_state.lock());
@@ -61,7 +63,7 @@ LocalStore::LocalStore(const Params & params)
     createDirs(linksDir);
     Path profilesDir = stateDir + "/profiles";
     createDirs(profilesDir);
-    createDirs(stateDir + "/temproots");
+    createDirs(tempRootsDir);
     createDirs(dbDir);
     Path gcRootsDir = stateDir + "/gcroots";
     if (!pathExists(gcRootsDir)) {
@@ -242,12 +244,24 @@ LocalStore::LocalStore(const Params & params)
 
 LocalStore::~LocalStore()
 {
-    auto state(_state.lock());
+    std::shared_future<void> future;
+
+    {
+        auto state(_state.lock());
+        if (state->gcRunning)
+            future = state->gcFuture;
+    }
+
+    if (future.valid()) {
+        printError("waiting for auto-GC to finish on exit...");
+        future.get();
+    }
 
     try {
+        auto state(_state.lock());
         if (state->fdTempRoots) {
             state->fdTempRoots = -1;
-            unlink(state->fnTempRoots.c_str());
+            unlink(fnTempRoots.c_str());
         }
     } catch (...) {
         ignoreException();
@@ -989,6 +1003,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
             StringSource source(*nar);
             restorePath(realPath, source);
 
+            autoGC();
+
             canonicalisePathMetaData(realPath, -1);
 
             optimisePath(realPath); // FIXME: combine with hashPath()
@@ -1023,6 +1039,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 
             deletePath(realPath);
 
+            autoGC();
+
             if (recursive) {
                 StringSource source(dump);
                 restorePath(realPath, source);
@@ -1095,6 +1113,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             deletePath(realPath);
 
+            autoGC();
+
             writeFile(realPath, s);
 
             canonicalisePathMetaData(realPath, -1);

src/libstore/local-store.hh

@@ -7,6 +7,8 @@
 #include "sync.hh"
 #include "util.hh"
 
+#include <chrono>
+#include <future>
 #include <string>
 #include <unordered_set>
 
@@ -59,8 +61,22 @@ private:
         SQLiteStmt stmtQueryValidPaths;
 
         /* The file to which we write our temporary roots. */
-        Path fnTempRoots;
         AutoCloseFD fdTempRoots;
+
+        /* The last time we checked whether to do an auto-GC, or an
+           auto-GC finished. */
+        std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
+
+        /* Whether auto-GC is running. If so, get gcFuture to wait for
+           the GC to finish. */
+        bool gcRunning = false;
+        std::shared_future<void> gcFuture;
+
+        /* How much disk space was available after the previous
+           auto-GC. If the current available disk space is below
+           minFree but not much below availAfterGC, then there is no
+           point in starting a new GC. */
+        uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
     };
 
     Sync<State, std::recursive_mutex> _state;
@@ -75,6 +91,8 @@ public:
     const Path reservedPath;
     const Path schemaPath;
     const Path trashDir;
+    const Path tempRootsDir;
+    const Path fnTempRoots;
 
 private:
 
@@ -195,6 +213,10 @@ public:
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
+    /* If free disk space in /nix/store if below minFree, delete
+       garbage until it exceeds maxFree. */
+    void autoGC(bool sync = true);
+
 private:
 
     int getSchema();