[lld][MachO] Follow-up to use madvise() for threaded file page-in.

lld/MachO/Driver.cpp

@@ -41,6 +41,7 @@
 #include "llvm/Object/Archive.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
@@ -53,6 +54,10 @@
 #include "llvm/TextAPI/Architecture.h"
 #include "llvm/TextAPI/PackedVersion.h"
 
+#if !_WIN32
+#include <sys/mman.h>
+#endif
+
 using namespace llvm;
 using namespace llvm::MachO;
 using namespace llvm::object;
@@ -291,12 +296,13 @@ struct DeferredFile {
 };
 using DeferredFiles = std::vector<DeferredFile>;
 
-class SerialBackgroundQueue {
+class SerialBackgroundWorkQueue {
   std::deque<std::function<void()>> queue;
   std::thread *running;
   std::mutex mutex;
 
 public:
+  bool stopAllWork = false;
   void queueWork(std::function<void()> work) {
     mutex.lock();
     if (running && queue.empty()) {
@@ -311,7 +317,7 @@ class SerialBackgroundQueue {
       queue.emplace_back(std::move(work));
       if (!running)
         running = new std::thread([&]() {
-          while (true) {
+          while (!stopAllWork) {
             mutex.lock();
             if (queue.empty()) {
               mutex.unlock();
@@ -330,61 +336,65 @@ class SerialBackgroundQueue {
   }
 };
 
+static SerialBackgroundWorkQueue pageInQueue;
+
 // Most input files have been mapped but not yet paged in.
 // This code forces the page-ins on multiple threads so
 // the process is not stalled waiting on disk buffer i/o.
 void multiThreadedPageInBackground(DeferredFiles &deferred) {
+  using namespace std::chrono;
   static const size_t pageSize = Process::getPageSizeEstimate();
   static const size_t largeArchive = 10 * 1024 * 1024;
-#ifndef NDEBUG
-  using namespace std::chrono;
-  std::atomic_int numDeferedFilesTouched = 0;
   static std::atomic_uint64_t totalBytes = 0;
+  std::atomic_int numDeferedFilesAdvised = 0;
   auto t0 = high_resolution_clock::now();
-#endif
 
   auto preloadDeferredFile = [&](const DeferredFile &deferredFile) {
     const StringRef &buff = deferredFile.buffer.getBuffer();
     if (buff.size() > largeArchive)
       return;
-#ifndef NDEBUG
+
     totalBytes += buff.size();
-    numDeferedFilesTouched += 1;
-#endif
+    numDeferedFilesAdvised += 1;
 
+#if _WIN32
     // Reference all file's mmap'd pages to load them into memory.
-    for (const char *page = buff.data(), *end = page + buff.size(); page < end;
-         page += pageSize) {
+    for (const char *page = buff.data(), *end = page + buff.size();
+         page < end && !pageInQueue.stopAllWork; page += pageSize) {
       LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
       (void)t;
     }
+#else
+#define DEBUG_TYPE "lld-madvise"
+    auto aligned = llvm::alignAddr(buff.data(), Align(pageSize));
+    if (madvise((void *)aligned, buff.size(), MADV_WILLNEED) < 0)
+      LLVM_DEBUG(llvm::dbgs() << "madvise error: " << strerror(errno) << "\n");
+#undef DEBUG_TYPE
+#endif
   };
-#if LLVM_ENABLE_THREADS
-  { // Create scope for waiting for the taskGroup
+
+  if (llvm_is_multithreaded()) { // Create scope for waiting for the taskGroup
     std::atomic_size_t index = 0;
     llvm::parallel::TaskGroup taskGroup;
     for (int w = 0; w < config->readWorkers; w++)
       taskGroup.spawn([&index, &preloadDeferredFile, &deferred]() {
-        while (true) {
+        while (!pageInQueue.stopAllWork) {
           size_t localIndex = index.fetch_add(1);
           if (localIndex >= deferred.size())
             break;
           preloadDeferredFile(deferred[localIndex]);
         }
       });
   }
-#endif
-#ifndef NDEBUG
+
   auto dt = high_resolution_clock::now() - t0;
   if (Process::GetEnv("LLD_MULTI_THREAD_PAGE"))
     llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
-                 << numDeferedFilesTouched << "/" << deferred.size() << "/"
+                 << numDeferedFilesAdvised << "/" << deferred.size() << "/"
                  << duration_cast<milliseconds>(dt).count() / 1000. << "\n";
-#endif
 }
 
 static void multiThreadedPageIn(const DeferredFiles &deferred) {
-  static SerialBackgroundQueue pageInQueue;
   pageInQueue.queueWork([=]() {
     DeferredFiles files = deferred;
     multiThreadedPageInBackground(files);
@@ -489,7 +499,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
             continue;
           }
 
-          if (archiveContents)
+          if (config->readWorkers && archiveContents)
             archiveContents->push_back({path, isLazy, *mb});
           if (!hasObjCSection(*mb))
             continue;
@@ -1446,6 +1456,8 @@ static void createFiles(const InputArgList &args) {
       multiThreadedPageIn(archiveContents);
     for (auto *archive : archives)
       archive->addLazySymbols();
+
+    pageInQueue.stopAllWork = true;
   }
 }
 
@@ -1834,13 +1846,16 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
   }
 
   if (auto *arg = args.getLastArg(OPT_read_workers)) {
-    StringRef v(arg->getValue());
-    unsigned workers = 0;
-    if (!llvm::to_integer(v, workers, 0))
-      error(arg->getSpelling() +
-            ": expected a non-negative integer, but got '" + arg->getValue() +
-            "'");
-    config->readWorkers = workers;
+    if (llvm_is_multithreaded()) {
+      StringRef v(arg->getValue());
+      unsigned workers = 0;
+      if (!llvm::to_integer(v, workers, 0))
+        error(arg->getSpelling() +
+              ": expected a non-negative integer, but got '" + arg->getValue() +
+              "'");
+      config->readWorkers = workers;
+    } else
+      error(arg->getSpelling() + ": option unavailable");
   }
   if (auto *arg = args.getLastArg(OPT_threads_eq)) {
     StringRef v(arg->getValue());

lld/MachO/InputFiles.cpp

@@ -217,7 +217,8 @@ std::optional<MemoryBufferRef> macho::readFile(StringRef path) {
   if (entry != cachedReads.end())
     return entry->second;
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr = MemoryBuffer::getFile(path);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> mbOrErr =
+      MemoryBuffer::getFile(path, false, /*RequiresNullTerminator=*/false);
   if (std::error_code ec = mbOrErr.getError()) {
     error("cannot open " + path + ": " + ec.message());
     return std::nullopt;

llvm/lib/Object/Archive.cpp

@@ -582,7 +582,8 @@ Expected<StringRef> Archive::Child::getBuffer() const {
   if (!FullNameOrErr)
     return FullNameOrErr.takeError();
   const std::string &FullName = *FullNameOrErr;
-  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf = MemoryBuffer::getFile(FullName);
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
+      MemoryBuffer::getFile(FullName, false, /*RequiresNullTerminator=*/false);
   if (std::error_code EC = Buf.getError())
     return errorCodeToError(EC);
   Parent->ThinBuffers.push_back(std::move(*Buf));