[DRAFT] "Builtins world" for LTO

lld/ELF/Driver.cpp

@@ -2709,7 +2709,19 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
   if (!ctx.bitcodeFiles.empty())
     markBuffersAsDontNeed(ctx, skipLinkedOutput);
 
-  ltoObjectFiles = lto->compile();
+  llvm::Triple tt(ctx.bitcodeFiles.front()->obj->getTargetTriple());
+  llvm::BumpPtrAllocator alloc;
+  llvm::StringSaver saver(alloc);
+  SmallVector<const char *> bitcodeLibFuncs;
+  for (const char *libFunc : lto::LTO::getLibFuncSymbols(tt, saver)) {
+    Symbol *sym = ctx.symtab->find(libFunc);
+    if (!sym)
+      continue;
+    if (isa<BitcodeFile>(sym->file))
+      bitcodeLibFuncs.push_back(libFunc);
+  }
+
+  ltoObjectFiles = lto->compile(bitcodeLibFuncs);
   for (auto &file : ltoObjectFiles) {
     auto *obj = cast<ObjFile<ELFT>>(file.get());
     obj->parse(/*ignoreComdats=*/true);

lld/ELF/LTO.cpp

@@ -311,7 +311,10 @@ static void thinLTOCreateEmptyIndexFiles(Ctx &ctx) {
 
 // Merge all the bitcode files we have seen, codegen the result
 // and return the resulting ObjectFile(s).
-SmallVector<std::unique_ptr<InputFile>, 0> BitcodeCompiler::compile() {
+SmallVector<std::unique_ptr<InputFile>, 0>
+BitcodeCompiler::compile(const SmallVector<const char *> &bitcodeLibFuncs) {
+  ltoObj->setBitcodeLibFuncs(bitcodeLibFuncs);
+
   unsigned maxTasks = ltoObj->getMaxTasks();
   buf.resize(maxTasks);
   files.resize(maxTasks);

lld/ELF/LTO.h

@@ -42,7 +42,8 @@ class BitcodeCompiler {
   ~BitcodeCompiler();
 
   void add(BitcodeFile &f);
-  SmallVector<std::unique_ptr<InputFile>, 0> compile();
+  SmallVector<std::unique_ptr<InputFile>, 0>
+  compile(const SmallVector<const char *> &bitcodeLibFuncs);
 
 private:
   Ctx &ctx;

llvm/include/llvm/LTO/LTO.h

@@ -264,7 +264,8 @@ class ThinBackendProc {
 using ThinBackendFunction = std::function<std::unique_ptr<ThinBackendProc>(
     const Config &C, ModuleSummaryIndex &CombinedIndex,
     const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-    AddStreamFn AddStream, FileCache Cache)>;
+    AddStreamFn AddStream, FileCache Cache,
+    const SmallVector<const char *> &BitcodeLibFuncs)>;
 
 /// This type defines the behavior following the thin-link phase during ThinLTO.
 /// It encapsulates a backend function and a strategy for thread pool
@@ -279,10 +280,11 @@ struct ThinBackend {
   std::unique_ptr<ThinBackendProc> operator()(
       const Config &Conf, ModuleSummaryIndex &CombinedIndex,
       const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-      AddStreamFn AddStream, FileCache Cache) {
+      AddStreamFn AddStream, FileCache Cache,
+      const SmallVector<const char *> &BitcodeLibFuncs) {
     assert(isValid() && "Invalid backend function");
     return Func(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
-                std::move(AddStream), std::move(Cache));
+                std::move(AddStream), std::move(Cache), BitcodeLibFuncs);
   }
   ThreadPoolStrategy getParallelism() const { return Parallelism; }
   bool isValid() const { return static_cast<bool>(Func); }
@@ -400,6 +402,8 @@ class LTO {
   LLVM_ABI Error add(std::unique_ptr<InputFile> Obj,
                      ArrayRef<SymbolResolution> Res);
 
+  LLVM_ABI void setBitcodeLibFuncs(const SmallVector<const char *> &BitcodeLibFuncs);
+
   /// Returns an upper bound on the number of tasks that the client may expect.
   /// This may only be called after all IR object files have been added. For a
   /// full description of tasks see LTOBackend.h.
@@ -420,6 +424,14 @@ class LTO {
   LLVM_ABI static SmallVector<const char *>
   getRuntimeLibcallSymbols(const Triple &TT);
 
+  /// Static method that returns a list of library function symbols that can be
+  /// generated by LTO but might not be visible from bitcode symbol table.
+  /// Unlike the runtime libcalls, the linker can report to the code generator
+  /// which of these are actually available in the link, and the code generator
+  /// can then only reference that set of symbols.
+  LLVM_ABI static SmallVector<const char *>
+  getLibFuncSymbols(const Triple &TT, llvm::StringSaver &Saver);
+
 private:
   Config Conf;
 
@@ -591,6 +603,9 @@ class LTO {
 
   // Diagnostic optimization remarks file
   LLVMRemarkFileHandle DiagnosticOutputFile;
+
+  const SmallVector<const char *> *BitcodeLibFuncs = &EmptyBitcodeLibFuncs;
+  SmallVector<const char *> EmptyBitcodeLibFuncs;
 };
 
 /// The resolution for a symbol. The linker must provide a SymbolResolution for

llvm/include/llvm/LTO/LTOBackend.h

@@ -39,13 +39,15 @@ LLVM_ABI bool opt(const Config &Conf, TargetMachine *TM, unsigned Task,
                   Module &Mod, bool IsThinLTO,
                   ModuleSummaryIndex *ExportSummary,
                   const ModuleSummaryIndex *ImportSummary,
-                  const std::vector<uint8_t> &CmdArgs);
+                  const std::vector<uint8_t> &CmdArgs,
+                  const SmallVector<const char*>& BitcodeLibFuncs);
 
 /// Runs a regular LTO backend. The regular LTO backend can also act as the
 /// regular LTO phase of ThinLTO, which may need to access the combined index.
 LLVM_ABI Error backend(const Config &C, AddStreamFn AddStream,
                        unsigned ParallelCodeGenParallelismLevel, Module &M,
-                       ModuleSummaryIndex &CombinedIndex);
+                       ModuleSummaryIndex &CombinedIndex,
+                       const SmallVector<const char *> &BitcodeLibFuncs);
 
 /// Runs a ThinLTO backend.
 /// If \p ModuleMap is not nullptr, all the module files to be imported have
@@ -62,6 +64,7 @@ thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M,
             const FunctionImporter::ImportMapTy &ImportList,
             const GVSummaryMapTy &DefinedGlobals,
             MapVector<StringRef, BitcodeModule> *ModuleMap, bool CodeGenOnly,
+            const SmallVector<const char *> &BitcodeLibFuncs,
             AddStreamFn IRAddStream = nullptr,
             const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>());
 

llvm/lib/LTO/LTO.cpp

@@ -762,6 +762,10 @@ Error LTO::add(std::unique_ptr<InputFile> Input,
   return Error::success();
 }
 
+void LTO::setBitcodeLibFuncs(const SmallVector<const char *> &BitcodeLibFuncs) {
+  this->BitcodeLibFuncs = &BitcodeLibFuncs;
+}
+
 Expected<ArrayRef<SymbolResolution>>
 LTO::addModule(InputFile &Input, ArrayRef<SymbolResolution> InputRes,
                unsigned ModI, ArrayRef<SymbolResolution> Res) {
@@ -1386,7 +1390,8 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
   if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
     if (Error Err =
             backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
-                    *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
+                    *RegularLTO.CombinedModule, ThinLTO.CombinedIndex,
+                    *BitcodeLibFuncs))
       return Err;
   }
 
@@ -1406,6 +1411,21 @@ SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
   return LibcallSymbols;
 }
 
+SmallVector<const char *> LTO::getLibFuncSymbols(const Triple &TT,
+                                                 StringSaver &Saver) {
+  auto TLII = std::make_unique<TargetLibraryInfoImpl>(TT);
+  TargetLibraryInfo TLI(*TLII);
+  SmallVector<const char *> LibFuncSymbols;
+  LibFuncSymbols.reserve(LibFunc::NumLibFuncs);
+  for (unsigned I = 0, E = static_cast<unsigned>(LibFunc::NumLibFuncs); I != E;
+       ++I) {
+    LibFunc F = static_cast<LibFunc>(I);
+    if (TLI.has(F))
+      LibFuncSymbols.push_back(Saver.save(TLI.getName(F)).data());
+  }
+  return LibFuncSymbols;
+}
+
 Error ThinBackendProc::emitFiles(
     const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath,
     const std::string &NewModulePath) const {
@@ -1483,18 +1503,20 @@ class CGThinBackend : public ThinBackendProc {
 class InProcessThinBackend : public CGThinBackend {
 protected:
   FileCache Cache;
+  const SmallVector<const char*> &BitcodeLibFuncs;
 
 public:
   InProcessThinBackend(
       const Config &Conf, ModuleSummaryIndex &CombinedIndex,
       ThreadPoolStrategy ThinLTOParallelism,
       const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
       AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
-      bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
+      bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,
+      const SmallVector<const char *> &BitcodeLibFuncs)
       : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
                       AddStream, OnWrite, ShouldEmitIndexFiles,
                       ShouldEmitImportsFiles, ThinLTOParallelism),
-        Cache(std::move(Cache)) {}
+        Cache(std::move(Cache)), BitcodeLibFuncs(BitcodeLibFuncs) {}
 
   virtual Error runThinLTOBackendThread(
       AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
@@ -1515,7 +1537,7 @@ class InProcessThinBackend : public CGThinBackend {
 
       return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
                          ImportList, DefinedGlobals, &ModuleMap,
-                         Conf.CodeGenOnly);
+                         Conf.CodeGenOnly, BitcodeLibFuncs);
     };
     if (ShouldEmitIndexFiles) {
       if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
@@ -1600,13 +1622,14 @@ class FirstRoundThinBackend : public InProcessThinBackend {
       const Config &Conf, ModuleSummaryIndex &CombinedIndex,
       ThreadPoolStrategy ThinLTOParallelism,
       const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-      AddStreamFn CGAddStream, FileCache CGCache, AddStreamFn IRAddStream,
+      AddStreamFn CGAddStream, FileCache CGCache,
+      const SmallVector<const char *> &BitcodeLibFuncs, AddStreamFn IRAddStream,
       FileCache IRCache)
       : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
                              ModuleToDefinedGVSummaries, std::move(CGAddStream),
                              std::move(CGCache), /*OnWrite=*/nullptr,
                              /*ShouldEmitIndexFiles=*/false,
-                             /*ShouldEmitImportsFiles=*/false),
+                             /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
         IRAddStream(std::move(IRAddStream)), IRCache(std::move(IRCache)) {}
 
   Error runThinLTOBackendThread(
@@ -1629,7 +1652,7 @@ class FirstRoundThinBackend : public InProcessThinBackend {
 
       return thinBackend(Conf, Task, CGAddStream, **MOrErr, CombinedIndex,
                          ImportList, DefinedGlobals, &ModuleMap,
-                         Conf.CodeGenOnly, IRAddStream);
+                         Conf.CodeGenOnly, BitcodeLibFuncs, IRAddStream);
     };
     // Like InProcessThinBackend, we produce index files as needed for
     // FirstRoundThinBackend. However, these files are not generated for
@@ -1696,14 +1719,15 @@ class SecondRoundThinBackend : public InProcessThinBackend {
       ThreadPoolStrategy ThinLTOParallelism,
       const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
       AddStreamFn AddStream, FileCache Cache,
+      const SmallVector<const char *> &BitcodeLibFuncs,
       std::unique_ptr<SmallVector<StringRef>> IRFiles,
       stable_hash CombinedCGDataHash)
       : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,
                              ModuleToDefinedGVSummaries, std::move(AddStream),
                              std::move(Cache),
                              /*OnWrite=*/nullptr,
                              /*ShouldEmitIndexFiles=*/false,
-                             /*ShouldEmitImportsFiles=*/false),
+                             /*ShouldEmitImportsFiles=*/false, BitcodeLibFuncs),
         IRFiles(std::move(IRFiles)), CombinedCGDataHash(CombinedCGDataHash) {}
 
   Error runThinLTOBackendThread(
@@ -1724,7 +1748,7 @@ class SecondRoundThinBackend : public InProcessThinBackend {
 
       return thinBackend(Conf, Task, AddStream, *LoadedModule, CombinedIndex,
                          ImportList, DefinedGlobals, &ModuleMap,
-                         /*CodeGenOnly=*/true);
+                         /*CodeGenOnly=*/true, BitcodeLibFuncs);
     };
     if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||
         all_of(CombinedIndex.getModuleHash(ModuleID),
@@ -1763,11 +1787,12 @@ ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,
   auto Func =
       [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
           const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-          AddStreamFn AddStream, FileCache Cache) {
+          AddStreamFn AddStream, FileCache Cache,
+          const SmallVector<const char *> &BitcodeLibFuncs) {
         return std::make_unique<InProcessThinBackend>(
             Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
             AddStream, Cache, OnWrite, ShouldEmitIndexFiles,
-            ShouldEmitImportsFiles);
+            ShouldEmitImportsFiles, BitcodeLibFuncs);
       };
   return ThinBackend(Func, Parallelism);
 }
@@ -1884,7 +1909,8 @@ ThinBackend lto::createWriteIndexesThinBackend(
   auto Func =
       [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
           const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-          AddStreamFn AddStream, FileCache Cache) {
+          AddStreamFn AddStream, FileCache Cache,
+          const SmallVector<const char *> &BitcodeLibFuncs) {
         return std::make_unique<WriteIndexesThinBackend>(
             Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
             OldPrefix, NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles,
@@ -2102,7 +2128,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
   if (!CodeGenDataThinLTOTwoRounds) {
     std::unique_ptr<ThinBackendProc> BackendProc =
         ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
-                        AddStream, Cache);
+                        AddStream, Cache, *BitcodeLibFuncs);
     return RunBackends(BackendProc.get());
   }
 
@@ -2125,7 +2151,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
   LLVM_DEBUG(dbgs() << "[TwoRounds] Running the first round of codegen\n");
   auto FirstRoundLTO = std::make_unique<FirstRoundThinBackend>(
       Conf, ThinLTO.CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
-      CG.AddStream, CG.Cache, IR.AddStream, IR.Cache);
+      CG.AddStream, CG.Cache, *BitcodeLibFuncs, IR.AddStream, IR.Cache);
   if (Error E = RunBackends(FirstRoundLTO.get()))
     return E;
 
@@ -2141,7 +2167,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
   LLVM_DEBUG(dbgs() << "[TwoRounds] Running the second round of codegen\n");
   auto SecondRoundLTO = std::make_unique<SecondRoundThinBackend>(
       Conf, ThinLTO.CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
-      AddStream, Cache, IR.getResult(), CombinedHash);
+      AddStream, Cache, *BitcodeLibFuncs, IR.getResult(), CombinedHash);
   return RunBackends(SecondRoundLTO.get());
 }
 
@@ -2525,7 +2551,8 @@ ThinBackend lto::createOutOfProcessThinBackend(
   auto Func =
       [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
           const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
-          AddStreamFn AddStream, FileCache /*Cache*/) {
+          AddStreamFn AddStream, FileCache /*Cache*/,
+          const SmallVector<const char *> &BitcodeLibFuncs) {
         return std::make_unique<OutOfProcessThinBackend>(
             Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
             AddStream, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles,