From 3a92171ca70f2f689797088f20f3b45f84021df4 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Sun, 24 Aug 2025 22:35:14 -0700 Subject: [PATCH 01/20] [DTLTO][ELF][COFF][MachO] Add archive support for DTLTO. This patch implements support for handling archive members in DTLTO. Unlike ThinLTO, where archive members are passed as in-memory buffers, DTLTO requires archive members to be materialized as individual files on the filesystem. This is necessary because DTLTO invokes clang externally, which expects file-based inputs. To support this, this implementation identifies archive members among the input files, saves them to the filesystem, and updates their module_id to match their file paths. --- cross-project-tests/CMakeLists.txt | 1 + cross-project-tests/dtlto/archive.test | 80 +++++++ .../dtlto/archives-mixed-lto-modes-test.test | 35 +++ .../dtlto/archives-same-module-id.test | 55 +++++ lld/COFF/InputFiles.cpp | 1 + lld/ELF/InputFiles.cpp | 1 + lld/ELF/LTO.cpp | 3 + lld/MachO/InputFiles.cpp | 1 + llvm/include/llvm/Bitcode/BitcodeReader.h | 5 + llvm/include/llvm/DTLTO/Dtlto.h | 23 ++ llvm/include/llvm/LTO/LTO.h | 48 ++++ llvm/lib/CMakeLists.txt | 1 + llvm/lib/DTLTO/CMakeLists.txt | 7 + llvm/lib/DTLTO/Dtlto.cpp | 226 ++++++++++++++++++ llvm/lib/LTO/CMakeLists.txt | 1 + llvm/lib/LTO/LTO.cpp | 20 +- 16 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 cross-project-tests/dtlto/archive.test create mode 100644 cross-project-tests/dtlto/archives-mixed-lto-modes-test.test create mode 100644 cross-project-tests/dtlto/archives-same-module-id.test create mode 100644 llvm/include/llvm/DTLTO/Dtlto.h create mode 100644 llvm/lib/DTLTO/CMakeLists.txt create mode 100644 llvm/lib/DTLTO/Dtlto.cpp diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index 192db87043177..6753a27698eae 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -20,6 +20,7 @@ set(CROSS_PROJECT_TEST_DEPS check-gdb-llvm-support count llvm-ar + llvm-ar llvm-config llvm-dwarfdump llvm-objdump diff --git a/cross-project-tests/dtlto/archive.test b/cross-project-tests/dtlto/archive.test new file mode 100644 index 0000000000000..1c01ae7a5691a --- /dev/null +++ b/cross-project-tests/dtlto/archive.test @@ -0,0 +1,80 @@ +REQUIRES: x86-registered-target,ld.lld,llvm-ar + +# Test that a DTLTO link succeeds and outputs the expected set of files +# correctly when archives are present. + +RUN: rm -rf %t && split-file %s %t && cd %t +# Compile sources into bitcode. -O2 is required for cross-module importing. +RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c foo.c boo.c moo.c loo.c voo.c main.c + +RUN: llvm-ar rcs archive.a foo.o boo.o moo.o +RUN: llvm-ar rcsT archive.thin.a loo.o voo.o + +# Build with DTLTO. +RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin \ +RUN: -fuse-ld=lld -nostdlib -e main \ +RUN: main.o archive.a archive.thin.a -o main.elf \ +RUN: -Wl,--thinlto-distributor=%python \ +RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +RUN: -Wl,--thinlto-remote-compiler=%clang \ +RUN: -Wl,--save-temps + +# Check that the required output files have been created. +RUN: ls | FileCheck %s --check-prefix=OUTPUTS + +# JSON jobs description. +OUTPUTS-DAG: {{^}}main.[[PID:[0-9]+]].dist-file.json + +# Main source. +OUTPUTS-DAG: {{^}}main.{{[0-9]+}}.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}main.{{[0-9]+}}.[[PID]].native.o.thinlto.bc{{$}} + +# Regular archive members. +# Filename composition: ( at ).....native.o[.thinlto.bc]. +OUTPUTS-DAG: {{^}}archive.a(boo.o at {{[0-9]+}}).2.[[HEXPID:[a-fA-F0-9]+]].2.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}archive.a(boo.o at {{[0-9]+}}).2.[[HEXPID]].2.[[PID]].native.o.thinlto.bc{{$}} + +OUTPUTS-DAG: {{^}}archive.a(foo.o at {{[0-9]+}}).3.[[HEXPID]].3.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}archive.a(foo.o at {{[0-9]+}}).3.[[HEXPID]].3.[[PID]].native.o.thinlto.bc{{$}} + +OUTPUTS-DAG: {{^}}archive.a(moo.o at {{[0-9]+}}).4.[[HEXPID]].4.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}archive.a(moo.o at {{[0-9]+}}).4.[[HEXPID]].4.[[PID]].native.o.thinlto.bc{{$}} + +# Thin archive members. +OUTPUTS-DAG: {{^}}voo.{{[0-9]+}}.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}voo.{{[0-9]+}}.[[PID]].native.o.thinlto.bc{{$}} + +OUTPUTS-DAG: {{^}}loo.{{[0-9]+}}.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}loo.{{[0-9]+}}.[[PID]].native.o.thinlto.bc{{$}} + +# Executable file. +OUTPUTS-DAG: {{^}}main.elf{{$}} + +#--- foo.c +volatile int foo_int; +__attribute__((retain)) int foo(int x) { return x + foo_int; } + +#--- boo.c +extern int foo(int x); +__attribute__((retain)) int boo(int x) { return foo(x); } + +#--- moo.c +__attribute__((retain)) int moo() { return 3; } + +#--- loo.c +extern int moo(int x); +__attribute__((retain)) int loo(int x) { return moo(x); } + +#--- voo.c +extern int foo(int x); +extern int loo(int x); +__attribute__((retain)) int voo(int x) { return foo(x) + loo(x + 1) + 7; } + +#--- main.c +extern int boo(int x); +extern int moo(); +extern int voo(int x); +__attribute__((retain)) int main(int argc, char** argv) { + return boo(argc) + moo() + voo(argc + 3); +} + diff --git a/cross-project-tests/dtlto/archives-mixed-lto-modes-test.test b/cross-project-tests/dtlto/archives-mixed-lto-modes-test.test new file mode 100644 index 0000000000000..74f146028b4b6 --- /dev/null +++ b/cross-project-tests/dtlto/archives-mixed-lto-modes-test.test @@ -0,0 +1,35 @@ +REQUIRES: x86-registered-target,ld.lld,llvm-ar + +# Test that DTLTO works with a mixture of FullLTO and ThinLTO bitcode archive members +# where there is more than one LTO partition. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: %clang --target=x86_64-linux-gnu -flto -c one.c two.c +RUN: %clang --target=x86_64-linux-gnu -flto=thin -c three.c + +RUN: llvm-ar rc archive.a one.o two.o three.o + +# Build with DTLTO. +RUN: %clang --target=x86_64-linux-gnu -Werror -flto -fuse-ld=lld -nostdlib \ +RUN: -Wl,--whole-archive archive.a \ +RUN: -Wl,--thinlto-distributor=%python \ +RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +RUN: -Wl,--thinlto-remote-compiler=%clang \ +RUN: -Wl,--save-temps,--lto-partitions=2 + +# Show that the FullLTO modules have been prepared for distribution, this is +# not optimal but has no functional impact. +RUN: FileCheck %s --input-file=a.out.resolution.txt +CHECK: archive.a(one.o at {{.*}}).1.[[PID:[a-zA-Z0-9_]+]].o +CHECK: archive.a(two.o at {{.*}}).2.[[PID]].o +CHECK: archive.a(three.o at {{.*}}).3.[[PID]].o + +#--- one.c +__attribute__((retain)) void one() {} + +#--- two.c +__attribute__((retain)) void two() {} + +#--- three.c +__attribute__((retain)) void three() {} diff --git a/cross-project-tests/dtlto/archives-same-module-id.test b/cross-project-tests/dtlto/archives-same-module-id.test new file mode 100644 index 0000000000000..09d5f7492bfa5 --- /dev/null +++ b/cross-project-tests/dtlto/archives-same-module-id.test @@ -0,0 +1,55 @@ +REQUIRES: x86-registered-target,ld.lld,llvm-ar + +# Test that a DTLTO link succeeds when there are two archive member files with +# the same filename path component. + +# Split this file into several sources. +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c start.c + +# Create first archive. +RUN: mkdir archive1 && cd archive1 +RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c ../t1.c ../t3.c +RUN: llvm-ar rc archive.a t3.o t1.o +RUN: cd .. + +# Create second archive. +RUN: mkdir archive2 && cd archive2 +RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -c ../t1.c ../t3.c +RUN: llvm-ar rc archive.a t3.o t1.o +RUN: cd .. + +RUN: %clang -O2 --target=x86_64-linux-gnu -Werror -flto=thin -fuse-ld=lld \ +RUN: -nostdlib -Wl,--undefined=t1,--undefined=t3 \ +RUN: start.o archive1/archive.a archive2/archive.a -o main.elf \ +RUN: -Wl,--save-temps \ +RUN: -Wl,--thinlto-distributor=%python \ +RUN: -Wl,--thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \ +RUN: -Wl,--thinlto-remote-compiler=%clang + +# Check that the required output files have been created. +RUN: ls | FileCheck %s --check-prefix=OUTPUTS + +# JSON jobs description. +OUTPUTS-DAG: {{^}}main.[[PID:[0-9]+]].dist-file.json + +# Sources. +OUTPUTS-DAG: {{^}}start.{{[0-9]+}}.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}start.{{[0-9]+}}.[[PID]].native.o.thinlto.bc{{$}} + +# Archive members. +# Filename composition: ( at ).....native.o[.thinlto.bc]. +OUTPUTS-DAG: {{^}}archive.a(t3.o at {{[0-9]+}}).2.[[HEXPID:[a-fA-F0-9]+]].2.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}archive.a(t3.o at {{[0-9]+}}).2.[[HEXPID]].2.[[PID]].native.o.thinlto.bc{{$}} +OUTPUTS-DAG: {{^}}archive.a(t1.o at {{[0-9]+}}).3.[[HEXPID]].3.[[PID]].native.o{{$}} +OUTPUTS-DAG: {{^}}archive.a(t1.o at {{[0-9]+}}).3.[[HEXPID]].3.[[PID]].native.o.thinlto.bc{{$}} + +#--- t1.c +__attribute__((retain)) void t1() { } + +#--- start.c +__attribute__((retain)) void _start() { } + +#--- t3.c +__attribute__((retain)) void t3() { } diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp index c08099b8810bb..d415955b6093b 100644 --- a/lld/COFF/InputFiles.cpp +++ b/lld/COFF/InputFiles.cpp @@ -1380,6 +1380,7 @@ BitcodeFile *BitcodeFile::create(COFFLinkerContext &ctx, MemoryBufferRef mb, utostr(offsetInArchive))); std::unique_ptr obj = check(lto::InputFile::create(mbref)); + obj->setArchivePathAndName(archiveName, mb.getBufferIdentifier()); return make(ctx.getSymtab(getMachineType(obj.get())), mb, obj, lazy); } diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index a5921feb18299..ec0af9d0c0f4e 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -1874,6 +1874,7 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, MemoryBufferRef mbref(mb.getBuffer(), name); obj = CHECK2(lto::InputFile::create(mbref), this); + obj->setArchivePathAndName(archiveName, mb.getBufferIdentifier()); Triple t(obj->getTargetTriple()); ekind = getBitcodeELFKind(t); diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 8d4a6c9e3a81e..13e9c63495e4d 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -202,6 +202,9 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { ctx.arg.ltoPartitions, ltoModes[ctx.arg.ltoKind]); + if(!ctx.arg.dtltoDistributor.empty()) + ltoObj->Dtlto = true; + // Initialize usedStartStop. if (ctx.bitcodeFiles.empty()) return; diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index 3b3023a94166f..bb40fcfb7701f 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -2360,6 +2360,7 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName, sys::path::filename(path) + ")" + utostr(offsetInArchive))); obj = check(lto::InputFile::create(mbref)); + obj->setArchivePathAndName(archiveName, mb.getBufferIdentifier()); if (lazy) parseLazy(); else diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h index 4f839d4cd1575..772ca82019278 100644 --- a/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -137,6 +137,11 @@ struct ParserCallbacks { StringRef getModuleIdentifier() const { return ModuleIdentifier; } + // Assign a new module identifier to this bitcode module. + void setModuleIdentifier(llvm::StringRef ModuleId) { + ModuleIdentifier = ModuleId; + } + /// Read the bitcode module and prepare for lazy deserialization of function /// bodies. If ShouldLazyLoadMetadata is true, lazily load metadata as well. /// If IsImporting is true, this module is being parsed for ThinLTO diff --git a/llvm/include/llvm/DTLTO/Dtlto.h b/llvm/include/llvm/DTLTO/Dtlto.h new file mode 100644 index 0000000000000..aa6af7d0cd9b7 --- /dev/null +++ b/llvm/include/llvm/DTLTO/Dtlto.h @@ -0,0 +1,23 @@ +//===- Dtlto.h - Distributed ThinLTO functions and classes ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_DTLTO_H +#define LLVM_DTLTO_H + +#include "llvm/LTO/LTO.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace dtlto { + +llvm::Expected addInput(llvm::lto::LTO *LtoObj, + std::unique_ptr Input); + +llvm::Error process(llvm::lto::LTO &LtoObj); +} // namespace dtlto + +#endif // LLVM_DTLTO_H diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 323c478691a92..7183a4429e665 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -32,6 +32,23 @@ #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" +namespace llvm { +namespace lto { +class LTO; +} +} // namespace llvm + +namespace dtlto { +class TempFilesRemover { + llvm::lto::LTO *Lto = nullptr; + +public: + TempFilesRemover(llvm::lto::LTO *LtoObj) : Lto{LtoObj} {} + ~TempFilesRemover(); +}; + +} // namespace dtlto + namespace llvm { class Error; @@ -135,6 +152,12 @@ class InputFile { std::vector DependentLibraries; std::vector> ComdatTable; + MemoryBufferRef MbRef; + bool IsMemberOfArchive = false; + bool IsThinLTO = false; + StringRef ArchivePath; + StringRef MemberName; + public: LLVM_ABI ~InputFile(); @@ -193,6 +216,20 @@ class InputFile { // Returns the only BitcodeModule from InputFile. LLVM_ABI BitcodeModule &getSingleBitcodeModule(); + // Returns the memory buffer reference for this input file. + MemoryBufferRef getFileBuffer() const { return MbRef; } + // Returns true if this input file is a member of an archive. + bool isMemberOfArchive() const { return IsMemberOfArchive; } + // Mark this input file as a member of archive. + void memberOfArchive(bool MA) { IsMemberOfArchive = MA; } + + // Returns true if bitcode is ThinLTO. + bool isThinLTO() const { return IsThinLTO; } + + // Store an archive path and a member name. + void setArchivePathAndName(StringRef Path, StringRef Name) { ArchivePath = Path; MemberName = Name; } + StringRef getArchivePath() const { return ArchivePath; } + StringRef getMemberName() const { return MemberName; } private: ArrayRef module_symbols(unsigned I) const { @@ -580,6 +617,17 @@ class LTO { // Diagnostic optimization remarks file std::unique_ptr DiagnosticOutputFile; + +public: + /// DTLTO mode. + bool Dtlto = false; + + BumpPtrAllocator PtrAlloc; + StringSaver Saver{PtrAlloc}; + + // Array of input bitcode files for LTO. + std::vector> InputFiles; + std::unique_ptr TempsRemover; }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt index a9432977718c6..0856af9058fef 100644 --- a/llvm/lib/CMakeLists.txt +++ b/llvm/lib/CMakeLists.txt @@ -22,6 +22,7 @@ add_subdirectory(Frontend) add_subdirectory(Transforms) add_subdirectory(Linker) add_subdirectory(Analysis) +add_subdirectory(DTLTO) add_subdirectory(LTO) add_subdirectory(MC) add_subdirectory(MCA) diff --git a/llvm/lib/DTLTO/CMakeLists.txt b/llvm/lib/DTLTO/CMakeLists.txt new file mode 100644 index 0000000000000..51fd8aad6f48b --- /dev/null +++ b/llvm/lib/DTLTO/CMakeLists.txt @@ -0,0 +1,7 @@ +add_llvm_component_library(LLVMDTLTO + Dtlto.cpp + + LINK_COMPONENTS + Core + Support + ) diff --git a/llvm/lib/DTLTO/Dtlto.cpp b/llvm/lib/DTLTO/Dtlto.cpp new file mode 100644 index 0000000000000..356f30188c5c4 --- /dev/null +++ b/llvm/lib/DTLTO/Dtlto.cpp @@ -0,0 +1,226 @@ +//===- Dtlto.cpp - Distributed ThinLTO implementation --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// \file +// This file implements support functions for Distributed ThinLTO, focusing on +// archive file handling. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DTLTO/Dtlto.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/LTO/LTO.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBufferRef.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +using namespace llvm; + +namespace dtlto { + +// Removes any temporary regular archive member files that were created during +// processing. +TempFilesRemover::~TempFilesRemover() { + if (!Lto) + return; + for (auto &Input : Lto->InputFiles) { + if (Input->isMemberOfArchive()) + sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); + } +} + +// Writes the content of a memory buffer into a file. +static llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { + std::error_code EC; + raw_fd_ostream OS(FilePath.str(), EC, sys::fs::OpenFlags::OF_None); + if (EC) { + return createStringError(inconvertibleErrorCode(), + "Failed to create file %s: %s", FilePath.data(), + EC.message().c_str()); + } + OS.write(FileBuffer.data(), FileBuffer.size()); + if (OS.has_error()) { + return createStringError(inconvertibleErrorCode(), + "Failed writing to file %s", FilePath.data()); + } + return Error::success(); +} + +// Compute the file path for a thin archive member. +// +// For thin archives, an archive member name is typically a file path relative +// to the archive file's directory. This function resolves that path. +SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, + const StringRef MemberName) { + assert(!ArchivePath.empty() && "An archive file path must be non empty."); + SmallString<64> MemberPath; + if (sys::path::is_relative(MemberName)) { + MemberPath = sys::path::parent_path(ArchivePath); + sys::path::append(MemberPath, MemberName); + } else + MemberPath = MemberName; + sys::path::remove_dots(MemberPath, /*remove_dot_dot=*/true); + return MemberPath; +} + +// Magic string identifying thin archive files. +static constexpr StringLiteral THIN_ARCHIVE_MAGIC = "!\n"; + +// Determines if a file at the given path is a thin archive file. +// +// This function uses a cache to avoid repeatedly reading the same file. +// It reads only the header portion (magic bytes) of the file to identify +// the archive type. +Expected isThinArchive(const StringRef ArchivePath) { + static StringMap ArchiveFiles; + + // Return cached result if available. + auto Cached = ArchiveFiles.find(ArchivePath); + if (Cached != ArchiveFiles.end()) + return Cached->second; + + uint64_t FileSize = -1; + bool IsThin = false; + std::error_code EC = sys::fs::file_size(ArchivePath, FileSize); + if (EC) + return createStringError(inconvertibleErrorCode(), + "Failed to get file size from archive %s: %s", + ArchivePath.data(), EC.message().c_str()); + if (FileSize < THIN_ARCHIVE_MAGIC.size()) + return createStringError(inconvertibleErrorCode(), + "Archive file size is too small %s", + ArchivePath.data()); + + // Read only the first few bytes containing the magic signature. + ErrorOr> MemBufferOrError = + MemoryBuffer::getFileSlice(ArchivePath, THIN_ARCHIVE_MAGIC.size(), 0); + + if (EC = MemBufferOrError.getError()) + return createStringError(inconvertibleErrorCode(), + "Failed to read from archive %s: %s", + ArchivePath.data(), EC.message().c_str()); + + StringRef MemBuf = (*MemBufferOrError.get()).getBuffer(); + if (file_magic::archive != identify_magic(MemBuf)) + return createStringError(inconvertibleErrorCode(), + "Unknown format for archive %s", + ArchivePath.data()); + + IsThin = MemBuf.starts_with(THIN_ARCHIVE_MAGIC); + + // Cache the result + ArchiveFiles[ArchivePath] = IsThin; + return IsThin; +} + +// This function performs the following tasks: +// 1. Adds the input file to the LTO object's list of input files. +// 2. For thin archive members, generates a new module ID which is a path to a +// thin archive member file. +// 3. For regular archive members, generates a new unique module ID. +// 4. Updates the bitcode module's identifier. +Expected addInput(lto::LTO *LtoObj, + std::unique_ptr InputPtr) { + + // Add the input file to the LTO object. + LtoObj->InputFiles.push_back(std::move(InputPtr)); + lto::InputFile *Input = LtoObj->InputFiles.back().get(); + + // Skip processing if not in DTLTO mode. + if (!LtoObj->Dtlto) + return Input; + + StringRef ModuleId = Input->getName(); + StringRef ArchivePath = Input->getArchivePath(); + + // Only process archive members. + if (ArchivePath.empty()) + return Input; + + SmallString<64> NewModuleId; + BitcodeModule &BM = Input->getSingleBitcodeModule(); + + // Check if the archive is a thin archive. + Expected IsThin = isThinArchive(ArchivePath); + if (!IsThin) + return IsThin.takeError(); + + if (*IsThin) { + // For thin archives, use the path to the actual file. + NewModuleId = + computeThinArchiveMemberPath(ArchivePath, Input->getMemberName()); + } else { + // For regular archives, generate a unique name. + Input->memberOfArchive(true); + + // Create unique identifier using process ID and sequence number. + std::string PID = utohexstr(sys::Process::getProcessId()); + std::string Seq = std::to_string(LtoObj->InputFiles.size()); + + NewModuleId = {sys::path::filename(ModuleId), ".", Seq, ".", PID, ".o"}; + } + + // Update the module identifier and save it. + BM.setModuleIdentifier(LtoObj->Saver.save(NewModuleId.str())); + + return Input; +} + +// Write the archive member content to a file named after the module ID. +// If a file with that name already exists, it's likely a leftover from a +// previously terminated linker process and can be safely overwritten. +Error saveInputArchiveMember(lto::LTO &LtoObj, lto::InputFile *Input) { + StringRef ModuleId = Input->getName(); + if (Input->isMemberOfArchive()) { + MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); + if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId)) + return EC; + } + return Error::success(); +} + +// Iterates through all ThinLTO-enabled input files and saves their content +// to separate files if they are regular archive members. +Error saveInputArchiveMembers(lto::LTO& LtoObj) { + for (auto &Input : LtoObj.InputFiles) { + if (!Input->isThinLTO()) + continue; + if (Error EC = saveInputArchiveMember(LtoObj, Input.get())) + return EC; + } + return Error::success(); +} + +// Entry point for DTLTO archives support. +// +// Sets up the temporary file remover and processes archive members. +// Must be called after all inputs are added but before optimization begins. +llvm::Error process(llvm::lto::LTO &LtoObj) { + if (!LtoObj.Dtlto) + return Error::success(); + + // Set up cleanup handler for temporary files + LtoObj.TempsRemover = std::make_unique(&LtoObj); + + // Process and save archive members to separate files if needed. + if (Error EC = saveInputArchiveMembers(LtoObj)) + return EC; + return Error::success(); +} + +} // namespace dtlto diff --git a/llvm/lib/LTO/CMakeLists.txt b/llvm/lib/LTO/CMakeLists.txt index 057d73b6349cf..499623eacf97c 100644 --- a/llvm/lib/LTO/CMakeLists.txt +++ b/llvm/lib/LTO/CMakeLists.txt @@ -25,6 +25,7 @@ add_llvm_component_library(LLVMLTO CodeGen CodeGenTypes Core + DTLTO Extensions IPO InstCombine diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 35d24c17bbd93..d0a7eceb38614 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -26,6 +26,7 @@ #include "llvm/CGData/CodeGenData.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/Config/llvm-config.h" +#include "llvm/DTLTO/Dtlto.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Intrinsics.h" @@ -570,6 +571,8 @@ Expected> InputFile::create(MemoryBufferRef Object) { File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts(); File->DependentLibraries = FOrErr->TheReader.getDependentLibraries(); File->ComdatTable = FOrErr->TheReader.getComdatTable(); + File->MbRef = + Object; // Save a memory buffer reference to an input file object. for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) { size_t Begin = File->Symbols.size(); @@ -729,12 +732,17 @@ static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, assert(ResI == Res.end()); } -Error LTO::add(std::unique_ptr Input, +Error LTO::add(std::unique_ptr InputPtr, ArrayRef Res) { assert(!CalledGetMaxTasks); + Expected InputOrErr = dtlto::addInput(this, std::move(InputPtr)); + if (!InputOrErr) + return InputOrErr.takeError(); + InputFile *Input = *InputOrErr; + if (Conf.ResolutionFile) - writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res); + writeToResolutionFile(*Conf.ResolutionFile, Input, Res); if (RegularLTO.CombinedModule->getTargetTriple().empty()) { Triple InputTriple(Input->getTargetTriple()); @@ -782,6 +790,10 @@ LTO::addModule(InputFile &Input, ArrayRef InputRes, LTOMode = LTOK_UnifiedThin; bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular); + // If any of the modules inside of a input bitcode file was compiled with + // ThinLTO, we assume that the whole input file also was compiled with + // ThinLTO. + Input.IsThinLTO = IsThinLTO; auto ModSyms = Input.module_symbols(ModI); addModuleToGlobalRes(ModSyms, Res, @@ -1193,6 +1205,10 @@ Error LTO::checkPartiallySplit() { } Error LTO::run(AddStreamFn AddStream, FileCache Cache) { + if (Dtlto) { + if (Error EC = dtlto::process(*this)) + return EC; + } // Compute "dead" symbols, we don't want to import/export these! DenseSet GUIDPreservedSymbols; DenseMap GUIDPrevailingResolutions; From 9afd973128f2c66f97616b70d4fc01d45178a0cb Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Fri, 5 Sep 2025 22:44:40 -0700 Subject: [PATCH 02/20] [DTLTO][ELF][COFF][MachO] Add archive support for DTLTO - Fixed compilation error on Linux. --- llvm/lib/DTLTO/Dtlto.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/DTLTO/Dtlto.cpp b/llvm/lib/DTLTO/Dtlto.cpp index 356f30188c5c4..97a15eda81674 100644 --- a/llvm/lib/DTLTO/Dtlto.cpp +++ b/llvm/lib/DTLTO/Dtlto.cpp @@ -110,7 +110,7 @@ Expected isThinArchive(const StringRef ArchivePath) { ErrorOr> MemBufferOrError = MemoryBuffer::getFileSlice(ArchivePath, THIN_ARCHIVE_MAGIC.size(), 0); - if (EC = MemBufferOrError.getError()) + if ((EC = MemBufferOrError.getError())) return createStringError(inconvertibleErrorCode(), "Failed to read from archive %s: %s", ArchivePath.data(), EC.message().c_str()); From ee4ec950532238d306657df35baecc5e3d5727e1 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Fri, 5 Sep 2025 22:55:07 -0700 Subject: [PATCH 03/20] [DTLTO][ELF][COFF][MachO] Add archive support for DTLTO - Fixed formating errors. --- lld/COFF/LTO.cpp | 3 +++ lld/ELF/LTO.cpp | 4 ++-- llvm/include/llvm/DTLTO/Dtlto.h | 4 ++-- llvm/include/llvm/LTO/LTO.h | 9 ++++++--- llvm/lib/DTLTO/Dtlto.cpp | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 1050874a1b10c..d656a54782c97 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -132,6 +132,9 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs)); } + if (!ctx.config.dtltoDistributor.empty()) + ltoObj->Dtlto = true; + ltoObj = std::make_unique(createConfig(), backend, ctx.config.ltoPartitions); } diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 13e9c63495e4d..5a8f9395b7174 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -202,8 +202,8 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { ctx.arg.ltoPartitions, ltoModes[ctx.arg.ltoKind]); - if(!ctx.arg.dtltoDistributor.empty()) - ltoObj->Dtlto = true; + if (!ctx.arg.dtltoDistributor.empty()) + ltoObj->Dtlto = true; // Initialize usedStartStop. if (ctx.bitcodeFiles.empty()) diff --git a/llvm/include/llvm/DTLTO/Dtlto.h b/llvm/include/llvm/DTLTO/Dtlto.h index aa6af7d0cd9b7..b908abf83c009 100644 --- a/llvm/include/llvm/DTLTO/Dtlto.h +++ b/llvm/include/llvm/DTLTO/Dtlto.h @@ -14,8 +14,8 @@ namespace dtlto { -llvm::Expected addInput(llvm::lto::LTO *LtoObj, - std::unique_ptr Input); +llvm::Expected +addInput(llvm::lto::LTO *LtoObj, std::unique_ptr Input); llvm::Error process(llvm::lto::LTO &LtoObj); } // namespace dtlto diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 7183a4429e665..195e9cc76003b 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -227,9 +227,12 @@ class InputFile { bool isThinLTO() const { return IsThinLTO; } // Store an archive path and a member name. - void setArchivePathAndName(StringRef Path, StringRef Name) { ArchivePath = Path; MemberName = Name; } - StringRef getArchivePath() const { return ArchivePath; } - StringRef getMemberName() const { return MemberName; } + void setArchivePathAndName(StringRef Path, StringRef Name) { + ArchivePath = Path; + MemberName = Name; + } + StringRef getArchivePath() const { return ArchivePath; } + StringRef getMemberName() const { return MemberName; } private: ArrayRef module_symbols(unsigned I) const { diff --git a/llvm/lib/DTLTO/Dtlto.cpp b/llvm/lib/DTLTO/Dtlto.cpp index 97a15eda81674..ee668d978db14 100644 --- a/llvm/lib/DTLTO/Dtlto.cpp +++ b/llvm/lib/DTLTO/Dtlto.cpp @@ -196,7 +196,7 @@ Error saveInputArchiveMember(lto::LTO &LtoObj, lto::InputFile *Input) { // Iterates through all ThinLTO-enabled input files and saves their content // to separate files if they are regular archive members. -Error saveInputArchiveMembers(lto::LTO& LtoObj) { +Error saveInputArchiveMembers(lto::LTO &LtoObj) { for (auto &Input : LtoObj.InputFiles) { if (!Input->isThinLTO()) continue; From 4171a334f022c829b66a8706ee0c98cdf340ece5 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Mon, 8 Sep 2025 04:36:25 -0700 Subject: [PATCH 04/20] [DTLTO][ELF][COFF][MachO] Add archive support for DTLTO - Addressed review comments from Tobias. --- cross-project-tests/CMakeLists.txt | 1 - llvm/include/llvm/DTLTO/{Dtlto.h => DTLTO.h} | 2 +- llvm/lib/DTLTO/{Dtlto.cpp => DTLTO.cpp} | 2 +- llvm/lib/LTO/LTO.cpp | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) rename llvm/include/llvm/DTLTO/{Dtlto.h => DTLTO.h} (90%) rename llvm/lib/DTLTO/{Dtlto.cpp => DTLTO.cpp} (99%) diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt index 6753a27698eae..192db87043177 100644 --- a/cross-project-tests/CMakeLists.txt +++ b/cross-project-tests/CMakeLists.txt @@ -20,7 +20,6 @@ set(CROSS_PROJECT_TEST_DEPS check-gdb-llvm-support count llvm-ar - llvm-ar llvm-config llvm-dwarfdump llvm-objdump diff --git a/llvm/include/llvm/DTLTO/Dtlto.h b/llvm/include/llvm/DTLTO/DTLTO.h similarity index 90% rename from llvm/include/llvm/DTLTO/Dtlto.h rename to llvm/include/llvm/DTLTO/DTLTO.h index b908abf83c009..dfbfcf79d7435 100644 --- a/llvm/include/llvm/DTLTO/Dtlto.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -1,4 +1,4 @@ -//===- Dtlto.h - Distributed ThinLTO functions and classes ----*- C++ -*-===// +//===- DTLTO.h - Distributed ThinLTO functions and classes ----*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/lib/DTLTO/Dtlto.cpp b/llvm/lib/DTLTO/DTLTO.cpp similarity index 99% rename from llvm/lib/DTLTO/Dtlto.cpp rename to llvm/lib/DTLTO/DTLTO.cpp index ee668d978db14..19ae6385f75cd 100644 --- a/llvm/lib/DTLTO/Dtlto.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/DTLTO/Dtlto.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index ca89ed6c24361..894b990a4263b 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -26,7 +26,7 @@ #include "llvm/CGData/CodeGenData.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/Config/llvm-config.h" -#include "llvm/DTLTO/Dtlto.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Intrinsics.h" From fa7376cef8495747e408730d77d518a971505c92 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Mon, 8 Sep 2025 05:01:15 -0700 Subject: [PATCH 05/20] [DTLTO][ELF][COFF][MachO] Add archive support for DTLTO - Compilation fix. --- llvm/lib/DTLTO/CMakeLists.txt | 2 +- llvm/lib/LTO/LTO.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/DTLTO/CMakeLists.txt b/llvm/lib/DTLTO/CMakeLists.txt index 51fd8aad6f48b..4a35de24c86db 100644 --- a/llvm/lib/DTLTO/CMakeLists.txt +++ b/llvm/lib/DTLTO/CMakeLists.txt @@ -1,5 +1,5 @@ add_llvm_component_library(LLVMDTLTO - Dtlto.cpp + DTLTO.cpp LINK_COMPONENTS Core diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 894b990a4263b..7c4bf54cd5fd9 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -735,7 +735,7 @@ static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, Error LTO::add(std::unique_ptr InputPtr, ArrayRef Res) { - llvm::TimeTraceScope timeScope("LTO add input", Input->getName()); + llvm::TimeTraceScope timeScope("LTO add input", InputPtr->getName()); assert(!CalledGetMaxTasks); Expected InputOrErr = dtlto::addInput(this, std::move(InputPtr)); From ccc8ae33d7f9f7e794a94fa80f18c1019dd32b5d Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Tue, 9 Sep 2025 01:11:50 -0700 Subject: [PATCH 06/20] [DTLTO][ELF][COFF] Archive support for DTLTO - Removed thin archives support from lld/ELF/InputFiles.cpp. --- lld/ELF/InputFiles.cpp | 58 ++++++++---------------------------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index ec0af9d0c0f4e..62b8ba8e93548 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -20,7 +20,6 @@ #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/STLExtras.h" #include "llvm/LTO/LTO.h" -#include "llvm/Object/Archive.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Support/AArch64AttributeParser.h" #include "llvm/Support/ARMAttributeParser.h" @@ -1812,39 +1811,6 @@ static uint8_t getOsAbi(const Triple &t) { } } -// For DTLTO, bitcode member names must be valid paths to files on disk. -// For thin archives, resolve `memberPath` relative to the archive's location. -// Returns true if adjusted; false otherwise. Non-thin archives are unsupported. -static bool dtltoAdjustMemberPathIfThinArchive(Ctx &ctx, StringRef archivePath, - std::string &memberPath) { - assert(!archivePath.empty()); - - if (ctx.arg.dtltoDistributor.empty()) - return false; - - // Read the archive header to determine if it's a thin archive. - auto bufferOrErr = - MemoryBuffer::getFileSlice(archivePath, sizeof(ThinArchiveMagic) - 1, 0); - if (std::error_code ec = bufferOrErr.getError()) { - ErrAlways(ctx) << "cannot open " << archivePath << ": " << ec.message(); - return false; - } - - if (!bufferOrErr->get()->getBuffer().starts_with(ThinArchiveMagic)) - return false; - - SmallString<128> resolvedPath; - if (path::is_relative(memberPath)) { - resolvedPath = path::parent_path(archivePath); - path::append(resolvedPath, memberPath); - } else - resolvedPath = memberPath; - - path::remove_dots(resolvedPath, /*remove_dot_dot=*/true); - memberPath = resolvedPath.str(); - return true; -} - BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, uint64_t offsetInArchive, bool lazy) : InputFile(ctx, BitcodeKind, mb) { @@ -1855,21 +1821,17 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, if (ctx.arg.thinLTOIndexOnly) path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier()); + // ThinLTO assumes that all MemoryBufferRefs given to it have a unique + // name. If two archives define two members with the same name, this + // causes a collision which result in only one of the objects being taken + // into consideration at LTO time (which very likely causes undefined + // symbols later in the link stage). So we append file offset to make + // filename unique. StringSaver &ss = ctx.saver; - StringRef name; - if (archiveName.empty() || - dtltoAdjustMemberPathIfThinArchive(ctx, archiveName, path)) { - name = ss.save(path); - } else { - // ThinLTO assumes that all MemoryBufferRefs given to it have a unique - // name. If two archives define two members with the same name, this - // causes a collision which result in only one of the objects being taken - // into consideration at LTO time (which very likely causes undefined - // symbols later in the link stage). So we append file offset to make - // filename unique. - name = ss.save(archiveName + "(" + path::filename(path) + " at " + - utostr(offsetInArchive) + ")"); - } + StringRef name = archiveName.empty() + ? ss.save(path) + : ss.save(archiveName + "(" + path::filename(path) + + " at " + utostr(offsetInArchive) + ")"); MemoryBufferRef mbref(mb.getBuffer(), name); From 70b51bd91a7da7267de758786a65a7038492d49a Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Tue, 9 Sep 2025 01:13:23 -0700 Subject: [PATCH 07/20] [DTLTO][ELF][COFF] Archive support for DTLTO - Removed archives support from lld/MachO/InputFiles.cpp for now. --- lld/MachO/InputFiles.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index 8cd4303441a22..442fc608865d2 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -2361,7 +2361,6 @@ BitcodeFile::BitcodeFile(MemoryBufferRef mb, StringRef archiveName, sys::path::filename(path) + ")" + utostr(offsetInArchive))); obj = check(lto::InputFile::create(mbref)); - obj->setArchivePathAndName(archiveName, mb.getBufferIdentifier()); if (lazy) parseLazy(); else From d61b60653dc54cab9aa952a34e27d08fe919657f Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Tue, 9 Sep 2025 02:38:57 -0700 Subject: [PATCH 08/20] [DTLTO][ELF][COFF] Archive support for DTLTO - Fixed crash in lld/COFF/LTO.cpp. --- lld/COFF/LTO.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index b31a198750483..1107fa36f5e11 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -133,11 +133,11 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs)); } - if (!ctx.config.dtltoDistributor.empty()) - ltoObj->Dtlto = true; - ltoObj = std::make_unique(createConfig(), backend, ctx.config.ltoPartitions); + + if (!ctx.config.dtltoDistributor.empty()) + ltoObj->Dtlto = true; } BitcodeCompiler::~BitcodeCompiler() = default; From fb43b8f48d0135d60fe22d57073b2a993236f8a2 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Tue, 9 Sep 2025 02:41:29 -0700 Subject: [PATCH 09/20] [DTLTO][ELF][COFF] Archive support for DTLTO - Fixed formatting in lld/ELF/InputFiles.cpp. --- lld/ELF/InputFiles.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 62b8ba8e93548..a34d007faa3df 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -1821,12 +1821,12 @@ BitcodeFile::BitcodeFile(Ctx &ctx, MemoryBufferRef mb, StringRef archiveName, if (ctx.arg.thinLTOIndexOnly) path = replaceThinLTOSuffix(ctx, mb.getBufferIdentifier()); - // ThinLTO assumes that all MemoryBufferRefs given to it have a unique - // name. If two archives define two members with the same name, this - // causes a collision which result in only one of the objects being taken - // into consideration at LTO time (which very likely causes undefined - // symbols later in the link stage). So we append file offset to make - // filename unique. + // ThinLTO assumes that all MemoryBufferRefs given to it have a unique + // name. If two archives define two members with the same name, this + // causes a collision which result in only one of the objects being taken + // into consideration at LTO time (which very likely causes undefined + // symbols later in the link stage). So we append file offset to make + // filename unique. StringSaver &ss = ctx.saver; StringRef name = archiveName.empty() ? ss.save(path) From 7ea9a04c26a16e4284ef884030efbfca85ad63c7 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Mon, 6 Oct 2025 00:27:12 -0700 Subject: [PATCH 10/20] [DTLTO][Archives] Fixed formatting issues after the merge. --- llvm/include/llvm/LTO/LTO.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 022403d24ca83..4e1e7231b0ee7 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -618,10 +618,10 @@ class LTO { // Identify symbols exported dynamically, and that therefore could be // referenced by a shared library not visible to the linker. DenseSet DynamicExportSymbols; - + // Diagnostic optimization remarks file LLVMRemarkFileHandle DiagnosticOutputFile; - + public: /// DTLTO mode. bool Dtlto = false; @@ -631,7 +631,7 @@ class LTO { // Array of input bitcode files for LTO. std::vector> InputFiles; - std::unique_ptr TempsRemover; + std::unique_ptr TempsRemover; }; /// The resolution for a symbol. The linker must provide a SymbolResolution for From 1582a50238fab7e4feb19fd80d9015c65b979c48 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Sun, 9 Nov 2025 22:43:15 -0800 Subject: [PATCH 11/20] [DTLTO][Archives] - Addressed review comment from Teresa - Removed 'if(Lto==NULL)' check from TempFilesRemover destructor. --- llvm/lib/DTLTO/DTLTO.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 19ae6385f75cd..7b11b7bbe78c0 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -36,8 +36,6 @@ namespace dtlto { // Removes any temporary regular archive member files that were created during // processing. TempFilesRemover::~TempFilesRemover() { - if (!Lto) - return; for (auto &Input : Lto->InputFiles) { if (Input->isMemberOfArchive()) sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); From 2e9421cc1b5398f0de4e45fcb7786c41f5287bf4 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Mon, 10 Nov 2025 01:57:18 -0800 Subject: [PATCH 12/20] [DTLTO][Archives] - Addressed review comment from Teresa - Replaced 'THIN_ARCHIVE_MAGIC' with object::ThinArchiveMagic. --- llvm/lib/DTLTO/DTLTO.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 7b11b7bbe78c0..50997313b5d7c 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/LTO/LTO.h" +#include "llvm/Object/Archive.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MemoryBufferRef.h" @@ -76,9 +77,6 @@ SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, return MemberPath; } -// Magic string identifying thin archive files. -static constexpr StringLiteral THIN_ARCHIVE_MAGIC = "!\n"; - // Determines if a file at the given path is a thin archive file. // // This function uses a cache to avoid repeatedly reading the same file. @@ -99,14 +97,15 @@ Expected isThinArchive(const StringRef ArchivePath) { return createStringError(inconvertibleErrorCode(), "Failed to get file size from archive %s: %s", ArchivePath.data(), EC.message().c_str()); - if (FileSize < THIN_ARCHIVE_MAGIC.size()) + if (FileSize < sizeof(object::ThinArchiveMagic)) return createStringError(inconvertibleErrorCode(), "Archive file size is too small %s", ArchivePath.data()); // Read only the first few bytes containing the magic signature. ErrorOr> MemBufferOrError = - MemoryBuffer::getFileSlice(ArchivePath, THIN_ARCHIVE_MAGIC.size(), 0); + MemoryBuffer::getFileSlice(ArchivePath, sizeof(object::ThinArchiveMagic), + 0); if ((EC = MemBufferOrError.getError())) return createStringError(inconvertibleErrorCode(), @@ -119,7 +118,7 @@ Expected isThinArchive(const StringRef ArchivePath) { "Unknown format for archive %s", ArchivePath.data()); - IsThin = MemBuf.starts_with(THIN_ARCHIVE_MAGIC); + IsThin = MemBuf.starts_with(object::ThinArchiveMagic); // Cache the result ArchiveFiles[ArchivePath] = IsThin; From cf91406244fc995ec0ec86717ef047ea7d41f960 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Wed, 3 Dec 2025 23:54:44 -0800 Subject: [PATCH 13/20] [DTLTO][Archives] - Addressed review comments from Teresa - Made addInput() a member of lto::LTO class. --- llvm/include/llvm/LTO/LTO.h | 2 ++ llvm/lib/DTLTO/DTLTO.cpp | 30 +++++++++++++++++------------- llvm/lib/LTO/LTO.cpp | 1 + 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 4e1e7231b0ee7..9e8cbede82cc8 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -632,6 +632,8 @@ class LTO { // Array of input bitcode files for LTO. std::vector> InputFiles; std::unique_ptr TempsRemover; + + Expected addInput(std::unique_ptr InputPtr); }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 50997313b5d7c..f577cbbc1d693 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -125,22 +125,24 @@ Expected isThinArchive(const StringRef ArchivePath) { return IsThin; } +} // namespace dtlto + // This function performs the following tasks: // 1. Adds the input file to the LTO object's list of input files. // 2. For thin archive members, generates a new module ID which is a path to a // thin archive member file. // 3. For regular archive members, generates a new unique module ID. // 4. Updates the bitcode module's identifier. -Expected addInput(lto::LTO *LtoObj, - std::unique_ptr InputPtr) { - - // Add the input file to the LTO object. - LtoObj->InputFiles.push_back(std::move(InputPtr)); - lto::InputFile *Input = LtoObj->InputFiles.back().get(); +Expected +lto::LTO::addInput(std::unique_ptr InputPtr) { // Skip processing if not in DTLTO mode. - if (!LtoObj->Dtlto) - return Input; + if (!Dtlto) + return InputPtr.release(); + + // Add the input file to the LTO object. + InputFiles.push_back(std::move(InputPtr)); + lto::InputFile *Input = InputFiles.back().get(); StringRef ModuleId = Input->getName(); StringRef ArchivePath = Input->getArchivePath(); @@ -153,31 +155,33 @@ Expected addInput(lto::LTO *LtoObj, BitcodeModule &BM = Input->getSingleBitcodeModule(); // Check if the archive is a thin archive. - Expected IsThin = isThinArchive(ArchivePath); + Expected IsThin = dtlto::isThinArchive(ArchivePath); if (!IsThin) return IsThin.takeError(); if (*IsThin) { // For thin archives, use the path to the actual file. - NewModuleId = - computeThinArchiveMemberPath(ArchivePath, Input->getMemberName()); + NewModuleId = dtlto::computeThinArchiveMemberPath(ArchivePath, + Input->getMemberName()); } else { // For regular archives, generate a unique name. Input->memberOfArchive(true); // Create unique identifier using process ID and sequence number. std::string PID = utohexstr(sys::Process::getProcessId()); - std::string Seq = std::to_string(LtoObj->InputFiles.size()); + std::string Seq = std::to_string(InputFiles.size()); NewModuleId = {sys::path::filename(ModuleId), ".", Seq, ".", PID, ".o"}; } // Update the module identifier and save it. - BM.setModuleIdentifier(LtoObj->Saver.save(NewModuleId.str())); + BM.setModuleIdentifier(Saver.save(NewModuleId.str())); return Input; } +namespace dtlto { + // Write the archive member content to a file named after the module ID. // If a file with that name already exists, it's likely a leftover from a // previously terminated linker process and can be safely overwritten. diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index b1f0105ccaafa..da7a38e38c7f8 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -761,6 +761,7 @@ Error LTO::add(std::unique_ptr InputPtr, } assert(Res.empty()); + if (!Dtlto) delete Input; return Error::success(); } From 85d2344d8cb85fd6e911c7e6c09cc26e273ed8c6 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Thu, 4 Dec 2025 22:44:50 -0800 Subject: [PATCH 14/20] [DTLTO][Archives] - Addressed review comments from Teresa - Fixed memory overhead for non-DTLTO mode from lto::LTO::add(). --- llvm/include/llvm/LTO/LTO.h | 7 ++++--- llvm/lib/DTLTO/DTLTO.cpp | 8 ++++---- llvm/lib/LTO/LTO.cpp | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 9e8cbede82cc8..5a5f772c735dc 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -623,17 +623,18 @@ class LTO { LLVMRemarkFileHandle DiagnosticOutputFile; public: - /// DTLTO mode. + // DTLTO mode. bool Dtlto = false; BumpPtrAllocator PtrAlloc; StringSaver Saver{PtrAlloc}; // Array of input bitcode files for LTO. - std::vector> InputFiles; + std::vector> InputFiles; std::unique_ptr TempsRemover; - Expected addInput(std::unique_ptr InputPtr); + Expected> + addInput(std::unique_ptr InputPtr); }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index f577cbbc1d693..adb9de84c03c7 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -133,16 +133,16 @@ Expected isThinArchive(const StringRef ArchivePath) { // thin archive member file. // 3. For regular archive members, generates a new unique module ID. // 4. Updates the bitcode module's identifier. -Expected +Expected> lto::LTO::addInput(std::unique_ptr InputPtr) { // Skip processing if not in DTLTO mode. if (!Dtlto) - return InputPtr.release(); + return std::shared_ptr(InputPtr.release()); // Add the input file to the LTO object. - InputFiles.push_back(std::move(InputPtr)); - lto::InputFile *Input = InputFiles.back().get(); + InputFiles.emplace_back(InputPtr.release()); + std::shared_ptr &Input = InputFiles.back(); StringRef ModuleId = Input->getName(); StringRef ArchivePath = Input->getArchivePath(); diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index da7a38e38c7f8..69ea2f20bebd9 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -739,10 +739,11 @@ Error LTO::add(std::unique_ptr InputPtr, llvm::TimeTraceScope timeScope("LTO add input", InputPtr->getName()); assert(!CalledGetMaxTasks); - Expected InputOrErr = dtlto::addInput(this, std::move(InputPtr)); + Expected> InputOrErr = + addInput(std::move(InputPtr)); if (!InputOrErr) return InputOrErr.takeError(); - InputFile *Input = *InputOrErr; + InputFile *Input = (*InputOrErr).get(); if (Conf.ResolutionFile) writeToResolutionFile(*Conf.ResolutionFile, Input, Res); @@ -761,7 +762,6 @@ Error LTO::add(std::unique_ptr InputPtr, } assert(Res.empty()); - if (!Dtlto) delete Input; return Error::success(); } From 413617096f0d613f4c0473e0b110a8973e7e6bb5 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Fri, 5 Dec 2025 00:24:12 -0800 Subject: [PATCH 15/20] [DTLTO][Archives] - Simplification - Made TempFilesRemover class a member of lto::LTO class. --- llvm/include/llvm/LTO/LTO.h | 20 +++++++++----------- llvm/lib/DTLTO/DTLTO.cpp | 9 ++++----- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 5a5f772c735dc..4cee468d134e7 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -39,16 +39,6 @@ class LTO; } } // namespace llvm -namespace dtlto { -class TempFilesRemover { - llvm::lto::LTO *Lto = nullptr; - -public: - TempFilesRemover(llvm::lto::LTO *LtoObj) : Lto{LtoObj} {} - ~TempFilesRemover(); -}; - -} // namespace dtlto namespace llvm { @@ -629,9 +619,17 @@ class LTO { BumpPtrAllocator PtrAlloc; StringSaver Saver{PtrAlloc}; + class TempFilesRemover { + LTO *Lto; + + public: + TempFilesRemover(LTO *P) : Lto{P} {}; + ~TempFilesRemover(); + }; + // Array of input bitcode files for LTO. std::vector> InputFiles; - std::unique_ptr TempsRemover; + TempFilesRemover TempsRemover{ this }; Expected> addInput(std::unique_ptr InputPtr); diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index adb9de84c03c7..ed82cea09a6b4 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -32,17 +32,19 @@ using namespace llvm; -namespace dtlto { // Removes any temporary regular archive member files that were created during // processing. -TempFilesRemover::~TempFilesRemover() { +lto::LTO::TempFilesRemover::~TempFilesRemover() { for (auto &Input : Lto->InputFiles) { if (Input->isMemberOfArchive()) sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); } } +namespace dtlto { + + // Writes the content of a memory buffer into a file. static llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { std::error_code EC; @@ -215,9 +217,6 @@ llvm::Error process(llvm::lto::LTO &LtoObj) { if (!LtoObj.Dtlto) return Error::success(); - // Set up cleanup handler for temporary files - LtoObj.TempsRemover = std::make_unique(&LtoObj); - // Process and save archive members to separate files if needed. if (Error EC = saveInputArchiveMembers(LtoObj)) return EC; From f89dc850117c4a0dd3c12c74c45a38fe79017bad Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Sat, 6 Dec 2025 22:16:07 -0800 Subject: [PATCH 16/20] [DTLTO][Archives] - Addressed review comments from Teresa - Introduced lto::DTLTO class derived from lto::LTO class. --- lld/COFF/LTO.cpp | 3 ++- lld/ELF/LTO.cpp | 9 +++++---- lld/MachO/LTO.cpp | 3 ++- lld/wasm/LTO.cpp | 5 +++-- llvm/include/llvm/DTLTO/DTLTO.h | 28 +++++++++++++++++++++----- llvm/include/llvm/LTO/LTO.h | 21 +++++--------------- llvm/lib/DTLTO/DTLTO.cpp | 35 +++++++++++++++------------------ llvm/lib/LTO/LTO.cpp | 2 +- 8 files changed, 57 insertions(+), 49 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 1107fa36f5e11..18dc19afbba55 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -21,6 +21,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" @@ -133,7 +134,7 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs)); } - ltoObj = std::make_unique(createConfig(), backend, + ltoObj = std::make_unique(createConfig(), backend, ctx.config.ltoPartitions); if (!ctx.config.dtltoDistributor.empty()) diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 5a8f9395b7174..55a51f34d93e3 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" @@ -197,10 +198,10 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { constexpr llvm::lto::LTO::LTOKind ltoModes[3] = {llvm::lto::LTO::LTOKind::LTOK_UnifiedThin, llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular, - llvm::lto::LTO::LTOKind::LTOK_Default}; - ltoObj = std::make_unique(createConfig(ctx), backend, - ctx.arg.ltoPartitions, - ltoModes[ctx.arg.ltoKind]); + llvm::lto::LTO::LTOKind::LTOK_Default}; + ltoObj = std::make_unique(createConfig(ctx), backend, + ctx.arg.ltoPartitions, + ltoModes[ctx.arg.ltoKind]); if (!ctx.arg.dtltoDistributor.empty()) ltoObj->Dtlto = true; diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp index 4695b639dcc96..777f895f2ec55 100644 --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -18,6 +18,7 @@ #include "lld/Common/Strings.h" #include "lld/Common/TargetOptionsCommandFlags.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" @@ -101,7 +102,7 @@ BitcodeCompiler::BitcodeCompiler() { config->thinLTOEmitImportsFiles); } - ltoObj = std::make_unique(createConfig(), backend); + ltoObj = std::make_unique(createConfig(), backend); } void BitcodeCompiler::add(BitcodeFile &f) { diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp index 71f18aa25a35c..d2ed44bdddbc7 100644 --- a/lld/wasm/LTO.cpp +++ b/lld/wasm/LTO.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/DTLTO/DTLTO.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" @@ -93,8 +94,8 @@ BitcodeCompiler::BitcodeCompiler() { onIndexWrite, ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles); } - ltoObj = std::make_unique(createConfig(), backend, - ctx.arg.ltoPartitions); + ltoObj = std::make_unique(createConfig(), backend, + ctx.arg.ltoPartitions); } BitcodeCompiler::~BitcodeCompiler() = default; diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index dfbfcf79d7435..88def0e83f550 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -12,12 +12,30 @@ #include "llvm/LTO/LTO.h" #include "llvm/Support/MemoryBuffer.h" -namespace dtlto { +namespace llvm { +namespace lto { -llvm::Expected -addInput(llvm::lto::LTO *LtoObj, std::unique_ptr Input); +class DTLTO : public LTO { +public: + // Inherit contructors from LTO base class. + using LTO::LTO; + ~DTLTO() { removeTempFiles(); } -llvm::Error process(llvm::lto::LTO &LtoObj); -} // namespace dtlto + BumpPtrAllocator PtrAlloc; + StringSaver Saver{PtrAlloc}; + + // Remove temporary files. + void removeTempFiles(); + + // Array of input bitcode files for LTO. + std::vector> InputFiles; + + virtual Expected> + addInput(std::unique_ptr InputPtr) override; + + virtual llvm::Error dtlto_process() override; +}; +} // namespace lto +} // namespace llvm #endif // LLVM_DTLTO_H diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 4cee468d134e7..db1383b7998c6 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -616,23 +616,12 @@ class LTO { // DTLTO mode. bool Dtlto = false; - BumpPtrAllocator PtrAlloc; - StringSaver Saver{PtrAlloc}; - - class TempFilesRemover { - LTO *Lto; - - public: - TempFilesRemover(LTO *P) : Lto{P} {}; - ~TempFilesRemover(); - }; - - // Array of input bitcode files for LTO. - std::vector> InputFiles; - TempFilesRemover TempsRemover{ this }; + virtual Expected> + addInput(std::unique_ptr InputPtr) { + return std::shared_ptr(InputPtr.release()); + } - Expected> - addInput(std::unique_ptr InputPtr); + virtual llvm::Error dtlto_process() { return llvm::Error::success(); } }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index ed82cea09a6b4..727e36a082812 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -33,18 +33,8 @@ using namespace llvm; -// Removes any temporary regular archive member files that were created during -// processing. -lto::LTO::TempFilesRemover::~TempFilesRemover() { - for (auto &Input : Lto->InputFiles) { - if (Input->isMemberOfArchive()) - sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); - } -} - namespace dtlto { - // Writes the content of a memory buffer into a file. static llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { std::error_code EC; @@ -129,6 +119,15 @@ Expected isThinArchive(const StringRef ArchivePath) { } // namespace dtlto +// Removes any temporary regular archive member files that were created during +// processing. +void lto::DTLTO::removeTempFiles() { + for (auto &Input : InputFiles) { + if (Input->isMemberOfArchive()) + sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); + } +} + // This function performs the following tasks: // 1. Adds the input file to the LTO object's list of input files. // 2. For thin archive members, generates a new module ID which is a path to a @@ -136,7 +135,7 @@ Expected isThinArchive(const StringRef ArchivePath) { // 3. For regular archive members, generates a new unique module ID. // 4. Updates the bitcode module's identifier. Expected> -lto::LTO::addInput(std::unique_ptr InputPtr) { +lto::DTLTO::addInput(std::unique_ptr InputPtr) { // Skip processing if not in DTLTO mode. if (!Dtlto) @@ -187,7 +186,7 @@ namespace dtlto { // Write the archive member content to a file named after the module ID. // If a file with that name already exists, it's likely a leftover from a // previously terminated linker process and can be safely overwritten. -Error saveInputArchiveMember(lto::LTO &LtoObj, lto::InputFile *Input) { +Error saveInputArchiveMember(lto::DTLTO &LtoObj, lto::InputFile *Input) { StringRef ModuleId = Input->getName(); if (Input->isMemberOfArchive()) { MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); @@ -199,7 +198,7 @@ Error saveInputArchiveMember(lto::LTO &LtoObj, lto::InputFile *Input) { // Iterates through all ThinLTO-enabled input files and saves their content // to separate files if they are regular archive members. -Error saveInputArchiveMembers(lto::LTO &LtoObj) { +Error saveInputArchiveMembers(lto::DTLTO &LtoObj) { for (auto &Input : LtoObj.InputFiles) { if (!Input->isThinLTO()) continue; @@ -209,18 +208,16 @@ Error saveInputArchiveMembers(lto::LTO &LtoObj) { return Error::success(); } +} // namespace dtlto + // Entry point for DTLTO archives support. // // Sets up the temporary file remover and processes archive members. // Must be called after all inputs are added but before optimization begins. -llvm::Error process(llvm::lto::LTO &LtoObj) { - if (!LtoObj.Dtlto) - return Error::success(); +llvm::Error lto::DTLTO::dtlto_process() { // Process and save archive members to separate files if needed. - if (Error EC = saveInputArchiveMembers(LtoObj)) + if (Error EC = dtlto::saveInputArchiveMembers(*this)) return EC; return Error::success(); } - -} // namespace dtlto diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 69ea2f20bebd9..a135ad18d4750 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1214,7 +1214,7 @@ Error LTO::checkPartiallySplit() { Error LTO::run(AddStreamFn AddStream, FileCache Cache) { if (Dtlto) { - if (Error EC = dtlto::process(*this)) + if (Error EC = dtlto_process()) return EC; } // Compute "dead" symbols, we don't want to import/export these! From 9571a3d3e3f4de100597ca962410688b027fb4f4 Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Sat, 6 Dec 2025 22:30:19 -0800 Subject: [PATCH 17/20] [DTLTO][Archives] - Fixed code formatting. --- lld/COFF/LTO.cpp | 2 +- lld/wasm/LTO.cpp | 2 +- llvm/include/llvm/DTLTO/DTLTO.h | 6 +++--- llvm/include/llvm/LTO/LTO.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 18dc19afbba55..186f3068979e9 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -20,8 +20,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/DTLTO/DTLTO.h" +#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp index d2ed44bdddbc7..c3f399fcd93d0 100644 --- a/lld/wasm/LTO.cpp +++ b/lld/wasm/LTO.cpp @@ -18,8 +18,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/DTLTO/DTLTO.h" +#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/LTO/Config.h" #include "llvm/LTO/LTO.h" #include "llvm/Support/Caching.h" diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index 88def0e83f550..97a47eef37337 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -25,15 +25,15 @@ class DTLTO : public LTO { StringSaver Saver{PtrAlloc}; // Remove temporary files. - void removeTempFiles(); + LLVM_ABI void removeTempFiles(); // Array of input bitcode files for LTO. std::vector> InputFiles; - virtual Expected> + LLVM_ABI virtual Expected> addInput(std::unique_ptr InputPtr) override; - virtual llvm::Error dtlto_process() override; + LLVM_ABI virtual llvm::Error dtlto_process() override; }; } // namespace lto } // namespace llvm diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index db1383b7998c6..9894c5a3bbbe3 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -425,7 +425,7 @@ class LTO { LLVM_ABI LTO(Config Conf, ThinBackend Backend = {}, unsigned ParallelCodeGenParallelismLevel = 1, LTOKind LTOMode = LTOK_Default); - LLVM_ABI ~LTO(); + LLVM_ABI virtual ~LTO(); /// Add an input file to the LTO link, using the provided symbol resolutions. /// The symbol resolutions must appear in the enumeration order given by From e7aebd4bb633baf06308c8a908d3e920d61da85a Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Sun, 7 Dec 2025 18:41:45 -0800 Subject: [PATCH 18/20] [DTLTO][Archives] - Fixed code formatting. --- lld/COFF/LTO.cpp | 2 +- lld/ELF/LTO.cpp | 6 +++--- llvm/lib/DTLTO/DTLTO.cpp | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 186f3068979e9..e296e48883ede 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -135,7 +135,7 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { } ltoObj = std::make_unique(createConfig(), backend, - ctx.config.ltoPartitions); + ctx.config.ltoPartitions); if (!ctx.config.dtltoDistributor.empty()) ltoObj->Dtlto = true; diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 55a51f34d93e3..4f002223f3abc 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -195,9 +195,9 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { ctx.arg.thinLTOEmitImportsFiles); } - constexpr llvm::lto::LTO::LTOKind ltoModes[3] = - {llvm::lto::LTO::LTOKind::LTOK_UnifiedThin, - llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular, + constexpr llvm::lto::LTO::LTOKind ltoModes[3] = { + llvm::lto::LTO::LTOKind::LTOK_UnifiedThin, + llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular, llvm::lto::LTO::LTOKind::LTOK_Default}; ltoObj = std::make_unique(createConfig(ctx), backend, ctx.arg.ltoPartitions, diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 727e36a082812..6ad4ee99e4a9e 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -32,7 +32,6 @@ using namespace llvm; - namespace dtlto { // Writes the content of a memory buffer into a file. From ea80ce2f4e4d54b900296036d74a4a059d35d63a Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Tue, 9 Dec 2025 20:57:04 -0800 Subject: [PATCH 19/20] [DTLTO][Archives] - Addressed review comments from Teresa - Removed Dtlto flag from lto::LTO class. --- lld/COFF/LTO.cpp | 9 +++++---- lld/ELF/LTO.cpp | 11 ++++++----- lld/MachO/LTO.cpp | 2 +- lld/wasm/LTO.cpp | 4 ++-- llvm/include/llvm/DTLTO/DTLTO.h | 4 +++- llvm/include/llvm/LTO/LTO.h | 5 +---- llvm/lib/DTLTO/DTLTO.cpp | 18 ++++++------------ llvm/lib/LTO/LTO.cpp | 7 +++---- 8 files changed, 27 insertions(+), 33 deletions(-) diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index e296e48883ede..36341cda5d130 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -134,11 +134,12 @@ BitcodeCompiler::BitcodeCompiler(COFFLinkerContext &c) : ctx(c) { llvm::heavyweight_hardware_concurrency(ctx.config.thinLTOJobs)); } - ltoObj = std::make_unique(createConfig(), backend, + if (ctx.config.dtltoDistributor.empty()) + ltoObj = std::make_unique(createConfig(), backend, ctx.config.ltoPartitions); - - if (!ctx.config.dtltoDistributor.empty()) - ltoObj->Dtlto = true; + else + ltoObj = std::make_unique(createConfig(), backend, + ctx.config.ltoPartitions); } BitcodeCompiler::~BitcodeCompiler() = default; diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp index 4f002223f3abc..b0c21dcc055f6 100644 --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -199,13 +199,14 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) { llvm::lto::LTO::LTOKind::LTOK_UnifiedThin, llvm::lto::LTO::LTOKind::LTOK_UnifiedRegular, llvm::lto::LTO::LTOKind::LTOK_Default}; - ltoObj = std::make_unique(createConfig(ctx), backend, + if (ctx.arg.dtltoDistributor.empty()) + ltoObj = std::make_unique(createConfig(ctx), backend, ctx.arg.ltoPartitions, ltoModes[ctx.arg.ltoKind]); - - if (!ctx.arg.dtltoDistributor.empty()) - ltoObj->Dtlto = true; - + else + ltoObj = std::make_unique(createConfig(ctx), backend, + ctx.arg.ltoPartitions, + ltoModes[ctx.arg.ltoKind]); // Initialize usedStartStop. if (ctx.bitcodeFiles.empty()) return; diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp index 777f895f2ec55..0e8eed1ad8a83 100644 --- a/lld/MachO/LTO.cpp +++ b/lld/MachO/LTO.cpp @@ -102,7 +102,7 @@ BitcodeCompiler::BitcodeCompiler() { config->thinLTOEmitImportsFiles); } - ltoObj = std::make_unique(createConfig(), backend); + ltoObj = std::make_unique(createConfig(), backend); } void BitcodeCompiler::add(BitcodeFile &f) { diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp index c3f399fcd93d0..1a2c7bc580875 100644 --- a/lld/wasm/LTO.cpp +++ b/lld/wasm/LTO.cpp @@ -94,8 +94,8 @@ BitcodeCompiler::BitcodeCompiler() { onIndexWrite, ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles); } - ltoObj = std::make_unique(createConfig(), backend, - ctx.arg.ltoPartitions); + ltoObj = std::make_unique(createConfig(), backend, + ctx.arg.ltoPartitions); } BitcodeCompiler::~BitcodeCompiler() = default; diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index 97a47eef37337..d6a0cd7050c1d 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -33,7 +33,9 @@ class DTLTO : public LTO { LLVM_ABI virtual Expected> addInput(std::unique_ptr InputPtr) override; - LLVM_ABI virtual llvm::Error dtlto_process() override; + LLVM_ABI virtual llvm::Error handleArchiveInputs() override; + + StringMap ArchiveFiles; }; } // namespace lto } // namespace llvm diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 9894c5a3bbbe3..d63d27c53a747 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -613,15 +613,12 @@ class LTO { LLVMRemarkFileHandle DiagnosticOutputFile; public: - // DTLTO mode. - bool Dtlto = false; - virtual Expected> addInput(std::unique_ptr InputPtr) { return std::shared_ptr(InputPtr.release()); } - virtual llvm::Error dtlto_process() { return llvm::Error::success(); } + virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); } }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 6ad4ee99e4a9e..94922638d5cc2 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -73,12 +73,10 @@ SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, // This function uses a cache to avoid repeatedly reading the same file. // It reads only the header portion (magic bytes) of the file to identify // the archive type. -Expected isThinArchive(const StringRef ArchivePath) { - static StringMap ArchiveFiles; - +Expected isThinArchive(lto::DTLTO *Dtlto, const StringRef ArchivePath) { // Return cached result if available. - auto Cached = ArchiveFiles.find(ArchivePath); - if (Cached != ArchiveFiles.end()) + auto Cached = Dtlto->ArchiveFiles.find(ArchivePath); + if (Cached != Dtlto->ArchiveFiles.end()) return Cached->second; uint64_t FileSize = -1; @@ -112,7 +110,7 @@ Expected isThinArchive(const StringRef ArchivePath) { IsThin = MemBuf.starts_with(object::ThinArchiveMagic); // Cache the result - ArchiveFiles[ArchivePath] = IsThin; + Dtlto->ArchiveFiles[ArchivePath] = IsThin; return IsThin; } @@ -136,10 +134,6 @@ void lto::DTLTO::removeTempFiles() { Expected> lto::DTLTO::addInput(std::unique_ptr InputPtr) { - // Skip processing if not in DTLTO mode. - if (!Dtlto) - return std::shared_ptr(InputPtr.release()); - // Add the input file to the LTO object. InputFiles.emplace_back(InputPtr.release()); std::shared_ptr &Input = InputFiles.back(); @@ -155,7 +149,7 @@ lto::DTLTO::addInput(std::unique_ptr InputPtr) { BitcodeModule &BM = Input->getSingleBitcodeModule(); // Check if the archive is a thin archive. - Expected IsThin = dtlto::isThinArchive(ArchivePath); + Expected IsThin = dtlto::isThinArchive(this, ArchivePath); if (!IsThin) return IsThin.takeError(); @@ -213,7 +207,7 @@ Error saveInputArchiveMembers(lto::DTLTO &LtoObj) { // // Sets up the temporary file remover and processes archive members. // Must be called after all inputs are added but before optimization begins. -llvm::Error lto::DTLTO::dtlto_process() { +llvm::Error lto::DTLTO::handleArchiveInputs() { // Process and save archive members to separate files if needed. if (Error EC = dtlto::saveInputArchiveMembers(*this)) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index a135ad18d4750..7a772363d2b75 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1213,10 +1213,9 @@ Error LTO::checkPartiallySplit() { } Error LTO::run(AddStreamFn AddStream, FileCache Cache) { - if (Dtlto) { - if (Error EC = dtlto_process()) - return EC; - } + if (Error EC = handleArchiveInputs()) + return EC; + // Compute "dead" symbols, we don't want to import/export these! DenseSet GUIDPreservedSymbols; DenseMap GUIDPrevailingResolutions; From ce11095beacf377be0d7cddc19f12782bdb3595d Mon Sep 17 00:00:00 2001 From: Konstantin Belochapka Date: Wed, 10 Dec 2025 22:38:58 -0800 Subject: [PATCH 20/20] [DTLTO][Archives] - Addressed review comments from Teresa - Made some functions and data members private for lto::DTLTO class. --- llvm/include/llvm/DTLTO/DTLTO.h | 24 +++++++++++++++++++--- llvm/include/llvm/LTO/LTO.h | 7 ------- llvm/lib/DTLTO/DTLTO.cpp | 36 +++++++++++++++------------------ 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index d6a0cd7050c1d..f9fad743e721c 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -21,21 +21,39 @@ class DTLTO : public LTO { using LTO::LTO; ~DTLTO() { removeTempFiles(); } +private: + // Bump allocator for a purpose of saving updated module IDs. BumpPtrAllocator PtrAlloc; StringSaver Saver{PtrAlloc}; - // Remove temporary files. + // Removes temporary files. LLVM_ABI void removeTempFiles(); + // Determines if a file at the given path is a thin archive file. + Expected isThinArchive(const StringRef ArchivePath); + + // Write the archive member content to a file named after the module ID. + Error saveInputArchiveMember(lto::InputFile *Input); + + // Iterates through all input files and saves their content + // to files if they are regular archive members. + Error saveInputArchiveMembers(); + // Array of input bitcode files for LTO. std::vector> InputFiles; + // A cache to avoid repeatedly reading the same archive file. + StringMap ArchiveFiles; + +public: + // Adds the input file to the LTO object's list of input files. + // For archive members, generates a new module ID which is a path to a real + // file on a filesystem. LLVM_ABI virtual Expected> addInput(std::unique_ptr InputPtr) override; + // Entry point for DTLTO archives support. LLVM_ABI virtual llvm::Error handleArchiveInputs() override; - - StringMap ArchiveFiles; }; } // namespace lto } // namespace llvm diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index d63d27c53a747..ae5402f64ac49 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -33,13 +33,6 @@ #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" -namespace llvm { -namespace lto { -class LTO; -} -} // namespace llvm - - namespace llvm { class Error; diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index 94922638d5cc2..7ba4bfd80b6ab 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -32,10 +32,10 @@ using namespace llvm; -namespace dtlto { +namespace { // Writes the content of a memory buffer into a file. -static llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { +llvm::Error saveBuffer(StringRef FileBuffer, StringRef FilePath) { std::error_code EC; raw_fd_ostream OS(FilePath.str(), EC, sys::fs::OpenFlags::OF_None); if (EC) { @@ -68,15 +68,17 @@ SmallString<64> computeThinArchiveMemberPath(const StringRef ArchivePath, return MemberPath; } +} // namespace + // Determines if a file at the given path is a thin archive file. // // This function uses a cache to avoid repeatedly reading the same file. // It reads only the header portion (magic bytes) of the file to identify // the archive type. -Expected isThinArchive(lto::DTLTO *Dtlto, const StringRef ArchivePath) { +Expected lto::DTLTO::isThinArchive(const StringRef ArchivePath) { // Return cached result if available. - auto Cached = Dtlto->ArchiveFiles.find(ArchivePath); - if (Cached != Dtlto->ArchiveFiles.end()) + auto Cached = ArchiveFiles.find(ArchivePath); + if (Cached != ArchiveFiles.end()) return Cached->second; uint64_t FileSize = -1; @@ -110,12 +112,10 @@ Expected isThinArchive(lto::DTLTO *Dtlto, const StringRef ArchivePath) { IsThin = MemBuf.starts_with(object::ThinArchiveMagic); // Cache the result - Dtlto->ArchiveFiles[ArchivePath] = IsThin; + ArchiveFiles[ArchivePath] = IsThin; return IsThin; } -} // namespace dtlto - // Removes any temporary regular archive member files that were created during // processing. void lto::DTLTO::removeTempFiles() { @@ -149,14 +149,14 @@ lto::DTLTO::addInput(std::unique_ptr InputPtr) { BitcodeModule &BM = Input->getSingleBitcodeModule(); // Check if the archive is a thin archive. - Expected IsThin = dtlto::isThinArchive(this, ArchivePath); + Expected IsThin = isThinArchive(ArchivePath); if (!IsThin) return IsThin.takeError(); if (*IsThin) { // For thin archives, use the path to the actual file. - NewModuleId = dtlto::computeThinArchiveMemberPath(ArchivePath, - Input->getMemberName()); + NewModuleId = + computeThinArchiveMemberPath(ArchivePath, Input->getMemberName()); } else { // For regular archives, generate a unique name. Input->memberOfArchive(true); @@ -174,12 +174,10 @@ lto::DTLTO::addInput(std::unique_ptr InputPtr) { return Input; } -namespace dtlto { - // Write the archive member content to a file named after the module ID. // If a file with that name already exists, it's likely a leftover from a // previously terminated linker process and can be safely overwritten. -Error saveInputArchiveMember(lto::DTLTO &LtoObj, lto::InputFile *Input) { +Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) { StringRef ModuleId = Input->getName(); if (Input->isMemberOfArchive()) { MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); @@ -191,18 +189,16 @@ Error saveInputArchiveMember(lto::DTLTO &LtoObj, lto::InputFile *Input) { // Iterates through all ThinLTO-enabled input files and saves their content // to separate files if they are regular archive members. -Error saveInputArchiveMembers(lto::DTLTO &LtoObj) { - for (auto &Input : LtoObj.InputFiles) { +Error lto::DTLTO::saveInputArchiveMembers() { + for (auto &Input : InputFiles) { if (!Input->isThinLTO()) continue; - if (Error EC = saveInputArchiveMember(LtoObj, Input.get())) + if (Error EC = saveInputArchiveMember(Input.get())) return EC; } return Error::success(); } -} // namespace dtlto - // Entry point for DTLTO archives support. // // Sets up the temporary file remover and processes archive members. @@ -210,7 +206,7 @@ Error saveInputArchiveMembers(lto::DTLTO &LtoObj) { llvm::Error lto::DTLTO::handleArchiveInputs() { // Process and save archive members to separate files if needed. - if (Error EC = dtlto::saveInputArchiveMembers(*this)) + if (Error EC = saveInputArchiveMembers()) return EC; return Error::success(); }