diff --git a/lld/MachO/SectionPriorities.cpp b/lld/MachO/SectionPriorities.cpp index 5faedd9b790a5..cf657aad5d145 100644 --- a/lld/MachO/SectionPriorities.cpp +++ b/lld/MachO/SectionPriorities.cpp @@ -247,15 +247,13 @@ DenseMap CallGraphSort::run() { } std::optional -macho::PriorityBuilder::getSymbolPriority(const Defined *sym) { - if (sym->isAbsolute()) - return std::nullopt; +macho::PriorityBuilder::getSymbolOrCStringPriority(const StringRef key, + InputFile *f) { - auto it = priorities.find(utils::getRootSymbol(sym->getName())); + auto it = priorities.find(key); if (it == priorities.end()) return std::nullopt; const SymbolPriorityEntry &entry = it->second; - const InputFile *f = sym->isec()->getFile(); if (!f) return entry.anyObjectFile; // We don't use toString(InputFile *) here because it returns the full path @@ -269,6 +267,14 @@ macho::PriorityBuilder::getSymbolPriority(const Defined *sym) { return std::min(entry.objectFiles.lookup(filename), entry.anyObjectFile); } +std::optional +macho::PriorityBuilder::getSymbolPriority(const Defined *sym) { + if (sym->isAbsolute()) + return std::nullopt; + return getSymbolOrCStringPriority(utils::getRootSymbol(sym->getName()), + sym->isec()->getFile()); +} + void macho::PriorityBuilder::extractCallGraphProfile() { TimeTraceScope timeScope("Extract call graph profile"); bool hasOrderFile = !priorities.empty(); @@ -301,7 +307,7 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) { int prio = std::numeric_limits::min(); MemoryBufferRef mbref = *buffer; for (StringRef line : args::getLines(mbref)) { - StringRef objectFile, symbol; + StringRef objectFile, symbolOrCStrHash; line = line.take_until([](char c) { return c == '#'; }); // ignore comments line = line.ltrim(); @@ -316,7 +322,6 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) { if (cpuType != CPU_TYPE_ANY && cpuType != target->cpuType) continue; - // Drop the CPU type as well as the colon if (cpuType != CPU_TYPE_ANY) line = line.drop_until([](char c) { return c == ':'; }).drop_front(); @@ -331,10 +336,20 @@ void macho::PriorityBuilder::parseOrderFile(StringRef path) { break; } } - symbol = utils::getRootSymbol(line.trim()); - if (!symbol.empty()) { - SymbolPriorityEntry &entry = priorities[symbol]; + // The rest of the line is either or + // CStringEntryPrefix + line = line.trim(); + if (line.starts_with(CStringEntryPrefix)) { + StringRef possibleHash = line.drop_front(CStringEntryPrefix.size()); + uint32_t hash = 0; + if (to_integer(possibleHash, hash)) + symbolOrCStrHash = possibleHash; + } else + symbolOrCStrHash = utils::getRootSymbol(line); + + if (!symbolOrCStrHash.empty()) { + SymbolPriorityEntry &entry = priorities[symbolOrCStrHash]; if (!objectFile.empty()) entry.objectFiles.insert(std::make_pair(objectFile, prio)); else @@ -389,3 +404,41 @@ macho::PriorityBuilder::buildInputSectionPriorities() { return sectionPriorities; } + +std::vector macho::PriorityBuilder::buildCStringPriorities( + ArrayRef inputs) { + // Split the input strings into hold and cold sets. + // Order hot set based on -order_file_cstring for performance improvement; + // TODO: Order cold set of cstrings for compression via BP. + std::vector> + hotStringPrioritiesAndStringPieces; + std::vector coldStringPieces; + std::vector orderedStringPieces; + + for (CStringInputSection *isec : inputs) { + for (const auto &[stringPieceIdx, piece] : llvm::enumerate(isec->pieces)) { + if (!piece.live) + continue; + + std::optional priority = getSymbolOrCStringPriority( + std::to_string(piece.hash), isec->getFile()); + if (!priority) + coldStringPieces.emplace_back(isec, stringPieceIdx); + else + hotStringPrioritiesAndStringPieces.emplace_back( + *priority, std::make_pair(isec, stringPieceIdx)); + } + } + + // Order hot set for perf + llvm::stable_sort(hotStringPrioritiesAndStringPieces); + for (auto &[priority, stringPiecePair] : hotStringPrioritiesAndStringPieces) + orderedStringPieces.push_back(stringPiecePair); + + // TODO: Order cold set for compression + + orderedStringPieces.insert(orderedStringPieces.end(), + coldStringPieces.begin(), coldStringPieces.end()); + + return orderedStringPieces; +} diff --git a/lld/MachO/SectionPriorities.h b/lld/MachO/SectionPriorities.h index 44fb101990c51..cc4e30fffc600 100644 --- a/lld/MachO/SectionPriorities.h +++ b/lld/MachO/SectionPriorities.h @@ -16,6 +16,7 @@ namespace lld::macho { using SectionPair = std::pair; +using StringPiecePair = std::pair; class PriorityBuilder { public: @@ -28,17 +29,28 @@ class PriorityBuilder { // // An order file has one entry per line, in the following format: // - // :: + // ::[ | CStringEntryPrefix ] // - // and are optional. If not specified, then that entry - // matches any symbol of that name. Parsing this format is not quite - // straightforward because the symbol name itself can contain colons, so when - // encountering a colon, we consider the preceding characters to decide if it - // can be a valid CPU type or file path. + // and are optional. + // If not specified, then that entry tries to match either, // + // 1) any symbol of the ; + // Parsing this format is not quite straightforward because the symbol name + // itself can contain colons, so when encountering a colon, we consider the + // preceding characters to decide if it can be a valid CPU type or file path. // If a symbol is matched by multiple entries, then it takes the // lowest-ordered entry (the one nearest to the front of the list.) // + // or 2) any cstring literal with the given hash, if the entry has the + // CStringEntryPrefix prefix defined below in the file. is the + // hash of cstring literal content. + // + // Cstring literals are not symbolized, we can't identify them by name + // However, cstrings are deduplicated, hence unique, so we use the hash of + // the content of cstring literals to identify them and assign priority to it. + // We use the same hash as used in StringPiece, i.e. 31 bit: + // xxh3_64bits(string) & 0x7fffffff + // // The file can also have line comments that start with '#'. void parseOrderFile(StringRef path); @@ -54,6 +66,8 @@ class PriorityBuilder { // Each section gets assigned the priority of the highest-priority symbol it // contains. llvm::DenseMap buildInputSectionPriorities(); + std::vector + buildCStringPriorities(ArrayRef); private: // The symbol with the smallest priority should be ordered first in the output @@ -65,8 +79,11 @@ class PriorityBuilder { // The priority given to a matching symbol from a particular object file. llvm::DenseMap objectFiles; }; + const llvm::StringRef CStringEntryPrefix = "CSTR;"; std::optional getSymbolPriority(const Defined *sym); + std::optional getSymbolOrCStringPriority(const StringRef key, + InputFile *f); llvm::DenseMap priorities; llvm::MapVector callGraphProfile; }; diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 2113aa88d97a0..0b7f233042487 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -14,6 +14,7 @@ #include "InputFiles.h" #include "ObjC.h" #include "OutputSegment.h" +#include "SectionPriorities.h" #include "SymbolTable.h" #include "Symbols.h" @@ -1760,26 +1761,25 @@ void DeduplicatedCStringSection::finalizeContents() { } } - // Assign an offset for each string and save it to the corresponding + // Sort the strings for performance and compression size win, and then + // assign an offset for each string and save it to the corresponding // StringPieces for easy access. - for (CStringInputSection *isec : inputs) { - for (const auto &[i, piece] : llvm::enumerate(isec->pieces)) { - if (!piece.live) - continue; - auto s = isec->getCachedHashStringRef(i); - auto it = stringOffsetMap.find(s); - assert(it != stringOffsetMap.end()); - StringOffset &offsetInfo = it->second; - if (offsetInfo.outSecOff == UINT64_MAX) { - offsetInfo.outSecOff = - alignToPowerOf2(size, 1ULL << offsetInfo.trailingZeros); - size = - offsetInfo.outSecOff + s.size() + 1; // account for null terminator - } - piece.outSecOff = offsetInfo.outSecOff; + for (auto &[isec, i] : priorityBuilder.buildCStringPriorities(inputs)) { + auto &piece = isec->pieces[i]; + auto s = isec->getCachedHashStringRef(i); + auto it = stringOffsetMap.find(s); + assert(it != stringOffsetMap.end()); + lld::macho::DeduplicatedCStringSection::StringOffset &offsetInfo = + it->second; + if (offsetInfo.outSecOff == UINT64_MAX) { + offsetInfo.outSecOff = + alignToPowerOf2(size, 1ULL << offsetInfo.trailingZeros); + size = offsetInfo.outSecOff + s.size() + 1; // account for null terminator } - isec->isFinal = true; + piece.outSecOff = offsetInfo.outSecOff; } + for (CStringInputSection *isec : inputs) + isec->isFinal = true; } void DeduplicatedCStringSection::writeTo(uint8_t *buf) const { diff --git a/lld/test/MachO/ordre-file-cstring.s b/lld/test/MachO/ordre-file-cstring.s new file mode 100644 index 0000000000000..3c6d2a377dc38 --- /dev/null +++ b/lld/test/MachO/ordre-file-cstring.s @@ -0,0 +1,228 @@ +# REQUIRES: aarch64 + +# RUN: rm -rf %t; split-file %s %t +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/test.s -o %t/test.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/more-cstrings.s -o %t/more-cstrings.o + +# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-0 %t/test.o %t/more-cstrings.o +# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SYM +# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-0 | FileCheck %s --check-prefix=ORIGIN_SEC + +# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-1 %t/test.o %t/more-cstrings.o -order_file %t/ord-1 +# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-1 | FileCheck %s --check-prefix=ONE_SYM +# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-1 | FileCheck %s --check-prefix=ONE_SEC + + +# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-2 %t/test.o %t/more-cstrings.o -order_file %t/ord-2 +# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-2 | FileCheck %s --check-prefix=TWO_SYM +# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-2 | FileCheck %s --check-prefix=TWO_SEC + +# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-3 %t/test.o %t/more-cstrings.o -order_file %t/ord-3 +# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-3 | FileCheck %s --check-prefix=THREE_SYM +# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-3 | FileCheck %s --check-prefix=THREE_SEC + +# RUN: %lld --deduplicate-strings -arch arm64 -lSystem -e _main -o %t/test-4 %t/test.o %t/more-cstrings.o -order_file %t/ord-4 +# RUN: llvm-nm --numeric-sort --format=just-symbols %t/test-4 | FileCheck %s --check-prefix=FOUR_SYM +# RUN: llvm-objdump --macho --section="__TEXT,__cstring" %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC +# RUN: llvm-readobj --string-dump=__cstring %t/test-4 | FileCheck %s --check-prefix=FOUR_SEC_ESCAPE + + +# We expect: +# 1) Covered cstring symbols are reordered +# 2) the rest of the cstring symbols remain original relative order within the cstring section + +# ORIGIN_SYM: _local_foo1 +# ORIGIN_SYM: _globl_foo2 +# ORIGIN_SYM: _local_foo2 +# ORIGIN_SYM: _bar +# ORIGIN_SYM: _baz +# ORIGIN_SYM: _baz_dup +# ORIGIN_SYM: _bar2 +# ORIGIN_SYM: _globl_foo3 + +# ORIGIN_SEC: foo1 +# ORIGIN_SEC: foo2 +# ORIGIN_SEC: bar +# ORIGIN_SEC: baz +# ORIGIN_SEC: bar2 +# ORIGIN_SEC: foo3 + +# original order, but only parital covered +#--- ord-1 +#foo2 +CSTR;1433942677 +#bar +CSTR;540201826 +#bar2 +CSTR;1496286555 +#foo3 +CSTR;1343999025 + +# ONE_SYM: _globl_foo2 +# ONE_SYM: _local_foo2 +# ONE_SYM: _bar +# ONE_SYM: _bar2 +# ONE_SYM: _globl_foo3 +# ONE_SYM: _local_foo1 +# ONE_SYM: _baz +# ONE_SYM: _baz_dup + +# ONE_SEC: foo2 +# ONE_SEC: bar +# ONE_SEC: bar2 +# ONE_SEC: foo3 +# ONE_SEC: foo1 +# ONE_SEC: baz + + +# TWO_SYM: _globl_foo2 +# TWO_SYM: _local_foo2 +# TWO_SYM: _local_foo1 +# TWO_SYM: _baz +# TWO_SYM: _baz_dup +# TWO_SYM: _bar +# TWO_SYM: _bar2 +# TWO_SYM: _globl_foo3 + +# TWO_SEC: foo2 +# TWO_SEC: foo1 +# TWO_SEC: baz +# TWO_SEC: bar +# TWO_SEC: bar2 +# TWO_SEC: foo3 + + +# THREE_SYM: _local_foo1 +# THREE_SYM: _baz +# THREE_SYM: _baz_dup +# THREE_SYM: _bar +# THREE_SYM: _bar2 +# THREE_SYM: _globl_foo2 +# THREE_SYM: _local_foo2 +# THREE_SYM: _globl_foo3 + +# THREE_SEC: foo1 +# THREE_SEC: baz +# THREE_SEC: bar +# THREE_SEC: bar2 +# THREE_SEC: foo2 +# THREE_SEC: foo3 + + +# FOUR_SYM: _local_escape_white_space +# FOUR_SYM: _globl_foo2 +# FOUR_SYM: _local_foo2 +# FOUR_SYM: _local_escape +# FOUR_SYM: _globl_foo3 +# FOUR_SYM: _bar +# FOUR_SYM: _local_foo1 +# FOUR_SYM: _baz +# FOUR_SYM: _baz_dup +# FOUR_SYM: _bar2 + +# FOUR_SEC: \t\n +# FOUR_SEC: foo2 +# FOUR_SEC: @\"NSDictionary\" +# FOUR_SEC: foo3 +# FOUR_SEC: bar +# FOUR_SEC: foo1 +# FOUR_SEC: baz +# FOUR_SEC: bar2 + +# FOUR_SEC_ESCAPE: .. +# FOUR_SEC_ESCAPE: foo2 +# FOUR_SEC_ESCAPE: @"NSDictionary" +# FOUR_SEC_ESCAPE: foo3 +# FOUR_SEC_ESCAPE: bar +# FOUR_SEC_ESCAPE: foo1 +# FOUR_SEC_ESCAPE: baz +# FOUR_SEC_ESCAPE: bar2 + + +# change order, parital covered +#--- ord-2 +#foo2 +CSTR;1433942677 +#foo1 +CSTR;1663475769 +#baz +CSTR;862947621 +#bar +CSTR;540201826 +#bar2 +CSTR;1496286555 + +# change order, parital covered, with mismatches, duplicates +#--- ord-3 +foo2222 +CSTR;0x11111111 +#bar (mismatched cpu and file name) +fakeCPU:fake-file-name.o:CSTR;540201826 +#not a hash +CSTR;xxx +#foo1 +CSTR;1663475769 +#baz +CSTR;862947621 +#bar +CSTR;540201826 +#bar2 +CSTR;1496286555 +#baz +CSTR;862947621 + +# test escape strings +#--- ord-4 +#\t\n +CSTR;1035903177 +#foo2 +CSTR;1433942677 +#@\"NSDictionary\" +CSTR;1202669430 +#foo3 +CSTR;1343999025 +#bar +CSTR;540201826 + + +#--- test.s +.text +.globl _main + +_main: + ret + +.cstring +.p2align 2 +_local_foo1: + .asciz "foo1" +_local_foo2: + .asciz "foo2" +L_.foo1_dup: + .asciz "foo1" +L_.foo2_dup: + .asciz "foo2" +_local_escape: + .asciz "@\"NSDictionary\"" +_local_escape_white_space: + .asciz "\t\n" + +_bar: + .asciz "bar" +_baz: + .asciz "baz" +_bar2: + .asciz "bar2" +_baz_dup: + .asciz "baz" + +.subsections_via_symbols + +#--- more-cstrings.s +.globl _globl_foo1, _globl_foo3 +.cstring +.p2align 4 +_globl_foo3: + .asciz "foo3" +_globl_foo2: + .asciz "foo2"