diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h index 7e5d0f1f91852..a8655dc6f14e3 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/ELF_loongarch.h @@ -33,6 +33,10 @@ Expected> createLinkGraphFromELFObject_loongarch( void link_ELF_loongarch(std::unique_ptr G, std::unique_ptr Ctx); +/// Returns a pass that performs linker relaxation. Should be added to +/// PostAllocationPasses. +LinkGraphPassFunction createRelaxationPass_ELF_loongarch(); + } // end namespace jitlink } // end namespace llvm diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h b/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h index 1db4b82218109..d6025edf7d110 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/loongarch.h @@ -225,6 +225,13 @@ enum EdgeKind_loongarch : Edge::Kind { /// out-of-range error will be returned. /// Call36PCRel, + + /// Alignment requirement used by linker relaxation. + /// + /// Linker relaxation will use this to ensure all code sequences are properly + /// aligned and then remove these edges from the graph. + /// + AlignRelaxable, }; /// Returns a string name for the given loongarch edge. For debugging purposes @@ -362,6 +369,9 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) { *(little32_t *)(FixupPtr + 4) = Jirl | Lo16; break; } + case AlignRelaxable: + // Ignore when the relaxation pass did not run + break; default: return make_error( "In graph " + G.getName() + ", section " + B.getSection().getName() + diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp index a12e9f33e80a6..3f0a7b645e83f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp @@ -45,6 +45,238 @@ class ELFJITLinker_loongarch : public JITLinker { } }; +namespace { + +struct SymbolAnchor { + uint64_t Offset; + Symbol *Sym; + bool End; // true for the anchor of getOffset() + getSize() +}; + +struct BlockRelaxAux { + // This records symbol start and end offsets which will be adjusted according + // to the nearest RelocDeltas element. + SmallVector Anchors; + // All edges that either 1) are R_LARCH_ALIGN or 2) have a R_LARCH_RELAX edge + // at the same offset. + SmallVector RelaxEdges; + // For RelaxEdges[I], the actual offset is RelaxEdges[I]->getOffset() - (I ? + // RelocDeltas[I - 1] : 0). + SmallVector RelocDeltas; + // For RelaxEdges[I], the actual type is EdgeKinds[I]. + SmallVector EdgeKinds; + // List of rewritten instructions. Contains one raw encoded instruction per + // element in EdgeKinds that isn't Invalid or R_LARCH_ALIGN. + SmallVector Writes; +}; + +struct RelaxAux { + DenseMap Blocks; +}; + +} // namespace + +static bool shouldRelax(const Section &S) { + return (S.getMemProt() & orc::MemProt::Exec) != orc::MemProt::None; +} + +static bool isRelaxable(const Edge &E) { + switch (E.getKind()) { + default: + return false; + case AlignRelaxable: + return true; + } +} + +static RelaxAux initRelaxAux(LinkGraph &G) { + RelaxAux Aux; + for (auto &S : G.sections()) { + if (!shouldRelax(S)) + continue; + for (auto *B : S.blocks()) { + auto BlockEmplaceResult = Aux.Blocks.try_emplace(B); + assert(BlockEmplaceResult.second && "Block encountered twice"); + auto &BlockAux = BlockEmplaceResult.first->second; + + for (auto &E : B->edges()) + if (isRelaxable(E)) + BlockAux.RelaxEdges.push_back(&E); + + if (BlockAux.RelaxEdges.empty()) { + Aux.Blocks.erase(BlockEmplaceResult.first); + continue; + } + + const auto NumEdges = BlockAux.RelaxEdges.size(); + BlockAux.RelocDeltas.resize(NumEdges, 0); + BlockAux.EdgeKinds.resize_for_overwrite(NumEdges); + + // Store anchors (offset and offset+size) for symbols. + for (auto *Sym : S.symbols()) { + if (!Sym->isDefined() || &Sym->getBlock() != B) + continue; + + BlockAux.Anchors.push_back({Sym->getOffset(), Sym, false}); + BlockAux.Anchors.push_back( + {Sym->getOffset() + Sym->getSize(), Sym, true}); + } + } + } + + // Sort anchors by offset so that we can find the closest relocation + // efficiently. For a zero size symbol, ensure that its start anchor precedes + // its end anchor. For two symbols with anchors at the same offset, their + // order does not matter. + for (auto &BlockAuxIter : Aux.Blocks) { + llvm::sort(BlockAuxIter.second.Anchors, [](auto &A, auto &B) { + return std::make_pair(A.Offset, A.End) < std::make_pair(B.Offset, B.End); + }); + } + + return Aux; +} + +static void relaxAlign(orc::ExecutorAddr Loc, const Edge &E, uint32_t &Remove, + Edge::Kind &NewEdgeKind) { + const uint64_t Addend = + !E.getTarget().isDefined() ? Log2_64(E.getAddend()) + 1 : E.getAddend(); + const uint64_t AllBytes = (1ULL << (Addend & 0xff)) - 4; + const uint64_t Align = 1ULL << (Addend & 0xff); + const uint64_t MaxBytes = Addend >> 8; + const uint64_t Off = Loc.getValue() & (Align - 1); + const uint64_t CurBytes = Off == 0 ? 0 : Align - Off; + // All bytes beyond the alignment boundary should be removed. + // If emit bytes more than max bytes to emit, remove all. + if (MaxBytes != 0 && CurBytes > MaxBytes) + Remove = AllBytes; + else + Remove = AllBytes - CurBytes; + + assert(static_cast(Remove) >= 0 && + "R_LARCH_ALIGN needs expanding the content"); + NewEdgeKind = AlignRelaxable; +} + +static bool relaxBlock(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) { + const auto BlockAddr = Block.getAddress(); + bool Changed = false; + ArrayRef SA = ArrayRef(Aux.Anchors); + uint32_t Delta = 0; + + Aux.EdgeKinds.assign(Aux.EdgeKinds.size(), Edge::Invalid); + Aux.Writes.clear(); + + for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { + const auto Loc = BlockAddr + E->getOffset() - Delta; + auto &Cur = Aux.RelocDeltas[I]; + uint32_t Remove = 0; + switch (E->getKind()) { + case AlignRelaxable: + relaxAlign(Loc, *E, Remove, Aux.EdgeKinds[I]); + break; + default: + llvm_unreachable("Unexpected relaxable edge kind"); + } + + // For all anchors whose offsets are <= E->getOffset(), they are preceded by + // the previous relocation whose RelocDeltas value equals Delta. + // Decrease their offset and update their size. + for (; SA.size() && SA[0].Offset <= E->getOffset(); SA = SA.slice(1)) { + if (SA[0].End) + SA[0].Sym->setSize(SA[0].Offset - Delta - SA[0].Sym->getOffset()); + else + SA[0].Sym->setOffset(SA[0].Offset - Delta); + } + + Delta += Remove; + if (Delta != Cur) { + Cur = Delta; + Changed = true; + } + } + + for (const SymbolAnchor &A : SA) { + if (A.End) + A.Sym->setSize(A.Offset - Delta - A.Sym->getOffset()); + else + A.Sym->setOffset(A.Offset - Delta); + } + + return Changed; +} + +static bool relaxOnce(LinkGraph &G, RelaxAux &Aux) { + bool Changed = false; + + for (auto &[B, BlockAux] : Aux.Blocks) + Changed |= relaxBlock(G, *B, BlockAux); + + return Changed; +} + +static void finalizeBlockRelax(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) { + auto Contents = Block.getAlreadyMutableContent(); + auto *Dest = Contents.data(); + uint32_t Offset = 0; + uint32_t Delta = 0; + + // Update section content: remove NOPs for R_LARCH_ALIGN and rewrite + // instructions for relaxed relocations. + for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { + uint32_t Remove = Aux.RelocDeltas[I] - Delta; + Delta = Aux.RelocDeltas[I]; + if (Remove == 0 && Aux.EdgeKinds[I] == Edge::Invalid) + continue; + + // Copy from last location to the current relocated location. + const auto Size = E->getOffset() - Offset; + std::memmove(Dest, Contents.data() + Offset, Size); + Dest += Size; + Offset = E->getOffset() + Remove; + } + + std::memmove(Dest, Contents.data() + Offset, Contents.size() - Offset); + + // Fixup edge offsets and kinds. + Delta = 0; + size_t I = 0; + for (auto &E : Block.edges()) { + E.setOffset(E.getOffset() - Delta); + + if (I < Aux.RelaxEdges.size() && Aux.RelaxEdges[I] == &E) { + if (Aux.EdgeKinds[I] != Edge::Invalid) + E.setKind(Aux.EdgeKinds[I]); + + Delta = Aux.RelocDeltas[I]; + ++I; + } + } + + // Remove AlignRelaxable edges: all other relaxable edges got modified and + // will be used later while linking. Alignment is entirely handled here so we + // don't need these edges anymore. + for (auto IE = Block.edges().begin(); IE != Block.edges().end();) { + if (IE->getKind() == AlignRelaxable) + IE = Block.removeEdge(IE); + else + ++IE; + } +} + +static void finalizeRelax(LinkGraph &G, RelaxAux &Aux) { + for (auto &[B, BlockAux] : Aux.Blocks) + finalizeBlockRelax(G, *B, BlockAux); +} + +static Error relax(LinkGraph &G) { + auto Aux = initRelaxAux(G); + while (relaxOnce(G, Aux)) { + } + finalizeRelax(G, Aux); + return Error::success(); +} + template class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder { private: @@ -74,6 +306,8 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder { return RequestGOTAndTransformToPageOffset12; case ELF::R_LARCH_CALL36: return Call36PCRel; + case ELF::R_LARCH_ALIGN: + return AlignRelaxable; } return make_error( @@ -81,6 +315,11 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder { object::getELFRelocationTypeName(ELF::EM_LOONGARCH, Type)); } + EdgeKind_loongarch getRelaxableRelocationKind(EdgeKind_loongarch Kind) { + // TODO: Implement more. Just ignore all relaxations now. + return Kind; + } + Error addRelocations() override { LLVM_DEBUG(dbgs() << "Processing relocations:\n"); @@ -99,6 +338,25 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder { Block &BlockToFix) { using Base = ELFLinkGraphBuilder; + uint32_t Type = Rel.getType(false); + int64_t Addend = Rel.r_addend; + + if (Type == ELF::R_LARCH_RELAX) { + if (BlockToFix.edges_empty()) + return make_error( + "R_LARCH_RELAX without preceding relocation", + inconvertibleErrorCode()); + + auto &PrevEdge = *std::prev(BlockToFix.edges().end()); + auto Kind = static_cast(PrevEdge.getKind()); + PrevEdge.setKind(getRelaxableRelocationKind(Kind)); + return Error::success(); + } + + Expected Kind = getRelocationKind(Type); + if (!Kind) + return Kind.takeError(); + uint32_t SymbolIndex = Rel.getSymbol(false); auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); if (!ObjSymbol) @@ -113,12 +371,6 @@ class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder { Base::GraphSymbols.size()), inconvertibleErrorCode()); - uint32_t Type = Rel.getType(false); - Expected Kind = getRelocationKind(Type); - if (!Kind) - return Kind.takeError(); - - int64_t Addend = Rel.r_addend; auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); Edge GE(*Kind, Offset, *GraphSymbol, Addend); @@ -209,6 +461,9 @@ void link_ELF_loongarch(std::unique_ptr G, // Add an in-place GOT/PLTStubs build pass. Config.PostPrunePasses.push_back(buildTables_ELF_loongarch); + + // Add a linker relaxation pass. + Config.PostAllocationPasses.push_back(relax); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) @@ -217,5 +472,7 @@ void link_ELF_loongarch(std::unique_ptr G, ELFJITLinker_loongarch::link(std::move(Ctx), std::move(G), std::move(Config)); } +LinkGraphPassFunction createRelaxationPass_ELF_loongarch() { return relax; } + } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp index cdb3da04354ee..a5579b074de7c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp @@ -52,6 +52,7 @@ const char *getEdgeKindName(Edge::Kind K) { KIND_NAME_CASE(RequestGOTAndTransformToPage20) KIND_NAME_CASE(RequestGOTAndTransformToPageOffset12) KIND_NAME_CASE(Call36PCRel) + KIND_NAME_CASE(AlignRelaxable) default: return getGenericEdgeKindName(K); } diff --git a/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s b/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s new file mode 100644 index 0000000000000..ec1aceb25b614 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/LoongArch/ELF_relax_align.s @@ -0,0 +1,95 @@ +# RUN: rm -rf %t && mkdir %t && cd %t + +# RUN: llvm-mc --filetype=obj --triple=loongarch32 -mattr=+relax %s -o %t.la32 +# RUN: llvm-jitlink --noexec \ +# RUN: -slab-allocate 100Kb -slab-address 0x0 -slab-page-size 16384 \ +# RUN: --check %s %t.la32 + +# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t.la64 +# RUN: llvm-jitlink --noexec \ +# RUN: -slab-allocate 100Kb -slab-address 0x0 -slab-page-size 16384 \ +# RUN: --check %s %t.la64 + +## Test that we can handle R_LARCH_ALIGN. + + .text + + .globl main,align4,align8,align16,align32,alignmax12,alignmax8 + .type main,@function +main: + bl f + .align 2 +align4: + bl f + .size align4, .-align4 + .align 3 +align8: + bl f + .size align8, .-align8 + .align 4 +align16: + bl f + .size align16, .-align16 + .align 5 +align32: + bl f + .size align32, .-align32 + .align 4,,12 +alignmax12: + bl f + .size alignmax12, .-alignmax12 + .align 4,,8 +alignmax8: + bl f + .size alignmax8, .-alignmax8 + .size main, .-main + + .globl f +f: + ret + .size f, .-f + +# jitlink-check: main = 0x0 +# jitlink-check: align4 = 0x4 +# jitlink-check: align8 = 0x8 +# jitlink-check: align16 = 0x10 +# jitlink-check: align32 = 0x20 +# jitlink-check: alignmax12 = 0x30 +## 3 nops (12 bytes) should be inserted to satisfy alignment. +## But the max bytes we can insert is 8. So alignment is ignored. +# jitlink-check: alignmax8 = 0x34 + +## main: bl f +# jitlink-check: (*{4}(main))[31:26] = 0x15 +# jitlink-check: decode_operand(main, 0)[27:0] = (f - main)[27:0] + +## align 4: bl f +# jitlink-check: (*{4}(align4))[31:26] = 0x15 +# jitlink-check: decode_operand(align4, 0)[27:0] = (f - align4)[27:0] + +## align8: bl f; nop +# jitlink-check: (*{4}(align8))[31:26] = 0x15 +# jitlink-check: decode_operand(align8, 0)[27:0] = (f - align8)[27:0] +# jitlink-check: (*{4}(align8+4)) = 0x3400000 + +## align16: bl f; nop; nop; nop +# jitlink-check: (*{4}(align16))[31:26] = 0x15 +# jitlink-check: decode_operand(align16, 0)[27:0] = (f - align16)[27:0] +# jitlink-check: (*{4}(align16+4)) = 0x3400000 +# jitlink-check: (*{4}(align16+8)) = 0x3400000 +# jitlink-check: (*{4}(align16+12)) = 0x3400000 + +## align32: bl f; nop; nop; nop +# jitlink-check: (*{4}(align32))[31:26] = 0x15 +# jitlink-check: decode_operand(align32, 0)[27:0] = (f - align32)[27:0] +# jitlink-check: (*{4}(align32+4)) = 0x3400000 +# jitlink-check: (*{4}(align32+8)) = 0x3400000 +# jitlink-check: (*{4}(align32+12)) = 0x3400000 + +## alignmax12: bl f +# jitlink-check: (*{4}(alignmax12))[31:26] = 0x15 +# jitlink-check: decode_operand(alignmax12, 0)[27:0] = (f - alignmax12)[27:0] + +## alignmax8: bl f +# jitlink-check: (*{4}(alignmax8))[31:26] = 0x15 +# jitlink-check: decode_operand(alignmax8, 0)[27:0] = (f - alignmax8)[27:0]