Skip to content

Commit 8d3a985

Browse files
mingmingl-llvmmingmingl-llvm
committed
Emit jump table with section suffix
1 parent dd74827 commit 8d3a985

File tree

8 files changed

+159
-61
lines changed

8 files changed

+159
-61
lines changed

‎llvm/include/llvm/CodeGen/AsmPrinter.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,10 @@ class AsmPrinter : public MachineFunctionPass {
453453
/// function to the current output stream.
454454
virtual void emitJumpTableInfo();
455455

456+
virtual void emitJumpTables(ArrayRef<unsigned> JumpTableIndices,
457+
MCSection *JumpTableSection, bool JTInDiffSection,
458+
const MachineJumpTableInfo &MJTI);
459+
456460
/// Emit the specified global variable to the .s file.
457461
virtual void emitGlobalVariable(const GlobalVariable *GV);
458462

@@ -892,10 +896,10 @@ class AsmPrinter : public MachineFunctionPass {
892896
// Internal Implementation Details
893897
//===------------------------------------------------------------------===//
894898

895-
void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
899+
void emitJumpTableEntry(const MachineJumpTableInfo &MJTI,
896900
const MachineBasicBlock *MBB, unsigned uid) const;
897901

898-
void emitJumpTableSizesSection(const MachineJumpTableInfo *MJTI,
902+
void emitJumpTableSizesSection(const MachineJumpTableInfo &MJTI,
899903
const Function &F) const;
900904

901905
void emitLLVMUsedList(const ConstantArray *InitList);

‎llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
7474

7575
MCSection *getSectionForJumpTable(const Function &F,
7676
const TargetMachine &TM) const override;
77+
MCSection *
78+
getSectionForJumpTable(const Function &F, const TargetMachine &TM,
79+
const MachineJumpTableEntry *JTE) const override;
7780
MCSection *getSectionForLSDA(const Function &F, const MCSymbol &FnSym,
7881
const TargetMachine &TM) const override;
7982

‎llvm/include/llvm/Target/TargetLoweringObjectFile.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class Function;
2727
class GlobalObject;
2828
class GlobalValue;
2929
class MachineBasicBlock;
30+
class MachineJumpTableEntry;
3031
class MachineModuleInfo;
3132
class Mangler;
3233
class MCContext;
@@ -132,6 +133,10 @@ class TargetLoweringObjectFile : public MCObjectFileInfo {
132133

133134
virtual MCSection *getSectionForJumpTable(const Function &F,
134135
const TargetMachine &TM) const;
136+
virtual MCSection *
137+
getSectionForJumpTable(const Function &F, const TargetMachine &TM,
138+
const MachineJumpTableEntry *JTE) const;
139+
135140
virtual MCSection *getSectionForLSDA(const Function &, const MCSymbol &,
136141
const TargetMachine &) const {
137142
return LSDASection;

‎llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp‎

Lines changed: 82 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,11 @@ static cl::opt<bool> BBAddrMapSkipEmitBBEntries(
168168
"unnecessary for some PGOAnalysisMap features."),
169169
cl::Hidden, cl::init(false));
170170

171+
static cl::opt<bool>
172+
EmitStaticDataHotnessSuffix("emit-static-data-hotness-suffix", cl::Hidden,
173+
cl::init(false), cl::ZeroOrMore,
174+
cl::desc("Emit static data hotness suffix"));
175+
171176
static cl::opt<bool> EmitJumpTableSizesSection(
172177
"emit-jump-table-sizes-section",
173178
cl::desc("Emit a section containing jump table addresses and sizes"),
@@ -2861,7 +2866,6 @@ void AsmPrinter::emitConstantPool() {
28612866
// Print assembly representations of the jump tables used by the current
28622867
// function.
28632868
void AsmPrinter::emitJumpTableInfo() {
2864-
const DataLayout &DL = MF->getDataLayout();
28652869
const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
28662870
if (!MJTI) return;
28672871
if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) return;
@@ -2876,42 +2880,94 @@ void AsmPrinter::emitJumpTableInfo() {
28762880
MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 ||
28772881
MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference64,
28782882
F);
2883+
2884+
std::vector<unsigned> JumpTableIndices;
2885+
if (!EmitStaticDataHotnessSuffix) {
2886+
for (unsigned JTI = 0, JTSize = JT.size(); JTI < JTSize; ++JTI)
2887+
JumpTableIndices.push_back(JTI);
2888+
emitJumpTables(JumpTableIndices, TLOF.getSectionForJumpTable(F, TM),
2889+
JTInDiffSection, *MJTI);
2890+
return;
2891+
}
2892+
2893+
// Iterate all jump tables, put hot jump table indices towards the beginning
2894+
// of the vector, and cold jump table indices towards the end.
2895+
int NextHotJumpTableIndex = 0, NextColdJumpTableIndex = JT.size() - 1;
2896+
JumpTableIndices.resize(JT.size());
2897+
for (unsigned JTI = 0, JTSize = JT.size(); JTI < JTSize; ++JTI) {
2898+
if (JT[JTI].Hotness == MachineFunctionDataHotness::Cold)
2899+
JumpTableIndices[NextColdJumpTableIndex--] = JTI;
2900+
else
2901+
JumpTableIndices[NextHotJumpTableIndex++] = JTI;
2902+
}
2903+
2904+
if (NextHotJumpTableIndex != 0) {
2905+
emitJumpTables(
2906+
ArrayRef<unsigned>(JumpTableIndices).take_front(NextHotJumpTableIndex),
2907+
TLOF.getSectionForJumpTable(F, TM, &JT[0]), JTInDiffSection, *MJTI);
2908+
}
2909+
2910+
if (NextHotJumpTableIndex != (int)JT.size()) {
2911+
// Retain the relative orders of original jump tables.
2912+
for (int L = NextHotJumpTableIndex, R = JT.size() - 1; L < R; ++L, --R)
2913+
std::swap(JumpTableIndices[L], JumpTableIndices[R]);
2914+
2915+
emitJumpTables(
2916+
ArrayRef<unsigned>(JumpTableIndices)
2917+
.take_back(JT.size() - NextHotJumpTableIndex),
2918+
TLOF.getSectionForJumpTable(F, TM, &JT[JumpTableIndices[NextHotJumpTableIndex]]),
2919+
JTInDiffSection, *MJTI);
2920+
}
2921+
2922+
return;
2923+
}
2924+
2925+
void AsmPrinter::emitJumpTables(ArrayRef<unsigned> JumpTableIndices,
2926+
MCSection *JumpTableSection,
2927+
bool JTInDiffSection,
2928+
const MachineJumpTableInfo &MJTI) {
2929+
if (JumpTableIndices.empty())
2930+
return;
2931+
2932+
const DataLayout &DL = MF->getDataLayout();
28792933
if (JTInDiffSection) {
2880-
// Drop it in the readonly section.
2881-
MCSection *ReadOnlySection = TLOF.getSectionForJumpTable(F, TM);
2882-
OutStreamer->switchSection(ReadOnlySection);
2934+
OutStreamer->switchSection(JumpTableSection);
28832935
}
28842936

2885-
emitAlignment(Align(MJTI->getEntryAlignment(DL)));
2937+
emitAlignment(Align(MJTI.getEntryAlignment(MF->getDataLayout())));
28862938

28872939
// Jump tables in code sections are marked with a data_region directive
28882940
// where that's supported.
28892941
if (!JTInDiffSection)
28902942
OutStreamer->emitDataRegion(MCDR_DataRegionJT32);
28912943

2892-
for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
2893-
const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
2944+
const auto &JT = MJTI.getJumpTables();
2945+
for (unsigned Index = 0, e = JumpTableIndices.size(); Index != e; ++Index) {
2946+
const std::vector<MachineBasicBlock *> &JTBBs =
2947+
JT[JumpTableIndices[Index]].MBBs;
28942948

28952949
// If this jump table was deleted, ignore it.
2896-
if (JTBBs.empty()) continue;
2950+
if (JTBBs.empty())
2951+
continue;
28972952

28982953
// For the EK_LabelDifference32 entry, if using .set avoids a relocation,
28992954
/// emit a .set directive for each unique entry.
2900-
if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
2955+
if (MJTI.getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
29012956
MAI->doesSetDirectiveSuppressReloc()) {
2902-
SmallPtrSet<const MachineBasicBlock*, 16> EmittedSets;
2957+
SmallPtrSet<const MachineBasicBlock *, 16> EmittedSets;
29032958
const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
2904-
const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF,JTI,OutContext);
2959+
const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(
2960+
MF, JumpTableIndices[Index], OutContext);
29052961
for (const MachineBasicBlock *MBB : JTBBs) {
29062962
if (!EmittedSets.insert(MBB).second)
29072963
continue;
29082964

29092965
// .set LJTSet, LBB32-base
29102966
const MCExpr *LHS =
2911-
MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
2912-
OutStreamer->emitAssignment(GetJTSetSymbol(JTI, MBB->getNumber()),
2913-
MCBinaryExpr::createSub(LHS, Base,
2914-
OutContext));
2967+
MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
2968+
OutStreamer->emitAssignment(
2969+
GetJTSetSymbol(JumpTableIndices[Index], MBB->getNumber()),
2970+
MCBinaryExpr::createSub(LHS, Base, OutContext));
29152971
}
29162972
}
29172973

@@ -2923,27 +2979,27 @@ void AsmPrinter::emitJumpTableInfo() {
29232979
// FIXME: This doesn't have to have any specific name, just any randomly
29242980
// named and numbered local label started with 'l' would work. Simplify
29252981
// GetJTISymbol.
2926-
OutStreamer->emitLabel(GetJTISymbol(JTI, true));
2982+
OutStreamer->emitLabel(GetJTISymbol(JumpTableIndices[Index], true));
29272983

2928-
MCSymbol* JTISymbol = GetJTISymbol(JTI);
2984+
MCSymbol *JTISymbol = GetJTISymbol(JumpTableIndices[Index]);
29292985
OutStreamer->emitLabel(JTISymbol);
29302986

29312987
// Defer MCAssembler based constant folding due to a performance issue. The
29322988
// label differences will be evaluated at write time.
29332989
for (const MachineBasicBlock *MBB : JTBBs)
2934-
emitJumpTableEntry(MJTI, MBB, JTI);
2990+
emitJumpTableEntry(MJTI, MBB, JumpTableIndices[Index]);
29352991
}
29362992

29372993
if (EmitJumpTableSizesSection)
2938-
emitJumpTableSizesSection(MJTI, F);
2994+
emitJumpTableSizesSection(MJTI, MF->getFunction());
29392995

29402996
if (!JTInDiffSection)
29412997
OutStreamer->emitDataRegion(MCDR_DataRegionEnd);
29422998
}
29432999

2944-
void AsmPrinter::emitJumpTableSizesSection(const MachineJumpTableInfo *MJTI,
3000+
void AsmPrinter::emitJumpTableSizesSection(const MachineJumpTableInfo &MJTI,
29453001
const Function &F) const {
2946-
const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
3002+
const std::vector<MachineJumpTableEntry> &JT = MJTI.getJumpTables();
29473003

29483004
if (JT.empty())
29493005
return;
@@ -2991,17 +3047,17 @@ void AsmPrinter::emitJumpTableSizesSection(const MachineJumpTableInfo *MJTI,
29913047

29923048
/// EmitJumpTableEntry - Emit a jump table entry for the specified MBB to the
29933049
/// current stream.
2994-
void AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
3050+
void AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo &MJTI,
29953051
const MachineBasicBlock *MBB,
29963052
unsigned UID) const {
29973053
assert(MBB && MBB->getNumber() >= 0 && "Invalid basic block");
29983054
const MCExpr *Value = nullptr;
2999-
switch (MJTI->getEntryKind()) {
3055+
switch (MJTI.getEntryKind()) {
30003056
case MachineJumpTableInfo::EK_Inline:
30013057
llvm_unreachable("Cannot emit EK_Inline jump table entry");
30023058
case MachineJumpTableInfo::EK_Custom32:
30033059
Value = MF->getSubtarget().getTargetLowering()->LowerCustomJumpTableEntry(
3004-
MJTI, MBB, UID, OutContext);
3060+
&MJTI, MBB, UID, OutContext);
30053061
break;
30063062
case MachineJumpTableInfo::EK_BlockAddress:
30073063
// EK_BlockAddress - Each entry is a plain address of block, e.g.:
@@ -3035,7 +3091,7 @@ void AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
30353091
// If the .set directive avoids relocations, this is emitted as:
30363092
// .set L4_5_set_123, LBB123 - LJTI1_2
30373093
// .word L4_5_set_123
3038-
if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
3094+
if (MJTI.getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 &&
30393095
MAI->doesSetDirectiveSuppressReloc()) {
30403096
Value = MCSymbolRefExpr::create(GetJTSetSymbol(UID, MBB->getNumber()),
30413097
OutContext);
@@ -3051,7 +3107,7 @@ void AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
30513107

30523108
assert(Value && "Unknown entry kind!");
30533109

3054-
unsigned EntrySize = MJTI->getEntrySize(getDataLayout());
3110+
unsigned EntrySize = MJTI.getEntrySize(getDataLayout());
30553111
OutStreamer->emitValue(Value, EntrySize);
30563112
}
30573113

‎llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
2525
#include "llvm/CodeGen/MachineBasicBlock.h"
2626
#include "llvm/CodeGen/MachineFunction.h"
27+
#include "llvm/CodeGen/MachineJumpTableInfo.h"
2728
#include "llvm/CodeGen/MachineModuleInfo.h"
2829
#include "llvm/CodeGen/MachineModuleInfoImpls.h"
2930
#include "llvm/IR/Comdat.h"
@@ -642,9 +643,11 @@ static StringRef getSectionPrefixForGlobal(SectionKind Kind, bool IsLarge) {
642643
static SmallString<128>
643644
getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
644645
Mangler &Mang, const TargetMachine &TM,
645-
unsigned EntrySize, bool UniqueSectionName) {
646+
unsigned EntrySize, bool UniqueSectionName,
647+
const MachineJumpTableEntry *JTE) {
646648
SmallString<128> Name =
647649
getSectionPrefixForGlobal(Kind, TM.isLargeGlobalValue(GO));
650+
648651
if (Kind.isMergeableCString()) {
649652
// We also need alignment here.
650653
// FIXME: this is getting the alignment of the character, not the
@@ -663,7 +666,12 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
663666

664667
bool HasPrefix = false;
665668
if (const auto *F = dyn_cast<Function>(GO)) {
666-
if (std::optional<StringRef> Prefix = F->getSectionPrefix()) {
669+
// Jump table hotness takes precedence over its enclosing function's hotness
670+
// if both are available.
671+
if (JTE) {
672+
if (JTE->Hotness == MachineFunctionDataHotness::Hot)
673+
raw_svector_ostream(Name) << ".hot";
674+
} else if (std::optional<StringRef> Prefix = F->getSectionPrefix()) {
667675
raw_svector_ostream(Name) << '.' << *Prefix;
668676
HasPrefix = true;
669677
}
@@ -761,8 +769,8 @@ calcUniqueIDUpdateFlagsAndSize(const GlobalObject *GO, StringRef SectionName,
761769
// implicitly for this symbol e.g. .rodata.str1.1, then we don't need
762770
// to unique the section as the entry size for this symbol will be
763771
// compatible with implicitly created sections.
764-
SmallString<128> ImplicitSectionNameStem =
765-
getELFSectionNameForGlobal(GO, Kind, Mang, TM, EntrySize, false);
772+
SmallString<128> ImplicitSectionNameStem = getELFSectionNameForGlobal(
773+
GO, Kind, Mang, TM, EntrySize, false, /*MJTE=*/nullptr);
766774
if (SymbolMergeable &&
767775
Ctx.isELFImplicitMergeableSectionNamePrefix(SectionName) &&
768776
SectionName.starts_with(ImplicitSectionNameStem))
@@ -862,7 +870,8 @@ MCSection *TargetLoweringObjectFileELF::getExplicitSectionGlobal(
862870
static MCSectionELF *selectELFSectionForGlobal(
863871
MCContext &Ctx, const GlobalObject *GO, SectionKind Kind, Mangler &Mang,
864872
const TargetMachine &TM, bool EmitUniqueSection, unsigned Flags,
865-
unsigned *NextUniqueID, const MCSymbolELF *AssociatedSymbol) {
873+
unsigned *NextUniqueID, const MCSymbolELF *AssociatedSymbol,
874+
const MachineJumpTableEntry *MJTE = nullptr) {
866875

867876
auto [Group, IsComdat, ExtraFlags] = getGlobalObjectInfo(GO, TM);
868877
Flags |= ExtraFlags;
@@ -881,7 +890,7 @@ static MCSectionELF *selectELFSectionForGlobal(
881890
}
882891
}
883892
SmallString<128> Name = getELFSectionNameForGlobal(
884-
GO, Kind, Mang, TM, EntrySize, UniqueSectionName);
893+
GO, Kind, Mang, TM, EntrySize, UniqueSectionName, MJTE);
885894

886895
// Use 0 as the unique ID for execute-only text.
887896
if (Kind.isExecuteOnly())
@@ -955,6 +964,12 @@ MCSection *TargetLoweringObjectFileELF::getUniqueSectionForFunction(
955964

956965
MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
957966
const Function &F, const TargetMachine &TM) const {
967+
return getSectionForJumpTable(F, TM, nullptr);
968+
}
969+
970+
MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
971+
const Function &F, const TargetMachine &TM,
972+
const MachineJumpTableEntry *JTE) const {
958973
// If the function can be removed, produce a unique section so that
959974
// the table doesn't prevent the removal.
960975
const Comdat *C = F.getComdat();
@@ -965,7 +980,7 @@ MCSection *TargetLoweringObjectFileELF::getSectionForJumpTable(
965980
return selectELFSectionForGlobal(getContext(), &F, SectionKind::getReadOnly(),
966981
getMangler(), TM, EmitUniqueSection,
967982
ELF::SHF_ALLOC, &NextUniqueID,
968-
/* AssociatedSymbol */ nullptr);
983+
/* AssociatedSymbol */ nullptr, JTE);
969984
}
970985

971986
MCSection *TargetLoweringObjectFileELF::getSectionForLSDA(

‎llvm/lib/CodeGen/TargetPassConfig.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1261,9 +1261,9 @@ void TargetPassConfig::addMachinePasses() {
12611261
"performance.\n";
12621262
}
12631263
}
1264-
addPass(createMachineFunctionSplitterPass());
12651264
if (SplitStaticData)
12661265
addPass(createStaticDataSplitterPass());
1266+
addPass(createMachineFunctionSplitterPass());
12671267
}
12681268
// We run the BasicBlockSections pass if either we need BB sections or BB
12691269
// address map (or both).

‎llvm/lib/Target/TargetLoweringObjectFile.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,12 @@ TargetLoweringObjectFile::SectionForGlobal(const GlobalObject *GO,
348348

349349
MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
350350
const Function &F, const TargetMachine &TM) const {
351+
return getSectionForJumpTable(F, TM, nullptr);
352+
}
353+
354+
MCSection *TargetLoweringObjectFile::getSectionForJumpTable(
355+
const Function &F, const TargetMachine &TM,
356+
const MachineJumpTableEntry *JTE) const {
351357
Align Alignment(1);
352358
return getSectionForConstant(F.getDataLayout(),
353359
SectionKind::getReadOnly(), /*C=*/nullptr,

0 commit comments

Comments
 (0)