Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ class LowerTypeTestsPass : public PassInfoMixin<LowerTypeTestsPass> {
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
};

class SimplifyTypeTestsPass : public PassInfoMixin<SimplifyTypeTestsPass> {
public:
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
};

} // end namespace llvm

#endif // LLVM_TRANSFORMS_IPO_LOWERTYPETESTS_H
3 changes: 3 additions & 0 deletions llvm/lib/Passes/PassBuilderPipelines.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1288,6 +1288,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
// and argument promotion.
MPM.addPass(DeadArgumentEliminationPass());

if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
MPM.addPass(SimplifyTypeTestsPass());

if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink)
MPM.addPass(CoroCleanupPass());

Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Passes/PassRegistry.def
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass())
MODULE_PASS("lower-emutls", LowerEmuTLSPass())
MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass())
MODULE_PASS("lower-ifunc", LowerIFuncPass())
MODULE_PASS("simplify-type-tests", SimplifyTypeTestsPass())
MODULE_PASS("lowertypetests", LowerTypeTestsPass())
MODULE_PASS("fatlto-cleanup", FatLtoCleanup())
MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default))
Expand Down
124 changes: 114 additions & 10 deletions llvm/lib/Transforms/IPO/LowerTypeTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/EquivalenceClasses.h"
#include "llvm/ADT/PointerUnion.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TypeMetadataUtils.h"
#include "llvm/Analysis/ValueTracking.h"
Expand Down Expand Up @@ -140,7 +143,7 @@ bool BitSetInfo::containsGlobalOffset(uint64_t Offset) const {
if (BitOffset >= BitSize)
return false;

return Bits.count(BitOffset);
return Bits.count(BitSize - 1 - BitOffset);
}

void BitSetInfo::print(raw_ostream &OS) const {
Expand Down Expand Up @@ -185,7 +188,11 @@ BitSetInfo BitSetBuilder::build() {
BSI.BitSize = ((Max - Min) >> BSI.AlignLog2) + 1;
for (uint64_t Offset : Offsets) {
Offset >>= BSI.AlignLog2;
BSI.Bits.insert(Offset);
// We invert the order of bits when adding them to the bitset. This is
// because the offset that we test against is computed by subtracting the
// address that we are testing from the global's address, which means that
// the offset increases as the tested address decreases.
BSI.Bits.insert(BSI.BitSize - 1 - Offset);
}

return BSI;
Expand Down Expand Up @@ -462,7 +469,8 @@ class LowerTypeTestsModule {
struct TypeIdLowering {
TypeTestResolution::Kind TheKind = TypeTestResolution::Unsat;

/// All except Unsat: the start address within the combined global.
/// All except Unsat: the address of the last element within the combined
/// global.
Constant *OffsetedGlobal;

/// ByteArray, Inline, AllOnes: log2 of the required global alignment
Expand Down Expand Up @@ -769,7 +777,11 @@ Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
if (TIL.TheKind == TypeTestResolution::Single)
return B.CreateICmpEQ(PtrAsInt, OffsetedGlobalAsInt);

Value *PtrOffset = B.CreateSub(PtrAsInt, OffsetedGlobalAsInt);
// Here we compute `last element - address`. The reason why we do this instead
// of computing `address - first element` is that it leads to a slightly
// shorter instruction sequence on x86. Because it doesn't matter how we do
// the subtraction on other architectures, we do so unconditionally.
Value *PtrOffset = B.CreateSub(OffsetedGlobalAsInt, PtrAsInt);

// We need to check that the offset both falls within our range and is
// suitably aligned. We can check both properties at the same time by
Expand All @@ -779,9 +791,8 @@ Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
// result, causing the comparison to fail if they are nonzero. The rotate
// also conveniently gives us a bit offset to use during the load from
// the bitset.
Value *BitOffset = B.CreateIntrinsic(
IntPtrTy, Intrinsic::fshr,
{PtrOffset, PtrOffset, B.CreateZExt(TIL.AlignLog2, IntPtrTy)});
Value *BitOffset = B.CreateIntrinsic(IntPtrTy, Intrinsic::fshr,
{PtrOffset, PtrOffset, TIL.AlignLog2});

Value *OffsetInRange = B.CreateICmpULE(BitOffset, TIL.SizeM1);

Expand Down Expand Up @@ -1033,7 +1044,7 @@ LowerTypeTestsModule::importTypeId(StringRef TypeId) {
if (TIL.TheKind == TypeTestResolution::ByteArray ||
TIL.TheKind == TypeTestResolution::Inline ||
TIL.TheKind == TypeTestResolution::AllOnes) {
TIL.AlignLog2 = ImportConstant("align", TTRes.AlignLog2, 8, Int8Ty);
TIL.AlignLog2 = ImportConstant("align", TTRes.AlignLog2, 8, IntPtrTy);
TIL.SizeM1 =
ImportConstant("size_m1", TTRes.SizeM1, TTRes.SizeM1BitWidth, IntPtrTy);
}
Expand Down Expand Up @@ -1152,9 +1163,12 @@ void LowerTypeTestsModule::lowerTypeTestCalls(

ByteArrayInfo *BAI = nullptr;
TypeIdLowering TIL;

uint64_t GlobalOffset =
BSI.ByteOffset + ((BSI.BitSize - 1) << BSI.AlignLog2);
TIL.OffsetedGlobal = ConstantExpr::getGetElementPtr(
Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, BSI.ByteOffset)),
TIL.AlignLog2 = ConstantInt::get(Int8Ty, BSI.AlignLog2);
Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, GlobalOffset)),
TIL.AlignLog2 = ConstantInt::get(IntPtrTy, BSI.AlignLog2);
TIL.SizeM1 = ConstantInt::get(IntPtrTy, BSI.BitSize - 1);
if (BSI.isAllOnes()) {
TIL.TheKind = (BSI.BitSize == 1) ? TypeTestResolution::Single
Expand Down Expand Up @@ -2472,3 +2486,93 @@ PreservedAnalyses LowerTypeTestsPass::run(Module &M,
return PreservedAnalyses::all();
return PreservedAnalyses::none();
}

PreservedAnalyses SimplifyTypeTestsPass::run(Module &M,
ModuleAnalysisManager &AM) {
bool Changed = false;
// Figure out whether inlining has exposed a constant address to a lowered
// type test, and remove the test if so and the address is known to pass the
// test. Unfortunately this pass ends up needing to reverse engineer what
// LowerTypeTests did; this is currently inherent to the design of ThinLTO
// importing where LowerTypeTests needs to run at the start.
//
// We look for things like:
//
// sub (i64 ptrtoint (ptr @_Z2fpv to i64), i64 ptrtoint (ptr @__typeid__ZTSFvvE_global_addr to i64))
//
// which gets replaced with 0 if _Z2fpv (more specifically _Z2fpv.cfi, the
// function referred to by the jump table) is a member of the type _ZTSFvv, as
// well as things like
//
// icmp eq ptr @_Z2fpv, @__typeid__ZTSFvvE_global_addr
//
// which gets replaced with true if _Z2fpv is a member.
for (auto &GV : M.globals()) {
if (!GV.getName().starts_with("__typeid_") ||
!GV.getName().ends_with("_global_addr"))
continue;
// __typeid_foo_global_addr -> foo
auto *MD = MDString::get(M.getContext(),
GV.getName().substr(9, GV.getName().size() - 21));
auto MaySimplifyPtr = [&](Value *Ptr) {
if (auto *GV = dyn_cast<GlobalValue>(Ptr))
if (auto *CFIGV = M.getNamedValue((GV->getName() + ".cfi").str()))
Ptr = CFIGV;
return isKnownTypeIdMember(MD, M.getDataLayout(), Ptr, 0);
};
auto MaySimplifyInt = [&](Value *Op) {
auto *PtrAsInt = dyn_cast<ConstantExpr>(Op);
if (!PtrAsInt || PtrAsInt->getOpcode() != Instruction::PtrToInt)
return false;
return MaySimplifyPtr(PtrAsInt->getOperand(0));
};
for (User *U : make_early_inc_range(GV.users())) {
if (auto *CI = dyn_cast<ICmpInst>(U)) {
if (CI->getPredicate() == CmpInst::ICMP_EQ &&
MaySimplifyPtr(CI->getOperand(0))) {
// This is an equality comparison (TypeTestResolution::Single case in
// lowerTypeTestCall). In this case we just replace the comparison
// with true.
CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext()));
CI->eraseFromParent();
Changed = true;
}
}
auto *CE = dyn_cast<ConstantExpr>(U);
if (!CE || CE->getOpcode() != Instruction::PtrToInt)
continue;
for (Use &U : make_early_inc_range(CE->uses())) {
auto *CE = dyn_cast<ConstantExpr>(U.getUser());
if (U.getOperandNo() == 0 && CE &&
CE->getOpcode() == Instruction::Sub &&
MaySimplifyInt(CE->getOperand(1))) {
// This is a computation of PtrOffset as generated by
// LowerTypeTestsModule::lowerTypeTestCall above. If
// isKnownTypeIdMember passes we just pretend it evaluated to 0. This
// should cause later passes to remove the range and alignment checks.
// The bitset checks won't be removed but those are uncommon.
CE->replaceAllUsesWith(ConstantInt::get(CE->getType(), 0));
Changed = true;
}
auto *CI = dyn_cast<ICmpInst>(U.getUser());
if (U.getOperandNo() == 1 && CI &&
CI->getPredicate() == CmpInst::ICMP_EQ &&
MaySimplifyInt(CI->getOperand(0))) {
// This is an equality comparison. Unlike in the case above it
// remained as an integer compare.
CI->replaceAllUsesWith(ConstantInt::getTrue(M.getContext()));
CI->eraseFromParent();
Changed = true;
}
}
}
}

if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA = PreservedAnalyses::none();
PA.preserve<DominatorTreeAnalysis>();
PA.preserve<PostDominatorTreeAnalysis>();
PA.preserve<LoopAnalysis>();
return PA;
}
1 change: 1 addition & 0 deletions llvm/test/Other/new-pm-thinlto-postlink-defaults.ll
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
; CHECK-O-NEXT: Running pass: CoroCleanupPass
; CHECK-POSTLINK-O-NEXT: Running pass: GlobalOptPass
; CHECK-POSTLINK-O-NEXT: Running pass: GlobalDCEPass
Expand Down
1 change: 1 addition & 0 deletions llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@
; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
; CHECK-O-NEXT: Running pass: CoroCleanupPass
; CHECK-O-NEXT: Running pass: GlobalOptPass
; CHECK-O-NEXT: Running pass: GlobalDCEPass
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@
; CHECK-O-NEXT: Invalidating analysis: ShouldNotRunFunctionPassesAnalysis
; CHECK-O-NEXT: Invalidating analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Running pass: DeadArgumentEliminationPass
; CHECK-O-NEXT: Running pass: SimplifyTypeTestsPass
; CHECK-O-NEXT: Running pass: CoroCleanupPass
; CHECK-O-NEXT: Running pass: GlobalOptPass
; CHECK-O-NEXT: Running pass: GlobalDCEPass
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/ThinLTO/X86/cfi-devirt.ll
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ cont2:
; CHECK-IR: br i1 {{.*}}, label %trap, label %cont2

; We still have to call it as virtual.
; CHECK-IR: %call3 = tail call i32 %4
; CHECK-IR: %call3 = tail call i32 %
%call3 = tail call i32 %5(ptr nonnull %obj, i32 %call)
ret i32 %call3
}
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/Transforms/LowerTypeTests/aarch64-jumptable.ll
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ define i1 @foo(ptr %p) {
; AARCH64-LABEL: define i1 @foo
; AARCH64-SAME: (ptr [[P:%.*]]) {
; AARCH64-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
; AARCH64-NEXT: [[TMP2:%.*]] = sub i64 [[TMP1]], ptrtoint (ptr @.cfi.jumptable to i64)
; AARCH64-NEXT: [[TMP2:%.*]] = sub i64 ptrtoint (ptr getelementptr (i8, ptr @.cfi.jumptable, i64 8) to i64), [[TMP1]]
; AARCH64-NEXT: [[TMP3:%.*]] = call i64 @llvm.fshr.i64(i64 [[TMP2]], i64 [[TMP2]], i64 3)
; AARCH64-NEXT: [[TMP4:%.*]] = icmp ule i64 [[TMP3]], 1
; AARCH64-NEXT: ret i1 [[TMP4]]
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/Transforms/LowerTypeTests/export-allones.ll
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,12 @@

; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer

; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 2)
; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 1 to ptr)
; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 1 to ptr)

; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 516)
; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i64 2 to ptr)
; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 128 to ptr)

; ARM-NOT: alias {{.*}} inttoptr
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@
; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
; CHECK: [[B:@[0-9]+]] = private constant [258 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01"

; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 130)
; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 1 to ptr)
; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 65 to ptr)
; CHECK: @__typeid_typeid1_byte_array = hidden alias i8, ptr @bits.1
; X86: @__typeid_typeid1_bit_mask = hidden alias i8, inttoptr (i8 2 to ptr)

; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 1032)
; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i64 2 to ptr)
; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 257 to ptr)
; CHECK: @__typeid_typeid2_byte_array = hidden alias i8, ptr @bits
; X86: @__typeid_typeid2_bit_mask = hidden alias i8, inttoptr (i8 1 to ptr)
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Transforms/LowerTypeTests/export-icall.ll
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ define void @f3(i32 %x) !type !8 {
!8 = !{i64 0, !"typeid3"}


; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[JT1:.*]]
; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 3 to ptr)
; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[JT1:.*]], i64 32)
; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i64 3 to ptr)
; CHECK-DAG: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 4 to ptr)

; CHECK-DAG: @h = alias void (i8), ptr [[JT1]]
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Transforms/LowerTypeTests/export-inline.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@

; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer

; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, ptr [[G]]
; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 6)
; CHECK-X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to ptr)
; CHECK-X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 3 to ptr)
; CHECK-X86: @__typeid_typeid1_inline_bits = hidden alias i8, inttoptr (i32 9 to ptr)

; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 4)
; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr (i8, ptr [[G]], i64 136)
; CHECK-X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to ptr)
; CHECK-X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 33 to ptr)
; CHECK-X86: @__typeid_typeid2_inline_bits = hidden alias i8, inttoptr (i64 8589934593 to ptr)
Expand Down
Loading
Loading