Skip to content

Commit faebbd7

Browse files
aschwaighoferaschwaighofer
authored
Merge pull request #6198 from aschwaighofer/relative_function_pointer_type_checked_load_support_20221013
[stable/20221013] Add support for swift's relative function pointer tables to type_checked_load
2 parents 237f6b8 + 1638905 commit faebbd7

File tree

9 files changed

+130
-29
lines changed

9 files changed

+130
-29
lines changed

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1839,6 +1839,11 @@ def int_type_checked_load : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_i1_ty],
18391839
[llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty],
18401840
[IntrNoMem, IntrWillReturn]>;
18411841

1842+
// Safely loads a relative function pointer from a virtual table pointer using type metadata.
1843+
def int_type_checked_load_relative : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_i1_ty],
1844+
[llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty],
1845+
[IntrNoMem, IntrWillReturn]>;
1846+
18421847
// Test whether a pointer is associated with a type metadata identifier. Used
18431848
// for public visibility classes that may later be refined to private
18441849
// visibility.

‎llvm/lib/Analysis/ModuleSummaryAnalysis.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ static void addIntrinsicToSummary(
192192
break;
193193
}
194194

195+
case Intrinsic::type_checked_load_relative:
195196
case Intrinsic::type_checked_load: {
196197
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
197198
auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());

‎llvm/lib/Analysis/TypeMetadataUtils.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,9 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad(
9999
SmallVectorImpl<Instruction *> &Preds, bool &HasNonCallUses,
100100
const CallInst *CI, DominatorTree &DT) {
101101
assert(CI->getCalledFunction()->getIntrinsicID() ==
102-
Intrinsic::type_checked_load);
102+
Intrinsic::type_checked_load ||
103+
CI->getCalledFunction()->getIntrinsicID() ==
104+
Intrinsic::type_checked_load_relative);
103105

104106
auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
105107
if (!Offset) {

‎llvm/lib/LTO/LTO.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -970,11 +970,16 @@ Error LTO::checkPartiallySplit() {
970970
Intrinsic::getName(Intrinsic::type_test));
971971
Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
972972
Intrinsic::getName(Intrinsic::type_checked_load));
973+
Function *TypeCheckedLoadRelativeFunc =
974+
RegularLTO.CombinedModule->getFunction(
975+
Intrinsic::getName(Intrinsic::type_checked_load_relative));
973976

974977
// First check if there are type tests / type checked loads in the
975978
// merged regular LTO module IR.
976979
if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
977-
(TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
980+
(TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
981+
(TypeCheckedLoadRelativeFunc &&
982+
!TypeCheckedLoadRelativeFunc->use_empty()))
978983
return make_error<StringError>(
979984
"inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
980985
inconvertibleErrorCode());

‎llvm/lib/Transforms/IPO/GlobalDCE.cpp

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -310,29 +310,36 @@ void GlobalDCEPass::ScanTypeCheckedLoadIntrinsics(Module &M) {
310310
LLVM_DEBUG(dbgs() << "Scanning type.checked.load intrinsics\n");
311311
Function *TypeCheckedLoadFunc =
312312
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
313-
314-
if (!TypeCheckedLoadFunc)
315-
return;
316-
317-
for (auto *U : TypeCheckedLoadFunc->users()) {
318-
auto CI = dyn_cast<CallInst>(U);
319-
if (!CI)
320-
continue;
321-
322-
auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
323-
Value *TypeIdValue = CI->getArgOperand(2);
324-
auto *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
325-
326-
if (Offset) {
327-
ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue());
328-
} else {
329-
// type.checked.load with a non-constant offset, so assume every entry in
330-
// every matching vtable is used.
331-
for (const auto &VTableInfo : TypeIdMap[TypeId]) {
332-
VFESafeVTablesAndFns.erase(VTableInfo.first);
313+
Function *TypeCheckedLoadRelativeFunc =
314+
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
315+
316+
auto scan = [&](Function *CheckedLoadFunc) {
317+
if (!CheckedLoadFunc)
318+
return;
319+
320+
for (auto U : CheckedLoadFunc->users()) {
321+
auto CI = dyn_cast<CallInst>(U);
322+
if (!CI)
323+
continue;
324+
325+
auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(1));
326+
Value *TypeIdValue = CI->getArgOperand(2);
327+
auto *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
328+
329+
if (Offset) {
330+
ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue());
331+
} else {
332+
// type.checked.load with a non-constant offset, so assume every entry
333+
// in every matching vtable is used.
334+
for (auto &VTableInfo : TypeIdMap[TypeId]) {
335+
VFESafeVTablesAndFns.erase(VTableInfo.first);
336+
}
333337
}
334338
}
335-
}
339+
};
340+
341+
scan(TypeCheckedLoadFunc);
342+
scan(TypeCheckedLoadRelativeFunc);
336343
}
337344

338345
void GlobalDCEPass::AddVirtualFunctionDependencies(Module &M) {

‎llvm/lib/Transforms/IPO/GlobalSplit.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,12 @@ static bool splitGlobals(Module &M) {
149149
M.getFunction(Intrinsic::getName(Intrinsic::type_test));
150150
Function *TypeCheckedLoadFunc =
151151
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
152+
Function *TypeCheckedLoadRelativeFunc =
153+
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
152154
if ((!TypeTestFunc || TypeTestFunc->use_empty()) &&
153-
(!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()))
155+
(!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) &&
156+
(!TypeCheckedLoadRelativeFunc ||
157+
TypeCheckedLoadRelativeFunc->use_empty()))
154158
return false;
155159

156160
bool Changed = false;

‎llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,14 @@ void promoteTypeIds(Module &M, StringRef ModuleId) {
148148
}
149149
}
150150

151+
if (Function *TypeCheckedLoadRelativeFunc = M.getFunction(
152+
Intrinsic::getName(Intrinsic::type_checked_load_relative))) {
153+
for (const Use &U : TypeCheckedLoadRelativeFunc->uses()) {
154+
auto CI = cast<CallInst>(U.getUser());
155+
ExternalizeTypeId(CI, 2);
156+
}
157+
}
158+
151159
for (GlobalObject &GO : M.global_objects()) {
152160
SmallVector<MDNode *, 1> MDs;
153161
GO.getMetadata(LLVMContext::MD_type, MDs);

‎llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,7 +1004,8 @@ bool DevirtModule::tryFindVirtualCallTargets(
10041004
return false;
10051005

10061006
Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(),
1007-
TM.Offset + ByteOffset, M);
1007+
TM.Offset + ByteOffset, M,
1008+
TM.Bits->GV);
10081009
if (!Ptr)
10091010
return false;
10101011

@@ -1949,9 +1950,23 @@ void DevirtModule::scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc) {
19491950
// This helps avoid unnecessary spills.
19501951
IRBuilder<> LoadB(
19511952
(LoadedPtrs.size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
1952-
Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
1953-
Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy));
1954-
Value *LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
1953+
1954+
Value *LoadedValue = nullptr;
1955+
if (TypeCheckedLoadFunc->getIntrinsicID() ==
1956+
Intrinsic::type_checked_load_relative) {
1957+
Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
1958+
Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int32Ty));
1959+
LoadedValue = LoadB.CreateLoad(Int32Ty, GEPPtr);
1960+
LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy);
1961+
GEP = LoadB.CreatePtrToInt(GEP, IntPtrTy);
1962+
LoadedValue = LoadB.CreateAdd(GEP, LoadedValue);
1963+
LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy);
1964+
} else {
1965+
Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset);
1966+
Value *GEPPtr =
1967+
LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy));
1968+
LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
1969+
}
19551970

19561971
for (Instruction *LoadedPtr : LoadedPtrs) {
19571972
LoadedPtr->replaceAllUsesWith(LoadedValue);
@@ -2132,6 +2147,8 @@ bool DevirtModule::run() {
21322147
M.getFunction(Intrinsic::getName(Intrinsic::type_test));
21332148
Function *TypeCheckedLoadFunc =
21342149
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
2150+
Function *TypeCheckedLoadRelativeFunc =
2151+
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative));
21352152
Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume));
21362153

21372154
// Normally if there are no users of the devirtualization intrinsics in the
@@ -2140,7 +2157,9 @@ bool DevirtModule::run() {
21402157
if (!ExportSummary &&
21412158
(!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc ||
21422159
AssumeFunc->use_empty()) &&
2143-
(!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()))
2160+
(!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) &&
2161+
(!TypeCheckedLoadRelativeFunc ||
2162+
TypeCheckedLoadRelativeFunc->use_empty()))
21442163
return false;
21452164

21462165
// Rebuild type metadata into a map for easy lookup.
@@ -2154,6 +2173,9 @@ bool DevirtModule::run() {
21542173
if (TypeCheckedLoadFunc)
21552174
scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
21562175

2176+
if (TypeCheckedLoadRelativeFunc)
2177+
scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);
2178+
21572179
if (ImportSummary) {
21582180
for (auto &S : CallSlots)
21592181
importResolution(S.first, S.second);

‎llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
2+
3+
target datalayout = "e-p:64:64"
4+
target triple = "x86_64-unknown-linux-gnu"
5+
6+
; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
7+
; CHECK: remark: <unknown>:0:0: devirtualized vf
8+
; CHECK-NOT: devirtualized
9+
10+
; A vtable with "relative pointers", slots don't contain pointers to implementations, but instead have an i32 offset from the vtable itself to the implementation.
11+
@vtable = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
12+
i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
13+
i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc2_dead to i64), i64 ptrtoint (ptr @vtable to i64)) to i32)
14+
]}, align 8, !type !0, !type !1
15+
;, !vcall_visibility !{i64 2}
16+
!0 = !{i64 0, !"vfunc1.type"}
17+
!1 = !{i64 4, !"vfunc2.type"}
18+
19+
define internal void @vfunc1_live() {
20+
ret void
21+
}
22+
23+
define internal void @vfunc2_dead() {
24+
ret void
25+
}
26+
27+
; CHECK: define void @call
28+
define void @call(ptr %obj) {
29+
%vtable = load ptr, ptr %obj
30+
%pair = call {ptr, i1} @llvm.type.checked.load.relative(ptr %vtable, i32 0, metadata !"vfunc1.type")
31+
%fptr = extractvalue {ptr, i1} %pair, 0
32+
%p = extractvalue {ptr, i1} %pair, 1
33+
; CHECK: br i1 true,
34+
br i1 %p, label %cont, label %trap
35+
36+
cont:
37+
; CHECK: call void @vfunc1_live(
38+
call void %fptr()
39+
ret void
40+
41+
trap:
42+
call void @llvm.trap()
43+
unreachable
44+
}
45+
46+
declare {ptr, i1} @llvm.type.checked.load.relative(ptr, i32, metadata)
47+
declare void @llvm.trap()

0 commit comments

Comments
 (0)