Skip to content

Commit d62fbb8

Browse files
atrickatrick
authored
Merge pull request #40296 from atrick/assert-align
Add Builtin.assumeAlignment with support for misaligned loads
2 parents b86a7fd + 4180acf commit d62fbb8

File tree

16 files changed

+328
-19
lines changed

16 files changed

+328
-19
lines changed

‎include/swift/AST/Builtins.def

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -820,6 +820,24 @@ BUILTIN_MISC_OPERATION(Move, "move", "", Special)
820820
/// the SILVerifier.
821821
BUILTIN_MISC_OPERATION(Copy, "copy", "", Special)
822822

823+
/// Unchecked pointer alignment assertion. Allows the compiler to assume
824+
/// alignment of the pointer to emit more efficient code.
825+
///
826+
/// %alignedPtr = builtin "assumeAlignment" (%ptr : $Builtin.RawPointer,
827+
/// %alignment : $Builtin.Int)
828+
/// : $Builtin.RawPointer
829+
/// %address = pointer_to_address %alignedPtr
830+
/// : $Builtin.RawPointer to [align=1] $*Int
831+
/// %val = load %address : $*Int
832+
///
833+
/// With compile-time knowledge of the value of `%alignment` the compiler can
834+
/// optimize any downstream 'pointer_to_address' instruction by refining its
835+
/// '[align=]' flag . That `[align=]` flag can be used by IRGen to refine the
836+
/// alignment on LLVM load instructions that use the resulting address.
837+
///
838+
/// (Builtin.RawPointer, Builtin.Word) -> Builtin.RawPointer
839+
BUILTIN_MISC_OPERATION(AssumeAlignment, "assumeAlignment", "n", Special)
840+
823841
// BUILTIN_MISC_OPERATION_WITH_SILGEN - Miscellaneous operations that are
824842
// specially emitted during SIL generation.
825843
//

‎include/swift/SIL/SILCloner.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,11 +1430,11 @@ template<typename ImplClass>
14301430
void
14311431
SILCloner<ImplClass>::visitPointerToAddressInst(PointerToAddressInst *Inst) {
14321432
getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
1433-
recordClonedInstruction(Inst, getBuilder().createPointerToAddress(
1434-
getOpLocation(Inst->getLoc()),
1435-
getOpValue(Inst->getOperand()),
1436-
getOpType(Inst->getType()),
1437-
Inst->isStrict(), Inst->isInvariant()));
1433+
recordClonedInstruction(
1434+
Inst, getBuilder().createPointerToAddress(
1435+
getOpLocation(Inst->getLoc()), getOpValue(Inst->getOperand()),
1436+
getOpType(Inst->getType()), Inst->isStrict(),
1437+
Inst->isInvariant(), Inst->alignment()));
14381438
}
14391439

14401440
template<typename ImplClass>

‎lib/AST/Builtins.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -861,6 +861,12 @@ static ValueDecl *getCopyOperation(ASTContext &ctx, Identifier id) {
861861
_parameters(_typeparam(0)), _typeparam(0));
862862
}
863863

864+
static ValueDecl *getAssumeAlignment(ASTContext &ctx, Identifier id) {
865+
// This is always "(Builtin.RawPointer, Builtin.Word) -> Builtin.RawPointer"
866+
return getBuiltinFunction(ctx, id, _thin, _parameters(_rawPointer, _word),
867+
_rawPointer);
868+
}
869+
864870
static ValueDecl *getTransferArrayOperation(ASTContext &ctx, Identifier id) {
865871
return getBuiltinFunction(ctx, id, _thin,
866872
_generics(_unrestricted),
@@ -2552,6 +2558,11 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
25522558
return nullptr;
25532559
return getCopyOperation(Context, Id);
25542560

2561+
case BuiltinValueKind::AssumeAlignment:
2562+
if (!Types.empty())
2563+
return nullptr;
2564+
return getAssumeAlignment(Context, Id);
2565+
25552566
#define BUILTIN(id, name, Attrs)
25562567
#define BUILTIN_BINARY_OPERATION(id, name, attrs)
25572568
#define BUILTIN_BINARY_OPERATION_OVERLOADED_STATIC(id, name, attrs, overload) \

‎lib/IRGen/GenBuiltin.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1338,6 +1338,18 @@ if (Builtin.ID == BuiltinValueKind::id) { \
13381338
addrTI.initializeWithCopy(IGF, resultAttr, inputAttr, addrTy, false);
13391339
return;
13401340
}
1341+
if (Builtin.ID == BuiltinValueKind::AssumeAlignment) {
1342+
// A no-op pointer cast that passes on its first value. Common occurences of
1343+
// this builtin should already be removed with the alignment guarantee moved
1344+
// to the subsequent load or store.
1345+
//
1346+
// TODO: Consider lowering to an LLVM intrinsic if there is any benefit:
1347+
// 'call void @llvm.assume(i1 true) ["align"(i32* %arg0, i32 %arg1)]'
1348+
auto pointerSrc = args.claimNext();
1349+
(void)args.claimAll();
1350+
out.add(pointerSrc);
1351+
return;
1352+
}
13411353

13421354
llvm_unreachable("IRGen unimplemented for this builtin!");
13431355
}

‎lib/IRGen/LoadableByAddress.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2113,7 +2113,7 @@ static void rewriteFunction(StructLoweringState &pass,
21132113
currSILType);
21142114
auto *newInstr = pointerBuilder.createPointerToAddress(
21152115
instr->getLoc(), instr->getOperand(), newSILType.getAddressType(),
2116-
instr->isStrict());
2116+
instr->isStrict(), instr->isInvariant(), instr->alignment());
21172117
instr->replaceAllUsesWith(newInstr);
21182118
instr->getParent()->erase(instr);
21192119
}

‎lib/SIL/IR/OperandOwnership.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,7 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssignCopyArrayNoAlias)
647647
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssignCopyArrayFrontToBack)
648648
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssignCopyArrayBackToFront)
649649
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssignTakeArray)
650+
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssumeAlignment)
650651
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssumeNonNegative)
651652
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AssumeTrue)
652653
BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, AtomicLoad)

‎lib/SIL/IR/ValueOwnership.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,7 @@ CONSTANT_OWNERSHIP_BUILTIN(None, Add)
393393
CONSTANT_OWNERSHIP_BUILTIN(None, GenericAdd)
394394
CONSTANT_OWNERSHIP_BUILTIN(None, And)
395395
CONSTANT_OWNERSHIP_BUILTIN(None, GenericAnd)
396+
CONSTANT_OWNERSHIP_BUILTIN(None, AssumeAlignment)
396397
CONSTANT_OWNERSHIP_BUILTIN(None, AssumeNonNegative)
397398
CONSTANT_OWNERSHIP_BUILTIN(None, AssumeTrue)
398399
CONSTANT_OWNERSHIP_BUILTIN(None, BitCast)

‎lib/SILGen/SILGenBuiltin.cpp

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ static ManagedValue emitBuiltinLoadOrTake(SILGenFunction &SGF,
101101
SGFContext C,
102102
IsTake_t isTake,
103103
bool isStrict,
104-
bool isInvariant) {
104+
bool isInvariant,
105+
llvm::MaybeAlign align) {
105106
assert(substitutions.getReplacementTypes().size() == 1 &&
106107
"load should have single substitution");
107108
assert(args.size() == 1 && "load should have a single argument");
@@ -113,9 +114,12 @@ static ManagedValue emitBuiltinLoadOrTake(SILGenFunction &SGF,
113114
SILType loadedType = rvalueTL.getLoweredType();
114115

115116
// Convert the pointer argument to a SIL address.
117+
//
118+
// Default to an unaligned pointer. This can be optimized in the presence of
119+
// Builtin.assumeAlignment.
116120
SILValue addr = SGF.B.createPointerToAddress(loc, args[0].getUnmanagedValue(),
117121
loadedType.getAddressType(),
118-
isStrict, isInvariant);
122+
isStrict, isInvariant, align);
119123
// Perform the load.
120124
return SGF.emitLoad(loc, addr, rvalueTL, C, isTake);
121125
}
@@ -125,39 +129,47 @@ static ManagedValue emitBuiltinLoad(SILGenFunction &SGF,
125129
SubstitutionMap substitutions,
126130
ArrayRef<ManagedValue> args,
127131
SGFContext C) {
132+
// Regular loads assume natural alignment.
128133
return emitBuiltinLoadOrTake(SGF, loc, substitutions, args,
129134
C, IsNotTake,
130-
/*isStrict*/ true, /*isInvariant*/ false);
135+
/*isStrict*/ true, /*isInvariant*/ false,
136+
llvm::MaybeAlign());
131137
}
132138

133139
static ManagedValue emitBuiltinLoadRaw(SILGenFunction &SGF,
134140
SILLocation loc,
135141
SubstitutionMap substitutions,
136142
ArrayRef<ManagedValue> args,
137143
SGFContext C) {
144+
// Raw loads cannot assume alignment.
138145
return emitBuiltinLoadOrTake(SGF, loc, substitutions, args,
139146
C, IsNotTake,
140-
/*isStrict*/ false, /*isInvariant*/ false);
147+
/*isStrict*/ false, /*isInvariant*/ false,
148+
llvm::MaybeAlign(1));
141149
}
142150

143151
static ManagedValue emitBuiltinLoadInvariant(SILGenFunction &SGF,
144152
SILLocation loc,
145153
SubstitutionMap substitutions,
146154
ArrayRef<ManagedValue> args,
147155
SGFContext C) {
156+
// Regular loads assume natural alignment.
148157
return emitBuiltinLoadOrTake(SGF, loc, substitutions, args,
149158
C, IsNotTake,
150-
/*isStrict*/ false, /*isInvariant*/ true);
159+
/*isStrict*/ false, /*isInvariant*/ true,
160+
llvm::MaybeAlign());
151161
}
152162

153163
static ManagedValue emitBuiltinTake(SILGenFunction &SGF,
154164
SILLocation loc,
155165
SubstitutionMap substitutions,
156166
ArrayRef<ManagedValue> args,
157167
SGFContext C) {
168+
// Regular loads assume natural alignment.
158169
return emitBuiltinLoadOrTake(SGF, loc, substitutions, args,
159170
C, IsTake,
160-
/*isStrict*/ true, /*isInvariant*/ false);
171+
/*isStrict*/ true, /*isInvariant*/ false,
172+
llvm::MaybeAlign());
161173
}
162174

163175
/// Specialized emitter for Builtin.destroy.

‎lib/SILOptimizer/SILCombiner/SILCombiner.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,8 @@ class SILCombiner :
406406
SingleValueInstruction *user, SingleValueInstruction *value,
407407
function_ref<SILValue()> newValueGenerator);
408408

409+
SILInstruction *optimizeAlignment(PointerToAddressInst *ptrAdrInst);
410+
409411
InstModCallbacks &getInstModCallbacks() { return deleter.getCallbacks(); }
410412

411413
private:

‎lib/SILOptimizer/SILCombiner/SILCombinerCastVisitors.cpp

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,95 @@ SILInstruction *SILCombiner::visitUpcastInst(UpcastInst *uci) {
218218
return nullptr;
219219
}
220220

221+
// Optimize Builtin.assumeAlignment -> pointer_to_address
222+
//
223+
// Case #1. Literal zero = natural alignment
224+
// %1 = integer_literal $Builtin.Int64, 0
225+
// %2 = builtin "assumeAlignment"
226+
// (%0 : $Builtin.RawPointer, %1 : $Builtin.Int64) : $Builtin.RawPointer
227+
// %3 = pointer_to_address %2 : $Builtin.RawPointer to [align=1] $*Int
228+
//
229+
// Erases the `pointer_to_address` `[align=]` attribute:
230+
//
231+
// Case #2. Literal nonzero = forced alignment.
232+
//
233+
// %1 = integer_literal $Builtin.Int64, 16
234+
// %2 = builtin "assumeAlignment"
235+
// (%0 : $Builtin.RawPointer, %1 : $Builtin.Int64) : $Builtin.RawPointer
236+
// %3 = pointer_to_address %2 : $Builtin.RawPointer to [align=1] $*Int
237+
//
238+
// Promotes the `pointer_to_address` `[align=]` attribute to a higher value.
239+
//
240+
// Case #3. Folded dynamic alignment
241+
//
242+
// %1 = builtin "alignof"<T>(%0 : $@thin T.Type) : $Builtin.Word
243+
// %2 = builtin "assumeAlignment"
244+
// (%0 : $Builtin.RawPointer, %1 : $Builtin.Int64) : $Builtin.RawPointer
245+
// %3 = pointer_to_address %2 : $Builtin.RawPointer to [align=1] $*T
246+
//
247+
// Erases the `pointer_to_address` `[align=]` attribute.
248+
SILInstruction *
249+
SILCombiner::optimizeAlignment(PointerToAddressInst *ptrAdrInst) {
250+
if (!ptrAdrInst->alignment())
251+
return nullptr;
252+
253+
llvm::Align oldAlign = ptrAdrInst->alignment().valueOrOne();
254+
255+
// TODO: stripCasts(ptrAdrInst->getOperand()) can be used to find the Builtin,
256+
// but then the Builtin could not be trivially removed. Ideally,
257+
// Builtin.assume will be the immediate operand so it can be removed in the
258+
// common case.
259+
BuiltinInst *assumeAlign = dyn_cast<BuiltinInst>(ptrAdrInst->getOperand());
260+
if (!assumeAlign
261+
|| assumeAlign->getBuiltinKind() != BuiltinValueKind::AssumeAlignment) {
262+
return nullptr;
263+
}
264+
SILValue ptrSrc = assumeAlign->getArguments()[0];
265+
SILValue alignOper = assumeAlign->getArguments()[1];
266+
267+
if (auto *integerInst = dyn_cast<IntegerLiteralInst>(alignOper)) {
268+
llvm::MaybeAlign newAlign(integerInst->getValue().getLimitedValue());
269+
if (newAlign && newAlign.valueOrOne() <= oldAlign)
270+
return nullptr;
271+
272+
// Case #1: the pointer is assumed naturally aligned
273+
//
274+
// Or Case #2: the pointer is assumed to have non-zero alignment greater
275+
// than it current alignment.
276+
//
277+
// In either case, rewrite the address alignment with the assumed alignment,
278+
// and bypass the Builtin.assumeAlign.
279+
return Builder.createPointerToAddress(
280+
ptrAdrInst->getLoc(), ptrSrc, ptrAdrInst->getType(),
281+
ptrAdrInst->isStrict(), ptrAdrInst->isInvariant(), newAlign);
282+
}
283+
// Handle possible 32-bit sign-extension.
284+
SILValue extendedAlignment;
285+
if (match(alignOper,
286+
m_ApplyInst(BuiltinValueKind::SExtOrBitCast,
287+
m_ApplyInst(BuiltinValueKind::TruncOrBitCast,
288+
m_SILValue(extendedAlignment))))) {
289+
alignOper = extendedAlignment;
290+
}
291+
MetatypeInst *metatype;
292+
if (match(alignOper,
293+
m_ApplyInst(BuiltinValueKind::Alignof, m_MetatypeInst(metatype)))) {
294+
SILType instanceType = ptrAdrInst->getFunction()->getLoweredType(
295+
metatype->getType().castTo<MetatypeType>().getInstanceType());
296+
297+
if (instanceType.getAddressType() != ptrAdrInst->getType())
298+
return nullptr;
299+
300+
// Case #3: the alignOf type matches the address type. Convert to a
301+
// naturally aligned pointer by erasing alignment and bypassing the
302+
// Builtin.assumeAlign.
303+
return Builder.createPointerToAddress(
304+
ptrAdrInst->getLoc(), ptrSrc, ptrAdrInst->getType(),
305+
ptrAdrInst->isStrict(), ptrAdrInst->isInvariant());
306+
}
307+
return nullptr;
308+
}
309+
221310
SILInstruction *
222311
SILCombiner::
223312
visitPointerToAddressInst(PointerToAddressInst *PTAI) {
@@ -374,7 +463,7 @@ visitPointerToAddressInst(PointerToAddressInst *PTAI) {
374463
}
375464
}
376465

377-
return nullptr;
466+
return optimizeAlignment(PTAI);
378467
}
379468

380469
SILInstruction *

0 commit comments

Comments
 (0)