Skip to content

Commit 364be60

Browse files
aeubanksmemfrob
aeubanks
authored and
memfrob
committed
Reland [IR] Increase max alignment to 4GB
Currently the max alignment representable is 1GB, see D108661. Setting the align of an object to 4GB is desirable in some cases to make sure the lower 32 bits are clear which can be used for some optimizations, e.g. https://crbug.com/1016945. This uses an extra bit in instructions that carry an alignment. We can store 15 bits of "free" information, and with this change some instructions (e.g. AtomicCmpXchgInst) use 14 bits. We can increase the max alignment representable above 4GB (up to 2^62) since we're only using 33 of the 64 values, but I've just limited it to 4GB for now. The one place we have to update the bitcode format is for the alloca instruction. It stores its alignment into 5 bits of a 32 bit bitfield. I've added another field which is 8 bits and should be future proof for a while. For backward compatibility, we check if the old field has a value and use that, otherwise use the new field. Updating clang's max allowed alignment will come in a future patch. Reviewed By: hans Differential Revision: https://reviews.llvm.org/D110451
1 parent a0d918c commit 364be60

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+176
-167
lines changed

‎clang/lib/CodeGen/CGBlocks.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2695,8 +2695,8 @@ const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
26952695
size = varOffset;
26962696

26972697
// Conversely, we might have to prevent LLVM from inserting padding.
2698-
} else if (CGM.getDataLayout().getABITypeAlignment(varTy)
2699-
> varAlign.getQuantity()) {
2698+
} else if (CGM.getDataLayout().getABITypeAlignment(varTy) >
2699+
uint64_t(varAlign.getQuantity())) {
27002700
packed = true;
27012701
}
27022702
types.push_back(varTy);

‎clang/lib/CodeGen/CGCall.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5021,12 +5021,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
50215021
auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
50225022

50235023
// Materialize to a temporary.
5024-
addr = CreateTempAlloca(
5025-
RV.getScalarVal()->getType(),
5026-
CharUnits::fromQuantity(std::max(
5027-
(unsigned)layout->getAlignment().value(), scalarAlign)),
5028-
"tmp",
5029-
/*ArraySize=*/nullptr, &AllocaAddr);
5024+
addr =
5025+
CreateTempAlloca(RV.getScalarVal()->getType(),
5026+
CharUnits::fromQuantity(std::max(
5027+
layout->getAlignment().value(), scalarAlign)),
5028+
"tmp",
5029+
/*ArraySize=*/nullptr, &AllocaAddr);
50305030
tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
50315031

50325032
Builder.CreateStore(RV.getScalarVal(), addr);

‎clang/lib/CodeGen/CGDecl.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1142,7 +1142,7 @@ Address CodeGenModule::createUnnamedGlobalFrom(const VarDecl &D,
11421142
GV->setAlignment(Align.getAsAlign());
11431143
GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
11441144
CacheEntry = GV;
1145-
} else if (CacheEntry->getAlignment() < Align.getQuantity()) {
1145+
} else if (CacheEntry->getAlignment() < uint64_t(Align.getQuantity())) {
11461146
CacheEntry->setAlignment(Align.getAsAlign());
11471147
}
11481148

‎clang/lib/CodeGen/CodeGenModule.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5386,7 +5386,7 @@ CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
53865386
if (!LangOpts.WritableStrings) {
53875387
Entry = &ConstantStringMap[C];
53885388
if (auto GV = *Entry) {
5389-
if (Alignment.getQuantity() > GV->getAlignment())
5389+
if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
53905390
GV->setAlignment(Alignment.getAsAlign());
53915391
return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
53925392
Alignment);
@@ -5449,7 +5449,7 @@ ConstantAddress CodeGenModule::GetAddrOfConstantCString(
54495449
if (!LangOpts.WritableStrings) {
54505450
Entry = &ConstantStringMap[C];
54515451
if (auto GV = *Entry) {
5452-
if (Alignment.getQuantity() > GV->getAlignment())
5452+
if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
54535453
GV->setAlignment(Alignment.getAsAlign());
54545454
return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
54555455
Alignment);

‎clang/test/CodeGen/builtin-assume-aligned.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ int test6() {
117117
// CHECK-NEXT: store i32* [[A]], i32** [[A_ADDR]], align 8
118118
// CHECK-NEXT: [[TMP0:%.*]] = load i32*, i32** [[A_ADDR]], align 8
119119
// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[TMP0]] to i8*
120-
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* [[TMP1]], i64 1073741824) ]
120+
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* [[TMP1]], i64 4294967296) ]
121121
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32*
122122
// CHECK-NEXT: store i32* [[TMP2]], i32** [[A_ADDR]], align 8
123123
// CHECK-NEXT: [[TMP3:%.*]] = load i32*, i32** [[A_ADDR]], align 8

‎clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-paramvar.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
// CHECK-SANITIZE-ANYRECOVER: @[[LINE_100_ALIGNMENT_ASSUMPTION:.*]] = {{.*}}, i32 100, i32 10 }, {{.*}}* @[[CHAR]] }
88

99
char **passthrough(__attribute__((align_value(0x80000000))) char **x) {
10-
// CHECK-NOSANITIZE: define{{.*}} i8** @{{.*}}(i8** align 1073741824 %[[X:.*]])
10+
// CHECK-NOSANITIZE: define{{.*}} i8** @{{.*}}(i8** align 2147483648 %[[X:.*]])
1111
// CHECK-SANITIZE: define{{.*}} i8** @{{.*}}(i8** %[[X:.*]])
1212
// CHECK-NEXT: [[entry:.*]]:
1313
// CHECK-NEXT: %[[X_ADDR:.*]] = alloca i8**, align 8

‎clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params-variable.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@ void *caller(char **x, unsigned long offset) {
1818
// CHECK-NEXT: %[[OFFSET_RELOADED:.*]] = load i64, i64* %[[OFFSET_ADDR]], align 8
1919
// CHECK-SANITIZE-NEXT: %[[PTRINT:.*]] = ptrtoint i8* %[[BITCAST]] to i64
2020
// CHECK-SANITIZE-NEXT: %[[OFFSETPTR:.*]] = sub i64 %[[PTRINT]], %[[OFFSET_RELOADED]]
21-
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[OFFSETPTR]], 1073741823
21+
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[OFFSETPTR]], 2147483647
2222
// CHECK-SANITIZE-NEXT: %[[MASKCOND:.*]] = icmp eq i64 %[[MASKEDPTR]], 0
2323
// CHECK-SANITIZE-NEXT: %[[PTRINT_DUP:.*]] = ptrtoint i8* %[[BITCAST]] to i64, !nosanitize
2424
// CHECK-SANITIZE-NEXT: br i1 %[[MASKCOND]], label %[[CONT:.*]], label %[[HANDLER_ALIGNMENT_ASSUMPTION:[^,]+]],{{.*}} !nosanitize
2525
// CHECK-SANITIZE: [[HANDLER_ALIGNMENT_ASSUMPTION]]:
26-
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
27-
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
26+
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
27+
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
2828
// CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
2929
// CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
3030
// CHECK-SANITIZE: [[CONT]]:
31-
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 1073741824, i64 %[[OFFSET_RELOADED]]) ]
31+
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 2147483648, i64 %[[OFFSET_RELOADED]]) ]
3232
// CHECK-NEXT: ret i8* %[[BITCAST]]
3333
// CHECK-NEXT: }
3434
#line 100

‎clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,17 @@ void *caller(char **x) {
1515
// CHECK-NEXT: %[[BITCAST:.*]] = bitcast i8** %[[X_RELOADED]] to i8*
1616
// CHECK-SANITIZE-NEXT: %[[PTRINT:.*]] = ptrtoint i8* %[[BITCAST]] to i64
1717
// CHECK-SANITIZE-NEXT: %[[OFFSETPTR:.*]] = sub i64 %[[PTRINT]], 42
18-
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[OFFSETPTR]], 1073741823
18+
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[OFFSETPTR]], 2147483647
1919
// CHECK-SANITIZE-NEXT: %[[MASKCOND:.*]] = icmp eq i64 %[[MASKEDPTR]], 0
2020
// CHECK-SANITIZE-NEXT: %[[PTRINT_DUP:.*]] = ptrtoint i8* %[[BITCAST]] to i64, !nosanitize
2121
// CHECK-SANITIZE-NEXT: br i1 %[[MASKCOND]], label %[[CONT:.*]], label %[[HANDLER_ALIGNMENT_ASSUMPTION:[^,]+]],{{.*}} !nosanitize
2222
// CHECK-SANITIZE: [[HANDLER_ALIGNMENT_ASSUMPTION]]:
23-
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 42){{.*}}, !nosanitize
24-
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 42){{.*}}, !nosanitize
23+
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 42){{.*}}, !nosanitize
24+
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 42){{.*}}, !nosanitize
2525
// CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
2626
// CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
2727
// CHECK-SANITIZE: [[CONT]]:
28-
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 1073741824, i64 42) ]
28+
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 2147483648, i64 42) ]
2929
// CHECK-NEXT: ret i8* %[[BITCAST]]
3030
// CHECK-NEXT: }
3131
#line 100

‎clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-two-params.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,17 @@ void *caller(char **x) {
1414
// CHECK-NEXT: %[[X_RELOADED:.*]] = load i8**, i8*** %[[X_ADDR]], align 8
1515
// CHECK-NEXT: %[[BITCAST:.*]] = bitcast i8** %[[X_RELOADED]] to i8*
1616
// CHECK-SANITIZE-NEXT: %[[PTRINT:.*]] = ptrtoint i8* %[[BITCAST]] to i64
17-
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[PTRINT]], 1073741823
17+
// CHECK-SANITIZE-NEXT: %[[MASKEDPTR:.*]] = and i64 %[[PTRINT]], 2147483647
1818
// CHECK-SANITIZE-NEXT: %[[MASKCOND:.*]] = icmp eq i64 %[[MASKEDPTR]], 0
1919
// CHECK-SANITIZE-NEXT: %[[PTRINT_DUP:.*]] = ptrtoint i8* %[[BITCAST]] to i64, !nosanitize
2020
// CHECK-SANITIZE-NEXT: br i1 %[[MASKCOND]], label %[[CONT:.*]], label %[[HANDLER_ALIGNMENT_ASSUMPTION:[^,]+]],{{.*}} !nosanitize
2121
// CHECK-SANITIZE: [[HANDLER_ALIGNMENT_ASSUMPTION]]:
22-
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 0){{.*}}, !nosanitize
23-
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 0){{.*}}, !nosanitize
22+
// CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
23+
// CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
2424
// CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
2525
// CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
2626
// CHECK-SANITIZE: [[CONT]]:
27-
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 1073741824) ]
27+
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 2147483648) ]
2828
// CHECK-NEXT: ret i8* %[[BITCAST]]
2929
// CHECK-NEXT: }
3030
#line 100

‎llvm/include/llvm/Bitcode/BitcodeCommon.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,14 @@
1919
namespace llvm {
2020

2121
struct AllocaPackedValues {
22-
using Align = Bitfield::Element<unsigned, 0, 5>;
23-
using UsedWithInAlloca = Bitfield::Element<bool, Align::NextBit, 1>;
22+
// We increased the number of bits needed to represent alignment to be more
23+
// than 5, but to preserve backward compatibility we store the upper bits
24+
// separately.
25+
using AlignLower = Bitfield::Element<unsigned, 0, 5>;
26+
using UsedWithInAlloca = Bitfield::Element<bool, AlignLower::NextBit, 1>;
2427
using ExplicitType = Bitfield::Element<bool, UsedWithInAlloca::NextBit, 1>;
2528
using SwiftError = Bitfield::Element<bool, ExplicitType::NextBit, 1>;
29+
using AlignUpper = Bitfield::Element<unsigned, SwiftError::NextBit, 3>;
2630
};
2731

2832
} // namespace llvm

0 commit comments

Comments
 (0)