Skip to content

Commit e0be1a0

Browse files
borsbors
committed
Auto merge of rust-lang#137271 - nikic:gep-nuw-2, r=scottmcm
Emit getelementptr inbounds nuw for pointer::add() Lower pointer::add (via intrinsic::offset with unsigned offset) to getelementptr inbounds nuw on LLVM versions that support it. This lets LLVM make use of the pre-condition that the offset addition does not wrap in an unsigned sense. Together with inbounds, this also implies that the offset is non-negative. Fixes rust-lang#137217.
2 parents 9af8985 + 9e7b184 commit e0be1a0

File tree

10 files changed

+93
-29
lines changed

10 files changed

+93
-29
lines changed

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ use tracing::{debug, instrument};
3131
use crate::abi::FnAbiLlvmExt;
3232
use crate::common::Funclet;
3333
use crate::context::{CodegenCx, SimpleCx};
34-
use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, Metadata, True};
34+
use crate::llvm::{
35+
self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
36+
};
3537
use crate::type_::Type;
3638
use crate::type_of::LayoutLlvmExt;
3739
use crate::value::Value;
@@ -910,13 +912,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
910912

911913
fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
912914
unsafe {
913-
llvm::LLVMBuildGEP2(
915+
llvm::LLVMBuildGEPWithNoWrapFlags(
914916
self.llbuilder,
915917
ty,
916918
ptr,
917919
indices.as_ptr(),
918920
indices.len() as c_uint,
919921
UNNAMED,
922+
GEPNoWrapFlags::default(),
920923
)
921924
}
922925
}
@@ -928,13 +931,33 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
928931
indices: &[&'ll Value],
929932
) -> &'ll Value {
930933
unsafe {
931-
llvm::LLVMBuildInBoundsGEP2(
934+
llvm::LLVMBuildGEPWithNoWrapFlags(
935+
self.llbuilder,
936+
ty,
937+
ptr,
938+
indices.as_ptr(),
939+
indices.len() as c_uint,
940+
UNNAMED,
941+
GEPNoWrapFlags::InBounds,
942+
)
943+
}
944+
}
945+
946+
fn inbounds_nuw_gep(
947+
&mut self,
948+
ty: &'ll Type,
949+
ptr: &'ll Value,
950+
indices: &[&'ll Value],
951+
) -> &'ll Value {
952+
unsafe {
953+
llvm::LLVMBuildGEPWithNoWrapFlags(
932954
self.llbuilder,
933955
ty,
934956
ptr,
935957
indices.as_ptr(),
936958
indices.len() as c_uint,
937959
UNNAMED,
960+
GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
938961
)
939962
}
940963
}

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -954,6 +954,17 @@ bitflags! {
954954
}
955955
}
956956

957+
// These values **must** match with LLVMGEPNoWrapFlags
958+
bitflags! {
959+
#[repr(transparent)]
960+
#[derive(Default)]
961+
pub struct GEPNoWrapFlags : c_uint {
962+
const InBounds = 1 << 0;
963+
const NUSW = 1 << 1;
964+
const NUW = 1 << 2;
965+
}
966+
}
967+
957968
unsafe extern "C" {
958969
pub type ModuleBuffer;
959970
}
@@ -1454,21 +1465,14 @@ unsafe extern "C" {
14541465

14551466
pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value;
14561467

1457-
pub(crate) fn LLVMBuildGEP2<'a>(
1458-
B: &Builder<'a>,
1459-
Ty: &'a Type,
1460-
Pointer: &'a Value,
1461-
Indices: *const &'a Value,
1462-
NumIndices: c_uint,
1463-
Name: *const c_char,
1464-
) -> &'a Value;
1465-
pub(crate) fn LLVMBuildInBoundsGEP2<'a>(
1468+
pub(crate) fn LLVMBuildGEPWithNoWrapFlags<'a>(
14661469
B: &Builder<'a>,
14671470
Ty: &'a Type,
14681471
Pointer: &'a Value,
14691472
Indices: *const &'a Value,
14701473
NumIndices: c_uint,
14711474
Name: *const c_char,
1475+
Flags: GEPNoWrapFlags,
14721476
) -> &'a Value;
14731477

14741478
// Casts

‎compiler/rustc_codegen_ssa/src/mir/place.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
423423
layout.size
424424
};
425425

426-
let llval = bx.inbounds_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]);
426+
let llval = bx.inbounds_nuw_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]);
427427
let align = self.val.align.restrict_for_offset(offset);
428428
PlaceValue::new_sized(llval, align).with_type(layout)
429429
}

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -666,9 +666,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
666666
lhs.layout.ty,
667667
),
668668

669-
(OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => {
670-
self.codegen_scalar_binop(bx, op, lhs_val, rhs_val, lhs.layout.ty)
671-
}
669+
(OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => self
670+
.codegen_scalar_binop(
671+
bx,
672+
op,
673+
lhs_val,
674+
rhs_val,
675+
lhs.layout.ty,
676+
rhs.layout.ty,
677+
),
672678

673679
_ => bug!(),
674680
};
@@ -889,10 +895,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
889895
op: mir::BinOp,
890896
lhs: Bx::Value,
891897
rhs: Bx::Value,
892-
input_ty: Ty<'tcx>,
898+
lhs_ty: Ty<'tcx>,
899+
rhs_ty: Ty<'tcx>,
893900
) -> Bx::Value {
894-
let is_float = input_ty.is_floating_point();
895-
let is_signed = input_ty.is_signed();
901+
let is_float = lhs_ty.is_floating_point();
902+
let is_signed = lhs_ty.is_signed();
896903
match op {
897904
mir::BinOp::Add => {
898905
if is_float {
@@ -958,17 +965,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
958965
mir::BinOp::BitAnd => bx.and(lhs, rhs),
959966
mir::BinOp::BitXor => bx.xor(lhs, rhs),
960967
mir::BinOp::Offset => {
961-
let pointee_type = input_ty
968+
let pointee_type = lhs_ty
962969
.builtin_deref(true)
963-
.unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty));
970+
.unwrap_or_else(|| bug!("deref of non-pointer {:?}", lhs_ty));
964971
let pointee_layout = bx.cx().layout_of(pointee_type);
965972
if pointee_layout.is_zst() {
966973
// `Offset` works in terms of the size of pointee,
967974
// so offsetting a pointer to ZST is a noop.
968975
lhs
969976
} else {
970977
let llty = bx.cx().backend_type(pointee_layout);
971-
bx.inbounds_gep(llty, lhs, &[rhs])
978+
if !rhs_ty.is_signed() {
979+
bx.inbounds_nuw_gep(llty, lhs, &[rhs])
980+
} else {
981+
bx.inbounds_gep(llty, lhs, &[rhs])
982+
}
972983
}
973984
}
974985
mir::BinOp::Shl | mir::BinOp::ShlUnchecked => {

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,14 @@ pub trait BuilderMethods<'a, 'tcx>:
325325
ptr: Self::Value,
326326
indices: &[Self::Value],
327327
) -> Self::Value;
328+
fn inbounds_nuw_gep(
329+
&mut self,
330+
ty: Self::Type,
331+
ptr: Self::Value,
332+
indices: &[Self::Value],
333+
) -> Self::Value {
334+
self.inbounds_gep(ty, ptr, indices)
335+
}
328336
fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
329337
self.gep(self.cx().type_i8(), ptr, &[offset])
330338
}

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,6 +1767,24 @@ extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS,
17671767
return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS), unwrap(RHS)));
17681768
}
17691769

1770+
#if LLVM_VERSION_LT(19, 0)
1771+
enum {
1772+
LLVMGEPFlagInBounds = (1 << 0),
1773+
LLVMGEPFlagNUSW = (1 << 1),
1774+
LLVMGEPFlagNUW = (1 << 2),
1775+
};
1776+
extern "C" LLVMValueRef
1777+
LLVMBuildGEPWithNoWrapFlags(LLVMBuilderRef B, LLVMTypeRef Ty,
1778+
LLVMValueRef Pointer, LLVMValueRef *Indices,
1779+
unsigned NumIndices, const char *Name,
1780+
unsigned NoWrapFlags) {
1781+
if (NoWrapFlags & LLVMGEPFlagInBounds)
1782+
return LLVMBuildInBoundsGEP2(B, Ty, Pointer, Indices, NumIndices, Name);
1783+
else
1784+
return LLVMBuildGEP2(B, Ty, Pointer, Indices, NumIndices, Name);
1785+
}
1786+
#endif
1787+
17701788
// Transfers ownership of DiagnosticHandler unique_ptr to the caller.
17711789
extern "C" DiagnosticHandler *
17721790
LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) {

‎tests/codegen/gep-index.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,27 @@ struct Foo(i32, i32);
1111
// CHECK-LABEL: @index_on_struct(
1212
#[no_mangle]
1313
fn index_on_struct(a: &[Foo], index: usize) -> &Foo {
14-
// CHECK: getelementptr inbounds %Foo, ptr %a.0, {{i64|i32}} %index
14+
// CHECK: getelementptr inbounds{{( nuw)?}} %Foo, ptr %a.0, {{i64|i32}} %index
1515
&a[index]
1616
}
1717

1818
// CHECK-LABEL: @offset_on_struct(
1919
#[no_mangle]
2020
fn offset_on_struct(a: *const Foo, index: usize) -> *const Foo {
21-
// CHECK: getelementptr inbounds %Foo, ptr %a, {{i64|i32}} %index
21+
// CHECK: getelementptr inbounds{{( nuw)?}} %Foo, ptr %a, {{i64|i32}} %index
2222
unsafe { a.add(index) }
2323
}
2424

2525
// CHECK-LABEL: @index_on_i32(
2626
#[no_mangle]
2727
fn index_on_i32(a: &[i32], index: usize) -> &i32 {
28-
// CHECK: getelementptr inbounds i32, ptr %a.0, {{i64|i32}} %index
28+
// CHECK: getelementptr inbounds{{( nuw)?}} i32, ptr %a.0, {{i64|i32}} %index
2929
&a[index]
3030
}
3131

3232
// CHECK-LABEL: @offset_on_i32(
3333
#[no_mangle]
3434
fn offset_on_i32(a: *const i32, index: usize) -> *const i32 {
35-
// CHECK: getelementptr inbounds i32, ptr %a, {{i64|i32}} %index
35+
// CHECK: getelementptr inbounds{{( nuw)?}} i32, ptr %a, {{i64|i32}} %index
3636
unsafe { a.add(index) }
3737
}

‎tests/codegen/intrinsics/offset.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
2727
// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
2828
#[no_mangle]
2929
pub unsafe fn offset_usize(p: *const u64, d: usize) -> *const u64 {
30-
// CHECK: %[[R:.*]] = getelementptr inbounds i64, ptr %p, [[SIZE]] %d
30+
// CHECK: %[[R:.*]] = getelementptr inbounds{{( nuw)?}} i64, ptr %p, [[SIZE]] %d
3131
// CHECK-NEXT: ret ptr %[[R]]
3232
offset(p, d)
3333
}

‎tests/codegen/intrinsics/ptr_metadata.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ pub unsafe fn dyn_byte_offset(
2828
p: *const dyn std::fmt::Debug,
2929
n: usize,
3030
) -> *const dyn std::fmt::Debug {
31-
// CHECK: %[[Q:.+]] = getelementptr inbounds i8, ptr %p.0, i64 %n
31+
// CHECK: %[[Q:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %p.0, i64 %n
3232
// CHECK: %[[TEMP1:.+]] = insertvalue { ptr, ptr } poison, ptr %[[Q]], 0
3333
// CHECK: %[[TEMP2:.+]] = insertvalue { ptr, ptr } %[[TEMP1]], ptr %p.1, 1
3434
// CHECK: ret { ptr, ptr } %[[TEMP2]]

‎tests/codegen/ptr-arithmetic.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
// CHECK-SAME: [[WORD:i[0-9]+]] noundef %n)
77
#[no_mangle]
88
pub unsafe fn i32_add(p: *const i32, n: usize) -> *const i32 {
9-
// CHECK: %[[TEMP:.+]] = getelementptr inbounds i32, ptr %p, [[WORD]] %n
9+
// CHECK: %[[TEMP:.+]] = getelementptr inbounds{{( nuw)?}} i32, ptr %p, [[WORD]] %n
1010
// CHECK: ret ptr %[[TEMP]]
1111
p.add(n)
1212
}

0 commit comments

Comments
 (0)