Skip to content

Commit ab92564

Browse files
borsbors
committed
Auto merge of rust-lang#147950 - durin42:llvm-22-intrinsics-changes, r=nikic
rustc_codegen_llvm: adapt for LLVM 22 change to pass masked intrinsic alignment as an attribute This was a bit more invasive than I had kind of hoped. An alternate approach would be to add an extra call_intrinsic_with_attrs() that would have the new-in-this-change signature for call_intrinsic, but this felt about equivalent and made it a little easier to audit the relevant callsites of call_intrinsic(). Related LLVM change is llvm/llvm-project#163802. `@rustbot` label llvm-main
2 parents 8aab621 + e3e342a commit ab92564

File tree

5 files changed

+113
-35
lines changed

5 files changed

+113
-35
lines changed

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 73 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1325,6 +1325,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
13251325
};
13261326
}
13271327

1328+
let llvm_version = crate::llvm_util::get_version();
1329+
13281330
/// Converts a vector mask, where each element has a bit width equal to the data elements it is used with,
13291331
/// down to an i1 based mask that can be used by llvm intrinsics.
13301332
///
@@ -1808,7 +1810,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18081810
);
18091811

18101812
// Alignment of T, must be a constant integer value:
1811-
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
1813+
let alignment = bx.align_of(in_elem).bytes();
18121814

18131815
// Truncate the mask vector to a vector of i1s:
18141816
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
@@ -1819,11 +1821,23 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18191821
// Type of the vector of elements:
18201822
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
18211823

1822-
return Ok(bx.call_intrinsic(
1823-
"llvm.masked.gather",
1824-
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
1825-
&[args[1].immediate(), alignment, mask, args[0].immediate()],
1826-
));
1824+
let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
1825+
let alignment = bx.const_i32(alignment as i32);
1826+
&[args[1].immediate(), alignment, mask, args[0].immediate()]
1827+
} else {
1828+
&[args[1].immediate(), mask, args[0].immediate()]
1829+
};
1830+
1831+
let call =
1832+
bx.call_intrinsic("llvm.masked.gather", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], args);
1833+
if llvm_version >= (22, 0, 0) {
1834+
crate::attributes::apply_to_callsite(
1835+
call,
1836+
crate::llvm::AttributePlace::Argument(0),
1837+
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
1838+
)
1839+
}
1840+
return Ok(call);
18271841
}
18281842

18291843
if name == sym::simd_masked_load {
@@ -1891,18 +1905,30 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
18911905
let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
18921906

18931907
// Alignment of T, must be a constant integer value:
1894-
let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1908+
let alignment = bx.align_of(values_elem).bytes();
18951909

18961910
let llvm_pointer = bx.type_ptr();
18971911

18981912
// Type of the vector of elements:
18991913
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
19001914

1901-
return Ok(bx.call_intrinsic(
1902-
"llvm.masked.load",
1903-
&[llvm_elem_vec_ty, llvm_pointer],
1904-
&[args[1].immediate(), alignment, mask, args[2].immediate()],
1905-
));
1915+
let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
1916+
let alignment = bx.const_i32(alignment as i32);
1917+
1918+
&[args[1].immediate(), alignment, mask, args[2].immediate()]
1919+
} else {
1920+
&[args[1].immediate(), mask, args[2].immediate()]
1921+
};
1922+
1923+
let call = bx.call_intrinsic("llvm.masked.load", &[llvm_elem_vec_ty, llvm_pointer], args);
1924+
if llvm_version >= (22, 0, 0) {
1925+
crate::attributes::apply_to_callsite(
1926+
call,
1927+
crate::llvm::AttributePlace::Argument(0),
1928+
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
1929+
)
1930+
}
1931+
return Ok(call);
19061932
}
19071933

19081934
if name == sym::simd_masked_store {
@@ -1964,18 +1990,29 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
19641990
let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
19651991

19661992
// Alignment of T, must be a constant integer value:
1967-
let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
1993+
let alignment = bx.align_of(values_elem).bytes();
19681994

19691995
let llvm_pointer = bx.type_ptr();
19701996

19711997
// Type of the vector of elements:
19721998
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
19731999

1974-
return Ok(bx.call_intrinsic(
1975-
"llvm.masked.store",
1976-
&[llvm_elem_vec_ty, llvm_pointer],
1977-
&[args[2].immediate(), args[1].immediate(), alignment, mask],
1978-
));
2000+
let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2001+
let alignment = bx.const_i32(alignment as i32);
2002+
&[args[2].immediate(), args[1].immediate(), alignment, mask]
2003+
} else {
2004+
&[args[2].immediate(), args[1].immediate(), mask]
2005+
};
2006+
2007+
let call = bx.call_intrinsic("llvm.masked.store", &[llvm_elem_vec_ty, llvm_pointer], args);
2008+
if llvm_version >= (22, 0, 0) {
2009+
crate::attributes::apply_to_callsite(
2010+
call,
2011+
crate::llvm::AttributePlace::Argument(1),
2012+
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2013+
)
2014+
}
2015+
return Ok(call);
19792016
}
19802017

19812018
if name == sym::simd_scatter {
@@ -2040,7 +2077,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
20402077
);
20412078

20422079
// Alignment of T, must be a constant integer value:
2043-
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
2080+
let alignment = bx.align_of(in_elem).bytes();
20442081

20452082
// Truncate the mask vector to a vector of i1s:
20462083
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
@@ -2050,12 +2087,25 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
20502087

20512088
// Type of the vector of elements:
20522089
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
2053-
2054-
return Ok(bx.call_intrinsic(
2090+
let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
2091+
let alignment = bx.const_i32(alignment as i32);
2092+
&[args[0].immediate(), args[1].immediate(), alignment, mask]
2093+
} else {
2094+
&[args[0].immediate(), args[1].immediate(), mask]
2095+
};
2096+
let call = bx.call_intrinsic(
20552097
"llvm.masked.scatter",
20562098
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
2057-
&[args[0].immediate(), args[1].immediate(), alignment, mask],
2058-
));
2099+
args,
2100+
);
2101+
if llvm_version >= (22, 0, 0) {
2102+
crate::attributes::apply_to_callsite(
2103+
call,
2104+
crate::llvm::AttributePlace::Argument(1),
2105+
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
2106+
)
2107+
}
2108+
return Ok(call);
20592109
}
20602110

20612111
macro_rules! arith_red {

‎tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-gather.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
//
22

33
//@ compile-flags: -C no-prepopulate-passes
4+
//@ revisions: LLVM21 LLVM22
5+
//@ [LLVM22] min-llvm-version: 22
6+
//@ [LLVM21] max-llvm-major-version: 21
7+
// ignore-tidy-linelength
48

59
#![crate_type = "lib"]
610
#![feature(repr_simd, core_intrinsics)]
@@ -24,7 +28,8 @@ pub unsafe fn gather_f32x2(
2428
) -> Vec2<f32> {
2529
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
2630
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
27-
// CHECK: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
31+
// LLVM21: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
32+
// LLVM22: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
2833
simd_gather(values, pointers, mask)
2934
}
3035

@@ -37,7 +42,8 @@ pub unsafe fn gather_f32x2_unsigned(
3742
) -> Vec2<f32> {
3843
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
3944
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
40-
// CHECK: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
45+
// LLVM21: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
46+
// LLVM22: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
4147
simd_gather(values, pointers, mask)
4248
}
4349

@@ -50,6 +56,7 @@ pub unsafe fn gather_pf32x2(
5056
) -> Vec2<*const f32> {
5157
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
5258
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
53-
// CHECK: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
59+
// LLVM21: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
60+
// LLVM22: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
5461
simd_gather(values, pointers, mask)
5562
}

‎tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
//@ compile-flags: -C no-prepopulate-passes
2+
//@ revisions: LLVM21 LLVM22
3+
//@ [LLVM22] min-llvm-version: 22
4+
//@ [LLVM21] max-llvm-major-version: 21
5+
// ignore-tidy-linelength
26

37
#![crate_type = "lib"]
48
#![feature(repr_simd, core_intrinsics)]
@@ -18,7 +22,8 @@ pub type Vec4<T> = Simd<T, 4>;
1822
pub unsafe fn load_f32x2(mask: Vec2<i32>, pointer: *const f32, values: Vec2<f32>) -> Vec2<f32> {
1923
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
2024
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
21-
// CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
25+
// LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
26+
// LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
2227
simd_masked_load(mask, pointer, values)
2328
}
2429

@@ -31,7 +36,8 @@ pub unsafe fn load_f32x2_unsigned(
3136
) -> Vec2<f32> {
3237
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
3338
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
34-
// CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
39+
// LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
40+
// LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
3541
simd_masked_load(mask, pointer, values)
3642
}
3743

@@ -44,6 +50,7 @@ pub unsafe fn load_pf32x4(
4450
) -> Vec4<*const f32> {
4551
// CHECK: [[A:%[0-9]+]] = lshr <4 x i32> {{.*}}, {{<i32 31, i32 31, i32 31, i32 31>|splat \(i32 31\)}}
4652
// CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
47-
// CHECK: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
53+
// LLVM21: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
54+
// LLVM22: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr align {{.*}} {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
4855
simd_masked_load(mask, pointer, values)
4956
}

‎tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
//@ compile-flags: -C no-prepopulate-passes
2+
//@ revisions: LLVM21 LLVM22
3+
//@ [LLVM22] min-llvm-version: 22
4+
//@ [LLVM21] max-llvm-major-version: 21
5+
// ignore-tidy-linelength
26

37
#![crate_type = "lib"]
48
#![feature(repr_simd, core_intrinsics)]
@@ -18,7 +22,8 @@ pub type Vec4<T> = Simd<T, 4>;
1822
pub unsafe fn store_f32x2(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>) {
1923
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
2024
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
21-
// CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
25+
// LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
26+
// LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
2227
simd_masked_store(mask, pointer, values)
2328
}
2429

@@ -27,7 +32,8 @@ pub unsafe fn store_f32x2(mask: Vec2<i32>, pointer: *mut f32, values: Vec2<f32>)
2732
pub unsafe fn store_f32x2_unsigned(mask: Vec2<u32>, pointer: *mut f32, values: Vec2<f32>) {
2833
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
2934
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
30-
// CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
35+
// LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
36+
// LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
3137
simd_masked_store(mask, pointer, values)
3238
}
3339

@@ -36,6 +42,7 @@ pub unsafe fn store_f32x2_unsigned(mask: Vec2<u32>, pointer: *mut f32, values: V
3642
pub unsafe fn store_pf32x4(mask: Vec4<i32>, pointer: *mut *const f32, values: Vec4<*const f32>) {
3743
// CHECK: [[A:%[0-9]+]] = lshr <4 x i32> {{.*}}, {{<i32 31, i32 31, i32 31, i32 31>|splat \(i32 31\)}}
3844
// CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
39-
// CHECK: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]])
45+
// LLVM21: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]])
46+
// LLVM22: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr align {{.*}} {{.*}}, <4 x i1> [[B]])
4047
simd_masked_store(mask, pointer, values)
4148
}

‎tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-scatter.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
//
22

33
//@ compile-flags: -C no-prepopulate-passes
4+
//@ revisions: LLVM21 LLVM22
5+
//@ [LLVM22] min-llvm-version: 22
6+
//@ [LLVM21] max-llvm-major-version: 21
7+
// ignore-tidy-linelength
48

59
#![crate_type = "lib"]
610
#![feature(repr_simd, core_intrinsics)]
@@ -20,7 +24,8 @@ pub type Vec4<T> = Simd<T, 4>;
2024
pub unsafe fn scatter_f32x2(pointers: Vec2<*mut f32>, mask: Vec2<i32>, values: Vec2<f32>) {
2125
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
2226
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
23-
// CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
27+
// LLVM21: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
28+
// LLVM22: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
2429
simd_scatter(values, pointers, mask)
2530
}
2631

@@ -29,7 +34,8 @@ pub unsafe fn scatter_f32x2(pointers: Vec2<*mut f32>, mask: Vec2<i32>, values: V
2934
pub unsafe fn scatter_f32x2_unsigned(pointers: Vec2<*mut f32>, mask: Vec2<u32>, values: Vec2<f32>) {
3035
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
3136
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
32-
// CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
37+
// LLVM21: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
38+
// LLVM22: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
3339
simd_scatter(values, pointers, mask)
3440
}
3541

@@ -42,6 +48,7 @@ pub unsafe fn scatter_pf32x2(
4248
) {
4349
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
4450
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
45-
// CHECK: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
51+
// LLVM21: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
52+
// LLVM22: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
4653
simd_scatter(values, pointers, mask)
4754
}

0 commit comments

Comments
 (0)