Skip to content

Commit 762d325

Browse files
zachs18zachs18
committed
Don't encode enum tag for uninhabited repr(Rust) enum variants
1 parent 9f60e66 commit 762d325

File tree

7 files changed

+99
-34
lines changed

7 files changed

+99
-34
lines changed

‎compiler/rustc_abi/src/layout.rs‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -842,11 +842,15 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
842842
let mut layout_variants = variants
843843
.iter_enumerated()
844844
.map(|(i, field_layouts)| {
845-
let mut st = self.univariant(
846-
field_layouts,
847-
repr,
848-
StructKind::Prefixed(min_ity.size(), prefix_align),
849-
)?;
845+
let uninhabited = field_layouts.iter().any(|f| f.is_uninhabited());
846+
// We don't need to encode the tag in uninhabited variants in repr(Rust) enums
847+
let struct_kind = if uninhabited && !repr.inhibit_enum_layout_opt() {
848+
StructKind::AlwaysSized
849+
} else {
850+
StructKind::Prefixed(min_ity.size(), prefix_align)
851+
};
852+
let mut st = self.univariant(field_layouts, repr, struct_kind)?;
853+
850854
st.variants = Variants::Single { index: i, variants: None };
851855
// Find the first field we can't move later
852856
// to make room for a larger discriminant.
@@ -916,6 +920,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
916920
let old_ity_size = min_ity.size();
917921
let new_ity_size = ity.size();
918922
for variant in &mut layout_variants {
923+
// Don't change field offsets of uninhabited variants in repr(Rust) enums,
924+
// they don't encode the tag and their fields may overlap with the tag.
925+
if variant.is_uninhabited() && !repr.inhibit_enum_layout_opt() {
926+
continue;
927+
}
919928
match variant.fields {
920929
FieldsShape::Arbitrary { ref mut offsets, .. } => {
921930
for i in offsets {
@@ -960,6 +969,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
960969
let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
961970
panic!("encountered a non-arbitrary layout during enum layout");
962971
};
972+
// Don't look in uninhabited variants for repr(Rust) enums, they will never be
973+
// passed over an ABI so they don't matter for the purpose of determining
974+
// BackendRepr.
975+
if layout_variant.is_uninhabited() && !repr.inhibit_enum_layout_opt() {
976+
continue;
977+
}
963978
// We skip *all* ZST here and later check if we are good in terms of alignment.
964979
// This lets us handle some cases involving aligned ZST.
965980
let mut fields = iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
@@ -1076,6 +1091,43 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
10761091
.map(|v| v.randomization_seed)
10771092
.fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
10781093

1094+
// If all variants are uninhabited, the repr does not inhibit layout optimizations,
1095+
// and all fields are ZSTs, then the tagged layout will not have room for the tag.
1096+
// So in this case, we return an uninhabited layout that is big enough and aligned
1097+
// enough for all variant fields, but do not say it has any fields itself.
1098+
// Doing this only when the layout is too small to fit the tag gives better error
1099+
// messages during const-eval in some cases, "constructing invalid value at .<enum-tag>:
1100+
// encountered an uninhabited enum variant" instead of "constructing invalid value:
1101+
// encountered a value of uninhabited type".
1102+
// Note the we only reach this case when there is at least one non-1-aligned ZST field,
1103+
// since the all-1-ZST case is handled by the "present_variants" check in
1104+
// `layout_of_struct_or_enum`.
1105+
if uninhabited && size < tag.size(&self.cx) {
1106+
// The only way for the size to be less than the tag's size is for it to be zero,
1107+
// which can only occur when the repr does not inhibit layout optimization.
1108+
debug_assert!(
1109+
size == Size::ZERO,
1110+
"size was non-zero but less than tag size: 0 < {size:?} < {:?}",
1111+
tag.size(&self.cx)
1112+
);
1113+
debug_assert!(
1114+
!repr.inhibit_enum_layout_opt(),
1115+
"enum size was zero with layout optimizations disabled"
1116+
);
1117+
return Ok(LayoutData {
1118+
fields: FieldsShape::Arbitrary { offsets: [].into(), memory_index: [].into() },
1119+
variants: Variants::Empty { variants: Some(layout_variants) },
1120+
backend_repr: BackendRepr::Memory { sized: true },
1121+
largest_niche: None,
1122+
uninhabited: true,
1123+
align: AbiAlign::new(align),
1124+
size,
1125+
max_repr_align,
1126+
unadjusted_abi_align,
1127+
randomization_seed: combined_seed,
1128+
});
1129+
}
1130+
10791131
let tagged_layout = LayoutData {
10801132
variants: Variants::Multiple {
10811133
tag,

‎tests/codegen-llvm/enum/enum-aggregate.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ fn make_uninhabited_err_indirectly(n: Never) -> Result<u32, Never> {
113113
fn make_fully_uninhabited_result(v: u32, n: Never) -> Result<(u32, Never), (Never, u32)> {
114114
// Actually reaching this would be UB, so we don't actually build a result.
115115

116-
// CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32{{( signext)?}} %v)
116+
// CHECK-LABEL: i32 @make_fully_uninhabited_result(i32{{( signext)?}} %v)
117117
// CHECK-NEXT: start:
118118
// CHECK-NEXT: call void @llvm.trap()
119119
// CHECK-NEXT: call void @llvm.trap()

‎tests/ui/consts/const-eval/ub-enum.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::transmute
9595

9696
// All variants are uninhabited but also have data.
9797
// Use `0` as constant to make behavior endianness-independent.
98-
const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) };
98+
const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u32) };
9999
//~^ ERROR uninhabited enum variant
100-
const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) };
100+
const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u32) };
101101
//~^ ERROR uninhabited enum variant
102102

103103
// All variants have same-size data but only one inhabited.

‎tests/ui/consts/const-eval/ub-enum.stderr‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,13 @@ LL | const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::tran
111111
error[E0080]: constructing invalid value at .<enum-tag>: encountered an uninhabited enum variant
112112
--> $DIR/ub-enum.rs:98:77
113113
|
114-
LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) };
114+
LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u32) };
115115
| ^^^^^^^^^^^^^^^^^^^^ evaluation of `BAD_UNINHABITED_WITH_DATA1` failed here
116116

117117
error[E0080]: constructing invalid value at .<enum-tag>: encountered an uninhabited enum variant
118118
--> $DIR/ub-enum.rs:100:77
119119
|
120-
LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) };
120+
LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u32) };
121121
| ^^^^^^^^^^^^^^^^^^^^ evaluation of `BAD_UNINHABITED_WITH_DATA2` failed here
122122

123123
error[E0080]: read discriminant of an uninhabited enum variant

‎tests/ui/layout/debug.stderr‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ LL | union EmptyUnion {}
55
| ^^^^^^^^^^^^^^^^^^^
66

77
error: layout_of(E) = Layout {
8-
size: Size(12 bytes),
8+
size: Size(8 bytes),
99
align: AbiAlign {
1010
abi: Align(4 bytes),
1111
},
@@ -65,23 +65,36 @@ error: layout_of(E) = Layout {
6565
randomization_seed: $SEED,
6666
},
6767
Layout {
68-
size: Size(12 bytes),
68+
size: Size(8 bytes),
6969
align: AbiAlign {
7070
abi: Align(4 bytes),
7171
},
72-
backend_repr: Memory {
73-
sized: true,
74-
},
72+
backend_repr: ScalarPair(
73+
Initialized {
74+
value: Int(
75+
I32,
76+
true,
77+
),
78+
valid_range: 0..=4294967295,
79+
},
80+
Initialized {
81+
value: Int(
82+
I32,
83+
true,
84+
),
85+
valid_range: 0..=4294967295,
86+
},
87+
),
7588
fields: Arbitrary {
7689
offsets: [
77-
Size(4 bytes),
78-
Size(4 bytes),
7990
Size(8 bytes),
91+
Size(0 bytes),
92+
Size(4 bytes),
8093
],
8194
memory_index: [
95+
2,
8296
0,
8397
1,
84-
2,
8598
],
8699
},
87100
largest_niche: None,
@@ -506,7 +519,7 @@ LL | union P3 { x: F32x4 }
506519
| ^^^^^^^^
507520

508521
error: layout_of(P4) = Layout {
509-
size: Size(12 bytes),
522+
size: Size(8 bytes),
510523
align: AbiAlign {
511524
abi: Align(1 bytes),
512525
},

‎tests/ui/layout/enum.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ enum UninhabitedVariantAlign { //~ERROR: abi: Align(2 bytes)
1212
}
1313

1414
#[rustc_layout(size)]
15-
enum UninhabitedVariantSpace { //~ERROR: size: Size(16 bytes)
15+
enum UninhabitedVariantSpace { //~ERROR: size: Size(15 bytes)
1616
A,
1717
B([u8; 15], !), // make sure there is space being reserved for this field.
1818
}
@@ -47,7 +47,7 @@ enum UninhabitedVariantUntagged { //~ERROR: size: Size(8 bytes)
4747
// the same size, but without a niche.
4848
#[rustc_layout(size, abi)]
4949
enum UninhabitedVariantUntaggedBigger { //~ERROR: size: Size(8 bytes)
50-
//~^ ERROR: abi: Memory
50+
//~^ ERROR: abi: ScalarPair
5151
A(i32),
5252
B([u8; 5], !),
5353
}
@@ -74,7 +74,7 @@ enum UninhabitedVariantLargeWithNiche {
7474
// This uses the tagged layout, but since all variants are uninhabited, none of them store the tag,
7575
// so we only need space for the fields, and the abi is Memory.
7676
#[rustc_layout(size, abi)]
77-
enum AllUninhabitedVariants { //~ERROR: size: Size(3 bytes)
77+
enum AllUninhabitedVariants { //~ERROR: size: Size(2 bytes)
7878
//~^ERROR: abi: Memory
7979
A(i8, bool, !),
8080
B(u8, u8, !),
@@ -103,15 +103,15 @@ enum TaggedI8 { //~ERROR: size: Size(4 bytes)
103103
// Tagged `(u16, i16)`
104104
#[rustc_layout(size, abi)]
105105
enum TaggedI16 { //~ERROR: size: Size(4 bytes)
106-
//~^ERROR: abi: Memory
106+
//~^ERROR: abi: ScalarPair
107107
A(i16),
108108
B(i8, i8, i8, AlignedNever)
109109
}
110110

111111
// This must not use tagged representation, since it's zero-sized.
112112
#[rustc_layout(size, abi)]
113-
enum AllUninhabitedVariantsAlignedZst { //~ERROR: size: Size(2 bytes)
114-
//~^ERROR: abi: Scalar
113+
enum AllUninhabitedVariantsAlignedZst { //~ERROR: size: Size(0 bytes)
114+
//~^ERROR: abi: Memory
115115
A(AlignedNever),
116116
B(AlignedNever),
117117
}

‎tests/ui/layout/enum.stderr‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ error: align: AbiAlign { abi: Align(2 bytes) }
44
LL | enum UninhabitedVariantAlign {
55
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
66

7-
error: size: Size(16 bytes)
7+
error: size: Size(15 bytes)
88
--> $DIR/enum.rs:15:1
99
|
1010
LL | enum UninhabitedVariantSpace {
@@ -46,7 +46,7 @@ error: size: Size(8 bytes)
4646
LL | enum UninhabitedVariantUntaggedBigger {
4747
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
4848

49-
error: abi: Memory { sized: true }
49+
error: abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=0 }, Initialized { value: Int(I32, true), valid_range: 0..=4294967295 })
5050
--> $DIR/enum.rs:49:1
5151
|
5252
LL | enum UninhabitedVariantUntaggedBigger {
@@ -65,7 +65,7 @@ LL | enum UninhabitedVariantWithNiche {
6565
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
6666

6767
error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
68-
size: Size(4 bytes),
68+
size: Size(3 bytes),
6969
align: AbiAlign {
7070
abi: Align(1 bytes),
7171
},
@@ -140,7 +140,7 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
140140
randomization_seed: 17394913183323368564,
141141
},
142142
Layout {
143-
size: Size(4 bytes),
143+
size: Size(3 bytes),
144144
align: AbiAlign {
145145
abi: Align(1 bytes),
146146
},
@@ -149,10 +149,10 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
149149
},
150150
fields: Arbitrary {
151151
offsets: [
152+
Size(0 bytes),
152153
Size(1 bytes),
153154
Size(2 bytes),
154155
Size(3 bytes),
155-
Size(4 bytes),
156156
],
157157
memory_index: [
158158
0,
@@ -182,7 +182,7 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
182182
LL | enum UninhabitedVariantLargeWithNiche {
183183
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
184184

185-
error: size: Size(3 bytes)
185+
error: size: Size(2 bytes)
186186
--> $DIR/enum.rs:77:1
187187
|
188188
LL | enum AllUninhabitedVariants {
@@ -224,19 +224,19 @@ error: size: Size(4 bytes)
224224
LL | enum TaggedI16 {
225225
| ^^^^^^^^^^^^^^
226226

227-
error: abi: Memory { sized: true }
227+
error: abi: ScalarPair(Initialized { value: Int(I16, false), valid_range: 0..=0 }, Initialized { value: Int(I16, true), valid_range: 0..=65535 })
228228
--> $DIR/enum.rs:105:1
229229
|
230230
LL | enum TaggedI16 {
231231
| ^^^^^^^^^^^^^^
232232

233-
error: size: Size(2 bytes)
233+
error: size: Size(0 bytes)
234234
--> $DIR/enum.rs:113:1
235235
|
236236
LL | enum AllUninhabitedVariantsAlignedZst {
237237
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
238238

239-
error: abi: Scalar(Initialized { value: Int(I16, false), valid_range: 0..=0 })
239+
error: abi: Memory { sized: true }
240240
--> $DIR/enum.rs:113:1
241241
|
242242
LL | enum AllUninhabitedVariantsAlignedZst {

0 commit comments

Comments
 (0)