Skip to content

Commit 55e56c1

Browse files
uweiganduweigand
authored
[s390x] Use VCodeConstant literal pool (#10756)
This switches s390x to use the common-code VCodeConstant literal pool instead of constants inlined into the instruction stream.
1 parent f6775a3 commit 55e56c1

22 files changed

+1048
-934
lines changed

‎cranelift/codegen/src/isa/s390x/inst.isle‎

Lines changed: 30 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -552,21 +552,6 @@
552552
(rn Reg)
553553
(rm Reg))
554554

555-
;; Load floating-point constant, half-precision (16 bit).
556-
(LoadFpuConst16
557-
(rd WritableReg)
558-
(const_data u16))
559-
560-
;; Load floating-point constant, single-precision (32 bit).
561-
(LoadFpuConst32
562-
(rd WritableReg)
563-
(const_data u32))
564-
565-
;; Load floating-point constant, double-precision (64 bit).
566-
(LoadFpuConst64
567-
(rd WritableReg)
568-
(const_data u64))
569-
570555
;; A binary vector operation with two vector register sources.
571556
(VecRRR
572557
(op VecBinaryOp)
@@ -765,17 +750,6 @@
765750
(rn Reg)
766751
(rm Reg))
767752

768-
;; Load 128-bit (big-endian) vector constant.
769-
(VecLoadConst
770-
(rd WritableReg)
771-
(const_data u128))
772-
773-
;; Load 128-bit (big-endian) replicated vector constant.
774-
(VecLoadConstReplicate
775-
(size u32)
776-
(rd WritableReg)
777-
(const_data u64))
778-
779753
;; Load vector immediate generated via byte mask.
780754
(VecImmByteMask
781755
(rd WritableReg)
@@ -878,6 +852,13 @@
878852
(imm i16)
879853
(lane_imm u8))
880854

855+
;; Same as VecInsertLaneImm, but allow undefined input VR.
856+
(VecInsertLaneImmUndef
857+
(size u32)
858+
(rd WritableReg)
859+
(imm i16)
860+
(lane_imm u8))
861+
881862
;; Vector lane replication with a VR source, a VR destination,
882863
;; and an immediate as lane index.
883864
(VecReplicateLane
@@ -1787,6 +1768,9 @@
17871768
(decl memarg_got () MemArg)
17881769
(extern constructor memarg_got memarg_got)
17891770

1771+
(decl memarg_const (VCodeConstant) MemArg)
1772+
(extern constructor memarg_const memarg_const)
1773+
17901774
;; Form the sum of two offset values, and check that the result is
17911775
;; a valid `MemArg::Symbol` offset (i.e. is even and fits into i32).
17921776
(decl pure partial memarg_symbol_offset_sum (i64 i64) i32)
@@ -2544,20 +2528,6 @@
25442528
(_ Unit (emit (MInst.MovToVec128 dst src1 src2))))
25452529
dst))
25462530

2547-
;; Helper for emitting `MInst.VecLoadConst` instructions.
2548-
(decl vec_load_const (Type u128) Reg)
2549-
(rule (vec_load_const (vr128_ty ty) n)
2550-
(let ((dst WritableReg (temp_writable_reg ty))
2551-
(_ Unit (emit (MInst.VecLoadConst dst n))))
2552-
dst))
2553-
2554-
;; Helper for emitting `MInst.VecLoadConstReplicate` instructions.
2555-
(decl vec_load_const_replicate (Type u64) Reg)
2556-
(rule (vec_load_const_replicate ty @ (multi_lane size _) n)
2557-
(let ((dst WritableReg (temp_writable_reg ty))
2558-
(_ Unit (emit (MInst.VecLoadConstReplicate size dst n))))
2559-
dst))
2560-
25612531
;; Helper for emitting `MInst.VecImmByteMask` instructions.
25622532
(decl vec_imm_byte_mask (Type u16) Reg)
25632533
(rule (vec_imm_byte_mask (vr128_ty ty) n)
@@ -2645,6 +2615,13 @@
26452615
(_ Unit (emit (MInst.VecInsertLaneImm size dst src imm lane_imm))))
26462616
dst))
26472617

2618+
;; Helper for emitting `MInst.VecInsertLaneImmUndef` instructions.
2619+
(decl vec_insert_lane_imm_undef (Type i16 u8) Reg)
2620+
(rule (vec_insert_lane_imm_undef ty @ (multi_lane size _) imm lane_imm)
2621+
(let ((dst WritableReg (temp_writable_reg ty))
2622+
(_ Unit (emit (MInst.VecInsertLaneImmUndef size dst imm lane_imm))))
2623+
dst))
2624+
26482625
;; Helper for emitting `MInst.VecReplicateLane` instructions.
26492626
(decl vec_replicate_lane (Type Reg u8) Reg)
26502627
(rule (vec_replicate_lane ty @ (multi_lane size _) src lane_imm)
@@ -2902,26 +2879,21 @@
29022879
(_ Unit (emit (MInst.Insert64UImm32Shifted dst src n))))
29032880
dst))
29042881

2905-
;; 16-bit floating-point type, any value. Loaded from literal pool.
2906-
;; TODO: use LZER to load 0.0
2882+
;; 16-bit floating-point type, any value.
29072883
(rule 8 (imm $F16 n)
2908-
(let ((dst WritableReg (temp_writable_reg $F16))
2909-
(_ Unit (emit (MInst.LoadFpuConst16 dst (u64_as_u16 n)))))
2910-
dst))
2884+
(vec_insert_lane_imm_undef $F16X8 (u64_as_i16 n) 0))
29112885

29122886
;; 32-bit floating-point type, any value. Loaded from literal pool.
2887+
;; FIXME: This wastes 4 bytes of constant pool space.
29132888
;; TODO: use LZER to load 0.0
29142889
(rule 8 (imm $F32 n)
2915-
(let ((dst WritableReg (temp_writable_reg $F32))
2916-
(_ Unit (emit (MInst.LoadFpuConst32 dst (u64_truncate_to_u32 n)))))
2917-
dst))
2890+
(vec_load_lane_undef $F32X4
2891+
(memarg_const (emit_u64_be_const (u32_pair (u64_truncate_to_u32 n) 0))) 0))
29182892

29192893
;; 64-bit floating-point type, any value. Loaded from literal pool.
29202894
;; TODO: use LZDR to load 0.0
29212895
(rule 8 (imm $F64 n)
2922-
(let ((dst WritableReg (temp_writable_reg $F64))
2923-
(_ Unit (emit (MInst.LoadFpuConst64 dst n))))
2924-
dst))
2896+
(vec_load_lane_undef $F64X2 (memarg_const (emit_u64_be_const n)) 0))
29252897

29262898
;; Variant used for negative constants.
29272899
(decl imm32 (Type i32) Reg)
@@ -2937,7 +2909,7 @@
29372909
(rule 1 (vec_imm (vr128_ty ty) (u64_pair n n))
29382910
(vec_imm_splat $I64X2 n))
29392911
(rule (vec_imm (vr128_ty ty) n)
2940-
(vec_load_const ty n))
2912+
(vec_load ty (memarg_const (emit_u128_be_const n))))
29412913

29422914
;; Variant with replicated immediate.
29432915
(decl vec_imm_splat (Type u64) Reg)
@@ -2957,9 +2929,12 @@
29572929
(vec_imm_splat $I16X8 (u16_as_u64 n)))
29582930
(rule 3 (vec_imm_splat (multi_lane 64 _) (u32_pair n n))
29592931
(vec_imm_splat $I32X4 (u32_as_u64 n)))
2960-
(rule 0 (vec_imm_splat (ty_vec128 ty) n)
2961-
(vec_load_const_replicate ty n))
2962-
2932+
;; FIXME: This wastes 4 bytes of constant pool space.
2933+
(rule 0 (vec_imm_splat ty @ (multi_lane 32 _) n)
2934+
(vec_load_replicate ty
2935+
(memarg_const (emit_u64_be_const (u32_pair (u64_truncate_to_u32 n) 0)))))
2936+
(rule 0 (vec_imm_splat ty @ (multi_lane 64 _) n)
2937+
(vec_load_replicate ty (memarg_const (emit_u64_be_const n))))
29632938

29642939
;; Helpers for generating extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
29652940

‎cranelift/codegen/src/isa/s390x/inst/args.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ pub enum MemArg {
3232
/// PC-relative Reference to a label.
3333
Label { target: MachLabel },
3434

35+
/// PC-relative Reference to a constant pool entry.
36+
Constant { constant: VCodeConstant },
37+
3538
/// PC-relative Reference to a near symbol.
3639
Symbol {
3740
name: Box<ExternalName>,
@@ -103,6 +106,7 @@ impl MemArg {
103106
MemArg::BXD20 { flags, .. } => *flags,
104107
MemArg::RegOffset { flags, .. } => *flags,
105108
MemArg::Label { .. } => MemFlags::trusted(),
109+
MemArg::Constant { .. } => MemFlags::trusted(),
106110
MemArg::Symbol { flags, .. } => *flags,
107111
MemArg::InitialSPOffset { .. } => MemFlags::trusted(),
108112
MemArg::IncomingArgOffset { .. } => MemFlags::trusted(),
@@ -226,6 +230,7 @@ impl PrettyPrint for MemArg {
226230
}
227231
}
228232
&MemArg::Label { target } => target.to_string(),
233+
&MemArg::Constant { constant } => format!("[const({})]", constant.as_u32()),
229234
&MemArg::Symbol {
230235
ref name, offset, ..
231236
} => format!("{} + {}", name.display(None), offset),

‎cranelift/codegen/src/isa/s390x/inst/emit.rs‎

Lines changed: 26 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,8 @@ pub fn mem_finalize(
151151

152152
// If this addressing mode cannot be handled by the instruction, use load-address.
153153
let need_load_address = match &mem {
154-
&MemArg::Label { .. } | &MemArg::Symbol { .. } if !mi.have_pcrel => true,
154+
&MemArg::Label { .. } | &MemArg::Constant { .. } if !mi.have_pcrel => true,
155+
&MemArg::Symbol { .. } if !mi.have_pcrel => true,
155156
&MemArg::Symbol { flags, .. } if !mi.have_unaligned_pcrel && !flags.aligned() => true,
156157
&MemArg::BXD20 { .. } if !mi.have_d20 => true,
157158
&MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !mi.have_index => {
@@ -243,6 +244,11 @@ pub fn mem_emit(
243244
sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
244245
put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
245246
}
247+
&MemArg::Constant { constant } => {
248+
let target = sink.get_label_for_constant(constant);
249+
sink.use_label_at_offset(sink.cur_offset(), target, LabelUse::BranchRIL);
250+
put(sink, &enc_ril_b(opcode_ril.unwrap(), rd, 0));
251+
}
246252
&MemArg::Symbol {
247253
ref name, offset, ..
248254
} => {
@@ -2386,42 +2392,6 @@ impl Inst {
23862392
put(sink, &enc_vrr_a(OPCODE_VLR, rd.to_reg(), rm, 0, 0, 0));
23872393
}
23882394
}
2389-
&Inst::LoadFpuConst16 { rd, const_data } => {
2390-
let reg = writable_spilltmp_reg().to_reg();
2391-
put(sink, &enc_ri_b(OPCODE_BRAS, reg, 6));
2392-
sink.put2(const_data.swap_bytes());
2393-
let inst = Inst::VecLoadLaneUndef {
2394-
size: 16,
2395-
rd,
2396-
mem: MemArg::reg(reg, MemFlags::trusted()),
2397-
lane_imm: 0,
2398-
};
2399-
inst.emit(sink, emit_info, state);
2400-
}
2401-
&Inst::LoadFpuConst32 { rd, const_data } => {
2402-
let reg = writable_spilltmp_reg().to_reg();
2403-
put(sink, &enc_ri_b(OPCODE_BRAS, reg, 8));
2404-
sink.put4(const_data.swap_bytes());
2405-
let inst = Inst::VecLoadLaneUndef {
2406-
size: 32,
2407-
rd,
2408-
mem: MemArg::reg(reg, MemFlags::trusted()),
2409-
lane_imm: 0,
2410-
};
2411-
inst.emit(sink, emit_info, state);
2412-
}
2413-
&Inst::LoadFpuConst64 { rd, const_data } => {
2414-
let reg = writable_spilltmp_reg().to_reg();
2415-
put(sink, &enc_ri_b(OPCODE_BRAS, reg, 12));
2416-
sink.put8(const_data.swap_bytes());
2417-
let inst = Inst::VecLoadLaneUndef {
2418-
size: 64,
2419-
rd,
2420-
mem: MemArg::reg(reg, MemFlags::trusted()),
2421-
lane_imm: 0,
2422-
};
2423-
inst.emit(sink, emit_info, state);
2424-
}
24252395
&Inst::FpuRR { fpu_op, rd, rn } => {
24262396
let (opcode, m3, m4, m5, opcode_fpr) = match fpu_op {
24272397
FPUOp1::Abs32 => (0xe7cc, 2, 8, 2, Some(0xb300)), // WFPSO, LPEBR
@@ -2909,35 +2879,6 @@ impl Inst {
29092879
let opcode = 0xe762; // VLVGP
29102880
put(sink, &enc_vrr_f(opcode, rd.to_reg(), rn, rm));
29112881
}
2912-
&Inst::VecLoadConst { rd, const_data } => {
2913-
let reg = writable_spilltmp_reg().to_reg();
2914-
put(sink, &enc_ri_b(OPCODE_BRAS, reg, 20));
2915-
for i in const_data.to_be_bytes().iter() {
2916-
sink.put1(*i);
2917-
}
2918-
let inst = Inst::VecLoad {
2919-
rd,
2920-
mem: MemArg::reg(reg, MemFlags::trusted()),
2921-
};
2922-
inst.emit(sink, emit_info, state);
2923-
}
2924-
&Inst::VecLoadConstReplicate {
2925-
size,
2926-
rd,
2927-
const_data,
2928-
} => {
2929-
let reg = writable_spilltmp_reg().to_reg();
2930-
put(sink, &enc_ri_b(OPCODE_BRAS, reg, (4 + size / 8) as i32));
2931-
for i in 0..size / 8 {
2932-
sink.put1((const_data >> (size - 8 - 8 * i)) as u8);
2933-
}
2934-
let inst = Inst::VecLoadReplicate {
2935-
size,
2936-
rd,
2937-
mem: MemArg::reg(reg, MemFlags::trusted()),
2938-
};
2939-
inst.emit(sink, emit_info, state);
2940-
}
29412882
&Inst::VecImmByteMask { rd, mask } => {
29422883
let opcode = 0xe744; // VGBM
29432884
put(sink, &enc_vri_a(opcode, rd.to_reg(), mask, 0));
@@ -3178,7 +3119,25 @@ impl Inst {
31783119

31793120
let opcode = match size {
31803121
8 => 0xe740, // VLEIB
3181-
16 => 0xe741, // LEIVH
3122+
16 => 0xe741, // VLEIH
3123+
32 => 0xe743, // VLEIF
3124+
64 => 0xe742, // VLEIG
3125+
_ => unreachable!(),
3126+
};
3127+
put(
3128+
sink,
3129+
&enc_vri_a(opcode, rd.to_reg(), imm as u16, lane_imm.into()),
3130+
);
3131+
}
3132+
&Inst::VecInsertLaneImmUndef {
3133+
size,
3134+
rd,
3135+
imm,
3136+
lane_imm,
3137+
} => {
3138+
let opcode = match size {
3139+
8 => 0xe740, // VLEIB
3140+
16 => 0xe741, // VLEIH
31823141
32 => 0xe743, // VLEIF
31833142
64 => 0xe742, // VLEIG
31843143
_ => unreachable!(),

0 commit comments

Comments
 (0)