Skip to content

Commit 33d9e4f

Browse files
d-sonugad-sonuga
committed
Remove over-constraint on available registers by removing clobbers only from the late available register set.
Allocate in order of late operands then early instead of def then use
1 parent 64b9757 commit 33d9e4f

File tree

2 files changed

+111
-89
lines changed

2 files changed

+111
-89
lines changed

‎src/fastalloc/iter.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::{Operand, OperandConstraint, OperandKind};
1+
use crate::{Operand, OperandConstraint, OperandKind, OperandPos};
22

33
pub struct Operands<'a>(pub &'a [Operand]);
44

@@ -37,6 +37,14 @@ impl<'a> Operands<'a> {
3737
pub fn any_reg(&self) -> impl Iterator<Item = (usize, Operand)> + 'a {
3838
self.matches(|op| matches!(op.constraint(), OperandConstraint::Reg))
3939
}
40+
41+
pub fn late(&self) -> impl Iterator<Item = (usize, Operand)> + 'a {
42+
self.matches(|op| op.pos() == OperandPos::Late)
43+
}
44+
45+
pub fn early(&self) -> impl Iterator<Item = (usize, Operand)> + 'a {
46+
self.matches(|op| op.pos() == OperandPos::Early)
47+
}
4048
}
4149

4250
impl<'a> core::ops::Index<usize> for Operands<'a> {

‎src/fastalloc/mod.rs‎

Lines changed: 102 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ impl<'a, F: Function> Env<'a, F> {
473473
self.available_pregs[OperandPos::Late].remove(preg);
474474
Ok(())
475475
} else {
476+
trace!("Can't get a scratch register for {class:?}");
476477
Err(RegAllocError::TooManyLiveRegs)
477478
}
478479
}
@@ -520,18 +521,21 @@ impl<'a, F: Function> Env<'a, F> {
520521
match (op.pos(), op.kind()) {
521522
(OperandPos::Early, OperandKind::Use) => {
522523
if op.as_fixed_nonallocatable().is_none() && !early_avail_pregs.contains(preg) {
524+
trace!("fixed {preg} for {op} isn't available");
523525
return Err(RegAllocError::TooManyLiveRegs);
524526
}
525527
self.available_pregs[OperandPos::Early].remove(preg);
526528
if self.reused_input_to_reuse_op[op_idx] != usize::MAX {
527529
if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) {
530+
trace!("fixed {preg} for {op} isn't available");
528531
return Err(RegAllocError::TooManyLiveRegs);
529532
}
530533
self.available_pregs[OperandPos::Late].remove(preg);
531534
}
532535
}
533536
(OperandPos::Late, OperandKind::Def) => {
534537
if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) {
538+
trace!("fixed {preg} for {op} isn't available");
535539
return Err(RegAllocError::TooManyLiveRegs);
536540
}
537541
self.available_pregs[OperandPos::Late].remove(preg);
@@ -540,6 +544,7 @@ impl<'a, F: Function> Env<'a, F> {
540544
if op.as_fixed_nonallocatable().is_none()
541545
&& (!early_avail_pregs.contains(preg) || !late_avail_pregs.contains(preg))
542546
{
547+
trace!("fixed {preg} for {op} isn't available");
543548
return Err(RegAllocError::TooManyLiveRegs);
544549
}
545550
self.available_pregs[OperandPos::Early].remove(preg);
@@ -671,7 +676,7 @@ impl<'a, F: Function> Env<'a, F> {
671676
_ => self.available_pregs[op.pos()],
672677
};
673678
if draw_from.is_empty(op.class()) {
674-
trace!("No registers available for {op}");
679+
trace!("No registers available for {op} in selection");
675680
return Err(RegAllocError::TooManyLiveRegs);
676681
}
677682
let Some(preg) = self.lrus[op.class()].last(draw_from) else {
@@ -888,21 +893,9 @@ impl<'a, F: Function> Env<'a, F> {
888893
}
889894

890895
fn remove_clobbers_from_available_pregs(&mut self, clobbers: PRegSet) {
891-
trace!("Removing clobbers {clobbers} from available reg sets");
892-
// Don't let defs get allocated to clobbers.
893-
// Consider a scenario:
894-
//
895-
// 1. (early|late) def v0 (reg). Clobbers: [p0]
896-
// 2. use v0 (fixed: p0)
897-
//
898-
// If p0 isn't removed from the both available reg sets, then
899-
// p0 could get allocated to v0 in inst 1, making it impossible
900-
// to restore it after the instruction.
901-
// To avoid this scenario, clobbers should be removed from both late
902-
// and early reg sets.
896+
trace!("Removing clobbers {clobbers} from late available reg sets");
903897
let all_but_clobbers = clobbers.invert();
904898
self.available_pregs[OperandPos::Late].intersect_from(all_but_clobbers);
905-
self.available_pregs[OperandPos::Early].intersect_from(all_but_clobbers);
906899
}
907900

908901
/// If instruction `inst` is a branch in `block`,
@@ -1030,6 +1023,90 @@ impl<'a, F: Function> Env<'a, F> {
10301023
Ok(())
10311024
}
10321025

1026+
fn alloc_def_op(&mut self, op_idx: usize, op: Operand, operands: &[Operand], block: Block, inst: Inst) -> Result<(), RegAllocError> {
1027+
trace!("Allocating def operand {op}");
1028+
if let OperandConstraint::Reuse(reused_idx) = op.constraint() {
1029+
let reused_op = operands[reused_idx];
1030+
// Alloc as an operand alive in both early and late phases
1031+
let new_reuse_op = Operand::new(
1032+
op.vreg(),
1033+
reused_op.constraint(),
1034+
OperandKind::Def,
1035+
OperandPos::Early,
1036+
);
1037+
trace!("allocating reuse op {op} as {new_reuse_op}");
1038+
self.process_operand_allocation(inst, new_reuse_op, op_idx)?;
1039+
} else if self.func.is_branch(inst) {
1040+
// If the defined vreg is used as a branch arg and it has an
1041+
// any or stack constraint, define it into the block param spillslot
1042+
let mut param_spillslot = None;
1043+
'outer: for (succ_idx, succ) in
1044+
self.func.block_succs(block).iter().cloned().enumerate()
1045+
{
1046+
for (param_idx, branch_arg_vreg) in self
1047+
.func
1048+
.branch_blockparams(block, inst, succ_idx)
1049+
.iter()
1050+
.cloned()
1051+
.enumerate()
1052+
{
1053+
if op.vreg() == branch_arg_vreg {
1054+
if matches!(
1055+
op.constraint(),
1056+
OperandConstraint::Any | OperandConstraint::Stack
1057+
) {
1058+
let block_param = self.func.block_params(succ)[param_idx];
1059+
param_spillslot = Some(self.get_spillslot(block_param));
1060+
}
1061+
break 'outer;
1062+
}
1063+
}
1064+
}
1065+
if let Some(param_spillslot) = param_spillslot {
1066+
let spillslot = self.vreg_spillslots[op.vreg().vreg()];
1067+
self.vreg_spillslots[op.vreg().vreg()] = param_spillslot;
1068+
let op = Operand::new(op.vreg(), OperandConstraint::Stack, op.kind(), op.pos());
1069+
self.process_operand_allocation(inst, op, op_idx)?;
1070+
self.vreg_spillslots[op.vreg().vreg()] = spillslot;
1071+
} else {
1072+
self.process_operand_allocation(inst, op, op_idx)?;
1073+
}
1074+
} else {
1075+
self.process_operand_allocation(inst, op, op_idx)?;
1076+
}
1077+
let slot = self.vreg_spillslots[op.vreg().vreg()];
1078+
if slot.is_valid() {
1079+
self.vreg_to_live_inst_range[op.vreg().vreg()].2 = Allocation::stack(slot);
1080+
let curr_alloc = self.vreg_allocs[op.vreg().vreg()];
1081+
let new_alloc = Allocation::stack(self.vreg_spillslots[op.vreg().vreg()]);
1082+
if curr_alloc != new_alloc {
1083+
self.add_move(inst, curr_alloc, new_alloc, op.class(), InstPosition::After)?;
1084+
}
1085+
}
1086+
self.vreg_to_live_inst_range[op.vreg().vreg()].0 = ProgPoint::after(inst);
1087+
self.freealloc(op.vreg());
1088+
Ok(())
1089+
}
1090+
1091+
fn alloc_use(&mut self, op_idx: usize, op: Operand, inst: Inst) -> Result<(), RegAllocError> {
1092+
trace!("Allocating use op {op}");
1093+
if self.reused_input_to_reuse_op[op_idx] != usize::MAX {
1094+
let reuse_op_idx = self.reused_input_to_reuse_op[op_idx];
1095+
let reuse_op_alloc = self.allocs[(inst.index(), reuse_op_idx)];
1096+
let Some(preg) = reuse_op_alloc.as_reg() else {
1097+
unreachable!();
1098+
};
1099+
let new_reused_input_constraint = OperandConstraint::FixedReg(preg);
1100+
let new_reused_input =
1101+
Operand::new(op.vreg(), new_reused_input_constraint, op.kind(), op.pos());
1102+
trace!("Allocating reused input {op} as {new_reused_input}");
1103+
self.process_operand_allocation(inst, new_reused_input, op_idx)?;
1104+
} else {
1105+
self.process_operand_allocation(inst, op, op_idx)?;
1106+
}
1107+
Ok(())
1108+
}
1109+
10331110
fn alloc_inst(&mut self, block: Block, inst: Inst) -> Result<(), RegAllocError> {
10341111
trace!("Allocating instruction {:?}", inst);
10351112
self.reset_available_pregs_and_scratch_regs();
@@ -1145,86 +1222,22 @@ impl<'a, F: Function> Env<'a, F> {
11451222
"Number of available pregs for int, float, vector any-reg and any ops: {:?}",
11461223
self.num_available_pregs
11471224
);
1148-
trace!("registers available for int, float, vector any-reg and any operands, respectively: {:?}", self.available_pregs);
1225+
trace!("registers available for early reg-only & any operands: {}", self.available_pregs[OperandPos::Early]);
1226+
trace!("registers available for late reg-only & any operands: {}", self.available_pregs[OperandPos::Late]);
11491227

1150-
for (op_idx, op) in operands.def_ops() {
1151-
trace!("Allocating def operands {op}");
1152-
if let OperandConstraint::Reuse(reused_idx) = op.constraint() {
1153-
let reused_op = operands[reused_idx];
1154-
// Alloc as an operand alive in both early and late phases
1155-
let new_reuse_op = Operand::new(
1156-
op.vreg(),
1157-
reused_op.constraint(),
1158-
OperandKind::Def,
1159-
OperandPos::Early,
1160-
);
1161-
trace!("allocating reuse op {op} as {new_reuse_op}");
1162-
self.process_operand_allocation(inst, new_reuse_op, op_idx)?;
1163-
} else if self.func.is_branch(inst) {
1164-
// If the defined vreg is used as a branch arg and it has an
1165-
// any or stack constraint, define it into the block param spillslot
1166-
let mut param_spillslot = None;
1167-
'outer: for (succ_idx, succ) in
1168-
self.func.block_succs(block).iter().cloned().enumerate()
1169-
{
1170-
for (param_idx, branch_arg_vreg) in self
1171-
.func
1172-
.branch_blockparams(block, inst, succ_idx)
1173-
.iter()
1174-
.cloned()
1175-
.enumerate()
1176-
{
1177-
if op.vreg() == branch_arg_vreg {
1178-
if matches!(
1179-
op.constraint(),
1180-
OperandConstraint::Any | OperandConstraint::Stack
1181-
) {
1182-
let block_param = self.func.block_params(succ)[param_idx];
1183-
param_spillslot = Some(self.get_spillslot(block_param));
1184-
}
1185-
break 'outer;
1186-
}
1187-
}
1188-
}
1189-
if let Some(param_spillslot) = param_spillslot {
1190-
let spillslot = self.vreg_spillslots[op.vreg().vreg()];
1191-
self.vreg_spillslots[op.vreg().vreg()] = param_spillslot;
1192-
let op = Operand::new(op.vreg(), OperandConstraint::Stack, op.kind(), op.pos());
1193-
self.process_operand_allocation(inst, op, op_idx)?;
1194-
self.vreg_spillslots[op.vreg().vreg()] = spillslot;
1195-
} else {
1196-
self.process_operand_allocation(inst, op, op_idx)?;
1197-
}
1228+
for (op_idx, op) in operands.late() {
1229+
if op.kind() == OperandKind::Def {
1230+
self.alloc_def_op(op_idx, op, operands.0, block, inst)?;
11981231
} else {
1199-
self.process_operand_allocation(inst, op, op_idx)?;
1232+
self.alloc_use(op_idx, op, inst)?;
12001233
}
1201-
let slot = self.vreg_spillslots[op.vreg().vreg()];
1202-
if slot.is_valid() {
1203-
self.vreg_to_live_inst_range[op.vreg().vreg()].2 = Allocation::stack(slot);
1204-
let curr_alloc = self.vreg_allocs[op.vreg().vreg()];
1205-
let new_alloc = Allocation::stack(self.vreg_spillslots[op.vreg().vreg()]);
1206-
if curr_alloc != new_alloc {
1207-
self.add_move(inst, curr_alloc, new_alloc, op.class(), InstPosition::After)?;
1208-
}
1209-
}
1210-
self.vreg_to_live_inst_range[op.vreg().vreg()].0 = ProgPoint::after(inst);
1211-
self.freealloc(op.vreg());
12121234
}
1213-
for (op_idx, op) in operands.use_ops() {
1235+
for (op_idx, op) in operands.early() {
12141236
trace!("Allocating use operand {op}");
1215-
if self.reused_input_to_reuse_op[op_idx] != usize::MAX {
1216-
let reuse_op_idx = self.reused_input_to_reuse_op[op_idx];
1217-
let reuse_op_alloc = self.allocs[(inst.index(), reuse_op_idx)];
1218-
let Some(preg) = reuse_op_alloc.as_reg() else {
1219-
unreachable!();
1220-
};
1221-
let new_reused_input_constraint = OperandConstraint::FixedReg(preg);
1222-
let new_reused_input =
1223-
Operand::new(op.vreg(), new_reused_input_constraint, op.kind(), op.pos());
1224-
trace!("Allocating reused input {op} as {new_reused_input}");
1225-
self.process_operand_allocation(inst, new_reused_input, op_idx)?;
1237+
if op.kind() == OperandKind::Use {
1238+
self.alloc_use(op_idx, op, inst)?;
12261239
} else {
1227-
self.process_operand_allocation(inst, op, op_idx)?;
1240+
self.alloc_def_op(op_idx, op, operands.0, block, inst)?;
12281241
}
12291242
}
12301243

@@ -1356,6 +1369,7 @@ impl<'a, F: Function> Env<'a, F> {
13561369
);
13571370
if self.edits.is_stack(prev_alloc) && self.edits.scratch_regs[vreg.class()].is_none() {
13581371
let Some(preg) = self.lrus[vreg.class()].last(avail_regs_for_scratch) else {
1372+
trace!("Can't find a scratch register for {vreg} in reload_at_begin");
13591373
return Err(RegAllocError::TooManyLiveRegs);
13601374
};
13611375
if self.vreg_in_preg[preg.index()] != VReg::invalid() {

0 commit comments

Comments
 (0)