Skip to content

Commit 89e33a1

Browse files
MacDueMacDue
committed
Small fixups
Change-Id: I64f503597717441461ede15dc50709f9af498288
1 parent e6e556b commit 89e33a1

File tree

9 files changed

+69
-46
lines changed

9 files changed

+69
-46
lines changed

‎llvm/include/llvm/CodeGen/MachineFrameInfo.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,8 +498,12 @@ class MachineFrameInfo {
498498
StackID == TargetStackID::ScalablePredicateVector;
499499
}
500500

501-
bool isScalableStackID(int ObjectIdx) const {
501+
bool hasScalableStackID(int ObjectIdx) const {
502502
uint8_t StackID = getStackID(ObjectIdx);
503+
return isScalableStackID(StackID);
504+
}
505+
506+
bool isScalableStackID(uint8_t StackID) const {
503507
return StackID == TargetStackID::ScalableVector ||
504508
StackID == TargetStackID::ScalablePredicateVector;
505509
}

‎llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ struct StackFrameLayoutAnalysis {
7272
: Slot(Idx), Size(MFI.getObjectSize(Idx)),
7373
Align(MFI.getObjectAlign(Idx).value()), Offset(Offset),
7474
SlotTy(Invalid), Scalable(false) {
75-
Scalable = MFI.isScalableStackID(Idx);
75+
Scalable = MFI.hasScalableStackID(Idx);
7676
if (MFI.isSpillSlotObjectIndex(Idx))
7777
SlotTy = SlotType::Spill;
7878
else if (MFI.isFixedObjectIndex(Idx))

‎llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1610,10 +1610,10 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
16101610
int BaseOffset = -AFI->getTaggedBasePointerOffset();
16111611
Register FrameReg;
16121612
StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1613-
MF, BaseOffset, false /*isFixed*/, false /*isSVE*/, FrameReg,
1613+
MF, BaseOffset, false /*isFixed*/, TargetStackID::Default /*StackID*/,
1614+
FrameReg,
16141615
/*PreferFP=*/false,
1615-
/*ForSimm=*/true,
1616-
/*FI=*/-1);
1616+
/*ForSimm=*/true);
16171617
Register SrcReg = FrameReg;
16181618
if (FrameRegOffset) {
16191619
// Use output register as temporary.

‎llvm/lib/Target/AArch64/AArch64FrameLowering.cpp‎

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,9 @@ AArch64FrameLowering::getZPRStackSize(const MachineFunction &MF) const {
354354

355355
StackOffset
356356
AArch64FrameLowering::getPPRStackSize(const MachineFunction &MF) const {
357+
// With split SVE objects, the hazard padding is added to the PPR region,
358+
// which places it between the [GPR, PPR] area and the [ZPR, FPR] area. This
359+
// avoids hazards between both GPRs and FPRs and ZPRs and PPRs.
357360
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
358361
return StackOffset::get(AFI->hasSplitSVEObjects() ? getStackHazardSize(MF)
359362
: 0,
@@ -374,7 +377,7 @@ static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL,
374377

375378
const MachineFrameInfo &MFI = MF.getFrameInfo();
376379
for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) {
377-
if (MFI.isScalableStackID(FI))
380+
if (MFI.hasScalableStackID(FI))
378381
return true;
379382
}
380383

@@ -522,10 +525,6 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
522525
if (!EnableRedZone)
523526
return false;
524527

525-
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
526-
if (AFI->hasSplitSVEObjects())
527-
return false;
528-
529528
// Don't use the red zone if the function explicitly asks us not to.
530529
// This is typically used for kernel code.
531530
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
@@ -535,6 +534,7 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
535534
return false;
536535

537536
const MachineFrameInfo &MFI = MF.getFrameInfo();
537+
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
538538
uint64_t NumBytes = AFI->getLocalStackSize();
539539

540540
// If neither NEON or SVE are available, a COPY from one Q-reg to
@@ -1258,14 +1258,16 @@ AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
12581258
const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
12591259
bool FPAfterSVECalleeSaves =
12601260
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
1261-
if (MFI.isScalableStackID(FI)) {
1261+
if (MFI.hasScalableStackID(FI)) {
12621262
if (FPAfterSVECalleeSaves &&
12631263
-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
12641264
assert(!AFI->hasSplitSVEObjects() &&
12651265
"split-sve-objects not supported with FPAfterSVECalleeSaves");
12661266
return StackOffset::getScalable(ObjectOffset);
12671267
}
12681268
StackOffset AccessOffset{};
1269+
// The scalable vectors are below (lower address) the scalable predicates
1270+
// with split SVE objects, so we must subtract the size of the predicates.
12691271
if (AFI->hasSplitSVEObjects() &&
12701272
MFI.getStackID(FI) == TargetStackID::ScalableVector)
12711273
AccessOffset = -PPRStackSize;
@@ -1332,14 +1334,15 @@ StackOffset AArch64FrameLowering::resolveFrameIndexReference(
13321334
const auto &MFI = MF.getFrameInfo();
13331335
int64_t ObjectOffset = MFI.getObjectOffset(FI);
13341336
bool isFixed = MFI.isFixedObjectIndex(FI);
1335-
bool isSVE = MFI.isScalableStackID(FI);
1336-
return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1337-
PreferFP, ForSimm, FI);
1337+
auto StackID = static_cast<TargetStackID::Value>(MFI.getStackID(FI));
1338+
return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, StackID,
1339+
FrameReg, PreferFP, ForSimm);
13381340
}
13391341

13401342
StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1341-
const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
1342-
Register &FrameReg, bool PreferFP, bool ForSimm, int64_t FI) const {
1343+
const MachineFunction &MF, int64_t ObjectOffset, bool isFixed,
1344+
TargetStackID::Value StackID, Register &FrameReg, bool PreferFP,
1345+
bool ForSimm) const {
13431346
const auto &MFI = MF.getFrameInfo();
13441347
const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
13451348
MF.getSubtarget().getRegisterInfo());
@@ -1350,6 +1353,7 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
13501353
int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
13511354
bool isCSR =
13521355
!isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1356+
bool isSVE = MFI.isScalableStackID(StackID);
13531357

13541358
StackOffset ZPRStackSize = getZPRStackSize(MF);
13551359
StackOffset PPRStackSize = getPPRStackSize(MF);
@@ -1428,19 +1432,25 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
14281432
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
14291433

14301434
if (isSVE) {
1431-
StackOffset AccessOffset{};
1432-
if (AFI->hasSplitSVEObjects() &&
1433-
MFI.getStackID(FI) == TargetStackID::ScalableVector)
1434-
AccessOffset = -PPRStackSize;
1435-
1436-
StackOffset FPOffset =
1437-
AccessOffset +
1438-
StackOffset::get(-AFI->getCalleeSaveBaseToFrameRecordOffset(),
1439-
ObjectOffset);
1435+
StackOffset FPOffset = StackOffset::get(
1436+
-AFI->getCalleeSaveBaseToFrameRecordOffset(), ObjectOffset);
14401437
StackOffset SPOffset =
1441-
SVEStackSize + AccessOffset +
1438+
SVEStackSize +
14421439
StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
14431440
ObjectOffset);
1441+
1442+
// With split SVE objects the ObjectOffset is relative to the split area
1443+
// (i.e. the PPR area or ZPR area respectively).
1444+
if (AFI->hasSplitSVEObjects() && StackID == TargetStackID::ScalableVector) {
1445+
// If we're accessing an SVE vector with split SVE objects...
1446+
// - From the FP we need to move down past the PPR area:
1447+
FPOffset -= PPRStackSize;
1448+
// - From the SP we only need to move up to the ZPR area:
1449+
SPOffset -= PPRStackSize;
1450+
// Note: `SPOffset = SVEStackSize + ...`, so `-= PPRStackSize` results in
1451+
// `SPOffset = ZPRStackSize + ...`.
1452+
}
1453+
14441454
if (FPAfterSVECalleeSaves) {
14451455
FPOffset += StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
14461456
if (-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
@@ -2323,7 +2333,7 @@ void AArch64FrameLowering::determineStackHazardSlot(
23232333
std::optional<int> FI = getLdStFrameID(MI, MFI);
23242334
if (!FI || FI < 0 || FI > int(SlotTypes.size()))
23252335
continue;
2326-
if (MFI.isScalableStackID(*FI)) {
2336+
if (MFI.hasScalableStackID(*FI)) {
23272337
SlotTypes[*FI] |=
23282338
isPPRAccess(MI) ? SlotType::PPR : SlotType::ZPRorFPR;
23292339
} else {
@@ -2398,7 +2408,7 @@ void AArch64FrameLowering::determineStackHazardSlot(
23982408
if (AArch64::FPR64RegClass.contains(Reg))
23992409
SubRegIdx = AArch64::dsub;
24002410
else if (AArch64::FPR128RegClass.contains(Reg))
2401-
SubRegIdx = AArch64::zsub; // TODO: Is the the right sub-register?
2411+
SubRegIdx = AArch64::zsub;
24022412
else
24032413
continue;
24042414
// Clear the bit for the FPR save.
@@ -2578,7 +2588,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
25782588
unsigned NumSavedRegs = SavedRegs.count();
25792589

25802590
// If we have hazard padding in the CS area add that to the size.
2581-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects())
2591+
if (AFI->isStackHazardIncludedInCalleeSaveArea())
25822592
CSStackSize += getStackHazardSize(MF);
25832593

25842594
// Increase the callee-saved stack size if the function has streaming mode
@@ -2748,7 +2758,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
27482758
const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
27492759

27502760
// Create a hazard slot as we switch between GPR and FPR CSRs.
2751-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects() &&
2761+
if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
27522762
(!LastReg || !AArch64InstrInfo::isFpOrNEON(LastReg)) &&
27532763
AArch64InstrInfo::isFpOrNEON(Reg)) {
27542764
assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
@@ -2787,7 +2797,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
27872797
}
27882798

27892799
// Add hazard slot in the case where no FPR CSRs are present.
2790-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects() &&
2800+
if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
27912801
HazardSlotIndex == std::numeric_limits<int>::max()) {
27922802
HazardSlotIndex = MFI.CreateStackObject(StackHazardSize, Align(8), true);
27932803
LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
@@ -2858,7 +2868,7 @@ static SVEStackSizes determineSVEStackSizes(MachineFunction &MF,
28582868
#ifndef NDEBUG
28592869
// First process all fixed stack objects.
28602870
for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2861-
assert(!MFI.isScalableStackID(I) &&
2871+
assert(!MFI.hasScalableStackID(I) &&
28622872
"SVE vectors should never be passed on the stack by value, only by "
28632873
"reference.");
28642874
#endif
@@ -3533,8 +3543,9 @@ void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
35333543

35343544
Register Reg;
35353545
FrameRegOffset = TFI->resolveFrameOffsetReference(
3536-
*MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
3537-
/*PreferFP=*/false, /*ForSimm=*/true, /*FI=*/-1);
3546+
*MF, FirstTagStore.Offset, false /*isFixed*/,
3547+
TargetStackID::Default /*StackID*/, Reg,
3548+
/*PreferFP=*/false, /*ForSimm=*/true);
35383549
FrameReg = Reg;
35393550
FrameRegUpdate = std::nullopt;
35403551

@@ -4256,7 +4267,7 @@ void AArch64FrameLowering::emitRemarks(
42564267
}
42574268

42584269
unsigned RegTy = StackAccess::AccessType::GPR;
4259-
if (MFI.isScalableStackID(FrameIdx)) {
4270+
if (MFI.hasScalableStackID(FrameIdx)) {
42604271
// SPILL_PPR_TO_ZPR_SLOT_PSEUDO and FILL_PPR_FROM_ZPR_SLOT_PSEUDO
42614272
// spill/fill the predicate as a data vector (so are an FPR access).
42624273
if (MI.getOpcode() != AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO &&

‎llvm/lib/Target/AArch64/AArch64FrameLowering.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ class AArch64FrameLowering : public TargetFrameLowering {
6969
bool ForSimm) const;
7070
StackOffset resolveFrameOffsetReference(const MachineFunction &MF,
7171
int64_t ObjectOffset, bool isFixed,
72-
bool isSVE, Register &FrameReg,
73-
bool PreferFP, bool ForSimm,
74-
int64_t FI) const;
72+
TargetStackID::Value StackID,
73+
Register &FrameReg, bool PreferFP,
74+
bool ForSimm) const;
7575
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
7676
MachineBasicBlock::iterator MI,
7777
ArrayRef<CalleeSavedInfo> CSI,

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7515,7 +7515,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
75157515
int FI = cast<FrameIndexSDNode>(N)->getIndex();
75167516
// We can only encode VL scaled offsets, so only fold in frame indexes
75177517
// referencing SVE objects.
7518-
if (MFI.isScalableStackID(FI)) {
7518+
if (MFI.hasScalableStackID(FI)) {
75197519
Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
75207520
OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
75217521
return true;
@@ -7561,7 +7561,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
75617561
int FI = cast<FrameIndexSDNode>(Base)->getIndex();
75627562
// We can only encode VL scaled offsets, so only fold in frame indexes
75637563
// referencing SVE objects.
7564-
if (MFI.isScalableStackID(FI))
7564+
if (MFI.hasScalableStackID(FI))
75657565
Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
75667566
}
75677567

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9171,7 +9171,7 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
91719171
(MI.getOpcode() == AArch64::ADDXri ||
91729172
MI.getOpcode() == AArch64::SUBXri)) {
91739173
const MachineOperand &MO = MI.getOperand(1);
9174-
if (MO.isFI() && MF.getFrameInfo().isScalableStackID(MO.getIndex()))
9174+
if (MO.isFI() && MF.getFrameInfo().hasScalableStackID(MO.getIndex()))
91759175
MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false,
91769176
/*IsImplicit=*/true));
91779177
}
@@ -29385,7 +29385,7 @@ void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
2938529385
// than doing it here in finalizeLowering.
2938629386
if (MFI.hasStackProtectorIndex()) {
2938729387
for (unsigned int i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
29388-
if (MFI.isScalableStackID(i) &&
29388+
if (MFI.hasScalableStackID(i) &&
2938929389
MFI.getObjectSSPLayout(i) != MachineFrameInfo::SSPLK_None) {
2939029390
MFI.setStackID(MFI.getStackProtectorIndex(),
2939129391
TargetStackID::ScalableVector);

‎llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
441441
}
442442

443443
unsigned getSVECalleeSavedStackSize() const {
444+
assert(!hasSplitSVEObjects() &&
445+
"ZPRs and PPRs are split. Use get[ZPR|PPR]CalleeSavedStackSize()");
444446
return getZPRCalleeSavedStackSize() + getPPRCalleeSavedStackSize();
445447
}
446448

@@ -449,6 +451,10 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
449451
return NumLocalDynamicTLSAccesses;
450452
}
451453

454+
bool isStackHazardIncludedInCalleeSaveArea() const {
455+
return hasStackHazardSlotIndex() && !hasSplitSVEObjects();
456+
}
457+
452458
std::optional<bool> hasRedZone() const { return HasRedZone; }
453459
void setHasRedZone(bool s) { HasRedZone = s; }
454460

‎llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,7 +1217,7 @@ void AArch64PrologueEmitter::emitCalleeSavedGPRLocations(
12171217
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
12181218
for (const auto &Info : CSI) {
12191219
unsigned FrameIdx = Info.getFrameIdx();
1220-
if (MFI.isScalableStackID(FrameIdx))
1220+
if (MFI.hasScalableStackID(FrameIdx))
12211221
continue;
12221222

12231223
assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
@@ -1246,7 +1246,7 @@ void AArch64PrologueEmitter::emitCalleeSavedSVELocations(
12461246
StackOffset PPRStackSize = AFL.getPPRStackSize(MF);
12471247
for (const auto &Info : CSI) {
12481248
int FI = Info.getFrameIdx();
1249-
if (!MFI.isScalableStackID(FI))
1249+
if (!MFI.hasScalableStackID(FI))
12501250
continue;
12511251

12521252
// Not all unwinders may know about SVE registers, so assume the lowest
@@ -1540,8 +1540,10 @@ void AArch64EpilogueEmitter::emitEpilogue() {
15401540
emitCalleeSavedSVERestores(RestoreEnd);
15411541
}
15421542
} else if (AFI->hasSplitSVEObjects() && SVEStackSize) {
1543+
// TODO: Support stack realigment and variable-sized objects.
15431544
assert(!AFI->isStackRealigned() && !MFI.hasVarSizedObjects() &&
1544-
"TODO: Support stack realigment / variable-sized objects");
1545+
"unexpected stack realignment or variable sized objects with split "
1546+
"SVE stack objects");
15451547
// SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
15461548
// areas.
15471549
auto ZPRCalleeSavedSize =
@@ -1764,7 +1766,7 @@ void AArch64EpilogueEmitter::emitCalleeSavedRestores(
17641766
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
17651767

17661768
for (const auto &Info : CSI) {
1767-
if (SVE != MFI.isScalableStackID(Info.getFrameIdx()))
1769+
if (SVE != MFI.hasScalableStackID(Info.getFrameIdx()))
17681770
continue;
17691771

17701772
MCRegister Reg = Info.getReg();

0 commit comments

Comments
 (0)