Skip to content

Commit 40dd12a

Browse files
MacDueMacDue
committed
Small fixups
Change-Id: I64f503597717441461ede15dc50709f9af498288
1 parent f2dc37e commit 40dd12a

File tree

9 files changed

+69
-46
lines changed

9 files changed

+69
-46
lines changed

‎llvm/include/llvm/CodeGen/MachineFrameInfo.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -501,8 +501,12 @@ class MachineFrameInfo {
501501
StackID == TargetStackID::ScalablePredicateVector;
502502
}
503503

504-
bool isScalableStackID(int ObjectIdx) const {
504+
bool hasScalableStackID(int ObjectIdx) const {
505505
uint8_t StackID = getStackID(ObjectIdx);
506+
return isScalableStackID(StackID);
507+
}
508+
509+
bool isScalableStackID(uint8_t StackID) const {
506510
return StackID == TargetStackID::ScalableVector ||
507511
StackID == TargetStackID::ScalablePredicateVector;
508512
}

‎llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ struct StackFrameLayoutAnalysis {
7272
: Slot(Idx), Size(MFI.getObjectSize(Idx)),
7373
Align(MFI.getObjectAlign(Idx).value()), Offset(Offset),
7474
SlotTy(Invalid), Scalable(false) {
75-
Scalable = MFI.isScalableStackID(Idx);
75+
Scalable = MFI.hasScalableStackID(Idx);
7676
if (MFI.isSpillSlotObjectIndex(Idx))
7777
SlotTy = SlotType::Spill;
7878
else if (MFI.isFixedObjectIndex(Idx))

‎llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1610,10 +1610,10 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
16101610
int BaseOffset = -AFI->getTaggedBasePointerOffset();
16111611
Register FrameReg;
16121612
StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1613-
MF, BaseOffset, false /*isFixed*/, false /*isSVE*/, FrameReg,
1613+
MF, BaseOffset, false /*isFixed*/, TargetStackID::Default /*StackID*/,
1614+
FrameReg,
16141615
/*PreferFP=*/false,
1615-
/*ForSimm=*/true,
1616-
/*FI=*/-1);
1616+
/*ForSimm=*/true);
16171617
Register SrcReg = FrameReg;
16181618
if (FrameRegOffset) {
16191619
// Use output register as temporary.

‎llvm/lib/Target/AArch64/AArch64FrameLowering.cpp‎

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,9 @@ AArch64FrameLowering::getZPRStackSize(const MachineFunction &MF) const {
354354

355355
StackOffset
356356
AArch64FrameLowering::getPPRStackSize(const MachineFunction &MF) const {
357+
// With split SVE objects, the hazard padding is added to the PPR region,
358+
// which places it between the [GPR, PPR] area and the [ZPR, FPR] area. This
359+
// avoids hazards between both GPRs and FPRs and ZPRs and PPRs.
357360
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
358361
return StackOffset::get(AFI->hasSplitSVEObjects() ? getStackHazardSize(MF)
359362
: 0,
@@ -374,7 +377,7 @@ static bool isLikelyToHaveSVEStack(const AArch64FrameLowering &AFL,
374377

375378
const MachineFrameInfo &MFI = MF.getFrameInfo();
376379
for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) {
377-
if (MFI.isScalableStackID(FI))
380+
if (MFI.hasScalableStackID(FI))
378381
return true;
379382
}
380383

@@ -522,10 +525,6 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
522525
if (!EnableRedZone)
523526
return false;
524527

525-
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
526-
if (AFI->hasSplitSVEObjects())
527-
return false;
528-
529528
// Don't use the red zone if the function explicitly asks us not to.
530529
// This is typically used for kernel code.
531530
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
@@ -535,6 +534,7 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
535534
return false;
536535

537536
const MachineFrameInfo &MFI = MF.getFrameInfo();
537+
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
538538
uint64_t NumBytes = AFI->getLocalStackSize();
539539

540540
// If neither NEON or SVE are available, a COPY from one Q-reg to
@@ -1258,14 +1258,16 @@ AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
12581258
const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
12591259
bool FPAfterSVECalleeSaves =
12601260
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
1261-
if (MFI.isScalableStackID(FI)) {
1261+
if (MFI.hasScalableStackID(FI)) {
12621262
if (FPAfterSVECalleeSaves &&
12631263
-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
12641264
assert(!AFI->hasSplitSVEObjects() &&
12651265
"split-sve-objects not supported with FPAfterSVECalleeSaves");
12661266
return StackOffset::getScalable(ObjectOffset);
12671267
}
12681268
StackOffset AccessOffset{};
1269+
// The scalable vectors are below (lower address) the scalable predicates
1270+
// with split SVE objects, so we must subtract the size of the predicates.
12691271
if (AFI->hasSplitSVEObjects() &&
12701272
MFI.getStackID(FI) == TargetStackID::ScalableVector)
12711273
AccessOffset = -PPRStackSize;
@@ -1332,14 +1334,15 @@ StackOffset AArch64FrameLowering::resolveFrameIndexReference(
13321334
const auto &MFI = MF.getFrameInfo();
13331335
int64_t ObjectOffset = MFI.getObjectOffset(FI);
13341336
bool isFixed = MFI.isFixedObjectIndex(FI);
1335-
bool isSVE = MFI.isScalableStackID(FI);
1336-
return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1337-
PreferFP, ForSimm, FI);
1337+
auto StackID = static_cast<TargetStackID::Value>(MFI.getStackID(FI));
1338+
return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, StackID,
1339+
FrameReg, PreferFP, ForSimm);
13381340
}
13391341

13401342
StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
1341-
const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
1342-
Register &FrameReg, bool PreferFP, bool ForSimm, int64_t FI) const {
1343+
const MachineFunction &MF, int64_t ObjectOffset, bool isFixed,
1344+
TargetStackID::Value StackID, Register &FrameReg, bool PreferFP,
1345+
bool ForSimm) const {
13431346
const auto &MFI = MF.getFrameInfo();
13441347
const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
13451348
MF.getSubtarget().getRegisterInfo());
@@ -1350,6 +1353,7 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
13501353
int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
13511354
bool isCSR =
13521355
!isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
1356+
bool isSVE = MFI.isScalableStackID(StackID);
13531357

13541358
StackOffset ZPRStackSize = getZPRStackSize(MF);
13551359
StackOffset PPRStackSize = getPPRStackSize(MF);
@@ -1428,19 +1432,25 @@ StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
14281432
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
14291433

14301434
if (isSVE) {
1431-
StackOffset AccessOffset{};
1432-
if (AFI->hasSplitSVEObjects() &&
1433-
MFI.getStackID(FI) == TargetStackID::ScalableVector)
1434-
AccessOffset = -PPRStackSize;
1435-
1436-
StackOffset FPOffset =
1437-
AccessOffset +
1438-
StackOffset::get(-AFI->getCalleeSaveBaseToFrameRecordOffset(),
1439-
ObjectOffset);
1435+
StackOffset FPOffset = StackOffset::get(
1436+
-AFI->getCalleeSaveBaseToFrameRecordOffset(), ObjectOffset);
14401437
StackOffset SPOffset =
1441-
SVEStackSize + AccessOffset +
1438+
SVEStackSize +
14421439
StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
14431440
ObjectOffset);
1441+
1442+
// With split SVE objects the ObjectOffset is relative to the split area
1443+
// (i.e. the PPR area or ZPR area respectively).
1444+
if (AFI->hasSplitSVEObjects() && StackID == TargetStackID::ScalableVector) {
1445+
// If we're accessing an SVE vector with split SVE objects...
1446+
// - From the FP we need to move down past the PPR area:
1447+
FPOffset -= PPRStackSize;
1448+
// - From the SP we only need to move up to the ZPR area:
1449+
SPOffset -= PPRStackSize;
1450+
// Note: `SPOffset = SVEStackSize + ...`, so `-= PPRStackSize` results in
1451+
// `SPOffset = ZPRStackSize + ...`.
1452+
}
1453+
14441454
if (FPAfterSVECalleeSaves) {
14451455
FPOffset += StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
14461456
if (-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
@@ -2323,7 +2333,7 @@ void AArch64FrameLowering::determineStackHazardSlot(
23232333
std::optional<int> FI = getLdStFrameID(MI, MFI);
23242334
if (!FI || FI < 0 || FI > int(SlotTypes.size()))
23252335
continue;
2326-
if (MFI.isScalableStackID(*FI)) {
2336+
if (MFI.hasScalableStackID(*FI)) {
23272337
SlotTypes[*FI] |=
23282338
isPPRAccess(MI) ? SlotType::PPR : SlotType::ZPRorFPR;
23292339
} else {
@@ -2398,7 +2408,7 @@ void AArch64FrameLowering::determineStackHazardSlot(
23982408
if (AArch64::FPR64RegClass.contains(Reg))
23992409
SubRegIdx = AArch64::dsub;
24002410
else if (AArch64::FPR128RegClass.contains(Reg))
2401-
SubRegIdx = AArch64::zsub; // TODO: Is the the right sub-register?
2411+
SubRegIdx = AArch64::zsub;
24022412
else
24032413
continue;
24042414
// Clear the bit for the FPR save.
@@ -2578,7 +2588,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
25782588
unsigned NumSavedRegs = SavedRegs.count();
25792589

25802590
// If we have hazard padding in the CS area add that to the size.
2581-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects())
2591+
if (AFI->isStackHazardIncludedInCalleeSaveArea())
25822592
CSStackSize += getStackHazardSize(MF);
25832593

25842594
// Increase the callee-saved stack size if the function has streaming mode
@@ -2748,7 +2758,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
27482758
const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
27492759

27502760
// Create a hazard slot as we switch between GPR and FPR CSRs.
2751-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects() &&
2761+
if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
27522762
(!LastReg || !AArch64InstrInfo::isFpOrNEON(LastReg)) &&
27532763
AArch64InstrInfo::isFpOrNEON(Reg)) {
27542764
assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
@@ -2787,7 +2797,7 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
27872797
}
27882798

27892799
// Add hazard slot in the case where no FPR CSRs are present.
2790-
if (AFI->hasStackHazardSlotIndex() && !AFI->hasSplitSVEObjects() &&
2800+
if (AFI->isStackHazardIncludedInCalleeSaveArea() &&
27912801
HazardSlotIndex == std::numeric_limits<int>::max()) {
27922802
HazardSlotIndex = MFI.CreateStackObject(StackHazardSize, Align(8), true);
27932803
LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
@@ -2858,7 +2868,7 @@ static SVEStackSizes determineSVEStackSizes(MachineFunction &MF,
28582868
#ifndef NDEBUG
28592869
// First process all fixed stack objects.
28602870
for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2861-
assert(!MFI.isScalableStackID(I) &&
2871+
assert(!MFI.hasScalableStackID(I) &&
28622872
"SVE vectors should never be passed on the stack by value, only by "
28632873
"reference.");
28642874
#endif
@@ -3533,8 +3543,9 @@ void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
35333543

35343544
Register Reg;
35353545
FrameRegOffset = TFI->resolveFrameOffsetReference(
3536-
*MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
3537-
/*PreferFP=*/false, /*ForSimm=*/true, /*FI=*/-1);
3546+
*MF, FirstTagStore.Offset, false /*isFixed*/,
3547+
TargetStackID::Default /*StackID*/, Reg,
3548+
/*PreferFP=*/false, /*ForSimm=*/true);
35383549
FrameReg = Reg;
35393550
FrameRegUpdate = std::nullopt;
35403551

@@ -4256,7 +4267,7 @@ void AArch64FrameLowering::emitRemarks(
42564267
}
42574268

42584269
unsigned RegTy = StackAccess::AccessType::GPR;
4259-
if (MFI.isScalableStackID(FrameIdx)) {
4270+
if (MFI.hasScalableStackID(FrameIdx)) {
42604271
// SPILL_PPR_TO_ZPR_SLOT_PSEUDO and FILL_PPR_FROM_ZPR_SLOT_PSEUDO
42614272
// spill/fill the predicate as a data vector (so are an FPR access).
42624273
if (MI.getOpcode() != AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO &&

‎llvm/lib/Target/AArch64/AArch64FrameLowering.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ class AArch64FrameLowering : public TargetFrameLowering {
6969
bool ForSimm) const;
7070
StackOffset resolveFrameOffsetReference(const MachineFunction &MF,
7171
int64_t ObjectOffset, bool isFixed,
72-
bool isSVE, Register &FrameReg,
73-
bool PreferFP, bool ForSimm,
74-
int64_t FI) const;
72+
TargetStackID::Value StackID,
73+
Register &FrameReg, bool PreferFP,
74+
bool ForSimm) const;
7575
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
7676
MachineBasicBlock::iterator MI,
7777
ArrayRef<CalleeSavedInfo> CSI,

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7497,7 +7497,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
74977497
int FI = cast<FrameIndexSDNode>(N)->getIndex();
74987498
// We can only encode VL scaled offsets, so only fold in frame indexes
74997499
// referencing SVE objects.
7500-
if (MFI.isScalableStackID(FI)) {
7500+
if (MFI.hasScalableStackID(FI)) {
75017501
Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
75027502
OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
75037503
return true;
@@ -7543,7 +7543,7 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
75437543
int FI = cast<FrameIndexSDNode>(Base)->getIndex();
75447544
// We can only encode VL scaled offsets, so only fold in frame indexes
75457545
// referencing SVE objects.
7546-
if (MFI.isScalableStackID(FI))
7546+
if (MFI.hasScalableStackID(FI))
75477547
Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
75487548
}
75497549

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9256,7 +9256,7 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
92569256
(MI.getOpcode() == AArch64::ADDXri ||
92579257
MI.getOpcode() == AArch64::SUBXri)) {
92589258
const MachineOperand &MO = MI.getOperand(1);
9259-
if (MO.isFI() && MF.getFrameInfo().isScalableStackID(MO.getIndex()))
9259+
if (MO.isFI() && MF.getFrameInfo().hasScalableStackID(MO.getIndex()))
92609260
MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false,
92619261
/*IsImplicit=*/true));
92629262
}
@@ -29608,7 +29608,7 @@ void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
2960829608
// than doing it here in finalizeLowering.
2960929609
if (MFI.hasStackProtectorIndex()) {
2961029610
for (unsigned int i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
29611-
if (MFI.isScalableStackID(i) &&
29611+
if (MFI.hasScalableStackID(i) &&
2961229612
MFI.getObjectSSPLayout(i) != MachineFrameInfo::SSPLK_None) {
2961329613
MFI.setStackID(MFI.getStackProtectorIndex(),
2961429614
TargetStackID::ScalableVector);

‎llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
441441
}
442442

443443
unsigned getSVECalleeSavedStackSize() const {
444+
assert(!hasSplitSVEObjects() &&
445+
"ZPRs and PPRs are split. Use get[ZPR|PPR]CalleeSavedStackSize()");
444446
return getZPRCalleeSavedStackSize() + getPPRCalleeSavedStackSize();
445447
}
446448

@@ -449,6 +451,10 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
449451
return NumLocalDynamicTLSAccesses;
450452
}
451453

454+
bool isStackHazardIncludedInCalleeSaveArea() const {
455+
return hasStackHazardSlotIndex() && !hasSplitSVEObjects();
456+
}
457+
452458
std::optional<bool> hasRedZone() const { return HasRedZone; }
453459
void setHasRedZone(bool s) { HasRedZone = s; }
454460

‎llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,7 +1224,7 @@ void AArch64PrologueEmitter::emitCalleeSavedGPRLocations(
12241224
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
12251225
for (const auto &Info : CSI) {
12261226
unsigned FrameIdx = Info.getFrameIdx();
1227-
if (MFI.isScalableStackID(FrameIdx))
1227+
if (MFI.hasScalableStackID(FrameIdx))
12281228
continue;
12291229

12301230
assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
@@ -1253,7 +1253,7 @@ void AArch64PrologueEmitter::emitCalleeSavedSVELocations(
12531253
StackOffset PPRStackSize = AFL.getPPRStackSize(MF);
12541254
for (const auto &Info : CSI) {
12551255
int FI = Info.getFrameIdx();
1256-
if (!MFI.isScalableStackID(FI))
1256+
if (!MFI.hasScalableStackID(FI))
12571257
continue;
12581258

12591259
// Not all unwinders may know about SVE registers, so assume the lowest
@@ -1547,8 +1547,10 @@ void AArch64EpilogueEmitter::emitEpilogue() {
15471547
emitCalleeSavedSVERestores(RestoreEnd);
15481548
}
15491549
} else if (AFI->hasSplitSVEObjects() && SVEStackSize) {
1550+
// TODO: Support stack realigment and variable-sized objects.
15501551
assert(!AFI->isStackRealigned() && !MFI.hasVarSizedObjects() &&
1551-
"TODO: Support stack realigment / variable-sized objects");
1552+
"unexpected stack realignment or variable sized objects with split "
1553+
"SVE stack objects");
15521554
// SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
15531555
// areas.
15541556
auto ZPRCalleeSavedSize =
@@ -1771,7 +1773,7 @@ void AArch64EpilogueEmitter::emitCalleeSavedRestores(
17711773
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
17721774

17731775
for (const auto &Info : CSI) {
1774-
if (SVE != MFI.isScalableStackID(Info.getFrameIdx()))
1776+
if (SVE != MFI.hasScalableStackID(Info.getFrameIdx()))
17751777
continue;
17761778

17771779
MCRegister Reg = Info.getReg();

0 commit comments

Comments
 (0)