Skip to content

Commit d9ed861

Browse files
pratikasharZuul
pratikashar
authored and
Zuul
committed
Make spill/fill and spill cleanup modules emit spill/fill intrinsics
instead of send message Change-Id: Ic715520c271b1c4dcffc0ce08489bc0122428bba
1 parent c9db13d commit d9ed861

File tree

9 files changed

+440
-338
lines changed

9 files changed

+440
-338
lines changed

‎visa/BuildIR.h‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,28 +1099,59 @@ class IR_Builder {
10991099
return inst;
11001100
}
11011101

1102-
G4_INST* createSpill(G4_SrcRegRegion* payload, uint16_t numRows, uint32_t offset, G4_Declare* fp, G4_InstOption option,
1103-
unsigned int lineno = 0, int CISAoff = -1, const char* srcFilename = nullptr)
1102+
G4_INST* createSpill(G4_DstRegRegion* dst, G4_SrcRegRegion* header, G4_SrcRegRegion* payload, unsigned int execSize,
1103+
uint16_t numRows, uint32_t offset, G4_Declare* fp, G4_InstOption option, unsigned int lineno = 0, int CISAoff = -1,
1104+
const char* srcFilename = nullptr)
1105+
{
1106+
G4_INST* spill = createIntrinsicInst(nullptr, Intrinsic::Spill, execSize, dst,
1107+
header, payload, nullptr, option, lineno);
1108+
spill->asSpillIntrinsic()->setSrcFilename(srcFilename);
1109+
spill->asSpillIntrinsic()->setCISAOff(CISAoff);
1110+
spill->asSpillIntrinsic()->setFP(fp);
1111+
spill->asSpillIntrinsic()->setOffset(offset);
1112+
spill->asSpillIntrinsic()->setNumRows(numRows);
1113+
return spill;
1114+
}
1115+
1116+
G4_INST* createSpill(G4_DstRegRegion* dst, G4_SrcRegRegion* payload, unsigned int execSize, uint16_t numRows, uint32_t offset,
1117+
G4_Declare* fp, G4_InstOption option, unsigned int lineno = 0, int CISAoff = -1, const char* srcFilename = nullptr)
11041118
{
11051119
auto builtInR0 = getBuiltinR0();
11061120
auto rd = getRegionStride1();
11071121
auto srcRgnr0 = createSrcRegRegion(Mod_src_undef, Direct, builtInR0->getRegVar(), 0, 0, rd, Type_UD);
1108-
G4_INST* spill = createInternalIntrinsicInst(nullptr, Intrinsic::Spill, 1, createNullDst(G4_Type::Type_UD),
1109-
srcRgnr0, payload, nullptr, option, lineno, CISAoff, srcFilename);
1122+
G4_INST* spill = createIntrinsicInst(nullptr, Intrinsic::Spill, execSize, dst,
1123+
srcRgnr0, payload, nullptr, option, lineno);
1124+
spill->asSpillIntrinsic()->setSrcFilename(srcFilename);
1125+
spill->asSpillIntrinsic()->setCISAOff(CISAoff);
11101126
spill->asSpillIntrinsic()->setFP(fp);
11111127
spill->asSpillIntrinsic()->setOffset(offset);
11121128
spill->asSpillIntrinsic()->setNumRows(numRows);
11131129
return spill;
11141130
}
11151131

1116-
G4_INST* createFill(G4_DstRegRegion* dstData, uint16_t numRows, uint32_t offset, G4_Declare* fp , G4_InstOption option,
1132+
G4_INST* createFill(G4_SrcRegRegion* header, G4_DstRegRegion* dstData, unsigned int execSize, uint16_t numRows, uint32_t offset,
1133+
G4_Declare* fp, G4_InstOption option, unsigned int lineno = 0, int CISAoff = -1, const char* srcFilename = nullptr)
1134+
{
1135+
G4_INST* fill = createIntrinsicInst(nullptr, Intrinsic::Fill, execSize, dstData,
1136+
header, nullptr, nullptr, option, lineno);
1137+
fill->asFillIntrinsic()->setSrcFilename(srcFilename);
1138+
fill->asFillIntrinsic()->setCISAOff(CISAoff);
1139+
fill->asFillIntrinsic()->setFP(fp);
1140+
fill->asFillIntrinsic()->setOffset(offset);
1141+
fill->asFillIntrinsic()->setNumRows(numRows);
1142+
return fill;
1143+
}
1144+
1145+
G4_INST* createFill(G4_DstRegRegion* dstData, unsigned int execSize, uint16_t numRows, uint32_t offset, G4_Declare* fp , G4_InstOption option,
11171146
unsigned int lineno = 0, int CISAoff = -1, const char* srcFilename = nullptr)
11181147
{
11191148
auto builtInR0 = getBuiltinR0();
11201149
auto rd = getRegionStride1();
11211150
auto srcRgnr0 = createSrcRegRegion(Mod_src_undef, Direct, builtInR0->getRegVar(), 0, 0, rd, Type_UD);
1122-
G4_INST* fill = createInternalIntrinsicInst(nullptr, Intrinsic::Fill, 1, dstData,
1123-
srcRgnr0, nullptr, nullptr, option, lineno, CISAoff, srcFilename);
1151+
G4_INST* fill = createIntrinsicInst(nullptr, Intrinsic::Fill, execSize, dstData,
1152+
srcRgnr0, nullptr, nullptr, option, lineno);
1153+
fill->asFillIntrinsic()->setSrcFilename(srcFilename);
1154+
fill->asFillIntrinsic()->setCISAOff(CISAoff);
11241155
fill->asFillIntrinsic()->setFP(fp);
11251156
fill->asFillIntrinsic()->setOffset(offset);
11261157
fill->asFillIntrinsic()->setNumRows(numRows);

‎visa/Gen4_IR.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7033,6 +7033,16 @@ void G4_INST::computeRightBound(G4_Operand* opnd)
70337033

70347034
done = true;
70357035
}
7036+
else if (done == false && isFillIntrinsic())
7037+
{
7038+
asFillIntrinsic()->computeRightBound(opnd);
7039+
done = true;
7040+
}
7041+
else if (done == false && isSpillIntrinsic())
7042+
{
7043+
asSpillIntrinsic()->computeRightBound(opnd);
7044+
done = true;
7045+
}
70367046

70377047
if( done == false )
70387048
{

‎visa/Gen4_IR.hpp‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1424,7 +1424,7 @@ static const IntrinsicInfo G4_Intrinsics[(int)Intrinsic::NumIntrinsics] =
14241424
{Intrinsic::Wait, "wait", 0, 0, Phase::Optimizer, { 0, 0, 0, false, false } },
14251425
{Intrinsic::Use, "use", 0, 1, Phase::Scheduler, { 0, 0, 0, false, false } },
14261426
{Intrinsic::MemFence, "mem_fence", 0, 0, Phase::BinaryEncoding, { 0, 0, 0, false, false } },
1427-
{Intrinsic::PseudoKill, "pseudo_kill", 1, 1, Phase::RA, { 0, 0, 0, false, false } },
1427+
{Intrinsic::PseudoKill, "pseudo_kill", 1, 1, Phase::RA, { 0, 0, 0, false, false} },
14281428
{Intrinsic::Spill, "spill", 1, 2, Phase::RA, { 0, 0, 0, false, false } },
14291429
{Intrinsic::Fill, "fill", 1, 1, Phase::RA, { 0, 0, 0, false, false } }
14301430
};
@@ -1951,6 +1951,8 @@ class G4_Operand
19511951
{
19521952
friend class G4_INST;
19531953
friend class G4_InstSend;
1954+
friend class G4_FillIntrinsic;
1955+
friend class G4_SpillIntrinsic;
19541956

19551957
public:
19561958
enum Kind {
@@ -3826,6 +3828,22 @@ inline const char* G4_InstCF::getUipLabelStr() const
38263828
return uip->asLabel()->getLabel();
38273829
}
38283830

3831+
static void computeSpillFillOperandBound(G4_Operand* opnd, unsigned int LB, int numReg)
3832+
{
3833+
if (numReg == 0)
3834+
{
3835+
return;
3836+
}
3837+
3838+
// read/write in units of GRF.
3839+
unsigned RB = std::min(opnd->getTopDcl()->getByteSize(),
3840+
LB + numReg * G4_GRF_REG_NBYTES) - 1;
3841+
3842+
unsigned NBytes = RB - LB + 1;
3843+
opnd->setBitVecFromSize(NBytes);
3844+
opnd->setRightBound(RB);
3845+
}
3846+
38293847
class G4_SpillIntrinsic : public G4_InstIntrinsic
38303848
{
38313849
public:
@@ -3844,6 +3862,8 @@ class G4_SpillIntrinsic : public G4_InstIntrinsic
38443862

38453863
}
38463864

3865+
const static unsigned int InvalidOffset = 0xffffffff;
3866+
38473867
bool isOffBP() const { return getFP() != nullptr; }
38483868

38493869
uint32_t getNumRows() const { return numRows; }
@@ -3854,10 +3874,26 @@ class G4_SpillIntrinsic : public G4_InstIntrinsic
38543874
void setOffset(uint32_t o) { offset = o; }
38553875
void setFP(G4_Declare* f) { fp = f; }
38563876

3877+
bool isOffsetValid() { return offset != InvalidOffset; }
3878+
3879+
void computeRightBound(G4_Operand* opnd)
3880+
{
3881+
uint16_t numReg = 0;
3882+
if (opnd == getSrc(1))
3883+
{
3884+
numReg = asSpillIntrinsic()->getNumRows();
3885+
}
3886+
else if (opnd->isSrcRegRegion() && opnd == getSrc(0))
3887+
{
3888+
numReg = 1;
3889+
}
3890+
computeSpillFillOperandBound(opnd, opnd->left_bound, numReg);
3891+
}
3892+
38573893
private:
38583894
G4_Declare* fp = nullptr;
38593895
uint32_t numRows = 0;
3860-
uint32_t offset = 0;
3896+
uint32_t offset = InvalidOffset;
38613897
};
38623898

38633899
class G4_FillIntrinsic : public G4_InstIntrinsic
@@ -3878,6 +3914,8 @@ class G4_FillIntrinsic : public G4_InstIntrinsic
38783914

38793915
}
38803916

3917+
const static unsigned int InvalidOffset = 0xffffffff;
3918+
38813919
bool isOffBP() const { return getFP() != nullptr; }
38823920

38833921
uint32_t getNumRows() const { return numRows; }
@@ -3888,10 +3926,27 @@ class G4_FillIntrinsic : public G4_InstIntrinsic
38883926
void setOffset(uint32_t o) { offset = o; }
38893927
void setFP(G4_Declare* f) { fp = f; }
38903928

3929+
bool isOffsetValid() { return offset != InvalidOffset; }
3930+
3931+
void computeRightBound(G4_Operand* opnd)
3932+
{
3933+
uint16_t numReg = 0;
3934+
if (opnd == getDst())
3935+
{
3936+
numReg = asFillIntrinsic()->getNumRows();
3937+
}
3938+
else if (opnd->isSrcRegRegion() &&
3939+
(opnd == getSrc(0) || opnd == getSrc(1)))
3940+
{
3941+
numReg = 1;
3942+
}
3943+
computeSpillFillOperandBound(opnd, opnd->left_bound, numReg);
3944+
}
3945+
38913946
private:
38923947
G4_Declare* fp = nullptr;
38933948
uint32_t numRows = 0;
3894-
uint32_t offset = 0;
3949+
uint32_t offset = InvalidOffset;
38953950
};
38963951

38973952
} // namespace vISA

‎visa/GraphColor.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1899,7 +1899,7 @@ void Interference::buildInterferenceWithinBB(G4_BB* bb, BitSet& live)
18991899
}
19001900
}
19011901

1902-
if (inst->isSend() && !dst->isNullReg())
1902+
if ((inst->isSend() || inst->isFillIntrinsic()) && !dst->isNullReg())
19031903
{
19041904
if (VISA_WA_CHECK(kernel.fg.builder->getPWaTable(), WaDisableSendSrcDstOverlap))
19051905
{
@@ -6170,14 +6170,15 @@ void GraphColor::saveRegs(
61706170
{
61716171
// add (1) r126.2<1>:ud r125.7<0;1,0>:ud 0x2:ud
61726172
// sends (8) null<1>:ud r126.0 r1.0 ...
6173+
uint8_t execSize = (owordSize > 2) ? 16 : 8;
61736174
unsigned messageLength = owordSize / 2;
61746175
G4_Declare* msgDcl = builder.createTempVar(messageLength * GENX_DATAPORT_IO_SZ,
61756176
Type_UD, GRFALIGN, StackCallStr);
61766177
msgDcl->getRegVar()->setPhyReg(regPool.getGreg(startReg), 0);
61776178
auto sendSrc2 = builder.createSrcRegRegion(Mod_src_undef, Direct, msgDcl->getRegVar(), 0, 0,
61786179
builder.getRegionStride1(), Type_UD);
6179-
MUST_BE_TRUE(frameOwordOffset % 2 == 0, "Frame oword offset is not aligned on 32-byte boundary");
6180-
auto spillIntrinsic = builder.createSpill(sendSrc2, messageLength, frameOwordOffset/2, framePtr, InstOpt_WriteEnable);
6180+
G4_DstRegRegion* dst = builder.createNullDst((execSize > 8) ? Type_UW : Type_UD);
6181+
auto spillIntrinsic = builder.createSpill(dst, sendSrc2, execSize, messageLength, frameOwordOffset/2, framePtr, InstOpt_WriteEnable);
61816182
bb->insert(insertIt, spillIntrinsic);
61826183
}
61836184
else if (owordSize > 8)
@@ -6257,9 +6258,7 @@ void GraphColor::restoreRegs(
62576258
Type_UD, GRFALIGN, GraphColor::StackCallStr);
62586259
dstDcl->getRegVar()->setPhyReg(regPool.getGreg(startReg), 0);
62596260
G4_DstRegRegion* dstRgn = builder.createDstRegRegion(Direct, dstDcl->getRegVar(), 0, 0, 1, (execSize > 8) ? Type_UW : Type_UD);
6260-
6261-
MUST_BE_TRUE(frameOwordOffset % 2 == 0, "Frame oword offset not aligned to 32-byte boundary");
6262-
auto fillIntrinsic = builder.createFill(dstRgn, responseLength, frameOwordOffset / 2, framePtr, InstOpt_WriteEnable);
6261+
auto fillIntrinsic = builder.createFill(dstRgn, execSize, responseLength, frameOwordOffset / 2, framePtr, InstOpt_WriteEnable);
62636262
bb->insert(insertIt, fillIntrinsic);
62646263
}
62656264
//
@@ -8906,16 +8905,17 @@ int GlobalRA::coloringRegAlloc()
89068905
kernel.dumpDotFile("Spill_GRF");
89078906
}
89088907

8909-
#ifdef FIX_SCRATCH_SPILL_MESSAGE
89108908
scratchOffset = std::max(scratchOffset, spillGMRF.getNextScratchOffset());
8909+
#ifdef FIX_SCRATCH_SPILL_MESSAGE
89118910
if (scratchOffset >= SCRATCH_MSG_LIMIT && useScratchMsgForSpill)
89128911
{
89138912
spillGMRF.fixSpillFillCode(&kernel);
89148913
}
89158914
#endif
89168915
bool disableSpillCoalecse = builder.getOption(vISA_DisableSpillCoalescing) ||
89178916
builder.getOption(vISA_FastSpill) || builder.getOption(vISA_Debug);
8918-
if (!reserveSpillReg && !disableSpillCoalecse && builder.useSends())
8917+
if (!reserveSpillReg && !disableSpillCoalecse && builder.useSends() &&
8918+
!kernel.fg.getHasStackCalls() && !kernel.fg.getIsStackCallFunc())
89198919
{
89208920
CoalesceSpillFills c(kernel, liveAnalysis, coloring, spillGMRF, iterationNo, rpe, *this);
89218921
c.run();

‎visa/Rematerialization.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,19 @@ namespace vISA
199199
return false;
200200
}
201201

202+
auto op = inst->opcode();
203+
if (op == G4_pseudo_callee_restore || op == G4_pseudo_callee_save ||
204+
op == G4_pseudo_caller_restore || op == G4_pseudo_caller_save)
205+
return false;
206+
207+
G4_Declare* dcl = nullptr;
208+
if (inst->getDst() && inst->getDst()->getTopDcl())
209+
dcl = inst->getDst()->getTopDcl();
210+
211+
if(kernel.fg.builder->isPreDefArg(dcl) || kernel.fg.builder->isPreDefRet(dcl) ||
212+
kernel.fg.builder->isPreDefFEStackVar(dcl))
213+
return false;
214+
202215
return true;
203216
}
204217

0 commit comments

Comments
 (0)