Skip to content

Commit 08398d5

Browse files
krystian-andrzejewskiigcbot
krystian-andrzejewski
authored and
igcbot
committed
Unification of pattern matching for imm global offset
This change is to unify the pattern matching case for imm global offset used in different functions. Moreover, this supports extra patterns with subtractionss in experimental code paths and resolves an issue with the immediate offsets in existing one.
1 parent bae47b7 commit 08398d5

File tree

7 files changed

+299
-108
lines changed

7 files changed

+299
-108
lines changed

‎IGC/Compiler/CISACodeGen/EmitVISAPass.cpp‎

Lines changed: 85 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -744,9 +744,8 @@ bool EmitPass::runOnFunction(llvm::Function &F) {
744744
m_encoder->SetIsCodePatchCandidate(false);
745745
}
746746

747-
if (m_pCtx->platform.supportDualSimd8PS() &&
748-
m_pCtx->platform.isCoreChildOf(IGFX_XE2_HPG_CORE) &&
749-
m_currShader->GetShaderType() == ShaderType::PIXEL_SHADER ) {
747+
if (m_pCtx->platform.supportDualSimd8PS() && m_pCtx->platform.isCoreChildOf(IGFX_XE2_HPG_CORE) &&
748+
m_currShader->GetShaderType() == ShaderType::PIXEL_SHADER) {
750749
m_encoder->SetIsCodePatchCandidate(false);
751750
}
752751
} else {
@@ -3999,7 +3998,8 @@ void EmitPass::emitVideoAnalyticGRF(llvm::GenIntrinsicInst *inst, const DWORD re
39993998

40003999
static bool isVectorTypeAllowed(Value *I) {
40014000
IGCLLVM::FixedVectorType *VecType = llvm::dyn_cast<IGCLLVM::FixedVectorType>(I->getType());
4002-
if (!VecType) return false;
4001+
if (!VecType)
4002+
return false;
40034003
auto ElType = VecType->getElementType();
40044004
return ElType->isIntegerTy(32) || ElType->isFloatTy();
40054005
}
@@ -4045,8 +4045,10 @@ void EmitPass::BinaryUnary(llvm::Instruction *inst, const SSource source[2], con
40454045
break;
40464046
case Instruction::FAdd:
40474047
case Instruction::Add:
4048-
if (inst->getType()->isVectorTy()) Add(source, modifier);
4049-
else EmitSimpleAlu(inst, source, modifier);
4048+
if (inst->getType()->isVectorTy())
4049+
Add(source, modifier);
4050+
else
4051+
EmitSimpleAlu(inst, source, modifier);
40504052
break;
40514053
case Instruction::Call:
40524054
EmitAluIntrinsic(cast<CallInst>(inst), source, modifier);
@@ -4245,7 +4247,6 @@ static unsigned getVectorSize(Value *I) {
42454247
return NumElements;
42464248
}
42474249

4248-
42494250
void EmitPass::FPTrunc(const SSource sources[2], const DstModifier &modifier) {
42504251

42514252
CVariable *src[2];
@@ -4351,13 +4352,18 @@ bool EmitPass::isVectorEmissionPossible(const SSource sources[2], CVariable *src
43514352
// any non ordinary state of source modifiers
43524353
// can be a sign of emission of a specific pattern,
43534354
// emission can interfere with
4354-
if (sources[0].elementOffset != 0) return false;
4355-
if (sources[1].elementOffset != 0) return false;
4356-
if (sources[0].SIMDOffset != 0) return false;
4357-
if (sources[1].SIMDOffset != 0) return false;
4355+
if (sources[0].elementOffset != 0)
4356+
return false;
4357+
if (sources[1].elementOffset != 0)
4358+
return false;
4359+
if (sources[0].SIMDOffset != 0)
4360+
return false;
4361+
if (sources[1].SIMDOffset != 0)
4362+
return false;
43584363

43594364
bool AllowedVectorTypes = isVectorTypeAllowed(sources[0].value) && isVectorTypeAllowed(sources[1].value);
4360-
if (!AllowedVectorTypes) return false;
4365+
if (!AllowedVectorTypes)
4366+
return false;
43614367

43624368
bool DestSizeEquals = false;
43634369
// it's possible have a case where two uniform vectors are
@@ -4732,7 +4738,8 @@ void EmitPass::VectorMad(const SSource sources[3], const DstModifier &modifier)
47324738
void EmitPass::FDiv(const SSource sources[2], const DstModifier &modifier) {
47334739

47344740
CVariable *src[2];
4735-
for (int i = 0; i < 2; ++i) src[i] = GetSrcVariable(sources[i]);
4741+
for (int i = 0; i < 2; ++i)
4742+
src[i] = GetSrcVariable(sources[i]);
47364743
bool IsPossible = isVectorEmissionPossible(sources, src);
47374744

47384745
if (IGC_IS_FLAG_ENABLED(EnableVectorEmitter) && IsPossible) {
@@ -5697,8 +5704,7 @@ void EmitPass::emitSimdShuffle(llvm::Instruction *inst) {
56975704
CVariable *tempCopy = m_currShader->GetNewVariable(simdChannel, "SanitizedIndexShuffleTmp");
56985705
m_encoder->And(tempCopy, simdChannel, m_currShader->ImmToVariable(maskOfValidLanes, ISA_TYPE_UW));
56995706
simdChannelUW = m_currShader->BitCast(tempCopy, ISA_TYPE_UW);
5700-
}
5701-
else {
5707+
} else {
57025708
m_encoder->SetSrcRegion(0, 2, 1, 0);
57035709
m_encoder->SetDstRegion(2);
57045710
m_encoder->And(simdChannelUW, simdChannelUW, m_currShader->ImmToVariable(maskOfValidLanes, ISA_TYPE_UW));
@@ -8747,12 +8753,13 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst *inst) {
87478753
break;
87488754
case GenISAIntrinsic::GenISA_ldrawvector_indexed:
87498755
case GenISAIntrinsic::GenISA_ldraw_indexed:
8750-
emitLoadRawIndexed(cast<LdRawIntrinsic>(inst), cast<LdRawIntrinsic>(inst)->getOffsetValue(), nullptr, nullptr);
8756+
emitLoadRawIndexed(cast<LdRawIntrinsic>(inst), cast<LdRawIntrinsic>(inst)->getOffsetValue(), nullptr, nullptr,
8757+
false /*flipVarOffsetSign*/);
87518758
break;
87528759
case GenISAIntrinsic::GenISA_storerawvector_indexed:
87538760
case GenISAIntrinsic::GenISA_storeraw_indexed:
87548761
emitStoreRawIndexed(cast<StoreRawIntrinsic>(inst), cast<StoreRawIntrinsic>(inst)->getOffsetValue(), nullptr,
8755-
nullptr);
8762+
nullptr, false /*flipVarOffsetSign*/);
87568763
break;
87578764
case GenISAIntrinsic::GenISA_GetBufferPtr:
87588765
emitGetBufferPtr(inst);
@@ -9218,6 +9225,7 @@ void EmitPass::EmitIntrinsicMessage(llvm::IntrinsicInst *inst) {
92189225
case Intrinsic::fabs:
92199226
case Intrinsic::trap:
92209227
case Intrinsic::experimental_noalias_scope_decl:
9228+
case Intrinsic::assume:
92219229
// do nothing
92229230
break;
92239231
case Intrinsic::stacksave:
@@ -10439,8 +10447,8 @@ void EmitPass::setRovCacheCtrl(GenIntrinsicInst *inst) {
1043910447
inst->setMetadata("lsc.cache.ctrl", node);
1044010448
}
1044110449

10442-
void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale,
10443-
ConstantInt *immOffset) {
10450+
void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale, ConstantInt *immOffset,
10451+
bool flipVarOffsetSign) {
1044410452
Value *bufPtrv = inst->getResourceValue();
1044510453

1044610454
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
@@ -10451,7 +10459,7 @@ void EmitPass::emitLoadRawIndexed(LdRawIntrinsic *inst, Value *varOffset, Consta
1045110459
}
1045210460
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, true);
1045310461
emitLSCVectorLoad(inst, bufPtrv,
10454-
varOffset, immOffset, immScale, cacheOpts, addrSpace
10462+
varOffset, immOffset, immScale, flipVarOffsetSign, cacheOpts, addrSpace
1045510463
);
1045610464
return;
1045710465
}
@@ -10710,7 +10718,8 @@ void EmitPass::emitLoad3DInner(LdRawIntrinsic *inst, ResourceDescriptor &resourc
1071010718
}
1071110719
}
1071210720

10713-
void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, ConstantInt *immScale
10721+
void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, ConstantInt *immScale,
10722+
bool flipVarOffsetSign
1071410723
) {
1071510724
if (m_pCtx->getModuleMetaData()->isHDCFastClearShader) {
1071610725
emitFastClear(inst);
@@ -10722,13 +10731,13 @@ void EmitPass::emitLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, C
1072210731
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1072310732

1072410733
emitLSCVectorLoad(inst, inst->getPointerOperand(),
10725-
offset, immOffset, immScale, cacheOpts, addrSpace
10734+
offset, immOffset, immScale, flipVarOffsetSign, cacheOpts, addrSpace
1072610735
);
1072710736
return;
1072810737
}
1072910738
IGC_ASSERT_MESSAGE(immScale ? immScale->getSExtValue() == 1 : true, "Immediate Scale not supported on non-LSC path!");
1073010739

10731-
emitVectorLoad(inst, offset, immOffset);
10740+
emitVectorLoad(inst, offset, immOffset, flipVarOffsetSign);
1073210741
}
1073310742

1073410743
void EmitPass::emitPredicatedLoad(Instruction *inst) {
@@ -10743,7 +10752,7 @@ void EmitPass::emitPredicatedLoad(Instruction *inst) {
1074310752
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1074410753

1074510754
emitLSCVectorLoad(inst, offset,
10746-
offset, nullptr, nullptr, cacheOpts, addrSpace
10755+
offset, nullptr, nullptr, false, cacheOpts, addrSpace
1074710756
);
1074810757
}
1074910758

@@ -11780,7 +11789,7 @@ void EmitPass::emitSymbolRelocation(Function &F) {
1178011789
}
1178111790

1178211791
void EmitPass::emitStoreRawIndexed(StoreRawIntrinsic *inst, Value *varOffset, ConstantInt *immScale,
11783-
ConstantInt *immOffset) {
11792+
ConstantInt *immOffset, bool flipVarOffsetSign) {
1178411793
Value *pBufPtr = inst->getResourceValue();
1178511794
Value *pValToStore = inst->getStoreValue();
1178611795

@@ -11791,8 +11800,8 @@ void EmitPass::emitStoreRawIndexed(StoreRawIntrinsic *inst, Value *varOffset, Co
1179111800

1179211801
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1179311802
emitLSCVectorStore(pBufPtr,
11794-
varOffset, immOffset, immScale, pValToStore, inst->getParent(), cacheOpts, inst->getAlignment(),
11795-
false, addrSpace
11803+
varOffset, immOffset, immScale, flipVarOffsetSign, pValToStore, inst->getParent(), cacheOpts,
11804+
inst->getAlignment(), false, addrSpace
1179611805
);
1179711806
return;
1179811807
}
@@ -11917,20 +11926,21 @@ void EmitPass::emitStore3DInner(Value *pllValToStore, Value *pllDstPtr, Value *p
1191711926
}
1191811927
}
1191911928

11920-
void EmitPass::emitStore(StoreInst *inst, Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale
11929+
void EmitPass::emitStore(StoreInst *inst, Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
11930+
bool flipVarOffsetSign
1192111931
) {
1192211932
if (shouldGenerateLSC(inst)) {
1192311933
LSC_DOC_ADDR_SPACE addrSpace = m_pCtx->getUserAddrSpaceMD().Get(inst);
1192411934

1192511935
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1192611936
emitLSCVectorStore(inst->getPointerOperand(),
11927-
varOffset, immOffset, immScale, inst->getValueOperand(), inst->getParent(), cacheOpts,
11928-
IGCLLVM::getAlignmentValue(inst), inst->getMetadata("enable.vmask"), addrSpace
11937+
varOffset, immOffset, immScale, flipVarOffsetSign, inst->getValueOperand(), inst->getParent(),
11938+
cacheOpts, IGCLLVM::getAlignmentValue(inst), inst->getMetadata("enable.vmask"), addrSpace
1192911939
);
1193011940
return;
1193111941
}
1193211942
IGC_ASSERT_MESSAGE(immScale ? immScale->getSExtValue() == 1 : true, "Immediate Scale not supported on non-LSC path!");
11933-
emitVectorStore(inst, varOffset, immOffset);
11943+
emitVectorStore(inst, varOffset, immOffset, flipVarOffsetSign);
1193411944
}
1193511945

1193611946
void EmitPass::emitPredicatedStore(Instruction *inst) {
@@ -11943,7 +11953,8 @@ void EmitPass::emitPredicatedStore(Instruction *inst) {
1194311953
LSC_CACHE_OPTS cacheOpts = translateLSCCacheControlsFromMetadata(inst, false);
1194411954

1194511955
emitLSCVectorStore(inst->getOperand(0),
11946-
inst->getOperand(0), nullptr, nullptr, inst->getOperand(1), inst->getParent(), cacheOpts,
11956+
inst->getOperand(0), nullptr, nullptr, false /*flipVarOffsetSign*/, inst->getOperand(1),
11957+
inst->getParent(), cacheOpts,
1194711958
cast<ConstantInt>(inst->getOperand(2))->getZExtValue(), // alignment
1194811959
inst->getMetadata("enable.vmask"), addrSpace
1194911960
,
@@ -14636,7 +14647,7 @@ CVariable *EmitPass::UnpackOrBroadcastIfUniform(CVariable *pVar) {
1463614647
}
1463714648

1463814649
void EmitPass::emitAtomicRaw(llvm::GenIntrinsicInst *pInst, Value *dstAddr, ConstantInt *immOffset,
14639-
ConstantInt *immScale
14650+
ConstantInt *immScale, bool flipVarOffsetSign
1464014651
) {
1464114652
ForceDMask();
1464214653
// Currently, Dword Atomics can be called by matching 2 intrinsics. One is the
@@ -14710,6 +14721,13 @@ void EmitPass::emitAtomicRaw(llvm::GenIntrinsicInst *pInst, Value *dstAddr, Cons
1471014721

1471114722
// Dst address in bytes.
1471214723
CVariable *pDstAddr = GetSymbol(dstAddr);
14724+
if (flipVarOffsetSign) {
14725+
CVariable *eNegBase = m_currShader->GetNewVariable(pDstAddr, CName(pDstAddr->getName(), "Neg"));
14726+
m_encoder->SetSrcModifier(0, EMOD_NEG);
14727+
m_encoder->Copy(eNegBase, pDstAddr); // MOV with -src modifier
14728+
m_encoder->Push();
14729+
pDstAddr = eNegBase;
14730+
}
1471314731

1471414732

1471514733
PointerType *PtrTy = dyn_cast<PointerType>(dstAddr->getType());
@@ -16631,7 +16649,7 @@ void EmitPass::emitFastClearSend(llvm::Instruction *pInst) {
1663116649
m_encoder->Push();
1663216650
}
1663316651

16634-
void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset) {
16652+
void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOffset, bool flipVarOffsetSign) {
1663516653
int immOffsetInt = 0;
1663616654
if (immOffset)
1663716655
immOffsetInt = static_cast<int>(immOffset->getSExtValue());
@@ -16645,6 +16663,13 @@ void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOff
1664516663
// eOffset is in bytes
1664616664
// offset corresponds to Int2Ptr operand obtained during pattern matching
1664716665
CVariable *eOffset = GetSymbol(immOffset ? offset : Ptr);
16666+
if (flipVarOffsetSign) {
16667+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
16668+
m_encoder->SetSrcModifier(0, EMOD_NEG);
16669+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
16670+
m_encoder->Push();
16671+
eOffset = eNegOffset;
16672+
}
1664816673
if (useA32) {
1664916674
eOffset = TruncatePointer(eOffset);
1665016675
}
@@ -17083,7 +17108,7 @@ void EmitPass::emitVectorLoad(LoadInst *inst, Value *offset, ConstantInt *immOff
1708317108
}
1708417109
}
1708517110

17086-
void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immOffset) {
17111+
void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immOffset, bool flipVarOffsetSign) {
1708717112
int immOffsetInt = 0;
1708817113
if (immOffset)
1708917114
immOffsetInt = static_cast<int>(immOffset->getSExtValue());
@@ -17100,6 +17125,13 @@ void EmitPass::emitVectorStore(StoreInst *inst, Value *offset, ConstantInt *immO
1710017125
// eOffset is in bytes
1710117126
// offset corresponds to Int2Ptr operand obtained during pattern matching
1710217127
CVariable *eOffset = GetSymbol(immOffset ? offset : Ptr);
17128+
if (flipVarOffsetSign) {
17129+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
17130+
m_encoder->SetSrcModifier(0, EMOD_NEG);
17131+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
17132+
m_encoder->Push();
17133+
eOffset = eNegOffset;
17134+
}
1710317135
bool useA32 = !isA64Ptr(ptrType, m_currShader->GetContext());
1710417136
if (useA32) {
1710517137
eOffset = TruncatePointer(eOffset);
@@ -17774,7 +17806,7 @@ void EmitPass::emitLSCVectorLoad_uniform(LSC_CACHE_OPTS CacheOpts, bool UseA32,
1777417806

1777517807
void EmitPass::emitLSCVectorLoad(Instruction *inst, Value *Ptr,
1777617808
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
17777-
LSC_CACHE_OPTS cacheOpts, LSC_DOC_ADDR_SPACE addrSpace
17809+
bool flipVarOffsetSign, LSC_CACHE_OPTS cacheOpts, LSC_DOC_ADDR_SPACE addrSpace
1777817810
) {
1777917811

1778017812
bool predicatedLoad = false;
@@ -17804,6 +17836,13 @@ void EmitPass::emitLSCVectorLoad(Instruction *inst, Value *Ptr,
1780417836
// eOffset is in bytes
1780517837
// offset corresponds to Int2Ptr operand obtained during pattern matching
1780617838
CVariable *eOffset = GetSymbol(varOffset);
17839+
if (flipVarOffsetSign) {
17840+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
17841+
m_encoder->SetSrcModifier(0, EMOD_NEG);
17842+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
17843+
m_encoder->Push();
17844+
eOffset = eNegOffset;
17845+
}
1780717846
if (useA32) {
1780817847
eOffset = TruncatePointer(eOffset);
1780917848
}
@@ -18155,9 +18194,9 @@ void EmitPass::emitLSCVectorStore_uniform(LSC_CACHE_OPTS CacheOpts, bool UseA32,
1815518194
}
1815618195

1815718196
void EmitPass::emitLSCVectorStore(Value *Ptr,
18158-
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale, Value *storedVal,
18159-
BasicBlock *BB, LSC_CACHE_OPTS cacheOpts, alignment_t align, bool dontForceDmask,
18160-
LSC_DOC_ADDR_SPACE addrSpace
18197+
Value *varOffset, ConstantInt *immOffset, ConstantInt *immScale,
18198+
bool flipVarOffsetSign, Value *storedVal, BasicBlock *BB, LSC_CACHE_OPTS cacheOpts,
18199+
alignment_t align, bool dontForceDmask, LSC_DOC_ADDR_SPACE addrSpace
1816118200
,
1816218201
Value *predicate) {
1816318202

@@ -18178,6 +18217,13 @@ void EmitPass::emitLSCVectorStore(Value *Ptr,
1817818217

1817918218
// offset corresponds to Int2Ptr operand obtained during pattern matching
1818018219
CVariable *eOffset = GetSymbol(varOffset);
18220+
if (flipVarOffsetSign) {
18221+
CVariable *eNegOffset = m_currShader->GetNewVariable(eOffset, CName(eOffset->getName(), "Neg"));
18222+
m_encoder->SetSrcModifier(0, EMOD_NEG);
18223+
m_encoder->Copy(eNegOffset, eOffset); // MOV with -src modifier
18224+
m_encoder->Push();
18225+
eOffset = eNegOffset;
18226+
}
1818118227
bool useA32 = !isA64Ptr(ptrType, m_currShader->GetContext());
1818218228
LSC_ADDR_SIZE addrSize = useA32 ? LSC_ADDR_SIZE_32b : LSC_ADDR_SIZE_64b;
1818318229
if (useA32) {

0 commit comments

Comments
 (0)