Skip to content

Commit 841b46b

Browse files
ytarasov-intelsys_zuul
ytarasov-intel
authored and
sys_zuul
committed
Add svm gather and scatter support to SplitGather
and SplitScatter functions to allow use TPM with SVM Change-Id: I19f86f489c83a2a9754548400ff68690ea673642
1 parent 1a559d0 commit 841b46b

File tree

1 file changed

+95
-38
lines changed

1 file changed

+95
-38
lines changed

‎IGC/VectorCompiler/lib/GenXCodeGen/GenXThreadPrivateMemory.cpp‎

Lines changed: 95 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -498,8 +498,17 @@ bool GenXThreadPrivateMemory::replaceLoad(LoadInst *LdI) {
498498

499499
if (!isa<VectorType>(LdI->getType()) &&
500500
isa<VectorType>(ProperGather->getType())) {
501-
Instruction *LdVal =
502-
CastInst::CreateBitOrPointerCast(ProperGather, LdI->getType());
501+
VectorType *GatheredTy = cast<VectorType>(ProperGather->getType());
502+
Builder.ClearInsertionPoint();
503+
Instruction *LdVal = nullptr;
504+
if (GatheredTy->getNumElements() == 1)
505+
LdVal = cast<Instruction>(Builder.CreateExtractElement(
506+
ProperGather, static_cast<uint64_t>(0ul),
507+
ProperGather->getName() + ".tpm.loadres"));
508+
else
509+
LdVal = cast<Instruction>(Builder.CreateBitOrPointerCast(
510+
ProperGather, LdI->getType(),
511+
ProperGather->getName() + ".tpm.loadres"));
503512
LdVal->insertAfter(ProperGather);
504513
ProperGather = LdVal;
505514
}
@@ -928,9 +937,16 @@ static void EraseUsers(Instruction *Inst) {
928937
}
929938

930939
void SplitScatter(CallInst *CI) {
931-
IGC_ASSERT(GenXIntrinsic::getAnyIntrinsicID(CI) ==
932-
llvm::GenXIntrinsic::genx_scatter_scaled);
933-
Type *DataTy = CI->getArgOperand(5)->getType();
940+
auto IID = static_cast<llvm::GenXIntrinsic::ID>(
941+
GenXIntrinsic::getAnyIntrinsicID(CI));
942+
IGC_ASSERT((IID == llvm::GenXIntrinsic::genx_scatter_scaled) ||
943+
(IID == llvm::GenXIntrinsic::genx_svm_scatter));
944+
Type *DataTy = nullptr;
945+
if (IID == llvm::GenXIntrinsic::genx_scatter_scaled) {
946+
DataTy = CI->getArgOperand(5)->getType();
947+
} else if (IID == llvm::GenXIntrinsic::genx_svm_scatter) {
948+
DataTy = CI->getArgOperand(2)->getType();
949+
}
934950
unsigned NumElts = cast<VectorType>(DataTy)->getNumElements();
935951
IGC_ASSERT(NumElts % 2 == 0);
936952

@@ -939,34 +955,54 @@ void SplitScatter(CallInst *CI) {
939955
Splitters.first = FillVecWithSeqVals(Splitters.first, 0, CI);
940956
Splitters.second = FillVecWithSeqVals(Splitters.second, NumElts / 2, CI);
941957

942-
Value *Pred = CI->getArgOperand(0);
958+
Value *Pred = nullptr;
959+
Value *EltOffsets = nullptr;
960+
Value *OldVal = nullptr;
961+
if (IID == llvm::GenXIntrinsic::genx_scatter_scaled) {
962+
Pred = CI->getArgOperand(0);
963+
EltOffsets = CI->getArgOperand(5);
964+
OldVal = CI->getArgOperand(6);
965+
} else if (IID == llvm::GenXIntrinsic::genx_svm_scatter) {
966+
Pred = CI->getArgOperand(0);
967+
EltOffsets = CI->getArgOperand(2);
968+
OldVal = CI->getArgOperand(3);
969+
}
970+
943971
std::pair<Value *, Value *> NewPreds = SplitVec(Pred, NumElts, CI, Splitters);
944972

945-
Value *EltOffsets = CI->getArgOperand(5);
946973
std::pair<Value *, Value *> NewEltOffsets =
947974
SplitVec(EltOffsets, NumElts, CI, Splitters);
948975

949-
Value *OldVal = CI->getArgOperand(6);
950976
std::pair<Value *, Value *> OldVals =
951977
SplitVec(OldVal, NumElts, CI, Splitters);
952978

953-
auto IID = llvm::GenXIntrinsic::genx_scatter_scaled;
954979
Function *F = GenXIntrinsic::getGenXDeclaration(CI->getModule(), IID,
955980
{NewPreds.first->getType(),
956981
NewEltOffsets.first->getType(),
957982
OldVals.first->getType()});
958983

959-
Value *LogNumBlock = CI->getArgOperand(1);
960-
Value *Scale = CI->getArgOperand(2);
961-
Value *Surface = CI->getArgOperand(3);
962-
Value *Offset = CI->getArgOperand(4);
963-
964-
CallInst *FirstScatter =
965-
IntrinsicInst::Create(F, {NewPreds.first, LogNumBlock, Scale, Surface,
966-
Offset, NewEltOffsets.first, OldVals.first});
967-
CallInst *SecondScatter =
968-
IntrinsicInst::Create(F, {NewPreds.second, LogNumBlock, Scale, Surface,
969-
Offset, NewEltOffsets.second, OldVals.second});
984+
CallInst *FirstScatter = nullptr;
985+
CallInst *SecondScatter = nullptr;
986+
if (IID == llvm::GenXIntrinsic::genx_scatter_scaled) {
987+
Value *LogNumBlock = CI->getArgOperand(1);
988+
Value *Scale = CI->getArgOperand(2);
989+
Value *Surface = CI->getArgOperand(3);
990+
Value *Offset = CI->getArgOperand(4);
991+
992+
FirstScatter =
993+
IntrinsicInst::Create(F, {NewPreds.first, LogNumBlock, Scale, Surface,
994+
Offset, NewEltOffsets.first, OldVals.first});
995+
SecondScatter = IntrinsicInst::Create(
996+
F, {NewPreds.second, LogNumBlock, Scale, Surface, Offset,
997+
NewEltOffsets.second, OldVals.second});
998+
} else if (IID == llvm::GenXIntrinsic::genx_svm_scatter) {
999+
Value *LogNumBlock = CI->getArgOperand(1);
1000+
FirstScatter = IntrinsicInst::Create(
1001+
F, {NewPreds.first, LogNumBlock, NewEltOffsets.first, OldVals.first});
1002+
SecondScatter =
1003+
IntrinsicInst::Create(F, {NewPreds.second, LogNumBlock,
1004+
NewEltOffsets.second, OldVals.second});
1005+
}
9701006

9711007
FirstScatter->insertAfter(CI);
9721008
SecondScatter->insertAfter(FirstScatter);
@@ -975,8 +1011,10 @@ void SplitScatter(CallInst *CI) {
9751011
}
9761012

9771013
void SplitGather(CallInst *CI) {
978-
IGC_ASSERT(GenXIntrinsic::getAnyIntrinsicID(CI) ==
979-
llvm::GenXIntrinsic::genx_gather_scaled);
1014+
auto IID = static_cast<llvm::GenXIntrinsic::ID>(
1015+
GenXIntrinsic::getAnyIntrinsicID(CI));
1016+
IGC_ASSERT((IID == llvm::GenXIntrinsic::genx_gather_scaled) ||
1017+
(IID == llvm::GenXIntrinsic::genx_svm_gather));
9801018
Type *DstTy = CI->getType();
9811019
unsigned NumElts = cast<VectorType>(DstTy)->getNumElements();
9821020
IGC_ASSERT(NumElts % 2 == 0);
@@ -986,33 +1024,52 @@ void SplitGather(CallInst *CI) {
9861024
Splitters.first = FillVecWithSeqVals(Splitters.first, 0, CI);
9871025
Splitters.second = FillVecWithSeqVals(Splitters.second, NumElts / 2, CI);
9881026

989-
Value *Pred = CI->getArgOperand(0);
1027+
Value *Pred = nullptr;
1028+
Value *EltOffsets = nullptr;
1029+
Value *OldVal = nullptr;
1030+
if (IID == llvm::GenXIntrinsic::genx_gather_scaled) {
1031+
Pred = CI->getArgOperand(0);
1032+
EltOffsets = CI->getArgOperand(5);
1033+
OldVal = CI->getArgOperand(6);
1034+
} else if (IID == llvm::GenXIntrinsic::genx_svm_gather) {
1035+
Pred = CI->getArgOperand(0);
1036+
EltOffsets = CI->getArgOperand(2);
1037+
OldVal = CI->getArgOperand(3);
1038+
}
1039+
9901040
std::pair<Value *, Value *> NewPreds = SplitVec(Pred, NumElts, CI, Splitters);
9911041

992-
Value *EltOffsets = CI->getArgOperand(5);
9931042
std::pair<Value *, Value *> NewEltOffsets =
9941043
SplitVec(EltOffsets, NumElts, CI, Splitters);
995-
996-
Value *OldVal = CI->getArgOperand(6);
9971044
std::pair<Value *, Value *> OldVals =
9981045
SplitVec(OldVal, NumElts, CI, Splitters);
999-
auto IID = llvm::GenXIntrinsic::genx_gather_scaled;
10001046
Function *F = GenXIntrinsic::getGenXDeclaration(CI->getModule(), IID,
10011047
{OldVals.first->getType(),
10021048
NewPreds.first->getType(),
10031049
NewEltOffsets.first->getType()});
10041050

1005-
Value *LogNumBlock = CI->getArgOperand(1);
1006-
Value *Scale = CI->getArgOperand(2);
1007-
Value *Surface = CI->getArgOperand(3);
1008-
Value *Offset = CI->getArgOperand(4);
1009-
1010-
CallInst *FirstGather =
1011-
IntrinsicInst::Create(F, {NewPreds.first, LogNumBlock, Scale, Surface,
1012-
Offset, NewEltOffsets.first, OldVals.first});
1013-
CallInst *SecondGather =
1014-
IntrinsicInst::Create(F, {NewPreds.second, LogNumBlock, Scale, Surface,
1015-
Offset, NewEltOffsets.second, OldVals.second});
1051+
CallInst *FirstGather = nullptr;
1052+
CallInst *SecondGather = nullptr;
1053+
if (IID == llvm::GenXIntrinsic::genx_gather_scaled) {
1054+
Value *LogNumBlock = CI->getArgOperand(1);
1055+
Value *Scale = CI->getArgOperand(2);
1056+
Value *Surface = CI->getArgOperand(3);
1057+
Value *Offset = CI->getArgOperand(4);
1058+
1059+
FirstGather =
1060+
IntrinsicInst::Create(F, {NewPreds.first, LogNumBlock, Scale, Surface,
1061+
Offset, NewEltOffsets.first, OldVals.first});
1062+
SecondGather = IntrinsicInst::Create(
1063+
F, {NewPreds.second, LogNumBlock, Scale, Surface, Offset,
1064+
NewEltOffsets.second, OldVals.second});
1065+
} else if (IID == llvm::GenXIntrinsic::genx_svm_gather) {
1066+
Value *LogNumBlock = CI->getArgOperand(1);
1067+
FirstGather = IntrinsicInst::Create(
1068+
F, {NewPreds.first, LogNumBlock, NewEltOffsets.first, OldVals.first});
1069+
SecondGather =
1070+
IntrinsicInst::Create(F, {NewPreds.second, LogNumBlock,
1071+
NewEltOffsets.second, OldVals.second});
1072+
}
10161073

10171074
FirstGather->insertAfter(CI);
10181075
SecondGather->insertAfter(FirstGather);

0 commit comments

Comments
 (0)