Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 516b973

Browse files
CarolEidtCarolEidt
authored
Merge pull request #15749 from fiigii/tabledrive
Table-driven Intel hardware intrinsic
2 parents 3abfaf7 + 654a8d5 commit 516b973

22 files changed

+1395
-1012
lines changed

‎src/jit/compiler.h‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3042,11 +3042,11 @@ class Compiler
30423042
NamedIntrinsic lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method);
30433043

30443044
#if FEATURE_HW_INTRINSICS
3045-
InstructionSet lookupHWIntrinsicISA(const char* className);
3046-
NamedIntrinsic lookupHWIntrinsic(const char* methodName, InstructionSet isa);
3047-
InstructionSet isaOfHWIntrinsic(NamedIntrinsic intrinsic);
3048-
bool isIntrinsicAnIsSupportedPropertyGetter(NamedIntrinsic intrinsic);
3049-
bool isFullyImplmentedISAClass(InstructionSet isa);
3045+
static InstructionSet lookupHWIntrinsicISA(const char* className);
3046+
static NamedIntrinsic lookupHWIntrinsic(const char* methodName, InstructionSet isa);
3047+
static InstructionSet isaOfHWIntrinsic(NamedIntrinsic intrinsic);
3048+
static bool isIntrinsicAnIsSupportedPropertyGetter(NamedIntrinsic intrinsic);
3049+
static bool isFullyImplmentedISAClass(InstructionSet isa);
30503050
#ifdef _TARGET_XARCH_
30513051
GenTree* impUnsupportedHWIntrinsic(unsigned helper,
30523052
CORINFO_METHOD_HANDLE method,
@@ -3119,7 +3119,12 @@ class Compiler
31193119
bool compSupportsHWIntrinsic(InstructionSet isa);
31203120
bool isScalarISA(InstructionSet isa);
31213121
static int ivalOfHWIntrinsic(NamedIntrinsic intrinsic);
3122+
static int numArgsOfHWIntrinsic(NamedIntrinsic intrinsic);
31223123
static instruction insOfHWIntrinsic(NamedIntrinsic intrinsic, var_types type);
3124+
static HWIntrinsicCategory categoryOfHWIntrinsic(NamedIntrinsic intrinsic);
3125+
static HWIntrinsicFlag flagsOfHWIntrinsic(NamedIntrinsic intrinsic);
3126+
GenTree* getArgForHWIntrinsic(var_types argType, CORINFO_CLASS_HANDLE argClass);
3127+
GenTreeArgList* buildArgList(CORINFO_SIG_INFO* sig);
31233128
#endif // _TARGET_XARCH_
31243129
#endif // FEATURE_HW_INTRINSICS
31253130
GenTreePtr impArrayAccessIntrinsic(CORINFO_CLASS_HANDLE clsHnd,

‎src/jit/emit.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,6 +938,7 @@ class emitter
938938
struct
939939
{
940940
regNumber _idReg3 : REGNUM_BITS;
941+
regNumber _idReg4 : REGNUM_BITS;
941942
};
942943
#endif // defined(_TARGET_XARCH_)
943944

@@ -1119,6 +1120,19 @@ class emitter
11191120
idAddr()->_idReg3 = reg;
11201121
assert(reg == idAddr()->_idReg3);
11211122
}
1123+
regNumber idReg4() const
1124+
{
1125+
assert(!idIsTiny());
1126+
assert(!idIsSmallDsc());
1127+
return idAddr()->_idReg4;
1128+
}
1129+
void idReg4(regNumber reg)
1130+
{
1131+
assert(!idIsTiny());
1132+
assert(!idIsSmallDsc());
1133+
idAddr()->_idReg4 = reg;
1134+
assert(reg == idAddr()->_idReg4);
1135+
}
11221136
#endif // defined(_TARGET_XARCH_)
11231137
#ifdef _TARGET_ARMARCH_
11241138
insOpts idInsOpt() const

‎src/jit/emitfmtsxarch.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ IF_DEF(RRW_RRW_CNS, IS_R1_RW|IS_R2_RW, SCNS) // r/w reg , r/w r
110110

111111
IF_DEF(RWR_RRD_RRD, IS_R1_WR|IS_R2_RD|IS_R3_RD, NONE) // write reg , read reg2 , read reg3
112112
IF_DEF(RWR_RRD_RRD_CNS, IS_R1_WR|IS_R2_RD|IS_R3_RD, SCNS) // write reg , read reg2 , read reg3, const
113+
114+
IF_DEF(RWR_RRD_RRD_RRD, IS_R1_WR|IS_R2_RD|IS_R3_RD|IS_R4_RD, NONE) // write reg , read reg2 , read reg3 , read reg4
113115
//----------------------------------------------------------------------------
114116
// The following formats are used for direct addresses (e.g. static data members)
115117
//----------------------------------------------------------------------------

‎src/jit/emitxarch.cpp‎

Lines changed: 123 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ bool emitter::IsDstDstSrcAVXInstruction(instruction ins)
146146
case INS_pminub:
147147
case INS_pminud:
148148
case INS_pminuw:
149+
case INS_pmuldq:
149150
case INS_pmulld:
150151
case INS_pmullw:
151152
case INS_pmuludq:
@@ -4227,6 +4228,45 @@ void emitter::emitIns_R_R_S_I(
42274228
emitCurIGsize += sz;
42284229
}
42294230

4231+
#ifdef DEBUG
4232+
static bool isAvxBlendv(instruction ins)
4233+
{
4234+
return ins == INS_vblendvps || ins == INS_vblendvpd || ins == INS_vpblendvb;
4235+
}
4236+
4237+
static bool isSse41Blendv(instruction ins)
4238+
{
4239+
return ins == INS_blendvps || ins == INS_blendvpd || ins == INS_pblendvb;
4240+
}
4241+
#endif
4242+
4243+
void emitter::emitIns_R_R_R_R(
4244+
instruction ins, emitAttr attr, regNumber targetReg, regNumber reg1, regNumber reg2, regNumber reg3)
4245+
{
4246+
assert(isAvxBlendv(ins));
4247+
assert(UseVEXEncoding());
4248+
// Currently vex prefix only use three bytes mode.
4249+
// size = vex + opcode + ModR/M + 1-byte-cns(Reg) = 3 + 1 + 1 + 1 = 6
4250+
// TODO-XArch-CQ: We should create function which can calculate all kinds of AVX instructions size in future
4251+
UNATIVE_OFFSET sz = 6;
4252+
4253+
// AVX/AVX2 supports 4-reg format for vblendvps/vblendvpd/vpblendvb,
4254+
// which encodes the fourth register into imm8[7:4]
4255+
int ival = (reg3 - XMMBASE) << 4; // convert reg3 to ival
4256+
4257+
instrDesc* id = emitNewInstrCns(attr, ival);
4258+
id->idIns(ins);
4259+
id->idInsFmt(IF_RWR_RRD_RRD_RRD);
4260+
id->idReg1(targetReg);
4261+
id->idReg2(reg1);
4262+
id->idReg3(reg2);
4263+
id->idReg4(reg3);
4264+
4265+
id->idCodeSize(sz);
4266+
dispIns(id);
4267+
emitCurIGsize += sz;
4268+
}
4269+
42304270
/*****************************************************************************
42314271
*
42324272
* Add an instruction with a register + static member operands.
@@ -5166,158 +5206,191 @@ void emitter::emitIns_AX_R(instruction ins, emitAttr attr, regNumber ireg, regNu
51665206
}
51675207

51685208
#if FEATURE_HW_INTRINSICS
5169-
void emitter::emitIns_SIMD_R_R(instruction ins, regNumber reg, regNumber reg1, var_types simdtype)
5209+
void emitter::emitIns_SIMD_R_R_A(instruction ins, emitAttr attr, regNumber reg, regNumber reg1, GenTreeIndir* indir)
51705210
{
5171-
emitIns_R_R(ins, emitTypeSize(simdtype), reg, reg1);
5211+
if (UseVEXEncoding())
5212+
{
5213+
emitIns_R_R_A(ins, attr, reg, reg1, indir, IF_RWR_RRD_ARD);
5214+
}
5215+
else
5216+
{
5217+
if (reg1 != reg)
5218+
{
5219+
emitIns_R_R(INS_movaps, attr, reg, reg1);
5220+
}
5221+
emitIns_R_A(ins, attr, reg, indir, IF_RRW_ARD);
5222+
}
51725223
}
51735224

5174-
void emitter::emitIns_SIMD_R_R_A(
5175-
instruction ins, regNumber reg, regNumber reg1, GenTreeIndir* indir, var_types simdtype)
5225+
void emitter::emitIns_SIMD_R_R_AR(instruction ins, emitAttr attr, regNumber reg, regNumber reg1, regNumber base)
51765226
{
51775227
if (UseVEXEncoding())
51785228
{
5179-
emitIns_R_R_A(ins, emitTypeSize(simdtype), reg, reg1, indir, IF_RWR_RRD_ARD);
5229+
emitIns_R_R_AR(ins, attr, reg, reg1, base, 0);
51805230
}
51815231
else
51825232
{
51835233
if (reg1 != reg)
51845234
{
5185-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5235+
emitIns_R_R(INS_movaps, attr, reg, reg1);
51865236
}
5187-
emitIns_R_A(ins, emitTypeSize(simdtype), reg, indir, IF_RRW_ARD);
5237+
emitIns_R_AR(ins, attr, reg, base, 0);
51885238
}
51895239
}
51905240

5191-
void emitter::emitIns_SIMD_R_R_AR(instruction ins, regNumber reg, regNumber reg1, regNumber base, var_types simdtype)
5241+
void emitter::emitIns_SIMD_R_R_C(
5242+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, CORINFO_FIELD_HANDLE fldHnd, int offs)
51925243
{
51935244
if (UseVEXEncoding())
51945245
{
5195-
emitIns_R_R_AR(ins, emitTypeSize(simdtype), reg, reg1, base, 0);
5246+
emitIns_R_R_C(ins, attr, reg, reg1, fldHnd, offs);
51965247
}
51975248
else
51985249
{
51995250
if (reg1 != reg)
52005251
{
5201-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5252+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52025253
}
5203-
emitIns_R_AR(ins, emitTypeSize(simdtype), reg, base, 0);
5254+
emitIns_R_C(ins, attr, reg, fldHnd, offs);
52045255
}
52055256
}
52065257

5207-
void emitter::emitIns_SIMD_R_R_C(
5208-
instruction ins, regNumber reg, regNumber reg1, CORINFO_FIELD_HANDLE fldHnd, int offs, var_types simdtype)
5258+
void emitter::emitIns_SIMD_R_R_R(instruction ins, emitAttr attr, regNumber reg, regNumber reg1, regNumber reg2)
52095259
{
52105260
if (UseVEXEncoding())
52115261
{
5212-
emitIns_R_R_C(ins, emitTypeSize(simdtype), reg, reg1, fldHnd, offs);
5262+
emitIns_R_R_R(ins, attr, reg, reg1, reg2);
52135263
}
52145264
else
52155265
{
52165266
if (reg1 != reg)
52175267
{
5218-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5268+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52195269
}
5220-
emitIns_R_C(ins, emitTypeSize(simdtype), reg, fldHnd, offs);
5270+
emitIns_R_R(ins, attr, reg, reg2);
52215271
}
52225272
}
52235273

5224-
void emitter::emitIns_SIMD_R_R_R(instruction ins, regNumber reg, regNumber reg1, regNumber reg2, var_types simdtype)
5274+
void emitter::emitIns_SIMD_R_R_R_R(
5275+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, regNumber reg2, regNumber reg3)
52255276
{
5277+
assert(isAvxBlendv(ins) || isSse41Blendv(ins));
52265278
if (UseVEXEncoding())
52275279
{
5228-
emitIns_R_R_R(ins, emitTypeSize(simdtype), reg, reg1, reg2);
5280+
// convert SSE encoding of SSE4.1 instructions to VEX encoding
5281+
switch (ins)
5282+
{
5283+
case INS_blendvps:
5284+
ins = INS_vblendvps;
5285+
break;
5286+
case INS_blendvpd:
5287+
ins = INS_vblendvpd;
5288+
break;
5289+
case INS_pblendvb:
5290+
ins = INS_vpblendvb;
5291+
break;
5292+
default:
5293+
break;
5294+
}
5295+
emitIns_R_R_R_R(ins, attr, reg, reg1, reg2, reg3);
52295296
}
52305297
else
52315298
{
5299+
assert(isSse41Blendv(ins));
5300+
// SSE4.1 blendv* hardcode the mask vector (op3) in XMM0
5301+
if (reg3 != REG_XMM0)
5302+
{
5303+
emitIns_R_R(INS_movaps, attr, REG_XMM0, reg3);
5304+
}
52325305
if (reg1 != reg)
52335306
{
5234-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5307+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52355308
}
5236-
emitIns_R_R(ins, emitTypeSize(simdtype), reg, reg2);
5309+
emitIns_R_R(ins, attr, reg, reg2);
52375310
}
52385311
}
52395312

5240-
void emitter::emitIns_SIMD_R_R_S(instruction ins, regNumber reg, regNumber reg1, int varx, int offs, var_types simdtype)
5313+
void emitter::emitIns_SIMD_R_R_S(instruction ins, emitAttr attr, regNumber reg, regNumber reg1, int varx, int offs)
52415314
{
52425315
if (UseVEXEncoding())
52435316
{
5244-
emitIns_R_R_S(ins, emitTypeSize(simdtype), reg, reg1, varx, offs);
5317+
emitIns_R_R_S(ins, attr, reg, reg1, varx, offs);
52455318
}
52465319
else
52475320
{
52485321
if (reg1 != reg)
52495322
{
5250-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5323+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52515324
}
5252-
emitIns_R_S(ins, emitTypeSize(simdtype), reg, varx, offs);
5325+
emitIns_R_S(ins, attr, reg, varx, offs);
52535326
}
52545327
}
52555328

52565329
void emitter::emitIns_SIMD_R_R_A_I(
5257-
instruction ins, regNumber reg, regNumber reg1, GenTreeIndir* indir, int ival, var_types simdtype)
5330+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, GenTreeIndir* indir, int ival)
52585331
{
52595332
if (UseVEXEncoding())
52605333
{
5261-
emitIns_R_R_A_I(ins, emitTypeSize(simdtype), reg, reg1, indir, ival, IF_RWR_RRD_ARD_CNS);
5334+
emitIns_R_R_A_I(ins, attr, reg, reg1, indir, ival, IF_RWR_RRD_ARD_CNS);
52625335
}
52635336
else
52645337
{
52655338
if (reg1 != reg)
52665339
{
5267-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5340+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52685341
}
5269-
emitIns_R_A_I(ins, emitTypeSize(simdtype), reg, indir, ival);
5342+
emitIns_R_A_I(ins, attr, reg, indir, ival);
52705343
}
52715344
}
52725345

52735346
void emitter::emitIns_SIMD_R_R_C_I(
5274-
instruction ins, regNumber reg, regNumber reg1, CORINFO_FIELD_HANDLE fldHnd, int offs, int ival, var_types simdtype)
5347+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, CORINFO_FIELD_HANDLE fldHnd, int offs, int ival)
52755348
{
52765349
if (UseVEXEncoding())
52775350
{
5278-
emitIns_R_R_C_I(ins, emitTypeSize(simdtype), reg, reg1, fldHnd, offs, ival);
5351+
emitIns_R_R_C_I(ins, attr, reg, reg1, fldHnd, offs, ival);
52795352
}
52805353
else
52815354
{
52825355
if (reg1 != reg)
52835356
{
5284-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5357+
emitIns_R_R(INS_movaps, attr, reg, reg1);
52855358
}
5286-
emitIns_R_C_I(ins, emitTypeSize(simdtype), reg, fldHnd, offs, ival);
5359+
emitIns_R_C_I(ins, attr, reg, fldHnd, offs, ival);
52875360
}
52885361
}
52895362

52905363
void emitter::emitIns_SIMD_R_R_R_I(
5291-
instruction ins, regNumber reg, regNumber reg1, regNumber reg2, int ival, var_types simdtype)
5364+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, regNumber reg2, int ival)
52925365
{
52935366
if (UseVEXEncoding())
52945367
{
5295-
emitIns_R_R_R_I(ins, emitTypeSize(simdtype), reg, reg1, reg2, ival);
5368+
emitIns_R_R_R_I(ins, attr, reg, reg1, reg2, ival);
52965369
}
52975370
else
52985371
{
52995372
if (reg1 != reg)
53005373
{
5301-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5374+
emitIns_R_R(INS_movaps, attr, reg, reg1);
53025375
}
5303-
emitIns_R_R_I(ins, emitTypeSize(simdtype), reg, reg2, ival);
5376+
emitIns_R_R_I(ins, attr, reg, reg2, ival);
53045377
}
53055378
}
53065379

53075380
void emitter::emitIns_SIMD_R_R_S_I(
5308-
instruction ins, regNumber reg, regNumber reg1, int varx, int offs, int ival, var_types simdtype)
5381+
instruction ins, emitAttr attr, regNumber reg, regNumber reg1, int varx, int offs, int ival)
53095382
{
53105383
if (UseVEXEncoding())
53115384
{
5312-
emitIns_R_R_S_I(ins, emitTypeSize(simdtype), reg, reg1, varx, offs, ival);
5385+
emitIns_R_R_S_I(ins, attr, reg, reg1, varx, offs, ival);
53135386
}
53145387
else
53155388
{
53165389
if (reg1 != reg)
53175390
{
5318-
emitIns_R_R(INS_movaps, emitTypeSize(simdtype), reg, reg1);
5391+
emitIns_R_R(INS_movaps, attr, reg, reg1);
53195392
}
5320-
emitIns_R_S_I(ins, emitTypeSize(simdtype), reg, varx, offs, ival);
5393+
emitIns_R_S_I(ins, attr, reg, varx, offs, ival);
53215394
}
53225395
}
53235396
#endif
@@ -7653,6 +7726,14 @@ void emitter::emitDispIns(
76537726
val = emitGetInsSC(id);
76547727
goto PRINT_CONSTANT;
76557728
break;
7729+
case IF_RWR_RRD_RRD_RRD:
7730+
assert(IsAVXOnlyInstruction(ins));
7731+
assert(UseVEXEncoding());
7732+
printf("%s, ", emitRegName(id->idReg1(), attr));
7733+
printf("%s, ", emitRegName(id->idReg2(), attr));
7734+
printf("%s, ", emitRegName(id->idReg3(), attr));
7735+
printf("%s", emitRegName(id->idReg4(), attr));
7736+
break;
76567737
case IF_RRW_RRW_CNS:
76577738
printf("%s,", emitRegName(id->idReg1(), attr));
76587739
printf(" %s", emitRegName(id->idReg2(), attr));
@@ -10304,7 +10385,7 @@ BYTE* emitter::emitOutputRRR(BYTE* dst, instrDesc* id)
1030410385

1030510386
instruction ins = id->idIns();
1030610387
assert(IsAVXInstruction(ins));
10307-
assert(IsThreeOperandAVXInstruction(ins));
10388+
assert(IsThreeOperandAVXInstruction(ins) || isAvxBlendv(ins));
1030810389
regNumber targetReg = id->idReg1();
1030910390
regNumber src1 = id->idReg2();
1031010391
regNumber src2 = id->idReg3();
@@ -11570,6 +11651,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
1157011651
sz = emitSizeOfInsDsc(id);
1157111652
break;
1157211653
case IF_RWR_RRD_RRD_CNS:
11654+
case IF_RWR_RRD_RRD_RRD:
1157311655
dst = emitOutputRRR(dst, id);
1157411656
sz = emitSizeOfInsDsc(id);
1157511657
dst += emitOutputByte(dst, emitGetInsSC(id));

0 commit comments

Comments
 (0)