@@ -14319,126 +14319,125 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
14319
14319
EltSize, NumElem, Idx->isDivergent(), getSubtarget());
14320
14320
}
14321
14321
14322
- SDValue
14323
- SITargetLowering::performExtractVectorEltCombine(SDNode *N,
14324
- DAGCombinerInfo &DCI) const {
14325
- SDValue Vec = N->getOperand(0);
14326
- SelectionDAG &DAG = DCI.DAG;
14322
+ SDValue SITargetLowering::performExtractVectorEltCombine(
14323
+ SDNode * N, DAGCombinerInfo & DCI) const {
14324
+ SDValue Vec = N->getOperand(0);
14325
+ SelectionDAG &DAG = DCI.DAG;
14327
14326
14328
- EVT VecVT = Vec.getValueType();
14329
- EVT VecEltVT = VecVT.getVectorElementType();
14330
- EVT ResVT = N->getValueType(0);
14327
+ EVT VecVT = Vec.getValueType();
14328
+ EVT VecEltVT = VecVT.getVectorElementType();
14329
+ EVT ResVT = N->getValueType(0);
14331
14330
14332
- unsigned VecSize = VecVT.getSizeInBits();
14333
- unsigned VecEltSize = VecEltVT.getSizeInBits();
14331
+ unsigned VecSize = VecVT.getSizeInBits();
14332
+ unsigned VecEltSize = VecEltVT.getSizeInBits();
14334
14333
14335
- if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14336
- allUsesHaveSourceMods(N)) {
14337
- SDLoc SL(N);
14338
- SDValue Idx = N->getOperand(1);
14339
- SDValue Elt =
14340
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
14341
- return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14342
- }
14343
-
14344
- // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14345
- // =>
14346
- // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14347
- // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14348
- // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14349
- if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14350
- SDLoc SL(N);
14351
- SDValue Idx = N->getOperand(1);
14352
- unsigned Opc = Vec.getOpcode();
14334
+ if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14335
+ allUsesHaveSourceMods(N)) {
14336
+ SDLoc SL(N);
14337
+ SDValue Idx = N->getOperand(1);
14338
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14339
+ Vec.getOperand(0), Idx);
14340
+ return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14341
+ }
14342
+
14343
+ // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14344
+ // =>
14345
+ // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14346
+ // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14347
+ // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14348
+ if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14349
+ SDLoc SL(N);
14350
+ SDValue Idx = N->getOperand(1);
14351
+ unsigned Opc = Vec.getOpcode();
14353
14352
14354
- switch (Opc) {
14355
- default:
14356
- break;
14357
- // TODO: Support other binary operations.
14358
- case ISD::FADD:
14359
- case ISD::FSUB:
14360
- case ISD::FMUL:
14361
- case ISD::ADD:
14362
- case ISD::UMIN:
14363
- case ISD::UMAX:
14364
- case ISD::SMIN:
14365
- case ISD::SMAX:
14366
- case ISD::FMAXNUM:
14367
- case ISD::FMINNUM:
14368
- case ISD::FMAXNUM_IEEE:
14369
- case ISD::FMINNUM_IEEE:
14370
- case ISD::FMAXIMUM:
14371
- case ISD::FMINIMUM: {
14372
- SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14373
- Vec.getOperand(0), Idx);
14374
- SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14375
- Vec.getOperand(1), Idx);
14376
-
14377
- DCI.AddToWorklist(Elt0.getNode());
14378
- DCI.AddToWorklist(Elt1.getNode());
14379
- return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14380
- }
14381
- }
14382
- }
14383
-
14384
- // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14385
- if (shouldExpandVectorDynExt(N)) {
14386
- SDLoc SL(N);
14387
- SDValue Idx = N->getOperand(1);
14388
- SDValue V;
14389
- for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14390
- SDValue IC = DAG.getVectorIdxConstant(I, SL);
14391
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14392
- if (I == 0)
14393
- V = Elt;
14394
- else
14395
- V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14353
+ switch (Opc) {
14354
+ default:
14355
+ break;
14356
+ // TODO: Support other binary operations.
14357
+ case ISD::FADD:
14358
+ case ISD::FSUB:
14359
+ case ISD::FMUL:
14360
+ case ISD::ADD:
14361
+ case ISD::UMIN:
14362
+ case ISD::UMAX:
14363
+ case ISD::SMIN:
14364
+ case ISD::SMAX:
14365
+ case ISD::FMAXNUM:
14366
+ case ISD::FMINNUM:
14367
+ case ISD::FMAXNUM_IEEE:
14368
+ case ISD::FMINNUM_IEEE:
14369
+ case ISD::FMAXIMUM:
14370
+ case ISD::FMINIMUM: {
14371
+ SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14372
+ Vec.getOperand(0), Idx);
14373
+ SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14374
+ Vec.getOperand(1), Idx);
14375
+
14376
+ DCI.AddToWorklist(Elt0.getNode());
14377
+ DCI.AddToWorklist(Elt1.getNode());
14378
+ return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14379
+ }
14380
+ }
14396
14381
}
14397
- return V;
14398
- }
14399
14382
14400
- if (!DCI.isBeforeLegalize())
14401
- return SDValue();
14383
+ // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14384
+ if (shouldExpandVectorDynExt(N)) {
14385
+ SDLoc SL(N);
14386
+ SDValue Idx = N->getOperand(1);
14387
+ SDValue V;
14388
+ for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14389
+ SDValue IC = DAG.getVectorIdxConstant(I, SL);
14390
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14391
+ if (I == 0)
14392
+ V = Elt;
14393
+ else
14394
+ V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14395
+ }
14396
+ return V;
14397
+ }
14402
14398
14403
- // Try to turn sub-dword accesses of vectors into accesses of the same
14404
- // 32-bit elements. This exposes more load reduction opportunities by
14405
- // replacing multiple small extract_vector_elements with a single 32-bit
14406
- // extract.
14407
- auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14408
- if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14409
- VecSize > 32 && VecSize % 32 == 0 && Idx) {
14410
- EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14411
-
14412
- unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14413
- unsigned EltIdx = BitIndex / 32;
14414
- unsigned LeftoverBitIdx = BitIndex % 32;
14415
- SDLoc SL(N);
14399
+ if (!DCI.isBeforeLegalize())
14400
+ return SDValue();
14401
+
14402
+ // Try to turn sub-dword accesses of vectors into accesses of the same
14403
+ // 32-bit elements. This exposes more load reduction opportunities by
14404
+ // replacing multiple small extract_vector_elements with a single 32-bit
14405
+ // extract.
14406
+ auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14407
+ if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14408
+ VecSize > 32 && VecSize % 32 == 0 && Idx) {
14409
+ EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14410
+
14411
+ unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14412
+ unsigned EltIdx = BitIndex / 32;
14413
+ unsigned LeftoverBitIdx = BitIndex % 32;
14414
+ SDLoc SL(N);
14416
14415
14417
- SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14418
- DCI.AddToWorklist(Cast.getNode());
14416
+ SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14417
+ DCI.AddToWorklist(Cast.getNode());
14419
14418
14420
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14421
- DAG.getConstant(EltIdx, SL, MVT::i32));
14422
- DCI.AddToWorklist(Elt.getNode());
14423
- SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14424
- DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14425
- DCI.AddToWorklist(Srl.getNode());
14419
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14420
+ DAG.getConstant(EltIdx, SL, MVT::i32));
14421
+ DCI.AddToWorklist(Elt.getNode());
14422
+ SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14423
+ DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14424
+ DCI.AddToWorklist(Srl.getNode());
14426
14425
14427
- EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14428
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
14429
- DCI.AddToWorklist(Trunc.getNode());
14426
+ EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14427
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
14428
+ DCI.AddToWorklist(Trunc.getNode());
14429
+
14430
+ if (VecEltVT == ResVT) {
14431
+ return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc);
14432
+ }
14430
14433
14431
- if (VecEltVT == ResVT) {
14432
- return DAG.getNode(ISD::BITCAST , SL, VecEltVT, Trunc );
14434
+ assert( ResVT.isScalarInteger());
14435
+ return DAG.getAnyExtOrTrunc(Trunc , SL, ResVT );
14433
14436
}
14434
14437
14435
- assert(ResVT.isScalarInteger());
14436
- return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
14438
+ return SDValue();
14437
14439
}
14438
14440
14439
- return SDValue();
14440
- }
14441
-
14442
14441
SDValue
14443
14442
SITargetLowering::performInsertVectorEltCombine(SDNode *N,
14444
14443
DAGCombinerInfo &DCI) const {
0 commit comments