@@ -14349,125 +14349,126 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
14349
14349
EltSize, NumElem, Idx->isDivergent(), getSubtarget());
14350
14350
}
14351
14351
14352
- SDValue SITargetLowering::performExtractVectorEltCombine(
14353
- SDNode * N, DAGCombinerInfo & DCI) const {
14354
- SDValue Vec = N->getOperand(0);
14355
- SelectionDAG &DAG = DCI.DAG;
14356
-
14357
- EVT VecVT = Vec.getValueType();
14358
- EVT VecEltVT = VecVT.getVectorElementType();
14359
- EVT ResVT = N->getValueType(0);
14352
+ SDValue
14353
+ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
14354
+ DAGCombinerInfo &DCI) const {
14355
+ SDValue Vec = N->getOperand(0);
14356
+ SelectionDAG &DAG = DCI.DAG;
14360
14357
14361
- unsigned VecSize = VecVT.getSizeInBits();
14362
- unsigned VecEltSize = VecEltVT.getSizeInBits();
14358
+ EVT VecVT = Vec.getValueType();
14359
+ EVT VecEltVT = VecVT.getVectorElementType();
14360
+ EVT ResVT = N->getValueType(0);
14363
14361
14364
- if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14365
- allUsesHaveSourceMods(N)) {
14366
- SDLoc SL(N);
14367
- SDValue Idx = N->getOperand(1);
14368
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14369
- Vec.getOperand(0), Idx);
14370
- return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14371
- }
14372
-
14373
- // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14374
- // =>
14375
- // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14376
- // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14377
- // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14378
- if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14379
- SDLoc SL(N);
14380
- SDValue Idx = N->getOperand(1);
14381
- unsigned Opc = Vec.getOpcode();
14362
+ unsigned VecSize = VecVT.getSizeInBits();
14363
+ unsigned VecEltSize = VecEltVT.getSizeInBits();
14382
14364
14383
- switch (Opc) {
14384
- default:
14385
- break;
14386
- // TODO: Support other binary operations.
14387
- case ISD::FADD:
14388
- case ISD::FSUB:
14389
- case ISD::FMUL:
14390
- case ISD::ADD:
14391
- case ISD::UMIN:
14392
- case ISD::UMAX:
14393
- case ISD::SMIN:
14394
- case ISD::SMAX:
14395
- case ISD::FMAXNUM:
14396
- case ISD::FMINNUM:
14397
- case ISD::FMAXNUM_IEEE:
14398
- case ISD::FMINNUM_IEEE:
14399
- case ISD::FMAXIMUM:
14400
- case ISD::FMINIMUM: {
14401
- SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14402
- Vec.getOperand(0), Idx);
14403
- SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14404
- Vec.getOperand(1), Idx);
14405
-
14406
- DCI.AddToWorklist(Elt0.getNode());
14407
- DCI.AddToWorklist(Elt1.getNode());
14408
- return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14409
- }
14410
- }
14411
- }
14365
+ if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14366
+ allUsesHaveSourceMods(N)) {
14367
+ SDLoc SL(N);
14368
+ SDValue Idx = N->getOperand(1);
14369
+ SDValue Elt =
14370
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
14371
+ return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14372
+ }
14373
+
14374
+ // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14375
+ // =>
14376
+ // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14377
+ // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14378
+ // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14379
+ if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14380
+ SDLoc SL(N);
14381
+ SDValue Idx = N->getOperand(1);
14382
+ unsigned Opc = Vec.getOpcode();
14412
14383
14413
- // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14414
- if (shouldExpandVectorDynExt(N)) {
14415
- SDLoc SL(N);
14416
- SDValue Idx = N->getOperand(1);
14417
- SDValue V;
14418
- for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14419
- SDValue IC = DAG.getVectorIdxConstant(I, SL);
14420
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14421
- if (I == 0)
14422
- V = Elt;
14423
- else
14424
- V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14425
- }
14426
- return V;
14384
+ switch (Opc) {
14385
+ default:
14386
+ break;
14387
+ // TODO: Support other binary operations.
14388
+ case ISD::FADD:
14389
+ case ISD::FSUB:
14390
+ case ISD::FMUL:
14391
+ case ISD::ADD:
14392
+ case ISD::UMIN:
14393
+ case ISD::UMAX:
14394
+ case ISD::SMIN:
14395
+ case ISD::SMAX:
14396
+ case ISD::FMAXNUM:
14397
+ case ISD::FMINNUM:
14398
+ case ISD::FMAXNUM_IEEE:
14399
+ case ISD::FMINNUM_IEEE:
14400
+ case ISD::FMAXIMUM:
14401
+ case ISD::FMINIMUM: {
14402
+ SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14403
+ Vec.getOperand(0), Idx);
14404
+ SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14405
+ Vec.getOperand(1), Idx);
14406
+
14407
+ DCI.AddToWorklist(Elt0.getNode());
14408
+ DCI.AddToWorklist(Elt1.getNode());
14409
+ return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14410
+ }
14411
+ }
14412
+ }
14413
+
14414
+ // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14415
+ if (shouldExpandVectorDynExt(N)) {
14416
+ SDLoc SL(N);
14417
+ SDValue Idx = N->getOperand(1);
14418
+ SDValue V;
14419
+ for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14420
+ SDValue IC = DAG.getVectorIdxConstant(I, SL);
14421
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14422
+ if (I == 0)
14423
+ V = Elt;
14424
+ else
14425
+ V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14427
14426
}
14427
+ return V;
14428
+ }
14428
14429
14429
- if (!DCI.isBeforeLegalize())
14430
- return SDValue();
14431
-
14432
- // Try to turn sub-dword accesses of vectors into accesses of the same
14433
- // 32-bit elements. This exposes more load reduction opportunities by
14434
- // replacing multiple small extract_vector_elements with a single 32-bit
14435
- // extract.
14436
- auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14437
- if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14438
- VecSize > 32 && VecSize % 32 == 0 && Idx) {
14439
- EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14440
-
14441
- unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14442
- unsigned EltIdx = BitIndex / 32;
14443
- unsigned LeftoverBitIdx = BitIndex % 32;
14444
- SDLoc SL(N);
14430
+ if (!DCI.isBeforeLegalize())
14431
+ return SDValue();
14445
14432
14446
- SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14447
- DCI.AddToWorklist(Cast.getNode());
14433
+ // Try to turn sub-dword accesses of vectors into accesses of the same
14434
+ // 32-bit elements. This exposes more load reduction opportunities by
14435
+ // replacing multiple small extract_vector_elements with a single 32-bit
14436
+ // extract.
14437
+ auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14438
+ if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14439
+ VecSize > 32 && VecSize % 32 == 0 && Idx) {
14440
+ EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14441
+
14442
+ unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14443
+ unsigned EltIdx = BitIndex / 32;
14444
+ unsigned LeftoverBitIdx = BitIndex % 32;
14445
+ SDLoc SL(N);
14448
14446
14449
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14450
- DAG.getConstant(EltIdx, SL, MVT::i32));
14451
- DCI.AddToWorklist(Elt.getNode());
14452
- SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14453
- DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14454
- DCI.AddToWorklist(Srl.getNode());
14447
+ SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14448
+ DCI.AddToWorklist(Cast.getNode());
14455
14449
14456
- EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14457
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
14458
- DCI.AddToWorklist(Trunc.getNode());
14450
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14451
+ DAG.getConstant(EltIdx, SL, MVT::i32));
14452
+ DCI.AddToWorklist(Elt.getNode());
14453
+ SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14454
+ DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14455
+ DCI.AddToWorklist(Srl.getNode());
14459
14456
14460
- if (VecEltVT == ResVT) {
14461
- return DAG.getNode(ISD::BITCAST , SL, VecEltVT, Trunc );
14462
- }
14457
+ EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14458
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE , SL, VecEltAsIntVT, Srl );
14459
+ DCI.AddToWorklist(Trunc.getNode());
14463
14460
14464
- assert( ResVT.isScalarInteger());
14465
- return DAG.getAnyExtOrTrunc(Trunc , SL, ResVT );
14461
+ if (VecEltVT == ResVT) {
14462
+ return DAG.getNode(ISD::BITCAST , SL, VecEltVT, Trunc );
14466
14463
}
14467
14464
14468
- return SDValue();
14465
+ assert(ResVT.isScalarInteger());
14466
+ return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
14469
14467
}
14470
14468
14469
+ return SDValue();
14470
+ }
14471
+
14471
14472
SDValue
14472
14473
SITargetLowering::performInsertVectorEltCombine(SDNode *N,
14473
14474
DAGCombinerInfo &DCI) const {
0 commit comments