@@ -14349,126 +14349,125 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
14349
14349
EltSize, NumElem, Idx->isDivergent(), getSubtarget());
14350
14350
}
14351
14351
14352
- SDValue
14353
- SITargetLowering::performExtractVectorEltCombine(SDNode *N,
14354
- DAGCombinerInfo &DCI) const {
14355
- SDValue Vec = N->getOperand(0);
14356
- SelectionDAG &DAG = DCI.DAG;
14352
+ SDValue SITargetLowering::performExtractVectorEltCombine(
14353
+ SDNode * N, DAGCombinerInfo & DCI) const {
14354
+ SDValue Vec = N->getOperand(0);
14355
+ SelectionDAG &DAG = DCI.DAG;
14357
14356
14358
- EVT VecVT = Vec.getValueType();
14359
- EVT VecEltVT = VecVT.getVectorElementType();
14360
- EVT ResVT = N->getValueType(0);
14357
+ EVT VecVT = Vec.getValueType();
14358
+ EVT VecEltVT = VecVT.getVectorElementType();
14359
+ EVT ResVT = N->getValueType(0);
14361
14360
14362
- unsigned VecSize = VecVT.getSizeInBits();
14363
- unsigned VecEltSize = VecEltVT.getSizeInBits();
14361
+ unsigned VecSize = VecVT.getSizeInBits();
14362
+ unsigned VecEltSize = VecEltVT.getSizeInBits();
14364
14363
14365
- if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14366
- allUsesHaveSourceMods(N)) {
14367
- SDLoc SL(N);
14368
- SDValue Idx = N->getOperand(1);
14369
- SDValue Elt =
14370
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec.getOperand(0), Idx);
14371
- return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14372
- }
14373
-
14374
- // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14375
- // =>
14376
- // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14377
- // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14378
- // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14379
- if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14380
- SDLoc SL(N);
14381
- SDValue Idx = N->getOperand(1);
14382
- unsigned Opc = Vec.getOpcode();
14364
+ if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
14365
+ allUsesHaveSourceMods(N)) {
14366
+ SDLoc SL(N);
14367
+ SDValue Idx = N->getOperand(1);
14368
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14369
+ Vec.getOperand(0), Idx);
14370
+ return DAG.getNode(Vec.getOpcode(), SL, ResVT, Elt);
14371
+ }
14372
+
14373
+ // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
14374
+ // =>
14375
+ // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
14376
+ // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
14377
+ // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
14378
+ if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
14379
+ SDLoc SL(N);
14380
+ SDValue Idx = N->getOperand(1);
14381
+ unsigned Opc = Vec.getOpcode();
14383
14382
14384
- switch (Opc) {
14385
- default:
14386
- break;
14387
- // TODO: Support other binary operations.
14388
- case ISD::FADD:
14389
- case ISD::FSUB:
14390
- case ISD::FMUL:
14391
- case ISD::ADD:
14392
- case ISD::UMIN:
14393
- case ISD::UMAX:
14394
- case ISD::SMIN:
14395
- case ISD::SMAX:
14396
- case ISD::FMAXNUM:
14397
- case ISD::FMINNUM:
14398
- case ISD::FMAXNUM_IEEE:
14399
- case ISD::FMINNUM_IEEE:
14400
- case ISD::FMAXIMUM:
14401
- case ISD::FMINIMUM: {
14402
- SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14403
- Vec.getOperand(0), Idx);
14404
- SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14405
- Vec.getOperand(1), Idx);
14406
-
14407
- DCI.AddToWorklist(Elt0.getNode());
14408
- DCI.AddToWorklist(Elt1.getNode());
14409
- return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14410
- }
14411
- }
14412
- }
14413
-
14414
- // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14415
- if (shouldExpandVectorDynExt(N)) {
14416
- SDLoc SL(N);
14417
- SDValue Idx = N->getOperand(1);
14418
- SDValue V;
14419
- for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14420
- SDValue IC = DAG.getVectorIdxConstant(I, SL);
14421
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14422
- if (I == 0)
14423
- V = Elt;
14424
- else
14425
- V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14383
+ switch (Opc) {
14384
+ default:
14385
+ break;
14386
+ // TODO: Support other binary operations.
14387
+ case ISD::FADD:
14388
+ case ISD::FSUB:
14389
+ case ISD::FMUL:
14390
+ case ISD::ADD:
14391
+ case ISD::UMIN:
14392
+ case ISD::UMAX:
14393
+ case ISD::SMIN:
14394
+ case ISD::SMAX:
14395
+ case ISD::FMAXNUM:
14396
+ case ISD::FMINNUM:
14397
+ case ISD::FMAXNUM_IEEE:
14398
+ case ISD::FMINNUM_IEEE:
14399
+ case ISD::FMAXIMUM:
14400
+ case ISD::FMINIMUM: {
14401
+ SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14402
+ Vec.getOperand(0), Idx);
14403
+ SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
14404
+ Vec.getOperand(1), Idx);
14405
+
14406
+ DCI.AddToWorklist(Elt0.getNode());
14407
+ DCI.AddToWorklist(Elt1.getNode());
14408
+ return DAG.getNode(Opc, SL, ResVT, Elt0, Elt1, Vec->getFlags());
14409
+ }
14410
+ }
14426
14411
}
14427
- return V;
14428
- }
14429
14412
14430
- if (!DCI.isBeforeLegalize())
14431
- return SDValue();
14413
+ // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
14414
+ if (shouldExpandVectorDynExt(N)) {
14415
+ SDLoc SL(N);
14416
+ SDValue Idx = N->getOperand(1);
14417
+ SDValue V;
14418
+ for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
14419
+ SDValue IC = DAG.getVectorIdxConstant(I, SL);
14420
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT, Vec, IC);
14421
+ if (I == 0)
14422
+ V = Elt;
14423
+ else
14424
+ V = DAG.getSelectCC(SL, Idx, IC, Elt, V, ISD::SETEQ);
14425
+ }
14426
+ return V;
14427
+ }
14432
14428
14433
- // Try to turn sub-dword accesses of vectors into accesses of the same
14434
- // 32-bit elements. This exposes more load reduction opportunities by
14435
- // replacing multiple small extract_vector_elements with a single 32-bit
14436
- // extract.
14437
- auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14438
- if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14439
- VecSize > 32 && VecSize % 32 == 0 && Idx) {
14440
- EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14441
-
14442
- unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14443
- unsigned EltIdx = BitIndex / 32;
14444
- unsigned LeftoverBitIdx = BitIndex % 32;
14445
- SDLoc SL(N);
14429
+ if (!DCI.isBeforeLegalize())
14430
+ return SDValue();
14431
+
14432
+ // Try to turn sub-dword accesses of vectors into accesses of the same
14433
+ // 32-bit elements. This exposes more load reduction opportunities by
14434
+ // replacing multiple small extract_vector_elements with a single 32-bit
14435
+ // extract.
14436
+ auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1));
14437
+ if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
14438
+ VecSize > 32 && VecSize % 32 == 0 && Idx) {
14439
+ EVT NewVT = getEquivalentMemType(*DAG.getContext(), VecVT);
14440
+
14441
+ unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
14442
+ unsigned EltIdx = BitIndex / 32;
14443
+ unsigned LeftoverBitIdx = BitIndex % 32;
14444
+ SDLoc SL(N);
14446
14445
14447
- SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14448
- DCI.AddToWorklist(Cast.getNode());
14446
+ SDValue Cast = DAG.getNode(ISD::BITCAST, SL, NewVT, Vec);
14447
+ DCI.AddToWorklist(Cast.getNode());
14449
14448
14450
- SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14451
- DAG.getConstant(EltIdx, SL, MVT::i32));
14452
- DCI.AddToWorklist(Elt.getNode());
14453
- SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14454
- DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14455
- DCI.AddToWorklist(Srl.getNode());
14449
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast,
14450
+ DAG.getConstant(EltIdx, SL, MVT::i32));
14451
+ DCI.AddToWorklist(Elt.getNode());
14452
+ SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt,
14453
+ DAG.getConstant(LeftoverBitIdx, SL, MVT::i32));
14454
+ DCI.AddToWorklist(Srl.getNode());
14456
14455
14457
- EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14458
- SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
14459
- DCI.AddToWorklist(Trunc.getNode());
14456
+ EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
14457
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VecEltAsIntVT, Srl);
14458
+ DCI.AddToWorklist(Trunc.getNode());
14459
+
14460
+ if (VecEltVT == ResVT) {
14461
+ return DAG.getNode(ISD::BITCAST, SL, VecEltVT, Trunc);
14462
+ }
14460
14463
14461
- if (VecEltVT == ResVT) {
14462
- return DAG.getNode(ISD::BITCAST , SL, VecEltVT, Trunc );
14464
+ assert( ResVT.isScalarInteger());
14465
+ return DAG.getAnyExtOrTrunc(Trunc , SL, ResVT );
14463
14466
}
14464
14467
14465
- assert(ResVT.isScalarInteger());
14466
- return DAG.getAnyExtOrTrunc(Trunc, SL, ResVT);
14468
+ return SDValue();
14467
14469
}
14468
14470
14469
- return SDValue();
14470
- }
14471
-
14472
14471
SDValue
14473
14472
SITargetLowering::performInsertVectorEltCombine(SDNode *N,
14474
14473
DAGCombinerInfo &DCI) const {
0 commit comments