@@ -102,6 +102,56 @@ static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
102
102
return false ;
103
103
}
104
104
105
+ // / Do the common operand retrieval and validition required by the
106
+ // / routines below.
107
+ static bool getMemOperands (unsigned Factor, VectorType *VTy, Type *XLenTy,
108
+ Instruction *I, Value *&Ptr, Value *&Mask,
109
+ Value *&VL, Align &Alignment) {
110
+
111
+ IRBuilder<> Builder (I);
112
+ const DataLayout &DL = I->getDataLayout ();
113
+ ElementCount EC = VTy->getElementCount ();
114
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
115
+ assert (LI->isSimple ());
116
+ Ptr = LI->getPointerOperand ();
117
+ Alignment = LI->getAlign ();
118
+ assert (!Mask && " Unexpected mask on a load" );
119
+ Mask = Builder.getAllOnesMask (EC);
120
+ VL = isa<FixedVectorType>(VTy) ? Builder.CreateElementCount (XLenTy, EC)
121
+ : Constant::getAllOnesValue (XLenTy);
122
+ return true ;
123
+ }
124
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
125
+ assert (SI->isSimple ());
126
+ Ptr = SI->getPointerOperand ();
127
+ Alignment = SI->getAlign ();
128
+ assert (!Mask && " Unexpected mask on a store" );
129
+ Mask = Builder.getAllOnesMask (EC);
130
+ VL = isa<FixedVectorType>(VTy) ? Builder.CreateElementCount (XLenTy, EC)
131
+ : Constant::getAllOnesValue (XLenTy);
132
+ return true ;
133
+ }
134
+ auto *VPLdSt = cast<VPIntrinsic>(I);
135
+ assert ((VPLdSt->getIntrinsicID () == Intrinsic::vp_load ||
136
+ VPLdSt->getIntrinsicID () == Intrinsic::vp_store) &&
137
+ " Unexpected intrinsic" );
138
+ Ptr = VPLdSt->getMemoryPointerParam ();
139
+ Alignment = VPLdSt->getPointerAlignment ().value_or (
140
+ DL.getABITypeAlign (VTy->getElementType ()));
141
+
142
+ assert (Mask && " vp.load and vp.store needs a mask!" );
143
+
144
+ Value *WideEVL = VPLdSt->getVectorLengthParam ();
145
+ // Conservatively check if EVL is a multiple of factor, otherwise some
146
+ // (trailing) elements might be lost after the transformation.
147
+ if (!isMultipleOfN (WideEVL, I->getDataLayout (), Factor))
148
+ return false ;
149
+
150
+ auto *FactorC = ConstantInt::get (WideEVL->getType (), Factor);
151
+ VL = Builder.CreateZExt (Builder.CreateExactUDiv (WideEVL, FactorC), XLenTy);
152
+ return true ;
153
+ }
154
+
105
155
// / Lower an interleaved load into a vlsegN intrinsic.
106
156
// /
107
157
// / E.g. Lower an interleaved load (Factor = 2):
@@ -127,32 +177,8 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
127
177
128
178
Value *Ptr, *VL;
129
179
Align Alignment;
130
- if (auto *LI = dyn_cast<LoadInst>(Load)) {
131
- assert (LI->isSimple ());
132
- Ptr = LI->getPointerOperand ();
133
- Alignment = LI->getAlign ();
134
- assert (!Mask && " Unexpected mask on a load\n " );
135
- Mask = Builder.getAllOnesMask (VTy->getElementCount ());
136
- VL = Builder.CreateElementCount (XLenTy, VTy->getElementCount ());
137
- } else {
138
- auto *VPLoad = cast<VPIntrinsic>(Load);
139
- assert (VPLoad->getIntrinsicID () == Intrinsic::vp_load &&
140
- " Unexpected intrinsic" );
141
- Ptr = VPLoad->getMemoryPointerParam ();
142
- Alignment = VPLoad->getPointerAlignment ().value_or (
143
- DL.getABITypeAlign (VTy->getElementType ()));
144
-
145
- assert (Mask && " vp.load needs a mask!" );
146
-
147
- Value *WideEVL = VPLoad->getVectorLengthParam ();
148
- // Conservatively check if EVL is a multiple of factor, otherwise some
149
- // (trailing) elements might be lost after the transformation.
150
- if (!isMultipleOfN (WideEVL, DL, Factor))
151
- return false ;
152
-
153
- auto *FactorC = ConstantInt::get (WideEVL->getType (), Factor);
154
- VL = Builder.CreateZExt (Builder.CreateExactUDiv (WideEVL, FactorC), XLenTy);
155
- }
180
+ if (!getMemOperands (Factor, VTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
181
+ return false ;
156
182
157
183
Type *PtrTy = Ptr->getType ();
158
184
unsigned AS = PtrTy->getPointerAddressSpace ();
@@ -296,34 +322,8 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
296
322
297
323
Value *Ptr, *VL;
298
324
Align Alignment;
299
- if (auto *LI = dyn_cast<LoadInst>(Load)) {
300
- assert (LI->isSimple ());
301
- Ptr = LI->getPointerOperand ();
302
- Alignment = LI->getAlign ();
303
- assert (!Mask && " Unexpected mask on a load\n " );
304
- Mask = Builder.getAllOnesMask (ResVTy->getElementCount ());
305
- VL = isa<FixedVectorType>(ResVTy)
306
- ? Builder.CreateElementCount (XLenTy, ResVTy->getElementCount ())
307
- : Constant::getAllOnesValue (XLenTy);
308
- } else {
309
- auto *VPLoad = cast<VPIntrinsic>(Load);
310
- assert (VPLoad->getIntrinsicID () == Intrinsic::vp_load &&
311
- " Unexpected intrinsic" );
312
- Ptr = VPLoad->getMemoryPointerParam ();
313
- Alignment = VPLoad->getPointerAlignment ().value_or (
314
- DL.getABITypeAlign (ResVTy->getElementType ()));
315
-
316
- assert (Mask && " vp.load needs a mask!" );
317
-
318
- Value *WideEVL = VPLoad->getVectorLengthParam ();
319
- // Conservatively check if EVL is a multiple of factor, otherwise some
320
- // (trailing) elements might be lost after the transformation.
321
- if (!isMultipleOfN (WideEVL, Load->getDataLayout (), Factor))
322
- return false ;
323
-
324
- auto *FactorC = ConstantInt::get (WideEVL->getType (), Factor);
325
- VL = Builder.CreateZExt (Builder.CreateExactUDiv (WideEVL, FactorC), XLenTy);
326
- }
325
+ if (!getMemOperands (Factor, ResVTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
326
+ return false ;
327
327
328
328
Type *PtrTy = Ptr->getType ();
329
329
unsigned AS = PtrTy->getPointerAddressSpace ();
@@ -385,34 +385,8 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
385
385
386
386
Value *Ptr, *VL;
387
387
Align Alignment;
388
- if (auto *SI = dyn_cast<StoreInst>(Store)) {
389
- assert (SI->isSimple ());
390
- Ptr = SI->getPointerOperand ();
391
- Alignment = SI->getAlign ();
392
- assert (!Mask && " Unexpected mask on a store" );
393
- Mask = Builder.getAllOnesMask (InVTy->getElementCount ());
394
- VL = isa<FixedVectorType>(InVTy)
395
- ? Builder.CreateElementCount (XLenTy, InVTy->getElementCount ())
396
- : Constant::getAllOnesValue (XLenTy);
397
- } else {
398
- auto *VPStore = cast<VPIntrinsic>(Store);
399
- assert (VPStore->getIntrinsicID () == Intrinsic::vp_store &&
400
- " Unexpected intrinsic" );
401
- Ptr = VPStore->getMemoryPointerParam ();
402
- Alignment = VPStore->getPointerAlignment ().value_or (
403
- DL.getABITypeAlign (InVTy->getElementType ()));
404
-
405
- assert (Mask && " vp.store needs a mask!" );
406
-
407
- Value *WideEVL = VPStore->getVectorLengthParam ();
408
- // Conservatively check if EVL is a multiple of factor, otherwise some
409
- // (trailing) elements might be lost after the transformation.
410
- if (!isMultipleOfN (WideEVL, DL, Factor))
411
- return false ;
412
-
413
- auto *FactorC = ConstantInt::get (WideEVL->getType (), Factor);
414
- VL = Builder.CreateZExt (Builder.CreateExactUDiv (WideEVL, FactorC), XLenTy);
415
- }
388
+ if (!getMemOperands (Factor, InVTy, XLenTy, Store, Ptr, Mask, VL, Alignment))
389
+ return false ;
416
390
Type *PtrTy = Ptr->getType ();
417
391
unsigned AS = Ptr->getType ()->getPointerAddressSpace ();
418
392
if (!isLegalInterleavedAccessType (InVTy, Factor, Alignment, AS, DL))
0 commit comments