@@ -2627,25 +2627,22 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
26272627 NewLI =
26282628 Builder.CreateIntrinsic (DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
26292629 nullptr , " wide.masked.gather" );
2630+ } else if (isReverse ()) {
2631+ auto *EltTy = DataTy->getElementType ();
2632+ auto *PtrTy = Addr->getType ();
2633+ Value *Operands[] = {
2634+ Addr,
2635+ ConstantInt::getSigned (Builder.getInt32Ty (),
2636+ -LI->getDataLayout ().getTypeAllocSize (EltTy)),
2637+ Mask, EVL};
2638+ NewLI = Builder.CreateIntrinsic (Intrinsic::experimental_vp_strided_load,
2639+ {DataTy, PtrTy, Builder.getInt32Ty ()},
2640+ Operands, nullptr , " vp.neg.strided.load" );
26302641 } else {
2631- if (isReverse ()) {
2632- auto *EltTy = DataTy->getElementType ();
2633- auto *PtrTy = Addr->getType ();
2634- Value *Operands[] = {
2635- Addr,
2636- ConstantInt::getSigned (
2637- Builder.getInt32Ty (),
2638- -static_cast <int64_t >(EltTy->getScalarSizeInBits ()) / 8 ),
2639- Mask, EVL};
2640- NewLI = Builder.CreateIntrinsic (Intrinsic::experimental_vp_strided_load,
2641- {DataTy, PtrTy, Builder.getInt32Ty ()},
2642- Operands, nullptr , " vp.neg.strided.load" );
2643- } else {
2644- VectorBuilder VBuilder (Builder);
2645- VBuilder.setEVL (EVL).setMask (Mask);
2646- NewLI = cast<CallInst>(VBuilder.createVectorInstruction (
2647- Instruction::Load, DataTy, Addr, " vp.op.load" ));
2648- }
2642+ VectorBuilder VBuilder (Builder);
2643+ VBuilder.setEVL (EVL).setMask (Mask);
2644+ NewLI = cast<CallInst>(VBuilder.createVectorInstruction (
2645+ Instruction::Load, DataTy, Addr, " vp.op.load" ));
26492646 }
26502647 NewLI->addParamAttr (
26512648 0 , Attribute::getWithAlignment (NewLI->getContext (), Alignment));
@@ -2759,27 +2756,24 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
27592756 NewSI = Builder.CreateIntrinsic (Type::getVoidTy (EVL->getContext ()),
27602757 Intrinsic::vp_scatter,
27612758 {StoredVal, Addr, Mask, EVL});
2759+ } else if (isReverse ()) {
2760+ Type *StoredValTy = StoredVal->getType ();
2761+ auto *EltTy = cast<VectorType>(StoredValTy)->getElementType ();
2762+ auto *PtrTy = Addr->getType ();
2763+ Value *Operands[] = {
2764+ StoredVal, Addr,
2765+ ConstantInt::getSigned (Builder.getInt32Ty (),
2766+ -SI->getDataLayout ().getTypeAllocSize (EltTy)),
2767+ Mask, EVL};
2768+ NewSI = Builder.CreateIntrinsic (Intrinsic::experimental_vp_strided_store,
2769+ {StoredValTy, PtrTy, Builder.getInt32Ty ()},
2770+ Operands);
27622771 } else {
2763- if (isReverse ()) {
2764- Type *StoredValTy = StoredVal->getType ();
2765- auto *EltTy = cast<VectorType>(StoredValTy)->getElementType ();
2766- auto *PtrTy = Addr->getType ();
2767- Value *Operands[] = {
2768- StoredVal, Addr,
2769- ConstantInt::getSigned (
2770- Builder.getInt32Ty (),
2771- -static_cast <int64_t >(EltTy->getScalarSizeInBits ()) / 8 ),
2772- Mask, EVL};
2773- NewSI = Builder.CreateIntrinsic (
2774- Intrinsic::experimental_vp_strided_store,
2775- {StoredValTy, PtrTy, Builder.getInt32Ty ()}, Operands);
2776- } else {
2777- VectorBuilder VBuilder (Builder);
2778- VBuilder.setEVL (EVL).setMask (Mask);
2779- NewSI = cast<CallInst>(VBuilder.createVectorInstruction (
2780- Instruction::Store, Type::getVoidTy (EVL->getContext ()),
2781- {StoredVal, Addr}));
2782- }
2772+ VectorBuilder VBuilder (Builder);
2773+ VBuilder.setEVL (EVL).setMask (Mask);
2774+ NewSI = cast<CallInst>(VBuilder.createVectorInstruction (
2775+ Instruction::Store, Type::getVoidTy (EVL->getContext ()),
2776+ {StoredVal, Addr}));
27832777 }
27842778 NewSI->addParamAttr (
27852779 1 , Attribute::getWithAlignment (NewSI->getContext (), Alignment));
0 commit comments