Skip to content

Commit 1f3f987

Browse files
fhahnfhahn
committed
[VPlan] Fix crash when narrowing interleave-groups with reuse.
If a wide load is used multiple times in an expression, it will be narrowed the first time. Re-use the already narrowed op in that case to fix crash.
1 parent ea60057 commit 1f3f987

File tree

2 files changed

+176
-0
lines changed

2 files changed

+176
-0
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3251,6 +3251,12 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
32513251
return L;
32523252
}
32533253

3254+
if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
3255+
assert(RepR->isSingleScalar() &&
3256+
isa<LoadInst>(RepR->getUnderlyingInstr()) &&
3257+
"must be a single scalar load");
3258+
return RepR;
3259+
}
32543260
auto *WideLoad = cast<VPWidenLoadRecipe>(R);
32553261

32563262
// Narrow wide load to uniform scalar load, as transformed VPlan will only

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,4 +277,174 @@ exit:
277277
ret void
278278
}
279279

280+
define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %n) #0 {
281+
; CHECK-LABEL: define void @single_fmul_used_by_each_member(
282+
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
283+
; CHECK-NEXT: [[ITER_CHECK:.*]]:
284+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
285+
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
286+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
287+
; CHECK: [[VECTOR_SCEVCHECK]]:
288+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 8
289+
; CHECK-NEXT: [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
290+
; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
291+
; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
292+
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 0, [[MUL_RESULT]]
293+
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
294+
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]
295+
; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
296+
; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
297+
; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
298+
; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
299+
; CHECK-NEXT: [[TMP5:%.*]] = sub i64 0, [[MUL_RESULT2]]
300+
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT2]]
301+
; CHECK-NEXT: [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[B]]
302+
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
303+
; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 8
304+
; CHECK-NEXT: [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
305+
; CHECK-NEXT: [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
306+
; CHECK-NEXT: [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
307+
; CHECK-NEXT: [[TMP9:%.*]] = sub i64 0, [[MUL_RESULT6]]
308+
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
309+
; CHECK-NEXT: [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
310+
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
311+
; CHECK-NEXT: [[MUL8:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
312+
; CHECK-NEXT: [[MUL_RESULT9:%.*]] = extractvalue { i64, i1 } [[MUL8]], 0
313+
; CHECK-NEXT: [[MUL_OVERFLOW10:%.*]] = extractvalue { i64, i1 } [[MUL8]], 1
314+
; CHECK-NEXT: [[TMP13:%.*]] = sub i64 0, [[MUL_RESULT9]]
315+
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT9]]
316+
; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[C]]
317+
; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW10]]
318+
; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP8]]
319+
; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP17]], [[TMP12]]
320+
; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP16]]
321+
; CHECK-NEXT: br i1 [[TMP19]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
322+
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
323+
; CHECK-NEXT: [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[TMP0]], 8
324+
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK11]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
325+
; CHECK: [[VECTOR_PH]]:
326+
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
327+
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
328+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
329+
; CHECK: [[VECTOR_BODY]]:
330+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
331+
; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 2
332+
; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[INDEX]], 4
333+
; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX]], 6
334+
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX]]
335+
; CHECK-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[TMP23]], i32 0
336+
; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, ptr [[TMP23]], i32 2
337+
; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[TMP23]], i32 4
338+
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[TMP23]], i32 6
339+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
340+
; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <2 x double>, ptr [[TMP25]], align 8
341+
; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <2 x double>, ptr [[TMP26]], align 8
342+
; CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <2 x double>, ptr [[TMP27]], align 8
343+
; CHECK-NEXT: [[TMP28:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00)
344+
; CHECK-NEXT: [[TMP29:%.*]] = fmul <2 x double> [[WIDE_LOAD12]], splat (double 5.000000e+00)
345+
; CHECK-NEXT: [[TMP30:%.*]] = fmul <2 x double> [[WIDE_LOAD13]], splat (double 5.000000e+00)
346+
; CHECK-NEXT: [[TMP31:%.*]] = fmul <2 x double> [[WIDE_LOAD14]], splat (double 5.000000e+00)
347+
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
348+
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP20]]
349+
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP21]]
350+
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP22]]
351+
; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> [[TMP28]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
352+
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP36]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
353+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP32]], align 8
354+
; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> [[TMP29]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
355+
; CHECK-NEXT: [[INTERLEAVED_VEC15:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
356+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP33]], align 8
357+
; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> [[TMP30]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
358+
; CHECK-NEXT: [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP38]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
359+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
360+
; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP31]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
361+
; CHECK-NEXT: [[INTERLEAVED_VEC17:%.*]] = shufflevector <4 x double> [[TMP39]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
362+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP35]], align 8
363+
; CHECK-NEXT: [[TMP40:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
364+
; CHECK-NEXT: [[TMP41:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP20]]
365+
; CHECK-NEXT: [[TMP42:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP21]]
366+
; CHECK-NEXT: [[TMP43:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP22]]
367+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP40]], align 8
368+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP41]], align 8
369+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP42]], align 8
370+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP43]], align 8
371+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
372+
; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
373+
; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
374+
; CHECK: [[MIDDLE_BLOCK]]:
375+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
376+
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
377+
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
378+
; CHECK-NEXT: [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
379+
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
380+
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
381+
; CHECK: [[VEC_EPILOG_PH]]:
382+
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
383+
; CHECK-NEXT: [[N_MOD_VF22:%.*]] = urem i64 [[TMP0]], 2
384+
; CHECK-NEXT: [[N_VEC23:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF22]]
385+
; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
386+
; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
387+
; CHECK-NEXT: [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
388+
; CHECK-NEXT: [[TMP45:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX24]]
389+
; CHECK-NEXT: [[TMP46:%.*]] = getelementptr double, ptr [[TMP45]], i32 0
390+
; CHECK-NEXT: [[TMP47:%.*]] = load double, ptr [[TMP46]], align 8
391+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP47]], i64 0
392+
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
393+
; CHECK-NEXT: [[TMP48:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT]], splat (double 5.000000e+00)
394+
; CHECK-NEXT: [[TMP49:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX24]]
395+
; CHECK-NEXT: store <2 x double> [[TMP48]], ptr [[TMP49]], align 8
396+
; CHECK-NEXT: [[TMP50:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX24]]
397+
; CHECK-NEXT: store <2 x double> [[TMP48]], ptr [[TMP50]], align 8
398+
; CHECK-NEXT: [[INDEX_NEXT25]] = add nuw i64 [[INDEX24]], 1
399+
; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
400+
; CHECK-NEXT: br i1 [[TMP51]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
401+
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
402+
; CHECK-NEXT: [[CMP_N26:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC23]]
403+
; CHECK-NEXT: br i1 [[CMP_N26]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
404+
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
405+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
406+
; CHECK-NEXT: br label %[[LOOP:.*]]
407+
; CHECK: [[LOOP]]:
408+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
409+
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr double, ptr [[A]], i64 [[IV]]
410+
; CHECK-NEXT: [[L:%.*]] = load double, ptr [[GEP_A]], align 8
411+
; CHECK-NEXT: [[DIV:%.*]] = fmul double [[L]], 5.000000e+00
412+
; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]], i32 1
413+
; CHECK-NEXT: store double [[DIV]], ptr [[GEP_B_1]], align 8
414+
; CHECK-NEXT: [[GEP_C_1:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]], i32 1
415+
; CHECK-NEXT: store double [[DIV]], ptr [[GEP_C_1]], align 8
416+
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]]
417+
; CHECK-NEXT: store double [[DIV]], ptr [[GEP_B]], align 8
418+
; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]]
419+
; CHECK-NEXT: store double [[DIV]], ptr [[GEP_C]], align 8
420+
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
421+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
422+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
423+
; CHECK: [[EXIT]]:
424+
; CHECK-NEXT: ret void
425+
;
426+
entry:
427+
br label %loop
428+
429+
loop:
430+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
431+
%gep.A = getelementptr double, ptr %A, i64 %iv
432+
%l = load double, ptr %gep.A, align 8
433+
%div = fmul double %l, 5.000000e+00
434+
%gep.B.1 = getelementptr {double, double}, ptr %B, i64 %iv, i32 1
435+
store double %div, ptr %gep.B.1, align 8
436+
%gep.C.1 = getelementptr {double, double}, ptr %C, i64 %iv, i32 1
437+
store double %div, ptr %gep.C.1, align 8
438+
%gep.B = getelementptr {double, double}, ptr %B, i64 %iv
439+
store double %div, ptr %gep.B, align 8
440+
%gep.C = getelementptr {double, double}, ptr %C, i64 %iv
441+
store double %div, ptr %gep.C, align 8
442+
%iv.next = add i64 %iv, 1
443+
%ec = icmp eq i64 %iv, %n
444+
br i1 %ec, label %exit, label %loop
445+
446+
exit:
447+
ret void
448+
}
449+
280450
attributes #0 = { "target-cpu"="neoverse-v2" }

0 commit comments

Comments
 (0)