[LoopUnroll] Introduce parallel accumulators when unrolling FP reductions. #166630

juliannagele · 2025-11-05T20:23:49Z

This is building on top of #149470, also introducing parallel accumulator PHIs when the reduction is for floating points, provided we have the reassoc flag. See also #166353, which aims to introduce parallel accumulators for reductions with vector instructions.

…ions.

llvmbot · 2025-11-05T20:24:28Z

@llvm/pr-subscribers-llvm-transforms

Author: Julian Nagele (juliannagele)

Changes

This is building on top of #149470, also introducing parallel accumulator PHIs when the reduction is for floating points, provided we have the reassoc flag. See also #166353, which aims to introduce parallel accumulators for reductions with vector instructions.

Full diff: https://github.com/llvm/llvm-project/pull/166630.diff

3 Files Affected:

(modified) llvm/lib/Transforms/Utils/LoopUnroll.cpp (+9-3)
(modified) llvm/test/Transforms/LoopUnroll/partial-unroll-reductions.ll (+14-7)
(modified) llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll (+133)

diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 94dfd3a974923..d3d7faf6ab7ff 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1094,6 +1094,7 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
       if (!RdxResult) {
         RdxResult = PartialReductions.front();
         IRBuilder Builder(ExitBlock, ExitBlock->getFirstNonPHIIt());
+        Builder.setFastMathFlags(Reductions.begin()->second.getFastMathFlags());
         RecurKind RK = Reductions.begin()->second.getRecurrenceKind();
         for (Instruction *RdxPart : drop_begin(PartialReductions)) {
           RdxResult = Builder.CreateBinOp(
@@ -1256,14 +1257,19 @@ llvm::canParallelizeReductionWhenUnrolling(PHINode &Phi, Loop *L,
     return std::nullopt;
   RecurKind RK = RdxDesc.getRecurrenceKind();
   // Skip unsupported reductions.
-  // TODO: Handle additional reductions, including FP and min-max
-  // reductions.
-  if (!RecurrenceDescriptor::isIntegerRecurrenceKind(RK) ||
+  // TODO: Handle additional reductions, including min-max reductions.
+  if (!(RecurrenceDescriptor::isIntegerRecurrenceKind(RK) ||
+        RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) ||
       RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
       RecurrenceDescriptor::isFindIVRecurrenceKind(RK) ||
       RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))
     return std::nullopt;
 
+  if (RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) {
+    if (!RdxDesc.getFastMathFlags().allowReassoc())
+      return std::nullopt;
+  }
+
   if (RdxDesc.IntermediateStore)
     return std::nullopt;
 
diff --git a/llvm/test/Transforms/LoopUnroll/partial-unroll-reductions.ll b/llvm/test/Transforms/LoopUnroll/partial-unroll-reductions.ll
index 2d48d20ba9c5c..d7965b3061c70 100644
--- a/llvm/test/Transforms/LoopUnroll/partial-unroll-reductions.ll
+++ b/llvm/test/Transforms/LoopUnroll/partial-unroll-reductions.ll
@@ -319,27 +319,33 @@ define float @test_fadd_with_ressaoc(ptr %src, i64 %n, float %start) {
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT_3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi float [ [[START]], %[[ENTRY]] ], [ [[RDX_NEXT_3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_1:%.*]] = phi float [ -0.000000e+00, %[[ENTRY]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_2:%.*]] = phi float [ -0.000000e+00, %[[ENTRY]] ], [ [[RDX_NEXT_2:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_3:%.*]] = phi float [ -0.000000e+00, %[[ENTRY]] ], [ [[RDX_NEXT_3:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi float [ [[START]], %[[ENTRY]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 1
-; CHECK-NEXT:    [[RDX_NEXT:%.*]] = fadd float [[RDX]], [[L]]
+; CHECK-NEXT:    [[RDX_NEXT]] = fadd reassoc float [[RDX]], [[L]]
 ; CHECK-NEXT:    [[IV_NEXT_1:%.*]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV_NEXT]]
 ; CHECK-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 1
-; CHECK-NEXT:    [[RDX_NEXT_1:%.*]] = fadd float [[RDX_NEXT]], [[L_1]]
+; CHECK-NEXT:    [[RDX_NEXT_1]] = fadd reassoc float [[RDX_1]], [[L_1]]
 ; CHECK-NEXT:    [[IV_NEXT_2:%.*]] = add nuw nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV_NEXT_1]]
 ; CHECK-NEXT:    [[L_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 1
-; CHECK-NEXT:    [[RDX_NEXT_2:%.*]] = fadd float [[RDX_NEXT_1]], [[L_2]]
+; CHECK-NEXT:    [[RDX_NEXT_2]] = fadd reassoc float [[RDX_2]], [[L_2]]
 ; CHECK-NEXT:    [[IV_NEXT_3]] = add nuw nsw i64 [[IV]], 4
 ; CHECK-NEXT:    [[GEP_SRC_24:%.*]] = getelementptr float, ptr [[SRC]], i64 [[IV_NEXT_2]]
 ; CHECK-NEXT:    [[L_24:%.*]] = load float, ptr [[GEP_SRC_24]], align 1
-; CHECK-NEXT:    [[RDX_NEXT_3]] = fadd float [[RDX_NEXT_2]], [[L_24]]
+; CHECK-NEXT:    [[RDX_NEXT_3]] = fadd reassoc float [[RDX_3]], [[L_24]]
 ; CHECK-NEXT:    [[EC_3:%.*]] = icmp ne i64 [[IV_NEXT_3]], 1000
 ; CHECK-NEXT:    br i1 [[EC_3]], label %[[LOOP]], label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RDX_NEXT_LCSSA:%.*]] = phi float [ [[RDX_NEXT_3]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_NEXT_LCSSA1:%.*]] = phi float [ [[RDX_NEXT_3]], %[[LOOP]] ]
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT:    [[BIN_RDX1:%.*]] = fadd reassoc float [[RDX_NEXT_2]], [[BIN_RDX]]
+; CHECK-NEXT:    [[RDX_NEXT_LCSSA:%.*]] = fadd reassoc float [[RDX_NEXT_3]], [[BIN_RDX1]]
 ; CHECK-NEXT:    ret float [[RDX_NEXT_LCSSA]]
 ;
 entry:
@@ -351,13 +357,14 @@ loop:
   %iv.next = add i64 %iv, 1
   %gep.src = getelementptr float, ptr %src, i64 %iv
   %l = load float, ptr %gep.src, align 1
-  %rdx.next = fadd float %rdx, %l
+  %rdx.next = fadd reassoc float %rdx, %l
   %ec = icmp ne i64 %iv.next, 1000
   br i1 %ec, label %loop, label %exit
 
 exit:
   ret float %rdx.next
 }
+
 define i32 @test_smin(ptr %src, i64 %n) {
 ; CHECK-LABEL: define i32 @test_smin(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
index a5ac2cf46653d..591401edc143a 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll
@@ -220,6 +220,137 @@ exit:
   ret i32 %res
 }
 
+define float @test_fadd_reduction(ptr %a, i64 %n) {
+; CHECK-LABEL: define float @test_fadd_reduction(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK:       [[ENTRY_NEW]]:
+; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_1:%.*]] = phi float [ -0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16
+; CHECK-NEXT:    [[RDX_NEXT]] = fadd reassoc float [[RDX]], [[TMP2]]
+; CHECK-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16
+; CHECK-NEXT:    [[RDX_NEXT_1]] = fadd reassoc float [[RDX_1]], [[TMP3]]
+; CHECK-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT:    br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT:    [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc float [[RDX_NEXT_1]], [[RDX_NEXT]]
+; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT:    [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[LCMP_MOD2]])
+; CHECK-NEXT:    br label %[[LOOP_EPIL:.*]]
+; CHECK:       [[LOOP_EPIL]]:
+; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16
+; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = fadd reassoc float [[RDX_EPIL_INIT]], [[TMP4]]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi float [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT:    ret float [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]
+  %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv
+  %1 = load float, ptr %gep.a, align 16
+  %rdx.next = fadd reassoc float %rdx, %1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop, !llvm.loop !0
+
+exit:
+  %res = phi float [ %rdx.next, %loop ]
+  ret float %res
+}
+
+define float @test_fadd_no_reassoc(ptr %a, i64 %n) {
+; CHECK-LABEL: define float @test_fadd_no_reassoc(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK:       [[ENTRY_NEW]]:
+; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[RDX_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[GEP_A]], align 16
+; CHECK-NEXT:    [[RDX_NEXT:%.*]] = fadd float [[RDX]], [[TMP2]]
+; CHECK-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_NEXT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[GEP_A_1]], align 16
+; CHECK-NEXT:    [[RDX_NEXT_1]] = fadd float [[RDX_NEXT]], [[TMP3]]
+; CHECK-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; CHECK-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; CHECK-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; CHECK-NEXT:    br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT:    [[RES_PH:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX_UNR:%.*]] = phi float [ [[RDX_NEXT_1]], %[[LOOP]] ]
+; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP_EPIL_PREHEADER]]:
+; CHECK-NEXT:    [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[RDX_EPIL_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT:    [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[LCMP_MOD2]])
+; CHECK-NEXT:    br label %[[LOOP_EPIL:.*]]
+; CHECK:       [[LOOP_EPIL]]:
+; CHECK-NEXT:    [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV_EPIL_INIT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[GEP_A_EPIL]], align 16
+; CHECK-NEXT:    [[RDX_NEXT_EPIL:%.*]] = fadd float [[RDX_EPIL_INIT]], [[TMP4]]
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RES:%.*]] = phi float [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ]
+; CHECK-NEXT:    ret float [[RES]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]
+  %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv
+  %1 = load float, ptr %gep.a, align 16
+  %rdx.next = fadd float %rdx, %1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop, !llvm.loop !0
+
+exit:
+  %res = phi float [ %rdx.next, %loop ]
+  ret float %res
+}
 
 
 !0 = distinct !{!0, !1}
@@ -234,4 +365,6 @@ exit:
 ; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
 ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
 ;.

fhahn · 2025-11-06T12:04:07Z

llvm/lib/Transforms/Utils/LoopUnroll.cpp

+  if (RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) {
+    if (!RdxDesc.getFastMathFlags().allowReassoc())
+      return std::nullopt;
+  }


Do we need the check here? I think the recurrence descriptor analysis should take care of checking the required fast-math flags (in RecurrenceDescriptor::isRecurrenceInstr), and return RecurKind::None, if the required flags are missing.

Hm, afaict isRecurrenceInstr doesn't return None but remembers the instruction that doesn't have reassoc

InstDesc(Kind == RecurKind::FAdd, I, I->hasAllowReassoc() ? nullptr : I);

So I think we do need a check but we could simplify it to

if (RdxDesc.hasExactFPMath()) return std::nullopt;

where

/// Returns true if the recurrence has floating-point math that requires /// precise (ordered) operations. bool hasExactFPMath() const { return ExactFPMathInst != nullptr; }

juliannagele · 2025-11-14T10:24:12Z

ping :)

fhahn · 2025-11-17T16:16:50Z

llvm/lib/Transforms/Utils/LoopUnroll.cpp

+  if (!(RecurrenceDescriptor::isIntegerRecurrenceKind(RK) ||
+        RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) ||


Suggested change

if (!(RecurrenceDescriptor::isIntegerRecurrenceKind(RK) ||

RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) ||

if (

I think the condition is always true, given how isFloatingPointRecurrenceKind is implemented, as long as RK != None, which RecurrenceDescriptor::isReductionPHI should already ensure

bool RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurKind Kind) { <<< return (Kind != RecurKind::None) && !isIntegerRecurrenceKind(Kind); }

Makes sense, thanks -- done!

… reductions.

…ll-fp-reductions

github-actions · 2025-11-20T17:43:37Z

✅ With the latest revision this PR passed the C/C++ code formatter.

github-actions · 2025-11-20T18:38:32Z

🐧 Linux x64 Test Results

186518 tests passed
4879 tests skipped

…#166353) In combination with #149470 this will introduce parallel accumulators when unrolling reductions with vector instructions. See also #166630, which aims to introduce parallel accumulators for FP reductions.

…structions. (#166353) In combination with llvm/llvm-project#149470 this will introduce parallel accumulators when unrolling reductions with vector instructions. See also llvm/llvm-project#166630, which aims to introduce parallel accumulators for FP reductions.

fhahn

LGTM, thanks

fhahn · 2025-11-24T13:51:48Z

llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll

+  %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]
+  %gep.a = getelementptr inbounds nuw float, ptr %a, i64 %iv
+  %1 = load float, ptr %gep.a, align 16
+  %rdx.next = fadd reassoc float %rdx, %1


Could you also add a test with a different fast-math flag, to make sure we don't introduce additional accumulators in that case?

done, thanks!

…ll-fp-reductions

… reductions.

…llvm#166353) In combination with llvm#149470 this will introduce parallel accumulators when unrolling reductions with vector instructions. See also llvm#166630, which aims to introduce parallel accumulators for FP reductions.

…llvm#166353) In combination with llvm#149470 this will introduce parallel accumulators when unrolling reductions with vector instructions. See also llvm#166630, which aims to introduce parallel accumulators for FP reductions. (cherry picked from commit c73de97)

…ions. (llvm#166630) This is building on top of llvm#149470, also introducing parallel accumulator PHIs when the reduction is for floating points, provided we have the reassoc flag. See also llvm#166353, which aims to introduce parallel accumulators for reductions with vector instructions. (cherry picked from commit b641509)

… reductions. (llvm#166630)

…llvm#166353) In combination with llvm#149470 this will introduce parallel accumulators when unrolling reductions with vector instructions. See also llvm#166630, which aims to introduce parallel accumulators for FP reductions.

…ions. (llvm#166630) This is building on top of llvm#149470, also introducing parallel accumulator PHIs when the reduction is for floating points, provided we have the reassoc flag. See also llvm#166353, which aims to introduce parallel accumulators for reductions with vector instructions.

[LoopUnroll] Introduce parallel accumulators when unrolling FP reduct…

214c03f

…ions.

juliannagele requested a review from fhahn November 5, 2025 20:23

llvmbot added the llvm:transforms label Nov 5, 2025

juliannagele mentioned this pull request Nov 5, 2025

[IVDesciptors] Support detecting reductions with vector instructions. #166353

Merged

fhahn requested review from efriedma-quic, nikic and preames November 6, 2025 12:01

fhahn reviewed Nov 6, 2025

View reviewed changes

fhahn reviewed Nov 17, 2025

View reviewed changes

juliannagele added 2 commits November 20, 2025 17:39

fixup! [LoopUnroll] Introduce parallel accumulators when unrolling FP…

7284232

… reductions.

Merge branch 'main' of https://github.com/llvm/llvm-project into unro…

3bdcf3c

…ll-fp-reductions

juliannagele and others added 2 commits November 20, 2025 17:44

!fixup Fix formatting

3e3db0b

Merge branch 'main' into unroll-fp-reductions

9c401f3

fhahn approved these changes Nov 24, 2025

View reviewed changes

juliannagele added 2 commits November 24, 2025 15:25

Merge branch 'main' of https://github.com/llvm/llvm-project into unro…

da24236

…ll-fp-reductions

fixup! [LoopUnroll] Introduce parallel accumulators when unrolling FP…

739c5e1

… reductions.

juliannagele merged commit b641509 into llvm:main Nov 27, 2025
10 checks passed

juliannagele added a commit to juliannagele/llvm-project that referenced this pull request Nov 28, 2025

fixup! [LoopUnroll] Introduce parallel accumulators when unrolling FP…

c431cb1

… reductions. (llvm#166630)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[LoopUnroll] Introduce parallel accumulators when unrolling FP reductions. #166630

[LoopUnroll] Introduce parallel accumulators when unrolling FP reductions. #166630

Uh oh!

juliannagele commented Nov 5, 2025

Uh oh!

llvmbot commented Nov 5, 2025

Uh oh!

fhahn Nov 6, 2025

Uh oh!

juliannagele Nov 7, 2025

Uh oh!

juliannagele commented Nov 14, 2025

Uh oh!

fhahn Nov 17, 2025

Uh oh!

juliannagele Nov 20, 2025

Uh oh!

github-actions bot commented Nov 20, 2025 •

edited

Loading

Uh oh!

github-actions bot commented Nov 20, 2025 •

edited

Loading

Uh oh!

fhahn left a comment

Uh oh!

fhahn Nov 24, 2025

Uh oh!

juliannagele Nov 24, 2025

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

		if (!(RecurrenceDescriptor::isIntegerRecurrenceKind(RK) \|\|
		RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) \|\|

	if (!(RecurrenceDescriptor::isIntegerRecurrenceKind(RK) \|\|
	RecurrenceDescriptor::isFloatingPointRecurrenceKind(RK)) \|\|
	if (

[LoopUnroll] Introduce parallel accumulators when unrolling FP reductions. #166630

[LoopUnroll] Introduce parallel accumulators when unrolling FP reductions. #166630

Uh oh!

Conversation

juliannagele commented Nov 5, 2025

Uh oh!

llvmbot commented Nov 5, 2025

Uh oh!

fhahn Nov 6, 2025

Choose a reason for hiding this comment

Uh oh!

juliannagele Nov 7, 2025

Choose a reason for hiding this comment

Uh oh!

juliannagele commented Nov 14, 2025

Uh oh!

fhahn Nov 17, 2025

Choose a reason for hiding this comment

Uh oh!

juliannagele Nov 20, 2025

Choose a reason for hiding this comment

Uh oh!

github-actions bot commented Nov 20, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

github-actions bot commented Nov 20, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

🐧 Linux x64 Test Results

Uh oh!

fhahn left a comment

Choose a reason for hiding this comment

Uh oh!

fhahn Nov 24, 2025

Choose a reason for hiding this comment

Uh oh!

juliannagele Nov 24, 2025

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

github-actions bot commented Nov 20, 2025 •

edited

Loading

github-actions bot commented Nov 20, 2025 •

edited

Loading