@@ -4915,12 +4915,12 @@ void CGOpenMPRuntime::emitPrivateReduction(
49154915 // - Thread enters critical section.
49164916 // - Reads its private value from LHSExprs[i].
49174917 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
4918- // LHSExprs [i]).
4918+ // Privates [i]).
49194919 // - Exits critical section.
49204920 //
49214921 // Call __kmpc_barrier after combining.
49224922 //
4923- // Each thread copies __shared_reduction_var[i] back to LHSExprs [i].
4923+ // Each thread copies __shared_reduction_var[i] back to RHSExprs [i].
49244924 //
49254925 // Final __kmpc_barrier to synchronize after broadcasting
49264926 QualType PrivateType = Privates->getType();
@@ -5025,7 +5025,7 @@ void CGOpenMPRuntime::emitPrivateReduction(
50255025 const Expr *ReductionOp = ReductionOps;
50265026 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
50275027 LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType);
5028- LValue LHSLV = CGF.EmitLValue(LHSExprs );
5028+ LValue LHSLV = CGF.EmitLValue(Privates );
50295029
50305030 auto EmitCriticalReduction = [&](auto ReductionGen) {
50315031 std::string CriticalName = getName({"reduction_critical"});
@@ -5114,7 +5114,7 @@ void CGOpenMPRuntime::emitPrivateReduction(
51145114 else
51155115 FinalResultVal = CGF.EmitLoadOfScalar(SharedLV1, Loc);
51165116
5117- LValue TargetLHSLV = CGF.EmitLValue(LHSExprs );
5117+ LValue TargetLHSLV = CGF.EmitLValue(RHSExprs );
51185118 if (IsAggregate) {
51195119 CGF.EmitAggregateCopy(TargetLHSLV,
51205120 CGF.MakeAddrLValue(FinalResultAddr, PrivateType),
@@ -5126,13 +5126,23 @@ void CGOpenMPRuntime::emitPrivateReduction(
51265126 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
51275127 CGM.getModule(), OMPRTL___kmpc_barrier),
51285128 BarrierArgs);
5129+
5130+ // Combiner with original list item
5131+ auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5132+ PrePostActionTy &Action) {
5133+ Action.Enter(CGF);
5134+ emitSingleReductionCombiner(CGF, ReductionOps, Privates,
5135+ cast<DeclRefExpr>(LHSExprs),
5136+ cast<DeclRefExpr>(RHSExprs));
5137+ };
5138+ EmitCriticalReduction(OriginalListCombiner);
51295139}
51305140
51315141void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5132- ArrayRef<const Expr *> Privates ,
5133- ArrayRef<const Expr *> LHSExprs ,
5134- ArrayRef<const Expr *> RHSExprs ,
5135- ArrayRef<const Expr *> ReductionOps ,
5142+ ArrayRef<const Expr *> OrgPrivates ,
5143+ ArrayRef<const Expr *> OrgLHSExprs ,
5144+ ArrayRef<const Expr *> OrgRHSExprs ,
5145+ ArrayRef<const Expr *> OrgReductionOps ,
51365146 ReductionOptionsTy Options) {
51375147 if (!CGF.HaveInsertPoint())
51385148 return;
@@ -5179,10 +5189,10 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
51795189
51805190 if (SimpleReduction) {
51815191 CodeGenFunction::RunCleanupsScope Scope(CGF);
5182- const auto *IPriv = Privates .begin();
5183- const auto *ILHS = LHSExprs .begin();
5184- const auto *IRHS = RHSExprs .begin();
5185- for (const Expr *E : ReductionOps ) {
5192+ const auto *IPriv = OrgPrivates .begin();
5193+ const auto *ILHS = OrgLHSExprs .begin();
5194+ const auto *IRHS = OrgRHSExprs .begin();
5195+ for (const Expr *E : OrgReductionOps ) {
51865196 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
51875197 cast<DeclRefExpr>(*IRHS));
51885198 ++IPriv;
@@ -5192,6 +5202,26 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
51925202 return;
51935203 }
51945204
5205+ // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5206+ // Only keep entries where the corresponding variable is not private.
5207+ SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5208+ FilteredRHSExprs, FilteredReductionOps;
5209+ for (unsigned I : llvm::seq<unsigned>(
5210+ std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
5211+ if (!Options.IsPrivateVarReduction[I]) {
5212+ FilteredPrivates.emplace_back(OrgPrivates[I]);
5213+ FilteredLHSExprs.emplace_back(OrgLHSExprs[I]);
5214+ FilteredRHSExprs.emplace_back(OrgRHSExprs[I]);
5215+ FilteredReductionOps.emplace_back(OrgReductionOps[I]);
5216+ }
5217+ }
5218+ // Wrap filtered vectors in ArrayRef for downstream shared reduction
5219+ // processing.
5220+ ArrayRef<const Expr *> Privates = FilteredPrivates;
5221+ ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5222+ ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5223+ ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5224+
51955225 // 1. Build a list of reduction variables.
51965226 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
51975227 auto Size = RHSExprs.size();
@@ -5439,11 +5469,12 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
54395469 "PrivateVarReduction: ReductionOps size mismatch");
54405470 assert(LHSExprs.size() == Options.IsPrivateVarReduction.size() &&
54415471 "PrivateVarReduction: IsPrivateVarReduction size mismatch");
5442- for (unsigned I :
5443- llvm::seq<unsigned>(std::min(ReductionOps.size(), LHSExprs.size()))) {
5472+
5473+ for (unsigned I : llvm::seq<unsigned>(
5474+ std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) {
54445475 if (Options.IsPrivateVarReduction[I])
5445- emitPrivateReduction(CGF, Loc, Privates [I], LHSExprs[I], RHSExprs [I],
5446- ReductionOps [I]);
5476+ emitPrivateReduction(CGF, Loc, OrgPrivates [I], OrgLHSExprs [I],
5477+ OrgRHSExprs[I], OrgReductionOps [I]);
54475478 }
54485479}
54495480
0 commit comments