Skip to content

Commit e45c30a

Browse files
author
Chandra Ghale
committed
Handle user-defined reduction and updated lit test
1 parent 59ab4be commit e45c30a

File tree

2 files changed

+393
-85
lines changed

2 files changed

+393
-85
lines changed

‎clang/lib/CodeGen/CGOpenMPRuntime.cpp‎

Lines changed: 86 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -4914,30 +4914,22 @@ void CGOpenMPRuntime::emitPrivateReduction(
49144914
QualType PrivateType = Privates[0]->getType();
49154915
llvm::Type *LLVMType = CGF.ConvertTypeForMem(PrivateType);
49164916

4917-
BinaryOperatorKind MainBO = BO_Comma;
4918-
if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionOps[0])) {
4919-
if (const auto *RHSExpr = BinOp->getRHS()) {
4920-
if (const auto *BORHS =
4921-
dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
4922-
MainBO = BORHS->getOpcode();
4923-
}
4924-
}
4925-
}
4926-
49274917
llvm::Constant *InitVal = llvm::Constant::getNullValue(LLVMType);
4928-
const Expr *Private = Privates[0];
4929-
4930-
if (const auto *DRE = dyn_cast<DeclRefExpr>(Private)) {
4918+
const Expr *InitExpr = nullptr;
4919+
if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates[0])) {
49314920
if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
4932-
if (const Expr *Init = VD->getInit()) {
4933-
if (Init->isConstantInitializer(CGF.getContext(), false)) {
4921+
InitExpr = VD->getInit();
4922+
if (InitExpr && !PrivateType->isAggregateType()) {
4923+
if (InitExpr->isConstantInitializer(CGF.getContext(), false)) {
49344924
Expr::EvalResult Result;
4935-
if (Init->EvaluateAsRValue(Result, CGF.getContext())) {
4925+
if (InitExpr->EvaluateAsRValue(Result, CGF.getContext())) {
49364926
APValue &InitValue = Result.Val;
49374927
if (InitValue.isInt()) {
49384928
InitVal = llvm::ConstantInt::get(LLVMType, InitValue.getInt());
49394929
}
49404930
}
4931+
} else {
4932+
InitVal = llvm::Constant::getNullValue(LLVMType);
49414933
}
49424934
}
49434935
}
@@ -4972,7 +4964,25 @@ void CGOpenMPRuntime::emitPrivateReduction(
49724964
CGF.Builder.CreateCondBr(IsWorker, InitBB, InitEndBB);
49734965

49744966
CGF.EmitBlock(InitBB);
4975-
CGF.Builder.CreateStore(InitVal, SharedResult);
4967+
if (InitExpr) {
4968+
RValue RV = CGF.EmitAnyExpr(InitExpr);
4969+
if (RV.isAggregate()) {
4970+
CGF.Builder.CreateMemCpy(SharedResult, RV.getAggregateAddress(),
4971+
llvm::ConstantInt::get(CGF.IntPtrTy, 4),
4972+
/*IsVolatile=*/false);
4973+
} else {
4974+
CGF.Builder.CreateStore(RV.getScalarVal(), SharedResult);
4975+
}
4976+
} else {
4977+
if (PrivateType->isAggregateType()) {
4978+
CGF.Builder.CreateMemSet(SharedResult,
4979+
llvm::ConstantInt::get(CGM.Int8Ty, 0),
4980+
llvm::ConstantInt::get(CGF.IntPtrTy, 4),
4981+
/*IsVolatile=*/false);
4982+
} else {
4983+
CGF.Builder.CreateStore(InitVal, SharedResult);
4984+
}
4985+
}
49764986
CGF.Builder.CreateBr(InitEndBB);
49774987

49784988
CGF.EmitBlock(InitEndBB);
@@ -4983,46 +4993,75 @@ void CGOpenMPRuntime::emitPrivateReduction(
49834993

49844994
for (unsigned I :
49854995
llvm::seq<unsigned>(std::min(ReductionOps.size(), LHSExprs.size()))) {
4996+
const Expr *ReductionClauseExpr = ReductionOps[I]->IgnoreParenCasts();
4997+
if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
4998+
ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
4999+
const Expr *AssignRHS = nullptr;
5000+
const Expr *AssignLHS = nullptr;
5001+
5002+
if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5003+
if (BinOp->getOpcode() == BO_Assign) {
5004+
AssignLHS = BinOp->getLHS();
5005+
AssignRHS = BinOp->getRHS();
5006+
}
5007+
} else if (const auto *OpCall =
5008+
dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5009+
if (OpCall->getOperator() == OO_Equal) {
5010+
AssignLHS = OpCall->getArg(0);
5011+
AssignRHS = OpCall->getArg(1);
5012+
}
5013+
}
49865014

4987-
const auto *BinOp = dyn_cast<BinaryOperator>(ReductionOps[I]);
4988-
if (!BinOp || BinOp->getOpcode() != BO_Assign)
4989-
continue;
4990-
4991-
const Expr *RHSExpr = BinOp->getRHS();
4992-
if (!RHSExpr)
5015+
if (!AssignRHS || !AssignLHS) {
49935016
continue;
5017+
}
49945018

4995-
BinaryOperatorKind BO = BO_Comma;
4996-
const Expr *StripRHS = RHSExpr->IgnoreParenImpCasts();
4997-
if (const auto *BORHS = dyn_cast<BinaryOperator>(StripRHS)) {
4998-
BO = BORHS->getOpcode();
4999-
} else if (const auto *OpCall = dyn_cast<CXXOperatorCallExpr>(StripRHS)) {
5000-
BO = BinaryOperator::getOverloadedOpcode(OpCall->getOperator());
5019+
const Expr *ReductionCombinerExpr = AssignRHS->IgnoreParenImpCasts();
5020+
if (const auto *MTE =
5021+
dyn_cast<MaterializeTemporaryExpr>(ReductionCombinerExpr)) {
5022+
ReductionCombinerExpr = MTE->getSubExpr()->IgnoreParenImpCasts();
50015023
}
50025024

5025+
BinaryOperatorKind BO = BO_Assign;
50035026
LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType);
50045027
LValue LHSLV = CGF.EmitLValue(LHSExprs[I]);
50055028
RValue PrivateRV = CGF.EmitLoadOfLValue(LHSLV, Loc);
5006-
auto UpdateOp = [&](RValue OldVal) {
5007-
if (BO == BO_Mul) {
5008-
llvm::Value *OldScalar = OldVal.getScalarVal();
5009-
llvm::Value *PrivateScalar = PrivateRV.getScalarVal();
5010-
llvm::Value *Result = CGF.Builder.CreateMul(OldScalar, PrivateScalar);
5011-
return RValue::get(Result);
5012-
} else {
5013-
OpaqueValueExpr OVE(BinOp->getLHS()->getExprLoc(),
5014-
BinOp->getLHS()->getType(),
5015-
ExprValueKind::VK_PRValue);
5016-
CodeGenFunction::OpaqueValueMapping OldValMapping(CGF, &OVE, OldVal);
5017-
return CGF.EmitAnyExpr(BinOp->getRHS());
5018-
}
5019-
};
5029+
if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionCombinerExpr)) {
5030+
BO = BinOp->getOpcode();
5031+
auto UpdateOp = [&](RValue OldVal) {
5032+
if (BO == BO_Mul) {
5033+
llvm::Value *OldScalar = OldVal.getScalarVal();
5034+
llvm::Value *PrivateScalar = PrivateRV.getScalarVal();
5035+
llvm::Value *Result = CGF.Builder.CreateMul(OldScalar, PrivateScalar);
5036+
return RValue::get(Result);
5037+
} else {
5038+
OpaqueValueExpr OVE(BinOp->getLHS()->getExprLoc(),
5039+
BinOp->getLHS()->getType(),
5040+
ExprValueKind::VK_PRValue);
5041+
CodeGenFunction::OpaqueValueMapping OldValMapping(CGF, &OVE, OldVal);
5042+
return CGF.EmitAnyExpr(BinOp->getRHS());
5043+
}
5044+
};
50205045

5021-
(void)CGF.EmitOMPAtomicSimpleUpdateExpr(
5022-
SharedLV, PrivateRV, BO, true,
5023-
llvm::AtomicOrdering::SequentiallyConsistent, Loc, UpdateOp);
5046+
(void)CGF.EmitOMPAtomicSimpleUpdateExpr(
5047+
SharedLV, PrivateRV, BO, true,
5048+
llvm::AtomicOrdering::SequentiallyConsistent, Loc, UpdateOp);
5049+
} else if (const auto *OpCall = dyn_cast<CallExpr>(ReductionClauseExpr)) {
5050+
auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5051+
Action.Enter(CGF);
5052+
CharUnits Alignment = CGF.getContext().getTypeAlignInChars(PrivateType);
5053+
Address TempResult =
5054+
CGF.CreateMemTemp(PrivateType, "reduction.temp.result");
5055+
ReturnValueSlot RVS(TempResult, /*IsVolatile=*/false);
5056+
RValue ResultRV = CGF.EmitCallExpr(OpCall, RVS, nullptr);
5057+
CGF.Builder.CreateMemCpy(SharedResult, ResultRV.getAggregateAddress(),
5058+
llvm::ConstantInt::get(CGF.IntPtrTy, 4),
5059+
Alignment.getQuantity());
5060+
};
5061+
std::string CriticalName = getName({"reduction_critical"});
5062+
emitCriticalRegion(CGF, CriticalName, ReductionGen, Loc);
5063+
}
50245064
}
5025-
50265065
// Final barrier
50275066
CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
50285067
CGM.getModule(), OMPRTL___kmpc_barrier),
@@ -5042,7 +5081,6 @@ void CGOpenMPRuntime::emitPrivateReduction(
50425081
CGM.getModule(), OMPRTL___kmpc_barrier),
50435082
BarrierArgs);
50445083
}
5045-
50465084
void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
50475085
ArrayRef<const Expr *> Privates,
50485086
ArrayRef<const Expr *> LHSExprs,

0 commit comments

Comments
 (0)