@@ -299,10 +299,6 @@ static LogicalResult checkImplementationStatus(Operation &op) {
299299 if (privatizer.getDataSharingType () ==
300300 omp::DataSharingClauseType::FirstPrivate)
301301 result = todo (" firstprivate" );
302-
303- if (!privatizer.getDeallocRegion ().empty ())
304- result = op.emitError (" not yet implemented: privatization of "
305- " structures in omp.target operation" );
306302 }
307303 }
308304 checkThreadLimit (op, result);
@@ -3813,6 +3809,43 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg,
38133809 return builder.saveIP ();
38143810}
38153811
3812+ // / Return the llvm::Value * corresponding to the `privateVar` that
3813+ // / is being privatized. It isn't always as simple as looking up
3814+ // / moduleTranslation with privateVar. For instance, in case of
3815+ // / an allocatable, the descriptor for the allocatable is privatized.
3816+ // / This descriptor is mapped using an MapInfoOp. So, this function
3817+ // / will return a pointer to the llvm::Value corresponding to the
3818+ // / block argument for the mapped descriptor.
3819+ static llvm::Value *
3820+ findHostAssociatedValue (Value privateVar, omp::TargetOp targetOp,
3821+ llvm::DenseMap<Value, int > &mappedPrivateVars,
3822+ llvm::IRBuilderBase &builder,
3823+ LLVM::ModuleTranslation &moduleTranslation) {
3824+ if (mappedPrivateVars.contains (privateVar)) {
3825+ int blockArgIndex = mappedPrivateVars[privateVar];
3826+ Value blockArg = targetOp.getRegion ().getArgument (blockArgIndex);
3827+ Type privVarType = privateVar.getType ();
3828+ Type blockArgType = blockArg.getType ();
3829+ assert (isa<LLVM::LLVMPointerType>(blockArgType) &&
3830+ " A block argument corresponding to a mapped var should have "
3831+ " !llvm.ptr type" );
3832+
3833+ if (privVarType == blockArg.getType ())
3834+ return moduleTranslation.lookupValue (blockArg);
3835+
3836+ if (!isa<LLVM::LLVMPointerType>(privVarType)) {
3837+ // This typically happens when the privatized type is lowered from
3838+ // boxchar<KIND> and gets lowered to !llvm.struct<(ptr, i64)>. That is the
3839+ // struct/pair is passed by value. But, mapped values are passed only as
3840+ // pointers, so before we privatize, we must load the pointer.
3841+ llvm::Value *load =
3842+ builder.CreateLoad (moduleTranslation.convertType (privVarType),
3843+ moduleTranslation.lookupValue (blockArg));
3844+ return load;
3845+ }
3846+ }
3847+ return moduleTranslation.lookupValue (privateVar);
3848+ }
38163849static LogicalResult
38173850convertOmpTarget (Operation &opInst, llvm::IRBuilderBase &builder,
38183851 LLVM::ModuleTranslation &moduleTranslation) {
@@ -3824,6 +3857,19 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
38243857 bool isTargetDevice = ompBuilder->Config .isTargetDevice ();
38253858 auto parentFn = opInst.getParentOfType <LLVM::LLVMFuncOp>();
38263859 auto &targetRegion = targetOp.getRegion ();
3860+ // Holds the private vars that have been mapped along with
3861+ // the block argument that corresponds to the MapInfoOp
3862+ // corresponding to the private var in question.
3863+ // So, for instance
3864+ //
3865+ // %10 = omp.map.info var_ptr(%6#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, ..)
3866+ // omp.target map_entries(%10 -> %arg0) private(@box.privatizer %6#0-> %arg1)
3867+ //
3868+ // Then, %10 has been created so that the descriptor can be used by the
3869+ // privatizer
3870+ // @box.privatizer on the device side. Here we'd record {%6#0, 0} in the
3871+ // mappedPrivateVars map.
3872+ llvm::DenseMap<Value, int > mappedPrivateVars;
38273873 DataLayout dl = DataLayout (opInst.getParentOfType <ModuleOp>());
38283874 SmallVector<Value> mapVars = targetOp.getMapVars ();
38293875 ArrayRef<BlockArgument> mapBlockArgs =
@@ -3835,6 +3881,55 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
38353881 bool isOffloadEntry =
38363882 isTargetDevice || !ompBuilder->Config .TargetTriples .empty ();
38373883
3884+ // For some private variables, the MapsForPrivatizedVariablesPass
3885+ // creates MapInfoOp instances. Go through the private variables and
3886+ // the mapped variables so that during codegeneration we are able
3887+ // to quickly look up the corresponding map variable, if any for each
3888+ // private variable.
3889+ if (!targetOp.getPrivateVars ().empty () && !targetOp.getMapVars ().empty ()) {
3890+ auto argIface = llvm::cast<omp::BlockArgOpenMPOpInterface>(*targetOp);
3891+ OperandRange privateVars = targetOp.getPrivateVars ();
3892+ std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms ();
3893+ std::optional<DenseI64ArrayAttr> privateMapIndices =
3894+ targetOp.getPrivateMapsAttr ();
3895+
3896+ for (auto [privVarIdx, privVarSymPair] :
3897+ llvm::enumerate (llvm::zip_equal (privateVars, *privateSyms))) {
3898+ auto privVar = std::get<0 >(privVarSymPair);
3899+ auto privSym = std::get<1 >(privVarSymPair);
3900+
3901+ SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(privSym);
3902+ omp::PrivateClauseOp privatizer =
3903+ findPrivatizer (targetOp, privatizerName);
3904+
3905+ if (!privatizer.needsMap ())
3906+ continue ;
3907+
3908+ mlir::Value mappedValue =
3909+ targetOp.getMappedValueForPrivateVar (privVarIdx);
3910+ assert (mappedValue);
3911+
3912+ // The MapInfoOp defining the map var isn't really needed later.
3913+ // So, we don't store it in any datastructure. Instead, we just
3914+ // do some sanity checks on it right now.
3915+ auto mapInfoOp = mappedValue.getDefiningOp <omp::MapInfoOp>();
3916+ Type varType = mapInfoOp.getVarType ();
3917+
3918+ // Check #1: Check that the type of the private variable matches
3919+ // the type of the variable being mapped.
3920+ if (!isa<LLVM::LLVMPointerType>(privVar.getType ()))
3921+ assert (
3922+ varType == privVar.getType () &&
3923+ " Type of private var doesn't match the type of the mapped value" );
3924+
3925+ // Ok, only 1 sanity check for now.
3926+ // Record the index of the block argument corresponding to this
3927+ // mapvar.
3928+ mappedPrivateVars.insert ({privVar, argIface.getMapBlockArgsStart () +
3929+ (*privateMapIndices)[privVarIdx]});
3930+ }
3931+ }
3932+
38383933 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
38393934 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP)
38403935 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
@@ -3861,9 +3956,10 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
38613956 moduleTranslation.lookupValue (mapInfoOp.getVarPtr ());
38623957 moduleTranslation.mapValue (arg, mapOpValue);
38633958 }
3864-
38653959 // Do privatization after moduleTranslation has already recorded
38663960 // mapped values.
3961+ SmallVector<llvm::Value *> llvmPrivateVars;
3962+ SmallVector<Region *> privateCleanupRegions;
38673963 if (!targetOp.getPrivateVars ().empty ()) {
38683964 builder.restoreIP (allocaIP);
38693965
@@ -3879,11 +3975,13 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
38793975 omp::PrivateClauseOp privatizer = findPrivatizer (&opInst, privSym);
38803976 assert (privatizer.getDataSharingType () !=
38813977 omp::DataSharingClauseType::FirstPrivate &&
3882- privatizer.getDeallocRegion ().empty () &&
38833978 " unsupported privatizer" );
3884- moduleTranslation.mapValue (privatizer.getAllocMoldArg (),
3885- moduleTranslation.lookupValue (privVar));
38863979 Region &allocRegion = privatizer.getAllocRegion ();
3980+ BlockArgument allocRegionArg = allocRegion.getArgument (0 );
3981+ moduleTranslation.mapValue (
3982+ allocRegionArg,
3983+ findHostAssociatedValue (privVar, targetOp, mappedPrivateVars,
3984+ builder, moduleTranslation));
38873985 SmallVector<llvm::Value *, 1 > yieldedValues;
38883986 if (failed (inlineConvertOmpRegions (
38893987 allocRegion, " omp.targetop.privatizer" , builder,
@@ -3892,7 +3990,12 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
38923990 " failed to inline `alloc` region of `omp.private`" );
38933991 }
38943992 assert (yieldedValues.size () == 1 );
3895- moduleTranslation.mapValue (privBlockArg, yieldedValues.front ());
3993+ llvm::Value *llvmReplacementValue = yieldedValues.front ();
3994+ moduleTranslation.mapValue (privBlockArg, llvmReplacementValue);
3995+ if (!privatizer.getDeallocRegion ().empty ()) {
3996+ llvmPrivateVars.push_back (llvmReplacementValue);
3997+ privateCleanupRegions.push_back (&privatizer.getDeallocRegion ());
3998+ }
38963999 moduleTranslation.forgetMapping (allocRegion);
38974000 builder.restoreIP (builder.saveIP ());
38984001 }
@@ -3904,6 +4007,19 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
39044007 return exitBlock.takeError ();
39054008
39064009 builder.SetInsertPoint (*exitBlock);
4010+ if (!llvmPrivateVars.empty ()) {
4011+ assert (llvmPrivateVars.size () == privateCleanupRegions.size () &&
4012+ " Number of private variables needing cleanup not equal to number"
4013+ " of privatizers with dealloc regions" );
4014+ if (failed (inlineOmpRegionCleanup (
4015+ privateCleanupRegions, llvmPrivateVars, moduleTranslation,
4016+ builder, " omp.targetop.private.cleanup" ,
4017+ /* shouldLoadCleanupRegionArg=*/ false ))) {
4018+ return llvm::createStringError (
4019+ " failed to inline `dealloc` region of `omp.private` "
4020+ " op in the target region" );
4021+ }
4022+ }
39074023 return builder.saveIP ();
39084024 };
39094025
0 commit comments