diff --git a/offload/include/device.h b/offload/include/device.h index f4b10abbaa3fd..d3f59d60fc41d 100644 --- a/offload/include/device.h +++ b/offload/include/device.h @@ -152,6 +152,9 @@ struct DeviceTy { /// Ask the device whether the runtime should use auto zero-copy. bool useAutoZeroCopy(); + /// Check if HstPtr is in pinned memory + bool isPinnedPtr(void *HstPtr); + /// Check if there are pending images for this device. bool hasPendingImages() const { return HasPendingImages; } diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h index 6971780c7bdb5..8af8c4f659b35 100644 --- a/offload/include/omptarget.h +++ b/offload/include/omptarget.h @@ -280,6 +280,7 @@ int omp_get_initial_device(void); void *omp_target_alloc(size_t Size, int DeviceNum); void omp_target_free(void *DevicePtr, int DeviceNum); int omp_target_is_present(const void *Ptr, int DeviceNum); +int omp_target_is_accessible(const void *Ptr, size_t Size, int DeviceNum); int omp_target_memcpy(void *Dst, const void *Src, size_t Length, size_t DstOffset, size_t SrcOffset, int DstDevice, int SrcDevice); diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp index b0f0573833713..c59734ae02d34 100644 --- a/offload/libomptarget/OpenMP/API.cpp +++ b/offload/libomptarget/OpenMP/API.cpp @@ -702,3 +702,65 @@ EXTERN void __tgt_target_sync(ident_t *loc_ref, int gtid, void *current_task, RTLOngoingSyncs--; } + +EXTERN int omp_target_is_accessible(const void *Ptr, size_t Size, + int DeviceNum) { + TIMESCOPE(); + // OpenMP 5.1, sec. 3.8.4 "omp_target_is_accessible", p. 417, L21-22: + // "This routine returns true if the storage of size bytes starting at the + // address given by Ptr is accessible from device device_num. Otherwise, it + // returns false." + // + // The meaning of "accessible" for unified shared memory is established in + // OpenMP 5.1, sec. 2.5.1 "requires directive". More generally, the specified + // host memory is accessible if it can be accessed from the device either + // directly (because of unified shared memory or because DeviceNum is the + // value returned by omp_get_initial_device()) or indirectly (because it's + // mapped to the device). + DP("Call to omp_target_is_accessible for device %d and address " DPxMOD "\n", + DeviceNum, DPxPTR(Ptr)); + + // FIXME: Is this right? + // + // Null pointer is permitted: + // + // OpenMP 5.1, sec. 3.8.4 "omp_target_is_accessible", p. 417, L15: + // "The value of ptr must be a valid host pointer or NULL (or C_NULL_PTR, for + // Fortran)." + // + // However, I found no specification of behavior in this case. + // omp_target_is_present has the same problem and is implemented the same way. + // Should Size have any effect on the result when Ptr is NULL? + if (!Ptr) { + DP("Call to omp_target_is_accessible with NULL Ptr, returning false\n"); + return false; + } + + if (DeviceNum == omp_get_initial_device()) { + DP("Call to omp_target_is_accessible on host, returning true\n"); + return true; + } + + auto DeviceOrErr = PM->getDevice(DeviceNum); + if (!DeviceOrErr) + FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str()); + + if (DeviceOrErr->isPinnedPtr(const_cast(Ptr))) { + DP("Call to omp_target_is_accessible with HostPtr in pinned memory, returning " + "true\n"); + return true; + } + // TODO: How does the spec intend for the Size=0 case to be handled? + // Currently, for the case where arr[N:M] is mapped, we return true for any + // address within arr[0:N+M]. However, Size>1 returns true only for arr[N:M]. + // This is based on the discussion so far at the time of this writing at + // . If the behavior + // changes, keep comments for omp_get_accessible_buffer in omp.h.var in sync. + TargetPointerResultTy TPR = + DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast(Ptr), Size, + /*UpdateRefCount=*/false, /*UseHoldRefCount=*/false, + /*MustContain=*/true); + int Rc = (TPR.isPresent() || TPR.isHostPointer()); + DP("Call to omp_target_is_accessible returns %d\n", Rc); + return Rc; +} diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp index f88e30ae9e76b..7e95ff18cc375 100644 --- a/offload/libomptarget/device.cpp +++ b/offload/libomptarget/device.cpp @@ -281,3 +281,7 @@ bool DeviceTy::useAutoZeroCopy() { return false; return RTL->use_auto_zero_copy(RTLDeviceID); } + +bool DeviceTy::isPinnedPtr(void *HstPtr) { + return RTL->is_pinned_ptr(RTLDeviceID, HstPtr); +} \ No newline at end of file diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports index 8e2db6ba8bba4..d4722fda51752 100644 --- a/offload/libomptarget/exports +++ b/offload/libomptarget/exports @@ -44,6 +44,7 @@ VERS1.0 { omp_target_alloc; omp_target_free; omp_target_is_present; + omp_target_is_accessible; omp_target_memcpy; omp_target_memcpy_rect; omp_target_memcpy_async; diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h index 1d64193c17f6b..c8ab7c009929c 100644 --- a/offload/plugins-nextgen/common/include/PluginInterface.h +++ b/offload/plugins-nextgen/common/include/PluginInterface.h @@ -1476,6 +1476,9 @@ struct GenericPluginTy { /// Returns if the plugin can support automatic copy. int32_t use_auto_zero_copy(int32_t DeviceId); + // Returns true, if the Ptr in pinned memory + int32_t is_pinned_ptr(int32_t DeviceId, void *HstPtr); + /// Look up a global symbol in the given binary. int32_t get_global(__tgt_device_binary Binary, uint64_t Size, const char *Name, void **DevicePtr); diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp index bcc91798f3f90..66aef3af5afeb 100644 --- a/offload/plugins-nextgen/common/src/PluginInterface.cpp +++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp @@ -2199,6 +2199,19 @@ int32_t GenericPluginTy::use_auto_zero_copy(int32_t DeviceId) { return getDevice(DeviceId).useAutoZeroCopy(); } + +int32_t GenericPluginTy::is_pinned_ptr(int32_t DeviceId, void *HstPtr) { + GenericDeviceTy &Device = getDevice(DeviceId); + + size_t BaseSize; + void *BaseHstPtr, *BaseDevAccessiblePtr; + auto Result = Device.isPinnedPtrImpl(HstPtr, BaseHstPtr,BaseDevAccessiblePtr, BaseSize); + if (!Result) + return false; + + return *Result; +} + int32_t GenericPluginTy::get_global(__tgt_device_binary Binary, uint64_t Size, const char *Name, void **DevicePtr) { assert(Binary.handle && "Invalid device binary handle"); diff --git a/openmp/runtime/src/kmp_ftn_os.h b/openmp/runtime/src/kmp_ftn_os.h index ae0ed067235e5..46afa800379e0 100644 --- a/openmp/runtime/src/kmp_ftn_os.h +++ b/openmp/runtime/src/kmp_ftn_os.h @@ -114,6 +114,7 @@ #define FTN_TARGET_ALLOC omp_target_alloc #define FTN_TARGET_FREE omp_target_free #define FTN_TARGET_IS_PRESENT omp_target_is_present +#define FTN_TARGET_IS_ACCESSIBLE omp_target_is_accessible #define FTN_TARGET_MEMCPY omp_target_memcpy #define FTN_TARGET_MEMCPY_RECT omp_target_memcpy_rect #define FTN_TARGET_MEMSET omp_target_memset @@ -263,6 +264,7 @@ #define FTN_TARGET_ALLOC omp_target_alloc_ #define FTN_TARGET_FREE omp_target_free_ #define FTN_TARGET_IS_PRESENT omp_target_is_present_ +#define FTN_TARGET_IS_ACCESSIBLE omp_target_is_accessible_ #define FTN_TARGET_MEMCPY omp_target_memcpy_ #define FTN_TARGET_MEMCPY_RECT omp_target_memcpy_rect_ #define FTN_TARGET_ASSOCIATE_PTR omp_target_associate_ptr_ @@ -412,6 +414,7 @@ #define FTN_TARGET_ALLOC OMP_TARGET_ALLOC #define FTN_TARGET_FREE OMP_TARGET_FREE #define FTN_TARGET_IS_PRESENT OMP_TARGET_IS_PRESENT +#define FTN_TARGET_IS_ACCESSIBLE OMP_TARGET_IS_ACCESSIBLE #define FTN_TARGET_MEMCPY OMP_TARGET_MEMCPY #define FTN_TARGET_MEMCPY_RECT OMP_TARGET_MEMCPY_RECT #define FTN_TARGET_ASSOCIATE_PTR OMP_TARGET_ASSOCIATE_PTR @@ -559,6 +562,7 @@ #define FTN_TARGET_ALLOC OMP_TARGET_ALLOC_ #define FTN_TARGET_FREE OMP_TARGET_FREE_ #define FTN_TARGET_IS_PRESENT OMP_TARGET_IS_PRESENT_ +#define FTN_TARGET_IS_ACCESSIBLE OMP_TARGET_IS_ACCESSIBLE_ #define FTN_TARGET_MEMCPY OMP_TARGET_MEMCPY_ #define FTN_TARGET_MEMCPY_RECT OMP_TARGET_MEMCPY_RECT_ #define FTN_TARGET_ASSOCIATE_PTR OMP_TARGET_ASSOCIATE_PTR_