Initial version of USMChecking (#124)

* initial version of USM checking

When USMChecking is enabled, log the type of USM allocation and
its associated device when it is set as a kernel argument.

* fix typo

Author: bashbaug

docs/controls.md

@@ -207,6 +207,10 @@ If set to a nonzero value, the Intercept Layer for OpenCL Applications will chec
 
 If set to a nonzero value, the Intercept Layer for OpenCL Applications will check for leaks of various OpenCL objects, such as memory objects and events.
 
+##### `USMChecking` (bool)
+
+If set to a nonzero value, the Intercept Layer for OpenCL Applications will check for incorrect usage of Unified Shared Memory (USM) pointers.
+
 ##### `CLInfoLogging` (bool)
 
 If set to a nonzero value, logs information about the platforms and devices in the system on the first call to clGetPlatformIDs().

intercept/src/cli_ext.h

@@ -705,7 +705,7 @@ cl_int CL_API_CALL clEnqueueReleaseVA_APIMediaSurfacesINTEL(
 #define CL_INVALID_PAK_REFERENCE_IMAGE_INDEX_INTEL      CL_INVALID_VALUE
 
 // cl_intel_unified_shared_memory POC
-// These enums are in sync with revision H of the USM spec.
+// These enums are in sync with revision O of the USM spec.
 
 /* cl_device_info */
 #define CL_DEVICE_HOST_MEM_CAPABILITIES_INTEL                   0x4190
@@ -714,8 +714,7 @@ cl_int CL_API_CALL clEnqueueReleaseVA_APIMediaSurfacesINTEL(
 #define CL_DEVICE_CROSS_DEVICE_SHARED_MEM_CAPABILITIES_INTEL    0x4193
 #define CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL          0x4194
 
-// TODO: should this be cl_device_unified_shared_memory_capabilities_intel?
-typedef cl_bitfield cl_unified_shared_memory_capabilities_intel;
+typedef cl_bitfield cl_device_unified_shared_memory_capabilities_intel;
 
 /* cl_unified_shared_memory_capabilities_intel - bitfield */
 #define CL_UNIFIED_SHARED_MEMORY_ACCESS_INTEL                   (1 << 0)
@@ -805,7 +804,7 @@ clSharedMemAllocINTEL(
 extern CL_API_ENTRY cl_int CL_API_CALL
 clMemFreeINTEL(
             cl_context context,
-            const void* ptr); // TBD: const?
+            void* ptr);
 
 extern CL_API_ENTRY cl_int CL_API_CALL
 clGetMemAllocInfoINTEL(

intercept/src/controls.h

@@ -50,6 +50,7 @@ CLI_CONTROL( cl_uint,       ContextHintLevel,                       0,     "If s
 CLI_CONTROL( bool,          EventCallbackLogging,                   false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will install its own callback for every event callback and log the call to the event callback.  The application's event callback will be invoked after the Intercept Layer for OpenCL Applications' event callback." )
 CLI_CONTROL( bool,          EventChecking,                          false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will check and log any events in an event wait list that are invalid or in an error state.  This can help to debug complex event dependency issues." )
 CLI_CONTROL( bool,          LeakChecking,                           false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will check for leaks of various OpenCL objects, such as memory objects and events." )
+CLI_CONTROL( bool,          USMChecking,                            false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will check for incorrect usage of Unified Shared Memory (USM) pointers." )
 CLI_CONTROL( bool,          CLInfoLogging,                          false, "If set to a nonzero value, logs information about the platforms and devices in the system on the first call to clGetPlatformIDs()." )
 CLI_CONTROL( std::string,   DumpDir,                                "",    "If set, the Intercept Layer for OpenCL Applications will emit logs and dumps to this directory instead of the default directory.  The default log and dump directory is \"%SYSTEMDRIVE%\\Intel\\CLIntercept_Dump\\<Process Name>\" on Windows and \"~/CLIntercept_Dump/<Process Name>\" on other operating systems.  The log and dump directory must be writeable, otherwise the Intercept Layer for OpenCL Applications will not be able to create or modify log or dump files." )
 CLI_CONTROL( bool,          AppendPid,                              false, "If set, the Intercept Layer for OpenCL Applications will append process ID to the log directory name." )

intercept/src/dispatch.cpp

@@ -9107,7 +9107,7 @@ clSharedMemAllocINTEL(
 CL_API_ENTRY cl_int CL_API_CALL
 clMemFreeINTEL(
     cl_context context,
-    const void* ptr)
+    void* ptr)
 {
     CLIntercept*    pIntercept = GetIntercept();
 
@@ -9196,7 +9196,7 @@ CL_API_ENTRY cl_int CL_API_CALL clSetKernelArgMemPointerINTEL(
             kernel,
             arg_index,
             arg_value );
-        SET_KERNEL_ARG_SVM_POINTER( kernel, arg_index, arg_value );
+        CHECK_KERNEL_ARG_USM_POINTER( kernel, arg_value );
         CPU_PERFORMANCE_TIMING_START();
 
         cl_int  retVal = CL_INVALID_OPERATION;

intercept/src/dispatch.h

@@ -1245,7 +1245,7 @@ struct CLdispatch
     // cl_intel_unified_shared_memory
     cl_int (CLI_API_CALL *clMemFreeINTEL) (
         cl_context context,
-        const void* ptr); // TBD: const?
+        void* ptr);
 
     // cl_intel_unified_shared_memory
     cl_int (CLI_API_CALL *clGetMemAllocInfoINTEL) (

intercept/src/enummap.cpp

@@ -806,10 +806,10 @@ CEnumNameMap::CEnumNameMap()
     ADD_ENUM_NAME( m_cl_int, CL_COMMAND_MIGRATEMEM_INTEL );
     ADD_ENUM_NAME( m_cl_int, CL_COMMAND_MEMADVISE_INTEL );
 
-    ADD_ENUM_NAME( m_cl_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_ACCESS_INTEL );
-    ADD_ENUM_NAME( m_cl_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_ATOMIC_ACCESS_INTEL );
-    ADD_ENUM_NAME( m_cl_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ACCESS_INTEL );
-    ADD_ENUM_NAME( m_cl_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ATOMIC_ACCESS_INTEL );
+    ADD_ENUM_NAME( m_cl_device_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_ACCESS_INTEL );
+    ADD_ENUM_NAME( m_cl_device_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_ATOMIC_ACCESS_INTEL );
+    ADD_ENUM_NAME( m_cl_device_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ACCESS_INTEL );
+    ADD_ENUM_NAME( m_cl_device_unified_shared_memory_capabilities_intel, CL_UNIFIED_SHARED_MEMORY_CONCURRENT_ATOMIC_ACCESS_INTEL );
 
     ADD_ENUM_NAME( m_cl_mem_alloc_flags_intel, CL_MEM_ALLOC_WRITE_COMBINED_INTEL );
 

intercept/src/enummap.h

@@ -120,7 +120,7 @@ class CEnumNameMap
     GENERATE_MAP_AND_FUNC(          name_program_binary_type,        cl_program_binary_type          );
     GENERATE_MAP_AND_BITFIELD_FUNC( name_svm_mem_flags,              cl_svm_mem_flags                );
     GENERATE_MAP_AND_FUNC(          name_gl,                         GLenum                          );
-    GENERATE_MAP_AND_BITFIELD_FUNC( name_device_usm_capabilities,    cl_unified_shared_memory_capabilities_intel );
+    GENERATE_MAP_AND_BITFIELD_FUNC( name_device_usm_capabilities,    cl_device_unified_shared_memory_capabilities_intel );
     GENERATE_MAP_AND_BITFIELD_FUNC( name_mem_alloc_flags,            cl_mem_alloc_flags_intel        );
 
     #undef GENERATE_MAP_AND_FUNC

intercept/src/intercept.cpp

@@ -1599,8 +1599,7 @@ void CLIntercept::getDeviceInfoString(
 {
     str = "";
 
-    unsigned int    i = 0;
-    for( i = 0; i < numDevices; i++ )
+    for( cl_uint i = 0; i < numDevices; i++ )
     {
         cl_int  errorCode = CL_SUCCESS;
 
@@ -2241,8 +2240,7 @@ void CLIntercept::logBuild(
 
     if( errorCode == CL_SUCCESS )
     {
-        size_t i = 0;
-        for( i = 0; i < numDevices; i++ )
+        for( cl_uint i = 0; i < numDevices; i++ )
         {
             if( m_Config.BuildLogging )
             {
@@ -2442,8 +2440,7 @@ void CLIntercept::logPreferredWorkGroupSizeMultiple(
             }
             if( errorCode == CL_SUCCESS )
             {
-                size_t i = 0;
-                for( i = 0; i < numDevices; i++ )
+                for( cl_uint i = 0; i < numDevices; i++ )
                 {
                     size_t  kernelPreferredWorkGroupSizeMultiple = 0;
                     errorCode = dispatch().clGetKernelWorkGroupInfo(
@@ -5982,6 +5979,190 @@ void CLIntercept::checkEventList(
     }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+//
+void CLIntercept::checkKernelArgUSMPointer(
+    cl_kernel kernel,
+    const void* arg )
+{
+    std::lock_guard<std::mutex> lock(m_Mutex);
+
+    if( arg == NULL )
+    {
+        logf( "mem pointer %p is NULL\n", arg );
+    }
+    else
+    {
+        cl_int errorCode = CL_SUCCESS;
+
+        cl_context  context = NULL;
+        if( errorCode == CL_SUCCESS )
+        {
+            errorCode = dispatch().clGetKernelInfo(
+                kernel,
+                CL_KERNEL_CONTEXT,
+                sizeof(context),
+                &context,
+                NULL );
+        }
+
+        cl_unified_shared_memory_type_intel memType = CL_MEM_TYPE_UNKNOWN_INTEL;
+        cl_device_id associatedDevice = NULL;
+        if( errorCode == CL_SUCCESS )
+        {
+            dispatch().clGetMemAllocInfoINTEL(
+                context,
+                arg,
+                CL_MEM_ALLOC_TYPE_INTEL,
+                sizeof(memType),
+                &memType,
+                NULL );
+            dispatch().clGetMemAllocInfoINTEL(
+                context,
+                arg,
+                CL_MEM_ALLOC_DEVICE_INTEL,
+                sizeof(associatedDevice),
+                &associatedDevice,
+                NULL );
+
+            char* deviceName = NULL;
+            if( associatedDevice )
+            {
+                allocateAndGetDeviceInfoString(
+                    associatedDevice,
+                    CL_DEVICE_NAME,
+                    deviceName );
+            }
+
+            if( memType == CL_MEM_TYPE_DEVICE_INTEL )
+            {
+                if( deviceName )
+                {
+                    logf( "mem pointer %p is a DEVICE pointer associated with device %s\n",
+                        arg,
+                        deviceName );
+                }
+                else if( associatedDevice )
+                {
+                    logf( "mem pointer %p is a DEVICE pointer associated with device %p\n",
+                        arg,
+                        associatedDevice );
+                }
+                else
+                {
+                    CLI_ASSERT( 0 );
+                    logf( "mem pointer %p is a DEVICE pointer but has no associated device?\n",
+                        arg );
+                }
+            }
+            else if( memType == CL_MEM_TYPE_HOST_INTEL )
+            {
+                logf( "mem pointer %p is a HOST pointer\n",
+                    arg );
+            }
+            else if( memType == CL_MEM_TYPE_SHARED_INTEL )
+            {
+                if( deviceName )
+                {
+                    logf( "mem pointer %p is a SHARED pointer associated with device %s\n",
+                        arg,
+                        deviceName );
+                }
+                else if( associatedDevice )
+                {
+                    logf( "mem pointer %p is a SHARED pointer associated with device %p\n",
+                        arg,
+                        associatedDevice );
+                }
+                else
+                {
+                    logf( "mem pointer %p is a SHARED pointer without an associated device\n",
+                        arg );
+                }
+            }
+            else if( memType == CL_MEM_TYPE_UNKNOWN_INTEL )
+            {
+                // This could be a system shared USM pointer, or this could be an error.
+                // Check the devices associatd with this kernel to see if any support
+                // system shared USM allocations.
+                cl_program  program = NULL;
+                if( errorCode == CL_SUCCESS )
+                {
+                    errorCode = dispatch().clGetKernelInfo(
+                        kernel,
+                        CL_KERNEL_PROGRAM,
+                        sizeof(program),
+                        &program,
+                        NULL );
+                }
+
+                cl_uint         numDevices = 0;
+                cl_device_id*   deviceList = NULL;
+                if( errorCode == CL_SUCCESS )
+                {
+                    errorCode = allocateAndGetProgramDeviceList(
+                        program,
+                        numDevices,
+                        deviceList );
+                }
+
+                if( errorCode == CL_SUCCESS )
+                {
+                    bool supportsSystemSharedUSM = false;
+                    for( cl_uint d = 0; d < numDevices; d++ )
+                    {
+                        cl_device_unified_shared_memory_capabilities_intel usmCaps = 0;
+                        dispatch().clGetDeviceInfo(
+                            deviceList[d],
+                            CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL,
+                            sizeof(usmCaps),
+                            &usmCaps,
+                            NULL);
+                        if( usmCaps != 0 )
+                        {
+                            supportsSystemSharedUSM = true;
+                            break;
+                        }
+                    }
+
+                    if( supportsSystemSharedUSM )
+                    {
+                        logf( "mem pointer %p is an UNKNOWN pointer and could be a shared system pointer\n",
+                            arg );
+                    }
+                    else
+                    {
+                        logf( "mem pointer %p is an UNKNOWN pointer and no device support shared system pointers!\n",
+                            arg );
+                    }
+                }
+                else
+                {
+                    logf( "mem pointer %p is an UNKNOWN pointer and additional queries returned an error!\n",
+                        arg );
+                }
+
+                delete [] deviceList;
+            }
+            else
+            {
+                CLI_ASSERT( 0 );
+                logf( "query for mem pointer %p returned an unknown memory type %08X!\n",
+                    arg,
+                    memType );
+            }
+
+            delete [] deviceName;
+        }
+        else
+        {
+            logf( "couldn't query context for kernel %p for mem pointer %p!\n",
+                kernel,
+                arg );
+        }
+    }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 void CLIntercept::startAubCapture(
@@ -6302,8 +6483,7 @@ void CLIntercept::initPrecompiledKernelOverrides(
 
                         if( tempErrorCode == CL_SUCCESS )
                         {
-                            cl_uint i = 0;
-                            for( i = 0; i < numDevices; i++ )
+                            for( cl_uint i = 0; i < numDevices; i++ )
                             {
                                 size_t  buildLogSize = 0;
                                 dispatch().clGetProgramBuildInfo(
@@ -6517,8 +6697,7 @@ void CLIntercept::initBuiltinKernelOverrides(
 
                         if( tempErrorCode == CL_SUCCESS )
                         {
-                            cl_uint i = 0;
-                            for( i = 0; i < numDevices; i++ )
+                            for( cl_uint i = 0; i < numDevices; i++ )
                             {
                                 size_t  buildLogSize = 0;
                                 dispatch().clGetProgramBuildInfo(

intercept/src/intercept.h

@@ -428,6 +428,9 @@ class CLIntercept
                 cl_uint numEvents,
                 const cl_event* eventList,
                 cl_event* event );
+    void    checkKernelArgUSMPointer(
+                cl_kernel kernel,
+                const void* arg );
 
     void    startAubCapture(
                 const std::string& functionName,
@@ -2044,6 +2047,16 @@ inline bool CLIntercept::checkAubCaptureEnqueueLimits() const
             _event );                                                       \
     }
 
+///////////////////////////////////////////////////////////////////////////////
+//
+#define CHECK_KERNEL_ARG_USM_POINTER( _kernel, _arg )                       \
+    if( pIntercept->config().USMChecking )                                  \
+    {                                                                       \
+        pIntercept->checkKernelArgUSMPointer(                               \
+            _kernel,                                                        \
+            _arg );                                                         \
+    }
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 inline std::string CLIntercept::getShortKernelName(