[OCL] Flag for allocating small buffers from pool

Improves performance in workloads that create small opencl buffers.

To enable, set env var ExperimentalSmallBufferPoolAllocator=1

Known issues (will be addressed in further commits):
- cannot create subBuffer from such buffer
- pool buffer allocation should be reused

Related-To: NEO-7332

Signed-off-by: Dominik Dabek <[email protected]>

Author: ddabek-i

opencl/source/context/context.cpp

@@ -45,6 +45,10 @@ Context::Context(
 Context::~Context() {
     gtpinNotifyContextDestroy((cl_context)this);
 
+    if (smallBufferPoolAllocator.isAggregatedSmallBuffersEnabled()) {
+        smallBufferPoolAllocator.releaseSmallBufferPool();
+    }
+
     delete[] properties;
 
     for (auto rootDeviceIndex = 0u; rootDeviceIndex < specialQueues.size(); rootDeviceIndex++) {
@@ -467,4 +471,65 @@ Platform *Context::getPlatformFromProperties(const cl_context_properties *proper
 bool Context::isSingleDeviceContext() {
     return devices[0]->getNumGenericSubDevices() == 0 && getNumDevices() == 1;
 }
+
+void Context::BufferPoolAllocator::initAggregatedSmallBuffers(Context *context) {
+    static constexpr cl_mem_flags flags{};
+    [[maybe_unused]] cl_int errcodeRet{};
+    this->mainStorage = Buffer::create(context,
+                                       flags,
+                                       BufferPoolAllocator::aggregatedSmallBuffersPoolSize,
+                                       nullptr,
+                                       errcodeRet);
+    if (this->mainStorage) {
+        this->chunkAllocator.reset(new HeapAllocator(BufferPoolAllocator::startingOffset,
+                                                     BufferPoolAllocator::aggregatedSmallBuffersPoolSize,
+                                                     BufferPoolAllocator::chunkAlignment));
+        context->decRefInternal();
+    }
+}
+
+Buffer *Context::BufferPoolAllocator::allocateBufferFromPool(const MemoryProperties &memoryProperties,
+                                                             cl_mem_flags flags,
+                                                             cl_mem_flags_intel flagsIntel,
+                                                             size_t size,
+                                                             void *hostPtr,
+                                                             cl_int &errcodeRet) {
+    errcodeRet = CL_MEM_OBJECT_ALLOCATION_FAILURE;
+    if (this->isAggregatedSmallBuffersEnabled() &&
+        this->isSizeWithinThreshold(size) &&
+        this->mainStorage) {
+        auto lock = std::unique_lock<std::mutex>(this->mutex);
+        cl_buffer_region bufferRegion{};
+        bufferRegion.origin = static_cast<size_t>(this->chunkAllocator->allocate(size));
+        if (bufferRegion.origin == 0) {
+            return nullptr;
+        }
+        bufferRegion.origin -= BufferPoolAllocator::startingOffset;
+        bufferRegion.size = size;
+        auto bufferFromPool = this->mainStorage->createSubBuffer(flags, flagsIntel, &bufferRegion, errcodeRet);
+        bufferFromPool->createFunction = this->mainStorage->createFunction;
+        return bufferFromPool;
+    }
+    return nullptr;
+}
+
+bool Context::BufferPoolAllocator::isPoolBuffer(const MemObj *buffer) const {
+    return this->mainStorage == buffer;
+}
+
+void Context::BufferPoolAllocator::tryFreeFromPoolBuffer(MemObj *possiblePoolBuffer, size_t offset, size_t size) {
+    if (this->isPoolBuffer(possiblePoolBuffer)) {
+        auto lock = std::unique_lock<std::mutex>(this->mutex);
+        DEBUG_BREAK_IF(!this->mainStorage);
+        auto internalBufferAddress = offset + BufferPoolAllocator::startingOffset;
+        this->chunkAllocator->free(internalBufferAddress, size);
+    }
+}
+
+void Context::BufferPoolAllocator::releaseSmallBufferPool() {
+    DEBUG_BREAK_IF(!this->mainStorage);
+    delete this->mainStorage;
+    this->mainStorage = nullptr;
+}
+
 } // namespace NEO

opencl/source/context/context.h

@@ -10,13 +10,15 @@
 #include "shared/source/helpers/common_types.h"
 #include "shared/source/helpers/string.h"
 #include "shared/source/unified_memory/unified_memory.h"
+#include "shared/source/utilities/heap_allocator.h"
 
 #include "opencl/source/cl_device/cl_device_vector.h"
 #include "opencl/source/context/context_type.h"
 #include "opencl/source/context/driver_diagnostics.h"
 #include "opencl/source/gtpin/gtpin_notify.h"
 #include "opencl/source/helpers/base_object.h"
 #include "opencl/source/helpers/destructor_callbacks.h"
+#include "opencl/source/mem_obj/buffer.h"
 #include "opencl/source/mem_obj/map_operations_handler.h"
 
 #include <map>
@@ -40,6 +42,42 @@ struct OpenCLObjectMapper<_cl_context> {
 
 class Context : public BaseObject<_cl_context> {
   public:
+    class BufferPoolAllocator {
+      public:
+        static constexpr auto aggregatedSmallBuffersPoolSize = 64 * KB;
+        static constexpr auto smallBufferThreshold = 4 * KB;
+        static constexpr auto chunkAlignment = 256u;
+        static constexpr auto startingOffset = chunkAlignment;
+
+        static_assert(aggregatedSmallBuffersPoolSize > smallBufferThreshold, "Largest allowed buffer needs to fit in pool");
+        Buffer *allocateBufferFromPool(const MemoryProperties &memoryProperties,
+                                       cl_mem_flags flags,
+                                       cl_mem_flags_intel flagsIntel,
+                                       size_t size,
+                                       void *hostPtr,
+                                       cl_int &errcodeRet);
+        void tryFreeFromPoolBuffer(MemObj *possiblePoolBuffer, size_t offset, size_t size);
+        void releaseSmallBufferPool();
+
+        inline bool isAggregatedSmallBuffersEnabled() const {
+            constexpr bool enable = false;
+            if (DebugManager.flags.ExperimentalSmallBufferPoolAllocator.get() != -1) {
+                return !!DebugManager.flags.ExperimentalSmallBufferPoolAllocator.get();
+            }
+            return enable;
+        }
+        void initAggregatedSmallBuffers(Context *context);
+
+        bool isPoolBuffer(const MemObj *buffer) const;
+
+      protected:
+        inline bool isSizeWithinThreshold(size_t size) const {
+            return BufferPoolAllocator::smallBufferThreshold >= size;
+        }
+        Buffer *mainStorage{nullptr};
+        std::unique_ptr<HeapAllocator> chunkAllocator;
+        std::mutex mutex;
+    };
     static const cl_ulong objectMagic = 0xA4234321DC002130LL;
 
     bool createImpl(const cl_context_properties *properties,
@@ -58,6 +96,11 @@ class Context : public BaseObject<_cl_context> {
         if (!pContext->createImpl(properties, devices, funcNotify, data, errcodeRet)) {
             delete pContext;
             pContext = nullptr;
+        } else {
+            auto &bufferPoolAllocator = pContext->getBufferPoolAllocator();
+            if (bufferPoolAllocator.isAggregatedSmallBuffersEnabled()) {
+                bufferPoolAllocator.initAggregatedSmallBuffers(pContext);
+            }
         }
         gtpinNotifyContextCreate(pContext);
         return pContext;
@@ -176,6 +219,9 @@ class Context : public BaseObject<_cl_context> {
     const std::map<uint32_t, DeviceBitfield> &getDeviceBitfields() const { return deviceBitfields; };
 
     static Platform *getPlatformFromProperties(const cl_context_properties *properties, cl_int &errcode);
+    BufferPoolAllocator &getBufferPoolAllocator() {
+        return this->smallBufferPoolAllocator;
+    }
 
   protected:
     struct BuiltInKernel {
@@ -211,6 +257,7 @@ class Context : public BaseObject<_cl_context> {
     MapOperationsStorage mapOperationsStorage = {};
     StackVec<CommandQueue *, 1> specialQueues;
     DriverDiagnostics *driverDiagnostics = nullptr;
+    BufferPoolAllocator smallBufferPoolAllocator;
 
     uint32_t maxRootDeviceIndex = std::numeric_limits<uint32_t>::max();
     cl_bool preferD3dSharedResources = 0u;

opencl/source/mem_obj/buffer.cpp

@@ -175,6 +175,58 @@ Buffer *Buffer::create(Context *context,
                   flags, 0, size, hostPtr, errcodeRet);
 }
 
+bool inline copyHostPointer(Buffer *buffer,
+                            size_t size,
+                            void *hostPtr,
+                            GraphicsAllocation *memory,
+                            GraphicsAllocation *mapAllocation,
+                            uint32_t rootDeviceIndex,
+                            bool isCompressionEnabled,
+                            bool implicitScalingEnabled,
+                            cl_int &errcodeRet) {
+    const bool isLocalMemory = !MemoryPoolHelper::isSystemMemoryPool(memory->getMemoryPool());
+    const bool gpuCopyRequired = isCompressionEnabled || isLocalMemory;
+    if (gpuCopyRequired) {
+        auto context = buffer->getContext();
+        auto &device = context->getDevice(0u)->getDevice();
+        auto &hwInfo = device.getHardwareInfo();
+        auto hwInfoConfig = HwInfoConfig::get(hwInfo.platform.eProductFamily);
+        bool copyOnCpuAllowed = implicitScalingEnabled == false &&
+                                size <= Buffer::maxBufferSizeForCopyOnCpu &&
+                                isCompressionEnabled == false &&
+                                hwInfoConfig->getLocalMemoryAccessMode(hwInfo) != LocalMemoryAccessMode::CpuAccessDisallowed &&
+                                memory->storageInfo.isLockable;
+        if (DebugManager.flags.CopyHostPtrOnCpu.get() != -1) {
+            copyOnCpuAllowed = DebugManager.flags.CopyHostPtrOnCpu.get() == 1;
+        }
+        if (auto lockedPointer = copyOnCpuAllowed ? device.getMemoryManager()->lockResource(memory) : nullptr) {
+            memcpy_s(ptrOffset(lockedPointer, buffer->getOffset()), size, hostPtr, size);
+            memory->setAubWritable(true, GraphicsAllocation::defaultBank);
+            memory->setTbxWritable(true, GraphicsAllocation::defaultBank);
+            return true;
+        } else {
+            auto blitMemoryToAllocationResult = BlitOperationResult::Unsupported;
+
+            if (hwInfoConfig->isBlitterFullySupported(hwInfo) && isLocalMemory) {
+                blitMemoryToAllocationResult = BlitHelperFunctions::blitMemoryToAllocation(device, memory, buffer->getOffset(), hostPtr, {size, 1, 1});
+            }
+
+            if (blitMemoryToAllocationResult != BlitOperationResult::Success) {
+                auto cmdQ = context->getSpecialQueue(rootDeviceIndex);
+                if (CL_SUCCESS != cmdQ->enqueueWriteBuffer(buffer, CL_TRUE, buffer->getOffset(), size, hostPtr, mapAllocation, 0, nullptr, nullptr)) {
+                    errcodeRet = CL_OUT_OF_RESOURCES;
+                    return false;
+                }
+            }
+            return true;
+        }
+    } else {
+        memcpy_s(ptrOffset(memory->getUnderlyingBuffer(), buffer->getOffset()), size, hostPtr, size);
+        return true;
+    }
+    return false;
+}
+
 Buffer *Buffer::create(Context *context,
                        const MemoryProperties &memoryProperties,
                        cl_mem_flags flags,
@@ -184,6 +236,47 @@ Buffer *Buffer::create(Context *context,
                        cl_int &errcodeRet) {
 
     errcodeRet = CL_SUCCESS;
+    Context::BufferPoolAllocator &bufferPoolAllocator = context->getBufferPoolAllocator();
+    const bool implicitScalingEnabled = ImplicitScalingHelper::isImplicitScalingEnabled(context->getDevice(0u)->getDeviceBitfield(), true);
+    const bool useHostPtr = memoryProperties.flags.useHostPtr;
+    const bool copyHostPtr = memoryProperties.flags.copyHostPtr;
+    if (implicitScalingEnabled == false &&
+        useHostPtr == false &&
+        memoryProperties.flags.forceHostMemory == false) {
+        cl_int poolAllocRet = CL_SUCCESS;
+        auto bufferFromPool = bufferPoolAllocator.allocateBufferFromPool(memoryProperties,
+                                                                         flags,
+                                                                         flagsIntel,
+                                                                         size,
+                                                                         hostPtr,
+                                                                         poolAllocRet);
+        if (CL_SUCCESS == poolAllocRet) {
+            const bool needsCopy = copyHostPtr;
+            if (needsCopy) {
+                for (auto &rootDeviceIndex : context->getRootDeviceIndices()) {
+                    auto graphicsAllocation = bufferFromPool->getGraphicsAllocation(rootDeviceIndex);
+                    auto mapAllocation = bufferFromPool->getMapAllocation(rootDeviceIndex);
+                    bool isCompressionEnabled = graphicsAllocation->isCompressionEnabled();
+                    if (copyHostPointer(bufferFromPool,
+                                        size,
+                                        hostPtr,
+                                        graphicsAllocation,
+                                        mapAllocation,
+                                        rootDeviceIndex,
+                                        isCompressionEnabled,
+                                        implicitScalingEnabled,
+                                        poolAllocRet)) {
+                        break;
+                    }
+                }
+            }
+            if (!needsCopy || poolAllocRet == CL_SUCCESS) {
+                return bufferFromPool;
+            } else {
+                clReleaseMemObject(bufferFromPool);
+            }
+        }
+    }
 
     MemoryManager *memoryManager = context->getMemoryManager();
     UNRECOVERABLE_IF(!memoryManager);
@@ -194,9 +287,6 @@ Buffer *Buffer::create(Context *context,
     AllocationInfoType allocationInfos;
     allocationInfos.resize(maxRootDeviceIndex + 1ull);
 
-    const bool useHostPtr = memoryProperties.flags.useHostPtr;
-    const bool copyHostPtr = memoryProperties.flags.copyHostPtr;
-
     void *allocationCpuPtr = nullptr;
     bool forceCopyHostPtr = false;
 
@@ -404,45 +494,15 @@ Buffer *Buffer::create(Context *context,
         pBuffer->setHostPtrMinSize(size);
 
         if (allocationInfo.copyMemoryFromHostPtr && !copyExecuted) {
-            auto isLocalMemory = !MemoryPoolHelper::isSystemMemoryPool(allocationInfo.memory->getMemoryPool());
-            bool gpuCopyRequired = isCompressionEnabled || isLocalMemory;
-
-            if (gpuCopyRequired) {
-                auto &device = pBuffer->getContext()->getDevice(0u)->getDevice();
-                auto &hwInfo = device.getHardwareInfo();
-                auto hwInfoConfig = HwInfoConfig::get(hwInfo.platform.eProductFamily);
-                bool copyOnCpuAllowed = false == ImplicitScalingHelper::isImplicitScalingEnabled(device.getDeviceBitfield(), true) &&
-                                        size <= Buffer::maxBufferSizeForCopyOnCpu &&
-                                        !isCompressionEnabled &&
-                                        hwInfoConfig->getLocalMemoryAccessMode(hwInfo) != LocalMemoryAccessMode::CpuAccessDisallowed &&
-                                        allocationInfo.memory->storageInfo.isLockable;
-                if (DebugManager.flags.CopyHostPtrOnCpu.get() != -1) {
-                    copyOnCpuAllowed = DebugManager.flags.CopyHostPtrOnCpu.get() == 1;
-                }
-                if (auto lockedPointer = copyOnCpuAllowed ? device.getMemoryManager()->lockResource(allocationInfo.memory) : nullptr) {
-                    memcpy_s(ptrOffset(lockedPointer, pBuffer->getOffset()), size, hostPtr, size);
-                    allocationInfo.memory->setAubWritable(true, GraphicsAllocation::defaultBank);
-                    allocationInfo.memory->setTbxWritable(true, GraphicsAllocation::defaultBank);
-                    copyExecuted = true;
-                } else {
-                    auto blitMemoryToAllocationResult = BlitOperationResult::Unsupported;
-
-                    if (hwInfoConfig->isBlitterFullySupported(hwInfo) && isLocalMemory) {
-                        blitMemoryToAllocationResult = BlitHelperFunctions::blitMemoryToAllocation(device, allocationInfo.memory, pBuffer->getOffset(), hostPtr, {size, 1, 1});
-                    }
-
-                    if (blitMemoryToAllocationResult != BlitOperationResult::Success) {
-                        auto cmdQ = context->getSpecialQueue(rootDeviceIndex);
-                        if (CL_SUCCESS != cmdQ->enqueueWriteBuffer(pBuffer, CL_TRUE, 0, size, hostPtr, allocationInfo.mapAllocation, 0, nullptr, nullptr)) {
-                            errcodeRet = CL_OUT_OF_RESOURCES;
-                        }
-                    }
-                    copyExecuted = true;
-                }
-            } else {
-                memcpy_s(allocationInfo.memory->getUnderlyingBuffer(), size, hostPtr, size);
-                copyExecuted = true;
-            }
+            copyExecuted = copyHostPointer(pBuffer,
+                                           size,
+                                           hostPtr,
+                                           allocationInfo.memory,
+                                           allocationInfo.mapAllocation,
+                                           rootDeviceIndex,
+                                           isCompressionEnabled,
+                                           implicitScalingEnabled,
+                                           errcodeRet);
         }
     }
 

opencl/source/mem_obj/mem_obj.cpp

@@ -104,6 +104,7 @@ MemObj::~MemObj() {
         }
         if (associatedMemObject) {
             associatedMemObject->decRefInternal();
+            context->getBufferPoolAllocator().tryFreeFromPoolBuffer(associatedMemObject, this->offset, this->size);
         }
         if (!associatedMemObject) {
             releaseAllocatedMapPtr();
@@ -112,7 +113,10 @@ MemObj::~MemObj() {
 
     destructorCallbacks.invoke(this);
 
-    context->decRefInternal();
+    const bool needDecrementContextRefCount = !context->getBufferPoolAllocator().isPoolBuffer(this);
+    if (needDecrementContextRefCount) {
+        context->decRefInternal();
+    }
 }
 
 cl_int MemObj::getMemObjectInfo(cl_mem_info paramName,

opencl/test/unit_test/mem_obj/CMakeLists.txt

@@ -1,12 +1,13 @@
 #
-# Copyright (C) 2018-2021 Intel Corporation
+# Copyright (C) 2018-2022 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
 
 set(IGDRCL_SRCS_tests_mem_obj
     ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
     ${CMAKE_CURRENT_SOURCE_DIR}/buffer_pin_tests.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/buffer_pool_alloc_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/buffer_set_arg_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/buffer_tests.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/buffer_bcs_tests.cpp