L0::Event to support dynamic size

Signed-off-by: Bartosz Dunajski <[email protected]>

Author: BartoszDunajski

level_zero/core/source/CMakeLists.txt

@@ -44,6 +44,7 @@ set(L0_RUNTIME_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/driver/driver_imp.h
     ${CMAKE_CURRENT_SOURCE_DIR}/driver/host_pointer_manager.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/driver/host_pointer_manager.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/event/event_impl.inl
     ${CMAKE_CURRENT_SOURCE_DIR}/event/event.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/event/event.h
     ${CMAKE_CURRENT_SOURCE_DIR}/fence/fence.cpp

level_zero/core/source/event/event.cpp

@@ -24,6 +24,7 @@
 #include "level_zero/core/source/device/device.h"
 #include "level_zero/core/source/device/device_imp.h"
 #include "level_zero/core/source/driver/driver_handle_imp.h"
+#include "level_zero/core/source/event/event_impl.inl"
 #include "level_zero/tools/source/metrics/metric.h"
 
 #include <set>
@@ -110,289 +111,16 @@ ze_result_t EventPoolImp::createEvent(const ze_event_desc_t *desc, ze_event_hand
     if (desc->index > (getNumEvents() - 1)) {
         return ZE_RESULT_ERROR_INVALID_ARGUMENT;
     }
-    *phEvent = Event::create(this, desc, this->getDevice());
+    *phEvent = Event::create<uint32_t>(this, desc, this->getDevice());
 
     return ZE_RESULT_SUCCESS;
 }
 
-uint64_t EventImp::getGpuAddress(Device *device) {
-    auto alloc = eventPool->getAllocation().getGraphicsAllocation(device->getNEODevice()->getRootDeviceIndex());
-    return (alloc->getGpuAddress() + (index * eventPool->getEventSize()));
-}
-
-Event *Event::create(EventPool *eventPool, const ze_event_desc_t *desc, Device *device) {
-    auto event = new EventImp(eventPool, desc->index, device);
-    UNRECOVERABLE_IF(event == nullptr);
-
-    if (eventPool->isEventPoolUsedForTimestamp) {
-        event->isTimestampEvent = true;
-        event->kernelTimestampsData = std::make_unique<NEO::TimestampPackets<uint32_t>[]>(EventPacketsCount::maxKernelSplit);
-    }
-
-    auto alloc = eventPool->getAllocation().getGraphicsAllocation(device->getNEODevice()->getRootDeviceIndex());
-
-    uint64_t baseHostAddr = reinterpret_cast<uint64_t>(alloc->getUnderlyingBuffer());
-    event->hostAddress = reinterpret_cast<void *>(baseHostAddr + (desc->index * eventPool->getEventSize()));
-    event->signalScope = desc->signal;
-    event->waitScope = desc->wait;
-    event->csr = static_cast<DeviceImp *>(device)->neoDevice->getDefaultEngine().commandStreamReceiver;
-    event->reset();
-
-    return event;
-}
-
-NEO::GraphicsAllocation &EventImp::getAllocation(Device *device) {
-    return *this->eventPool->getAllocation().getGraphicsAllocation(device->getNEODevice()->getRootDeviceIndex());
-}
-
-void Event::resetPackets() {
-    for (uint32_t i = 0; i < kernelCount; i++) {
-        kernelTimestampsData[i].setPacketsUsed(1);
-    }
-    kernelCount = 1;
-}
-
-uint32_t Event::getPacketsInUse() {
-    if (isTimestampEvent) {
-        uint32_t packetsInUse = 0;
-        for (uint32_t i = 0; i < kernelCount; i++) {
-            packetsInUse += kernelTimestampsData[i].getPacketsUsed();
-        };
-        return packetsInUse;
-    } else {
-        return 1;
-    }
-}
-
-void Event::setPacketsInUse(uint32_t value) {
-    kernelTimestampsData[getCurrKernelDataIndex()].setPacketsUsed(value);
-};
-
-uint64_t Event::getPacketAddress(Device *device) {
-    uint64_t address = getGpuAddress(device);
-    if (isTimestampEvent && kernelCount > 1) {
-        for (uint32_t i = 0; i < kernelCount - 1; i++) {
-            address += kernelTimestampsData[i].getPacketsUsed() *
-                       NEO::TimestampPackets<uint32_t>::getSinglePacketSize();
-        }
-    }
-    return address;
-}
-
-ze_result_t EventImp::calculateProfilingData() {
-    globalStartTS = kernelTimestampsData[0].getGlobalStartValue(0);
-    globalEndTS = kernelTimestampsData[0].getGlobalEndValue(0);
-    contextStartTS = kernelTimestampsData[0].getContextStartValue(0);
-    contextEndTS = kernelTimestampsData[0].getContextEndValue(0);
-
-    for (uint32_t i = 0; i < kernelCount; i++) {
-        for (auto packetId = 0u; packetId < kernelTimestampsData[i].getPacketsUsed(); packetId++) {
-            if (globalStartTS > kernelTimestampsData[i].getGlobalStartValue(packetId)) {
-                globalStartTS = kernelTimestampsData[i].getGlobalStartValue(packetId);
-            }
-            if (contextStartTS > kernelTimestampsData[i].getContextStartValue(packetId)) {
-                contextStartTS = kernelTimestampsData[i].getContextStartValue(packetId);
-            }
-            if (contextEndTS < kernelTimestampsData[i].getContextEndValue(packetId)) {
-                contextEndTS = kernelTimestampsData[i].getContextEndValue(packetId);
-            }
-            if (globalEndTS < kernelTimestampsData[i].getGlobalEndValue(packetId)) {
-                globalEndTS = kernelTimestampsData[i].getGlobalEndValue(packetId);
-            }
-        }
-    }
-
-    return ZE_RESULT_SUCCESS;
-}
-
-void EventImp::assignTimestampData(void *address) {
-    for (uint32_t i = 0; i < kernelCount; i++) {
-        uint32_t packetsToCopy = kernelTimestampsData[i].getPacketsUsed();
-        for (uint32_t packetId = 0; packetId < packetsToCopy; packetId++) {
-            kernelTimestampsData[i].assignDataToAllTimestamps(packetId, address);
-            address = ptrOffset(address, NEO::TimestampPackets<uint32_t>::getSinglePacketSize());
-        }
-    }
-}
-
 ze_result_t Event::destroy() {
     delete this;
     return ZE_RESULT_SUCCESS;
 }
 
-ze_result_t EventImp::queryStatusKernelTimestamp() {
-    assignTimestampData(hostAddress);
-    for (uint32_t i = 0; i < kernelCount; i++) {
-        uint32_t packetsToCheck = kernelTimestampsData[i].getPacketsUsed();
-        for (uint32_t packetId = 0; packetId < packetsToCheck; packetId++) {
-            if (kernelTimestampsData[i].getContextEndValue(packetId) == Event::STATE_CLEARED) {
-                return ZE_RESULT_NOT_READY;
-            }
-        }
-    }
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t EventImp::queryStatus() {
-    uint64_t *hostAddr = static_cast<uint64_t *>(hostAddress);
-    uint32_t queryVal = Event::STATE_CLEARED;
-    ze_result_t retVal;
-
-    if (metricStreamer != nullptr) {
-        *hostAddr = metricStreamer->getNotificationState();
-    }
-    this->csr->downloadAllocations();
-    if (isTimestampEvent) {
-        return queryStatusKernelTimestamp();
-    }
-    memcpy_s(static_cast<void *>(&queryVal), sizeof(uint32_t), static_cast<void *>(hostAddr), sizeof(uint32_t));
-    retVal = (queryVal == Event::STATE_CLEARED) ? ZE_RESULT_NOT_READY : ZE_RESULT_SUCCESS;
-
-    if (retVal == ZE_RESULT_NOT_READY) {
-        return retVal;
-    }
-
-    if (updateTaskCountEnabled) {
-        this->csr->flushTagUpdate();
-        updateTaskCountEnabled = false;
-    }
-
-    return retVal;
-}
-
-ze_result_t EventImp::hostEventSetValueTimestamps(uint32_t eventVal) {
-
-    auto baseAddr = reinterpret_cast<uint64_t>(hostAddress);
-    auto signalScopeFlag = this->signalScope;
-
-    auto eventTsSetFunc = [&eventVal, &signalScopeFlag](auto tsAddr) {
-        auto tsptr = reinterpret_cast<void *>(tsAddr);
-
-        memcpy_s(tsptr, sizeof(uint32_t), static_cast<void *>(&eventVal), sizeof(uint32_t));
-        if (!signalScopeFlag) {
-            NEO::CpuIntrinsics::clFlush(tsptr);
-        }
-    };
-    for (uint32_t i = 0; i < kernelCount; i++) {
-        uint32_t packetsToSet = kernelTimestampsData[i].getPacketsUsed();
-        for (uint32_t i = 0; i < packetsToSet; i++) {
-            eventTsSetFunc(baseAddr + NEO::TimestampPackets<uint32_t>::getContextStartOffset());
-            eventTsSetFunc(baseAddr + NEO::TimestampPackets<uint32_t>::getGlobalStartOffset());
-            eventTsSetFunc(baseAddr + NEO::TimestampPackets<uint32_t>::getContextEndOffset());
-            eventTsSetFunc(baseAddr + NEO::TimestampPackets<uint32_t>::getGlobalEndOffset());
-            baseAddr += NEO::TimestampPackets<uint32_t>::getSinglePacketSize();
-        }
-    }
-    assignTimestampData(hostAddress);
-
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t EventImp::hostEventSetValue(uint32_t eventVal) {
-    if (isTimestampEvent) {
-        return hostEventSetValueTimestamps(eventVal);
-    }
-
-    auto hostAddr = static_cast<uint64_t *>(hostAddress);
-    UNRECOVERABLE_IF(hostAddr == nullptr);
-    memcpy_s(static_cast<void *>(hostAddr), sizeof(uint32_t), static_cast<void *>(&eventVal), sizeof(uint32_t));
-
-    if (updateTaskCountEnabled) {
-        this->csr->flushTagUpdate();
-        updateTaskCountEnabled = false;
-    }
-
-    NEO::CpuIntrinsics::clFlush(hostAddr);
-
-    return ZE_RESULT_SUCCESS;
-}
-
-ze_result_t EventImp::hostSignal() {
-    return hostEventSetValue(Event::STATE_SIGNALED);
-}
-
-ze_result_t EventImp::hostSynchronize(uint64_t timeout) {
-    std::chrono::high_resolution_clock::time_point time1, time2;
-    uint64_t timeDiff = 0;
-
-    ze_result_t ret = ZE_RESULT_NOT_READY;
-
-    if (this->csr->getType() == NEO::CommandStreamReceiverType::CSR_AUB) {
-        return ZE_RESULT_SUCCESS;
-    }
-
-    if (timeout == 0) {
-        return queryStatus();
-    }
-
-    time1 = std::chrono::high_resolution_clock::now();
-    while (true) {
-        ret = queryStatus();
-        if (ret == ZE_RESULT_SUCCESS) {
-            return ret;
-        }
-
-        NEO::WaitUtils::waitFunction(nullptr, 0u);
-
-        if (timeout == std::numeric_limits<uint32_t>::max()) {
-            continue;
-        }
-
-        time2 = std::chrono::high_resolution_clock::now();
-        timeDiff = std::chrono::duration_cast<std::chrono::nanoseconds>(time2 - time1).count();
-
-        if (timeDiff >= timeout) {
-            break;
-        }
-    }
-
-    return ret;
-}
-
-ze_result_t EventImp::reset() {
-    if (isTimestampEvent) {
-        kernelCount = EventPacketsCount::maxKernelSplit;
-        for (uint32_t i = 0; i < kernelCount; i++) {
-            kernelTimestampsData[i].setPacketsUsed(NEO::TimestampPacketSizeControl::preferredPacketCount);
-        }
-        hostEventSetValue(Event::STATE_INITIAL);
-        resetPackets();
-        return ZE_RESULT_SUCCESS;
-    } else {
-        return hostEventSetValue(Event::STATE_INITIAL);
-    }
-}
-
-ze_result_t EventImp::queryKernelTimestamp(ze_kernel_timestamp_result_t *dstptr) {
-
-    ze_kernel_timestamp_result_t &result = *dstptr;
-
-    if (queryStatus() != ZE_RESULT_SUCCESS) {
-        return ZE_RESULT_NOT_READY;
-    }
-
-    assignTimestampData(hostAddress);
-    calculateProfilingData();
-
-    auto eventTsSetFunc = [&](uint64_t &timestampFieldToCopy, uint64_t &timestampFieldForWriting) {
-        memcpy_s(&(timestampFieldForWriting), sizeof(uint64_t), static_cast<void *>(&timestampFieldToCopy), sizeof(uint64_t));
-    };
-
-    if (!NEO::HwHelper::get(device->getHwInfo().platform.eRenderCoreFamily).useOnlyGlobalTimestamps()) {
-        eventTsSetFunc(contextStartTS, result.context.kernelStart);
-        eventTsSetFunc(globalStartTS, result.global.kernelStart);
-        eventTsSetFunc(contextEndTS, result.context.kernelEnd);
-        eventTsSetFunc(globalEndTS, result.global.kernelEnd);
-    } else {
-        eventTsSetFunc(globalStartTS, result.context.kernelStart);
-        eventTsSetFunc(globalStartTS, result.global.kernelStart);
-        eventTsSetFunc(globalEndTS, result.context.kernelEnd);
-        eventTsSetFunc(globalEndTS, result.global.kernelEnd);
-    }
-
-    return ZE_RESULT_SUCCESS;
-}
-
 EventPool *EventPool::create(DriverHandle *driver, Context *context, uint32_t numDevices,
                              ze_device_handle_t *phDevices,
                              const ze_event_pool_desc_t *desc) {

level_zero/core/source/event/event.h

@@ -42,6 +42,7 @@ struct Event : _ze_event_handle_t {
         STATE_INITIAL = STATE_CLEARED
     };
 
+    template <typename TagSizeT>
     static Event *create(EventPool *eventPool, const ze_event_desc_t *desc, Device *device);
 
     static Event *fromHandle(ze_event_handle_t handle) { return static_cast<Event *>(handle); }
@@ -51,11 +52,11 @@ struct Event : _ze_event_handle_t {
     virtual NEO::GraphicsAllocation &getAllocation(Device *device) = 0;
 
     virtual uint64_t getGpuAddress(Device *device) = 0;
-    uint32_t getPacketsInUse();
-    uint64_t getPacketAddress(Device *device);
-    void resetPackets();
+    virtual uint32_t getPacketsInUse() = 0;
+    virtual uint64_t getPacketAddress(Device *device) = 0;
+    virtual void resetPackets() = 0;
     void *getHostAddress() { return hostAddress; }
-    void setPacketsInUse(uint32_t value);
+    virtual void setPacketsInUse(uint32_t value) = 0;
     uint32_t getCurrKernelDataIndex() const { return kernelCount - 1; }
     void *hostAddress = nullptr;
     uint32_t kernelCount = 1u;
@@ -64,7 +65,6 @@ struct Event : _ze_event_handle_t {
     bool isTimestampEvent = false;
     bool updateTaskCountEnabled = false;
 
-    std::unique_ptr<NEO::TimestampPackets<uint32_t>[]> kernelTimestampsData = nullptr;
     uint64_t globalStartTS;
     uint64_t globalEndTS;
     uint64_t contextStartTS;
@@ -79,6 +79,7 @@ struct Event : _ze_event_handle_t {
     NEO::GraphicsAllocation *allocation = nullptr;
 };
 
+template <typename TagSizeT>
 struct EventImp : public Event {
     EventImp(EventPool *eventPool, int index, Device *device)
         : device(device), index(index), eventPool(eventPool) {}
@@ -99,6 +100,13 @@ struct EventImp : public Event {
 
     uint64_t getGpuAddress(Device *device) override;
 
+    void resetPackets() override;
+    uint64_t getPacketAddress(Device *device) override;
+    uint32_t getPacketsInUse() override;
+    void setPacketsInUse(uint32_t value) override;
+
+    std::unique_ptr<NEO::TimestampPackets<TagSizeT>[]> kernelTimestampsData;
+
     Device *device;
     int index;
     EventPool *eventPool;
@@ -107,7 +115,7 @@ struct EventImp : public Event {
     ze_result_t calculateProfilingData();
     ze_result_t queryStatusKernelTimestamp();
     ze_result_t hostEventSetValue(uint32_t eventValue);
-    ze_result_t hostEventSetValueTimestamps(uint32_t eventVal);
+    ze_result_t hostEventSetValueTimestamps(TagSizeT eventVal);
     void assignTimestampData(void *address);
 };