Skip to content

Commit 5f4a5a2

Browse files
martygrantmartygrant
authored
Merge pull request #2334 from winstonzhang-intel/interrupt-based
[L0] Interrupt-based event implementation
2 parents 4111306 + 039cf14 commit 5f4a5a2

File tree

8 files changed

+250
-133
lines changed

8 files changed

+250
-133
lines changed

‎source/adapters/level_zero/command_buffer.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,8 @@ ur_result_t createSyncPointAndGetZeEvents(
215215
UR_CALL(EventCreate(CommandBuffer->Context, nullptr /*Queue*/,
216216
false /*IsMultiDevice*/, HostVisible, &LaunchEvent,
217217
false /*CounterBasedEventEnabled*/,
218-
!CommandBuffer->IsProfilingEnabled));
218+
!CommandBuffer->IsProfilingEnabled,
219+
false /*InterruptBasedEventEnabled*/));
219220
LaunchEvent->CommandType = CommandType;
220221
ZeLaunchEvent = LaunchEvent->ZeEvent;
221222

@@ -683,13 +684,15 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
683684
if (Device->hasMainCopyEngine()) {
684685
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false, false,
685686
&CopyFinishedEvent, UseCounterBasedEvents,
686-
!EnableProfiling));
687+
!EnableProfiling,
688+
false /*InterruptBasedEventEnabled*/));
687689
}
688690

689691
if (EnableProfiling) {
690692
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
691693
false /*HostVisible*/, &ComputeFinishedEvent,
692-
UseCounterBasedEvents, !EnableProfiling));
694+
UseCounterBasedEvents, !EnableProfiling,
695+
false /*InterruptBasedEventEnabled*/));
693696
}
694697
}
695698

@@ -698,7 +701,8 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
698701
if (WaitEventPath) {
699702
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
700703
false /*HostVisible*/, &WaitEvent,
701-
false /*CounterBasedEventEnabled*/, !EnableProfiling));
704+
false /*CounterBasedEventEnabled*/, !EnableProfiling,
705+
false /*InterruptBasedEventEnabled*/));
702706
}
703707

704708
// Create ZeCommandListResetEvents only if counter-based events are not being
@@ -710,15 +714,17 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
710714
if (!UseCounterBasedEvents) {
711715
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
712716
false /*HostVisible*/, &AllResetEvent,
713-
false /*CounterBasedEventEnabled*/, !EnableProfiling));
717+
false /*CounterBasedEventEnabled*/, !EnableProfiling,
718+
false /*InterruptBasedEventEnabled*/));
714719

715720
UR_CALL(createMainCommandList(Context, Device, false, false, false,
716721
ZeCommandListResetEvents));
717722

718723
// The ExecutionFinishedEvent is only waited on by ZeCommandListResetEvents.
719724
UR_CALL(EventCreate(Context, nullptr /*Queue*/, false /*IsMultiDevice*/,
720725
false /*HostVisible*/, &ExecutionFinishedEvent,
721-
false /*CounterBasedEventEnabled*/, !EnableProfiling));
726+
false /*CounterBasedEventEnabled*/, !EnableProfiling,
727+
false /*InterruptBased*/));
722728
}
723729

724730
try {

‎source/adapters/level_zero/context.cpp‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
478478
ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
479479
ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
480480
bool ProfilingEnabled, ur_device_handle_t Device,
481-
bool CounterBasedEventEnabled, bool UsingImmCmdList) {
481+
bool CounterBasedEventEnabled, bool UsingImmCmdList,
482+
bool InterruptBasedEventEnabled) {
482483
// Lock while updating event pool machinery.
483484
std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
484485

@@ -487,9 +488,9 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
487488
if (Device) {
488489
ZeDevice = Device->ZeDevice;
489490
}
490-
std::list<ze_event_pool_handle_t> *ZePoolCache =
491-
getZeEventPoolCache(HostVisible, ProfilingEnabled,
492-
CounterBasedEventEnabled, UsingImmCmdList, ZeDevice);
491+
std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
492+
HostVisible, ProfilingEnabled, CounterBasedEventEnabled, UsingImmCmdList,
493+
InterruptBasedEventEnabled, ZeDevice);
493494

494495
if (!ZePoolCache->empty()) {
495496
if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -537,6 +538,14 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
537538
counterBasedExt.flags);
538539
ZeEventPoolDesc.pNext = &counterBasedExt;
539540
}
541+
if (InterruptBasedEventEnabled) {
542+
ze_intel_event_sync_mode_exp_desc_t eventSyncMode = {
543+
ZE_INTEL_STRUCTURE_TYPE_EVENT_SYNC_MODE_EXP_DESC, nullptr, 0};
544+
eventSyncMode.syncModeFlags =
545+
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_LOW_POWER_WAIT |
546+
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_SIGNAL_INTERRUPT;
547+
ZeEventPoolDesc.pNext = &eventSyncMode;
548+
}
540549

541550
std::vector<ze_device_handle_t> ZeDevices;
542551
if (ZeDevice) {
@@ -563,27 +572,31 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
563572

564573
ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
565574
bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
566-
bool CounterBasedEventEnabled) {
575+
bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) {
567576
std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
568-
auto Cache = getEventCache(HostVisible, WithProfiling, Device,
569-
CounterBasedEventEnabled);
577+
auto Cache =
578+
getEventCache(HostVisible, WithProfiling, Device,
579+
CounterBasedEventEnabled, InterruptBasedEventEnabled);
570580
if (Cache->empty()) {
571581
logger::info("Cache empty (Host Visible: {}, Profiling: {}, Counter: {}, "
572-
"Device: {})",
573-
HostVisible, WithProfiling, CounterBasedEventEnabled, Device);
582+
"Interrupt: {}, Device: {})",
583+
HostVisible, WithProfiling, CounterBasedEventEnabled,
584+
InterruptBasedEventEnabled, Device);
574585
return nullptr;
575586
}
576587

577588
auto It = Cache->begin();
578589
ur_event_handle_t Event = *It;
590+
579591
Cache->erase(It);
580592
// We have to reset event before using it.
581593
Event->reset();
582594

583595
logger::info("Using {} event (Host Visible: {}, Profiling: {}, Counter: {}, "
584-
"Device: {}) from cache {}",
596+
"Interrupt: {}, Device: {}) from cache {}",
585597
Event, Event->HostVisibleEvent, Event->isProfilingEnabled(),
586-
Event->CounterBasedEventsEnabled, Device, Cache);
598+
Event->CounterBasedEventsEnabled,
599+
Event->InterruptBasedEventsEnabled, Device, Cache);
587600

588601
return Event;
589602
}
@@ -596,9 +609,9 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
596609
Device = Event->UrQueue->Device;
597610
}
598611

599-
auto Cache =
600-
getEventCache(Event->isHostVisible(), Event->isProfilingEnabled(), Device,
601-
Event->CounterBasedEventsEnabled);
612+
auto Cache = getEventCache(
613+
Event->isHostVisible(), Event->isProfilingEnabled(), Device,
614+
Event->CounterBasedEventsEnabled, Event->InterruptBasedEventsEnabled);
602615
logger::info("Inserting {} event (Host Visible: {}, Profiling: {}, Counter: "
603616
"{}, Device: {}) into cache {}",
604617
Event, Event->HostVisibleEvent, Event->isProfilingEnabled(),
@@ -627,7 +640,8 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
627640

628641
std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
629642
Event->isHostVisible(), Event->isProfilingEnabled(),
630-
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists, ZeDevice);
643+
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists,
644+
Event->InterruptBasedEventsEnabled, ZeDevice);
631645

632646
// Put the empty pool to the cache of the pools.
633647
if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)

‎source/adapters/level_zero/context.hpp‎

Lines changed: 94 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,24 @@ struct l0_command_list_cache_info {
3333
bool IsImmediate = false;
3434
};
3535

36+
typedef uint32_t ze_intel_event_sync_mode_exp_flags_t;
37+
typedef enum _ze_intel_event_sync_mode_exp_flag_t {
38+
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_LOW_POWER_WAIT = ZE_BIT(0),
39+
ZE_INTEL_EVENT_SYNC_MODE_EXP_FLAG_SIGNAL_INTERRUPT = ZE_BIT(1),
40+
ZE_INTEL_EVENT_SYNC_MODE_EXP_EXP_FLAG_FORCE_UINT32 = 0x7fffffff
41+
42+
} ze_intel_event_sync_mode_exp_flag_t;
43+
44+
#define ZE_INTEL_STRUCTURE_TYPE_EVENT_SYNC_MODE_EXP_DESC \
45+
(ze_structure_type_t)0x00030016
46+
47+
typedef struct _ze_intel_event_sync_mode_exp_desc_t {
48+
ze_structure_type_t stype;
49+
const void *pNext;
50+
51+
ze_intel_event_sync_mode_exp_flags_t syncModeFlags;
52+
} ze_intel_event_sync_mode_exp_desc_t;
53+
3654
struct ur_context_handle_t_ : _ur_object {
3755
ur_context_handle_t_(ze_context_handle_t ZeContext, uint32_t NumDevices,
3856
const ur_device_handle_t *Devs, bool OwnZeContext)
@@ -150,9 +168,9 @@ struct ur_context_handle_t_ : _ur_object {
150168
// head.
151169
//
152170
// Cache of event pools to which host-visible events are added to.
153-
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{12};
171+
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{30};
154172
std::vector<std::unordered_map<ze_device_handle_t, size_t>>
155-
ZeEventPoolCacheDeviceMap{12};
173+
ZeEventPoolCacheDeviceMap{30};
156174

157175
// This map will be used to determine if a pool is full or not
158176
// by storing number of empty slots available in the pool.
@@ -199,13 +217,15 @@ struct ur_context_handle_t_ : _ur_object {
199217
bool ProfilingEnabled,
200218
ur_device_handle_t Device,
201219
bool CounterBasedEventEnabled,
202-
bool UsingImmCmdList);
220+
bool UsingImmCmdList,
221+
bool InterruptBasedEventEnabled);
203222

204223
// Get ur_event_handle_t from cache.
205224
ur_event_handle_t getEventFromContextCache(bool HostVisible,
206225
bool WithProfiling,
207226
ur_device_handle_t Device,
208-
bool CounterBasedEventEnabled);
227+
bool CounterBasedEventEnabled,
228+
bool InterruptBasedEventEnabled);
209229

210230
// Add ur_event_handle_t to cache.
211231
void addEventToContextCache(ur_event_handle_t);
@@ -216,17 +236,29 @@ struct ur_context_handle_t_ : _ur_object {
216236
HostVisibleCounterBasedRegularCacheType,
217237
HostInvisibleCounterBasedRegularCacheType,
218238
HostVisibleCounterBasedImmediateCacheType,
219-
HostInvisibleCounterBasedImmediateCacheType
239+
HostInvisibleCounterBasedImmediateCacheType,
240+
241+
HostVisibleInterruptBasedRegularCacheType,
242+
HostInvisibleInterruptBasedRegularCacheType,
243+
HostVisibleInterruptBasedImmediateCacheType,
244+
HostInvisibleInterruptBasedImmediateCacheType,
245+
246+
HostVisibleInterruptAndCounterBasedRegularCacheType,
247+
HostInvisibleInterruptAndCounterBasedRegularCacheType,
248+
HostVisibleInterruptAndCounterBasedImmediateCacheType,
249+
HostInvisibleInterruptAndCounterBasedImmediateCacheType
220250
};
221251

222252
std::list<ze_event_pool_handle_t> *
223253
getZeEventPoolCache(bool HostVisible, bool WithProfiling,
224254
bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
255+
bool InterruptBasedEventEnabled,
225256
ze_device_handle_t ZeDevice) {
226257
EventPoolCacheType CacheType;
227258

228259
calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
229-
UsingImmediateCmdList, CacheType);
260+
UsingImmediateCmdList, InterruptBasedEventEnabled,
261+
CacheType);
230262
if (ZeDevice) {
231263
auto ZeEventPoolCacheMap =
232264
WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType * 2]
@@ -246,23 +278,57 @@ struct ur_context_handle_t_ : _ur_object {
246278
ur_result_t calculateCacheIndex(bool HostVisible,
247279
bool CounterBasedEventEnabled,
248280
bool UsingImmediateCmdList,
281+
bool InterruptBasedEventEnabled,
249282
EventPoolCacheType &CacheType) {
250-
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
251-
CacheType = HostVisibleCounterBasedRegularCacheType;
252-
} else if (CounterBasedEventEnabled && !HostVisible &&
253-
!UsingImmediateCmdList) {
254-
CacheType = HostInvisibleCounterBasedRegularCacheType;
255-
} else if (CounterBasedEventEnabled && HostVisible &&
256-
UsingImmediateCmdList) {
257-
CacheType = HostVisibleCounterBasedImmediateCacheType;
258-
} else if (CounterBasedEventEnabled && !HostVisible &&
259-
UsingImmediateCmdList) {
260-
CacheType = HostInvisibleCounterBasedImmediateCacheType;
261-
} else if (!CounterBasedEventEnabled && HostVisible) {
262-
CacheType = HostVisibleCacheType;
283+
if (InterruptBasedEventEnabled) {
284+
if (CounterBasedEventEnabled) {
285+
if (HostVisible) {
286+
if (UsingImmediateCmdList) {
287+
CacheType = HostVisibleInterruptAndCounterBasedImmediateCacheType;
288+
} else {
289+
CacheType = HostVisibleInterruptAndCounterBasedRegularCacheType;
290+
}
291+
} else {
292+
if (UsingImmediateCmdList) {
293+
CacheType = HostInvisibleInterruptAndCounterBasedImmediateCacheType;
294+
} else {
295+
CacheType = HostInvisibleInterruptAndCounterBasedRegularCacheType;
296+
}
297+
}
298+
} else {
299+
if (HostVisible) {
300+
if (UsingImmediateCmdList) {
301+
CacheType = HostVisibleInterruptBasedImmediateCacheType;
302+
} else {
303+
CacheType = HostVisibleInterruptBasedRegularCacheType;
304+
}
305+
} else {
306+
if (UsingImmediateCmdList) {
307+
CacheType = HostInvisibleInterruptBasedImmediateCacheType;
308+
} else {
309+
CacheType = HostInvisibleInterruptBasedRegularCacheType;
310+
}
311+
}
312+
}
263313
} else {
264-
CacheType = HostInvisibleCacheType;
314+
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
315+
CacheType = HostVisibleCounterBasedRegularCacheType;
316+
} else if (CounterBasedEventEnabled && !HostVisible &&
317+
!UsingImmediateCmdList) {
318+
CacheType = HostInvisibleCounterBasedRegularCacheType;
319+
} else if (CounterBasedEventEnabled && HostVisible &&
320+
UsingImmediateCmdList) {
321+
CacheType = HostVisibleCounterBasedImmediateCacheType;
322+
} else if (CounterBasedEventEnabled && !HostVisible &&
323+
UsingImmediateCmdList) {
324+
CacheType = HostInvisibleCounterBasedImmediateCacheType;
325+
} else if (!CounterBasedEventEnabled && HostVisible) {
326+
CacheType = HostVisibleCacheType;
327+
} else {
328+
CacheType = HostInvisibleCacheType;
329+
}
265330
}
331+
266332
return UR_RESULT_SUCCESS;
267333
}
268334

@@ -308,9 +374,10 @@ struct ur_context_handle_t_ : _ur_object {
308374
EVENT_FLAG_HOST_VISIBLE = UR_BIT(0),
309375
EVENT_FLAG_WITH_PROFILING = UR_BIT(1),
310376
EVENT_FLAG_COUNTER = UR_BIT(2),
311-
EVENT_FLAG_DEVICE = UR_BIT(3), // if set, subsequent bits are device id
377+
EVENT_FLAG_INTERRUPT = UR_BIT(3),
378+
EVENT_FLAG_DEVICE = UR_BIT(4), // if set, subsequent bits are device id
312379
MAX_EVENT_FLAG_BITS =
313-
4, // this is used as an offset for embedding device id
380+
5, // this is used as an offset for embedding device id
314381
};
315382

316383
// Mutex to control operations on event caches.
@@ -322,7 +389,8 @@ struct ur_context_handle_t_ : _ur_object {
322389

323390
// Get the cache of events for a provided scope and profiling mode.
324391
EventCache *getEventCache(bool HostVisible, bool WithProfiling,
325-
ur_device_handle_t Device, bool Counter) {
392+
ur_device_handle_t Device, bool Counter,
393+
bool Interrupt) {
326394

327395
size_t index = 0;
328396
if (HostVisible) {
@@ -334,6 +402,9 @@ struct ur_context_handle_t_ : _ur_object {
334402
if (Counter) {
335403
index |= EVENT_FLAG_COUNTER;
336404
}
405+
if (Interrupt) {
406+
index |= EVENT_FLAG_INTERRUPT;
407+
}
337408
if (Device) {
338409
index |= EVENT_FLAG_DEVICE | (*Device->Id << MAX_EVENT_FLAG_BITS);
339410
}

‎source/adapters/level_zero/device.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,8 @@ ur_result_t urDeviceGetInfo(
485485
case UR_DEVICE_INFO_BUILT_IN_KERNELS:
486486
// TODO: To find out correct value
487487
return ReturnValue("");
488+
case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
489+
return ReturnValue(static_cast<ur_bool_t>(true));
488490
case UR_DEVICE_INFO_QUEUE_PROPERTIES:
489491
return ReturnValue(
490492
ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE |
@@ -1155,8 +1157,6 @@ ur_result_t urDeviceGetInfo(
11551157
return ReturnValue(true);
11561158
case UR_DEVICE_INFO_USM_POOL_SUPPORT:
11571159
return ReturnValue(true);
1158-
case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
1159-
return ReturnValue(false);
11601160
case UR_DEVICE_INFO_2D_BLOCK_ARRAY_CAPABILITIES_EXP: {
11611161
#ifdef ZE_INTEL_DEVICE_BLOCK_ARRAY_EXP_NAME
11621162
const auto ZeDeviceBlockArrayFlags =

0 commit comments

Comments
 (0)