feature: add debug flags and instrumentation of waitpkg calls

Related-To: NEO-9737

Signed-off-by: Zbigniew Zdanowicz <[email protected]>

Author: zzdanowicz

shared/source/debug_settings/debug_variables_base.inl

@@ -98,6 +98,9 @@ DECLARE_DEBUG_VARIABLE(std::string, WddmResidencyLoggerOutputDirectory, std::str
 DECLARE_DEBUG_VARIABLE(int64_t, OverrideMultiStoragePlacement, -1, "Place memory only in selected tiles indicated by bit mask; ignore when -1")
 DECLARE_DEBUG_VARIABLE(int64_t, ForceCompressionDisabledForCompressedBlitCopies, -1, "If compression is required, set AUX_CCS_E, but force CompressionEnable filed; 0 should result in uncompressed read/write; values = -1: default, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int64_t, WddmPagingFenceCpuWaitDelayTime, 0, "Amount of microseconds after waitng for paging fence on CPU")
+DECLARE_DEBUG_VARIABLE(int64_t, OverrideEventSynchronizeTimeout, -1, "-1: default - user provided timeout value,  >0: timeout in nanoseconds")
+DECLARE_DEBUG_VARIABLE(int64_t, WaitpkgCounterValue, -1, "-1: use default, >=0: use constant value added for umwait counter")
+DECLARE_DEBUG_VARIABLE(int32_t, WaitpkgControlValue, -1, "-1: use default, 0: slower wakeup - larger power savings, 1: faster wakeup - smaller power savings")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceL1Caching, -1, "Program L1 cache policy for surface state and stateless accesses; values = -1: default, 0: disable, 1: enable")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceAuxTranslationEnabled, -1, "Require AUX translation for kernels; values = -1: default, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableExperimentalCommandBuffer, 0, "Inject experimental command buffer")
@@ -247,7 +250,6 @@ DECLARE_DEBUG_VARIABLE(int32_t, ExitOnSubmissionMode, 0, "Exit on X submission m
 DECLARE_DEBUG_VARIABLE(int32_t, ForceInOrderImmediateCmdListExecution, -1, "-1: default, 0: disabled, 1: all Immediate Command Lists are switched to in-order execution")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceInOrderEvents, -1, "-1: default, 0: disabled, 1: Enable all Events as in-order, to rely on command list counter value")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableImplicitConvertionToCounterBasedEvents, -1, "-1: default, 0: Disable, 1: Enable. If enabled, try to convert Regular Events used on Immediate CL to CounterBased")
-DECLARE_DEBUG_VARIABLE(int64_t, OverrideEventSynchronizeTimeout, -1, "-1: default - user provided timeout value,  >0: timeout in nanoseconds")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceTlbFlush, -1, "-1: default,  0: Tlb flush disabled, 1: Tlb Flush enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, DebugSetMemoryDiagnosticsDelay, -1, "-1: default, >=0: delay time in minutes necessary for completion of Memory diagnostics")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableDeviceStateVerification, -1, "-1: default, 0: disable, 1: enable check of device state before submit on Windows")
@@ -415,6 +417,7 @@ DECLARE_DEBUG_VARIABLE(bool, EngineInstancedSubDevices, false, "Create subdevice
 DECLARE_DEBUG_VARIABLE(bool, AllowSingleTileEngineInstancedSubDevices, false, "Create subdevices assigned to specific engine on single tile config")
 DECLARE_DEBUG_VARIABLE(bool, EnablePrivateBO, false, "Enable PRELIM_I915_GEM_CREATE_EXT_VM_PRIVATE extension creating VM_PRIVATE BOs")
 DECLARE_DEBUG_VARIABLE(bool, UseDeprecatedClDeviceIpVersion, false, "When enabled, the deprecated ip version scheme distinguishing between families and integrated devices will be queried in OCL")
+DECLARE_DEBUG_VARIABLE(bool, EnableAIL, true, "Enables AIL")
 DECLARE_DEBUG_VARIABLE(int32_t, ForceRunAloneContext, -1, "Control creation of run-alone HW context, -1:default, 0:disable, 1:enable")
 DECLARE_DEBUG_VARIABLE(int32_t, AddClGlSharing, -1, "Add cl-gl extension")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableKernelTunning, -1, "Perform a tunning of enqueue kernel, -1:default(disabled), 0:disable, 1:enable simple kernel tunning, 2:enable full kernel tunning")
@@ -456,6 +459,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, UseBindlessMode, -1, "Use precompiled builtins i
 DECLARE_DEBUG_VARIABLE(int32_t, OverrideSlmSize, -1, "Force different slm size than default in kB")
 DECLARE_DEBUG_VARIABLE(int32_t, UseCyclesPerSecondTimer, 0, "0: default behavior, 0: disabled: Report L0 timer in nanosecond units, 1: enabled: Report L0 timer in cycles per second")
 DECLARE_DEBUG_VARIABLE(int32_t, WaitLoopCount, -1, "-1: use default, >=0: number of iterations in wait loop")
+DECLARE_DEBUG_VARIABLE(int32_t, EnableWaitpkg, -1, "-1: use default, 0: disable, 1: enable")
 DECLARE_DEBUG_VARIABLE(int32_t, GTPinAllocateBufferInSharedMemory, -1, "Force GTPin to allocate buffer in shared memory")
 DECLARE_DEBUG_VARIABLE(int32_t, AlignLocalMemoryVaTo2MB, -1, "Allow 2MB pages for allocations with size>=2MB. On Linux it means aligned VA, on Windows it means aligned size. -1: default, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableUserFenceForCompletionWait, -1, "-1: default (disabled), 0: disable, 1: enable : Use Wait User Fence instead Gem Wait")
@@ -485,7 +489,6 @@ DECLARE_DEBUG_VARIABLE(int32_t, CompactL3FlushEventPacket, -1, "Compact COMPUTE_
 DECLARE_DEBUG_VARIABLE(int32_t, UseDynamicEventPacketsCount, -1, "Use dynamic estimation for event packet count based on a given device configuration, -1: default , 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, SignalAllEventPackets, -1, "All packets of event are signaled, reset and waited/synchronized, -1: default, 0: disabled, 1: enabled")
 DECLARE_DEBUG_VARIABLE(int32_t, EnableBcsSwControlWa, -1, "Enable BCS WA via BCSSWCONTROL MMIO. -1: default, 0: disabled, 1: if src in system mem, 2: if dst in system mem, 3: if src and dst in system mem, 4: always")
-DECLARE_DEBUG_VARIABLE(bool, EnableAIL, true, "Enables AIL")
 
 /* IMPLICIT SCALING */
 DECLARE_DEBUG_VARIABLE(int32_t, EnableWalkerPartition, -1, "-1: default, 0: disable, 1: enable, Enables Walker Partitioning via WPARID.")

shared/source/utilities/wait_util.cpp

@@ -8,14 +8,50 @@
 #include "shared/source/utilities/wait_util.h"
 
 #include "shared/source/debug_settings/debug_settings_manager.h"
+#include "shared/source/utilities/cpu_info.h"
 
 namespace NEO {
 
 namespace WaitUtils {
 
+constexpr uint64_t defaultCounterValue = 10000;
+constexpr uint32_t defaultControlValue = 0;
+constexpr bool defaultEnableWaitPkg = false;
+
+uint64_t counterValue = defaultCounterValue;
+uint32_t controlValue = defaultControlValue;
+
 uint32_t waitCount = defaultWaitCount;
 
+#ifdef SUPPORTS_WAITPKG
+bool waitpkgSupport = SUPPORTS_WAITPKG;
+#else
+bool waitpkgSupport = false;
+#endif
+bool waitpkgUse = false;
+
 void init() {
+    bool enableWaitPkg = defaultEnableWaitPkg;
+    int32_t overrideEnableWaitPkg = debugManager.flags.EnableWaitpkg.get();
+    if (overrideEnableWaitPkg != -1) {
+        enableWaitPkg = !!(overrideEnableWaitPkg);
+    }
+    if (enableWaitPkg && waitpkgSupport) {
+        if (CpuInfo::getInstance().isFeatureSupported(CpuInfo::featureWaitPkg)) {
+            waitpkgUse = true;
+            waitCount = 0;
+        }
+    }
+
+    int64_t overrideWaitPkgCounter = debugManager.flags.WaitpkgCounterValue.get();
+    if (overrideWaitPkgCounter != -1) {
+        counterValue = static_cast<uint64_t>(overrideWaitPkgCounter);
+    }
+    int32_t overrideWaitPkgControl = debugManager.flags.WaitpkgControlValue.get();
+    if (overrideWaitPkgControl != -1) {
+        controlValue = static_cast<uint32_t>(overrideWaitPkgControl);
+    }
+
     int32_t overrideWaitCount = debugManager.flags.WaitLoopCount.get();
     if (overrideWaitCount != -1) {
         waitCount = static_cast<uint32_t>(overrideWaitCount);

shared/source/utilities/wait_util.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2022 Intel Corporation
+ * Copyright (C) 2021-2023 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -18,7 +18,12 @@ namespace NEO {
 namespace WaitUtils {
 
 constexpr uint32_t defaultWaitCount = 1u;
+
+extern uint64_t counterValue;
+extern uint32_t controlValue;
 extern uint32_t waitCount;
+extern bool waitpkgSupport;
+extern bool waitpkgUse;
 
 template <typename T>
 inline bool waitFunctionWithPredicate(volatile T const *pollAddress, T expectedValue, std::function<bool(T, T)> predicate) {

shared/test/common/test_files/igdrcl.config

@@ -362,6 +362,9 @@ HBMSizePerTileInGigabytes = 0
 OverrideSystolicPipelineSelect = -1
 OverrideSystolicInComputeWalker = -1
 SkipFlushingEventsOnGetStatusCalls = 0
+EnableWaitpkg = -1
+WaitpkgControlValue = -1
+WaitpkgCounterValue = -1
 AllowUnrestrictedSize = 0
 ForceDefaultThreadArbitrationPolicyIfNotSpecified = 0
 DoNotFreeResources = 0

shared/test/unit_test/mocks/CMakeLists.txt

@@ -1,12 +1,14 @@
 #
-# Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2021-2023 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
 
 target_sources(neo_shared_tests PRIVATE
                ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
                ${CMAKE_CURRENT_SOURCE_DIR}/mock_aub_stream.h
+               ${CMAKE_CURRENT_SOURCE_DIR}/mock_cpuid_functions.cpp
+               ${CMAKE_CURRENT_SOURCE_DIR}/mock_cpuid_functions.h
                ${CMAKE_CURRENT_SOURCE_DIR}/mock_csr_simulated_common_hw.h
                ${CMAKE_CURRENT_SOURCE_DIR}/mock_direct_submission_diagnostic_collector.h
                ${CMAKE_CURRENT_SOURCE_DIR}/mock_dispatch_kernel_encoder_interface.h

shared/test/unit_test/mocks/mock_cpuid_functions.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/test/unit_test/mocks/mock_cpuid_functions.h"
+
+void mockCpuidEnableAll(int *cpuInfo, int functionId) {
+    cpuInfo[0] = -1;
+    cpuInfo[1] = -1;
+    cpuInfo[2] = -1;
+    cpuInfo[3] = -1;
+}
+
+void mockCpuidFunctionAvailableDisableAll(int *cpuInfo, int functionId) {
+    cpuInfo[0] = -1;
+    cpuInfo[1] = 0;
+    cpuInfo[2] = 0;
+    cpuInfo[3] = 0;
+}
+
+void mockCpuidFunctionNotAvailableDisableAll(int *cpuInfo, int functionId) {
+    cpuInfo[0] = 0;
+    cpuInfo[1] = 0;
+    cpuInfo[2] = 0;
+    cpuInfo[3] = 0;
+}
+
+void mockCpuidReport36BitVirtualAddressSize(int *cpuInfo, int functionId) {
+    if (static_cast<unsigned int>(functionId) == 0x80000008) {
+        cpuInfo[0] = 36 << 8;
+        cpuInfo[1] = 0;
+        cpuInfo[2] = 0;
+        cpuInfo[3] = 0;
+    } else {
+        mockCpuidEnableAll(cpuInfo, functionId);
+    }
+}

shared/test/unit_test/mocks/mock_cpuid_functions.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#pragma once
+
+void mockCpuidEnableAll(int *cpuInfo, int functionId);
+
+void mockCpuidFunctionAvailableDisableAll(int *cpuInfo, int functionId);
+
+void mockCpuidFunctionNotAvailableDisableAll(int *cpuInfo, int functionId);
+
+void mockCpuidReport36BitVirtualAddressSize(int *cpuInfo, int functionId);

shared/test/unit_test/utilities/x86_64/CMakeLists.txt

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2021-2022 Intel Corporation
+# Copyright (C) 2021-2023 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -8,6 +8,7 @@ if(${NEO_TARGET_PROCESSOR} STREQUAL "x86_64")
   target_sources(neo_shared_tests PRIVATE
                  ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
                  ${CMAKE_CURRENT_SOURCE_DIR}/cpuinfo_tests_x86_64.cpp
+                 ${CMAKE_CURRENT_SOURCE_DIR}/wait_util_tests_x86_64.cpp
   )
 
   add_subdirectories()

shared/test/unit_test/utilities/x86_64/cpuinfo_tests_x86_64.cpp

@@ -6,43 +6,12 @@
  */
 
 #include "shared/source/utilities/cpu_info.h"
+#include "shared/test/unit_test/mocks/mock_cpuid_functions.h"
 
 #include "gtest/gtest.h"
 
 using namespace NEO;
 
-void mockCpuidEnableAll(int *cpuInfo, int functionId) {
-    cpuInfo[0] = -1;
-    cpuInfo[1] = -1;
-    cpuInfo[2] = -1;
-    cpuInfo[3] = -1;
-}
-
-void mockCpuidFunctionAvailableDisableAll(int *cpuInfo, int functionId) {
-    cpuInfo[0] = -1;
-    cpuInfo[1] = 0;
-    cpuInfo[2] = 0;
-    cpuInfo[3] = 0;
-}
-
-void mockCpuidFunctionNotAvailableDisableAll(int *cpuInfo, int functionId) {
-    cpuInfo[0] = 0;
-    cpuInfo[1] = 0;
-    cpuInfo[2] = 0;
-    cpuInfo[3] = 0;
-}
-
-void mockCpuidReport36BitVirtualAddressSize(int *cpuInfo, int functionId) {
-    if (static_cast<uint32_t>(functionId) == 0x80000008) {
-        cpuInfo[0] = 36 << 8;
-        cpuInfo[1] = 0;
-        cpuInfo[2] = 0;
-        cpuInfo[3] = 0;
-    } else {
-        mockCpuidEnableAll(cpuInfo, functionId);
-    }
-}
-
 TEST(CpuInfoTest, giveFunctionIsNotAvailableWhenFeatureIsNotSupportedThenMaskBitIsOff) {
     void (*defaultCpuidFunc)(int *, int) = CpuInfo::cpuidFunc;
     CpuInfo::cpuidFunc = mockCpuidFunctionNotAvailableDisableAll;