Skip to content

Commit dfb935f

Browse files
BartoszDunajskiCompute-Runtime-Automation
BartoszDunajski
authored and
Compute-Runtime-Automation
committed
Engine instanced SubDevices [1/n]
Signed-off-by: Bartosz Dunajski <[email protected]>
1 parent 576ac84 commit dfb935f

File tree

8 files changed

+169
-19
lines changed

8 files changed

+169
-19
lines changed

‎opencl/test/unit_test/device/sub_device_tests.cpp‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77

88
#include "shared/source/device/sub_device.h"
9+
#include "shared/source/os_interface/device_factory.h"
910
#include "shared/source/os_interface/os_context.h"
1011
#include "shared/test/common/helpers/debug_manager_state_restore.h"
1112
#include "shared/test/common/helpers/ult_hw_config.h"
@@ -265,6 +266,91 @@ TEST(SubDevicesTest, givenRootDeviceWithSubDevicesWhenGettingGlobalMemorySizeThe
265266
}
266267
}
267268

269+
TEST(SubDevicesTest, whenCreatingEngineInstancedSubDeviceThenSetCorrectSubdeviceIndex) {
270+
class MyRootDevice : public RootDevice {
271+
public:
272+
using RootDevice::createEngineInstancedSubDevice;
273+
using RootDevice::RootDevice;
274+
};
275+
276+
auto executionEnvironment = new ExecutionEnvironment();
277+
executionEnvironment->prepareRootDeviceEnvironments(1);
278+
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
279+
DeviceFactory::createMemoryManagerFunc(*executionEnvironment);
280+
281+
auto rootDevice = std::unique_ptr<MyRootDevice>(Device::create<MyRootDevice>(executionEnvironment, 0));
282+
283+
auto subDevice = std::unique_ptr<SubDevice>(rootDevice->createEngineInstancedSubDevice(1, defaultHwInfo->capabilityTable.defaultEngineType));
284+
285+
ASSERT_NE(nullptr, subDevice.get());
286+
287+
EXPECT_EQ(2u, subDevice->getDeviceBitfield().to_ulong());
288+
}
289+
290+
TEST(SubDevicesTest, givenDebugFlagSetAndMoreThanOneCcsWhenCreatingRootDeviceWithoutGenericSubDevicesThenCreateEngineInstanced) {
291+
DebugManagerStateRestore restorer;
292+
DebugManager.flags.EngineInstancedSubDevices.set(true);
293+
294+
auto executionEnvironment = new ExecutionEnvironment();
295+
executionEnvironment->prepareRootDeviceEnvironments(1);
296+
297+
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
298+
executionEnvironment->rootDeviceEnvironments[0]->getMutableHardwareInfo()->gtSystemInfo.CCSInfo.NumberOfCCSEnabled = 2;
299+
300+
UltDeviceFactory deviceFactory(1, 1, *executionEnvironment);
301+
302+
auto &rootDevice = deviceFactory.rootDevices[0];
303+
auto &hwInfo = rootDevice->getHardwareInfo();
304+
uint32_t ccsCount = hwInfo.gtSystemInfo.CCSInfo.NumberOfCCSEnabled;
305+
306+
EXPECT_EQ(ccsCount, rootDevice->getNumAvailableDevices());
307+
308+
for (uint32_t i = 0; i < ccsCount; i++) {
309+
auto engineType = static_cast<aub_stream::EngineType>(aub_stream::EngineType::ENGINE_CCS + i);
310+
auto subDevice = static_cast<MockSubDevice *>(rootDevice->getDeviceById(i));
311+
ASSERT_NE(nullptr, subDevice);
312+
313+
EXPECT_TRUE(subDevice->engineInstanced);
314+
EXPECT_EQ(engineType, subDevice->engineType);
315+
}
316+
}
317+
318+
TEST(SubDevicesTest, givenDebugFlagSetAndSingleCcsWhenCreatingRootDeviceWithoutGenericSubDevicesThenCreateEngineInstanced) {
319+
DebugManagerStateRestore restorer;
320+
DebugManager.flags.EngineInstancedSubDevices.set(true);
321+
322+
auto executionEnvironment = new ExecutionEnvironment();
323+
executionEnvironment->prepareRootDeviceEnvironments(1);
324+
325+
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(defaultHwInfo.get());
326+
executionEnvironment->rootDeviceEnvironments[0]->getMutableHardwareInfo()->gtSystemInfo.CCSInfo.NumberOfCCSEnabled = 1;
327+
328+
UltDeviceFactory deviceFactory(1, 1, *executionEnvironment);
329+
330+
auto &rootDevice = deviceFactory.rootDevices[0];
331+
332+
EXPECT_EQ(1u, rootDevice->getNumAvailableDevices());
333+
EXPECT_FALSE(rootDevice->getDeviceById(0)->isSubDevice());
334+
}
335+
336+
TEST(SubDevicesTest, givenDebugFlagSetWhenCreatingRootDeviceWithGenericSubDevicesThenDontCreateEngineInstanced) {
337+
DebugManagerStateRestore restorer;
338+
DebugManager.flags.EngineInstancedSubDevices.set(true);
339+
340+
UltDeviceFactory deviceFactory(1, 2);
341+
342+
auto &rootDevice = deviceFactory.rootDevices[0];
343+
344+
for (uint32_t i = 0; i < 2; i++) {
345+
auto subDevice = static_cast<MockSubDevice *>(rootDevice->getDeviceById(i));
346+
ASSERT_NE(nullptr, subDevice);
347+
348+
EXPECT_FALSE(subDevice->engineInstanced);
349+
EXPECT_EQ(1u, subDevice->getNumAvailableDevices());
350+
EXPECT_EQ(aub_stream::EngineType::NUM_ENGINES, subDevice->engineType);
351+
}
352+
}
353+
268354
TEST(SubDevicesTest, whenInitializeRootCsrThenDirectSubmissionIsNotInitialized) {
269355
auto device = std::make_unique<MockDevice>();
270356
device->initializeRootCommandStreamReceiver();

‎opencl/test/unit_test/test_files/igdrcl.config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,3 +238,4 @@ DebuggerLogBitmask = 0
238238
GTPinAllocateBufferInSharedMemory = -1
239239
DeferOsContextInitialization = -1
240240
DebuggerOptDisable = -1
241+
EngineInstancedSubDevices = 0

‎shared/source/debug_settings/debug_variables_base.inl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ DECLARE_DEBUG_VARIABLE(bool, AddClGlSharing, false, "Add cl-gl extension")
187187
DECLARE_DEBUG_VARIABLE(bool, EnableFormatQuery, false, "Enable sharing format querying")
188188
DECLARE_DEBUG_VARIABLE(bool, EnableFreeMemory, false, "Enable freeMemory in memory manager")
189189
DECLARE_DEBUG_VARIABLE(bool, ForceSamplerLowFilteringPrecision, false, "Force Low Filtering Precision Sampler mode")
190+
DECLARE_DEBUG_VARIABLE(bool, EngineInstancedSubDevices, false, "Create subdevices assigned to specific engine")
190191
DECLARE_DEBUG_VARIABLE(int32_t, EnableKernelTunning, -1, "Perform a tunning of enqueue kernel, -1:default(disabled), 0:disable, 1:enable simple kernel tunning, 2:enable full kernel tunning")
191192
DECLARE_DEBUG_VARIABLE(int32_t, EnableBOMmapCreate, -1, "Create BOs using mmap, -1:default, 0:disable(GEM_USERPTR), 1:enable")
192193
DECLARE_DEBUG_VARIABLE(int32_t, EnableGemCloseWorker, -1, "Use asynchronous gem object closing, -1:default, 0:disable, 1:enable")

‎shared/source/device/device.cpp‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,11 @@ SubDevice *Device::createSubDevice(uint32_t subDeviceIndex) {
6161
return Device::create<SubDevice>(executionEnvironment, subDeviceIndex, *getRootDevice());
6262
}
6363

64-
bool Device::createSubDevices() {
65-
if (!subDevicesAllowed()) {
66-
return true;
67-
}
64+
SubDevice *Device::createEngineInstancedSubDevice(uint32_t subDeviceIndex, aub_stream::EngineType engineType) {
65+
return Device::create<SubDevice>(executionEnvironment, subDeviceIndex, *getRootDevice(), engineType);
66+
}
6867

68+
bool Device::genericSubDevicesAllowed() {
6969
auto deviceMask = executionEnvironment->rootDeviceEnvironments[getRootDeviceIndex()]->deviceAffinityMask;
7070
uint32_t subDeviceCount = HwHelper::getSubDevicesCount(&getHardwareInfo());
7171
deviceBitfield = maxNBitValue(subDeviceCount);
@@ -74,19 +74,60 @@ bool Device::createSubDevices() {
7474
if (numSubDevices == 1) {
7575
numSubDevices = 0;
7676
}
77+
78+
return (numSubDevices > 0);
79+
}
80+
81+
bool Device::engineInstancedSubDevicesAllowed() const {
82+
return (DebugManager.flags.EngineInstancedSubDevices.get() &&
83+
(getHardwareInfo().gtSystemInfo.CCSInfo.NumberOfCCSEnabled > 1));
84+
}
85+
86+
bool Device::createEngineInstancedSubDevices() {
87+
UNRECOVERABLE_IF(deviceBitfield.count() != 1);
88+
UNRECOVERABLE_IF(!subdevices.empty());
89+
90+
numSubDevices = getHardwareInfo().gtSystemInfo.CCSInfo.NumberOfCCSEnabled;
91+
subdevices.resize(numSubDevices, nullptr);
92+
uint32_t subDeviceIndex = Math::log2(static_cast<uint32_t>(deviceBitfield.to_ulong()));
93+
94+
for (uint32_t i = 0; i < numSubDevices; i++) {
95+
auto engineType = static_cast<aub_stream::EngineType>(aub_stream::EngineType::ENGINE_CCS + i);
96+
auto subDevice = createEngineInstancedSubDevice(subDeviceIndex, engineType);
97+
UNRECOVERABLE_IF(!subDevice);
98+
subdevices[i] = subDevice;
99+
}
100+
101+
return true;
102+
}
103+
104+
bool Device::createGenericSubDevices() {
77105
UNRECOVERABLE_IF(!subdevices.empty());
78-
if (numSubDevices) {
79-
subdevices.resize(subDeviceCount, nullptr);
80-
for (auto i = 0u; i < subDeviceCount; i++) {
81-
if (!deviceBitfield.test(i)) {
82-
continue;
83-
}
84-
auto subDevice = createSubDevice(i);
85-
if (!subDevice) {
86-
return false;
87-
}
88-
subdevices[i] = subDevice;
106+
uint32_t subDeviceCount = HwHelper::getSubDevicesCount(&getHardwareInfo());
107+
108+
subdevices.resize(subDeviceCount, nullptr);
109+
110+
for (auto i = 0u; i < subDeviceCount; i++) {
111+
if (!deviceBitfield.test(i)) {
112+
continue;
113+
}
114+
auto subDevice = createSubDevice(i);
115+
if (!subDevice) {
116+
return false;
89117
}
118+
subdevices[i] = subDevice;
119+
}
120+
121+
return true;
122+
}
123+
124+
bool Device::createSubDevices() {
125+
if (genericSubDevicesAllowed()) {
126+
return createGenericSubDevices();
127+
}
128+
129+
if (engineInstancedSubDevicesAllowed()) {
130+
return createEngineInstancedSubDevices();
90131
}
91132

92133
return true;

‎shared/source/device/device.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,14 @@ class Device : public ReferenceTrackedObject<Device> {
132132
bool createEngine(uint32_t deviceCsrIndex, EngineTypeUsage engineTypeUsage);
133133
MOCKABLE_VIRTUAL std::unique_ptr<CommandStreamReceiver> createCommandStreamReceiver() const;
134134
MOCKABLE_VIRTUAL SubDevice *createSubDevice(uint32_t subDeviceIndex);
135+
MOCKABLE_VIRTUAL SubDevice *createEngineInstancedSubDevice(uint32_t subDeviceIndex, aub_stream::EngineType engineType);
135136
virtual uint64_t getGlobalMemorySize(uint32_t deviceBitfield) const;
136137
virtual void createBindlessHeapsHelper() {}
137138
bool createSubDevices();
138-
virtual bool subDevicesAllowed() const { return true; };
139+
bool createGenericSubDevices();
140+
bool createEngineInstancedSubDevices();
141+
virtual bool genericSubDevicesAllowed();
142+
virtual bool engineInstancedSubDevicesAllowed() const;
139143

140144
DeviceInfo deviceInfo = {};
141145

‎shared/source/device/sub_device.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,18 @@
1212
namespace NEO {
1313

1414
SubDevice::SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t subDeviceIndex, Device &rootDevice)
15-
: Device(executionEnvironment), subDeviceIndex(subDeviceIndex), rootDevice(static_cast<RootDevice &>(rootDevice)) {
15+
: Device(executionEnvironment), rootDevice(static_cast<RootDevice &>(rootDevice)), subDeviceIndex(subDeviceIndex) {
1616
UNRECOVERABLE_IF(rootDevice.isSubDevice());
1717
deviceBitfield = 0;
1818
deviceBitfield.set(subDeviceIndex);
1919
}
2020

21+
SubDevice::SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t subDeviceIndex, Device &rootDevice, aub_stream::EngineType engineType)
22+
: SubDevice(executionEnvironment, subDeviceIndex, rootDevice) {
23+
this->engineType = engineType;
24+
engineInstanced = true;
25+
}
26+
2127
void SubDevice::incRefInternal() {
2228
rootDevice.incRefInternal();
2329
}

‎shared/source/device/sub_device.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ class RootDevice;
1313
class SubDevice : public Device {
1414
public:
1515
SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t subDeviceIndex, Device &rootDevice);
16+
SubDevice(ExecutionEnvironment *executionEnvironment, uint32_t subDeviceIndex, Device &rootDevice, aub_stream::EngineType engineType);
1617
void incRefInternal() override;
1718
unique_ptr_if_unused<Device> decRefInternal() override;
1819

@@ -25,8 +26,12 @@ class SubDevice : public Device {
2526

2627
protected:
2728
uint64_t getGlobalMemorySize(uint32_t deviceBitfield) const override;
28-
bool subDevicesAllowed() const override { return false; };
29-
const uint32_t subDeviceIndex;
29+
bool genericSubDevicesAllowed() override { return false; };
30+
bool engineInstancedSubDevicesAllowed() const override { return false; };
31+
3032
RootDevice &rootDevice;
33+
const uint32_t subDeviceIndex;
34+
aub_stream::EngineType engineType = aub_stream::EngineType::NUM_ENGINES;
35+
bool engineInstanced = false;
3136
};
3237
} // namespace NEO

‎shared/test/common/mocks/mock_device.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ extern CommandStreamReceiver *createCommandStream(ExecutionEnvironment &executio
2828
const DeviceBitfield deviceBitfield);
2929

3030
struct MockSubDevice : public SubDevice {
31+
using SubDevice::engineInstanced;
32+
using SubDevice::engineType;
3133
using SubDevice::getDeviceBitfield;
3234
using SubDevice::getGlobalMemorySize;
3335
using SubDevice::SubDevice;
@@ -124,6 +126,10 @@ class MockDevice : public RootDevice {
124126
return Device::create<MockSubDevice>(executionEnvironment, subDeviceIndex, *this);
125127
}
126128

129+
SubDevice *createEngineInstancedSubDevice(uint32_t subDeviceIndex, aub_stream::EngineType engineType) override {
130+
return Device::create<MockSubDevice>(executionEnvironment, subDeviceIndex, *this, engineType);
131+
}
132+
127133
std::unique_ptr<CommandStreamReceiver> createCommandStreamReceiver() const override {
128134
return std::unique_ptr<CommandStreamReceiver>(createCommandStreamReceiverFunc(*executionEnvironment, getRootDeviceIndex(), getDeviceBitfield()));
129135
}

0 commit comments

Comments
 (0)