Sysman: Add support to check if resizable Bar supported

Signed-off-by: Jitendra Sharma <[email protected]>

Author: jitendrasharma1989

level_zero/tools/source/sysman/linux/fs_access.cpp

@@ -577,4 +577,8 @@ bool SysfsAccess::isMyDeviceFile(const std::string dev) {
     return false;
 }
 
+bool SysfsAccess::isRootUser() {
+    return FsAccess::isRootUser();
+}
+
 } // namespace L0

level_zero/tools/source/sysman/linux/fs_access.h

@@ -111,6 +111,7 @@ class SysfsAccess : protected FsAccess {
     MOCKABLE_VIRTUAL bool fileExists(const std::string file) override;
     MOCKABLE_VIRTUAL bool isMyDeviceFile(const std::string dev);
     MOCKABLE_VIRTUAL bool directoryExists(const std::string path) override;
+    MOCKABLE_VIRTUAL bool isRootUser() override;
 
   private:
     SysfsAccess(const std::string file);

level_zero/tools/source/sysman/pci/linux/os_pci_imp.cpp

@@ -12,6 +12,8 @@
 
 #include "sysman/pci/pci_imp.h"
 
+#include <linux/pci_regs.h>
+
 namespace L0 {
 
 const std::string LinuxPciImp::deviceDir("device");
@@ -150,7 +152,34 @@ ze_result_t LinuxPciImp::initializeBarProperties(std::vector<zes_pci_bar_propert
     return result;
 }
 
+// Parse PCIe configuration space to see if resizable Bar is supported
 bool LinuxPciImp::resizableBarSupported() {
+    uint32_t pos = PCI_CFG_SPACE_SIZE;
+    uint32_t header = 0;
+
+    if (!configMemory) {
+        return false;
+    }
+
+    // Minimum 8 bytes per capability. Hence maximum capabilities that
+    // could be present in PCI extended configuration space are
+    // represented by loopCount.
+    auto loopCount = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
+    header = getDwordFromConfig(pos);
+    if (!header) {
+        return false;
+    }
+
+    while (loopCount-- > 0) {
+        if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_REBAR) {
+            return true;
+        }
+        pos = PCI_EXT_CAP_NEXT(header);
+        if (pos < PCI_CFG_SPACE_SIZE) {
+            return false;
+        }
+        header = getDwordFromConfig(pos);
+    }
     return false;
 }
 
@@ -161,12 +190,30 @@ bool LinuxPciImp::resizableBarEnabled() {
 ze_result_t LinuxPciImp::getState(zes_pci_state_t *state) {
     return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
+
+void LinuxPciImp::pciExtendedConfigRead() {
+    std::string pciConfigNode;
+    pSysfsAccess->getRealPath("device/config", pciConfigNode);
+    int fdConfig = -1;
+    fdConfig = this->openFunction(pciConfigNode.c_str(), O_RDONLY);
+    if (fdConfig < 0) {
+        return;
+    }
+    configMemory = std::make_unique<uint8_t[]>(PCI_CFG_SPACE_EXP_SIZE);
+    memset(configMemory.get(), 0, PCI_CFG_SPACE_EXP_SIZE);
+    this->preadFunction(fdConfig, configMemory.get(), PCI_CFG_SPACE_EXP_SIZE, 0);
+    this->closeFunction(fdConfig);
+}
+
 LinuxPciImp::LinuxPciImp(OsSysman *pOsSysman) {
     pLinuxSysmanImp = static_cast<LinuxSysmanImp *>(pOsSysman);
     pSysfsAccess = &pLinuxSysmanImp->getSysfsAccess();
     pfsAccess = &pLinuxSysmanImp->getFsAccess();
     Device *pDevice = pLinuxSysmanImp->getDeviceHandle();
     isLmemSupported = pDevice->getDriverHandle()->getMemoryManager()->isLocalMemorySupported(pDevice->getRootDeviceIndex());
+    if (pSysfsAccess->isRootUser()) {
+        pciExtendedConfigRead();
+    }
 }
 
 OsPci *OsPci::create(OsSysman *pOsSysman) {

level_zero/tools/source/sysman/pci/linux/os_pci_imp.h

@@ -7,6 +7,7 @@
 
 #pragma once
 #include "shared/source/helpers/non_copyable_or_moveable.h"
+#include "shared/source/os_interface/linux/sys_calls.h"
 
 #include "sysman/linux/os_sysman_imp.h"
 #include "sysman/pci/os_pci.h"
@@ -34,13 +35,22 @@ class LinuxPciImp : public OsPci, NEO::NonCopyableOrMovableClass {
     SysfsAccess *pSysfsAccess = nullptr;
     FsAccess *pfsAccess = nullptr;
     LinuxSysmanImp *pLinuxSysmanImp = nullptr;
+    std::unique_ptr<uint8_t[]> configMemory;
+    void pciExtendedConfigRead();
+    decltype(&NEO::SysCalls::open) openFunction = NEO::SysCalls::open;
+    decltype(&NEO::SysCalls::close) closeFunction = NEO::SysCalls::close;
+    decltype(&NEO::SysCalls::pread) preadFunction = NEO::SysCalls::pread;
 
   private:
     static const std::string deviceDir;
     static const std::string resourceFile;
     static const std::string maxLinkSpeedFile;
     static const std::string maxLinkWidthFile;
     bool isLmemSupported = false;
+    uint32_t getDwordFromConfig(uint32_t pos) {
+        return configMemory[pos] | (configMemory[pos + 1] << 8) |
+               (configMemory[pos + 2] << 16) | (configMemory[pos + 3] << 24);
+    }
 };
 
 } // namespace L0

level_zero/tools/source/sysman/pci/pci_imp.cpp

@@ -106,8 +106,8 @@ ze_result_t PciImp::pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properti
                 pBarPropsExt->index = pciBarProperties[i]->index;
                 pBarPropsExt->size = pciBarProperties[i]->size;
                 pBarPropsExt->type = pciBarProperties[i]->type;
-                pBarPropsExt->resizableBarEnabled = resizableBarEnabled;
-                pBarPropsExt->resizableBarSupported = resizableBarSupported;
+                pBarPropsExt->resizableBarEnabled = static_cast<ze_bool_t>(resizableBarEnabled);
+                pBarPropsExt->resizableBarSupported = static_cast<ze_bool_t>(resizableBarSupported);
             }
         }
     }
@@ -117,11 +117,8 @@ ze_result_t PciImp::pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properti
 ze_result_t PciImp::pciGetState(zes_pci_state_t *pState) {
     return pOsPci->getState(pState);
 }
-void PciImp::init() {
-    if (pOsPci == nullptr) {
-        pOsPci = OsPci::create(pOsSysman);
-    }
-    UNRECOVERABLE_IF(nullptr == pOsPci);
+
+void PciImp::pciGetStaticFields() {
     pOsPci->getProperties(&pciProperties);
     resizableBarEnabled = pOsPci->resizableBarEnabled();
     resizableBarSupported = pOsPci->resizableBarSupported();
@@ -154,6 +151,15 @@ void PciImp::init() {
     pOsPci->initializeBarProperties(pciBarProperties);
 }
 
+void PciImp::init() {
+    if (pOsPci == nullptr) {
+        pOsPci = OsPci::create(pOsSysman);
+    }
+    UNRECOVERABLE_IF(nullptr == pOsPci);
+
+    pciGetStaticFields();
+}
+
 PciImp::~PciImp() {
     for (zes_pci_bar_properties_t *pProperties : pciBarProperties) {
         delete pProperties;

level_zero/tools/source/sysman/pci/pci_imp.h

@@ -25,6 +25,7 @@ class PciImp : public Pci, NEO::NonCopyableOrMovableClass {
     ze_result_t pciStaticProperties(zes_pci_properties_t *pProperties) override;
     ze_result_t pciGetInitializedBars(uint32_t *pCount, zes_pci_bar_properties_t *pProperties) override;
     ze_result_t pciGetState(zes_pci_state_t *pState) override;
+    void pciGetStaticFields();
 
     PciImp() = default;
     PciImp(OsSysman *pOsSysman) : pOsSysman(pOsSysman){};

level_zero/tools/test/black_box_tests/zello_sysman.cpp

@@ -302,13 +302,21 @@ void testSysmanPci(ze_device_handle_t &device) {
         std::cout << "Bar count = " << count << std::endl;
     }
     std::vector<zes_pci_bar_properties_t> pciBarProps(count);
+    zes_pci_bar_properties_1_2_t props1_2 = {};
+    memset(&props1_2, 0, sizeof(zes_pci_bar_properties_1_2_t));
+    for (uint32_t i = 0; i < count; i++) {
+        pciBarProps[i].pNext = static_cast<void *>(&props1_2);
+        pciBarProps[i].stype = ZES_STRUCTURE_TYPE_PCI_BAR_PROPERTIES_1_2;
+    }
     VALIDATECALL(zesDevicePciGetBars(device, &count, pciBarProps.data()));
     if (verbose) {
         for (uint32_t i = 0; i < count; i++) {
             std::cout << "pciBarProps.type = " << std::hex << pciBarProps[i].type << std::endl;
             std::cout << "pciBarProps.index = " << std::hex << pciBarProps[i].index << std::endl;
             std::cout << "pciBarProps.base = " << std::hex << pciBarProps[i].base << std::endl;
             std::cout << "pciBarProps.size = " << std::hex << pciBarProps[i].size << std::endl;
+            std::cout << "pci_bar_properties_1_2_t.resizableBarSupported = " << static_cast<uint32_t>(props1_2.resizableBarSupported) << std::endl;
+            std::cout << "pci_bar_properties_1_2_t.resizableBarEnabled = " << static_cast<uint32_t>(props1_2.resizableBarEnabled) << std::endl;
         }
     }
 }

level_zero/tools/test/unit_tests/sources/sysman/pci/linux/mock_sysfs_pci.h

@@ -21,6 +21,7 @@ const std::string maxLinkSpeedFile("device/max_link_speed");
 const std::string maxLinkWidthFile("device/max_link_width");
 const std::string mockBdf = "0000:00:02.0";
 const std::string mockRealPath = "/sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0/0000:02:01.0/" + mockBdf;
+const std::string mockRealPathConfig = mockRealPath + "/config";
 const std::string mockRealPath2LevelsUp = "/sys/devices/pci0000:00/0000:00:01.0/0000:01:00.0";
 
 constexpr double mockMaxLinkSpeed = 2.5;
@@ -83,6 +84,11 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
     MOCK_METHOD(ze_result_t, read, (const std::string file, std::vector<std::string> &val), (override));
     MOCK_METHOD(ze_result_t, readSymLink, (const std::string file, std::string &buf), (override));
     MOCK_METHOD(ze_result_t, getRealPath, (const std::string file, std::string &buf), (override));
+    MOCK_METHOD(bool, isRootUser, (), (override));
+
+    bool checkRootUser() {
+        return true;
+    }
 
     ze_result_t getValDouble(const std::string file, double &val) {
         if (file.compare(maxLinkSpeedFile) == 0) {
@@ -120,6 +126,10 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
             val = mockRealPath;
             return ZE_RESULT_SUCCESS;
         }
+        if (file.compare("device/config") == 0) {
+            val = mockRealPathConfig;
+            return ZE_RESULT_SUCCESS;
+        }
         return ZE_RESULT_ERROR_NOT_AVAILABLE;
     }
 
@@ -136,7 +146,13 @@ struct Mock<PciSysfsAccess> : public PciSysfsAccess {
 
 class PublicLinuxPciImp : public L0::LinuxPciImp {
   public:
+    PublicLinuxPciImp(OsSysman *pOsSysman) : LinuxPciImp(pOsSysman) {}
+    using LinuxPciImp::closeFunction;
+    using LinuxPciImp::configMemory;
+    using LinuxPciImp::openFunction;
+    using LinuxPciImp::pciExtendedConfigRead;
     using LinuxPciImp::pfsAccess;
+    using LinuxPciImp::preadFunction;
     using LinuxPciImp::pSysfsAccess;
 };