feat: AMD hardware API

docs/docs/architecture/cortex-db.mdx

@@ -14,6 +14,13 @@ import TabItem from "@theme/TabItem";
 This document outlines the architecture of the database designed to store and manage various types of entities and their associated metadata.
 
 ## Table Structure
+### schema Table
+The `schema` table is designed to hold schema version for cortex database. Below is the structure of the table:
+
+| Column Name        | Data Type | Description                                             |
+|--------------------|-----------|---------------------------------------------------------|
+| schema_version     | INTEGER   | A unique schema version for database.                   |
+
 ### models Table
 The `models` table is designed to hold metadata about various AI models. Below is the structure of the table:
 
@@ -22,4 +29,23 @@ The `models` table is designed to hold metadata about various AI models. Below i
 | model_id           | TEXT      | A unique identifier for each model (Primary Key).      |
 | author_repo_id     | TEXT      | The identifier for the repository where the model is stored. |
 | branch_name        | TEXT      | The branch name in the repository that contains the model. |
-| path_to_model_yaml | TEXT      | The file path to the YAML configuration file for the model. |
+| path_to_model_yaml | TEXT      | The file path to the YAML configuration file for the model. |
+| model_alias        | TEXT      | The optional alias or friendly name for the model. |
+| model_format       | TEXT      | The format or type of the model (e.g., TensorFlow, PyTorch, GGUF, etc.). |
+| model_source       | TEXT      | The source or origin of the model (e.g., a URL or file path). |
+| status             | TEXT      | Current status of the model (e.g., "downloaded", "downloadable").. |
+| engine             | TEXT      | The name or identifier of the engine or framework used for running or deploying the model.. |
+| metadata           | TEXT      | Additional metadata or information about the model, such as a JSON string containing various attributes or properties. |
+
+### hardware Table
+The `hardware` table is designed to hold metadata about hardware information. Below is the structure of the table:
+
+| Column Name        | Data Type | Description                                             |
+|--------------------|-----------|---------------------------------------------------------|
+| uuid               | TEXT      | the primary key for the table, meaning that each row must have a unique value in this column.      |
+| type               | TEXT      | The type of hardware. |
+| hardware_id        | INTEGER   | An integer value representing the hardware ID. |
+| software_id        | INTEGER   | An integer value representing the software ID associated with the hardware. |
+| activated          | INTEGER   | A boolean value (0 or 1) indicating whether the hardware is activated or not. |
+| priority           | INTEGER   | An integer value representing the priority associated with the hardware. |
+

engine/.gitignore

@@ -32,6 +32,7 @@
 *.so
 *.so.*
 *.dylib
+!**/libvulkan.so
 
 # Executables
 *.exe

engine/CMakeLists.txt

@@ -68,6 +68,8 @@ endif()
 
 add_subdirectory(cli)
 
+
+
 find_package(jsoncpp CONFIG REQUIRED)
 find_package(Drogon CONFIG REQUIRED)
 find_package(yaml-cpp CONFIG REQUIRED)

engine/cli/CMakeLists.txt

@@ -136,3 +136,9 @@ set_target_properties(${TARGET_NAME} PROPERTIES
                       RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}
                       RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
 )
+
+if(UNIX AND NOT APPLE)
+configure_file("${PROJECT_SOURCE_DIR}/../deps/linux/libvulkan.so" "${CMAKE_BINARY_DIR}/libvulkan.so" COPYONLY)
+elseif(MSVC)
+configure_file("${PROJECT_SOURCE_DIR}/../deps/windows/vulkan-1.dll" "${CMAKE_BINARY_DIR}/vulkan-1.dll" COPYONLY)
+endif()

engine/cli/commands/hardware_list_cmd.cc

@@ -130,6 +130,7 @@ bool HardwareListCmd::Exec(const std::string& host, int port,
           std::get<cortex::hw::NvidiaAddInfo>(gpu.add_info).compute_cap);
       row.emplace_back(gpu.is_activated ? "Yes" : "No");
       table.add_row({row.begin(), row.end()});
+      count++;
     }
 
     std::cout << table << std::endl;

engine/common/hardware_common.h

@@ -64,6 +64,7 @@ struct AmdAddInfo {};
 using GPUAddInfo = std::variant<NvidiaAddInfo, AmdAddInfo>;
 struct GPU {
   std::string id;
+  uint32_t device_id;
   std::string name;
   std::string version;
   GPUAddInfo add_info;
@@ -77,7 +78,7 @@ inline Json::Value ToJson(const std::vector<GPU>& gpus) {
   Json::Value res(Json::arrayValue);
   for (size_t i = 0; i < gpus.size(); i++) {
     Json::Value gpu;
-    gpu["id"] = std::to_string(i);
+    gpu["id"] = gpus[i].id;
     gpu["name"] = gpus[i].name;
     gpu["version"] = gpus[i].version;
     Json::Value add_info;

engine/database/hardware.cc

@@ -98,4 +98,41 @@ cpp::result<bool, std::string> Hardware::DeleteHardwareEntry(
     return cpp::fail(e.what());
   }
 }
+
+bool Hardware::HasHardwareEntry(const std::string& id) {
+   try {
+    SQLite::Statement query(db_,
+                            "SELECT COUNT(*) FROM hardware WHERE uuid = ?");
+    query.bind(1, id);
+    if (query.executeStep()) {
+      return query.getColumn(0).getInt() > 0;
+    }
+    return false;
+  } catch (const std::exception& e) {
+    CTL_WRN(e.what());
+    return false;
+  }
+}
+
+cpp::result<bool, std::string> Hardware::UpdateHardwareEntry(const std::string& id,
+                                                     int hw_id,
+                                                     int sw_id) const {
+ try {
+    SQLite::Statement upd(
+        db_,
+        "UPDATE hardware "
+        "SET hardware_id = ?, software_id = ? "
+        "WHERE uuid = ?");
+    upd.bind(1, hw_id);
+    upd.bind(2, sw_id);
+    upd.bind(3, id);
+    if (upd.exec() == 1) {
+      CTL_INF("Updated: " << id << " " << hw_id << " " << sw_id);
+      return true;
+    }
+    return false;
+  } catch (const std::exception& e) {
+    return cpp::fail(e.what());
+  }
+                                                     }
 }  // namespace cortex::db

engine/database/hardware.h

@@ -43,5 +43,9 @@ class Hardware {
   cpp::result<bool, std::string> UpdateHardwareEntry(
       const std::string& id, const HardwareEntry& updated_entry);
   cpp::result<bool, std::string> DeleteHardwareEntry(const std::string& id);
+  bool HasHardwareEntry(const std::string& id);
+  cpp::result<bool, std::string> UpdateHardwareEntry(const std::string& id,
+                                                     int hw_id,
+                                                     int sw_id) const;
 };
 }  // namespace cortex::db

engine/services/hardware_service.cc

@@ -86,13 +86,15 @@ bool HardwareService::Restart(const std::string& host, int port) {
 
 #if defined(_WIN32) || defined(_WIN64) || defined(__linux__)
   std::string cuda_visible_devices = "";
-  for (auto i : (*ahc_).gpus) {
+  auto cuda_config = GetCudaConfig();
+  for (auto i : cuda_config) {
     if (!cuda_visible_devices.empty())
       cuda_visible_devices += ",";
     cuda_visible_devices += std::to_string(i);
   }
   if (cuda_visible_devices.empty())
     cuda_visible_devices += " ";
+
   // Set the CUDA_VISIBLE_DEVICES environment variable
   if (!set_env("CUDA_VISIBLE_DEVICES", cuda_visible_devices)) {
     LOG_WARN << "Error setting CUDA_VISIBLE_DEVICES";
@@ -105,6 +107,28 @@ bool HardwareService::Restart(const std::string& host, int port) {
   } else {
     LOG_WARN << "CUDA_VISIBLE_DEVICES is not set.";
   }
+
+  std::string vk_visible_devices = "";
+  for (auto i : (*ahc_).gpus) {
+    if (!vk_visible_devices.empty())
+      vk_visible_devices += ",";
+    vk_visible_devices += std::to_string(i);
+  }
+
+  if (vk_visible_devices.empty())
+    vk_visible_devices += " ";
+
+  if (!set_env("GGML_VK_VISIBLE_DEVICES", vk_visible_devices)) {
+    LOG_WARN << "Error setting GGML_VK_VISIBLE_DEVICES";
+    return false;
+  }
+
+  const char* vk_value = std::getenv("GGML_VK_VISIBLE_DEVICES");
+  if (vk_value) {
+    LOG_INFO << "GGML_VK_VISIBLE_DEVICES is set to: " << vk_value;
+  } else {
+    LOG_WARN << "GGML_VK_VISIBLE_DEVICES is not set.";
+  }
 #endif
 
 #if defined(_WIN32) || defined(_WIN64)
@@ -273,6 +297,22 @@ void HardwareService::UpdateHardwareInfos() {
   auto gpus = cortex::hw::GetGPUInfo();
   cortex::db::Hardware hw_db;
   auto b = hw_db.LoadHardwareList();
+  // delete if not exists
+  auto exists = [&gpus](const std::string& uuid) {
+    for (auto const& g : gpus) {
+      if (g.uuid == uuid)
+        return true;
+    }
+    return false;
+  };
+  for (auto const& he : b.value()) {
+    if (!exists(he.uuid)) {
+      hw_db.DeleteHardwareEntry(he.uuid);
+    }
+  }
+
+  // Get updated list
+  b = hw_db.LoadHardwareList();
   std::vector<std::pair<int, int>> activated_gpu_bf;
   std::string debug_b;
   for (auto const& he : b.value()) {
@@ -285,14 +325,23 @@ void HardwareService::UpdateHardwareInfos() {
   for (auto const& gpu : gpus) {
     // ignore error
     // Note: only support NVIDIA for now, so hardware_id = software_id
-    auto res = hw_db.AddHardwareEntry(HwEntry{.uuid = gpu.uuid,
-                                              .type = "gpu",
-                                              .hardware_id = std::stoi(gpu.id),
-                                              .software_id = std::stoi(gpu.id),
-                                              .activated = true,
-                                              .priority = INT_MAX});
-    if (res.has_error()) {
-      CTL_WRN(res.error());
+    if (hw_db.HasHardwareEntry(gpu.uuid)) {
+      auto res = hw_db.UpdateHardwareEntry(gpu.uuid, std::stoi(gpu.id),
+                                           std::stoi(gpu.id));
+      if (res.has_error()) {
+        CTL_WRN(res.error());
+      }
+    } else {
+      auto res =
+          hw_db.AddHardwareEntry(HwEntry{.uuid = gpu.uuid,
+                                         .type = "gpu",
+                                         .hardware_id = std::stoi(gpu.id),
+                                         .software_id = std::stoi(gpu.id),
+                                         .activated = true,
+                                         .priority = INT_MAX});
+      if (res.has_error()) {
+        CTL_WRN(res.error());
+      }
     }
   }
 
@@ -317,7 +366,7 @@ void HardwareService::UpdateHardwareInfos() {
     need_restart = true;
   } else {
     for (size_t i = 0; i < activated_gpu_bf.size(); i++) {
-      if (activated_gpu_bf[i].first != activated_gpu_af[i].first) {
+      if (activated_gpu_bf[i].first != activated_gpu_af[i].first) {       
         need_restart = true;
         break;
       }
@@ -333,6 +382,13 @@ void HardwareService::UpdateHardwareInfos() {
     } else {
       need_restart = true;
     }
+
+    const char* vk_value = std::getenv("GGML_VK_VISIBLE_DEVICES");
+    if (vk_value) {
+      LOG_INFO << "GGML_VK_VISIBLE_DEVICES: " << vk_value;
+    } else {
+      need_restart = true;
+    }
   }
 #endif
 
@@ -364,4 +420,31 @@ bool HardwareService::IsValidConfig(
   }
   return false;
 }
+
+std::vector<int> HardwareService::GetCudaConfig() {
+  std::vector<int> res;
+  if (!ahc_)
+    return res;
+  auto nvidia_gpus = system_info_utils::GetGpuInfoList();
+  auto all_gpus = cortex::hw::GetGPUInfo();
+  // Map id with uuid
+  std::vector<std::string> uuids;
+  for (auto i : (*ahc_).gpus) {
+    for (auto const& gpu : all_gpus) {
+      if (i == std::stoi(gpu.id)) {
+        uuids.push_back(gpu.uuid);
+      }
+    }
+  }
+
+  // Map uuid back to nvidia id
+  for (auto const& uuid : uuids) {
+    for (auto const& ngpu : nvidia_gpus) {
+      if (uuid == ngpu.uuid) {
+        res.push_back(std::stoi(ngpu.id));
+      }
+    }
+  }
+  return res;
+}
 }  // namespace services

engine/services/hardware_service.h

@@ -31,6 +31,8 @@ class HardwareService {
   void UpdateHardwareInfos();
   bool IsValidConfig(const cortex::hw::ActivateHardwareConfig& ahc);
 
+private:
+  std::vector<int> GetCudaConfig();
  private:
   std::optional<cortex::hw::ActivateHardwareConfig> ahc_;
 };