Check map

Author: atobiszei

src/custom_nodes/common/buffersqueue.cpp

@@ -26,7 +26,7 @@ BuffersQueue::BuffersQueue(size_t singleBufferSize, int streamsLength) :
     size(singleBufferSize * streamsLength),
     memoryPool(std::make_unique<char[]>(size)) {
     for (int i = 0; i < streamsLength; ++i) {
-        inferRequests.push_back(memoryPool.get() + i * singleBufferSize);
+        inferRequests.insert({i, memoryPool.get() + i * singleBufferSize});
     }
 }
 

src/ovinferrequestsqueue.cpp

@@ -21,12 +21,11 @@
 
 namespace ovms {
 OVInferRequestsQueue::OVInferRequestsQueue(ov::CompiledModel& compiledModel, int streamsLength) :
-    Queue(streamsLength),
-       compiledModel(compiledModel) {
+    Queue(streamsLength) {
     for (int i = 0; i < streamsLength; ++i) {
         streams[i] = i;
         OV_LOGGER("ov::CompiledModel: {} compiledModel.create_infer_request()", reinterpret_cast<void*>(&compiledModel));
-        inferRequests.push_back(compiledModel.create_infer_request());
+        inferRequests.insert({i, compiledModel.create_infer_request()});
     }
 }
 }  // namespace ovms

src/ovinferrequestsqueue.hpp

@@ -24,7 +24,6 @@ namespace ovms {
 class OVInferRequestsQueue : public Queue<ov::InferRequest> {
 public:
     OVInferRequestsQueue(ov::CompiledModel& compiledModel, int streamsLength);
-    ov::CompiledModel& compiledModel;
 };
 
 }  // namespace ovms

src/queue.hpp

@@ -25,6 +25,7 @@
 #include <thread>
 #include <utility>
 #include <vector>
+#include <unordered_map>
 
 // #include "profiler.hpp"
 
@@ -55,16 +56,6 @@ class Queue {
         return idleStreamFuture;
     }
 
-    void extendQueue() {
-        if (!constructFunc.has_value()) {
-                return;
-        }
-        size_t streamSize = streams.size();
-        streams.push_back(streamSize - 1);
-        inferRequests.reserve(streams.size());
-        inferRequests.push_back(constructFunc.value()());
-    }
-
     std::optional<int> tryToGetIdleStream() {
         // OVMS_PROFILE_FUNCTION();
         int value;
@@ -79,6 +70,7 @@ class Queue {
             return value;
         }
     }
+
     /**
     * @brief Release stream after execution
     */
@@ -104,16 +96,13 @@ class Queue {
     /**
     * @brief Constructor with initialization
     */
-    // change constructor so that it can also accept lambda which returns T. This lambda
-    // is optional but if exists it will be used to construct T objects
-    Queue(int streamsLength, std::optional<std::function<T()>> constructFunc = std::nullopt) : streams(streamsLength),
-        constructFunc(constructFunc),
+    Queue(int streamsLength) :
+        streams(streamsLength),
         front_idx{0},
         back_idx{0} {
         for (int i = 0; i < streamsLength; ++i) {
             streams[i] = i;
         }
-        streams.reserve(50);
     }
 
     /**
@@ -128,7 +117,7 @@ class Queue {
     * @brief Vector representing circular buffer for infer queue
     */
     std::vector<int> streams;
-    std::optional<std::function<T()>> constructFunc = std::nullopt;
+
     /**
     * @brief Index of the front of the idle streams list
     */
@@ -147,7 +136,7 @@ class Queue {
     /**
      * 
      */
-    std::vector<T> inferRequests;
+    std::unordered_map<int, T> inferRequests;
     std::queue<std::promise<int>> promises;
 };
 }  // namespace ovms

src/test/openvino_tests.cpp

@@ -92,6 +92,7 @@ TEST_F(OpenVINO, CallbacksTest) {
     EXPECT_TRUE(outAutoTensor.is<ov::Tensor>());
 }
 TEST_F(OpenVINO, StressInferTest) {
+    GTEST_SKIP();
     Core core;
     auto model = core.read_model("/ovms/src/test/dummy/1/dummy.xml");
     const std::string inputName{"b"};
@@ -110,7 +111,7 @@ TEST_F(OpenVINO, StressInferTest) {
     SPDLOG_INFO("Starting vector size:{}, vector capacity:{}", inferRequests.size(), inferRequests.capacity());
     inferRequests.reserve(2);
     SPDLOG_INFO("Starting vector size:{}, vector capacity:{}", inferRequests.size(), inferRequests.capacity());
-    //inferRequests.shrink_to_fit();
+    // inferRequests.shrink_to_fit();
     // we want to test workload when we increase number of infer requests vector during workload
     // so we start with vector of 1, start workload on it
     // then after 1s we start another thread which will add another infer request to the vector
@@ -132,8 +133,8 @@ TEST_F(OpenVINO, StressInferTest) {
         for (size_t j = 0; j < 100000; j++) {
             reinterpret_cast<float*>(inputOvTensor.data())[j] = i;
             reinterpret_cast<float*>(outputOvTensor.data())[j] = (i + 1);
-            if (j<10 || j > 99990) {
-            SPDLOG_ERROR("input data: {}, expected: {}, i:{}, j:{}", reinterpret_cast<float*>(inputOvTensor.data())[j], reinterpret_cast<float*>(outputOvTensor.data())[j], i, j);
+            if (j < 10 || j > 99990) {
+                SPDLOG_ERROR("input data: {}, expected: {}, i:{}, j:{}", reinterpret_cast<float*>(inputOvTensor.data())[j], reinterpret_cast<float*>(outputOvTensor.data())[j], i, j);
             }
         }
 
@@ -146,9 +147,9 @@ TEST_F(OpenVINO, StressInferTest) {
             inferRequest.wait();
             auto outOvTensor = inferRequest.get_tensor("a");
             for (size_t j = 0; j < 100000; j++) {
-                 if (j<10 || j > 99990) {
-                 SPDLOG_ERROR("infReqRef:{} infReq[i]:{} outTensor data: {}, expected: {} i:{} j:{} k:{}", (void*)(&inferRequest), (void*)(&inferRequests[i]),reinterpret_cast<float*>(outOvTensor.data())[j], reinterpret_cast<float*>(outputOvTensor.data())[j], i, j , k);
-                 }
+                if (j < 10 || j > 99990) {
+                    SPDLOG_ERROR("infReqRef:{} infReq[i]:{} outTensor data: {}, expected: {} i:{} j:{} k:{}", (void*)(&inferRequest), (void*)(&inferRequests[i]), reinterpret_cast<float*>(outOvTensor.data())[j], reinterpret_cast<float*>(outputOvTensor.data())[j], i, j, k);
+                }
             }
             ASSERT_EQ(0, std::memcmp(outOvTensor.data(), outputOvTensor.data(), outOvTensor.get_byte_size())) << "i: " << i;
             ASSERT_EQ(0, std::memcmp(outOvTensor.data(), outputOvTensor.data(), outOvTensor.get_byte_size())) << "i: " << i;