Applying comments, added dropping batch info, fixed bug, reduced memory usage

Author: Pashchenkov Maxim

demos/gesture_recognition_demo/cpp_gapi/gesture_recognition_demo_gapi.hpp

@@ -52,10 +52,10 @@ static void showUsage() {
     std::cout << "    -o \"<path>\"              " << output_message << std::endl;
     std::cout << "    -limit \"<num>\"           " << limit_message << std::endl;
     std::cout << "    -res \"<WxH>\"             " << camera_resolution_message << std::endl;
-    std::cout << "    -m_d \"<path>\"           " << person_detection_model_message << std::endl;
-    std::cout << "    -m_a \"<path>\"           " << action_recognition_model_message << std::endl;
-    std::cout << "    -d_d \"<device>\"         " << target_device_message_d << std::endl;
-    std::cout << "    -d_a \"<device>\"         " << target_device_message_a << std::endl;
+    std::cout << "    -m_d \"<path>\"            " << person_detection_model_message << std::endl;
+    std::cout << "    -m_a \"<path>\"            " << action_recognition_model_message << std::endl;
+    std::cout << "    -d_d \"<device>\"          " << target_device_message_d << std::endl;
+    std::cout << "    -d_a \"<device>\"          " << target_device_message_a << std::endl;
     std::cout << "    -no_show                 " << no_show_message << std::endl;
     std::cout << "    -c                       " << class_map_message << std::endl;
     std::cout << "    -s                       " << samples_dir_message << std::endl;

demos/gesture_recognition_demo/cpp_gapi/include/custom_kernels.hpp

@@ -11,8 +11,6 @@
 
 #include "tracker.hpp"
 
-static std::atomic<size_t> current_person_id{ 0 };
-
 namespace custom {
 G_API_OP(GetFastFrame,
          <cv::GMat(cv::GArray<cv::GMat>, cv::Size)>, "custom.get_fast_frame") {
@@ -48,12 +46,14 @@ G_API_OP(ConstructClip,
          <cv::GArray<cv::GMat>(const cv::GArray<cv::GMat>,
                                const cv::GArray<TrackedObject>,
                                const cv::Scalar,
-                               const cv::Size)>,
+                               const cv::Size,
+                               const cv::GOpaque<std::shared_ptr<size_t>>)>,
          "custom.construct_clip") {
     static cv::GArrayDesc outMeta(const cv::GArrayDesc&,
                                   const cv::GArrayDesc&,
                                   const cv::Scalar&,
-                                  const cv::Size&) {
+                                  const cv::Size&,
+                                  const cv::GOpaqueDesc&) {
         return cv::empty_array_desc();
     }
 };

demos/gesture_recognition_demo/cpp_gapi/include/stream_source.hpp

@@ -11,41 +11,43 @@
 #include <thread>
 #include <mutex>
 
-std::mutex batch_lock; // batch frames filling will be locked
-
 namespace custom {
 class BatchProducer {
 public:
-    BatchProducer(const int batch_size, const float batch_fps)
-        : batch_fps(batch_fps) {
+    BatchProducer(const int batch_size, const float batch_fps, const std::shared_ptr<bool>& drop_batch)
+        : batch_fps(batch_fps), drop_batch(drop_batch) {
         /** Create batch memory space for batch_size + 2 size
          *  first additional element is fast image
          *  second additional Mat is sacriface of memory for data about first element
         **/
         batch = std::vector<cv::Mat>(batch_size + 1 + 1); // 16(8) 15FPS-batch imgaes + one fast image +  batch description
-        batch[batch_size + 1].create(cv::Size{1, 2}, CV_8U); // 1x2 Mat for first element position and is_filled batch state
+        batch[batch_size + 1].create(cv::Size{ 1, 2 }, CV_8U); // 1x2 Mat for first element position and is_filled batch state
         auto ptr = batch[batch.size() - 1].ptr<uint8_t>();
         ptr[1] = 0; // set is_filled to NO
     }
     std::vector<cv::Mat> getBatch() {
-        return batch;
+        batch_lock.lock();
+        std::vector<cv::Mat> temp_batch = batch;
+        batch_lock.unlock();
+        return temp_batch;
     }
 
     void fillFastFrame(const cv::Mat& frame) {
-        /** Copy fast frame from VideoCapture to batch memory as 17th (9) image **/
-        batch[batch.size() - 2] = frame.clone(); // 16th (from 0)
+            /** Copy fast frame from VideoCapture to batch memory as 17th (9) image **/
+            frame.copyTo(batch[batch.size() - 2]); // 16th (from 0)
     }
 
     void fillBatch(const cv::Mat& frame, std::chrono::steady_clock::time_point time) {
+        if (*drop_batch > 0) {
+            DropBatchInfo();
+        }
         /** Place of new frame in batch **/
         const int step = updateStep(batch.size() - 2);
-        batch_lock.lock();
         /** Adding of new image to batch. **/
-        batch[step] = frame.clone();
+        frame.copyTo(batch[step]);
         /** Putting of info about batch to additional element **/
         auto ptr = batch[batch.size() - 1].ptr<uint8_t>();
         ptr[0] = first_el; // position of start of batch in cyclic buffer
-        batch_lock.unlock();
         const auto cur_step = std::chrono::steady_clock::now() - time;
         const auto gap = std::chrono::duration_cast<std::chrono::milliseconds>(cur_step);
         const auto time_step = std::chrono::milliseconds(int(1000.f / batch_fps)); // 1/15 sec
@@ -55,46 +57,59 @@ class BatchProducer {
     }
 private:
     float batch_fps = 0; // constant FPS for batch
+    const std::shared_ptr<bool>& drop_batch;
     std::vector<cv::Mat> batch; // pack of images for graph
     size_t first_el = 0; // place of first image in batch
     size_t images_in_batch_count = 0; // number of images in batch
     bool is_filled = false; // is batch filled
+    std::mutex batch_lock; // batch frames filling will be locked
 
     int updateStep(const size_t batch_size) {
         if (images_in_batch_count < batch_size) {
             /** case when batch isn't filled **/
             return images_in_batch_count++;
-        } else {
+        }
+        else {
             if (!is_filled) {
-                batch_lock.lock();
                 auto ptr = batch[batch.size() - 1].ptr<uint8_t>();
                 ptr[1] = 1;
-                batch_lock.unlock();
                 is_filled = true;
             }
             /** Cyclic buffer if filled. Counting of step for next image **/
             first_el = (first_el + 1) % batch_size;
             return first_el;
         }
     }
+
+    void DropBatchInfo() {
+        /** Drop batch information.
+          * Processing will continue when the batch will be filled
+          * Data of the batch will be overwritten */
+        auto ptr = batch[batch.size() - 1].ptr<uint8_t>();
+        ptr[0] = 0; // first position
+        ptr[1] = 0; // batch if filled
+        first_el = 0;
+        images_in_batch_count = 0;
+        is_filled = false;
+    }
 };
 
 void runBatchFill(const cv::Mat& frame,
-                        BatchProducer& producer,
-                        std::chrono::steady_clock::time_point& time) {
-    while(!frame.empty()) {
+                  BatchProducer& producer,
+    std::chrono::steady_clock::time_point& time) {
+    while (!frame.empty()) {
         producer.fillBatch(frame, time);
     }
 }
 
-class CustomCapSource : public cv::gapi::wip::IStreamSource
-{
+class CustomCapSource : public cv::gapi::wip::IStreamSource {
 public:
     explicit CustomCapSource(const std::shared_ptr<ImagesCapture>& cap,
                              const cv::Size& frame_size,
                              const int batch_size,
-                             const float batch_fps)
-        : cap(cap), producer(batch_size, batch_fps), source_fps(cap->fps()) {
+                             const float batch_fps,
+                             const std::shared_ptr<bool>& drop_batch)
+        : cap(cap), producer(batch_size, batch_fps, drop_batch), source_fps(cap->fps()) {
         if (source_fps <= 0.) {
             source_fps = 30.;
             wait_gap = true;
@@ -106,19 +121,21 @@ class CustomCapSource : public cv::gapi::wip::IStreamSource
             GAPI_Assert(false && "Batch must contain more than one image");
         }
 
-        fast_frame.create(frame_size, CV_8UC3);
-
         /** Reading of frame with ImagesCapture class **/
         read_time = std::chrono::steady_clock::now();
-        fast_frame = cap->read();
+        cv::Mat fast_frame = cap->read();
         if (!fast_frame.data) {
             GAPI_Assert(false && "Couldn't grab the frame");
         }
 
+        producer.fillFastFrame(fast_frame);
+        fast_frame.copyTo(thread_frame);
         /** Batch filling with constant time step **/
+        std::thread fill_bath_thr(runBatchFill,
+                                  std::cref(thread_frame),
+                                  std::ref(producer),
+                                  std::ref(read_time));
         fill_bath_thr.detach();
-
-        producer.fillFastFrame(fast_frame);
         first_batch = producer.getBatch();
     }
 
@@ -129,14 +146,11 @@ class CustomCapSource : public cv::gapi::wip::IStreamSource
     bool wait_gap = false; // waiting for fast frame reading (stop main thread when got a non-positive FPS value)
     bool first_pulled = false; // is first already pulled
     std::vector<cv::Mat> first_batch; // batch from constructor
-    cv::Mat fast_frame; // frame from cv::VideoCapture
+    cv::Mat thread_frame; // frame for batch constant filling
+    std::mutex thread_frame_lock;
     std::chrono::steady_clock::time_point read_time; // timepoint from cv::read()
-    std::thread fill_bath_thr = std::thread(runBatchFill,
-                                            std::ref(fast_frame),
-                                            std::ref(producer),
-                                            std::ref(read_time));
 
-    virtual bool pull(cv::gapi::wip::Data &data) override {
+    virtual bool pull(cv::gapi::wip::Data& data) override {
         /** Is first already pulled **/
         if (!first_pulled) {
             GAPI_Assert(!first_batch.empty());
@@ -148,11 +162,16 @@ class CustomCapSource : public cv::gapi::wip::IStreamSource
 
         /** Frame reading with ImagesCapture class **/
         read_time = std::chrono::steady_clock::now();
-        fast_frame = cap->read();
+        cv::Mat fast_frame = cap->read();
+
         if (!fast_frame.data) {
             return false;
         }
 
+        thread_frame_lock.lock();
+        fast_frame.copyTo(thread_frame);
+        thread_frame_lock.unlock();
+
         /** Put fast frame to the batch **/
         producer.fillFastFrame(fast_frame);
         if (wait_gap) {
@@ -175,5 +194,4 @@ class CustomCapSource : public cv::gapi::wip::IStreamSource
         return cv::GMetaArg{ cv::empty_array_desc() };
     }
 };
-
 } // namespace custom

demos/gesture_recognition_demo/cpp_gapi/include/visualizer.hpp

@@ -14,7 +14,6 @@ class Visualizer {
 private:
     cv::Mat storageFrame;
     bool no_show_;
-    std::string main_window_name_;
     std::string storage_window_name_;
     std::vector<std::string> labels_;
     std::string storage_path_;
@@ -43,18 +42,15 @@ class Visualizer {
                const std::string& storage_window_name,
                const std::vector<std::string>& labels,
                const std::string& storage_path) :
-        no_show_(no_show), main_window_name_(main_window_name),
-        storage_window_name_(storage_window_name), labels_(labels),
-        storage_path_(storage_path) {
+        no_show_(no_show), storage_window_name_(storage_window_name),
+        labels_(labels), storage_path_(storage_path) {
         if (storage_path_.size() > 0) {
             getStorageElements();
         }
         if (no_show) {
             return;
         }
 
-        cv::namedWindow(main_window_name_);
-
         if (storage_path_.size() > 0) {
             cv::namedWindow(storage_window_name_);
             gesture_cap_.open(storage_elements_.front().second);
@@ -66,6 +62,4 @@ class Visualizer {
               const int out_label_number,
               const size_t current_id,
               const int key);
-
-    void finalize();
 };

demos/gesture_recognition_demo/cpp_gapi/main.cpp

@@ -65,9 +65,13 @@ int main(int argc, char *argv[]) {
         cap = openImagesCapture(FLAGS_i, FLAGS_loop, 0,
             std::numeric_limits<size_t>::max(), stringToSize(FLAGS_res));
 
+        /** Share runtime id with graph **/
+        auto current_person_id_m = std::make_shared<size_t>(0);
+
         /** ---------------- Main graph of demo ---------------- **/
         /** Graph inputs **/
         cv::GArray<cv::GMat> batch;
+        cv::GOpaque<std::shared_ptr<size_t>> current_person_id;
 
         cv::GMat fast_frame = custom::GetFastFrame::on(batch, frame_size);
 
@@ -81,7 +85,7 @@ int main(int argc, char *argv[]) {
         cv::GArray<TrackedObject> tracked = custom::TrackPerson::on(fast_frame, objects);
 
         /** Create clip for AR net **/
-        cv::GArray<cv::GMat> clip = custom::ConstructClip::on(batch, tracked, ar_net_shape, frame_size);
+        cv::GArray<cv::GMat> clip = custom::ConstructClip::on(batch, tracked, ar_net_shape, frame_size, current_person_id);
 
         /** Action recognition **/
         cv::GArray<cv::GMat> actions = cv::gapi::infer2<nets::ActionRecognition>(fast_frame, clip);
@@ -90,7 +94,7 @@ int main(int argc, char *argv[]) {
         cv::GOpaque<int> label = custom::GestureRecognitionPostprocessing::on(actions, float(FLAGS_t));
 
         /** Inputs and outputs of graph **/
-        auto graph = cv::GComputation(cv::GIn(batch), cv::GOut(fast_frame, tracked, label));
+        auto graph = cv::GComputation(cv::GIn(batch, current_person_id), cv::GOut(fast_frame, tracked, label));
         /** ---------------- End of graph ---------------- **/
         /** Configure networks **/
         auto person_detection = cv::gapi::ie::Params<nets::PersonDetection> {
@@ -118,17 +122,16 @@ int main(int argc, char *argv[]) {
         TrackedObjects out_detections;
         int out_label_number;
 
-        auto out_vector = cv::gout(out_frame, out_detections, out_label_number);
-
         /** ---------------- The execution part ---------------- **/
         const float batch_constant_FPS = 15;
-
+        auto drop_batch = std::make_shared<bool>(false);
         pipeline.setSource(cv::gin(cv::gapi::wip::make_src<custom::CustomCapSource>(cap,
                                                                                     frame_size,
                                                                                     int(ar_net_shape[1]),
-                                                                                    batch_constant_FPS)));
+                                                                                    batch_constant_FPS,
+                                                                                    drop_batch),
+                                   current_person_id_m));
 
-        std::string mainWindowName = "Gesture Recognition demo G-API";
         std::string gestureWindowName = "Gesture";
 
         cv::Size graphSize{static_cast<int>(frame_size.width / 4), 60};
@@ -144,16 +147,16 @@ int main(int argc, char *argv[]) {
         /** Fill labels container from file with classes **/
         const auto labels = fill_labels(FLAGS_c);
         size_t current_id = 0;
-        size_t id = 0;
+        size_t last_id = current_id;
         int gesture = 0;
 
         /** Configure drawing utilities **/
-        Visualizer visualizer(FLAGS_no_show, mainWindowName, gestureWindowName, labels, FLAGS_s);
+        Visualizer visualizer(FLAGS_no_show, gestureWindowName, labels, FLAGS_s);
 
         bool isStart = true;
         const auto startTime = std::chrono::steady_clock::now();
         pipeline.start();
-        while (pipeline.pull(std::move(out_vector))) {
+        while (pipeline.pull(std::move(cv::gout(out_frame, out_detections, out_label_number)))) {
             /** Put FPS to frame**/
             if (isStart) {
                 metrics.update(startTime, out_frame, { 10, 22 }, cv::FONT_HERSHEY_COMPLEX,
@@ -177,21 +180,20 @@ int main(int argc, char *argv[]) {
             /** Controls **/
             int key = cv::waitKey(1);
             if      (key == 0x1B) break;  // (esc button) exit
-            else if (key >= 48 && key <= 57) id = key - 48; // buttons for person id
+            else if (key >= 48 && key <= 57) current_id = key - 48; // buttons for person id
             else if (key == 0x0D) out_label_number = -1; // (Enter) reset last gesture
             else if (key == 'f') gesture = 1; // next gesture
             else if (key == 'b') gesture = -1; // prev gesture
             else
                 presenter.handleKey(key);
 
             /** Share id with graph **/
-            if (id < out_detections.size()) {
-                current_person_id = id;
-                current_id = id;
+            if (current_id < out_detections.size()) {
+                *drop_batch = !(last_id != current_id);
+                *current_person_id_m = current_id;
+                last_id = current_id;
             }
         }
-        /** Destroy windows if exist **/
-        visualizer.finalize();
         slog::info << "Metrics report:" << slog::endl;
         slog::info << "\tFPS: " << std::fixed << std::setprecision(1) << metrics.getTotal().fps << slog::endl;
         slog::info << presenter.reportMeans() << slog::endl;