From 7461f7fa8e524bf5b0e039b97c58f6e75b3bef26 Mon Sep 17 00:00:00 2001 From: Pavel Druzhkov Date: Thu, 23 Jun 2016 10:17:38 +0300 Subject: [PATCH 1/4] add tracking sample --- include/tracking.hpp | 36 +++++++ samples/tracking_demo.cpp | 102 ++++++++++++++++++++ src/tracking.cpp | 194 +++++++++++++++++++++++++++++++++++++- 3 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 samples/tracking_demo.cpp diff --git a/include/tracking.hpp b/include/tracking.hpp index f80004c..65718e3 100644 --- a/include/tracking.hpp +++ b/include/tracking.hpp @@ -11,3 +11,39 @@ class Tracker { virtual bool Init(const cv::Mat &frame, const cv::Rect &roi) = 0; virtual cv::Rect Track(const cv::Mat &frame) = 0; }; + +class MedianFlowTracker : public Tracker { + public: + virtual bool Init(const cv::Mat &frame, const cv::Rect &roi); + virtual cv::Rect Track(const cv::Mat &frame); + + protected: + cv::Rect position_; + cv::Mat frame_; + + float Median(const std::vector &v) const; + + bool FilterCorners(std::vector &corners, + std::vector &corners_next_frame, + std::vector &status, + std::vector &errors) const; + + bool ComputeMedianShift(const std::vector &corners, + const std::vector &corners_next_frame, + cv::Point2f &shift) const; + + bool ComputePointDistances(const std::vector &corners, + std::vector &dist) const; + + bool ComputeDistScales(const std::vector &dist, + const std::vector &dist_next_frame, + std::vector &scales) const; + + bool ComputeScaleFactor(const std::vector &corners, + const std::vector &corners_next_frame, + float &scale) const; + + bool RestoreBoundingBox(const std::vector &corners, + const std::vector &corners_next_frame, + cv::Rect &new_position) const; +}; diff --git a/samples/tracking_demo.cpp b/samples/tracking_demo.cpp new file mode 100644 index 0000000..53166e0 --- /dev/null +++ b/samples/tracking_demo.cpp @@ -0,0 +1,102 @@ +#include +#include + +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/objdetect.hpp" + +#include "tracking.hpp" + +using namespace std; +using namespace cv; + +const char* kAbout = "This is tracking sample application."; + +const char* kOptions = + "{ v video | | video to process }" + "{ h ? help usage | | print help message }"; + +struct MouseCallbackState { + bool is_selection_started; + bool is_selection_finished; + Point point_first; + Point point_second; +}; + +static void OnMouse(int event, int x, int y, int, void* s) { + MouseCallbackState* state = reinterpret_cast(s); + CV_Assert(state != nullptr); + switch (event) { + case cv::EVENT_LBUTTONDOWN: + state->is_selection_started = true; + state->is_selection_finished = false; + state->point_first = Point(x, y); + break; + case cv::EVENT_LBUTTONUP: + state->is_selection_finished = true; + state->is_selection_started = false; + break; + case cv::EVENT_MOUSEMOVE: + if (state->is_selection_started && !state->is_selection_finished) { + state->point_second = Point(x, y); + } + break; + } +} + +int main(int argc, const char** argv) { + // Parse command line arguments. + CommandLineParser parser(argc, argv, kOptions); + parser.about(kAbout); + + // If help option is given, print help message and exit. + if (parser.get("help")) { + parser.printMessage(); + return 0; + } + + // Load input video. + string video_path = parser.get("video"); + VideoCapture video(video_path); + if (!video.isOpened()) { + cout << "Failed to open video file '" << video_path << "'" << endl; + return 0; + } + + const string kWindowName = "video"; + const int kWaitKeyDelay = 100; + const int kEscapeKey = 27; + const Scalar kColorBlue = CV_RGB(0, 0, 255); + + namedWindow(kWindowName); + MouseCallbackState mouse_state; + mouse_state.is_selection_started = false; + mouse_state.is_selection_finished = false; + setMouseCallback(kWindowName, OnMouse, &mouse_state); + + Mat frame; + video >> frame; + imshow(kWindowName, frame); + while (!mouse_state.is_selection_finished) { + waitKey(kWaitKeyDelay); + } + Rect roi = Rect(mouse_state.point_first, mouse_state.point_second); + + MedianFlowTracker tracker; + tracker.Init(frame, roi); + video >> frame; + + while (!frame.empty()) { + roi = tracker.Track(frame); + rectangle(frame, roi, kColorBlue, 1); + imshow(kWindowName, frame); + int key = waitKey(kWaitKeyDelay) & 0x00FF; + if (key == kEscapeKey) { + break; + } + video >> frame; + } + + return 0; +} diff --git a/src/tracking.cpp b/src/tracking.cpp index 26cbe06..a942acd 100644 --- a/src/tracking.cpp +++ b/src/tracking.cpp @@ -1,13 +1,203 @@ #include "tracking.hpp" +#include #include +#include "opencv2/imgproc.hpp" +#include "opencv2/video/tracking.hpp" + using std::string; using std::shared_ptr; +using std::vector; using namespace cv; shared_ptr Tracker::CreateTracker(const string &name) { - std::cerr << "Failed to create tracker with name '" << name << "'" - << std::endl; + if (name == "median_flow") { + return std::make_shared(); + } else { + std::cerr << "Failed to create tracker with name '" << name << "'" + << std::endl; + } return nullptr; } + +bool MedianFlowTracker::Init(const Mat &frame, const Rect &roi) { + if (frame.channels() == 3) { + cvtColor(frame, frame_, CV_BGR2GRAY); + } + Rect image_bounding_rect(Point(0, 0), frame.size()); + if ((image_bounding_rect & roi) == roi) { + position_ = roi; + return true; + } else { + std::cerr << "ROI " << roi << "is outside of image."; + } + return false; +} + +float MedianFlowTracker::Median(const vector &v) const { + auto values = v; + size_t middle = values.size() / 2; + std::nth_element(values.begin(), values.begin() + middle, values.end()); + return values[middle]; +} + +bool MedianFlowTracker::FilterCorners(vector &corners, + vector &corners_next_frame, + vector &status, + vector &errors) const { + for (int i = status.size() - 1; i >= 0; i--) { + if (!status[i]) { + status.erase(status.begin() + i); + corners.erase(corners.begin() + i); + corners_next_frame.erase(corners_next_frame.begin() + i); + errors.erase(errors.begin() + i); + } + } + if (corners.empty()) { + return false; + } + vector errors_copy(errors.size()); + std::copy(errors.begin(), errors.end(), errors_copy.begin()); + float median_error = Median(errors_copy); + + for (int i = errors.size() - 1; i >= 0; i--) { + if (errors[i] > median_error) { + errors.erase(errors.begin() + i); + corners.erase(corners.begin() + i); + corners_next_frame.erase(corners_next_frame.begin() + i); + status.erase(status.begin() + i); + } + } + if (corners.empty()) { + return false; + } + + return true; +} + +bool MedianFlowTracker::ComputeMedianShift(const vector &corners, + const vector &nextCorners, + Point2f &shift) const { + vector shifts_x, shifts_y; + for (size_t i = 0; i < corners.size(); ++i) { + shifts_x.emplace_back(nextCorners.at(i).x - corners.at(i).x); + shifts_y.emplace_back(nextCorners.at(i).y - corners.at(i).y); + } + float dx = Median(shifts_x); + float dy = Median(shifts_y); + shift = Point2f(dx, dy); + return true; +} + +bool MedianFlowTracker::ComputePointDistances(const vector &corners, + vector &dist) const { + dist.clear(); + for (int i = 0; i < corners.size(); i++) { + for (int j = i + 1; j < corners.size(); j++) { + dist.push_back(cv::norm(corners.at(i) - corners.at(j))); + } + } + return true; +} + +bool MedianFlowTracker::ComputeDistScales(const vector &dist, + const vector &dist_next_frame, + vector &scales) const { + scales.clear(); + if (dist.size() != dist_next_frame.size()) { + return false; + } + for (size_t i = 0; i < dist.size(); ++i) { + if (dist.at(i) != 0.0f) { + scales.emplace_back(dist_next_frame.at(i) / dist.at(i)); + } + } + if (scales.empty()) { + return false; + } + return true; +} + +bool MedianFlowTracker::ComputeScaleFactor( + const vector &corners, const vector &corners_next_frame, + float &scale) const { + if (corners.size() <= 1 || corners_next_frame.size() <= 1) { + return false; + } + vector dist, dist_next_frame, scales; + ComputePointDistances(corners, dist); + ComputePointDistances(corners_next_frame, dist_next_frame); + ComputeDistScales(dist, dist_next_frame, scales); + scale = Median(scales); + return true; +} + +bool MedianFlowTracker::RestoreBoundingBox( + const vector &corners, const vector &corners_next_frame, + Rect &new_position) const { + Point2f shift; + ComputeMedianShift(corners, corners_next_frame, shift); + + float scale; + if (!ComputeScaleFactor(corners, corners_next_frame, scale)) { + return false; + } + + new_position = position_ + Point(shift); + new_position.width *= scale; + new_position.height *= scale; + Rect image_bounding_box(Point(0, 0), frame_.size()); + new_position = image_bounding_box & new_position; + if (new_position.area() == 0) { + return false; + } + + return true; +} + +Rect MedianFlowTracker::Track(const Mat &frame) { + CV_Assert(!frame.empty()); + Mat object = frame_(position_); + vector corners; + + const int kMaxCorners = 100; + const double kQualityLevel = 0.01; + const double kMinDistance = 5.0; + goodFeaturesToTrack(object, corners, kMaxCorners, kQualityLevel, + kMinDistance); + if (corners.empty()) { + std::cout << "Tracked object is lost." << std::endl; + return Rect(); + } + + for (auto &corner : corners) { + corner += Point2f(position_.tl()); + } + + vector corners_next_frame; + vector status; + vector errors; + Mat next_frame = frame.clone(); + if (next_frame.channels() == 3) { + cvtColor(next_frame, next_frame, CV_BGR2GRAY); + } + calcOpticalFlowPyrLK(frame_, next_frame, corners, corners_next_frame, status, + errors); + + if (!FilterCorners(corners, corners_next_frame, status, errors)) { + std::cout << "There are not enough points for tracking." << std::endl; + return Rect(); + } + + Rect new_position; + if (!RestoreBoundingBox(corners, corners_next_frame, new_position)) { + std::cout << "There are not enough points to restore bounding box." + << std::endl; + return Rect(); + } + + position_ = new_position; + frame_ = next_frame; + return position_; +} From fb1714d35eefd3deccd2b4cefb03b80d99e071c6 Mon Sep 17 00:00:00 2001 From: Pavel Druzhkov Date: Thu, 7 Jul 2016 19:04:55 +0400 Subject: [PATCH 2/4] fix issue with scale --- include/tracking.hpp | 4 +++- samples/tracking_demo.cpp | 36 ++++++++++++++++++++++++++++-------- src/tracking.cpp | 34 ++++++++++++++++++++++++++-------- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/include/tracking.hpp b/include/tracking.hpp index 65718e3..c8192fc 100644 --- a/include/tracking.hpp +++ b/include/tracking.hpp @@ -20,6 +20,8 @@ class MedianFlowTracker : public Tracker { protected: cv::Rect position_; cv::Mat frame_; + cv::Size initial_size_; + float scale_; float Median(const std::vector &v) const; @@ -45,5 +47,5 @@ class MedianFlowTracker : public Tracker { bool RestoreBoundingBox(const std::vector &corners, const std::vector &corners_next_frame, - cv::Rect &new_position) const; + cv::Rect &new_position, float& scale) const; }; diff --git a/samples/tracking_demo.cpp b/samples/tracking_demo.cpp index 53166e0..4271435 100644 --- a/samples/tracking_demo.cpp +++ b/samples/tracking_demo.cpp @@ -14,8 +14,9 @@ using namespace cv; const char* kAbout = "This is tracking sample application."; const char* kOptions = - "{ v video | | video to process }" - "{ h ? help usage | | print help message }"; + "{ v video | | video to process }" + "{ c camera | | camera id to capture video from }" + "{ h ? help usage | | print help message }"; struct MouseCallbackState { bool is_selection_started; @@ -57,17 +58,29 @@ int main(int argc, const char** argv) { } // Load input video. - string video_path = parser.get("video"); - VideoCapture video(video_path); + VideoCapture video; + bool is_live_stream = false; + if (parser.has("video")) { + string video_path = parser.get("video"); + video.open(video_path); + is_live_stream = false; + } + if (parser.has("camera")) { + video.open(parser.get("camera")); + is_live_stream = true; + } + if (!video.isOpened()) { - cout << "Failed to open video file '" << video_path << "'" << endl; + cout << "Failed to open video." << endl; return 0; } const string kWindowName = "video"; - const int kWaitKeyDelay = 100; + const int kWaitKeyDelay = 20; const int kEscapeKey = 27; const Scalar kColorBlue = CV_RGB(0, 0, 255); + const Scalar kColorGreen = CV_RGB(0, 255, 0); + const int kLineThickness = 2; namedWindow(kWindowName); MouseCallbackState mouse_state; @@ -76,12 +89,19 @@ int main(int argc, const char** argv) { setMouseCallback(kWindowName, OnMouse, &mouse_state); Mat frame; + Rect roi; video >> frame; imshow(kWindowName, frame); while (!mouse_state.is_selection_finished) { + Mat frame_with_selection = frame.clone(); + roi = Rect(mouse_state.point_first, mouse_state.point_second); + rectangle(frame_with_selection, roi, kColorGreen, kLineThickness); + imshow(kWindowName, frame_with_selection); waitKey(kWaitKeyDelay); + if (is_live_stream) { + video >> frame; + } } - Rect roi = Rect(mouse_state.point_first, mouse_state.point_second); MedianFlowTracker tracker; tracker.Init(frame, roi); @@ -89,7 +109,7 @@ int main(int argc, const char** argv) { while (!frame.empty()) { roi = tracker.Track(frame); - rectangle(frame, roi, kColorBlue, 1); + rectangle(frame, roi, kColorBlue, kLineThickness); imshow(kWindowName, frame); int key = waitKey(kWaitKeyDelay) & 0x00FF; if (key == kEscapeKey) { diff --git a/src/tracking.cpp b/src/tracking.cpp index a942acd..9a7c292 100644 --- a/src/tracking.cpp +++ b/src/tracking.cpp @@ -3,8 +3,7 @@ #include #include -#include "opencv2/imgproc.hpp" -#include "opencv2/video/tracking.hpp" +#include "opencv2/opencv.hpp" using std::string; using std::shared_ptr; @@ -28,9 +27,11 @@ bool MedianFlowTracker::Init(const Mat &frame, const Rect &roi) { Rect image_bounding_rect(Point(0, 0), frame.size()); if ((image_bounding_rect & roi) == roi) { position_ = roi; + initial_size_ = roi.size(); + scale_ = 1.0f; return true; } else { - std::cerr << "ROI " << roi << "is outside of image."; + std::cerr << "ROI " << roi << " is outside of image."; } return false; } @@ -135,18 +136,16 @@ bool MedianFlowTracker::ComputeScaleFactor( bool MedianFlowTracker::RestoreBoundingBox( const vector &corners, const vector &corners_next_frame, - Rect &new_position) const { + Rect &new_position, float& scale) const { Point2f shift; ComputeMedianShift(corners, corners_next_frame, shift); - float scale; if (!ComputeScaleFactor(corners, corners_next_frame, scale)) { return false; } - - new_position = position_ + Point(shift); new_position.width *= scale; new_position.height *= scale; + new_position = position_ + Point(shift); Rect image_bounding_box(Point(0, 0), frame_.size()); new_position = image_bounding_box & new_position; if (new_position.area() == 0) { @@ -190,13 +189,32 @@ Rect MedianFlowTracker::Track(const Mat &frame) { return Rect(); } + Mat debug_image(frame_.rows, frame_.cols * 2, frame_.type()); + Mat im1 = debug_image.colRange(Range(0, frame_.cols)); + Mat im2 = debug_image.colRange(Range(frame_.cols, frame_.cols * 2)); + frame_.copyTo(im1); + next_frame.copyTo(im2); + + for (size_t i = 0; i < corners.size(); ++i) { + line(debug_image, Point(corners.at(i)), Point(corners_next_frame.at(i)) + Point(frame_.cols, 0), CV_RGB(0, 255, 255)); + } + + imshow("debug", debug_image); + + Rect new_position; - if (!RestoreBoundingBox(corners, corners_next_frame, new_position)) { + float scale_factor; + if (!RestoreBoundingBox(corners, corners_next_frame, new_position, scale_factor)) { std::cout << "There are not enough points to restore bounding box." << std::endl; return Rect(); } + scale_ *= scale_factor; + new_position = Rect(new_position.tl(), Size2f(initial_size_) * scale_); + Rect image_bounding_box(Point(0, 0), frame_.size()); + new_position = image_bounding_box & new_position; + position_ = new_position; frame_ = next_frame; return position_; From 1ae8873bd61739b1a8b23d382e177021d467200a Mon Sep 17 00:00:00 2001 From: Pavel Druzhkov Date: Thu, 7 Jul 2016 19:25:44 +0400 Subject: [PATCH 3/4] remove debug output --- src/tracking.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/tracking.cpp b/src/tracking.cpp index 9a7c292..8d1072e 100644 --- a/src/tracking.cpp +++ b/src/tracking.cpp @@ -189,19 +189,6 @@ Rect MedianFlowTracker::Track(const Mat &frame) { return Rect(); } - Mat debug_image(frame_.rows, frame_.cols * 2, frame_.type()); - Mat im1 = debug_image.colRange(Range(0, frame_.cols)); - Mat im2 = debug_image.colRange(Range(frame_.cols, frame_.cols * 2)); - frame_.copyTo(im1); - next_frame.copyTo(im2); - - for (size_t i = 0; i < corners.size(); ++i) { - line(debug_image, Point(corners.at(i)), Point(corners_next_frame.at(i)) + Point(frame_.cols, 0), CV_RGB(0, 255, 255)); - } - - imshow("debug", debug_image); - - Rect new_position; float scale_factor; if (!RestoreBoundingBox(corners, corners_next_frame, new_position, scale_factor)) { From 13aefd0392344b52c5477ec7dbc7173d72d0702a Mon Sep 17 00:00:00 2001 From: Pavel Druzhkov Date: Thu, 7 Jul 2016 19:36:44 +0400 Subject: [PATCH 4/4] code cleanup --- include/tracking.hpp | 4 ---- src/tracking.cpp | 39 +++++++++------------------------------ 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/include/tracking.hpp b/include/tracking.hpp index c8192fc..a33653d 100644 --- a/include/tracking.hpp +++ b/include/tracking.hpp @@ -44,8 +44,4 @@ class MedianFlowTracker : public Tracker { bool ComputeScaleFactor(const std::vector &corners, const std::vector &corners_next_frame, float &scale) const; - - bool RestoreBoundingBox(const std::vector &corners, - const std::vector &corners_next_frame, - cv::Rect &new_position, float& scale) const; }; diff --git a/src/tracking.cpp b/src/tracking.cpp index 8d1072e..b791312 100644 --- a/src/tracking.cpp +++ b/src/tracking.cpp @@ -47,7 +47,7 @@ bool MedianFlowTracker::FilterCorners(vector &corners, vector &corners_next_frame, vector &status, vector &errors) const { - for (int i = status.size() - 1; i >= 0; i--) { + for (int i = static_cast(status.size()) - 1; i >= 0; i--) { if (!status[i]) { status.erase(status.begin() + i); corners.erase(corners.begin() + i); @@ -62,7 +62,7 @@ bool MedianFlowTracker::FilterCorners(vector &corners, std::copy(errors.begin(), errors.end(), errors_copy.begin()); float median_error = Median(errors_copy); - for (int i = errors.size() - 1; i >= 0; i--) { + for (int i = static_cast(errors.size()) - 1; i >= 0; i--) { if (errors[i] > median_error) { errors.erase(errors.begin() + i); corners.erase(corners.begin() + i); @@ -96,7 +96,7 @@ bool MedianFlowTracker::ComputePointDistances(const vector &corners, dist.clear(); for (int i = 0; i < corners.size(); i++) { for (int j = i + 1; j < corners.size(); j++) { - dist.push_back(cv::norm(corners.at(i) - corners.at(j))); + dist.push_back(static_cast(cv::norm(corners.at(i) - corners.at(j)))); } } return true; @@ -134,27 +134,6 @@ bool MedianFlowTracker::ComputeScaleFactor( return true; } -bool MedianFlowTracker::RestoreBoundingBox( - const vector &corners, const vector &corners_next_frame, - Rect &new_position, float& scale) const { - Point2f shift; - ComputeMedianShift(corners, corners_next_frame, shift); - - if (!ComputeScaleFactor(corners, corners_next_frame, scale)) { - return false; - } - new_position.width *= scale; - new_position.height *= scale; - new_position = position_ + Point(shift); - Rect image_bounding_box(Point(0, 0), frame_.size()); - new_position = image_bounding_box & new_position; - if (new_position.area() == 0) { - return false; - } - - return true; -} - Rect MedianFlowTracker::Track(const Mat &frame) { CV_Assert(!frame.empty()); Mat object = frame_(position_); @@ -189,16 +168,16 @@ Rect MedianFlowTracker::Track(const Mat &frame) { return Rect(); } - Rect new_position; + Point2f shift; + ComputeMedianShift(corners, corners_next_frame, shift); + float scale_factor; - if (!RestoreBoundingBox(corners, corners_next_frame, new_position, scale_factor)) { - std::cout << "There are not enough points to restore bounding box." - << std::endl; + if (!ComputeScaleFactor(corners, corners_next_frame, scale_factor)) { + std::cout << "Failed to compute scale factor." << std::endl; return Rect(); } - scale_ *= scale_factor; - new_position = Rect(new_position.tl(), Size2f(initial_size_) * scale_); + Rect new_position = Rect(position_.tl() + Point(shift), Size2f(initial_size_) * scale_); Rect image_bounding_box(Point(0, 0), frame_.size()); new_position = image_bounding_box & new_position;