diff --git a/include/detection.hpp b/include/detection.hpp index 46930d3..4d064b7 100644 --- a/include/detection.hpp +++ b/include/detection.hpp @@ -4,6 +4,7 @@ #include #include "opencv2/core/core.hpp" +#include "opencv2/objdetect.hpp" class Detector { public: @@ -12,3 +13,13 @@ class Detector { virtual void Detect(const cv::Mat& frame, std::vector& objects, std::vector& scores) = 0; }; + +class CascadeDetector : public Detector { +public: + virtual bool Init(const std::string& model_file_path); + virtual void Detect(const cv::Mat& frame, std::vector& objects, + std::vector& scores); + +protected: + cv::CascadeClassifier detector; +}; diff --git a/include/image_processing.hpp b/include/image_processing.hpp index ca1f116..1a7c2a3 100644 --- a/include/image_processing.hpp +++ b/include/image_processing.hpp @@ -3,7 +3,9 @@ #include #include -#include "opencv2/core/core.hpp" +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc.hpp" class ImageProcessor { public: @@ -15,4 +17,16 @@ class ImageProcessor { const int kernelSize) = 0; virtual cv::Mat Pixelize(const cv::Mat &src, const cv::Rect &roi, const int kDivs) = 0; +}; + +class ImageProcessorImpl { + public: + virtual cv::Mat CvtColor(const cv::Mat &src, const cv::Rect &roi); + virtual cv::Mat Filter(const cv::Mat &src, const cv::Rect &roi, + const int kSize); + virtual cv::Mat DetectEdges(const cv::Mat &src, const cv::Rect &roi, + const int filterSize, const int lowThreshold, const int ratio, + const int kernelSize); + virtual cv::Mat Pixelize(const cv::Mat &src, const cv::Rect &roi, + const int kDivs); }; \ No newline at end of file diff --git a/include/tracking.hpp b/include/tracking.hpp index f80004c..7533fb6 100644 --- a/include/tracking.hpp +++ b/include/tracking.hpp @@ -11,3 +11,14 @@ class Tracker { virtual bool Init(const cv::Mat &frame, const cv::Rect &roi) = 0; virtual cv::Rect Track(const cv::Mat &frame) = 0; }; + +class MedianFlowTracker : public Tracker { +public: + virtual bool Init(const cv::Mat &frame, const cv::Rect &roi); + virtual cv::Rect Track(const cv::Mat &frame); + +protected: + cv::Rect position_; + cv::Mat frame_; +}; + diff --git a/samples/detection_demo.cpp b/samples/detection_demo.cpp new file mode 100644 index 0000000..dc72770 --- /dev/null +++ b/samples/detection_demo.cpp @@ -0,0 +1,80 @@ +#include +#include + +#include "opencv2/core.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "detection.hpp" + +using namespace std; +using namespace cv; + +const char* kAbout = + "This is an empty application that can be treated as a template for your " + "own doing-something-cool applications."; + +const char* kOptions = + "{ i image | | image to process }" + "{ v video | | video to process }" + "{ c camera | | camera to get video from }" + "{ m model | | detector model }" + "{ h ? help usage | | print help message }"; + +void processImage(string & windowName, Mat & frame, CascadeDetector & detector) { + vector objects; + vector scores; + detector.Detect(frame, objects, scores); + Mat frame_copy = frame.clone(); + for each (Rect object in objects) + { + rectangle(frame_copy, object, Scalar(255, 0, 0)); + } + imshow(windowName, frame_copy); + waitKey(1); +} + +int main(int argc, const char** argv) { + // Parse command line arguments. + CommandLineParser parser(argc, argv, kOptions); + parser.about(kAbout); + + // If help option is given, print help message and exit. + if (parser.get("help")) { + parser.printMessage(); + return 0; + } + + if (!parser.has("model")) { + cerr << "Specify detector model" << endl; + return -1; + } + + string mfilepath = parser.get("model"); + + CascadeDetector detector; + detector.CreateDetector("cascade"); + if (!detector.Init(mfilepath)) { + cerr << "Can't load cascade!" << endl; + return -1; + } + + string windowName("Detected objects"); + + if (parser.has("video")) { + string vfilepath = parser.get("video"); + VideoCapture cap(vfilepath); + Mat frame; + if(!cap.isOpened()) { + cerr << "Can't open video capture!" << parser.get("video") << endl; + return -1; + } + cap >> frame; + while (frame.cols * frame.rows) + { + processImage(windowName, frame, detector); + cap >> frame; + } + } + + return 0; +} diff --git a/samples/imgproc_demo.cpp b/samples/imgproc_demo.cpp new file mode 100644 index 0000000..2009412 --- /dev/null +++ b/samples/imgproc_demo.cpp @@ -0,0 +1,112 @@ +#include +#include + +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" + +using namespace std; +using namespace cv; + +#include "image_processing.hpp" + + +struct MouseCallbackState { + bool is_selection_started; + bool is_selection_finished; + Point point_first; + Point point_second; + + MouseCallbackState() : is_selection_started(false), is_selection_finished(false) {} +}; + +void CallBackFunc(int event, int x, int y, int flags, void* userdata) +{ + MouseCallbackState * p_mc_state = (MouseCallbackState *)userdata; + if (event == EVENT_LBUTTONDOWN) + { + p_mc_state->point_first = Point(x, y); + p_mc_state->is_selection_started = true; + } + else if (event == EVENT_LBUTTONUP) + { + p_mc_state->is_selection_finished = true; + } + p_mc_state->point_second = Point(x, y); +} + +const char* kAbout = + "This is image processing test application"; + +const char* kOptions = + "{ @image | | image to process }" + "{ gray | | convert ROI to gray scale }" + "{ median | | apply median filter for ROI }" + "{ edges | | detect edges in ROI }" + "{ pix | | pixelize ROI }" + "{ h ? help usage | | print help message }"; + +int main(int argc, const char** argv) { + // Parse command line arguments. + CommandLineParser parser(argc, argv, kOptions); + parser.about(kAbout); + + // If help option is given, print help message and exit. + if (parser.has("help")) { + parser.printMessage(); + return 0; + } + + // Read image. + Mat src = imread(parser.get(0)); + if (src.empty()) { + cout << "Failed to open image file '" + parser.get(0) + "'." + << endl; + return 0; + } + + //set the callback function for any mouse event + // Show source image. + const string kSrcWindowName = "Source image"; + imshow(kSrcWindowName, src); + MouseCallbackState mc_state; + setMouseCallback(kSrcWindowName, CallBackFunc, (void *)&mc_state); + + Rect roi; + + while (!mc_state.is_selection_finished) { + if (mc_state.is_selection_started) { + Mat src_copy = src.clone(); + roi = Rect(mc_state.point_first, mc_state.point_second); + rectangle(src_copy, roi, Scalar(255, 0, 0)); + imshow(kSrcWindowName, src_copy); + } + waitKey(30); + } + + ImageProcessorImpl proc; + Mat res = src.clone(); + if (parser.has("gray")) { + res = proc.CvtColor(src, roi); + } + else if (parser.has("median")) { + res = proc.Filter(src, roi, 11); + } + else if (parser.has("edges")) { + int filterSize = 5; + int lowThreshold = 50; + int ratio = 3; + int kernelSize = 5; + res = proc.DetectEdges(src, roi, + filterSize, lowThreshold, ratio, kernelSize); + } + else if (parser.has("pix")) { + res = proc.Pixelize(src, roi, 5); + } + + const string kResWindowName = "Processed image"; + imshow(kResWindowName, res); + const int kWaitKeyDelay = 0; + waitKey(kWaitKeyDelay); + + return 0; +} diff --git a/samples/tracking_demo.cpp b/samples/tracking_demo.cpp new file mode 100644 index 0000000..2723498 --- /dev/null +++ b/samples/tracking_demo.cpp @@ -0,0 +1,99 @@ +#include +#include + +#include "opencv2/core.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc.hpp" +#include "tracking.hpp" + +using namespace std; +using namespace cv; + +const char* kAbout = + "This is an empty application that can be treated as a template for your " + "own doing-something-cool applications."; + +const char* kOptions = + "{ v video | | video to process }" + "{ h ? help usage | | print help message }"; + +struct MouseCallbackState { + bool is_selection_started; + bool is_selection_finished; + Point point_first; + Point point_second; + + MouseCallbackState() : is_selection_started(false), is_selection_finished(false) {} +}; + +void CallBackFunc(int event, int x, int y, int flags, void* userdata) +{ + MouseCallbackState * p_mc_state = (MouseCallbackState *)userdata; + if (event == EVENT_LBUTTONDOWN) + { + p_mc_state->point_first = Point(x, y); + p_mc_state->is_selection_started = true; + } + else if (event == EVENT_LBUTTONUP) + { + p_mc_state->is_selection_finished = true; + } + p_mc_state->point_second = Point(x, y); +} + +int main(int argc, const char** argv) { + // Parse command line arguments. + CommandLineParser parser(argc, argv, kOptions); + parser.about(kAbout); + + // If help option is given, print help message and exit. + if (parser.get("help")) { + parser.printMessage(); + return 0; + } + + if (parser.has("video")) { + string vfilepath = parser.get("video"); + VideoCapture cap(vfilepath); + Mat frame; + if (!cap.isOpened()) { + cerr << "Can't open video capture!" << parser.get("video") << endl; + return -1; + } + cap >> frame; + + const string kSrcWindowName = "Tracking object"; + imshow(kSrcWindowName, frame); + MouseCallbackState mc_state; + setMouseCallback(kSrcWindowName, CallBackFunc, (void *)&mc_state); + Rect roi; + + while (!mc_state.is_selection_finished) { + if (mc_state.is_selection_started) { + Mat frame_copy = frame.clone(); + roi = Rect(mc_state.point_first, mc_state.point_second); + rectangle(frame_copy, roi, Scalar(255, 0, 0)); + imshow(kSrcWindowName, frame_copy); + } + waitKey(30); + } + + MedianFlowTracker tracker; + tracker.Init(frame, roi); + while (frame.cols * frame.rows) + { + Rect obj = tracker.Track(frame); + Mat tracking_frame = frame.clone(); + if (! (obj.width * obj.height)) { + cout << "Tracking lost!" << endl; + break; + } + rectangle(tracking_frame, obj, Scalar(255, 0, 0)); + imshow("Tracking object", tracking_frame); + waitKey(30); + cap >> frame; + } + } + + return 0; +} diff --git a/src/detection.cpp b/src/detection.cpp index 15e7fd1..32e1fb1 100644 --- a/src/detection.cpp +++ b/src/detection.cpp @@ -4,10 +4,32 @@ using std::string; using std::shared_ptr; +using std::vector; using namespace cv; shared_ptr Detector::CreateDetector(const string& name) { - std::cerr << "Failed to create detector with name '" << name << "'" - << std::endl; - return nullptr; + if (name == "cascade") { + return std::make_shared(); + } + else { + std::cerr << "Failed to create detector with name '" << name << "'" + << std::endl; + return nullptr; + } } + +bool CascadeDetector::Init(const std::string& model_file_path) { + return detector.load(model_file_path); +} + +void CascadeDetector::Detect(const cv::Mat& frame, std::vector& objects, + std::vector& scores) { + if (!detector.empty()) { + vector dscores; + detector.detectMultiScale(frame, objects, dscores); + scores.resize(dscores.size()); + for (int i = 0; i < dscores.size(); i++) { + scores[i] = (double)dscores[i]; + } + } +} \ No newline at end of file diff --git a/src/image_processing.cpp b/src/image_processing.cpp new file mode 100644 index 0000000..857ac1a --- /dev/null +++ b/src/image_processing.cpp @@ -0,0 +1,59 @@ +#pragma once +#include "image_processing.hpp" + +using namespace std; +using namespace cv; + +Mat ImageProcessorImpl::CvtColor(const cv::Mat &src, const cv::Rect &roi) { + Mat src_copy = src.clone(); + Mat src_copy_roi = src_copy(roi); + Mat dst_gray_roi; + cvtColor(src_copy_roi, dst_gray_roi, CV_BGR2GRAY); + vector channels(src.channels(), dst_gray_roi); + Mat dst_roi; + merge(channels, dst_roi); + dst_roi.copyTo(src_copy_roi); + return src_copy; +} + +Mat ImageProcessorImpl::Filter(const cv::Mat &src, const cv::Rect &roi, + const int kSize) { + Mat src_copy = src.clone(); + Mat src_copy_roi = src_copy(roi); + medianBlur(src_copy_roi, src_copy_roi, kSize); + return src_copy; +} + +Mat ImageProcessorImpl::DetectEdges(const cv::Mat &src, const cv::Rect &roi, + const int filterSize, const int lowThreshold, const int ratio, + const int kernelSize) { + Mat src_roi = src.clone()(roi); + Mat src_gray_roi; + cvtColor(src_roi, src_gray_roi, CV_BGR2GRAY); + Mat gray_blurred; + blur(src_gray_roi, gray_blurred, Size(filterSize, filterSize)); + Mat detected_edges; + Canny(gray_blurred, detected_edges, lowThreshold, lowThreshold * ratio); + Mat dst = src.clone(); + Mat dst_roi = dst(roi); + dst_roi.setTo(Scalar::all(0)); + src_roi.copyTo(dst_roi, detected_edges); + return dst; +} + +Mat ImageProcessorImpl::Pixelize(const cv::Mat &src, const cv::Rect &roi, + const int kDivs) { + Mat src_copy = src.clone(); + Mat src_copy_roi = src_copy(roi); + int block_size_x = roi.width / kDivs; + int block_size_y = roi.height / kDivs; + for (int i = 0; i < kDivs; i++) { + for (int j = 0; j < kDivs; j ++) { + int x = i*block_size_x; + int y = j*block_size_y; + Mat src_roi_block = src_copy_roi(Rect(x, y, block_size_x, block_size_y)); + blur(src_roi_block, src_roi_block, Size(block_size_x, block_size_y)); + } + } + return src_copy; +} \ No newline at end of file diff --git a/src/tracking.cpp b/src/tracking.cpp index 26cbe06..96ea66d 100644 --- a/src/tracking.cpp +++ b/src/tracking.cpp @@ -4,10 +4,134 @@ using std::string; using std::shared_ptr; +using std::vector; using namespace cv; +#include "opencv2/features2d.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/video.hpp" + +template bool eraseElements(vector & data, vector & status) { + int erase_index = 0; + for (int i = 0; i < data.size(); i++) { + if (!status[i]) { + data.erase(data.begin() + erase_index); + } + else { + erase_index++; + } + } + return data.size() > 0; +}; + +template T getMedian(vector & data) { + std::nth_element(data.begin(), data.begin() + data.size() / 2, data.end()); + return data[data.size() / 2]; +} + +Rect rescaleRect(Rect& rect, float scale) { + Rect res = rect; + int new_width = res.width / scale + 0.5; + int new_height = res.height / scale + 0.5; + res.x += (res.width - new_width) / 2; + res.y += (res.height - new_height) / 2; + res.width = new_width; + res.height = new_height; + return res; +} + shared_ptr Tracker::CreateTracker(const string &name) { - std::cerr << "Failed to create tracker with name '" << name << "'" - << std::endl; + if (name == "median_flow") { + return std::make_shared(); + } + else { + std::cerr << "Failed to create tracker with name '" << name << "'" + << std::endl; + } return nullptr; } + + +bool MedianFlowTracker::Init(const cv::Mat &frame, const cv::Rect &roi) { + frame_ = frame; + position_ = roi; + return frame.cols && frame.rows && roi.width && roi.height; +} + +Rect MedianFlowTracker::Track(const Mat &frame) { + vector corners; + const int maxCorners = 50; + const double qualityLevel = 0.01; + const double minDistance = 5.0; + const int minCorners = 5; + + Mat mask = Mat::zeros(frame.size(), CV_8UC1); + rectangle(mask, position_, 255, -1); + Mat gray_frame_; + cvtColor(frame_, gray_frame_, CV_BGR2GRAY); + + goodFeaturesToTrack(gray_frame_, corners, maxCorners, qualityLevel, minDistance, mask); + if (corners.size() < minCorners) return Rect(); + + Mat gray_frame; + cvtColor(frame, gray_frame, CV_BGR2GRAY); + + vector corners_forward, corners_backward; + vector status_forward, status_backward; + Mat error_forward, error_backward; + + calcOpticalFlowPyrLK(gray_frame_, gray_frame, corners, corners_forward, status_forward, error_forward); + eraseElements(corners, status_forward); + eraseElements(corners_forward, status_forward); + if (corners.size() < minCorners) return Rect(); + + calcOpticalFlowPyrLK(gray_frame, gray_frame_, corners_forward, corners_backward, status_backward, error_backward); + eraseElements(corners, status_backward); + eraseElements(corners_forward, status_backward); + eraseElements(corners_backward, status_backward); + if (corners.size() < minCorners) return Rect(); + + vector fb_error(corners.size()); + for (int i = 0; i < fb_error.size(); i++) { + fb_error[i] = norm(corners[i] - corners_backward[i]); + } + float median_error = getMedian(fb_error); + + vector error_status(fb_error.size()); + for (int i = 0; i < error_status.size(); i++) { + error_status[i] = fb_error[i] > median_error ? 0 : 1; + } + eraseElements(corners, error_status); + eraseElements(corners_forward, error_status); + eraseElements(corners_backward, error_status); + if (corners.size() < minCorners) return Rect(); + + vector shifts_x(corners.size()); + vector shifts_y(corners.size()); + for (int i = 0; i < corners.size(); i++) { + shifts_x[i] = (corners_forward[i].x - corners[i].x); + shifts_y[i] = (corners_forward[i].y - corners[i].y); + } + + position_.x += getMedian(shifts_x) + 0.5; + position_.y += getMedian(shifts_y) + 0.5; + + frame_ = frame.clone(); + + vector scales; + float old_distance(0), new_distance(0); + + for (int i = 0; i < corners.size(); i++) { + for (int j = i+1; j < corners.size(); j++) { + old_distance = norm(corners[i] - corners[j]); + new_distance = norm(corners_forward[i] - corners_forward[j]); + scales.push_back(old_distance / new_distance); + } + } + float median_scale = getMedian(scales); + + position_ = rescaleRect(position_, median_scale); + + return position_; +} diff --git a/src/workaround.cpp b/src/workaround.cpp index 2eba458..caeba3d 100644 --- a/src/workaround.cpp +++ b/src/workaround.cpp @@ -6,5 +6,8 @@ using namespace std; void MatrixProcessor::Threshold(unsigned char* const data, const int width, const int height, const int threshold) { - // TODO: Add thresholding logic here. + // TODO: Add thresholding logic here. + for (int i = 0; i < width*height; i++) { + data[i] = data[i] > threshold ? 255 : 0; + } }