Skip to content

Commit 831523c

Browse files
sbokovsbokov
committed
Sparse-to-dense OF algorithm is added
* Sparse match interpolator interface and EdgeAwareInterpolator were added to the ximgproc module * New optical flow algorithm, based on PyrLK sparse OF and sparse match interpolation, is added to the optflow module
1 parent 6a0545e commit 831523c

File tree

11 files changed

+1543
-3
lines changed

11 files changed

+1543
-3
lines changed

‎modules/optflow/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
set(the_description "Optical Flow Algorithms")
2-
ocv_define_module(optflow opencv_core opencv_imgproc opencv_video opencv_highgui WRAP python)
2+
ocv_define_module(optflow opencv_core opencv_imgproc opencv_video opencv_highgui opencv_ximgproc WRAP python)

‎modules/optflow/include/opencv2/optflow.hpp

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,30 @@ CV_EXPORTS_W void calcOpticalFlowSF( InputArray from, InputArray to, OutputArray
109109
double sigma_dist, double sigma_color, int postprocess_window,
110110
double sigma_dist_fix, double sigma_color_fix, double occ_thr,
111111
int upscale_averaging_radius, double upscale_sigma_dist,
112-
double upscale_sigma_color, double speed_up_thr );
112+
double upscale_sigma_color, double speed_up_thr );
113+
114+
/** @brief Fast dense optical flow based on PyrLK sparse matches interpolation.
115+
116+
@param from first 8-bit 3-channel or 1-channel image.
117+
@param to second 8-bit 3-channel or 1-channel image of the same size as from
118+
@param flow computed flow image that has the same size as from and CV_32FC2 type
119+
@param grid_step stride used in sparse match computation. Lower values usually
120+
result in higher quality but slow down the algorithm.
121+
@param k number of nearest-neighbor matches considered, when fitting a locally affine
122+
model. Lower values can make the algorithm noticeably faster at the cost of
123+
some quality degradation.
124+
@param sigma parameter defining how fast the weights decrease in the locally-weighted affine
125+
fitting. Higher values can help preserve fine details, lower values can help to get rid
126+
of the noise in the output flow.
127+
@param use_post_proc defines whether the ximgproc::fastGlobalSmootherFilter() is used
128+
for post-processing after interpolation
129+
@param fgs_lambda see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
130+
@param fgs_sigma see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
131+
*/
132+
CV_EXPORTS_W void calcOpticalFlowSparseToDense ( InputArray from, InputArray to, OutputArray flow,
133+
int grid_step = 8, int k = 128, float sigma = 0.05f,
134+
bool use_post_proc = true, float fgs_lambda = 500.0f,
135+
float fgs_sigma = 1.5f );
113136

114137
/** @brief Read a .flo file
115138
@@ -165,6 +188,9 @@ CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SimpleFlow();
165188
//! Additional interface to the Farneback's algorithm - calcOpticalFlowFarneback()
166189
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_Farneback();
167190

191+
//! Additional interface to the SparseToDenseFlow algorithm - calcOpticalFlowSparseToDense()
192+
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SparseToDense();
193+
168194
//! @}
169195

170196
} //optflow

‎modules/optflow/samples/optical_flow_evaluation.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ using namespace optflow;
1010
const String keys = "{help h usage ? | | print this message }"
1111
"{@image1 | | image1 }"
1212
"{@image2 | | image2 }"
13-
"{@algorithm | | [farneback, simpleflow, tvl1 or deepflow] }"
13+
"{@algorithm | | [farneback, simpleflow, tvl1, deepflow or sparsetodenseflow] }"
1414
"{@groundtruth | | path to the .flo file (optional), Middlebury format }"
1515
"{m measure |endpoint| error measure - [endpoint or angular] }"
1616
"{r region |all | region to compute stats about [all, discontinuities, untextured] }"
@@ -249,6 +249,8 @@ int main( int argc, char** argv )
249249
algorithm = createOptFlow_DualTVL1();
250250
else if ( method == "deepflow" )
251251
algorithm = createOptFlow_DeepFlow();
252+
else if ( method == "sparsetodenseflow" )
253+
algorithm = createOptFlow_SparseToDense();
252254
else
253255
{
254256
printf("Wrong method!\n");

‎modules/optflow/src/interfaces.cpp

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,5 +177,42 @@ Ptr<DenseOpticalFlow> createOptFlow_Farneback()
177177
{
178178
return makePtr<OpticalFlowFarneback>();
179179
}
180+
181+
class OpticalFlowSparseToDense : public DenseOpticalFlow
182+
{
183+
public:
184+
OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma);
185+
void calc(InputArray I0, InputArray I1, InputOutputArray flow);
186+
void collectGarbage();
187+
protected:
188+
int grid_step;
189+
int k;
190+
float sigma;
191+
bool use_post_proc;
192+
float fgs_lambda;
193+
float fgs_sigma;
194+
};
195+
196+
OpticalFlowSparseToDense::OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma)
197+
{
198+
grid_step = _grid_step;
199+
k = _k;
200+
sigma = _sigma;
201+
use_post_proc = _use_post_proc;
202+
fgs_lambda = _fgs_lambda;
203+
fgs_sigma = _fgs_sigma;
204+
}
205+
206+
void OpticalFlowSparseToDense::calc(InputArray I0, InputArray I1, InputOutputArray flow)
207+
{
208+
calcOpticalFlowSparseToDense(I0,I1,flow,grid_step,k,sigma,use_post_proc,fgs_lambda,fgs_sigma);
209+
}
210+
211+
void OpticalFlowSparseToDense::collectGarbage() {}
212+
213+
Ptr<DenseOpticalFlow> createOptFlow_SparseToDense()
214+
{
215+
return makePtr<OpticalFlowSparseToDense>(8,128,0.05f,true,500.0f,1.5f);
216+
}
180217
}
181218
}

‎modules/optflow/src/sparsetodenseflow.cpp

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*M///////////////////////////////////////////////////////////////////////////////////////
2+
//
3+
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4+
//
5+
// By downloading, copying, installing or using the software you agree to this license.
6+
// If you do not agree to this license, do not download, install,
7+
// copy or use the software.
8+
//
9+
//
10+
// License Agreement
11+
// For Open Source Computer Vision Library
12+
//
13+
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
14+
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
15+
// Third party copyrights are property of their respective owners.
16+
//
17+
// Redistribution and use in source and binary forms, with or without modification,
18+
// are permitted provided that the following conditions are met:
19+
//
20+
// * Redistribution's of source code must retain the above copyright notice,
21+
// this list of conditions and the following disclaimer.
22+
//
23+
// * Redistribution's in binary form must reproduce the above copyright notice,
24+
// this list of conditions and the following disclaimer in the documentation
25+
// and/or other materials provided with the distribution.
26+
//
27+
// * The name of the copyright holders may not be used to endorse or promote products
28+
// derived from this software without specific prior written permission.
29+
//
30+
// This software is provided by the copyright holders and contributors "as is" and
31+
// any express or implied warranties, including, but not limited to, the implied
32+
// warranties of merchantability and fitness for a particular purpose are disclaimed.
33+
// In no event shall the Intel Corporation or contributors be liable for any direct,
34+
// indirect, incidental, special, exemplary, or consequential damages
35+
// (including, but not limited to, procurement of substitute goods or services;
36+
// loss of use, data, or profits; or business interruption) however caused
37+
// and on any theory of liability, whether in contract, strict liability,
38+
// or tort (including negligence or otherwise) arising in any way out of
39+
// the use of this software, even if advised of the possibility of such damage.
40+
//
41+
//M*/
42+
43+
#include "precomp.hpp"
44+
#include "opencv2/ximgproc/sparse_match_interpolator.hpp"
45+
using namespace std;
46+
47+
namespace cv {
48+
namespace optflow {
49+
50+
CV_EXPORTS_W void calcOpticalFlowSparseToDense(InputArray from, InputArray to, OutputArray flow,
51+
int grid_step, int k,
52+
float sigma, bool use_post_proc,
53+
float fgs_lambda, float fgs_sigma)
54+
{
55+
CV_Assert( grid_step>1 && k>3 && sigma>0.0001f && fgs_lambda>1.0f && fgs_sigma>0.01f );
56+
CV_Assert( !from.empty() && from.depth() == CV_8U && (from.channels() == 3 || from.channels() == 1) );
57+
CV_Assert( !to .empty() && to .depth() == CV_8U && (to .channels() == 3 || to .channels() == 1) );
58+
CV_Assert( from.sameSize(to) );
59+
60+
Mat prev = from.getMat();
61+
Mat cur = to.getMat();
62+
Mat prev_grayscale, cur_grayscale;
63+
64+
while( (prev.cols/grid_step)*(prev.rows/grid_step) > SHRT_MAX ) //ensure that the number matches is not too big
65+
grid_step*=2;
66+
67+
if(prev.channels()==3)
68+
{
69+
cvtColor(prev,prev_grayscale,COLOR_BGR2GRAY);
70+
cvtColor(cur, cur_grayscale, COLOR_BGR2GRAY);
71+
}
72+
else
73+
{
74+
prev.copyTo(prev_grayscale);
75+
cur .copyTo(cur_grayscale);
76+
}
77+
78+
vector<Point2f> points;
79+
vector<Point2f> dst_points;
80+
vector<unsigned char> status;
81+
vector<float> err;
82+
vector<Point2f> points_filtered, dst_points_filtered;
83+
84+
for(int i=0;i<prev.rows;i+=grid_step)
85+
for(int j=0;j<prev.cols;j+=grid_step)
86+
points.push_back(Point2f((float)j,(float)i));
87+
88+
calcOpticalFlowPyrLK(prev_grayscale,cur_grayscale,points,dst_points,status,err,Size(21,21));
89+
90+
for(unsigned int i=0;i<points.size();i++)
91+
{
92+
if(status[i]!=0)
93+
{
94+
points_filtered.push_back(points[i]);
95+
dst_points_filtered.push_back(dst_points[i]);
96+
}
97+
}
98+
99+
flow.create(from.size(),CV_32FC2);
100+
Mat dense_flow = flow.getMat();
101+
102+
Ptr<ximgproc::EdgeAwareInterpolator> gd = ximgproc::createEdgeAwareInterpolator();
103+
gd->setK(k);
104+
gd->setSigma(sigma);
105+
gd->setUsePostProcessing(use_post_proc);
106+
gd->setFGSLambda(fgs_lambda);
107+
gd->setFGSSigma (fgs_sigma);
108+
gd->interpolate(prev,points_filtered,cur,dst_points_filtered,dense_flow);
109+
}
110+
111+
}
112+
}

‎modules/optflow/test/test_sparsetodenseflow.cpp

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*M///////////////////////////////////////////////////////////////////////////////////////
2+
//
3+
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4+
//
5+
// By downloading, copying, installing or using the software you agree to this license.
6+
// If you do not agree to this license, do not download, install,
7+
// copy or use the software.
8+
//
9+
//
10+
// Intel License Agreement
11+
// For Open Source Computer Vision Library
12+
//
13+
// Copyright (C) 2000, Intel Corporation, all rights reserved.
14+
// Third party copyrights are property of their respective owners.
15+
//
16+
// Redistribution and use in source and binary forms, with or without modification,
17+
// are permitted provided that the following conditions are met:
18+
//
19+
// * Redistribution's of source code must retain the above copyright notice,
20+
// this list of conditions and the following disclaimer.
21+
//
22+
// * Redistribution's in binary form must reproduce the above copyright notice,
23+
// this list of conditions and the following disclaimer in the documentation
24+
// and/or other materials provided with the distribution.
25+
//
26+
// * The name of Intel Corporation may not be used to endorse or promote products
27+
// derived from this software without specific prior written permission.
28+
//
29+
// This software is provided by the copyright holders and contributors "as is" and
30+
// any express or implied warranties, including, but not limited to, the implied
31+
// warranties of merchantability and fitness for a particular purpose are disclaimed.
32+
// In no event shall the Intel Corporation or contributors be liable for any direct,
33+
// indirect, incidental, special, exemplary, or consequential damages
34+
// (including, but not limited to, procurement of substitute goods or services;
35+
// loss of use, data, or profits; or business interruption) however caused
36+
// and on any theory of liability, whether in contract, strict liability,
37+
// or tort (including negligence or otherwise) arising in any way out of
38+
// the use of this software, even if advised of the possibility of such damage.
39+
//
40+
//M*/
41+
42+
#include "test_precomp.hpp"
43+
44+
#include <string>
45+
46+
using namespace std;
47+
using namespace cv;
48+
49+
/* ///////////////////// sparsetodenseflow_test ///////////////////////// */
50+
51+
class CV_SparseToDenseFlowTest : public cvtest::BaseTest
52+
{
53+
protected:
54+
void run(int);
55+
};
56+
57+
static bool isFlowCorrect(float u) {
58+
return !cvIsNaN(u) && (fabs(u) < 1e9);
59+
}
60+
61+
static float calc_rmse(Mat flow1, Mat flow2) {
62+
float sum = 0;
63+
int counter = 0;
64+
const int rows = flow1.rows;
65+
const int cols = flow1.cols;
66+
67+
for (int y = 0; y < rows; ++y) {
68+
for (int x = 0; x < cols; ++x) {
69+
Vec2f flow1_at_point = flow1.at<Vec2f>(y, x);
70+
Vec2f flow2_at_point = flow2.at<Vec2f>(y, x);
71+
72+
float u1 = flow1_at_point[0];
73+
float v1 = flow1_at_point[1];
74+
float u2 = flow2_at_point[0];
75+
float v2 = flow2_at_point[1];
76+
77+
if (isFlowCorrect(u1) && isFlowCorrect(u2) && isFlowCorrect(v1) && isFlowCorrect(v2)) {
78+
sum += (u1-u2)*(u1-u2) + (v1-v2)*(v1-v2);
79+
counter++;
80+
}
81+
}
82+
}
83+
return (float)sqrt(sum / (1e-9 + counter));
84+
}
85+
86+
void CV_SparseToDenseFlowTest::run(int) {
87+
const float MAX_RMSE = 0.6f;
88+
const string frame1_path = ts->get_data_path() + "optflow/RubberWhale1.png";
89+
const string frame2_path = ts->get_data_path() + "optflow/RubberWhale2.png";
90+
const string gt_flow_path = ts->get_data_path() + "optflow/RubberWhale.flo";
91+
92+
Mat frame1 = imread(frame1_path);
93+
Mat frame2 = imread(frame2_path);
94+
95+
if (frame1.empty()) {
96+
ts->printf(cvtest::TS::LOG, "could not read image %s\n", frame2_path.c_str());
97+
ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
98+
return;
99+
}
100+
101+
if (frame2.empty()) {
102+
ts->printf(cvtest::TS::LOG, "could not read image %s\n", frame2_path.c_str());
103+
ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
104+
return;
105+
}
106+
107+
if (frame1.rows != frame2.rows && frame1.cols != frame2.cols) {
108+
ts->printf(cvtest::TS::LOG, "images should be of equal sizes (%s and %s)",
109+
frame1_path.c_str(), frame2_path.c_str());
110+
ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
111+
return;
112+
}
113+
114+
if (frame1.type() != 16 || frame2.type() != 16) {
115+
ts->printf(cvtest::TS::LOG, "images should be of equal type CV_8UC3 (%s and %s)",
116+
frame1_path.c_str(), frame2_path.c_str());
117+
ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
118+
return;
119+
}
120+
121+
Mat flow_gt = optflow::readOpticalFlow(gt_flow_path);
122+
if(flow_gt.empty()) {
123+
ts->printf(cvtest::TS::LOG, "error while reading flow data from file %s",
124+
gt_flow_path.c_str());
125+
ts->set_failed_test_info(cvtest::TS::FAIL_MISSING_TEST_DATA);
126+
return;
127+
}
128+
129+
Mat flow;
130+
optflow::calcOpticalFlowSparseToDense(frame1, frame2, flow);
131+
132+
float rmse = calc_rmse(flow_gt, flow);
133+
134+
ts->printf(cvtest::TS::LOG, "Optical flow estimation RMSE for SparseToDenseFlow algorithm : %lf\n",
135+
rmse);
136+
137+
if (rmse > MAX_RMSE) {
138+
ts->printf( cvtest::TS::LOG,
139+
"Too big rmse error : %lf ( >= %lf )\n", rmse, MAX_RMSE);
140+
ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY);
141+
return;
142+
}
143+
}
144+
145+
146+
TEST(Video_OpticalFlowSparseToDenseFlow, accuracy) { CV_SparseToDenseFlowTest test; test.safe_run(); }

‎modules/ximgproc/doc/ximgproc.bib

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,11 @@ @inproceedings{Farbman2008
7777
year={2008},
7878
organization={ACM}
7979
}
80+
81+
@inproceedings{Revaud2015,
82+
title={EpicFlow: Edge-Preserving Interpolation of Correspondences for Optical Flow},
83+
author={Revaud, Jerome and Weinzaepfel, Philippe and Harchaoui, Zaid and Schmid, Cordelia},
84+
booktitle={Computer Vision and Pattern Recognition (CVPR), IEEE Conference on},
85+
pages={1164--1172},
86+
year={2015}
87+
}

‎modules/ximgproc/include/opencv2/ximgproc.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939

4040
#include "ximgproc/edge_filter.hpp"
4141
#include "ximgproc/disparity_filter.hpp"
42+
#include "ximgproc/sparse_match_interpolator.hpp"
4243
#include "ximgproc/structured_edge_detection.hpp"
4344
#include "ximgproc/seeds.hpp"
4445

0 commit comments

Comments
 (0)