Improving DisparityWLSFilter interface and adding a tutorial

Now the filter natively supports StereoBM and StereoSGBM with no
parameter tuning required. Also, now user won't need to set the ROI and
the right matcher parameters manually, it is all done in the respective
convenience factory method based on the left matcher instance. Tutorial
was added to clarify the provided example of use.

Author: sbokov

modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp

@@ -39,6 +39,7 @@
 #ifdef __cplusplus
 
 #include <opencv2/core.hpp>
+#include <opencv2/calib3d.hpp>
 
 namespace cv {
 namespace ximgproc {
@@ -63,15 +64,15 @@ class CV_EXPORTS_W DisparityFilter : public Algorithm
 
     @param filtered_disparity_map output disparity map.
 
-    @param ROI region of the disparity map to filter.
-
     @param disparity_map_right optional argument, some implementations might also use the disparity map
     of the right view to compute confidence maps, for instance.
 
+    @param ROI region of the disparity map to filter. Optional, usually it should be set automatically.
+
     @param right_view optional argument, some implementations might also use the right view of the original
     stereo-pair.
      */
-    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right = Mat(), InputArray right_view = Mat()) = 0;
+    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right = Mat(), Rect ROI = Rect(), InputArray right_view = Mat()) = 0;
 };
 
 /** @brief Disparity map filter based on Weighted Least Squares filter (in form of Fast Global Smoother that
@@ -106,8 +107,7 @@ class CV_EXPORTS_W DisparityWLSFilter : public DisparityFilter
     /** @see getLRCthresh */
     CV_WRAP virtual void setLRCthresh(int _LRC_thresh) = 0;
     /** @brief DepthDiscontinuityRadius is a parameter used in confidence computation. It defines the size of
-    low-confidence regions around depth discontinuities. For typical window sizes used in stereo matching the
-    optimal value is around 5.
+    low-confidence regions around depth discontinuities.
      */
     CV_WRAP virtual int getDepthDiscontinuityRadius() = 0;
     /** @see getDepthDiscontinuityRadius */
@@ -117,16 +117,36 @@ class CV_EXPORTS_W DisparityWLSFilter : public DisparityFilter
     correct disparity values with a high degree of confidence).
      */
     CV_WRAP virtual Mat getConfidenceMap() = 0;
-
+    /** @brief Get the ROI used in the last filter call
+     */
+    CV_WRAP virtual Rect getROI() = 0;
 };
 
-/** @brief Factory method, create instance of DisparityWLSFilter and execute the initialization routines.
+/** @brief Convenience factory method that creates an instance of DisparityWLSFilter and sets up all the relevant
+filter parameters automatically based on the matcher instance. Currently supports only StereoBM and StereoSGBM.
+
+@param matcher_left stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<DisparityWLSFilter> createDisparityWLSFilter(Ptr<StereoMatcher> matcher_left);
+
+/** @brief Convenience method to set up the matcher for computing the right-view disparity map
+that is required in case of filtering with confidence.
+
+@param matcher_left main stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<StereoMatcher> createRightMatcher(Ptr<StereoMatcher> matcher_left);
+
+/** @brief More generic factory method, create instance of DisparityWLSFilter and execute basic
+initialization routines. When using this method you will need to set-up the ROI, matchers and
+other parameters by yourself.
 
 @param use_confidence filtering with confidence requires two disparity maps (for the left and right views) and is
 approximately two times slower. However, quality is typically significantly better.
 */
 CV_EXPORTS_W
-Ptr<DisparityWLSFilter> createDisparityWLSFilter(bool use_confidence);
+Ptr<DisparityWLSFilter> createDisparityWLSFilterGeneric(bool use_confidence);
 
 //////////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////

modules/ximgproc/perf/perf_disparity_wls_filter.cpp

@@ -85,8 +85,8 @@ PERF_TEST_P( DisparityWLSFilterPerfTest, perf, Combine(GuideTypes::all(), SrcTyp
     cv::setNumThreads(cv::getNumberOfCPUs());
     TEST_CYCLE_N(10)
     {
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(use_conf);
-        wls_filter->filter(disp_left,guide,dst,ROI,disp_right);
+        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilterGeneric(use_conf);
+        wls_filter->filter(disp_left,guide,dst,disp_right,ROI);
     }
 
     SANITY_CHECK(dst);

modules/ximgproc/samples/disparity_filtering.cpp

@@ -23,13 +23,13 @@ const String keys =
     "{algorithm      |bm                | stereo matching method (bm or sgbm)                               }"
     "{filter         |wls_conf          | used post-filtering (wls_conf or wls_no_conf)                     }"
     "{no-display     |                  | don't display results                                             }"
-    "{no-downscale   |                  | prevent stereo matching on downscaled views                       }"
+    "{no-downscale   |                  | force stereo matching on full-sized views to improve quality      }"
     "{dst_conf_path  |None              | optional path to save the confidence map used in filtering        }"
     "{vis_mult       |1.0               | coefficient used to scale disparity map visualizations            }"
     "{max_disparity  |160               | parameter of stereo matching                                      }"
-    "{window_size    |19                | parameter of stereo matching                                      }"
+    "{window_size    |-1                | parameter of stereo matching                                      }"
     "{wls_lambda     |8000.0            | parameter of post-filtering                                       }"
-    "{wls_sigma      |1.0               | parameter of post-filtering                                       }"
+    "{wls_sigma      |1.5               | parameter of post-filtering                                       }"
     ;
 
 int main(int argc, char** argv)
@@ -54,17 +54,30 @@ int main(int argc, char** argv)
     bool no_display = parser.has("no-display");
     bool no_downscale = parser.has("no-downscale");
     int max_disp = parser.get<int>("max_disparity");
-    int wsize = parser.get<int>("window_size");
     double lambda = parser.get<double>("wls_lambda");
     double sigma  = parser.get<double>("wls_sigma");
     double vis_mult = parser.get<double>("vis_mult");
 
+    int wsize;
+    if(parser.get<int>("window_size")>=0) //user provided window_size value
+        wsize = parser.get<int>("window_size");
+    else
+    {
+        if(algo=="sgbm")
+            wsize = 3; //default window size for SGBM
+        else if(!no_downscale && algo=="bm" && filter=="wls_conf")
+            wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf)
+        else
+            wsize = 15; //default window size for BM on full-sized views
+    }
+
     if (!parser.check())
     {
         parser.printErrors();
         return -1;
     }
 
+    //! [load_views]
     Mat left  = imread(left_im ,IMREAD_COLOR);
     if ( left.empty() )
     {
@@ -78,6 +91,7 @@ int main(int argc, char** argv)
         cout<<"Cannot read image file: "<<right_im;
         return -1;
     }
+    //! [load_views]
 
     bool noGT;
     Mat GT_disp;
@@ -99,6 +113,7 @@ int main(int argc, char** argv)
     Mat conf_map = Mat(left.rows,left.cols,CV_8U);
     conf_map = Scalar(255);
     Rect ROI;
+    Ptr<DisparityWLSFilter> wls_filter;
     double matching_time, filtering_time;
     if(max_disp<=0 || max_disp%16!=0)
     {
@@ -110,56 +125,51 @@ int main(int argc, char** argv)
         cout<<"Incorrect window_size value: it should be positive and odd";
         return -1;
     }
-    if(filter=="wls_conf")
+    if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf)
     {
         if(!no_downscale)
         {
-            wsize = wsize/2;
-            if(wsize%2==0) wsize++;
+            // downscale the views to speed-up the matching stage, as we will need to compute both left
+            // and right disparity maps for confidence map computation
+            //! [downscale]
             max_disp/=2;
             if(max_disp%16!=0)
                 max_disp += 16-(max_disp%16);
             resize(left ,left_for_matcher ,Size(),0.5,0.5);
             resize(right,right_for_matcher,Size(),0.5,0.5);
+            //! [downscale]
         }
         else
         {
             left_for_matcher  = left.clone();
             right_for_matcher = right.clone();
         }
 
-
         if(algo=="bm")
         {
-            Ptr<StereoBM> left_matcher  = StereoBM::create(max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoBM> right_matcher = StereoBM::create(max_disp,wsize);
-            right_matcher->setMinDisparity(-max_disp+1);
-            left_matcher->setTextureThreshold(0);
-            left_matcher->setUniquenessRatio(0);
-            right_matcher->setTextureThreshold(0);
-            right_matcher->setUniquenessRatio(0);
-            cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
+            //! [matching]
+            Ptr<StereoBM> left_matcher = StereoBM::create(max_disp,wsize);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
+
+            cvtColor(left_for_matcher,  left_for_matcher,  COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
             right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
             matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
+            //! [matching]
         }
         else if(algo=="sgbm")
         {
             Ptr<StereoSGBM> left_matcher  = StereoSGBM::create(0,max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoSGBM> right_matcher = StereoSGBM::create(-max_disp+1,max_disp,wsize);
-            left_matcher->setUniquenessRatio(0);
-            left_matcher->setDisp12MaxDiff(1000000);
-            left_matcher->setSpeckleWindowSize(0);
-            right_matcher->setUniquenessRatio(0);
-            right_matcher->setDisp12MaxDiff(1000000);
-            right_matcher->setSpeckleWindowSize(0);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
+            left_matcher->setP1(24*wsize*wsize);
+            left_matcher->setP2(96*wsize*wsize);
+            left_matcher->setPreFilterCap(63);
+            left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
@@ -172,14 +182,17 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(true);
+        //! [filtering]
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI,right_disp);
+        wls_filter->filter(left_disp,left,filtered_disp,right_disp);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
+        //! [filtering]
         conf_map = wls_filter->getConfidenceMap();
 
+        // Get the ROI that was used in the last filter call:
+        ROI = wls_filter->getROI();
         if(!no_downscale)
         {
             // upscale raw disparity and ROI back for a proper comparison:
@@ -190,6 +203,9 @@ int main(int argc, char** argv)
     }
     else if(filter=="wls_no_conf")
     {
+        /* There is no convenience function for the case of filtering with no confidence, so we
+        will need to set the ROI and matcher parameters manually */
+
         left_for_matcher  = left.clone();
         right_for_matcher = right.clone();
 
@@ -201,6 +217,8 @@ int main(int argc, char** argv)
             cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -212,7 +230,12 @@ int main(int argc, char** argv)
             matcher->setUniquenessRatio(0);
             matcher->setDisp12MaxDiff(1000000);
             matcher->setSpeckleWindowSize(0);
+            matcher->setP1(24*wsize*wsize);
+            matcher->setP2(96*wsize*wsize);
+            matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -224,11 +247,10 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(false);
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI);
+        wls_filter->filter(left_disp,left,filtered_disp,Mat(),ROI);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
     }
     else
@@ -292,6 +314,7 @@ int main(int argc, char** argv)
             imshow("ground-truth disparity", GT_disp_vis);
         }
 
+        //! [visualization]
         Mat raw_disp_vis;
         getDisparityVis(left_disp,raw_disp_vis,vis_mult);
         namedWindow("raw disparity", WINDOW_AUTOSIZE);
@@ -301,6 +324,7 @@ int main(int argc, char** argv)
         namedWindow("filtered disparity", WINDOW_AUTOSIZE);
         imshow("filtered disparity", filtered_disp_vis);
         waitKey();
+        //! [visualization]
     }
 
     return 0;