Merge pull request #2367 from tsenst:add_robust_interpolation_of_correspondence

Add RIC method for sparse match interpolation

* * add RIC

* sparse_match_interpolators EdgeAwareInterpolation - enhance limitation of the maximal number of matches to be interpolated from SHORT_MAX to INT_MAX by substituting short by int types

* * add RIC citation

* * update EdgeAwareInterpolatorImpl

* * add intermediate RIC implementation

* * implementation of new paralelization classes. First test where sucessfull.

* * update documentation RICInterpolatorImpl and RLOF

* * update CMakeLists.txt remove highgui
* add test for RIC
* add ASSERTION for curr image

* * add cost map interface

* * fix internal cost map allocation bug

* * remove white spaces
* fix warnings

* *fix compiler warnings

* * remove double whitespaces
* underscore from parameters
* substitute parallel_for_() classes with lambda functions
* remove complex assertion statements
* remove dead code
* substitute swap function with std::swap()
* ocv_define_module now contains video instead of tracking module

* * remove whitespace endings

* * documentation update

* * remove double declarations that lead to warnings

* * unrole tracker.py
*  remove double space
* use static for inner functions
* update create function
* modify mem init. to avoid manual memory management

* * uncomment parallel_for_ for parallelization

* * unrole unwanted changes
* uncomment paralellization

* * remove empty comment
* change CHECK to CHK_GD
* remove not necessary ;

* *documentation remove double space

Author: tsenst

modules/optflow/README.md

@@ -1,4 +1,4 @@
 Optical Flow Algorithms
 =======================
 
-Algorithms for running and evaluating deepflow, simpleflow, sparsetodenseflow and motion templates (silhouette flow).
+Algorithms for running and evaluating deepflow, simpleflow, sparsetodenseflow, robust local optical flow and motion templates (silhouette flow).

modules/optflow/doc/optflow.bib

@@ -104,9 +104,26 @@ @article{Senst2012
   number = {9},
 }
 
+@phdthesis{Senst2019,
+  title={Estimation and analysis of motion in video data},
+  author={Senst, Tobias},
+  year={2019},
+  school={Technical Univerity Berlin},
+  doi = {10.14279/depositonce-9085},
+  url = {https://doi.org/10.14279/depositonce-9085}
+}
+
 @article{Tibshirani2008,
   title={Fast computation of the median by successive binning},
   author={Tibshirani, Ryan J},
   journal={arXiv preprint arXiv:0806.3301},
   year={2008}
 }
+
+@inproceedings{Hu2017,
+  title={Robust interpolation of correspondences for large displacement optical flow},
+  author={Hu, Yinlin and Li, Yunsong and Song, Rui},
+  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={481--489},
+  year={2017}
+}

modules/optflow/include/opencv2/optflow/rlofflow.hpp

@@ -30,14 +30,16 @@ enum SolverType {
 enum InterpolationType
 {
     INTERP_GEO = 0,    /**<  Fast geodesic interpolation, see @cite Geistert2016 */
-    INTERP_EPIC = 1,    /**<  Edge-preserving interpolation, see @cite Revaud2015,Geistert2016. */
+    INTERP_EPIC = 1,   /**<  Edge-preserving interpolation using ximgproc::EdgeAwareInterpolator, see @cite Revaud2015,Geistert2016. */
+    INTERP_RIC = 2,    /**<  SLIC based robust interpolation using ximgproc::RICInterpolator, see @cite Hu2017. */
 };
 
 /** @brief This is used store and set up the parameters of the robust local optical flow (RLOF) algoritm.
  *
  * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
  * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
- * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
+ * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
+ * The implementation is derived from optflow::calcOpticalFlowPyrLK().
  * This RLOF implementation can be seen as an improved pyramidal iterative Lucas-Kanade and includes
  * a set of improving modules. The main improvements in respect to the pyramidal iterative Lucas-Kanade
  * are:
@@ -200,7 +202,8 @@ class CV_EXPORTS_W RLOFOpticalFlowParameter{
  *
  * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
  * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
- * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
+ * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
+ * The implementation is derived from optflow::calcOpticalFlowPyrLK().
  *
  * The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
  * For this scheme the following steps are applied:
@@ -324,6 +327,35 @@ class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow
      *    @see ximgproc::fastGlobalSmootherFilter, setUsePostProc
      */
     CV_WRAP virtual bool getUsePostProc() const = 0;
+    //! @brief enables VariationalRefinement
+    /**
+     * @see getUseVariationalRefinement
+     */
+    CV_WRAP virtual void setUseVariationalRefinement(bool val) = 0;
+    /** @copybrief setUseVariationalRefinement
+     *    @see ximgproc::fastGlobalSmootherFilter, setUsePostProc
+     */
+    CV_WRAP virtual bool getUseVariationalRefinement() const = 0;
+    //! @brief Parameter to tune the approximate size of the superpixel used for oversegmentation.
+    /**
+     * @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator
+     */
+    CV_WRAP virtual void setRICSPSize(int val) = 0;
+    /** @copybrief setRICSPSize
+    *    @see setRICSPSize
+    */
+    CV_WRAP virtual int  getRICSPSize() const = 0;
+    /** @brief Parameter to choose superpixel algorithm variant to use:
+     * - cv::ximgproc::SLICType SLIC segments image using a desired region_size (value: 100)
+     * - cv::ximgproc::SLICType SLICO will optimize using adaptive compactness factor (value: 101)
+     * - cv::ximgproc::SLICType MSLIC will optimize using manifold methods resulting in more content-sensitive superpixels (value: 102).
+     *  @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator
+    */
+    CV_WRAP virtual void setRICSLICType(int val) = 0;
+    /** @copybrief setRICSLICType
+     *    @see setRICSLICType
+     */
+    CV_WRAP virtual int  getRICSLICType() const = 0;
     //! @brief Creates instance of optflow::DenseRLOFOpticalFlow
     /**
      *    @param rlofParam see optflow::RLOFOpticalFlowParameter
@@ -333,9 +365,12 @@ class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow
      *    @param epicK see setEPICK
      *    @param epicSigma see setEPICSigma
      *    @param epicLambda see setEPICLambda
+     *    @param ricSPSize see setRICSPSize
+     *    @param ricSLICType see setRICSLICType
      *    @param use_post_proc see setUsePostProc
      *    @param fgsLambda see setFgsLambda
      *    @param fgsSigma see setFgsSigma
+     *    @param use_variational_refinement see setUseVariationalRefinement
     */
     CV_WRAP static Ptr<DenseRLOFOpticalFlow> create(
         Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
@@ -345,16 +380,20 @@ class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow
         int epicK = 128,
         float epicSigma = 0.05f,
         float epicLambda = 999.0f,
+        int ricSPSize = 15,
+        int ricSLICType = 100,
         bool use_post_proc = true,
         float fgsLambda = 500.0f,
-        float fgsSigma = 1.5f);
+        float fgsSigma = 1.5f,
+        bool use_variational_refinement = false);
 };
 
 /** @brief Class used for calculation sparse optical flow and feature tracking with robust local optical flow (RLOF) algorithms.
 *
 * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
  * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
-* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
+* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
+* The implementation is derived from optflow::calcOpticalFlowPyrLK().
 *
 * For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
 * Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014 and @cite Senst2016.
@@ -401,7 +440,8 @@ class CV_EXPORTS_W SparseRLOFOpticalFlow : public SparseOpticalFlow
  *
  * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
  * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
- * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
+ * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
+ * The implementation is derived from optflow::calcOpticalFlowPyrLK().
  *
  * The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
  * For this scheme the following steps are applied:
@@ -430,12 +470,15 @@ class CV_EXPORTS_W SparseRLOFOpticalFlow : public SparseOpticalFlow
  * supported:
  * - **INTERP_GEO** applies the fast geodesic interpolation, see @cite Geistert2016.
  * - **INTERP_EPIC_RESIDUAL** applies the edge-preserving interpolation, see @cite Revaud2015,Geistert2016.
- * @param epicK see ximgproc::EdgeAwareInterpolator() sets the respective parameter.
- * @param epicSigma see ximgproc::EdgeAwareInterpolator() sets the respective parameter.
- * @param epicLambda see ximgproc::EdgeAwareInterpolator() sets the respective parameter.
+ * @param epicK see ximgproc::EdgeAwareInterpolator sets the respective parameter.
+ * @param epicSigma see ximgproc::EdgeAwareInterpolator sets the respective parameter.
+ * @param epicLambda see ximgproc::EdgeAwareInterpolator sets the respective parameter.
+ * @param ricSPSize  see ximgproc::RICInterpolator sets the respective parameter.
+ * @param ricSLICType see ximgproc::RICInterpolator sets the respective parameter.
  * @param use_post_proc enables ximgproc::fastGlobalSmootherFilter() parameter.
  * @param fgsLambda sets the respective ximgproc::fastGlobalSmootherFilter() parameter.
  * @param fgsSigma sets the respective ximgproc::fastGlobalSmootherFilter() parameter.
+ * @param use_variational_refinement enables VariationalRefinement
  *
  * Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014, @cite Senst2016.
  * For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
@@ -451,14 +494,17 @@ CV_EXPORTS_W void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOu
     float forwardBackwardThreshold = 0, Size gridStep = Size(6, 6),
     InterpolationType interp_type = InterpolationType::INTERP_EPIC,
     int epicK = 128, float epicSigma = 0.05f, float epicLambda = 100.f,
-    bool use_post_proc = true, float fgsLambda = 500.0f, float fgsSigma = 1.5f);
+    int ricSPSize = 15, int ricSLICType = 100,
+    bool use_post_proc = true, float fgsLambda = 500.0f, float fgsSigma = 1.5f,
+    bool use_variational_refinement = false);
 
 /** @brief Calculates fast optical flow for a sparse feature set using the robust local optical flow (RLOF) similar
 * to optflow::calcOpticalFlowPyrLK().
 *
 * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
  * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
-* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
+* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
+* The implementation is derived from optflow::calcOpticalFlowPyrLK().
 *
 * @param prevImg first 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType
 * = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.

modules/optflow/samples/optical_flow_evaluation.cpp

@@ -12,7 +12,7 @@ using namespace optflow;
 const String keys = "{help h usage ? |      | print this message   }"
         "{@image1        |      | image1               }"
         "{@image2        |      | image2               }"
-        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, sparsetodenseflow, pcaflow, DISflow_ultrafast, DISflow_fast, DISflow_medium] }"
+        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, sparsetodenseflow, RLOF_EPIC, RLOF_RIC, pcaflow, DISflow_ultrafast, DISflow_fast, DISflow_medium] }"
         "{@groundtruth   |      | path to the .flo file  (optional), Middlebury format }"
         "{m measure      |endpoint| error measure - [endpoint or angular] }"
         "{r region       |all   | region to compute stats about [all, discontinuities, untextured] }"
@@ -259,6 +259,20 @@ int main( int argc, char** argv )
         algorithm = createOptFlow_DeepFlow();
     else if ( method == "sparsetodenseflow" )
         algorithm = createOptFlow_SparseToDense();
+    else if (method == "RLOF_EPIC")
+    {
+        algorithm = createOptFlow_DenseRLOF();
+        Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();
+        rlof->setInterpolation(INTERP_EPIC);
+        rlof->setForwardBackward(1.f);
+    }
+    else if (method == "RLOF_RIC")
+    {
+        algorithm = createOptFlow_DenseRLOF();
+        Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();;
+        rlof->setInterpolation(INTERP_RIC);
+        rlof->setForwardBackward(1.f);
+    }
     else if ( method == "pcaflow" ) {
         if ( parser.has("prior") ) {
             String prior = parser.get<String>("prior");

modules/optflow/src/rlofflow.cpp

@@ -70,6 +70,9 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
         , fgs_lambda(500.0f)
         , fgs_sigma(1.5f)
         , use_post_proc(true)
+        , use_variational_refinement(false)
+        , sp_size(15)
+        , slic_type(ximgproc::SLIC)
 
     {
         prevPyramid[0] = cv::Ptr<CImageBuffer>(new CImageBuffer);
@@ -107,6 +110,15 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
     virtual bool getUsePostProc() const CV_OVERRIDE { return use_post_proc; }
     virtual void setUsePostProc(bool val) CV_OVERRIDE { use_post_proc = val; }
 
+    virtual void setUseVariationalRefinement(bool val) CV_OVERRIDE { use_variational_refinement = val; }
+    virtual bool getUseVariationalRefinement() const CV_OVERRIDE { return use_variational_refinement; }
+
+    virtual void setRICSPSize(int val) CV_OVERRIDE { sp_size = val; }
+    virtual int  getRICSPSize() const CV_OVERRIDE { return sp_size; }
+
+    virtual void setRICSLICType(int val) CV_OVERRIDE { slic_type = static_cast<ximgproc::SLICType>(val); }
+    virtual int  getRICSLICType() const CV_OVERRIDE { return slic_type; }
+
     virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE
     {
         CV_Assert(!I0.empty() && I0.depth() == CV_8U && (I0.channels() == 3 || I0.channels() == 1));
@@ -116,7 +128,7 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
             param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter());
         if (param->supportRegionType == SR_CROSS)
             CV_Assert( I0.channels() == 3 && I1.channels() == 3);
-        CV_Assert(interp_type == InterpolationType::INTERP_EPIC || interp_type == InterpolationType::INTERP_GEO);
+        CV_Assert(interp_type == InterpolationType::INTERP_EPIC || interp_type == InterpolationType::INTERP_GEO || interp_type == InterpolationType::INTERP_RIC);
         // if no parameter is used use the default parameter
 
         Mat prevImage = I0.getMat();
@@ -184,9 +196,23 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
             gd->setK(k);
             gd->setSigma(sigma);
             gd->setLambda(lambda);
+            gd->setFGSLambda(fgs_lambda);
+            gd->setFGSSigma(fgs_sigma);
             gd->setUsePostProcessing(false);
             gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);
         }
+        else if (interp_type == InterpolationType::INTERP_RIC)
+        {
+            Ptr<ximgproc::RICInterpolator> gd = ximgproc::createRICInterpolator();
+            gd->setK(k);
+            gd->setFGSLambda(fgs_lambda);
+            gd->setFGSSigma(fgs_sigma);
+            gd->setSuperpixelSize(sp_size);
+            gd->setSuperpixelMode(slic_type);
+            gd->setUseGlobalSmootherFilter(false);
+            gd->setUseVariationalRefinement(false);
+            gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);
+        }
         else
         {
             Mat blurredPrevImage, blurredCurrImage;
@@ -200,6 +226,15 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
             cv::bilateralFilter(vecMats[1], vecMats2[1], 5, 2, 20);
             cv::merge(vecMats2, dense_flow);
         }
+        if (use_variational_refinement)
+        {
+            Mat prevGrey, currGrey;
+            Ptr<VariationalRefinement > variationalrefine = VariationalRefinement::create();
+            cvtColor(prevImage, prevGrey, COLOR_BGR2GRAY);
+            cvtColor(currImage, currGrey, COLOR_BGR2GRAY);
+            variationalrefine->setOmega(1.9f);
+            variationalrefine->calc(prevGrey, currGrey, flow);
+        }
         if (use_post_proc)
         {
             ximgproc::fastGlobalSmootherFilter(prevImage, flow, flow, fgs_lambda, fgs_sigma);
@@ -227,6 +262,9 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
     float                         fgs_lambda;
     float                         fgs_sigma;
     bool                          use_post_proc;
+    bool                          use_variational_refinement;
+    int                           sp_size;
+    ximgproc::SLICType            slic_type;
 };
 
 Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create(
@@ -237,9 +275,12 @@ Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create(
     int epicK,
     float epicSigma,
     float epicLambda,
+    int ricSPSize,
+    int ricSLICType,
     bool use_post_proc,
     float fgs_lambda,
-    float fgs_sigma)
+    float fgs_sigma,
+    bool use_variational_refinement)
 {
     Ptr<DenseRLOFOpticalFlow> algo = makePtr<DenseOpticalFlowRLOFImpl>();
     algo->setRLOFOpticalFlowParameter(rlofParam);
@@ -252,6 +293,9 @@ Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create(
     algo->setUsePostProc(use_post_proc);
     algo->setFgsLambda(fgs_lambda);
     algo->setFgsSigma(fgs_sigma);
+    algo->setRICSLICType(ricSLICType);
+    algo->setRICSPSize(ricSPSize);
+    algo->setUseVariationalRefinement(use_variational_refinement);
     return algo;
 }
 
@@ -381,11 +425,13 @@ void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOutputArray flo
     float forewardBackwardThreshold, Size gridStep,
     InterpolationType interp_type,
     int epicK, float epicSigma, float epicLambda,
-    bool use_post_proc, float fgsLambda, float fgsSigma)
+    int superpixelSize, int superpixelType,
+    bool use_post_proc, float fgsLambda, float fgsSigma, bool use_variational_refinement)
 {
     Ptr<DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create(
         rlofParam, forewardBackwardThreshold, gridStep, interp_type,
-        epicK, epicSigma, epicLambda, use_post_proc, fgsLambda, fgsSigma);
+        epicK, epicSigma, epicLambda, superpixelSize, superpixelType,
+        use_post_proc, fgsLambda, fgsSigma, use_variational_refinement);
     algo->calc(I0, I1, flow);
     algo->collectGarbage();
 }

modules/ximgproc/CMakeLists.txt

@@ -1,2 +1,2 @@
 set(the_description "Extended image processing module. It includes edge-aware filters and etc.")
-ocv_define_module(ximgproc opencv_core opencv_imgproc opencv_calib3d opencv_imgcodecs WRAP python java)
+ocv_define_module(ximgproc opencv_core opencv_imgproc opencv_calib3d opencv_imgcodecs opencv_video WRAP python java)

modules/ximgproc/doc/ximgproc.bib

@@ -328,3 +328,11 @@ @inproceedings{BarronPoole2016
 publisher={Springer International Publishing},
 pages={617--632},
 }
+
+@inproceedings{Hu2017,
+  title={Robust interpolation of correspondences for large displacement optical flow},
+  author={Hu, Yinlin and Li, Yunsong and Song, Rui},
+  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
+  pages={481--489},
+  year={2017}
+}