Merge pull request #3277 from iago-suarez:4.x

Add BAD descriptor to xfeatures2d module

* Adding new BAD descriptor to xfeatures2d module

* Changing BAD name by TEBLID and using int threshold again for BEBLID

* Changing link to AKAZE tutorial and moved parameters initialization to ::create()

* Adding f suffixes to floating-point parameters and using arrays again

Author: iago-suarez

modules/xfeatures2d/README.md

@@ -5,4 +5,4 @@ Extra 2D Features Framework
 2. Non-free 2D feature algorithms
 
 Extra 2D Features Framework containing experimental and non-free 2D feature detector/descriptor algorithms:
- SURF, BRIEF, Censure, Freak, LUCID, Daisy, BEBLID, Self-similar.
+ SURF, BRIEF, Censure, Freak, LUCID, Daisy, BEBLID, TEBLID, Self-similar.

modules/xfeatures2d/doc/xfeatures2d.bib

@@ -154,6 +154,18 @@ @article{Suarez2020BEBLID
   author = {Iago Su\'arez and Ghesn Sfeir and Jos\'e M. Buenaposada and Luis Baumela},
 }
 
+@article{Suarez2021TEBLID,
+  title = {Revisiting Binary Local Image Description for Resource Limited Devices},
+  journal = {IEEE Robotics and Automation Letters},
+  volume = {6},
+  pages = {8317--8324},
+  year = {2021},
+  number = {4},
+  doi = {https://doi.org/10.1109/LRA.2021.3107024},
+  url = {https://arxiv.org/pdf/2108.08380.pdf},
+  author = {Iago Su\'arez and Jos\'e M. Buenaposada and Luis Baumela},
+}
+
 @inproceedings{winder2007learning,
   title= {Learning Local Image Descriptors},
   author= {Winder, Simon AJ and Brown, Matthew},

modules/xfeatures2d/include/opencv2/xfeatures2d.hpp

@@ -224,6 +224,49 @@ class CV_EXPORTS_W BEBLID : public Feature2D
     CV_WRAP static Ptr<BEBLID> create(float scale_factor, int n_bits = BEBLID::SIZE_512_BITS);
 };
 
+/** @brief Class implementing TEBLID (Triplet-based Efficient Binary Local Image Descriptor),
+ * described in @cite Suarez2021TEBLID.
+
+TEBLID stands for Triplet-based Efficient Binary Local Image Descriptor, although originally it was called BAD
+\cite Suarez2021TEBLID. It is an improvement over BEBLID \cite Suarez2020BEBLID, that uses triplet loss,
+hard negative mining, and anchor swap to improve the image matching results.
+It is able to describe keypoints from any detector just by changing the scale_factor parameter.
+TEBLID is as efficient as ORB, BEBLID or BRISK, but the triplet-based training objective selected more
+discriminative features that explain the accuracy gain. It is also more compact than BEBLID,
+when running the [AKAZE example](https://github.com/opencv/opencv/blob/4.x/samples/cpp/tutorial_code/features2D/AKAZE_match.cpp)
+with 10000 keypoints detected by ORB, BEBLID obtains 561 inliers (75%) with 512 bits, whereas
+TEBLID obtains 621 (75.2%) with 256 bits. ORB obtains only 493 inliers (63%).
+
+If you find this code useful, please add a reference to the following paper:
+<BLOCKQUOTE> Iago Suárez, José M. Buenaposada, and Luis Baumela.
+Revisiting Binary Local Image Description for Resource Limited Devices.
+IEEE Robotics and Automation Letters, vol. 6, no. 4, pp. 8317-8324, Oct. 2021. </BLOCKQUOTE>
+
+The descriptor was trained in Liberty split of the UBC datasets \cite winder2007learning .
+*/
+class CV_EXPORTS_W TEBLID : public Feature2D
+{
+public:
+    /**
+     * @brief  Descriptor number of bits, each bit is a box average difference.
+     * The user can choose between 256 or 512 bits.
+     */
+    enum TeblidSize
+    {
+        SIZE_256_BITS = 102, SIZE_512_BITS = 103,
+    };
+    /** @brief Creates the TEBLID descriptor.
+    @param scale_factor Adjust the sampling window around detected keypoints:
+    - <b> 1.00f </b> should be the scale for ORB keypoints
+    - <b> 6.75f </b> should be the scale for SIFT detected keypoints
+    - <b> 6.25f </b> is default and fits for KAZE, SURF detected keypoints
+    - <b> 5.00f </b> should be the scale for AKAZE, MSD, AGAST, FAST, BRISK keypoints
+    @param n_bits Determine the number of bits in the descriptor. Should be either
+     TEBLID::SIZE_256_BITS or TEBLID::SIZE_512_BITS.
+    */
+    CV_WRAP static Ptr<TEBLID> create(float scale_factor, int n_bits = TEBLID::SIZE_256_BITS);
+};
+
 /** @brief Class implementing DAISY descriptor, described in @cite Tola10
 
 @param radius radius of the descriptor at the initial scale

modules/xfeatures2d/perf/perf_teblid.cpp

@@ -0,0 +1,36 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+#include "perf_precomp.hpp"
+
+namespace opencv_test { namespace {
+
+typedef perf::TestBaseWithParam<std::string> teblid;
+
+#define TEBLID_IMAGES \
+    "cv/detectors_descriptors_evaluation/images_datasets/leuven/img1.png",\
+    "stitching/a3.png"
+
+#ifdef OPENCV_ENABLE_NONFREE
+PERF_TEST_P(teblid, extract, testing::Values(TEBLID_IMAGES))
+{
+    string filename = getDataPath(GetParam());
+    Mat frame = imread(filename, IMREAD_GRAYSCALE);
+    ASSERT_FALSE(frame.empty()) << "Unable to load source image " << filename;
+
+    Mat mask;
+    declare.in(frame).time(90);
+
+    Ptr<SURF> detector = SURF::create();
+    vector<KeyPoint> points;
+    detector->detect(frame, points, mask);
+
+    Ptr<TEBLID> descriptor = TEBLID::create(6.25f);
+    cv::Mat descriptors;
+    TEST_CYCLE() descriptor->compute(frame, points, descriptors);
+
+    SANITY_CHECK_NOTHING();
+}
+#endif // NONFREE
+
+}} // namespace

modules/xfeatures2d/src/beblid.cpp

@@ -30,14 +30,21 @@ struct ABWLParams
 {
     int x1, y1, x2, y2, boxRadius, th;
 };
+// Same as previous with floating point threshold
+struct ABWLParamsFloatTh
+{
+    int x1, y1, x2, y2, boxRadius;
+    float th;
+};
 
 // BEBLID implementation
+template <class WeakLearnerT>
 class BEBLID_Impl CV_FINAL: public BEBLID
 {
 public:
 
     // constructor
-    explicit BEBLID_Impl(float scale_factor, int n_bits = SIZE_512_BITS);
+    explicit BEBLID_Impl(float scale_factor, const std::vector<WeakLearnerT>& wl_params);
 
     // destructor
     ~BEBLID_Impl() CV_OVERRIDE = default;
@@ -55,15 +62,65 @@ class BEBLID_Impl CV_FINAL: public BEBLID
     void compute(InputArray image, vector<KeyPoint> &keypoints, OutputArray descriptors) CV_OVERRIDE;
 
 private:
-    std::vector<ABWLParams> wl_params_;
+    std::vector<WeakLearnerT> wl_params_;
     float scale_factor_;
     cv::Size patch_size_;
 
-    void computeBEBLID(const cv::Mat &integralImg,
-                       const std::vector<cv::KeyPoint> &keypoints,
-                       cv::Mat &descriptors);
+    void computeBoxDiffsDescriptor(const cv::Mat &integralImg,
+                                   const std::vector<cv::KeyPoint> &keypoints,
+                                   cv::Mat &descriptors);
 }; // END BEBLID_Impl CLASS
 
+
+// TEBLID implementation
+class TEBLID_Impl CV_FINAL: public TEBLID
+{
+public:
+
+    // constructor
+    explicit TEBLID_Impl(float scale_factor, const std::vector<ABWLParamsFloatTh>& wl_params) :
+        impl(scale_factor, wl_params){}
+
+    // destructor
+    ~TEBLID_Impl() CV_OVERRIDE = default;
+
+    // returns the descriptor length in bytes
+    int descriptorSize() const CV_OVERRIDE { return impl.descriptorSize(); }
+
+    // returns the descriptor type
+    int descriptorType() const CV_OVERRIDE { return impl.descriptorType(); }
+
+    // returns the default norm type
+    int defaultNorm() const CV_OVERRIDE { return impl.defaultNorm();  }
+
+    // compute descriptors given keypoints
+    void compute(InputArray image, vector<KeyPoint> &keypoints, OutputArray descriptors) CV_OVERRIDE
+    {
+        impl.compute(image, keypoints, descriptors);
+    }
+
+private:
+    BEBLID_Impl<ABWLParamsFloatTh> impl;
+}; // END TEBLID_Impl CLASS
+
+Ptr<TEBLID> TEBLID::create(float scale_factor, int n_bits)
+{
+    if (n_bits == TEBLID::SIZE_512_BITS)
+    {
+        #include "teblid.p512.hpp"
+        return makePtr<TEBLID_Impl>(scale_factor, teblid_wl_params_512);
+    }
+    else if(n_bits == TEBLID::SIZE_256_BITS)
+    {
+        #include "teblid.p256.hpp"
+        return makePtr<TEBLID_Impl>(scale_factor, teblid_wl_params_256);
+    }
+    else
+    {
+        CV_Error(Error::StsBadArg, "n_bits should be either TEBLID::SIZE_512_BITS or TEBLID::SIZE_256_BITS");
+    }
+}
+
 /**
  * @brief Function that determines if a keypoint is close to the image border.
  * @param kp The detected keypoint
@@ -100,8 +157,9 @@ static inline bool isKeypointInTheBorder(const cv::KeyPoint &kp,
  * @param scaleFactor A scale factor that magnifies the measurement functions w.r.t. the keypoint.
  * @param patchSize The size of the normalized patch where the measurement functions were learnt.
  */
-static inline void rectifyABWL(const std::vector<ABWLParams> &wlPatchParams,
-                               std::vector<ABWLParams> &wlImageParams,
+template< typename WeakLearnerT>
+static inline void rectifyABWL(const std::vector<WeakLearnerT> &wlPatchParams,
+                               std::vector<WeakLearnerT> &wlImageParams,
                                const cv::KeyPoint &kp,
                                float scaleFactor = 1,
                                const cv::Size &patchSize = cv::Size(32, 32))
@@ -151,7 +209,8 @@ static inline void rectifyABWL(const std::vector<ABWLParams> &wlPatchParams,
  * @param integralImage The integral image used to compute the average gray value in the square regions.
  * @return The difference of gray level in the two squares defined by wlImageParams
  */
-static inline float computeABWLResponse(const ABWLParams &wlImageParams,
+template <typename WeakLearnerT>
+static inline float computeABWLResponse(const WeakLearnerT &wlImageParams,
                                         const cv::Mat &integralImage)
 {
     CV_DbgAssert(!integralImage.empty());
@@ -239,7 +298,8 @@ static inline float computeABWLResponse(const ABWLParams &wlImageParams,
 }
 
 // descriptor computation using keypoints
-void BEBLID_Impl::compute(InputArray _image, vector<KeyPoint> &keypoints, OutputArray _descriptors)
+template <class WeakLearnerT>
+void BEBLID_Impl<WeakLearnerT>::compute(InputArray _image, vector<KeyPoint> &keypoints, OutputArray _descriptors)
 {
     Mat image = _image.getMat();
 
@@ -281,27 +341,21 @@ void BEBLID_Impl::compute(InputArray _image, vector<KeyPoint> &keypoints, Output
     CV_DbgAssert(descriptors.type() == CV_8UC1);
 
     // Compute the BEBLID descriptors
-    computeBEBLID(integralImg, keypoints, descriptors);
+    computeBoxDiffsDescriptor(integralImg, keypoints, descriptors);
 }
 
 // constructor
-BEBLID_Impl::BEBLID_Impl(float scale_factor, int n_bits)
-    : scale_factor_(scale_factor), patch_size_(32, 32)
+template <class WeakLearnerT>
+BEBLID_Impl<WeakLearnerT>::BEBLID_Impl(float scale_factor, const std::vector<WeakLearnerT>& wl_params)
+    :  wl_params_(wl_params), scale_factor_(scale_factor),patch_size_(32, 32)
 {
-    #include "beblid.p512.hpp"
-    #include "beblid.p256.hpp"
-    if (n_bits == SIZE_512_BITS)
-        wl_params_.assign(wl_params_512, wl_params_512 + sizeof(wl_params_512) / sizeof(wl_params_512[0]));
-    else if(n_bits == SIZE_256_BITS)
-        wl_params_.assign(wl_params_256, wl_params_256 + sizeof(wl_params_256) / sizeof(wl_params_256[0]));
-    else
-        CV_Error(Error::StsBadArg, "n_wls should be either SIZE_512_BITS or SIZE_256_BITS");
 }
 
 // Internal function that implements the core of BEBLID descriptor
-void BEBLID_Impl::computeBEBLID(const cv::Mat &integralImg,
-                                const std::vector<cv::KeyPoint> &keypoints,
-                                cv::Mat &descriptors)
+template<class WeakLearnerT>
+void BEBLID_Impl<WeakLearnerT>::computeBoxDiffsDescriptor(const cv::Mat &integralImg,
+                                                          const std::vector<cv::KeyPoint> &keypoints,
+                                                          cv::Mat &descriptors)
 {
     CV_DbgAssert(!integralImg.empty());
     CV_DbgAssert(size_t(descriptors.rows) == keypoints.size());
@@ -316,13 +370,13 @@ void BEBLID_Impl::computeBEBLID(const cv::Mat &integralImg,
 #endif
     {
         // Get a pointer to the first element in the range
-        ABWLParams *wl;
+        WeakLearnerT *wl;
         float responseFun;
         int areaResponseFun, kpIdx;
         size_t wlIdx;
         int box1x1, box1y1, box1x2, box1y2, box2x1, box2y1, box2x2, box2y2, bit_idx, side;
         uchar byte = 0;
-        std::vector<ABWLParams> imgWLParams(wl_params_.size());
+        std::vector<WeakLearnerT> imgWLParams(wl_params_.size());
         uchar *d = &descriptors.at<uchar>(range.start, 0);
 
         for (kpIdx = range.start; kpIdx < range.end; kpIdx++)
@@ -397,7 +451,20 @@ void BEBLID_Impl::computeBEBLID(const cv::Mat &integralImg,
 
 Ptr<BEBLID> BEBLID::create(float scale_factor, int n_bits)
 {
-    return makePtr<BEBLID_Impl>(scale_factor, n_bits);
+    if (n_bits == BEBLID::SIZE_512_BITS)
+    {
+        #include "beblid.p512.hpp"
+        return makePtr<BEBLID_Impl<ABWLParams>>(scale_factor, beblid_wl_params_512);
+    }
+    else if(n_bits == BEBLID::SIZE_256_BITS)
+    {
+        #include "beblid.p256.hpp"
+        return makePtr<BEBLID_Impl<ABWLParams>>(scale_factor, beblid_wl_params_256);
+    }
+    else
+    {
+        CV_Error(Error::StsBadArg, "n_bits should be either BEBLID::SIZE_512_BITS or BEBLID::SIZE_256_BITS");
+    }
 }
 } // END NAMESPACE XFEATURES2D
 } // END NAMESPACE CV

modules/xfeatures2d/src/beblid.p256.hpp

@@ -12,7 +12,7 @@
 
 // Pre-trained parameters of BEBLID-256 trained in Liberty data set with
 // a million of patch pairs, 20% positives and 80% negatives
-static const ABWLParams wl_params_256[] = {
+static const ABWLParams beblid_wl_params_256_[] = {
     {26, 20, 14, 16, 5, 16}, {17, 17, 15, 15, 2, 7}, {18, 16, 8, 13, 3, 18},
     {19, 15, 13, 14, 3, 17}, {16, 16, 5, 15, 4, 10}, {25, 10, 16, 16, 6, 11},
     {16, 15, 12, 15, 1, 12}, {18, 17, 14, 17, 1, 13}, {15, 14, 5, 21, 5, 6}, {14, 14, 11, 7, 4, 2},
@@ -79,3 +79,5 @@ static const ABWLParams wl_params_256[] = {
     {2, 14, 1, 9, 1, 1}, {6, 25, 6, 21, 1, 1}, {6, 2, 2, 1, 1, 1}, {30, 19, 29, 20, 1, 0},
     {25, 21, 23, 20, 1, 0}, {16, 10, 16, 9, 1, 0}
 };
+static const std::vector<ABWLParams> beblid_wl_params_256(std::begin(beblid_wl_params_256_),
+                                                          std::end(beblid_wl_params_256_));

modules/xfeatures2d/src/beblid.p512.hpp

@@ -12,7 +12,7 @@
 
 // Pre-trained parameters of BEBLID-512 trained in Liberty data set with
 // a million of patch pairs, 20% positives and 80% negatives
-static const ABWLParams wl_params_512[] = {
+static const ABWLParams beblid_wl_params_512_[] = {
     {24, 18, 15, 17, 6, 13}, {19, 14, 13, 17, 2, 18}, {23, 19, 12, 15, 6, 19},
     {24, 14, 16, 16, 6, 11}, {16, 15, 12, 16, 1, 12}, {16, 15, 7, 10, 4, 10},
     {17, 12, 8, 17, 3, 16}, {24, 12, 11, 17, 7, 19}, {19, 17, 14, 11, 3, 13},
@@ -144,3 +144,5 @@ static const ABWLParams wl_params_512[] = {
     {26, 4, 26, 1, 1, 0}, {5, 21, 2, 20, 1, -1}, {14, 1, 13, 3, 1, 1}, {30, 9, 28, 8, 1, 0},
     {13, 15, 12, 12, 1, 1}, {7, 23, 6, 25, 1, -1}
 };
+static const std::vector<ABWLParams> beblid_wl_params_512(std::begin(beblid_wl_params_512_),
+                                                          std::end(beblid_wl_params_512_));