Merge pull request #1199 from sukhad-app:face_alignment

Face alignment (#1199)

* This commit will add a new functionality of one millisecond face_alignment to OpenCV.
Face alignment is a computer vision technology for identifying the geometric structure of human faces in digital images.
Given the location and size of a face, it automatically determines the shape
 of the face components such as eyes and nose.
Added following functions :
 1) Application to train a face landmark detector.
 2) Application to detect face landmarks using a trained model.
 3) Application to swap faces using face landmark detection
 4) Application to detect landmarks in a video.
Merged the code with a global facemark API.
Added Doxygen Documentation for the Class created.
Added tutorials for the samples added.
Added visualisations depicting error rate and training time.

Made desired changes

fix

fix

fix

fix

fix

fix

fix

fix

fix

* face: drop duplicated file

-face_alignmentImpl.hpp
+face_alignmentimpl.hpp

* face: minor refactoring

- replace license headers
- fix usage of "precomp.hpp"

Author: IAmUnbounded

modules/face/CMakeLists.txt

@@ -1,3 +1,27 @@
 set(the_description "Face recognition etc")
-ocv_define_module(face opencv_core opencv_imgproc opencv_objdetect WRAP python)
+ocv_define_module(face opencv_core
+    opencv_imgproc
+    opencv_objdetect
+    opencv_tracking     # estimateRigidTransform()
+    opencv_photo        # samples
+    WRAP python
+)
 # NOTE: objdetect module is needed for one of the samples
+
+set(__commit_hash "8afa57abc8229d611c4937165d20e2a2d9fc5a12")
+set(__file_hash "7505c44ca4eb54b4ab1e4777cb96ac05")
+ocv_download(
+    FILENAME face_landmark_model.dat
+    HASH ${__file_hash}
+    URL
+      "${OPENCV_FACE_ALIGNMENT_URL}"
+      "$ENV{OPENCV_FACE_ALIGNMENT_URL}"
+      "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${__commit_hash}/"
+    DESTINATION_DIR "${CMAKE_BINARY_DIR}/${OPENCV_TEST_DATA_INSTALL_PATH}/cv/face/"
+    ID "data"
+    RELATIVE_URL
+    STATUS res
+)
+if(NOT res)
+  message(WARNING "Face: Can't get model file for face alignment.")
+endif()

modules/face/include/opencv2/face.hpp

@@ -377,4 +377,6 @@ class CV_EXPORTS_W FaceRecognizer : public Algorithm
 #include "opencv2/face/facemark.hpp"
 #include "opencv2/face/facemarkLBF.hpp"
 #include "opencv2/face/facemarkAAM.hpp"
-#endif
+#include "opencv2/face/face_alignment.hpp"
+
+#endif // __OPENCV_FACE_HPP__

modules/face/include/opencv2/face/face_alignment.hpp

@@ -0,0 +1,71 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+#ifndef __OPENCV_FACE_ALIGNMENT_HPP__
+#define __OPENCV_FACE_ALIGNMENT_HPP__
+
+#include "facemark.hpp"
+
+namespace cv{
+namespace face{
+class CV_EXPORTS_W FacemarkKazemi : public Algorithm
+{
+public:
+    struct CV_EXPORTS Params
+    {
+        /**
+        * \brief Constructor
+        */
+        Params();
+        /// cascade_depth This stores the deapth of cascade used for training.
+        unsigned long cascade_depth;
+        /// tree_depth This stores the max height of the regression tree built.
+        unsigned long tree_depth;
+        /// num_trees_per_cascade_level This stores number of trees fit per cascade level.
+        unsigned long num_trees_per_cascade_level;
+        /// learning_rate stores the learning rate in gradient boosting, also reffered as shrinkage.
+        float learning_rate;
+        /// oversampling_amount stores number of initialisations used to create training samples.
+        unsigned long oversampling_amount;
+        /// num_test_coordinates stores number of test coordinates.
+        unsigned long num_test_coordinates;
+        /// lambda stores a value to calculate probability of closeness of two coordinates.
+        float lambda;
+        /// num_test_splits stores number of random test splits generated.
+        unsigned long num_test_splits;
+        /// configfile stores the name of the file containing the values of training parameters
+        String configfile;
+    };
+    static Ptr<FacemarkKazemi> create(const FacemarkKazemi::Params &parameters = FacemarkKazemi::Params());
+    virtual ~FacemarkKazemi();
+
+    /// @brief training the facemark model, input are the file names of image list and landmark annotation
+    virtual void training(String imageList, String groundTruth)=0;
+    /** @brief This function is used to train the model using gradient boosting to get a cascade of regressors
+    *which can then be used to predict shape.
+    *@param images A vector of type cv::Mat which stores the images which are used in training samples.
+    *@param landmarks A vector of vectors of type cv::Point2f which stores the landmarks detected in a particular image.
+    *@param scale A size of type cv::Size to which all images and landmarks have to be scaled to.
+    *@param configfile A variable of type std::string which stores the name of the file storing parameters for training the model.
+    *@param modelFilename A variable of type std::string which stores the name of the trained model file that has to be saved.
+    *@returns A boolean value. The function returns true if the model is trained properly or false if it is not trained.
+    */
+    virtual bool training(std::vector<Mat>& images, std::vector< std::vector<Point2f> >& landmarks,std::string configfile,Size scale,std::string modelFilename = "face_landmarks.dat")=0;
+    /** @brief This function is used to load the trained model..
+    *@param filename A variable of type cv::String which stores the name of the file in which trained model is stored.
+    */
+    virtual void loadModel(String filename)=0;
+    /** @brief This functions retrieves a centered and scaled face shape, according to the bounding rectangle.
+    *@param image A variable of type cv::InputArray which stores the image whose landmarks have to be found
+    *@param faces A variable of type cv::InputArray which stores the bounding boxes of faces found in a given image.
+    *@param landmarks A variable of type cv::InputOutputArray which stores the landmarks of all the faces found in the image
+    */
+    virtual bool fit( InputArray image, InputArray faces, InputOutputArray landmarks )=0;//!< from many ROIs
+    /// set the custom face detector
+    virtual bool setFaceDetector(bool(*f)(InputArray , OutputArray, void*), void* userData)=0;
+    /// get faces using the custom detector
+    virtual bool getFaces(InputArray image, OutputArray faces)=0;
+};
+
+}} // namespace
+#endif

modules/face/include/opencv2/face/facemark.hpp

@@ -3,7 +3,7 @@
 // of this distribution and at http://opencv.org/license.html.
 
 /*
-This file contains results of GSoC Project: Facemark API for OpenCV
+This file was part of GSoC Project: Facemark API for OpenCV
 Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
 Student: Laksono Kurnianggoro
 Mentor: Delia Passalacqua
@@ -73,6 +73,8 @@ cv::imshow("detection", frame);
 */
 CV_EXPORTS bool getFaces(InputArray image, OutputArray faces, CParams* params);
 
+CV_EXPORTS_W bool getFacesHAAR(InputArray image, OutputArray faces, const String& face_cascade_name);
+
 /** @brief A utility to load list of paths to training image and annotation file.
 @param imageList The specified file contains paths to the training images.
 @param annotationList The specified file contains paths to the training annotations.
@@ -163,6 +165,25 @@ CV_EXPORTS_W bool loadTrainingData( String imageList, String groundTruth,
                                     OutputArray facePoints,
                                     float offset = 0.0f);
 
+/** @brief This function extracts the data for training from .txt files which contains the corresponding image name and landmarks.
+*The first file in each file should give the path of the image whose
+*landmarks are being described in the file. Then in the subsequent
+*lines there should be coordinates of the landmarks in the image
+*i.e each line should be of the form x,y
+*where x represents the x coordinate of the landmark and y represents
+*the y coordinate of the landmark.
+*
+*For reference you can see the files as provided in the
+*<a href="http://www.ifp.illinois.edu/~vuongle2/helen/">HELEN dataset</a>
+*
+* @param filename A vector of type cv::String containing name of the .txt files.
+* @param trainlandmarks A vector of type cv::Point2f that would store shape or landmarks of all images.
+* @param trainimages A vector of type cv::String which stores the name of images whose landmarks are tracked
+* @returns A boolean value. It returns true when it reads the data successfully and false otherwise
+*/
+CV_EXPORTS_W bool loadTrainingData(std::vector<String> filename,std::vector< std::vector<Point2f> >
+                          &trainlandmarks,std::vector<String> & trainimages);
+
 /** @brief A utility to load facial landmark information from a given file.
 
 @param filename The filename of file contains the facial landmarks data.

modules/face/include/opencv2/face/facemarkAAM.hpp

@@ -1,9 +1,34 @@
-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-
 /*
-This file contains results of GSoC Project: Facemark API for OpenCV
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+                          License Agreement
+               For Open Source Computer Vision Library
+                       (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+  * Neither the names of the copyright holders nor the names of the contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
 Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
 Student: Laksono Kurnianggoro
 Mentor: Delia Passalacqua
@@ -55,7 +80,7 @@ class CV_EXPORTS_W FacemarkAAM : public Facemark
     struct CV_EXPORTS Config
     {
         Config( Mat rot = Mat::eye(2,2,CV_32F),
-                Point2f trans = Point2f(0.0f,0.0f),
+                Point2f trans = Point2f(0.0f, 0.0f),
                 float scaling = 1.0f,
                 int scale_id=0
         );

modules/face/include/opencv2/face/facemarkLBF.hpp

@@ -1,9 +1,34 @@
-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-
 /*
-This file contains results of GSoC Project: Facemark API for OpenCV
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+                          License Agreement
+               For Open Source Computer Vision Library
+                       (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+  * Neither the names of the copyright holders nor the names of the contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
 Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
 Student: Laksono Kurnianggoro
 Mentor: Delia Passalacqua

modules/face/samples/facemark_lbf_fitting.cpp

@@ -1,5 +1,34 @@
 /*
-This file contains results of GSoC Project: Facemark API for OpenCV
+By downloading, copying, installing or using the software you agree to this
+license. If you do not agree to this license, do not download, install,
+copy or use the software.
+                          License Agreement
+               For Open Source Computer Vision Library
+                       (3-clause BSD License)
+Copyright (C) 2013, OpenCV Foundation, all rights reserved.
+Third party copyrights are property of their respective owners.
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+  * Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+  * Neither the names of the copyright holders nor the names of the contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+This software is provided by the copyright holders and contributors "as is" and
+any express or implied warranties, including, but not limited to, the implied
+warranties of merchantability and fitness for a particular purpose are
+disclaimed. In no event shall copyright holders or contributors be liable for
+any direct, indirect, incidental, special, exemplary, or consequential damages
+(including, but not limited to, procurement of substitute goods or services;
+loss of use, data, or profits; or business interruption) however caused
+and on any theory of liability, whether in contract, strict liability,
+or tort (including negligence or otherwise) arising in any way out of
+the use of this software, even if advised of the possibility of such damage.
+
+This file was part of GSoC Project: Facemark API for OpenCV
 Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
 Student: Laksono Kurnianggoro
 Mentor: Delia Passalacqua
@@ -18,7 +47,7 @@ Mentor: Delia Passalacqua
 
 #include <stdio.h>
 #include <ctime>
-#include <iostream>
+ #include <iostream>
 #include "opencv2/core.hpp"
 #include "opencv2/highgui.hpp"
 #include "opencv2/imgproc.hpp"
@@ -137,34 +166,34 @@ bool parseArguments(int argc, char** argv, CommandLineParser & parser,
     String & model,
     String & video
 ){
-    const String keys =
-        "{ @c cascade         |      | (required) path to the cascade model file for the face detector }"
-        "{ @m model           |      | (required) path to the trained model }"
-        "{ @v video           |      | (required) path input video}"
-        "{ help h usage ?     |      | facemark_lbf_fitting -cascade -model -video [-t]\n"
-             " example: facemark_lbf_fitting ../face_cascade.xml ../LBF.model ../video.mp4}"
-    ;
-    parser = CommandLineParser(argc, argv,keys);
-    parser.about("hello");
-
-    if (parser.has("help")){
-        parser.printMessage();
-        return false;
-    }
-
-    cascade = String(parser.get<String>("cascade"));
-    model = String(parser.get<string>("model"));
-    video = String(parser.get<string>("video"));
-
-
-    if(cascade.empty() || model.empty() || video.empty() ){
-        std::cerr << "one or more required arguments are not found" << '\n';
-        cout<<"cascade : "<<cascade.c_str()<<endl;
-        cout<<"model : "<<model.c_str()<<endl;
-        cout<<"video : "<<video.c_str()<<endl;
-        parser.printMessage();
-        return false;
-    }
-
-    return true;
+   const String keys =
+       "{ @c cascade         |      | (required) path to the cascade model file for the face detector }"
+       "{ @m model           |      | (required) path to the trained model }"
+       "{ @v video           |      | (required) path input video}"
+       "{ help h usage ?     |      | facemark_lbf_fitting -cascade -model -video [-t]\n"
+            " example: facemark_lbf_fitting ../face_cascade.xml ../LBF.model ../video.mp4}"
+   ;
+   parser = CommandLineParser(argc, argv,keys);
+   parser.about("hello");
+
+   if (parser.has("help")){
+       parser.printMessage();
+       return false;
+   }
+
+   cascade = String(parser.get<String>("cascade"));
+   model = String(parser.get<string>("model"));
+   video = String(parser.get<string>("video"));
+
+
+   if(cascade.empty() || model.empty() || video.empty() ){
+       std::cerr << "one or more required arguments are not found" << '\n';
+       cout<<"cascade : "<<cascade.c_str()<<endl;
+       cout<<"model : "<<model.c_str()<<endl;
+       cout<<"video : "<<video.c_str()<<endl;
+       parser.printMessage();
+       return false;
+   }
+
+   return true;
 }