Merge pull request #62 from tue-robotics/robocup

Robocup RWC2018

Author: reinzor

README.md

@@ -15,6 +15,7 @@ Package | Build status Xenial Kinetic x64 | Description
 [openface_ros](https://github.com/tue-robotics/image_recognition/tree/master/openface_ros) | [![Build Status](http://build.ros.org/job/Ksrc_uX__openface_ros__ubuntu_xenial__source/1//badge/icon)](http://build.ros.org/job/Ksrc_uX__openface_ros__ubuntu_xenial__source/1/) | ROS wrapper for Openface (https://github.com/cmusatyalab/openface) to detect and recognize faces in images.
 [skybiometry_ros](https://github.com/tue-robotics/image_recognition/tree/master/skybiometry_ros) | [![Build Status](http://build.ros.org/job/Ksrc_uX__skybiometry_ros__ubuntu_xenial__source/1//badge/icon)](http://build.ros.org/job/Ksrc_uX__skybiometry_ros_ubuntu_xenial__source/1/) | ROS wrapper for Skybiometry (https://skybiometry.com/) for getting face properties of a detected face, e.g. age estimation, gender estimation etc.
 [openpose_ros](https://github.com/tue-robotics/image_recognition/tree/master/openpose_ros) | [![Build Status](http://build.ros.org/job/Ksrc_uX__openpose_ros__ubuntu_xenial__source/1//badge/icon)](http://build.ros.org/job/Ksrc_uX__openpose_ros_ubuntu_xenial__source/1/) | ROS wrapper for Openpose (https://github.com/CMU-Perceptual-Computing-Lab/) for getting poses of 2D images.
+[keras_ros](https://github.com/tue-robotics/image_recognition/tree/master/keras_ros) | [![Build Status](http://build.ros.org/job/Ksrc_uX__keras_ros__ubuntu_xenial__source/1//badge/icon)](http://build.ros.org/job/Ksrc_uX__keras_ros_ubuntu_xenial__source/1/) | ROS wrappers for Keras neural nets.
 
 ## Travis CI Build Status
 

image_recognition/package.xml

@@ -12,6 +12,7 @@
   <run_depend>image_recognition_msgs</run_depend>
 
   <run_depend>tensorflow_ros</run_depend>
+  <run_depend>keras_ros</run_depend>
   <run_depend>openface_ros</run_depend>
   <run_depend>skybiometry_ros</run_depend>
   <run_depend>openpose_ros</run_depend>

image_recognition_util/src/image_recognition_util/image_writer.py

@@ -32,6 +32,69 @@ def bitget(byteval, idx):
     return cmap
 
 
+def write_estimation(dir_path, image, label, annotated_original_image=None, suffix=""):
+    """
+    Write estimation to a directory, for the estimation, a directory of the run will be created
+    """
+    if dir_path is None:
+        return False
+
+    # Check if path exists, otherwise created it
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+
+    # Check if path exists, otherwise created it
+    estimations_dir = dir_path + "/estimations"
+    if not os.path.exists(estimations_dir):
+        os.makedirs(estimations_dir)
+
+    # Make a directory of the estimation with current time
+    estimation_dir = "%s/%s%s" % (estimations_dir, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S_%f"), suffix)
+    os.makedirs(estimation_dir)
+
+    filename = "%s/%s.jpg" % (estimation_dir, label)
+    cv2.imwrite(filename, image)
+
+    if annotated_original_image is not None:
+        filename = "%s/annotated_original_image.jpg" % estimation_dir
+        cv2.imwrite(filename, annotated_original_image)
+
+    return True
+
+
+def write_estimations(dir_path, images, labels, annotated_original_image=None, suffix=""):
+    """
+    Write estimations to a directory, for each estimation cycle, a directory of the run will be created
+    """
+    assert len(images) == len(labels)
+
+    if dir_path is None:
+        return False
+
+    # Check if path exists, otherwise created it
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+
+    # Check if path exists, otherwise created it
+    estimations_dir = dir_path + "/estimations"
+    if not os.path.exists(estimations_dir):
+        os.makedirs(estimations_dir)
+
+    # Make a directory of the estimation with current time
+    estimation_dir = "%s/%s%s" % (estimations_dir, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S_%f"), suffix)
+    os.makedirs(estimation_dir)
+
+    for i, (image, label) in enumerate(zip(images, labels)):
+        filename = "%s/%s_%d.jpg" % (estimation_dir, label, i)
+        cv2.imwrite(filename, image)
+
+    if annotated_original_image is not None:
+        filename = "%s/annotated_original_image.jpg" % estimation_dir
+        cv2.imwrite(filename, annotated_original_image)
+
+    return True
+
+
 def write_annotated(dir_path, image, label, verified=False):
     """
     Write an image with an annotation to a folder

keras_ros/CMakeLists.txt

@@ -0,0 +1,19 @@
+cmake_minimum_required(VERSION 2.8.3)
+project(keras_ros)
+
+find_package(catkin REQUIRED COMPONENTS
+  rospy
+  image_recognition_msgs
+  image_recognition_util
+)
+
+catkin_python_setup()
+
+catkin_package()
+
+install(PROGRAMS
+  scripts/face_properties_node
+  scripts/get_face_properties
+  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+)
+

keras_ros/README.md

@@ -0,0 +1,47 @@
+# keras_ros
+
+Image recognition with use of Keras.
+
+## Installation
+
+See https://github.com/tue-robotics/image_recognition for installation instructions. 
+
+## ROS Node (face_properties_node)
+
+Age and gender estimation with use of WideResNet from https://github.com/yu4u/age-gender-estimation. You can download the pre-trained model here: https://github.com/yu4u/age-gender-estimation/releases/download/v0.5/weights.18-4.06.hdf5
+
+```
+rosrun keras_ros face_properties_node _weights_file_path:=[path_to_model]
+```
+
+Run the image_recognition_rqt test gui (https://github.com/tue-robotics/image_recognition_rqt)
+
+    rosrun image_recognition_rqt test_gui
+    
+Configure the service you want to call with the gear-wheel in the top-right corner of the screen. If everything is set-up, draw a rectangle in the image around a face:
+
+![Wide ResNet](doc/wide_resnet_test.png)
+
+## Scripts
+
+### Get face properties (get_face_properties)
+
+Get the classification result of an input image:
+
+```
+rosrun keras_ros get_face_properties --image doc/face.png --weights-path [path_to_model]
+```
+
+![Example](doc/face.png)
+
+Output: 
+
+    [(50.5418073660112, array([0.5845756 , 0.41542447], dtype=float32))]
+
+## Troubleshooting
+
+```
+An error occurred: softmax() got an unexpected keyword argument 'axis'
+```
+
+Make sure you have tensorflow version >= 1.5.

keras_ros/doc/face.png

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<package>
+  <name>keras_ros</name>
+  <version>0.0.1</version>
+  <description>The keras_ros package</description>
+
+  <maintainer email="[email protected]">Rein Appeldoorn</maintainer>
+
+  <license>TODO</license>
+
+  <buildtool_depend>catkin</buildtool_depend>
+  <build_depend>rospy</build_depend>
+  <build_depend>image_recognition_msgs</build_depend>
+  <build_depend>image_recognition_util</build_depend>
+
+  <run_depend>rospy</run_depend>
+  <run_depend>image_recognition_msgs</run_depend>
+  <run_depend>python-keras-pip</run_depend>
+  <run_depend>python-numpy</run_depend>
+  <run_depend>python-opencv</run_depend>
+  <run_depend>python-tensorflow-pip</run_depend>
+</package>

keras_ros/doc/wide_resnet_test.png

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+import datetime
+import os
+import sys
+
+import cv2
+import rospy
+from cv_bridge import CvBridge, CvBridgeError
+from image_recognition_msgs.msg import FaceProperties
+from image_recognition_msgs.srv import GetFaceProperties
+from image_recognition_util import image_writer
+from keras_ros.age_gender_estimator import AgeGenderEstimator
+
+
+class WideResnetFaceProperties:
+    def __init__(self, weights_file_path, img_size, depth, width, save_images_folder):
+        """
+        ROS node that wraps the Wide Resnet age gender estimator
+        """
+        self._bridge = CvBridge()
+        self._properties_srv = rospy.Service('get_face_properties', GetFaceProperties, self._get_face_properties_srv)
+        self._estimator = AgeGenderEstimator(weights_file_path, img_size, depth, width)
+
+        if save_images_folder:
+            self._save_images_folder = os.path.expanduser(save_images_folder)
+            if not os.path.exists(self._save_images_folder):
+                os.makedirs(self._save_images_folder)
+        else:
+            self._save_images_folder = None
+
+        rospy.loginfo("WideResnetFaceProperties node initialized:")
+        rospy.loginfo(" - weights_file_path=%s", weights_file_path)
+        rospy.loginfo(" - img_size=%s", img_size)
+        rospy.loginfo(" - depth=%s", depth)
+        rospy.loginfo(" - width=%s", width)
+        rospy.loginfo(" - save_images_folder=%s", save_images_folder)
+
+    def _get_face_properties_srv(self, req):
+        """
+        Callback when the GetFaceProperties service is called
+        :param req: Input images
+        :return: properties
+        """
+        # Convert to opencv images
+        try:
+            bgr_images = [self._bridge.imgmsg_to_cv2(image, "bgr8") for image in req.face_image_array]
+        except CvBridgeError as e:
+            raise Exception("Could not convert image to opencv image: %s" % str(e))
+
+        rospy.loginfo("Estimating the age and gender of %d incoming images ...", len(bgr_images))
+        estimations = self._estimator.estimate(bgr_images)
+        rospy.loginfo("Done")
+
+        face_properties_array = []
+        for (age, gender) in estimations:
+            face_properties_array.append(FaceProperties(
+                age=int(age),
+                gender=FaceProperties.FEMALE if gender[0] > 0.5 else FaceProperties.MALE,
+            ))
+
+        # Store images if specified
+        if self._save_images_folder:
+            def _get_label(p):
+                return "age_%d_gender_%s" % (p.age, "male" if p.gender == FaceProperties.MALE else "female")
+
+            image_writer.write_estimations(self._save_images_folder, bgr_images,
+                                           [_get_label(p) for p in face_properties_array],
+                                           suffix="_face_properties")
+
+        # Service response
+        return {"properties_array": face_properties_array}
+
+if __name__ == '__main__':
+    rospy.init_node("face_properties")
+
+    try:
+        weights_file_path = rospy.get_param("~weights_file_path")
+        img_size = rospy.get_param("~image_size", 64)
+        depth = rospy.get_param("~depth", 16)
+        width = rospy.get_param("~width", 8)
+        save_images = rospy.get_param("~save_images", True)
+
+        save_images_folder = None
+        if save_images:
+            save_images_folder = rospy.get_param("~save_images_folder", "/tmp/keras_ros")
+    except KeyError as e:
+        rospy.logerr("Parameter %s not found" % e)
+        sys.exit(1)
+
+    try:
+        WideResnetFaceProperties(weights_file_path, img_size, depth, width, save_images_folder)
+        rospy.spin()
+    except Exception as e:
+        rospy.logfatal(e)
+

keras_ros/package.xml

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import argparse
+from keras_ros.age_gender_estimator import AgeGenderEstimator
+import cv2
+
+# Assign description to the help doc
+parser = argparse.ArgumentParser(description='Get face properties using WideResnet')
+
+# Add arguments
+parser.add_argument('--image', type=str, help='Image', required=True)
+parser.add_argument('--weights-path', type=str, help='Path to the weights of the WideResnet model', required=True)
+parser.add_argument('--image-size', type=int, help='Size of the input image', default=64)
+parser.add_argument('--depth', type=int, help='Depth of the network', default=16)
+parser.add_argument('--width', type=int, help='Width of the network', default=8)
+
+args = parser.parse_args()
+
+# Read the image
+img = cv2.imread(args.image)
+
+# Create Skybiometry iface
+estimator = AgeGenderEstimator(args.weights_path, args.image_size, args.depth, args.width)
+
+# Pretty print the output
+try:
+    print estimator.estimate([img])
+except Exception as e:
+    print "An error occurred: %s" % e