diff --git a/cpp/DarknetDetection/CMakeLists.txt b/cpp/DarknetDetection/CMakeLists.txt index b3135945..85576ee0 100644 --- a/cpp/DarknetDetection/CMakeLists.txt +++ b/cpp/DarknetDetection/CMakeLists.txt @@ -31,7 +31,7 @@ set(CMAKE_CXX_STANDARD 11) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 COMPONENTS opencv_core opencv_highgui) +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_core opencv_highgui) find_package(mpfComponentInterface REQUIRED) diff --git a/cpp/DarknetDetection/darknet_lib/CMakeLists.txt b/cpp/DarknetDetection/darknet_lib/CMakeLists.txt index b87fe280..3bd85f60 100644 --- a/cpp/DarknetDetection/darknet_lib/CMakeLists.txt +++ b/cpp/DarknetDetection/darknet_lib/CMakeLists.txt @@ -78,7 +78,7 @@ set(DARKNET_LIB_SRC_FILES src/yolo_layer.c src/iseg_layer.c) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 COMPONENTS opencv_core opencv_highgui) +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_core opencv_highgui) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/cpp/DarknetDetection/darknet_wrapper/CMakeLists.txt b/cpp/DarknetDetection/darknet_wrapper/CMakeLists.txt index c319b6c1..227e2f8c 100644 --- a/cpp/DarknetDetection/darknet_wrapper/CMakeLists.txt +++ b/cpp/DarknetDetection/darknet_wrapper/CMakeLists.txt @@ -30,7 +30,7 @@ project(darknet-wrapper) set(CMAKE_CXX_STANDARD 11) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 COMPONENTS opencv_core opencv_highgui) +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_core opencv_highgui) find_package(mpfComponentInterface REQUIRED) find_package(mpfDetectionComponentApi REQUIRED) diff --git a/cpp/DlibFaceDetection/CMakeLists.txt b/cpp/DlibFaceDetection/CMakeLists.txt index b82c5695..9dccaef1 100644 --- a/cpp/DlibFaceDetection/CMakeLists.txt +++ b/cpp/DlibFaceDetection/CMakeLists.txt @@ -37,7 +37,7 @@ set_property(TARGET dlib PROPERTY POSITION_INDEPENDENT_CODE ON) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 COMPONENTS opencv_highgui) +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_highgui) find_package(mpfComponentInterface REQUIRED) find_package(mpfDetectionComponentApi REQUIRED) find_package(mpfComponentUtils REQUIRED) diff --git a/cpp/OalprLicensePlateTextDetection/CMakeLists.txt b/cpp/OalprLicensePlateTextDetection/CMakeLists.txt index 355b9650..50d7f5aa 100644 --- a/cpp/OalprLicensePlateTextDetection/CMakeLists.txt +++ b/cpp/OalprLicensePlateTextDetection/CMakeLists.txt @@ -32,7 +32,7 @@ set(CMAKE_CXX_STANDARD 11) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7) +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION}) find_package(mpfComponentInterface REQUIRED) find_package(mpfDetectionComponentApi REQUIRED) find_package(mpfComponentUtils REQUIRED) diff --git a/cpp/OcvDnnDetection/CMakeLists.txt b/cpp/OcvDnnDetection/CMakeLists.txt index dfa8365f..f5927f85 100644 --- a/cpp/OcvDnnDetection/CMakeLists.txt +++ b/cpp/OcvDnnDetection/CMakeLists.txt @@ -32,7 +32,7 @@ set(CMAKE_CXX_STANDARD 11) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_dnn) find_package(mpfComponentInterface REQUIRED) diff --git a/cpp/OcvFaceDetection/CMakeLists.txt b/cpp/OcvFaceDetection/CMakeLists.txt index 626e9bab..eb70822c 100644 --- a/cpp/OcvFaceDetection/CMakeLists.txt +++ b/cpp/OcvFaceDetection/CMakeLists.txt @@ -32,7 +32,7 @@ set(CMAKE_CXX_STANDARD 11) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_highgui opencv_objdetect opencv_features2d opencv_ml opencv_flann opencv_video) find_package(mpfComponentInterface REQUIRED) diff --git a/cpp/OcvSsdFaceDetection/.devcontainer/Dockerfile b/cpp/OcvSsdFaceDetection/.devcontainer/Dockerfile new file mode 100644 index 00000000..47202726 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.devcontainer/Dockerfile @@ -0,0 +1,57 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# +from openmpf_cpp_component_build:gpu + +RUN yum install --assumeyes \ + git gdb which mlocate \ + jasper-devel openexr-devel libwebp-devel \ + bzip2 libtiff-devel libdc1394-devel tbb-devel libv4l-devel \ + gstreamer-plugins-base-devel ffmpeg-devel \ + openblas openblas-devel lapack lapack-devel \ + gcc-gfortran gcc-c++ + +RUN echo '/usr/local/lib64' > /etc/ld.so.conf.d/usr_local_lib64.conf; \ + ldconfig; + +RUN mkdir -p /tmp/dlib; \ + cd /tmp/dlib; \ + [[ -e dlib-19.20 ]] || curl --location 'http://dlib.net/files/dlib-19.20.tar.bz2' | tar --extract --bzip2; \ + cd dlib-19.20; \ + mkdir build; \ + cd build; \ + cmake3 -DBUILD_SHARED_LIBS:BOOL="1" ..; \ + cmake3 --build . --config Release; \ + make --jobs "$(nproc)" install; \ + ldconfig; + +RUN ln -s /usr/bin/cmake3 /usr/bin/cmake; + +LABEL org.label-schema.license="Mixed" \ + org.label-schema.name="OpenMPF OpenCV SSD Face Detection Dev GPU" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.url="https://openmpf.github.io" \ + org.label-schema.vcs-url="https://github.com/openmpf/openmpf-components" \ + org.label-schema.vendor="MITRE" \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/.devcontainer/devcontainer.json b/cpp/OcvSsdFaceDetection/.devcontainer/devcontainer.json new file mode 100644 index 00000000..d6efd82c --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +{ + "name": "OcvSsdFaceDetection", + + // The order of the files is important since later files override previous ones + "dockerComposeFile": ["./docker-compose.yml"], + + "service": "OcvSsdFaceDetection", + "workspaceFolder": "/home/mpf/openmpf-components/cpp/OcvSsdFaceDetection", + "shutdownAction": "stopCompose", + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "sleistner.vscode-fileutils", + "bierner.markdown-preview-github-styles", + "ryanmcalister.unotes", + "redhat.vscode-yaml", + "matepek.vscode-catch2-test-adapter" + ], + "settings":{ + "gtest-adapter.debugConfig": "Debug", + "cmake.ctestPath": "/usr/bin/ctest3", + "files.exclude": { + // "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "${workspaceRoot}/build": true, + "${workspaceRoot}/.devcontainer/fsroot": true + } + } +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/.devcontainer/docker-compose.yml b/cpp/OcvSsdFaceDetection/.devcontainer/docker-compose.yml new file mode 100644 index 00000000..cd15ad06 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.devcontainer/docker-compose.yml @@ -0,0 +1,35 @@ +version: '2.3' + +services: + OcvSsdFaceDetection: + cpuset: "0,1,2,3" + #cpus: "1.0" + image: openmpf/openmpf_cpp_component_dev + runtime: nvidia + build: + context: . + dockerfile: Dockerfile + environment: + MPF_LOG_PATH: . + THIS_MPF_NODE: . + NVIDIA_VISIBLE_DEVICES: all + CUDA_DEVICE_ORDER: PCI_BUS_ID + volumes: + # Mounts the project folder to '/workspace'. While this file is in .devcontainer, + # mounts are relative to the first file in the list, which is a level up. + - ../../../../openmpf-components:/home/mpf/openmpf-components:cached + + + # [Optional] Required for ptrace-based debuggers like C++, Go, and Rust + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "while sleep 1000; do :; done" + +networks: + default: + external: + name: localdev diff --git a/cpp/OcvSsdFaceDetection/.vscode/c_cpp_properties.json b/cpp/OcvSsdFaceDetection/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..331096eb --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.vscode/c_cpp_properties.json @@ -0,0 +1,23 @@ +{ + "configurations": [ + { + "name": "Linux", + "defines": [], + "compilerPath": "/usr/bin/gcc", + "intelliSenseMode": "gcc-x64", + "configurationProvider": "vector-of-bool.cmake-tools", + "includePath": [ + "${workspaceFolder}", + "/usr/include/**", + "/opt/opencv-${OPENCV_VERSION}/include/**", + "/home/mpf/mpf-sdk-install/include/**" + ], + "browse": { + "path": ["${workspaceFolder}" + ], + "limitSymbolsToIncludedHeaders": true + } + } + ], + "version": 4 +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/.vscode/launch.json b/cpp/OcvSsdFaceDetection/.vscode/launch.json new file mode 100644 index 00000000..982b2608 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.vscode/launch.json @@ -0,0 +1,33 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/build//test/OcvSsdFaceDetectionTest", + // "args": ["./test/data/test_imgs/meds_faces_image.png"], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/build/test", + "environment": [], + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Enable all-exceptions", + "text": "catch throw", + "ignoreFailures": true + } + ] + } + + ] +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/.vscode/settings.json b/cpp/OcvSsdFaceDetection/.vscode/settings.json new file mode 100644 index 00000000..b5a3136a --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.vscode/settings.json @@ -0,0 +1,36 @@ +{ + "cmake.ctestPath": "/usr/bin/ctest3", + "editor.tabSize": 2, + "editor.insertSpaces": true, + "editor.detectIndentation": false, + "files.exclude": { + "**/.git":true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "${workspaceRoot}/build": true, + "${workspaceRoot}/.devcontainer/fsroot": true + }, + "files.watcherExclude": { + "**/.git/objects/**": true, + "**/.git/subtree-cache/**": true, + "**/node_modules/*/**": true, + "/usr/include/**": true, + "/home/mpf/mpf-sdk-install/include/**": true, + "${workspaceRoot}/build": true, + "${workspaceRoot}/.devcontainer/fsroot": true + }, + "git.ignoreLimitWarning": true, + "files.associations": { + "*.ipp": "cpp", + "array": "cpp", + "*.tcc": "cpp", + "functional": "cpp", + "future": "cpp", + "istream": "cpp", + "tuple": "cpp", + "utility": "cpp", + "limits": "cpp" + } +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/.vscode/tasks.json b/cpp/OcvSsdFaceDetection/.vscode/tasks.json new file mode 100644 index 00000000..4a36ee94 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/.vscode/tasks.json @@ -0,0 +1,19 @@ +{ + "tasks": [ + { + "type": "shell", + "label": "gcc build active file", + "command": "/usr/bin/gcc", + "args": [ + "-g", + "${file}", + "-o", + "${fileDirname}/${fileBasenameNoExtension}" + ], + "options": { + "cwd": "/usr/bin" + } + } + ], + "version": "2.0.0" +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/CMakeLists.txt b/cpp/OcvSsdFaceDetection/CMakeLists.txt new file mode 100644 index 00000000..d0e43abb --- /dev/null +++ b/cpp/OcvSsdFaceDetection/CMakeLists.txt @@ -0,0 +1,106 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + + +cmake_minimum_required(VERSION 3.6) +project(ocv-ssd-face-detection) + +set(CMAKE_CXX_STANDARD 11) + +#set(CMAKE_BUILD_TYPE RelWithDebInfo) # build release version with debug symbols +set(CMAKE_BUILD_TYPE Debug) # build debug version + +add_definitions(-Wfatal-errors) +include(../ComponentSetup.cmake) + +# Specify dlib source directory here. +# Otherwise, disable the following three lines if using find_package(dlib) instead. +#set(DLIB_DIR /apps/source/dlib-sources/dlib-19.19/dlib) +#include(${DLIB_DIR}/cmake) +#set_property(TARGET dlib PROPERTY POSITION_INDEPENDENT_CODE ON) + +# Uncomment if not specifying dlib source directory. +# Warning: may not work with dlib release installation. +# find_package(dlib REQUIRED) + +find_package(OpenCV + REQUIRED + COMPONENTS + opencv_dnn + opencv_video + opencv_face + opencv_tracking +) +if(${OpenCV_VERSION} VERSION_LESS 3.4.7) + message(FATAL_ERROR "OpenCV ${OpenCV_VERSION} is below minimum 3.4.7") +endif() + +find_package(mpfComponentInterface REQUIRED) +find_package(mpfDetectionComponentApi REQUIRED) +find_package(mpfComponentUtils REQUIRED) +find_package(Qt4 REQUIRED) + +IF(CUDA_FOUND) + MESSAGE( STATUS "I found CUDA !" ) + SET(HAVE_CUDA ${CUDA_FOUND} CACHE BOOL "Set to TRUE if CUDA is found, FALSE otherwise") + set (CMAKE_CXX_FLAGS "-DHAVE_CUDA=1") +ENDIF(CUDA_FOUND) + +find_package(GTest) +if (${GTEST_FOUND}) + message("-- found GTest, configuring tests") + enable_testing() + include(Dart) +endif() + +set(OCV_SSD_FACE_DETECTION_SOURCE_FILES + util.cpp + JobConfig.cpp + DetectionLocation.cpp + Track.cpp + KFTracker.cpp + OcvSsdFaceDetection.cpp) + +add_library(mpfOcvSsdFaceDetection SHARED ${OCV_SSD_FACE_DETECTION_SOURCE_FILES}) +target_link_libraries(mpfOcvSsdFaceDetection + mpfComponentInterface + mpfDetectionComponentApi + mpfComponentUtils + ${OpenCV_LIBS} + dlib) + +configure_mpf_component(OcvSsdFaceDetection TARGETS mpfOcvSsdFaceDetection) + +add_subdirectory(test) + +add_executable( sample_ocv_ssd_face_detector sample_ocv_ssd_face_detector.cpp) +target_link_libraries(sample_ocv_ssd_face_detector mpfOcvSsdFaceDetection) + +# update configs if they change +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + plugin-files/config/mpfOcvSsdFaceDetection.ini + plugin-files/config/Log4cxxConfig.xml + plugin-files/descriptor/descriptor.json) \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/COPYING b/cpp/OcvSsdFaceDetection/COPYING new file mode 100644 index 00000000..19dc35b2 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/COPYING @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/DetectionLocation.cpp b/cpp/OcvSsdFaceDetection/DetectionLocation.cpp new file mode 100644 index 00000000..ccd0602b --- /dev/null +++ b/cpp/OcvSsdFaceDetection/DetectionLocation.cpp @@ -0,0 +1,631 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include +#include +#include +#include + +#include "util.h" +#include "DetectionLocation.h" +#include "Track.h" + +using namespace MPF::COMPONENT; + +// Shared static members (might need mutex locks and condition variable if multithreading... ) +log4cxx::LoggerPtr DetectionLocation::_log; +cv::dnn::Net DetectionLocation::_ssdNet; ///< single shot DNN face detector network +cv::dnn::Net DetectionLocation::_openFaceNet; ///< feature generator +unique_ptr DetectionLocation::_shapePredFuncPtr = NULL; ///< landmark detector function pointer +string DetectionLocation::_modelsPath; ///< where to find model files + +/** **************************************************************************** +* Draw polylines to visualize landmark features +* +* \param im image to draw landmarks on +* \param landmarks all landmark point only some of which will be drawn +* \param start start landmark point index for polyline +* \param end end landmark point index for polyline +* \param isClosed if true polyline draw a closed shape (end joined to start) +* \param drawColor color to use for drawing +* +***************************************************************************** */ +void drawPolyline(cv::Mat &im, const cvPoint2fVec &landmarks, + const int start, const int end, bool isClosed = false, + const cv::Scalar drawColor = cv::Scalar(255, 200,0)){ + cvPointVec points; + for (int i = start; i <= end; i++){ + points.push_back(cv::Point(landmarks[i].x, landmarks[i].y)); + } + cv::polylines(im, points, isClosed, drawColor, 2, 16); +} + +/** **************************************************************************** +* Visualize landmark point on image by drawing them. If 68 landmarks are +* available, they are drawn as polygons, otherwise just as points +* +* \param [in,out] im image to draw landmarks on +* \param drawColor color to use for drawing +* +***************************************************************************** */ +void DetectionLocation::drawLandmarks(cv::Mat &img, + const cv::Scalar drawColor = cv::Scalar(255, 200,0)) const { + cvPoint2fVec landmarks = getLandmarks(); + if(landmarks.size() == 68){ + drawPolyline(img, landmarks, 0, 16, false, drawColor); // Jaw line + drawPolyline(img, landmarks, 17, 21, false, drawColor); // Left eyebrow + drawPolyline(img, landmarks, 22, 26, false, drawColor); // Right eyebrow + drawPolyline(img, landmarks, 27, 30, false, drawColor); // Nose bridge + drawPolyline(img, landmarks, 30, 35, true, drawColor); // Lower nose + drawPolyline(img, landmarks, 36, 41, true, drawColor); // Left eye + drawPolyline(img, landmarks, 42, 47, true, drawColor); // Right Eye + drawPolyline(img, landmarks, 48, 59, true, drawColor); // Outer lip + drawPolyline(img, landmarks, 60, 67, true, drawColor); // Inner lip + }else { + for(size_t i = 0; i < landmarks.size(); i++){ + cv::circle(img,landmarks[i],3, drawColor, cv::FILLED); + } + } +} + +/** ************************************************************************** +* Compute (1 - Intersection Over Union) metric between a rectangel and detection +* comprised of 1 - the ratio of the area of the intersection of the detection +* rectangles divided by the area of the union of the detection rectangles. +* +* \param rect rectangle to compute iou with +* \returns 1- intersection over union [0.0 ... 1.0] +* +*************************************************************************** */ +float DetectionLocation::_iouDist(const cv::Rect2i &rect) const { + int ulx = max(x_left_upper , rect.x ); + int uly = max(y_left_upper , rect.y ); + int lrx = min(x_left_upper + width , rect.x + rect.width ); + int lry = min(y_left_upper + height, rect.y + rect.height); + + float inter_area = max(0, lrx - ulx) * max(0, lry - uly); + float union_area = width * height + rect.width * rect.height - inter_area; + float dist = 1.0f - inter_area / union_area; LOG4CXX_TRACE(_log,"iou dist = " << dist); + return dist; +} + +/** ************************************************************************** +* Compute 1 - Intersection Over Union metric between track tail and detection +* +* \param tr track +* \returns 1- intersection over union [0.0 ... 1.0] +* +*************************************************************************** */ +float DetectionLocation::iouDist(const Track &tr) const { + return _iouDist(cv::Rect2i(tr.back()->x_left_upper, + tr.back()->y_left_upper, + tr.back()->width, + tr.back()->height)); +} + +/** ************************************************************************** +* Compute 1 - Intersection Over Union metric between Kalman filter predicted +* location of track tail and a detection +* +* \param tr track +* \returns 1- intersection over union [0.0 ... 1.0] +* +*************************************************************************** */ +float DetectionLocation::kfIouDist(const Track &tr) const { + return _iouDist(tr.kalmanPredictedBox()); +} + +/** ************************************************************************** +* Compute the temporal distance (frame count) between track tail and detection +* +* \param tr track +* \returns absolute difference in frame indices +* +*************************************************************************** */ +float DetectionLocation::frameDist(const Track &tr) const { + if(frameIdx > tr.back()->frameIdx){ + return frameIdx - tr.back()->frameIdx; + }else{ + return tr.back()->frameIdx - frameIdx; + } +} + +/** ************************************************************************** +* Compute euclidean center to distance center distance from normalized centers +* +* \param tr track +* \returns normalized center to center distance [0 ... Sqrt(2)] +* +*************************************************************************** */ +float DetectionLocation::center2CenterDist(const Track &tr) const { + float dx = center.x - tr.back()->center.x; + float dy = center.y - tr.back()->center.y; + float dist = sqrt( dx*dx + dy*dy ); LOG4CXX_TRACE(_log,"center-2-center dist = " << dist); + return dist; +} + + +/** ************************************************************************** +* Compute feature distance (similarity) to track's tail detection's +* feature vectors +* +* \param tr track +* \returns cos distance [0 ... 1.0] +* +* \note Feature vectors are expected to be of unit magnitude +* +*************************************************************************** */ +float DetectionLocation::featureDist(const Track &tr) const { + float dist = 1.0f - max(0.0f,static_cast(getFeature().dot(tr.back()->getFeature()))); LOG4CXX_TRACE(_log,"feature dist = " << dist); + return dist; +} + +/** ************************************************************************** +* Lazy accessor method to get/compute landmark points +* 5-Landmark detector returns outside and inside eye corners and bottom of nose +* +* +* \returns landmarks +* +*************************************************************************** */ +const cvPoint2fVec& DetectionLocation::getLandmarks() const { + if(_landmarks.empty()){ + + try{ + dlib::cv_image cimg(_bgrFrameRot); + cv::Rect2i bbox = rotate(getRect(), + _detectionOrientation, + cv::Size2i(_bgrFrameRot.cols-1,_bgrFrameRot.rows-1)); + dlib::full_object_detection + shape = (*_shapePredFuncPtr)(cimg, dlib::rectangle( bbox.x, // left + bbox.y, // top + bbox.x + bbox.width-1, // right + bbox.y + bbox.height-1)); // bottom + + cv::Size2f canvas(_bgrFrame.cols-1,_bgrFrame.rows-1); + for(size_t i=0; i(degCCWfromVertical(vec) + 0.5); + if(abs(angleDiff(angle, _angle)) < 45){ // check if it makes sense + _angle = angle; + }else{ // guess landmarks based on orientation + cv::Point2f orig(bbox.x,bbox.y); + _landmarks[2] = rotate(orig + cv::Point2f(bbox.width*0.1941570,bbox.height*0.16926692),inv(_detectionOrientation),canvas); + _landmarks[0] = rotate(orig + cv::Point2f(bbox.width*0.7888591,bbox.height*0.15817115),inv(_detectionOrientation),canvas); + _landmarks[4] = rotate(orig + cv::Point2f(bbox.width*0.4949509,bbox.height*0.51444140),inv(_detectionOrientation),canvas); + _landmarks[1] = _landmarks[3] = (_landmarks[0] + _landmarks[2]) / 2.0; + } + + }catch(...){ + exception_ptr eptr = current_exception(); + LOG4CXX_FATAL(_log, "failed to determine landmarks"); + rethrow_exception(eptr); + } + } + return _landmarks; +} + +/** ************************************************************************** +* Lazy accessor method to get/copy-create 92x92 thumbnail image for +* feature generation +* +* \returns thumbnail image of detection +* +*************************************************************************** */ +const cv::Mat& DetectionLocation::getThumbnail() const { + if(_thumbnail.empty()){ + const int THUMBNAIL_SIZE_X = 96; + const int THUMBNAIL_SIZE_Y = 96; + const cv::Size THUMB_SIZE(THUMBNAIL_SIZE_X,THUMBNAIL_SIZE_Y); + + // Landmark indices for OpenFace nn4.v2, nn4.small1.v1, nn4.small2.v1 + const size_t lmIdx[] = {2,0,4}; // if using 5 pt landmarks + const cv::Mat dst = (cv::Mat1f(3,2) + << 0.1941570 * THUMBNAIL_SIZE_X, 0.16926692 * THUMBNAIL_SIZE_Y, + 0.7888591 * THUMBNAIL_SIZE_X, 0.15817115 * THUMBNAIL_SIZE_Y, + 0.4949509 * THUMBNAIL_SIZE_X, 0.51444140 * THUMBNAIL_SIZE_Y); + + cv::Mat src = cv::Mat1f(3,2); + for(size_t r=0; r<3; r++){ + float* rowPtr = src.ptr(r); + rowPtr[0] = getLandmarks()[lmIdx[r]].x; + rowPtr[1] = getLandmarks()[lmIdx[r]].y; + } + cv::Mat xfrm = cv::getAffineTransform(src,dst); + try{ + _thumbnail = cv::Mat(THUMB_SIZE,_bgrFrame.type()); + cv::warpAffine(_bgrFrame,_thumbnail,cv::getAffineTransform(src,dst), + THUMB_SIZE,cv::INTER_CUBIC,cv::BORDER_REPLICATE); + }catch (...) { + exception_ptr eptr = current_exception(); + LOG4CXX_FATAL(_log, "failed to generate thumbnail for f" << frameIdx << *this); + rethrow_exception(eptr); + } + + } + return _thumbnail; +} + + +/** ************************************************************************** +* accessor method to get image associate with detection +* +* \returns image associated with detection +* +*************************************************************************** */ +const cv::Mat& DetectionLocation::getBGRFrame() const{ + #ifndef NDEBUG // debug build + if(! _bgrFrame.empty()){ + return _bgrFrame; + }else{ + LOG4CXX_TRACE(_log,"BRG frame missing for detection: f" << this->frameIdx << *this); + THROW_EXCEPTION("BRG frame is not allocated"); + } + #else // release build + return _bgrFrame; + #endif + +} + +/** ************************************************************************** +* accessor method to get face rotation angle +* +* \returns angle of face +* +*************************************************************************** */ +const int DetectionLocation::getAngle() const{ + return _angle; +} + +/** ************************************************************************** +* release reference to image frame +*************************************************************************** */ +void DetectionLocation::releaseBGRFrame() { LOG4CXX_TRACE(_log,"releasing bgrFrames for f" << this->frameIdx << *this); + _bgrFrameRot.release(); + _bgrFrame.release(); +} + +/** ************************************************************************** +* get the location as an opencv rectange +*************************************************************************** */ +const cv::Rect2i DetectionLocation::getRect() const { + return cv::Rect2i(x_left_upper,y_left_upper,width,height); +} + +/** ************************************************************************** +* set the location from an opencv rectange +*************************************************************************** */ +void DetectionLocation::setRect(const cv::Rect2i& rec){ + x_left_upper = rec.x; + y_left_upper = rec.y; + width = rec.width; + height = rec.height; +} + +/** ************************************************************************** +* copy feature vector and orientations from another detection +* +* \param d detection from which to copy feature vector +* +*************************************************************************** */ +void DetectionLocation::copyFeature(const DetectionLocation& d){ + _feature = d.getFeature(); + _angle = d._angle; + _detectionOrientation = d._detectionOrientation; +} + +/** ************************************************************************** +* Lazy accessor method to get/compute feature vector based on thumbnail +* +* \returns unit magnitude feature vector +* +*************************************************************************** */ +const cv::Mat& DetectionLocation::getFeature() const { + if(_feature.empty()){ + float aspect_ratio = width / height; + if( ( x_left_upper > 0 + && y_left_upper > 0 + && x_left_upper + width < _bgrFrame.cols - 1 + && y_left_upper + height < _bgrFrame.rows - 1) + || ( 0.5 < aspect_ratio && aspect_ratio < 2.0)){ + const double inScaleFactor = 1.0 / 255.0; + const cv::Size blobSize(96, 96); + const cv::Scalar meanVal(0.0, 0.0, 0.0); // BGR mean pixel color + + cv::Mat inputBlob = cv::dnn::blobFromImage(getThumbnail(), // BGR image + inScaleFactor, // no pixel value scaling (e.g. 1.0/255.0) + blobSize, // expected network input size: 90x90 + meanVal, // mean BGR pixel value + true, // swap RB channels + false); // center crop + _openFaceNet.setInput(inputBlob); + _feature = _openFaceNet.forward().clone(); // need to clone as mem gets reused + + }else{ // can't trust feature of detections on the edge + LOG4CXX_TRACE(_log,"'Zero-feature' for detection at frame edge with poor aspect ratio = " << aspect_ratio); + _feature = cv::Mat::zeros(1, 128, CV_32F); // zero feature will wipe out any dot products... + } + } + return _feature; +} + +/** **************************************************************************** +* Detect objects using SSD DNN opencv face detector network +* +* \param cfg job configuration setting including image frame +* +* \returns found face detections that meet size requirements. +* +* \note each detection hang on to a reference to the bgrFrame which +* should be released once no longer needed (i.e. features care computed) +* +*************************************************************************** */ +DetectionLocationPtrVec DetectionLocation::createDetections(const JobConfig &cfg){ + + DetectionLocationPtrVec detections; // detection vector to be returned + + // some vars for NMS + cvRectVec bboxes; // bounding boxes found for all orientations + cvPoint2fVec centers; // centers of found bounding boxes for all orientations + vector scores; // confidence scores of found detections for all orientations + vector orientations; // orientation for each found bounding box + + // setup related image variables + map rotImages; // rotated image cache for making detections objects + int maxSide = max(cfg.bgrFrame.cols,cfg.bgrFrame.rows); // maximum image side length + cv::Rect2i canvas(0,0,cfg.bgrFrame.cols,cfg.bgrFrame.rows); // canvas size for rotations + + // create zero padded square version of bgrFrame + cv::Mat square; + cv::copyMakeBorder(cfg.bgrFrame, square, + 0, maxSide - cfg.bgrFrame.rows, + 0, maxSide - cfg.bgrFrame.cols, + cv::BORDER_CONSTANT,cv::Scalar(0,0,0)); + int infSize = (cfg.inferenceSize > 0 ) ? cfg.inferenceSize : maxSide; // size to use for inferencing + + for(auto& orientation:cfg.inferenceOrientations){ + + cv::Mat image = rotate(square, orientation); + cv::Rect2i canvasRot = rotate(canvas, orientation,cv::Size2i(maxSide,maxSide)); + rotImages.insert(pair(orientation,image(canvasRot))); + + cv::Mat inputBlob = cv::dnn::blobFromImage(image, // BGR image + 1.0, // no pixel value scaling (e.g. 1.0/255.0) + cv::Size(infSize,infSize), // expected network input size: e.g.300x300 + cv::Scalar(104.0, 117.0, 124.0), // mean BGR pixel value + true, // swap RB channels + false); // center crop + + _ssdNet.setInput(inputBlob,"data"); + cv::Mat netOut = _ssdNet.forward("detection_out"); + cv::Mat_ res = cv::Mat(netOut.size[2], netOut.size[3], CV_32F, netOut.ptr(0)); + for(int i = 0; i < res.rows; i++){ + float conf = res(i,2); + if(conf > cfg.confThresh){ + cv::Rect2f rbox(res(i,3), res(i,4), res(i,5) - res(i,3),res(i,6) - res(i,4)); // normalized coords bounding box on rotated image + cv::Rect2f bbox = rotate(rbox, inv(orientation),cv::Size2f(1.0, 1.0)); // normalized coords cords bounding box on brgFrame + cv::Point2f center = 0.5 * (bbox.tl() + bbox.br()); // normalized coords bounding box center + cv::Point2f wh = cv::Point2f(bbox.width,bbox.height) * cfg.bboxScaleFactor; // normalized scaled width and height + + int width = static_cast(wh.x * maxSide); // width in image pixels + int height = static_cast(wh.y * maxSide); // height in image pixels + + if(width >= cfg.minDetectionSize && height >= cfg.minDetectionSize){ + cv::Point2f ul = center - 0.5 * wh; // normalized coords upper left coordinate + int x1 = static_cast(ul.x * maxSide); // upper left x in image pixels + int y1 = static_cast(ul.y * maxSide); // upper left y in image pixels + + cv::Rect2i iBbox = cv::Rect2i(x1,y1,width,height) & canvas; // bounding box in pixels on image + if(iBbox.width > 0 && iBbox.height > 0){ + bboxes.push_back(iBbox); + scores.push_back(conf); + centers.push_back(center); + orientations.push_back(orientation); + } + } + } + } + } + //#define DIAGNOSTIC_IMAGES + #ifdef DIAGNOSTIC_IMAGES + cv::Mat lm = cfg.bgrFrame.clone(); + #endif + // Perform non maximum supression (NMS) + vector keepIdxs; + cv::dnn::NMSBoxes(bboxes, scores, cfg.confThresh, cfg.nmsThresh, keepIdxs); + for(int i:keepIdxs){ + detections.push_back(DetectionLocationPtr( + new DetectionLocation(bboxes[i].x, bboxes[i].y, bboxes[i].width, bboxes[i].height, + scores[i], centers[i], + cfg.frameIdx, cfg.frameTimeInSec, + cfg.bgrFrame, + rotImages[orientations[i]], + orientations[i]))); LOG4CXX_TRACE(_log,"detection:" << *detections.back()); + #ifdef DIAGNOSTIC_IMAGES + stringstream fn; fn << "thumb_"<< cfg.frameIdx <<"_" << i << ".png"; + cv::imwrite(fn.str(),detections.back()->getThumbnail()); + detections.back()->drawLandmarks(lm,cv::Scalar(0,0,0)); + #endif + } + #ifdef DIAGNOSTIC_IMAGES + stringstream fn; fn << "landmarks_"<< cfg.frameIdx << ".png"; + cv::imwrite(fn.str(),lm); + #endif + + return detections; +} + +/** ************************************************************************** +*************************************************************************** */ +void DetectionLocation::_loadNets(const bool enabled){ + + // Load SSD Tensor Flow Network + string tf_model_path = _modelsPath + "opencv_face_detector_uint8.pb"; + string tf_config_path = _modelsPath + "opencv_face_detector.pbtxt"; + + string tr_model_path = _modelsPath + "nn4.small2.v1.t7"; + + // load detector net + _ssdNet = cv::dnn::readNetFromTensorflow(tf_model_path, tf_config_path); + // load feature generator + _openFaceNet = cv::dnn::readNetFromTorch(tr_model_path); + + #ifdef HAVE_CUDA + if(enabled){ + _ssdNet.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + _ssdNet.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + _openFaceNet.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + _openFaceNet.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); LOG4CXX_TRACE(_log,"Enabled CUDA acceleration on device " << cv::cuda::getDevice()); + }else{ + _ssdNet.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT); + _ssdNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + _openFaceNet.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT); + _openFaceNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); LOG4CXX_INFO(_log,"Disabled CUDA acceleration"); + } + #endif +} + +/** ************************************************************************** +* try set CUDA to use specified GPU device +* +* \param cudaDeviceId device to use for hardware acceleration (-1 to disable) +* +* \returns true if successful false otherwise +*************************************************************************** */ +bool DetectionLocation::loadNetToCudaDevice(const int cudaDeviceId){ + static int lastCudaDeviceId = -1; + const string err_msg = "Failed to configure CUDA for deviceID="; + try{ + #ifdef HAVE_CUDA + if(lastCudaDeviceId != cudaDeviceId){ + if(lastCudaDeviceId >=0){ + if(!_ssdNet.empty()) _ssdNet.~Net(); // need to release network this prior to device reset + if(!_openFaceNet.empty()) _openFaceNet.~Net(); + cv::cuda::resetDevice(); + } + if(cudaDeviceId >=0){ + cv::cuda::setDevice(cudaDeviceId); + _loadNets(true); + lastCudaDeviceId = cudaDeviceId; + }else{ + _loadNets(false); + lastCudaDeviceId = -1; + } + } + return true; + #endif + }catch(const runtime_error& re){ LOG4CXX_FATAL(_log, err_msg << cudaDeviceId << " Runtime error: " << re.what()); + }catch(const exception& ex){ LOG4CXX_FATAL(_log, err_msg << cudaDeviceId << " Exception: " << ex.what()); + }catch(...){ LOG4CXX_FATAL(_log, err_msg << cudaDeviceId << " Unknown failure occurred. Possible memory corruption"); + } + _loadNets(false); + lastCudaDeviceId = -1; + return false; +} + +/** ************************************************************************** +* Setup class shared static configurations and initialize / load shared +* detectors and feature generator objects. Set default CUDA acceleration +* +* \param log logger object for logging +* \param plugin_path plugin directory so configs and models can be loaded +* +* \return true if everything was properly initialized, false otherwise +*************************************************************************** */ +bool DetectionLocation::Init(log4cxx::LoggerPtr log, string plugin_path){ + + int cudaDeviceId = getEnv ({},"CUDA_DEVICE_ID",0); + + _log = log; + _modelsPath = plugin_path + "/data/"; + + const string err_msg = "Failed to load models from " + _modelsPath; + try{ + // load detector net + loadNetToCudaDevice(cudaDeviceId); + + _shapePredFuncPtr = unique_ptr(new dlib::shape_predictor()); + dlib::deserialize(_modelsPath + "shape_predictor_5_face_landmarks.dat") >> *_shapePredFuncPtr; + + }catch(const runtime_error& re){ LOG4CXX_FATAL(_log, err_msg << " Runtime error: " << re.what()); + return false; + }catch(const exception& ex){ LOG4CXX_FATAL(_log, err_msg << " Exception: " << ex.what()); + return false; + }catch(...){ LOG4CXX_FATAL(_log, err_msg << " Unknown failure occurred. Possible memory corruption"); + return false; + } + + return true; +} + + +/** ************************************************************************** +* Private constructor for a detection object +* +* \param x top left x coordinate of bounding box +* \param y top left y coordinate of bounding box +* \param width bounding box width +* \param height bounding box height +* \param conf confidence score of detection +* \param center normalized center coord of detection +* \param frameIdx video frame sequence number +* \param frameTimeInSec time stamp of video frame in sec +* \param bgrFrame frame in which detection was found +* \param bgrFrameRot bgrFrame rotated to detectionOrientation +* \param detectionOrientation orientation of the frame when detected +* +**************************************************************************** */ +DetectionLocation::DetectionLocation(int x,int y,int width,int height,float conf, + cv::Point2f center, + size_t frameIdx, double frameTimeInSec, + cv::Mat bgrFrame, + cv::Mat bgrFrameRot, + OrientationType detectionOrientation): + MPFImageLocation(x,y,width,height,conf), + center(center), + frameIdx(frameIdx), + frameTimeInSec(frameTimeInSec), + _detectionOrientation(detectionOrientation), + _angle(degCCWfromVertical(detectionOrientation)){ + _bgrFrame = bgrFrame; //.clone(); + _bgrFrameRot = bgrFrameRot; +} + +/** ************************************************************************** +* Dump MPFLocation to a stream +*************************************************************************** */ +ostream& MPF::COMPONENT::operator<< (ostream& out, const DetectionLocation& d) { + out << "[" << (MPFImageLocation)d + << " F[" << d.getFeature().size() << "] T[" + << d.getThumbnail().rows << "," << d.getThumbnail().cols << "]"; + return out; +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/DetectionLocation.h b/cpp/OcvSsdFaceDetection/DetectionLocation.h new file mode 100644 index 00000000..d5e50eb2 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/DetectionLocation.h @@ -0,0 +1,116 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#ifndef OCVSSDFACEDETECTION_DETECTIONLOCATION_H +#define OCVSSDFACEDETECTION_DETECTIONLOCATION_H + +#include + +#include +#include + +#include +#include + +#include "types.h" +#include "util.h" +#include "JobConfig.h" + +#define CALL_MEMBER_FUNC(object,ptrToMember) ((object).*(ptrToMember)) + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + using DetectionLocationCostFunc = float (DetectionLocation::*)(const Track &tr) const; ///< cost member-function pointer type + + class DetectionLocation: public MPFImageLocation{ // extend MPFImageLocation + + public: + using MPFImageLocation::MPFImageLocation; // C++11 inherit all constructors for MPFImageLocation + + const cv::Point2f center; ///< bounding box center normalized to image dimensions + const size_t frameIdx; ///< frame index frame where detection is located (for videos) + const double frameTimeInSec; ///< frame time in sec where detection is located (for videos) + + static bool Init(log4cxx::LoggerPtr log, const string plugin_path); ///< setup class shared members + static DetectionLocationPtrVec createDetections(const JobConfig &cfg); ///< created detection objects from image frame + + const cvPoint2fVec& getLandmarks() const; ///< get landmark points for detection + const cv::Mat& getThumbnail() const; ///< get thumbnail image for detection + const cv::Mat& getFeature() const; ///< get DNN features for detection + const cv::Mat& getBGRFrame() const; ///< get image data associated with detection + const int getAngle() const; ///< get face rotation angle + + const cv::Rect2i getRect() const; ///< get location as an opencv rectange + void setRect(const cv::Rect2i& rec); ///< set location from an opencv rectangle + + void copyFeature(const DetectionLocation& d); ///< copy DNN feature from another detection + + float iouDist(const Track &tr) const; ///< 1 - compute intersection over union + float kfIouDist(const Track &tr) const; ///< 1 - compute intersection over union using kalman predicted location + float frameDist(const Track &tr) const; ///< compute temporal frame gap + float center2CenterDist(const Track &tr) const; ///< compute normalized center to center distance + float featureDist(const Track &tr) const; ///< compute deep feature similarity distance + + void drawLandmarks(cv::Mat &img, const cv::Scalar drawColor) const; ///< draw landmark point on image + void releaseBGRFrame(); ///< release reference to image frame + static bool loadNetToCudaDevice(const int cudaDeviceId); ///< try set CUDA to use specified GPU device + + DetectionLocation(int x,int y,int width,int height,float conf, + cv::Point2f center, + size_t frameIdx, double frameTimeInMillis, + cv::Mat bgrFrame, + cv::Mat bgrFrameRot, + OrientationType detectionOrientation); ///< private constructor for createDetections() + private: + + static log4cxx::LoggerPtr _log; ///< shared log object + static cv::dnn::Net _ssdNet; ///< single shot DNN face detector network + static cv::dnn::Net _openFaceNet; ///< feature generator + static unique_ptr _shapePredFuncPtr; ///< landmark detector + static string _modelsPath; ///< where to find models files + + float _iouDist(const cv::Rect2i &rect) const; ///< compute intersectino over union + + mutable OrientationType _detectionOrientation; ///< rotation under which detection was aquired + mutable int _angle; ///< rotation angle of face; + mutable cvPoint2fVec _landmarks; ///< vector of landmarks (e.g. eyes, nose, etc.) + mutable cv::Mat _thumbnail; ///< 96x96 image comprising an aligned thumbnail + mutable cv::Mat _feature; ///< DNN feature for matching-up detections + cv::Mat _bgrFrame; ///< frame associated with detection (openCV memory managed :( ) + cv::Mat _bgrFrameRot; ///< rotated version of _bgrFrame + static void _loadNets(const bool enabled); ///< turn on or off cuda backend for inferencing + + }; + + ostream& operator<< (ostream& out, const DetectionLocation& d); + + } +} + +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/Dockerfile b/cpp/OcvSsdFaceDetection/Dockerfile new file mode 100644 index 00000000..02ca3aea --- /dev/null +++ b/cpp/OcvSsdFaceDetection/Dockerfile @@ -0,0 +1,71 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +ARG BUILD_REGISTRY +ARG BUILD_TAG=latest + +# Download library source code so it doesn't need to be re-downloaded when base images change +FROM centos:7 as download_library_source + +RUN yum install --assumeyes bzip2 \ + && yum clean all \ + && rm --recursive /var/cache/yum/* + +RUN mkdir --parents /apps/source/dlib-sources; \ + curl --location 'http://dlib.net/files/dlib-19.19.tar.bz2' \ + | tar --extract --bzip2 --directory /apps/source/dlib-sources + + +FROM ${BUILD_REGISTRY}openmpf_cpp_component_build:${BUILD_TAG} as build_component + +COPY --from=download_library_source /apps/source/dlib-sources /apps/source/dlib-sources +COPY . . + +RUN cd /apps/source/dlib-sources/dlib-19.19/dlib && mkdir build && cd build && cmake3 -DBUILD_SHARED_LIBS:BOOL="1" .. \ + && cmake3 --build . --config Release && make --jobs "$(nproc)" install && ldconfig + +RUN build-component.sh + +ARG RUN_TESTS=false +RUN if [ "${RUN_TESTS,,}" == true ]; then cd $BUILD_DIR/test && ./OcvSsdFaceDetectionTest; fi + + + +FROM ${BUILD_REGISTRY}openmpf_cpp_executor:${BUILD_TAG} as install_component + +ENV COMPONENT_LOG_NAME ocv-ssd-face-detection.log + +COPY --from=build_component $BUILD_DIR/plugin/OcvSsdFaceDetection $PLUGINS_DIR/OcvSsdFaceDetection + +COPY --from=build_component $BUILD_DIR/libmpfOcvSsdFaceDetection.so \ + $PLUGINS_DIR/OcvSsdFaceDetection/lib/ + +LABEL org.label-schema.license="Apache 2.0" \ + org.label-schema.name="OpenMPF OpenCV SSD Face Detection" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.url="https://openmpf.github.io" \ + org.label-schema.vcs-url="https://github.com/openmpf/openmpf-components" \ + org.label-schema.vendor="MITRE" diff --git a/cpp/OcvSsdFaceDetection/JobConfig.cpp b/cpp/OcvSsdFaceDetection/JobConfig.cpp new file mode 100644 index 00000000..43377ed1 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/JobConfig.cpp @@ -0,0 +1,221 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include "types.h" +#include "util.h" +#include "JobConfig.h" + +using namespace MPF::COMPONENT; + +/** ************************************************************************** +* Parse a string into a vector +* e.g. [1,2,3,4] +*************************************************************************** */ +template +vector _fromString(const string data){ + string::size_type begin = data.find('[') + 1; + string::size_type end = data.find(']', begin); + stringstream ss( data.substr(begin, end - begin) ); + vector ret; + T val; + while(ss >> val){ + ret.push_back(val); + ss.ignore(); // ignore seperating tokens e.g. ',' + } + return ret; +} + + +/** ************************************************************************** +* Parse argument fromMPFJob structure to our job objects +*************************************************************************** */ +void JobConfig::_parse(const MPFJob &job){ + const Properties jpr = job.job_properties; + minDetectionSize = abs(getEnv (jpr,"MIN_DETECTION_SIZE", minDetectionSize)); LOG4CXX_TRACE(_log, "MIN_DETECTION_SIZE: " << minDetectionSize); + confThresh = abs(getEnv(jpr,"DETECTION_CONFIDENCE_THRESHOLD", confThresh)); LOG4CXX_TRACE(_log, "DETECTION_CONFIDENCE_THRESHOLD: " << confThresh); + nmsThresh = abs(getEnv(jpr,"DETECTION_NMS_THRESHOLD", nmsThresh)); LOG4CXX_TRACE(_log, "DETECTION_NMS_THRESHOLD: " << nmsThresh); + inferenceSize = getEnv (jpr,"DETECTION_INFERENCE_SIZE", inferenceSize); LOG4CXX_TRACE(_log, "DETECTION_INFERENCE_SIZE: " << inferenceSize); + rotateDetect = getEnv (jpr,"ROTATE_AND_DETECT", rotateDetect); LOG4CXX_TRACE(_log, "ROTATE_AND_DETECT: " << rotateDetect); + detFrameInterval = abs(getEnv (jpr,"DETECTION_FRAME_INTERVAL", detFrameInterval)); LOG4CXX_TRACE(_log, "DETECTION_FRAME_INTERVAL: " << detFrameInterval); + bboxScaleFactor = abs(getEnv(jpr,"DETECTION_BOUNDING_BOX_SCALE_FACTOR",bboxScaleFactor)); LOG4CXX_TRACE(_log, "DETECTION_BOUNDING_BOX_SCALE_FACTOR: "<< bboxScaleFactor); + + maxFeatureDist = abs(getEnv(jpr,"TRACKING_MAX_FEATURE_DIST", maxFeatureDist)); LOG4CXX_TRACE(_log, "TRACKING_MAX_FEATURE_DIST: " << maxFeatureDist); + maxFrameGap = abs(getEnv (jpr,"TRACKING_MAX_FRAME_GAP", maxFrameGap)); LOG4CXX_TRACE(_log, "TRACKING_MAX_FRAME_GAP: " << maxFrameGap); + maxCenterDist = abs(getEnv(jpr,"TRACKING_MAX_CENTER_DIST", maxCenterDist)); LOG4CXX_TRACE(_log, "TRACKING_MAX_CENTER_DIST: " << maxCenterDist); + maxIOUDist = abs(getEnv(jpr,"TRACKING_MAX_IOU_DIST", maxIOUDist)); LOG4CXX_TRACE(_log, "TRACKING_MAX_IOU_DIST: " << maxIOUDist); + + kfDisabled = getEnv(jpr,"KF_DISABLED", kfDisabled); LOG4CXX_TRACE(_log, "KF_DISABLED: " << kfDisabled); + + _strRN = getEnv(jpr,"KF_RN",_strRN); LOG4CXX_TRACE(_log, "KF_RN: " << _strRN); + _strQN = getEnv(jpr,"KF_QN",_strQN); LOG4CXX_TRACE(_log, "KF_QN: " << _strQN); + RN = fromString(_strRN, 4, 1, "f"); + QN = fromString(_strQN, 4, 1, "f"); + //convert stddev to variances + RN = RN.mul(RN); + QN = QN.mul(QN); + + + _strOrientations = (rotateDetect) ? getEnv(jpr, "ROTATE_ORIENTATIONS",_strOrientations) : "[0]"; + inferenceOrientations = _fromString(_strOrientations); + + fallback2CpuWhenGpuProblem = getEnv(jpr,"FALLBACK_TO_CPU_WHEN_GPU_PROBLEM", + fallback2CpuWhenGpuProblem); LOG4CXX_TRACE(_log, "FALLBACK_TO_CPU_WHEN_GPU_PROBLEM: " << fallback2CpuWhenGpuProblem); + cudaDeviceId = getEnv (jpr,"CUDA_DEVICE_ID", + cudaDeviceId); LOG4CXX_TRACE(_log, "CUDA_DEVICE_ID: " << cudaDeviceId); +} + +/** ************************************************************************** +* Dump config to a stream +*************************************************************************** */ +ostream& operator<< (ostream& out, const JobConfig& cfg) { + out << "{" + << "\"minDetectionSize\": " << cfg.minDetectionSize << "," + << "\"confThresh\":" << cfg.confThresh << "," + << "\"nmsThresh\":" << cfg.nmsThresh << "," + << "\"rotateDetect\":" << (cfg.rotateDetect ? "1":"0") << "," + << "\"inferenceOrientations\":" << cfg.inferenceOrientations << "," + << "\"bboxScaleFactor\":" << cfg.bboxScaleFactor << "," + << "\"detFrameInterval\":" << cfg.detFrameInterval << "," + << "\"maxFeatureDist\":" << cfg.maxFeatureDist << "," + << "\"maxFrameGap\":" << cfg.maxFrameGap << "," + << "\"maxCenterDist\":" << cfg.maxCenterDist << "," + << "\"maxIOUDist\":" << cfg.maxIOUDist << "," + << "\"kfDisabled\":" << (cfg.kfDisabled ? "1":"0") << "," + << "\"kfProcessVar\":" << format(cfg.QN) << "," + << "\"kfMeasurementVar\":" << format(cfg.RN) << "," + << "\"fallback2CpuWhenGpuProblem\":" << (cfg.fallback2CpuWhenGpuProblem ? "1" : "0") << "," + << "\"cudaDeviceId\":" << cfg.cudaDeviceId + << "}"; + return out; +} + +/** ************************************************************************** +* Default constructor with default values +*************************************************************************** */ +JobConfig::JobConfig(): + minDetectionSize(46), + confThresh(0.3), + nmsThresh(0.3), + inferenceSize(-1), + rotateDetect(true), + bboxScaleFactor(1.0), + maxFrameGap(4), + detFrameInterval(1), + maxFeatureDist(0.25), + maxCenterDist(0.0), + maxIOUDist(0.5), + kfDisabled(false), + cudaDeviceId(0), + fallback2CpuWhenGpuProblem(true), + frameIdx(-1), + frameTimeInSec(0), + frameTimeStep(0), + lastError(MPF_DETECTION_SUCCESS), + _imreaderPtr(unique_ptr()), + _videocapPtr(unique_ptr()){ + + // Kalman filter motion model noise / acceleration stddev for covariance matrix Q + _strQN = "[100.0,100.0,100.0,100.0]"; + QN = fromString(_strQN, 4, 1, "f"); + + // Kalman Bounding Box Measurement noise sdtdev for covariance Matrix R + _strRN = "[6.0, 6.0, 6.0, 6.0]"; + RN = fromString(_strRN, 4, 1, "f"); + + // Inference rotations + _strOrientations = "[0, 90, 180, 270]"; + inferenceOrientations = fromString(_strOrientations); + + } + +/** ************************************************************************** +* Constructor that parses parameters from MPFImageJob and load image +*************************************************************************** */ +JobConfig::JobConfig(const MPFImageJob &job): + JobConfig() { + LOG4CXX_DEBUG(_log, "[" << job.job_name << "Data URI = " << job.data_uri); + _parse(job); + + if(job.data_uri.empty()) { LOG4CXX_ERROR(_log, "[" << job.job_name << "Invalid image url"); + lastError = MPF_INVALID_DATAFILE_URI; + }else{ + _imreaderPtr = unique_ptr(new MPFImageReader(job)); + bgrFrame = _imreaderPtr->GetImage(); + if(bgrFrame.empty()){ LOG4CXX_ERROR(_log, "[" << job.job_name << "] Could not read image file: " << job.data_uri); + lastError = MPF_IMAGE_READ_ERROR; + } LOG4CXX_DEBUG(_log, "[" << job.job_name << "] image.width = " << bgrFrame.cols); + LOG4CXX_DEBUG(_log, "[" << job.job_name << "] image.height = " << bgrFrame.rows); + } +} + +/** ************************************************************************** +* Constructor that parses parameters from MPFVideoJob and initializes +* video capture / reader +*************************************************************************** */ +JobConfig::JobConfig(const MPFVideoJob &job): + JobConfig() { + + _parse(job); + + if(job.data_uri.empty()) { LOG4CXX_ERROR(_log, "[" << job.job_name << "Invalid video url"); + lastError = MPF_INVALID_DATAFILE_URI; + }else{ + _videocapPtr = unique_ptr(new MPFVideoCapture(job, true, true)); + if(!_videocapPtr->IsOpened()){ LOG4CXX_ERROR(_log, "[" << job.job_name << "] Could not initialize capturing"); + lastError = MPF_COULD_NOT_OPEN_DATAFILE; + } + // pre-compute diagonal normalization factor for distance normalizations + cv::Size fs = _videocapPtr->GetFrameSize(); + float diag = sqrt(fs.width*fs.width + fs.height*fs.height); + widthOdiag = fs.width / diag; + heightOdiag = fs.height / diag; + + frameTimeStep = 1.0 / _videocapPtr->GetFrameRate(); + } +} + +/** ************************************************************************** +* Read next frame of video into current bgrFrame member variable and advance +* frame index counter. +*************************************************************************** */ +bool JobConfig::nextFrame(){ + frameIdx = _videocapPtr->GetCurrentFramePosition(); + frameTimeInSec = _videocapPtr->GetCurrentTimeInMillis() * 0.001; + return _videocapPtr->Read(bgrFrame); +} + +/** ************************************************************************** +* Destructor to release image / video readers +*************************************************************************** */ +JobConfig::~JobConfig(){ + if(_imreaderPtr){ + _imreaderPtr.reset(); + } + if(_videocapPtr){ + _videocapPtr->Release(); + _videocapPtr.reset(); + } +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/JobConfig.h b/cpp/OcvSsdFaceDetection/JobConfig.h new file mode 100644 index 00000000..3ccea8fc --- /dev/null +++ b/cpp/OcvSsdFaceDetection/JobConfig.h @@ -0,0 +1,105 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#ifndef OCVSSDFACEDETECTION_JOBCONFIG_H +#define OCVSSDFACEDETECTION_JOBCONFIG_H + +#include +#include + +#include "detectionComponentUtils.h" +#include "adapters/MPFImageAndVideoDetectionComponentAdapter.h" +#include "MPFImageReader.h" +#include "MPFVideoCapture.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + /* ************************************************************************** + * Configuration parameters populated with appropriate values & defaults + *************************************************************************** */ + class JobConfig{ + public: + static log4cxx::LoggerPtr _log; ///< shared log object + size_t minDetectionSize; ///< minimum bounding box dimension + float confThresh; ///< detection confidence threshold + float nmsThresh; ///< non-maximum suppression threshold for removing redundant overlapping bounding boxes + float bboxScaleFactor; ///< scale factor for width and height of the detector bounding box + bool rotateDetect; ///< perform multiple passes at different image rotations to increase detections found + OrientVec inferenceOrientations; ///< ccw rotations of frame to inference (only multiples of 90 are accepted) + int inferenceSize; ///< max image dimension to use for inferencing e.g. 300 (-1 will use original but run slower) + long detFrameInterval; ///< number of frames between looking for new detection (tracking only) + + float maxFeatureDist; ///< maximum feature distance to maintain track continuity + float maxCenterDist; ///< maximum spatial distance normalized by diagonal to maintain track continuity + long maxFrameGap; ///< maximum temporal distance (frames) to maintain track continuity + float maxIOUDist; ///< maximum for (1 - Intersection/Union) to maintain track continuity + + float widthOdiag; ///< image (width/diagonal) + float heightOdiag; ///< image (height/diagonal) + size_t frameIdx; ///< index of current frame + double frameTimeInSec; ///< time of current frame in sec + double frameTimeStep; ///< time interval between frames in sec + + cv::Mat bgrFrame; ///< current BGR image frame + + cv::Mat1f RN; ///< kalman filter measurement noise matrix + cv::Mat1f QN; ///< kalman filter process noise variances (i.e. unknown accelerations) + bool kfDisabled; ///< if true kalman filtering is disabled + + bool fallback2CpuWhenGpuProblem; ///< fallback to cpu if there is a gpu problem + int cudaDeviceId; ///< gpu device id to use for cuda + + mutable MPFDetectionError lastError; ///< last MPF error that should be returned + + JobConfig(); + JobConfig(const MPFImageJob &job); + JobConfig(const MPFVideoJob &job); + ~JobConfig(); + + void ReverseTransform(MPFImageLocation loc){_imreaderPtr->ReverseTransform(loc);} + void ReverseTransform(MPFVideoTrack track){_videocapPtr->ReverseTransform(track);} + bool nextFrame(); + + private: + unique_ptr _imreaderPtr; + unique_ptr _videocapPtr; + string _strRN; ///< kalman filter measurement noise matrix serialized to string + string _strQN; ///< kalman filter process noise matrix serialized to string + string _strOrientations; ///< ccw rotations of frame to inference (only multiples of 90 are accepted) + + void _parse(const MPFJob &job); + + }; + + ostream& operator<< (ostream& out, const JobConfig& cfg); ///< Dump JobConfig to a stream + + } +} + +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/KFTracker.cpp b/cpp/OcvSsdFaceDetection/KFTracker.cpp new file mode 100644 index 00000000..94d1eb01 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/KFTracker.cpp @@ -0,0 +1,351 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include "KFTracker.h" + +#define PROCESS_NOISE CONTINUOUS_WHITE +//#define PROCESS_NOISE PIECEWISE_WHITE + +using namespace MPF::COMPONENT; + +log4cxx::LoggerPtr KFTracker::_log; + +// Kalman Filter Dimensions (4x constant acceleration model) +const int KF_STATE_DIM = 12; ///< dimensionality of kalman state vector [x, v_x, ,a_x, y, v_y, a_y, w, v_w, a_w, h, v_h, ah] +const int KF_MEAS_DIM = 4; ///< dimensionality of kalman measurement vector [x, y, w, h] +const int KF_CTRL_DIM = 0; ///< dimensionality of kalman control input [] + +/** ************************************************************************** + * Update the model state transision matrix F and the model noise covariance + * matrix Q to be suitable of a timestep of dt + * + * \param dt new time step in sec + * + * \note F and Q are block diagonal with 4 blocks, one for each x, y, w, h + * +*************************************************************************** */ +void KFTracker::_setTimeStep(float dt){ + + if(fabs(_dt - dt) > 2 * numeric_limits::epsilon()){ + _dt = dt; LOG4CXX_TRACE(_log,"kd dt:" << _dt); + + float dt2 = dt * dt; + float dt3 = dt2 * dt; + float dt4 = dt2 * dt2; + + float half_dt2 = 0.5*dt2; + + #if PROCESS_NOISE == CONTINUOUS_WHITE + float third_dt3 = dt3 / 3.0; + float sixth_dt3 = dt3 / 6.0; + float eighth_dt4 = dt4 / 8.0; + float twentieth_dt5 = dt2 * dt3 / 20.0; + #elif PROCESS_NOISE == PIECEWISE_WHITE + float half_dt3 = dt3 / 2.0; + float quarter_dt4 = dt4 / 4.0; + #endif + + for(int b=0; b < 4; b++){ + int i = 3*b; + // update state transition matrix F + _kf.transitionMatrix.at( i, 1+i) = + _kf.transitionMatrix.at( 1+i, 2+i) = dt; + _kf.transitionMatrix.at( i, 2+i) = half_dt2; + + // update process noise matrix Q + #if PROCESS_NOISE == CONTINUOUS_WHITE + _kf.processNoiseCov.at( i, i) = _qn.at(b,0) * twentieth_dt5; + _kf.processNoiseCov.at(1+i, i) = + _kf.processNoiseCov.at( i,1+i) = _qn.at(b,0) * eighth_dt4; + _kf.processNoiseCov.at(2+i, i) = + _kf.processNoiseCov.at( i,2+i) = _qn.at(b,0) * sixth_dt3; + _kf.processNoiseCov.at(1+i,1+i) = _qn.at(b,0) * third_dt3; + _kf.processNoiseCov.at(1+i,2+i) = + _kf.processNoiseCov.at(2+i,1+i) = _qn.at(b,0) * half_dt2; + _kf.processNoiseCov.at(2+i,2+i) = _qn.at(b,0) * dt; + #elif PROCESS_NOISE == PIECEWISE_WHITE + _kf.processNoiseCov.at( i, i) = _qn.at(b,0) * quarter_dt4; + _kf.processNoiseCov.at(1+i, i) = + _kf.processNoiseCov.at( i,1+i) = _qn.at(b,0) * half_dt3; + _kf.processNoiseCov.at(2+i, i) = + _kf.processNoiseCov.at( i,2+i) = _qn.at(b,0) * half_dt2; + _kf.processNoiseCov.at(1+i,1+i) = _qn.at(b,0) * dt2; + _kf.processNoiseCov.at(1+i,2+i) = + _kf.processNoiseCov.at(2+i,1+i) = _qn.at(b,0) * dt; + _kf.processNoiseCov.at(2+i,2+i) = _qn.at(b,0); + #endif + } + + } +} + +/** ************************************************************************** + * Convert bbox to a measurement vector consisting of center coordinate and + * width and height dimensions + * + * \param r bounding box to convert + * + * \returns measurement vector + * +*************************************************************************** */ +cv::Mat1f KFTracker::measurementFromBBox(const cv::Rect2i& r){ + cv::Mat1f z(KF_MEAS_DIM,1); + z.at(0) = r.x + r.width / 2.0f; + z.at(1) = r.y + r.height / 2.0f; + z.at(2) = r.width; + z.at(3) = r.height; + return z; +} + +/** ************************************************************************** + * Convert a filter state [ x,vx,ax, y,vy,ay, w,wv,aw, h,vh,ah] to roi(x,y,w,h) + * 0 1 2 3 4 5 6 7 8 9 10 11 + * + * \param state filter state vector to convert + * + * \returns bounding box corresponding to state + * +*************************************************************************** */ +cv::Rect2i KFTracker::bboxFromState(const cv::Mat1f state){ + return cv::Rect2i(static_cast(state.at(0) - state.at(6)/2.0f + 0.5f), + static_cast(state.at(3) - state.at(9)/2.0f + 0.5f), + static_cast(state.at(6) + 0.5f), + static_cast(state.at(9) + 0.5f)); +} + +/** ************************************************************************** + * Get bounding box prediction by advancing filter state to time t + * and update filter time to t + * + * \param t time in sec to advance filter to with prediction + * +*************************************************************************** */ +void KFTracker::predict(float t){ + _setTimeStep(t - _t); + _t = t; + _kf.predict(); + _kf.errorCovPre += _kf.errorCovPre.t(); // guarantee cov symmetry to help stability + _kf.errorCovPre /= 2.0f; +} + +/** ************************************************************************** + * Get bounding box from state after corrected by a measurement at filter time t + * + * \param rec measurement to use for correcting filter state at current filter + * time + * + * \note filter will record and dump error statistics in _state_trace if + * compiled for debug + * +*************************************************************************** */ +void KFTracker::correct(const cv::Rect2i &rec){ + _kf.correct(measurementFromBBox(rec)); + _kf.errorCovPost += _kf.errorCovPost.t(); // guarantee cov symmetry to help stability + _kf.errorCovPost /= 2.0f; + #ifdef DIAGNOSTIC_FILES + _state_trace << (*this) << endl; + #endif +} + +/** ************************************************************************** + * Construct and initialize kalman filter motion tracker + * + * \param t time corresponding to initial bounding box rec0 + * \param dt timestep to use for the filter (sec) + * \param rec0 inital bounding box measurement + * \param roi clipping constraints so BBoxes don't wonder off frame + * \param rn 4x1 column vector of variances for measurement noise, var([x,y,w,h]) + * \param qn 4x1 column vector of variances for model noise, var([ax,ay,aw,ah]) + * +*************************************************************************** */ +KFTracker::KFTracker(const float t, + const float dt, + const cv::Rect2i &rec0, + const cv::Rect2i &roi, + const cv::Mat1f &rn, + const cv::Mat1f &qn): + _t(t), + _dt(-1.0f), + _roi(roi), + _qn(qn), + _kf(KF_STATE_DIM,KF_MEAS_DIM,KF_CTRL_DIM,CV_32F) // kalman(state_dim,meas_dim,contr_dim) +{ + assert(_log); + assert( rn.rows == KF_MEAS_DIM && rn.cols == 1); + assert( qn.rows == KF_MEAS_DIM && qn.cols == 1); // only accelerations model noise should be specified + + assert(_roi.x == 0); + assert(_roi.y == 0); + assert(_roi.width > 0); + assert(_roi.height > 0); + + // state transition matrix F + // | 0 1 2 3 4 5 6 7 8 9 10 11 + //-------------------------------------------------------- + // 0 | 1 dt .5dt^2 0 0 0 0 0 0 0 0 0 | | x| + // 1 | 0 1 dt 0 0 0 0 0 0 0 0 0 | |vx| + // 2 | 0 0 1 0 0 0 0 0 0 0 0 0 | |ax| + // 3 | 0 0 0 1 dt .5dt^2 0 0 0 0 0 0 | | y| + // 4 | 0 0 0 0 1 dt 0 0 0 0 0 0 | |vy| + // 5 | 0 0 0 0 0 1 0 0 0 0 0 0 | |vy| + // 6 | 0 0 0 0 0 0 1 dt .5dt^2 0 0 0 | | w| + // 7 | 0 0 0 0 0 0 0 1 dt 0 0 0 | |vw| + // 8 | 0 0 0 0 0 0 0 0 1 0 0 0 | |aw| + // 9 | 0 0 0 0 0 0 0 0 0 1 dt .5dt^2 | | h| + // 10 | 0 0 0 0 0 0 0 0 0 0 1 dt | |vh| + // 11 | 0 0 0 0 0 0 0 0 0 0 0 1 | |ah| + + // measurement matrix H + // 0 1 2 3 4 5 6 7 8 9 10 11 + //----------------------------------------- + // 0 | 1 0 0 0 0 0 0 0 0 0 0 0 | | x | + // 1 | 0 0 0 1 0 0 0 0 0 0 0 0 | | y | + // 2 | 0 0 0 0 0 0 1 0 0 0 0 0 | * A = | w | + // 3 | 0 0 0 0 0 0 0 0 0 1 0 0 | | h | + _kf.measurementMatrix.at(0,0) = + _kf.measurementMatrix.at(1,3) = + _kf.measurementMatrix.at(2,6) = + _kf.measurementMatrix.at(3,9) = 1.0f; + + // measurement noise covariance matrix R + // | 0 1 2 3 + //------------------ + // 0 | xx 0 0 0 + // 1 | 0 yy 0 0 + // 2 | 0 0 ww 0 + // 3 | 0 0 0 hh + _kf.measurementNoiseCov.at(0,0) = rn.at(0); + _kf.measurementNoiseCov.at(1,1) = rn.at(1); + _kf.measurementNoiseCov.at(2,2) = rn.at(2); + _kf.measurementNoiseCov.at(3,3) = rn.at(3); + + //adjust F and setup process noise covariance matrix Q + _setTimeStep(dt); + + //initialize filter state + cv::Mat1f z0 = measurementFromBBox(rec0); + _kf.statePost.at(0) = z0.at(0); + _kf.statePost.at(3) = z0.at(1); + _kf.statePost.at(6) = z0.at(2); + _kf.statePost.at(9) = z0.at(3); + + //initialize error covariance matrix P + // z: [ x, y, w, h] + // x: [ x,vx,ax, y,vy,ay, w,wv,aw, h,vh,ah] + // 0 1 2 3 4 5 6 7 8 9 10 11 + _kf.errorCovPost.at(0,0) = rn.at(0); + _kf.errorCovPost.at(1,1) = (z0.at(2)/dt) * (z0.at(2)/dt); // guess max vx as one width/dt + _kf.errorCovPost.at(2,2) = 10.0 * _kf.processNoiseCov.at(2,2); // guess ~3 sigma ?! + + _kf.errorCovPost.at(3,3) = rn.at(1); + _kf.errorCovPost.at(4,4) = (z0.at(3)/dt) * (z0.at(3)/dt); // guess max vy as one height/dt + _kf.errorCovPost.at(5,5) = 10.0 * _kf.processNoiseCov.at(5,5); // guess ~3 sigma ?! + + _kf.errorCovPost.at(6,6) = rn.at(2); + _kf.errorCovPost.at(7,7) = 10.0 * _kf.processNoiseCov.at(7,7); // guess ~3 sigma + _kf.errorCovPost.at(8,8) = 10.0 * _kf.processNoiseCov.at(8,8); // guess ~3 sigma + + _kf.errorCovPost.at(9,9) = rn.at(3); + _kf.errorCovPost.at(10,10) = 10.0 * _kf.processNoiseCov.at(10,10); // guess ~3 sigma + _kf.errorCovPost.at(11,11) = 10.0 * _kf.processNoiseCov.at(11,11); // guess ~3 sigma + + #ifdef DIAGNOSTIC_FILES + _state_trace << (*this) << endl; // trace filter initial error stats + _myId = _objId; // used for output filename + _objId++; + #endif +} + +/** ************************************************************************** +* Setup class shared static configurations and initialize +* +* \param log logger object for logging +* \param plugin_path plugin directory +* +* \return true if everything was properly initialized, false otherwise +*************************************************************************** */ +bool KFTracker::Init(log4cxx::LoggerPtr log, string plugin_path=""){ + + _log = log; + + return true; +} + +#ifdef DIAGNOSTIC_FILES + +#include +size_t KFTracker::_objId = 0; // class static sequence variable + +/** ************************************************************************** +* Write out error statistics over time to csv file for filter tuning +*************************************************************************** */ +void KFTracker::dump(){ + stringstream filename; + filename << "kf_" << setfill('0') << setw(4) << _myId; + filename << ".csv"; + ofstream dump; + dump.open(filename.str(),ofstream::out | ofstream::trunc); + dump << "t,"; + dump << "px,pvx,pax, py,pvy,pay, pw,pvw,paw, ph,pvh,pah, "; + dump << "cx,cvx,cax, cy,cvy,cay, cw,cvw,caw, ch,cvh,cah, "; + dump << "err_x, err_y, err_w, err_h,"; + for(int r=0;r<12;r++){ + dump << "P" << setfill('0') << setw(2) << r << "_" << r ; + dump << ","; + } + dump << endl; + dump << _state_trace.rdbuf(); + dump.close(); +} +/** ************************************************************************** +* Dump Diagnostics for MPF::COMPONENT::KFTracker to a stream +*************************************************************************** */ +ostream& MPF::COMPONENT::operator<< (ostream& out, const KFTracker& kft) { + out << kft._t << ","; + + for(int i=0; i < kft._kf.statePre.rows; i++){ + out << kft._kf.statePre.at(i) <<","; + } + out << " "; + + for(int i=0; i < kft._kf.statePost.rows; i++){ + out << kft._kf.statePost.at(i) <<","; + } + out << " "; + + for(int i=0; i < kft._kf.temp5.rows; i++){ + out << kft._kf.temp5.at(i) << ","; + } + out << " "; + + for(int r=0; r(r,r)) << ","; + //} + } + return out; +} +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/KFTracker.h b/cpp/OcvSsdFaceDetection/KFTracker.h new file mode 100644 index 00000000..912ba1a3 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/KFTracker.h @@ -0,0 +1,90 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#ifndef KF_TRACKER_H +#define KF_TRACKER_H + +#include + +#include +#include +#include + +#include "types.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + class KFTracker{ + + public: + + static bool Init(log4cxx::LoggerPtr log, const string plugin_path); ///< setup class shared members + static cv::Mat1f measurementFromBBox(const cv::Rect2i& r); + static cv::Rect2i bboxFromState(const cv::Mat1f state); + + + const cv::Rect2i predictedBBox() const {return bboxFromState(_kf.statePre) & _roi;}; + const cv::Rect2i correctedBBox() const {return bboxFromState(_kf.statePost) & _roi;}; + + void predict(const float t); ///< advance Kalman state to time t and get predicted bbox + void correct(const cv::Rect2i &rec); ///< correct current filter state with measurement rec + + KFTracker(const float t, + const float dt, + const cv::Rect2i &rec0, + const cv::Rect2i &roi, + const cv::Mat1f &rn, + const cv::Mat1f &qn); + + //#define DIAGNOSTIC_FILES + #ifdef DIAGNOSTIC_FILES + static size_t _objId; + size_t _myId; + void dump(); + stringstream _state_trace; + friend ostream& operator<< (ostream& out, const KFTracker& kft); + #endif + + private: + static log4cxx::LoggerPtr _log; ///< shared log object + cv::KalmanFilter _kf; ///< kalman filter for bounding box + float _t; ///< time corresponding to kalman filter state + float _dt; ///< time step to use for filter updates + const cv::Rect2i _roi; ///< canvas clipping limits for bboxes returned by filter + const cv::Mat1f _qn; ///< kalman filter process noise variances (i.e. unknown accelerations) [ax,ay,aw,ah] + + void _setTimeStep(float dt); ///< update model variables Q F for time step size dt + + }; + + + } +} + +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/LICENSE b/cpp/OcvSsdFaceDetection/LICENSE new file mode 100644 index 00000000..3c537fbf --- /dev/null +++ b/cpp/OcvSsdFaceDetection/LICENSE @@ -0,0 +1,29 @@ +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + + This project contains content developed by The MITRE Corporation. If this code + is used in a deployment or embedded within another project, it is requested + that you send an email to opensource@mitre.org in order to let us know where + this software is being used. + + When this software is packaged and/or distributed with the pre-trained model: + + opencv_face_detector_uint8.pb + [https://github.com/opencv/opencv_3rdparty/raw/8033c2bc31b3256f0d461c919ecc01c2428ca03b/opencv_face_detector_uint8.pb], + + the resulting product inherits the license and usage restrictions of that model, + which are not explicitly stated by the original author. Please contact the author + before using this component for commercial purposes. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/NOTICE b/cpp/OcvSsdFaceDetection/NOTICE new file mode 100644 index 00000000..f4133536 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/NOTICE @@ -0,0 +1,7 @@ +# NOTICE + +This software (or technical data) was produced for the U.S. Government +under contract, and is subject to the Rights in Data-General Clause +52.227-14, Alt. IV (DEC 2007). + +Copyright 2020 The MITRE Corporation. All Rights Reserved. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.code-workspace b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.code-workspace new file mode 100644 index 00000000..5e6d2008 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.code-workspace @@ -0,0 +1,104 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": { + "C_Cpp.default.cppStandard": "c++11", + "C_Cpp.default.cStandard": "c11", + "C_Cpp.loggingLevel": "Error", + "C_Cpp.includePath": [ + "${workspaceFolder}" + ], + "cmake.cmakePath": "cmake3", + "files.associations": { + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "array": "cpp", + "strstream": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "chrono": "cpp", + "complex": "cpp", + "cstdint": "cpp", + "deque": "cpp", + "list": "cpp", + "unordered_map": "cpp", + "vector": "cpp", + "exception": "cpp", + "fstream": "cpp", + "functional": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "new": "cpp", + "ostream": "cpp", + "numeric": "cpp", + "ratio": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "cfenv": "cpp", + "cinttypes": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "algorithm": "cpp", + "random": "cpp", + "qhash": "cpp", + "csignal": "cpp", + "atomic": "cpp", + "hash_map": "cpp", + "hash_set": "cpp", + "condition_variable": "cpp", + "forward_list": "cpp", + "unordered_set": "cpp", + "future": "cpp", + "mutex": "cpp", + "system_error": "cpp", + "thread": "cpp", + "valarray": "cpp", + "*.ipp": "cpp", + "string": "cpp" + }, + "editor.tabSize": 2, + "editor.tokenColorCustomizations": { + "textMateRules": [ + { + "scope": "googletest.failed", + "settings": { + "foreground": "#f00" + } + }, + { + "scope": "googletest.passed", + "settings": { + "foreground": "#0f0" + } + }, + { + "scope": "googletest.run", + "settings": { + "foreground": "#0f0" + } + } + ] + } + }, + "gtest-adapter.debugConfig": "" +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.cpp b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.cpp new file mode 100644 index 00000000..808fff7c --- /dev/null +++ b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.cpp @@ -0,0 +1,481 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include "OcvSsdFaceDetection.h" + +#include +#include +#include + +#include + +#include +#include +#include +#include + +// 3rd party code for solving assignment problem +#include +#include + +// MPF-SDK header files +#include "Utils.h" +#include "MPFSimpleConfigLoader.h" +#include "detectionComponentUtils.h" + + +using namespace MPF::COMPONENT; + +// Temporary initializer for static member variable +log4cxx::LoggerPtr JobConfig::_log = log4cxx::Logger::getRootLogger(); + +/** **************************************************************************** + * print out dlib matrix on a single line + * + * \param m matrix to serialize to single line string + * \returns single line string representation of matrix + * +***************************************************************************** */ +template +string dformat(dlib::matrix m){ + stringstream ss; + ss << "{"; + for(size_t r=0;r params; + if(LoadConfig(config_params_path, params)) { LOG4CXX_ERROR(_log, "Failed to load the OcvSsdFaceDetection config from: " << config_params_path); + return false; + } LOG4CXX_TRACE(_log,"read config file:" << config_params_path); + Properties props; + for(auto p = params.begin(); p != params.end(); p++){ + const string key = p.key().toStdString(); + const string val = p.value().toStdString(); LOG4CXX_TRACE(_log,"Config Vars:" << key << "=" << val); + const char* env_val = getenv(key.c_str()); LOG4CXX_TRACE(_log,"Verifying ENVs:" << key << "=" << env_val); + if(env_val == NULL){ + if(setenv(key.c_str(), val.c_str(), 0) !=0){ LOG4CXX_ERROR(_log,"Failed to convert config to env variable: " << key << "=" << val); + return false; + } + }else if(string(env_val) != val){ LOG4CXX_INFO(_log,"Keeping existing env variable:" << key << "=" << string(env_val)); + } + } + + // initialize adaptive histogram equalizer + _equalizerPtr = cv::createCLAHE(40.0,cv::Size(8,8)); + + bool detectionLocationInitalizedOK = DetectionLocation::Init(_log, plugin_path); + + Properties fromEnv; + int cudaDeviceId = getEnv (fromEnv,"CUDA_DEVICE_ID", -1); + bool fallbackToCPU = getEnv(fromEnv,"FALLBACK_TO_CPU_WHEN_GPU_PROBLEM", true); + bool defaultCudaDeviceOK = DetectionLocation::loadNetToCudaDevice(cudaDeviceId) || fallbackToCPU; + + bool trackInitializedOK = Track::Init(_log, plugin_path); + bool kfTrackerInitializedOK = KFTracker::Init(_log, plugin_path); + + return detectionLocationInitalizedOK + && defaultCudaDeviceOK + && trackInitializedOK + && kfTrackerInitializedOK; + + +} + +/** **************************************************************************** +* Clean up and release any created detector objects +* +* \returns true on success +***************************************************************************** */ +bool OcvSsdFaceDetection::Close() { + return true; +} + +/** **************************************************************************** +* Compute cost vector and solve it to give detection to track assignment matrix +* +* \param COST_FUNC cost function to use +* \param TrackPtrList list of existing tracks to consider for assignment +* \param detections vector of detections that need assigned to tracks +* \param maxCost maximum assignment cost, if exceeded the particular +* detection to track assignment will be removed from result +* \returns assignment vector am[track,detection] with dim (# tracks x # detections) +* if am[x,y]==0 then detection[y] should be assigned to track[x] +* +***************************************************************************** */ +template +vector OcvSsdFaceDetection::_calcAssignmentVector(const TrackPtrList &tracks, + const DetectionLocationPtrVec &detections, + const float maxCost = INT_MAX){ + vector av; //assignment vector to return + + if(tracks.size() == 0 || detections.size() ==0) return av; // nothing to do + + // rows -> tracks, cols -> detections, but matrix has to be square! + size_t n = max(tracks.size(),detections.size()); + dlib::matrix costs = dlib::zeros_matrix(n,n); + + // fill in actual costs for non-dummy entries + size_t r = 0; + vector uniqueCosts; + for(auto &track:tracks){ + for(size_t c=0; cback()->frameIdx < detections[c]->frameIdx){ + //float cost = CALL_MEMBER_FUNC(*(track->back()),COST_FUNC)(*detections[c]); + float cost = CALL_MEMBER_FUNC(*detections[c],COST_FUNC)(*track); + int iCost = ((cost <= maxCost) ? (INT_MAX - static_cast(1.E9 * cost)) : 0); + if(iCost !=0 ) + while(find(uniqueCosts.begin(),uniqueCosts.end(),iCost) != uniqueCosts.end()) iCost--; // cost have to be unique or dlib::max_cost_assignment() hangs... + costs(r,c) = iCost; + uniqueCosts.push_back(iCost); + } + } + r++; + } LOG4CXX_TRACE(_log,"cost matrix[tr=" << costs.nr() << ",det=" << costs.nc() << "]: " << dformat(costs)); + + // solve cost matrix, track av[track] get assigned detection[av[track]] + av = dlib::max_cost_assignment(costs); LOG4CXX_TRACE(_log, "solved assignment vec["<< av.size() << "] = " << av); + + // drop dummy tracks used to pad cost matrix + av.resize(tracks.size()); + + // knock out assignments that are too costly (i.e. new track needed) + for(long t=0; tframeIdx << " " << ((MPFImageLocation)*(detections[assignmentVector[t]])) << " to track " << *track); + track->releaseTracker(); + track->push_back(move(detections.at(assignmentVector[t]))); + track->kalmanCorrect(); + assignedTracks.push_back(move(track)); + } + t++; + } + + //remove tracks that received and assignment + tracks.erase(remove_if(tracks.begin(), + tracks.end(), + [](TrackPtr const& d){return !d;}), + tracks.end()); + + // remove detections that were assigned to tracks + detections.erase(remove_if(detections.begin(), + detections.end(), + [](DetectionLocationPtr const& d){return !d;}), + detections.end()); + +} + +/** **************************************************************************** +* Read an image and get object detections and features +* +* \param job MPF Image job +* +* \returns locations collection to which detections have been added +* +***************************************************************************** */ +MPFImageLocationVec OcvSsdFaceDetection::GetDetections(const MPFImageJob &job) { + + try { LOG4CXX_DEBUG(_log, "[" << job.job_name << "Data URI = " << job.data_uri); + JobConfig cfg(job); + if(cfg.lastError != MPF_DETECTION_SUCCESS){ + throw MPFDetectionException(cfg.lastError,"failed to parse image job configuration parameters"); + } + DetectionLocationPtrVec detections = DetectionLocation::createDetections(cfg); + LOG4CXX_DEBUG(_log, "[" << job.job_name << "] Number of faces detected = " << detections.size()); + MPFImageLocationVec locations; + for(auto &det:detections){ + det->detection_properties["ROTATION"] = to_string(det->getAngle()); + MPFImageLocation loc = *det; + det.reset(); // release frame object + cfg.ReverseTransform(loc); + locations.push_back(loc); + } + return locations; + + }catch(const runtime_error& re){ + LOG4CXX_FATAL(_log, "[" << job.job_name << "] runtime error: " << re.what()); + throw MPFDetectionException(re.what()); + }catch(const exception& ex){ + LOG4CXX_FATAL(_log, "[" << job.job_name << "] exception: " << ex.what()); + throw MPFDetectionException(ex.what()); + }catch (...){ + LOG4CXX_FATAL(_log, "[" << job.job_name << "] unknown error"); + throw MPFDetectionException(" Unknown error processing image job"); + } + LOG4CXX_DEBUG(_log,"[" << job.job_name << "] complete."); +} + +/** **************************************************************************** +* Convert track (list of detection ptrs) to an MPFVideoTrack object +* +* \param[in,out] tracks Tracks collection to which detections will be added +* +* \returns MPFVideoTrack object resulting from conversion +* +* \note detection pts are released on conversion and confidence is assigned +* as the average of the detection confidences +* +***************************************************************************** */ +MPFVideoTrack OcvSsdFaceDetection::_convert_track(Track &track){ + + MPFVideoTrack mpf_track; + mpf_track.start_frame = track.front()->frameIdx; + mpf_track.stop_frame = track.back()->frameIdx; + + stringstream start_feature; + stringstream stop_feature; + start_feature << track.front()->getFeature(); // make sure we have computed features to serialize + stop_feature << track.back()->getFeature(); // for the start and end detections. + mpf_track.detection_properties["START_FEATURE"] = start_feature.str(); + mpf_track.detection_properties["STOP_FEATURE"] = stop_feature.str(); + + #ifdef DIAGNOSTIC_FILES + track.kalmanDump(); + #endif + + for(auto &det:track){ + mpf_track.confidence += det->confidence; + det->detection_properties["ROTATION"] = to_string(det->getAngle()); + mpf_track.frame_locations.insert(mpf_track.frame_locations.end(),{det->frameIdx,*det}); + det.reset(); + } + mpf_track.confidence /= static_cast(track.size()); + + return mpf_track; +} + +/** **************************************************************************** +* Read frames from a video, get object detections and make tracks +* +* \param job MPF Video job +* +* \returns Tracks collection to which detections have been added +* +***************************************************************************** */ +MPFVideoTrackVec OcvSsdFaceDetection::GetDetections(const MPFVideoJob &job){ + + MPFVideoTrackVec mpf_tracks; + TrackPtrList trackPtrs; + + try{ + JobConfig cfg(job); + if(cfg.lastError != MPF_DETECTION_SUCCESS){ + throw MPFDetectionException(cfg.lastError,"failed to parse video job configuration parameters"); + } + + size_t detectTrigger = 0; + while(cfg.nextFrame()) { LOG4CXX_TRACE(_log, "."); + LOG4CXX_TRACE(_log, "processing frame " << cfg.frameIdx); + // remove any tracks too far in the past + trackPtrs.remove_if([&](unique_ptr& tPtr){ + if(cfg.frameIdx - tPtr->back()->frameIdx > cfg.maxFrameGap){ LOG4CXX_TRACE(_log,"dropping old track: " << *tPtr); + mpf_tracks.push_back(_convert_track(*tPtr)); + return true; + } + return false; + }); + + // advance kalman predictions + if(! cfg.kfDisabled){ + for(auto &trackPtr:trackPtrs){ + trackPtr->kalmanPredict(cfg.frameTimeInSec); + } + } + + TrackPtrList assignedTracks; + + if(detectTrigger == 0){ LOG4CXX_TRACE(_log,"checking for new detections"); + DetectionLocationPtrVec detections = DetectionLocation::createDetections(cfg); // look for new detections + + if(detections.size() > 0){ // found some detections in current frame + if(trackPtrs.size() >= 0 ){ // not all tracks were dropped + vector av; LOG4CXX_TRACE(_log, detections.size() <<" detections to be matched to " << trackPtrs.size() << " tracks"); + // intersection over union tracking and assignment + if(! cfg.kfDisabled){ + av = _calcAssignmentVector<&DetectionLocation::kfIouDist>(trackPtrs,detections,cfg.maxIOUDist); + }else{ + av = _calcAssignmentVector<&DetectionLocation::iouDist>(trackPtrs,detections,cfg.maxIOUDist); + } + _assignDetections2Tracks(trackPtrs, detections, av, assignedTracks); LOG4CXX_TRACE(_log,"IOU assignment complete"); + + // feature-based tracking tracking and assignment + if(detections.size() > 0){ LOG4CXX_TRACE(_log, detections.size() <<" detections to be matched to " << trackPtrs.size() << " tracks"); + av = _calcAssignmentVector<&DetectionLocation::featureDist>(trackPtrs,detections,cfg.maxFeatureDist); + _assignDetections2Tracks(trackPtrs, detections, av, assignedTracks); LOG4CXX_TRACE(_log,"Feature assignment complete"); + } + + // center-to-center distance tracking and assignment + if(detections.size() > 0){ LOG4CXX_TRACE(_log, detections.size() <<" detections to be matched to " << trackPtrs.size() << " tracks"); + av = _calcAssignmentVector<&DetectionLocation::center2CenterDist>(trackPtrs,detections,cfg.maxCenterDist); + _assignDetections2Tracks(trackPtrs, detections, av, assignedTracks); LOG4CXX_TRACE(_log,"Center2Center assignment complete"); + } + + } + LOG4CXX_TRACE(_log, detections.size() <<" detections left for new tracks"); + // any detection not assigned up to this point becomes a new track + for(auto &det:detections){ // make any unassigned detections into new tracks + det->getFeature(); // start of tracks always get feature calculated + trackPtrs.push_back(unique_ptr(new Track(move(det),cfg))); LOG4CXX_TRACE(_log,"created new track " << *(trackPtrs.back())); + } + } + } + + // check any tracks that didn't get a detection and use tracker to continue them if possible + for(auto &track:trackPtrs){ + if(track->back()->frameIdx < cfg.frameIdx){ // no detections for track in current frame, try tracking + DetectionLocationPtr detPtr = track->ocvTrackerPredict(cfg); + if(detPtr){ // tracker returned something + track->push_back(move(detPtr)); // add new location as tracks's tail + track->kalmanCorrect(); + } + } + } + + // put all the tracks that were assigned a detection back into trackPtrs + for(auto& track:assignedTracks){ + trackPtrs.push_back(move(track)); + } + assignedTracks.clear(); + + detectTrigger++; + detectTrigger = detectTrigger % (cfg.detFrameInterval + 1); + + } LOG4CXX_DEBUG(_log, "[" << job.job_name << "] Number of tracks detected = " << trackPtrs.size()); + + // convert any remaining active tracks to MPFVideoTracks + for(auto &trackPtr:trackPtrs){ + mpf_tracks.push_back(_convert_track(*trackPtr)); + } + + // reverse transform all mpf tracks + for(auto &mpf_track:mpf_tracks){ + cfg.ReverseTransform(mpf_track); + } + + return mpf_tracks; + + }catch(const runtime_error& re){ LOG4CXX_FATAL(_log, "[" << job.job_name << "] runtime error: " << re.what()); + throw MPFDetectionException(re.what()); + }catch(const exception& ex){ LOG4CXX_FATAL(_log, "[" << job.job_name << "] exception: " << ex.what()); + throw MPFDetectionException(ex.what()); + }catch (...) { LOG4CXX_FATAL(_log, "[" << job.job_name << "] unknown error"); + throw MPFDetectionException("Unknown error processing video job"); + } + LOG4CXX_DEBUG(_log,"[" << job.job_name << "] complete."); +} + +/** ************************************************************************** +* Perform histogram equalization +* +* \param cfg job structure containing image frame to be equalized +* +*************************************************************************** */ +void OcvSsdFaceDetection::_equalizeHistogram(JobConfig &cfg){ + cv::Mat3b hsvFrame; + cv::cvtColor(cfg.bgrFrame,hsvFrame,cv::COLOR_BGR2HSV); + + vector hsvComponents; + cv::split(hsvFrame, hsvComponents); + + cv::equalizeHist(hsvComponents[2],hsvComponents[2]); + cv::merge(hsvComponents,hsvFrame); + cv::cvtColor(hsvFrame,cfg.bgrFrame,cv::COLOR_HSV2BGR); +} + + +/** ************************************************************************** +* Perform image normalization +* +* \param cfg job structure containing image frame to be normalized +* +*************************************************************************** */ +void OcvSsdFaceDetection::_normalizeFrame(JobConfig &cfg){ + cv::Mat fltImg; + cfg.bgrFrame.convertTo(fltImg,CV_32F); + + cv::Mat mu, mu_sq, sigma; + cv::blur(fltImg, mu, cv::Size(3,3)); + cv::blur(fltImg.mul(fltImg), mu_sq, cv::Size(3,3)); + cv::sqrt(mu_sq - mu.mul(mu), sigma); + + fltImg = (fltImg - mu) / sigma; + + cv::normalize(fltImg, cfg.bgrFrame, 255,0, cv::NORM_MINMAX, CV_8U); + +} + +MPF_COMPONENT_CREATOR(OcvSsdFaceDetection); +MPF_COMPONENT_DELETER(); diff --git a/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.h b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.h new file mode 100644 index 00000000..ecdc2125 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/OcvSsdFaceDetection.h @@ -0,0 +1,79 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + + +#ifndef OPENMPF_COMPONENTS_OCVSsdFACEDETECTION_H +#define OPENMPF_COMPONENTS_OCVSsdFACEDETECTION_H + +#include +#include + +#include "adapters/MPFImageAndVideoDetectionComponentAdapter.h" + +#include "types.h" +#include "Track.h" +#include "DetectionLocation.h" +#include "JobConfig.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + + class OcvSsdFaceDetection : public MPFImageAndVideoDetectionComponentAdapter { + + public: + bool Init() override; + bool Close() override; + string GetDetectionType(){return "FACE";}; + MPFVideoTrackVec GetDetections(const MPFVideoJob &job) override; + MPFImageLocationVec GetDetections(const MPFImageJob &job) override; + + private: + + log4cxx::LoggerPtr _log; ///< log object + cv::Ptr _equalizerPtr; ///< adaptive histogram equalizer + + template + vector _calcAssignmentVector(const TrackPtrList &tracks, + const DetectionLocationPtrVec &detections, + const float maxCost); ///< determine costs of assigning detections to tracks + + MPFVideoTrack _convert_track(Track &track); ///< convert to MFVideoTrack and release + + void _equalizeHistogram(JobConfig &cfg); ///< perform histogram equalization on the frame + void _normalizeFrame(JobConfig &cfg); ///< perform image normalization + + void _assignDetections2Tracks(TrackPtrList &tracks, + DetectionLocationPtrVec &detections, + const vector &assignmentVector, + TrackPtrList &assignedTracks); ///< assign detections to tracks + + }; + } +} +#endif //OPENMPF_COMPONENTS_OCVFACEDETECTION_H diff --git a/cpp/OcvSsdFaceDetection/OrientationType.h b/cpp/OcvSsdFaceDetection/OrientationType.h new file mode 100644 index 00000000..75bd9bcc --- /dev/null +++ b/cpp/OcvSsdFaceDetection/OrientationType.h @@ -0,0 +1,202 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include +#include +#include +#include + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + enum OrientationType {ROTATE_90_CLOCKWISE = cv::ROTATE_90_CLOCKWISE, + ROTATE_180 = cv::ROTATE_180, + ROTATE_90_COUNTERCLOCKWISE = cv::ROTATE_90_COUNTERCLOCKWISE, + ROTATE_0}; + + using OrientVec = vector; ///< vector of 90 Orientations + + /** **************************************************************************** + * Write OrientationType to stream + ****************************************************************************** */ + inline + ostream& operator<< (ostream& os, const OrientationType& t) { + switch(t){ + case ROTATE_0: os << "0"; break; + case ROTATE_180: os << "180"; break; + case ROTATE_90_CLOCKWISE: os << "270"; break; + case ROTATE_90_COUNTERCLOCKWISE: os << "90"; break; + } + return os; + } + + /** **************************************************************************** + * Convet OrientationType to CCW angle from vertical [0-360) + * + * \param orientation enum to convert to float + * + * \return floatingpoint value corresponding to angle + * + ****************************************************************************** */ + inline + float degCCWfromVertical(const OrientationType& orientation){ + switch(orientation){ + case ROTATE_0: return 0.0; + case ROTATE_90_CLOCKWISE: return 90.0; + case ROTATE_180: return 180.0; + case ROTATE_90_COUNTERCLOCKWISE: return 270.0; + } + } + + /** **************************************************************************** + * Calculate CCW angle from vector as measued from vertical in range of [0,360) + * + * \param vec vector making the angle with the vertical axis + * + * \return ccw angle mad with vertical axis [0,360) + * + ****************************************************************************** */ + inline + float degCCWfromVertical(const cv::Point2f vec){ + float angle = -90.0 - atan2(vec.y,vec.x) * 180.0 / M_PI; + if(angle < 0.0){ angle += 360.0;} + return angle; + } + + /** **************************************************************************** + * Calculate angle difference between two angles a,b + * + * \param a 'a' angle in a - b + * \param b 'b' angle in a - b + * + * \return a - b in range [-180,180) + * + ****************************************************************************** */ + inline + int angleDiff(int a, int b){ + return (a - b + 540) % 360 - 180; + } + + /** **************************************************************************** + * Read OrientationType from ints in a stream + ****************************************************************************** */ + inline + istream& operator>> (istream& is, OrientationType& t ){ + unsigned int x; + if(is >> x){ + switch(x){ + case 0: t = ROTATE_0; break; + case 90: t = ROTATE_90_COUNTERCLOCKWISE; break; + case 180: t = ROTATE_180; break; + case 270: t = ROTATE_90_CLOCKWISE; break; + default: throw new invalid_argument("Value " + to_string(x) + " of enum OrientationType is not supported"); + } + } + return is; + } + + /** **************************************************************************** + * Inverse transfrom + * + * \param orientation for which to get the inverse operation + * + * \return operation to undo rotation + * + ****************************************************************************** */ + inline + OrientationType inv(const OrientationType& orientation){ + switch(orientation){ + case ROTATE_0: return ROTATE_0; + case ROTATE_90_COUNTERCLOCKWISE: return ROTATE_90_CLOCKWISE; + case ROTATE_180: return ROTATE_180; + case ROTATE_90_CLOCKWISE: return ROTATE_90_COUNTERCLOCKWISE; + } + } + + + /** **************************************************************************** + * Rotate a point to in an image to a corresponding position if the image + * were rotated to the given orientation + * + * \param pt point to be rotated + * \param orientation orientation of image to which to rotate the point to + * \param canvasSize size of oriented/rotated destination image + * + * \return equivalent coordinates in rotated/oriented image + * + ***************************************************************************** */ + template inline + cv::Point_ rotate(const cv::Point_ &pt,const OrientationType &orientation, const cv::Size_ &canvasSize){ + cv::Point_ p(pt); + switch(orientation) { + case ROTATE_90_COUNTERCLOCKWISE: swap(p.x, p.y); p.y = canvasSize.height - p.y; break; + case ROTATE_90_CLOCKWISE: swap(p.x, p.y); p.x = canvasSize.width - p.x; break; + case ROTATE_180: p.x = canvasSize.width - p.x; p.y = canvasSize.height - p.y; break; + } + return p; + } + + /** **************************************************************************** + * Rotate a rectangel to in an image to a corresponding position if the image + * were rotated to the given orientation + * + * \param rec rectangle for which to find corresponding recangle + * \param orientation orientation of image to which to rotate the point to + * \param canvasSize size of oriented/rotated destination image + * + * \return equivalent rectangle in rotated/oriented image + * + ***************************************************************************** */ + template inline + cv::Rect_ rotate(const cv::Rect_ &rec, const OrientationType &orientation, const cv::Size_ &canvasSize){ + cv::Point_ p1 = rotate(rec.tl(),orientation,canvasSize); + cv::Point_ p2 = rotate(rec.br(),orientation,canvasSize); + return cv::Rect_(min(p1.x,p2.x),min(p1.y,p2.y),abs(p1.x-p2.x),abs(p1.y-p2.y)); + } + + /** **************************************************************************** + * Rotate an image to the specified orientation + * + * \param img image to be rotated + * \param orientation orientation to rotate image to + * + * \return rotated image + * + ***************************************************************************** */ + inline + cv::Mat rotate(cv::Mat img,const OrientationType &orientation){ + cv::Mat rot; + switch(orientation){ + case ROTATE_0: rot = img; break; + case ROTATE_90_COUNTERCLOCKWISE: cv::rotate(img,rot,cv::ROTATE_90_COUNTERCLOCKWISE); break; + case ROTATE_180: cv::rotate(img,rot,cv::ROTATE_180); break; + case ROTATE_90_CLOCKWISE: cv::rotate(img,rot,cv::ROTATE_90_CLOCKWISE); break; + } + return rot; + } + } +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/README.md b/cpp/OcvSsdFaceDetection/README.md new file mode 100644 index 00000000..149363e5 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/README.md @@ -0,0 +1,29 @@ +# Overview + +This repository contains source code for the MPF OpenCV SSD face detection component. This component detects faces in images and videos using +the OpenCV Single Shot Detector. + +## Trained Models + +The OpenCV SSD face detection component comes with three trained models: + +* A discretized tensorflow version of OpenCV's Single Shot Detector (SSD) for faces, [opencv_face_detector_uint8.pbtxt](https://github.com/opencv/opencv_extra/tree/master/testdata/dnn/opencv_face_detector.pbtxt) and [opencv_face_detector_uint8.pb](https://github.com/opencv/opencv_3rdparty/raw/8033c2bc31b3256f0d461c919ecc01c2428ca03b/opencv_face_detector_uint8.pb). This model was discretized from an existing Caffe model, [opencv_face_detector.caffemodel](https://github.com/opencv/opencv_3rdparty/raw/dnn_samples_face_detector_20170830/res10_300x300_ssd_iter_140000.caffemodel). More details about how this model was trained can be found [here](https://github.com/opencv/opencv/blob/3.4.3/samples/dnn/face_detector/how_to_train_face_detector.txt). Details of the SSD model architecture are presented in [Liu et al.](https://arxiv.org/abs/1512.02325), *"SSD: Single Shot MultiBox Detector"*, 2016 +* A 5 landmark detector model for faces, [shape_predictor_5_face_landmarks.dat](http://dlib.net/files/shape_predictor_5_face_landmarks.dat.bz2) from DLIB and trained by its author on 7198 faces, see [here](https://github.com/davisking/dlib-models/blob/master/README.md#shape_predictor_5_face_landmarksdatbz2). This model is used to align faces prior to feature generation. It is an implementation of the Ensemble of Regression Trees (ERT) presented by [Kazemi and Sullivan](https://www.cv-foundation.org/openaccess/content_cvpr_2014/html/Kazemi_One_Millisecond_Face_2014_CVPR_paper.html), *"One Millisecond Face Alignment with an Ensemble of Regression Trees"*, 2014 +* A face feature generator [Torch](http://torch.ch/) model, [nn4.small2.v1.t7](https://storage.cmusatyalab.org/openface-models/nn4.small2.v1.t7) from [OpenFace](https://cmusatyalab.github.io/openface/). This model was trained with publicly-available face recognition datasets based on names: FaceScrub and CASIA-WebFace and generates a 128 dimensional embedding for faces. It is used in the tracking portion of the MPF component. More details about OpenFace can be found in [Amos, Ludwiczuk and Satyanarayanan](http://elijah.cs.cmu.edu/DOCS/CMU-CS-16-118.pdf), *"Openface: A general-purpose face recognition library with mobile applications"*, 2016 + +## Algorithms used +Both [OpenCV](https://opencv.org) and [DLIB](http://dlib.net) algorithms are used in MPF. This component specifically uses: + +* A Minimum Output Sum of Squared Error (MOSSE) tracker implemented in OpenCV's tracking API based on [Bolme et al.](https://ieeexplore.ieee.org/document/5539960), *"Visual object tracking using adaptive correlation filters."*, 2010 +* A linear assignment cost solver, a DLIB implementation of the Hungarian algorithm (also know as the Kuhn-Munkres algorithm) to solve detection to track assignment. See [Kuhn](https://doi.org/10.1002/nav.3800020109), *"The Hungarian Method for the assignment problem"*, 1955 for original method. + +## Tracking implementation +Detected faces in videos are aggregated into tracks as frames are processed. This is done using multiple stages and linear assignment cost solver: While processing each frame, tracks that have not had any detections of some number of frames are terminated, then detections are assigned to the remaining tracks using Intersection Over Union (IoU) as a cost, any unassigned detections are attempted to be assigned to tracks using the cos distance with the OpenFace features. Any tracks that did not receive a detection are then continued using a correlation tracker prediction. Any unassigned new detections become new tracks. + +Since detection and feature computation are computationally expensive operations, they can be performed only every so many frames and the faster correlation tracker will have to fill in the gaps. The processing speed gained by doing this comes at the cost of potentially missed detections and more track fragmentation. + +- - - + +### Note + +The component was coded using [Visual Studio Code](https://code.visualstudio.com) in a docker container.  The `.vscode` and `.devcontainer` subdirectories contain setting for the development tool but are in no way required to use or build the project. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/Track.cpp b/cpp/OcvSsdFaceDetection/Track.cpp new file mode 100644 index 00000000..164e0196 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/Track.cpp @@ -0,0 +1,160 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include "Track.h" + +using namespace MPF::COMPONENT; + +// Shared static members (might need mutex locks and condition variable if multithreading... ) +log4cxx::LoggerPtr Track::_log; + +string format(cv::Mat m); + +/** *************************************************************************** +* Constructor +* +* \param detPtr a 1st detection to add to the new track +* \param cfg job struct to initialize kalman filter params from +* +**************************************************************************** */ +Track::Track(unique_ptr detPtr, const JobConfig &cfg){ + if(! cfg.kfDisabled){ + _kfPtr = unique_ptr( + new KFTracker(cfg.frameTimeInSec, + cfg.frameTimeStep, + detPtr->getRect(), + cv::Rect2i(0,0,cfg.bgrFrame.cols-1,cfg.bgrFrame.rows-1), + cfg.RN, + cfg.QN)); + } + _locationPtrs.push_back(move(detPtr)); +} + +/** ************************************************************************** +* Get a new DetectionLocation from an existing one based on a frame +* +* \param cfg job struct to access current image frame from +* +* \returns ptr to new location based on tracker's estimation +* +* \note tracker is passed on to the new location on success +* +**************************************************************************** */ +DetectionLocationPtr Track::ocvTrackerPredict(const JobConfig &cfg){ + + if(_trackerPtr.empty()){ // initialize a new tracker if we don't have one already + cv::Rect2i bbox = _locationPtrs.back()->getRect(); + cv::Rect2i overlap = bbox & cv::Rect2i(0,0,_locationPtrs.back()->getBGRFrame().cols -1, + _locationPtrs.back()->getBGRFrame().rows -1); + if(overlap.width > 1 && overlap.height > 1){ + _trackerPtr = cv::TrackerMOSSE::create(); // could try different trackers here. e.g. cv::TrackerKCF::create(); + _trackerPtr->init(_locationPtrs.back()->getBGRFrame(), bbox); LOG4CXX_TRACE(_log,"tracker created for " << (MPFImageLocation)*_locationPtrs.back()); + _trackerStartFrameIdx = cfg.frameIdx; + }else{ LOG4CXX_TRACE(_log,"can't create tracker created for " << (MPFImageLocation)*_locationPtrs.back()); + return nullptr; + } + } + + cv::Rect2d p; + DetectionLocationPtr detPtr; + if(cfg.frameIdx - _trackerStartFrameIdx <= cfg.maxFrameGap){ + if(_trackerPtr->update(cfg.bgrFrame,p)){ + detPtr = DetectionLocationPtr(new DetectionLocation( + p.x, p.y, p.width, p.height, 0.0, + cv::Point2f((p.x + p.width/2.0f )/static_cast(cfg.bgrFrame.cols), + (p.y + p.height/2.0f)/static_cast(cfg.bgrFrame.rows)), + cfg.frameIdx, + cfg.frameTimeInSec, + cfg.bgrFrame, + cfg.bgrFrame, + OrientationType::ROTATE_0)); LOG4CXX_TRACE(_log,"tracking " << (MPFImageLocation)*_locationPtrs.back() << " to " << (MPFImageLocation)*detPtr); + // clone feature of prior detection + detPtr->copyFeature(*(_locationPtrs.back())); + }else{ + LOG4CXX_TRACE(_log,"could not track " << (MPFImageLocation)*_locationPtrs.back() << " to new location"); + } + }else{ + LOG4CXX_TRACE(_log,"extrapolation tracking stopped" << (MPFImageLocation)*_locationPtrs.back() << " frame gap = " << cfg.frameIdx - _trackerStartFrameIdx << " > " << cfg.maxFrameGap); + } + return detPtr; +} + +/** ************************************************************************** +* Add detection pointer to the end of the track +* +* \param d pointer to the detection to append to the track +* +* \note the previous track tail will have its image frame released +*************************************************************************** */ +void Track::push_back(DetectionLocationPtr d){ + + assert(_locationPtrs.size() > 0); + _locationPtrs.back()->releaseBGRFrame(); + _locationPtrs.push_back(move(d)); +} + +/** ************************************************************************** + * Advance Kalman filter state to predict next bbox at time t + * + * \param t time in sec to which kalamn filter state is advanced to + * +*************************************************************************** */ +void Track::kalmanPredict(float t){ + _kfPtr->predict(t); LOG4CXX_TRACE(_log,"kf pred:" << _locationPtrs.back()->getRect() << " => " << _kfPtr->predictedBBox()); +} + +/** ************************************************************************** + * apply kalman correction to tail detection using tail's measurement +*************************************************************************** */ +void Track::kalmanCorrect(){ + if(_kfPtr){ LOG4CXX_TRACE(_log,"kf meas:" << _locationPtrs.back()->getRect()); + _kfPtr->correct(_locationPtrs.back()->getRect()); LOG4CXX_TRACE(_log,"kf corr:" << _locationPtrs.back()->getRect()); + back()->setRect(_kfPtr->correctedBBox()); + } +} + +/** ************************************************************************** +* Setup class shared static configurations and initialize +* +* \param log logger object for logging +* \param plugin_path plugin directory +* +* \return true if everything was properly initialized, false otherwise +*************************************************************************** */ +bool Track::Init(log4cxx::LoggerPtr log, string plugin_path=""){ + _log = log; + return true; +} + +/** ************************************************************************** +* Dump MPF::COMPONENT::Track to a stream +*************************************************************************** */ +ostream& MPF::COMPONENT::operator<< (ostream& out, const Track& t) { + out << "frameIdx << (MPFImageLocation)(*t.front()) + << "...f" << t.back()->frameIdx << (MPFImageLocation)(*t.back()) + << ">("< + +#include + +#include "types.h" +#include "JobConfig.h" +#include "DetectionLocation.h" +#include "KFTracker.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + class Track{ + + public: + + static bool Init(log4cxx::LoggerPtr log, const string plugin_path); ///< setup class shared members + + DetectionLocationPtr ocvTrackerPredict(const JobConfig &cfg); ///< predict a new detection from an exiting one using a tracker + void releaseTracker() {_trackerPtr.release();} ///< release tracker so it can be reinitialized + + // Vector like interface detection pointer in tack + const DetectionLocationPtr &at (size_t i) const {return _locationPtrs.at(i);} + const DetectionLocationPtr &operator[](size_t i) const {return _locationPtrs[i];} + const DetectionLocationPtr &front() const {return _locationPtrs.front();} + const DetectionLocationPtr &back() const {return _locationPtrs.back();} + const size_t size() const {return _locationPtrs.size();} + DetectionLocationPtrVec::iterator begin() {return _locationPtrs.begin();} + DetectionLocationPtrVec::iterator end() {return _locationPtrs.end();} + void push_back(DetectionLocationPtr d); + + void kalmanPredict(float t); + void kalmanCorrect(); + const cv::Rect2i kalmanPredictedBox() const {return _kfPtr->predictedBBox();} + + #ifdef DIAGNOSTIC_FILES + void kalmanDump(){_kfPtr->dump();}; + #endif + + Track(DetectionLocationPtr detPtr, const JobConfig &cfg); + + private: + static log4cxx::LoggerPtr _log; ///< shared log object + + DetectionLocationPtrVec _locationPtrs; ///< vector of pointers to locations making up track + cv::Ptr _trackerPtr; ///< openCV tracker to help bridge gaps when detector fails + size_t _trackerStartFrameIdx; ///< frame index at which the tracker was initialized + + unique_ptr _kfPtr; ///< kalman filter tracker + + }; + + ostream& operator<< (ostream& out, const Track& t); + + } +} + +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/plugin-files/config/Log4cxxConfig.xml b/cpp/OcvSsdFaceDetection/plugin-files/config/Log4cxxConfig.xml new file mode 100644 index 00000000..8e8ad288 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/plugin-files/config/Log4cxxConfig.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cpp/OcvSsdFaceDetection/plugin-files/config/mpfOcvSsdFaceDetection.ini b/cpp/OcvSsdFaceDetection/plugin-files/config/mpfOcvSsdFaceDetection.ini new file mode 100644 index 00000000..bbffce8b --- /dev/null +++ b/cpp/OcvSsdFaceDetection/plugin-files/config/mpfOcvSsdFaceDetection.ini @@ -0,0 +1,78 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +# minimum x and y pixel size passed to the opencv face detector +MIN_DETECTION_SIZE: 25 + +# Max image dimension to use for inferencing e.g. 300 +DETECTION_INFERENCE_SIZE: 300 + +# minimum face detection confidence value needed to start a track, must be greater than 0 +DETECTION_CONFIDENCE_THRESHOLD: 0.3 + +# perform multiple passes at different image orientations to increase detections found +ROTATE_AND_DETECT: true + +# ccw rotations of frame to inference (only multiples of 90 are accepted) +ROTATE_ORIENTATIONS: [0, 90, 180, 270] + +# detection non-maximum suppression threshold for removing redundant overlapping bounding boxes +DETECTION_NMS_THRESHOLD: 0.3 + +# number of frames to skip detection and rely on just tracking +DETECTION_FRAME_INTERVAL: 0 + +# scale the width and height of the detector bounding box +DETECTION_BOUNDING_BOX_SCALE_FACTOR: 1.0 + +# tracks without detections for this many frames will be terminated +TRACKING_MAX_FRAME_GAP: 4 + +# threshold for 1-iou distance below which detection will be considered part of same track +TRACKING_MAX_IOU_DIST: 0.5 + +# threshold for 1-cos_similarity in DNN feature vectors below which detection will be considered part of same track +TRACKING_MAX_FEATURE_DIST: 0.35 + +# threshold for (centroid to centroid) / frame_diag below which detection will be considered part of same track +TRACKING_MAX_CENTER_DIST: 0.0 + +# Disable Kalman Filter processing +KF_DISABLED: false + +# detector measurement noise: standard deviations for kalman filter measurment noise covariance matrix R +# [x,y,width,height] +KF_RN: [ 6.0, 6.0, 6.0, 6.0 ] + +# motion model inaccuracies/noise: standard deviations of accelerations for kalman filter process noise covariances matrix Q +# [ax, ay, aw, ah] +KF_QN: [ 10000.0, 10000.0, 1000.0, 1000.0 ] + +# GPU device ID to use. If negative, only CPU will be used +CUDA_DEVICE_ID: 1 + +# If using GPU fails, the use CPU +FALLBACK_TO_CPU_WHEN_GPU_PROBLEM: false diff --git a/cpp/OcvSsdFaceDetection/plugin-files/data/NOTICE b/cpp/OcvSsdFaceDetection/plugin-files/data/NOTICE new file mode 100644 index 00000000..6bc1d5ba --- /dev/null +++ b/cpp/OcvSsdFaceDetection/plugin-files/data/NOTICE @@ -0,0 +1,20 @@ +This software makes use of a data model derived from third party software: + +-------------------------------------------------------------------------- + +# shape_predictor_5_face_landmarks.dat +originally created by DLIB author downloaded from DLIB https://github.com/davisking/dlib-models/blob/master/shape_predictor_5_face_landmarks.dat.bz2 +Creative Commons Zero v1.0 Universal License http://creativecommons.org/publicdomain/zero/1.0/ + +# opencv_face_detector_uint8.pb and opencv_face_detector.pbtxt +Single shot multi box face detector for 300x300 images +derived from res10_300x300_ssd_iter_140000_fp16.caffemodel by quantization and conversion to tensorflow +downloaded from https://github.com/opencv/opencv_3rdparty/raw/8033c2bc31b3256f0d461c919ecc01c2428ca03b/opencv_face_detector_uint8.pb +and https://github.com/opencv/opencv_extra/tree/master/testdata/dnn/opencv_face_detector.pbtxt +The training data used to create this model is not specified, the procedure used is detailed here +https://github.com/opencv/opencv/blob/master/samples/dnn/face_detector/how_to_train_face_detector.txt + +# nn4.small2.v1.t7 +OpenFace feature generator +downloaded from https://storage.cmusatyalab.org/openface-models/nn4.small2.v1.t7 +Apache License 2.0 http://www.apache.org/licenses/LICENSE-2.0 diff --git a/cpp/OcvSsdFaceDetection/plugin-files/data/nn4.small2.v1.t7 b/cpp/OcvSsdFaceDetection/plugin-files/data/nn4.small2.v1.t7 new file mode 100644 index 00000000..8a0d0197 Binary files /dev/null and b/cpp/OcvSsdFaceDetection/plugin-files/data/nn4.small2.v1.t7 differ diff --git a/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector.pbtxt b/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector.pbtxt new file mode 100644 index 00000000..60daa5e6 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector.pbtxt @@ -0,0 +1,2368 @@ +node { + name: "data" + op: "Placeholder" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } +} +node { + name: "data_bn/FusedBatchNorm" + op: "FusedBatchNorm" + input: "data:0" + input: "data_bn/gamma" + input: "data_bn/beta" + input: "data_bn/mean" + input: "data_bn/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "data_scale/Mul" + op: "Mul" + input: "data_bn/FusedBatchNorm" + input: "data_scale/mul" +} +node { + name: "data_scale/BiasAdd" + op: "BiasAdd" + input: "data_scale/Mul" + input: "data_scale/add" +} +node { + name: "SpaceToBatchND/block_shape" + op: "Const" + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + } + int_val: 1 + int_val: 1 + } + } + } +} +node { + name: "SpaceToBatchND/paddings" + op: "Const" + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + dim { + size: 2 + } + } + int_val: 3 + int_val: 3 + int_val: 3 + int_val: 3 + } + } + } +} +node { + name: "Pad" + op: "SpaceToBatchND" + input: "data_scale/BiasAdd" + input: "SpaceToBatchND/block_shape" + input: "SpaceToBatchND/paddings" +} +node { + name: "conv1_h/Conv2D" + op: "Conv2D" + input: "Pad" + input: "conv1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "conv1_h/BiasAdd" + op: "BiasAdd" + input: "conv1_h/Conv2D" + input: "conv1_h/bias" +} +node { + name: "BatchToSpaceND" + op: "BatchToSpaceND" + input: "conv1_h/BiasAdd" +} +node { + name: "conv1_bn_h/FusedBatchNorm" + op: "FusedBatchNorm" + input: "BatchToSpaceND" + input: "conv1_bn_h/gamma" + input: "conv1_bn_h/beta" + input: "conv1_bn_h/mean" + input: "conv1_bn_h/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "conv1_scale_h/Mul" + op: "Mul" + input: "conv1_bn_h/FusedBatchNorm" + input: "conv1_scale_h/mul" +} +node { + name: "conv1_scale_h/BiasAdd" + op: "BiasAdd" + input: "conv1_scale_h/Mul" + input: "conv1_scale_h/add" +} +node { + name: "Relu" + op: "Relu" + input: "conv1_scale_h/BiasAdd" +} +node { + name: "conv1_pool/MaxPool" + op: "MaxPool" + input: "Relu" + attr { + key: "ksize" + value { + list { + i: 1 + i: 3 + i: 3 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "layer_64_1_conv1_h/Conv2D" + op: "Conv2D" + input: "conv1_pool/MaxPool" + input: "layer_64_1_conv1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "layer_64_1_bn2_h/FusedBatchNorm" + op: "BiasAdd" + input: "layer_64_1_conv1_h/Conv2D" + input: "layer_64_1_conv1_h/Conv2D_bn_offset" +} +node { + name: "layer_64_1_scale2_h/Mul" + op: "Mul" + input: "layer_64_1_bn2_h/FusedBatchNorm" + input: "layer_64_1_scale2_h/mul" +} +node { + name: "layer_64_1_scale2_h/BiasAdd" + op: "BiasAdd" + input: "layer_64_1_scale2_h/Mul" + input: "layer_64_1_scale2_h/add" +} +node { + name: "Relu_1" + op: "Relu" + input: "layer_64_1_scale2_h/BiasAdd" +} +node { + name: "layer_64_1_conv2_h/Conv2D" + op: "Conv2D" + input: "Relu_1" + input: "layer_64_1_conv2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "add" + op: "Add" + input: "layer_64_1_conv2_h/Conv2D" + input: "conv1_pool/MaxPool" +} +node { + name: "layer_128_1_bn1_h/FusedBatchNorm" + op: "FusedBatchNorm" + input: "add" + input: "layer_128_1_bn1_h/gamma" + input: "layer_128_1_bn1_h/beta" + input: "layer_128_1_bn1_h/mean" + input: "layer_128_1_bn1_h/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "layer_128_1_scale1_h/Mul" + op: "Mul" + input: "layer_128_1_bn1_h/FusedBatchNorm" + input: "layer_128_1_scale1_h/mul" +} +node { + name: "layer_128_1_scale1_h/BiasAdd" + op: "BiasAdd" + input: "layer_128_1_scale1_h/Mul" + input: "layer_128_1_scale1_h/add" +} +node { + name: "Relu_2" + op: "Relu" + input: "layer_128_1_scale1_h/BiasAdd" +} +node { + name: "layer_128_1_conv_expand_h/Conv2D" + op: "Conv2D" + input: "Relu_2" + input: "layer_128_1_conv_expand_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "layer_128_1_conv1_h/Conv2D" + op: "Conv2D" + input: "Relu_2" + input: "layer_128_1_conv1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "layer_128_1_bn2/FusedBatchNorm" + op: "BiasAdd" + input: "layer_128_1_conv1_h/Conv2D" + input: "layer_128_1_conv1_h/Conv2D_bn_offset" +} +node { + name: "layer_128_1_scale2/Mul" + op: "Mul" + input: "layer_128_1_bn2/FusedBatchNorm" + input: "layer_128_1_scale2/mul" +} +node { + name: "layer_128_1_scale2/BiasAdd" + op: "BiasAdd" + input: "layer_128_1_scale2/Mul" + input: "layer_128_1_scale2/add" +} +node { + name: "Relu_3" + op: "Relu" + input: "layer_128_1_scale2/BiasAdd" +} +node { + name: "layer_128_1_conv2/Conv2D" + op: "Conv2D" + input: "Relu_3" + input: "layer_128_1_conv2/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "add_1" + op: "Add" + input: "layer_128_1_conv2/Conv2D" + input: "layer_128_1_conv_expand_h/Conv2D" +} +node { + name: "layer_256_1_bn1/FusedBatchNorm" + op: "FusedBatchNorm" + input: "add_1" + input: "layer_256_1_bn1/gamma" + input: "layer_256_1_bn1/beta" + input: "layer_256_1_bn1/mean" + input: "layer_256_1_bn1/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "layer_256_1_scale1/Mul" + op: "Mul" + input: "layer_256_1_bn1/FusedBatchNorm" + input: "layer_256_1_scale1/mul" +} +node { + name: "layer_256_1_scale1/BiasAdd" + op: "BiasAdd" + input: "layer_256_1_scale1/Mul" + input: "layer_256_1_scale1/add" +} +node { + name: "Relu_4" + op: "Relu" + input: "layer_256_1_scale1/BiasAdd" +} +node { + name: "SpaceToBatchND_1/paddings" + op: "Const" + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 2 + } + dim { + size: 2 + } + } + int_val: 1 + int_val: 1 + int_val: 1 + int_val: 1 + } + } + } +} +node { + name: "layer_256_1_conv_expand/Conv2D" + op: "Conv2D" + input: "Relu_4" + input: "layer_256_1_conv_expand/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "conv4_3_norm/l2_normalize" + op: "L2Normalize" + input: "Relu_4:0" + input: "conv4_3_norm/l2_normalize/Sum/reduction_indices" +} +node { + name: "conv4_3_norm/mul_1" + op: "Mul" + input: "conv4_3_norm/l2_normalize" + input: "conv4_3_norm/mul" +} +node { + name: "conv4_3_norm_mbox_loc/Conv2D" + op: "Conv2D" + input: "conv4_3_norm/mul_1" + input: "conv4_3_norm_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv4_3_norm_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "conv4_3_norm_mbox_loc/Conv2D" + input: "conv4_3_norm_mbox_loc/bias" +} +node { + name: "flatten/Reshape" + op: "Flatten" + input: "conv4_3_norm_mbox_loc/BiasAdd" +} +node { + name: "conv4_3_norm_mbox_conf/Conv2D" + op: "Conv2D" + input: "conv4_3_norm/mul_1" + input: "conv4_3_norm_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv4_3_norm_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "conv4_3_norm_mbox_conf/Conv2D" + input: "conv4_3_norm_mbox_conf/bias" +} +node { + name: "flatten_6/Reshape" + op: "Flatten" + input: "conv4_3_norm_mbox_conf/BiasAdd" +} +node { + name: "Pad_1" + op: "SpaceToBatchND" + input: "Relu_4" + input: "SpaceToBatchND/block_shape" + input: "SpaceToBatchND_1/paddings" +} +node { + name: "layer_256_1_conv1/Conv2D" + op: "Conv2D" + input: "Pad_1" + input: "layer_256_1_conv1/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "layer_256_1_bn2/FusedBatchNorm" + op: "BiasAdd" + input: "layer_256_1_conv1/Conv2D" + input: "layer_256_1_conv1/Conv2D_bn_offset" +} +node { + name: "BatchToSpaceND_1" + op: "BatchToSpaceND" + input: "layer_256_1_bn2/FusedBatchNorm" +} +node { + name: "layer_256_1_scale2/Mul" + op: "Mul" + input: "BatchToSpaceND_1" + input: "layer_256_1_scale2/mul" +} +node { + name: "layer_256_1_scale2/BiasAdd" + op: "BiasAdd" + input: "layer_256_1_scale2/Mul" + input: "layer_256_1_scale2/add" +} +node { + name: "Relu_5" + op: "Relu" + input: "layer_256_1_scale2/BiasAdd" +} +node { + name: "layer_256_1_conv2/Conv2D" + op: "Conv2D" + input: "Relu_5" + input: "layer_256_1_conv2/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "add_2" + op: "Add" + input: "layer_256_1_conv2/Conv2D" + input: "layer_256_1_conv_expand/Conv2D" +} +node { + name: "layer_512_1_bn1/FusedBatchNorm" + op: "FusedBatchNorm" + input: "add_2" + input: "layer_512_1_bn1/gamma" + input: "layer_512_1_bn1/beta" + input: "layer_512_1_bn1/mean" + input: "layer_512_1_bn1/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "layer_512_1_scale1/Mul" + op: "Mul" + input: "layer_512_1_bn1/FusedBatchNorm" + input: "layer_512_1_scale1/mul" +} +node { + name: "layer_512_1_scale1/BiasAdd" + op: "BiasAdd" + input: "layer_512_1_scale1/Mul" + input: "layer_512_1_scale1/add" +} +node { + name: "Relu_6" + op: "Relu" + input: "layer_512_1_scale1/BiasAdd" +} +node { + name: "layer_512_1_conv_expand_h/Conv2D" + op: "Conv2D" + input: "Relu_6" + input: "layer_512_1_conv_expand_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "layer_512_1_conv1_h/Conv2D" + op: "Conv2D" + input: "Relu_6" + input: "layer_512_1_conv1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "layer_512_1_bn2_h/FusedBatchNorm" + op: "BiasAdd" + input: "layer_512_1_conv1_h/Conv2D" + input: "layer_512_1_conv1_h/Conv2D_bn_offset" +} +node { + name: "layer_512_1_scale2_h/Mul" + op: "Mul" + input: "layer_512_1_bn2_h/FusedBatchNorm" + input: "layer_512_1_scale2_h/mul" +} +node { + name: "layer_512_1_scale2_h/BiasAdd" + op: "BiasAdd" + input: "layer_512_1_scale2_h/Mul" + input: "layer_512_1_scale2_h/add" +} +node { + name: "Relu_7" + op: "Relu" + input: "layer_512_1_scale2_h/BiasAdd" +} +node { + name: "layer_512_1_conv2_h/convolution/SpaceToBatchND" + op: "SpaceToBatchND" + input: "Relu_7" + input: "layer_512_1_conv2_h/convolution/SpaceToBatchND/block_shape" + input: "layer_512_1_conv2_h/convolution/SpaceToBatchND/paddings" +} +node { + name: "layer_512_1_conv2_h/convolution" + op: "Conv2D" + input: "layer_512_1_conv2_h/convolution/SpaceToBatchND" + input: "layer_512_1_conv2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "layer_512_1_conv2_h/convolution/BatchToSpaceND" + op: "BatchToSpaceND" + input: "layer_512_1_conv2_h/convolution" + input: "layer_512_1_conv2_h/convolution/BatchToSpaceND/block_shape" + input: "layer_512_1_conv2_h/convolution/BatchToSpaceND/crops" +} +node { + name: "add_3" + op: "Add" + input: "layer_512_1_conv2_h/convolution/BatchToSpaceND" + input: "layer_512_1_conv_expand_h/Conv2D" +} +node { + name: "last_bn_h/FusedBatchNorm" + op: "FusedBatchNorm" + input: "add_3" + input: "last_bn_h/gamma" + input: "last_bn_h/beta" + input: "last_bn_h/mean" + input: "last_bn_h/std" + attr { + key: "epsilon" + value { + f: 1.00099996416e-05 + } + } +} +node { + name: "last_scale_h/Mul" + op: "Mul" + input: "last_bn_h/FusedBatchNorm" + input: "last_scale_h/mul" +} +node { + name: "last_scale_h/BiasAdd" + op: "BiasAdd" + input: "last_scale_h/Mul" + input: "last_scale_h/add" +} +node { + name: "last_relu" + op: "Relu" + input: "last_scale_h/BiasAdd" +} +node { + name: "conv6_1_h/Conv2D" + op: "Conv2D" + input: "last_relu" + input: "conv6_1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv6_1_h/BiasAdd" + op: "BiasAdd" + input: "conv6_1_h/Conv2D" + input: "conv6_1_h/bias" +} +node { + name: "conv6_1_h/Relu" + op: "Relu" + input: "conv6_1_h/BiasAdd" +} +node { + name: "conv6_2_h/Conv2D" + op: "Conv2D" + input: "conv6_1_h/Relu" + input: "conv6_2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "conv6_2_h/BiasAdd" + op: "BiasAdd" + input: "conv6_2_h/Conv2D" + input: "conv6_2_h/bias" +} +node { + name: "conv6_2_h/Relu" + op: "Relu" + input: "conv6_2_h/BiasAdd" +} +node { + name: "conv7_1_h/Conv2D" + op: "Conv2D" + input: "conv6_2_h/Relu" + input: "conv7_1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv7_1_h/BiasAdd" + op: "BiasAdd" + input: "conv7_1_h/Conv2D" + input: "conv7_1_h/bias" +} +node { + name: "conv7_1_h/Relu" + op: "Relu" + input: "conv7_1_h/BiasAdd" +} +node { + name: "Pad_2" + op: "SpaceToBatchND" + input: "conv7_1_h/Relu" + input: "SpaceToBatchND/block_shape" + input: "SpaceToBatchND_1/paddings" +} +node { + name: "conv7_2_h/Conv2D" + op: "Conv2D" + input: "Pad_2" + input: "conv7_2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 2 + i: 2 + i: 1 + } + } + } +} +node { + name: "conv7_2_h/BiasAdd" + op: "BiasAdd" + input: "conv7_2_h/Conv2D" + input: "conv7_2_h/bias" +} +node { + name: "BatchToSpaceND_2" + op: "BatchToSpaceND" + input: "conv7_2_h/BiasAdd" +} +node { + name: "conv7_2_h/Relu" + op: "Relu" + input: "BatchToSpaceND_2" +} +node { + name: "conv8_1_h/Conv2D" + op: "Conv2D" + input: "conv7_2_h/Relu" + input: "conv8_1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv8_1_h/BiasAdd" + op: "BiasAdd" + input: "conv8_1_h/Conv2D" + input: "conv8_1_h/bias" +} +node { + name: "conv8_1_h/Relu" + op: "Relu" + input: "conv8_1_h/BiasAdd" +} +node { + name: "conv8_2_h/Conv2D" + op: "Conv2D" + input: "conv8_1_h/Relu" + input: "conv8_2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv8_2_h/BiasAdd" + op: "BiasAdd" + input: "conv8_2_h/Conv2D" + input: "conv8_2_h/bias" +} +node { + name: "conv8_2_h/Relu" + op: "Relu" + input: "conv8_2_h/BiasAdd" +} +node { + name: "conv9_1_h/Conv2D" + op: "Conv2D" + input: "conv8_2_h/Relu" + input: "conv9_1_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv9_1_h/BiasAdd" + op: "BiasAdd" + input: "conv9_1_h/Conv2D" + input: "conv9_1_h/bias" +} +node { + name: "conv9_1_h/Relu" + op: "Relu" + input: "conv9_1_h/BiasAdd" +} +node { + name: "conv9_2_h/Conv2D" + op: "Conv2D" + input: "conv9_1_h/Relu" + input: "conv9_2_h/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "VALID" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv9_2_h/BiasAdd" + op: "BiasAdd" + input: "conv9_2_h/Conv2D" + input: "conv9_2_h/bias" +} +node { + name: "conv9_2_h/Relu" + op: "Relu" + input: "conv9_2_h/BiasAdd" +} +node { + name: "conv9_2_mbox_loc/Conv2D" + op: "Conv2D" + input: "conv9_2_h/Relu" + input: "conv9_2_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv9_2_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "conv9_2_mbox_loc/Conv2D" + input: "conv9_2_mbox_loc/bias" +} +node { + name: "flatten_5/Reshape" + op: "Flatten" + input: "conv9_2_mbox_loc/BiasAdd" +} +node { + name: "conv9_2_mbox_conf/Conv2D" + op: "Conv2D" + input: "conv9_2_h/Relu" + input: "conv9_2_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv9_2_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "conv9_2_mbox_conf/Conv2D" + input: "conv9_2_mbox_conf/bias" +} +node { + name: "flatten_11/Reshape" + op: "Flatten" + input: "conv9_2_mbox_conf/BiasAdd" +} +node { + name: "conv8_2_mbox_loc/Conv2D" + op: "Conv2D" + input: "conv8_2_h/Relu" + input: "conv8_2_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv8_2_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "conv8_2_mbox_loc/Conv2D" + input: "conv8_2_mbox_loc/bias" +} +node { + name: "flatten_4/Reshape" + op: "Flatten" + input: "conv8_2_mbox_loc/BiasAdd" +} +node { + name: "conv8_2_mbox_conf/Conv2D" + op: "Conv2D" + input: "conv8_2_h/Relu" + input: "conv8_2_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv8_2_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "conv8_2_mbox_conf/Conv2D" + input: "conv8_2_mbox_conf/bias" +} +node { + name: "flatten_10/Reshape" + op: "Flatten" + input: "conv8_2_mbox_conf/BiasAdd" +} +node { + name: "conv7_2_mbox_loc/Conv2D" + op: "Conv2D" + input: "conv7_2_h/Relu" + input: "conv7_2_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv7_2_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "conv7_2_mbox_loc/Conv2D" + input: "conv7_2_mbox_loc/bias" +} +node { + name: "flatten_3/Reshape" + op: "Flatten" + input: "conv7_2_mbox_loc/BiasAdd" +} +node { + name: "conv7_2_mbox_conf/Conv2D" + op: "Conv2D" + input: "conv7_2_h/Relu" + input: "conv7_2_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv7_2_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "conv7_2_mbox_conf/Conv2D" + input: "conv7_2_mbox_conf/bias" +} +node { + name: "flatten_9/Reshape" + op: "Flatten" + input: "conv7_2_mbox_conf/BiasAdd" +} +node { + name: "conv6_2_mbox_loc/Conv2D" + op: "Conv2D" + input: "conv6_2_h/Relu" + input: "conv6_2_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv6_2_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "conv6_2_mbox_loc/Conv2D" + input: "conv6_2_mbox_loc/bias" +} +node { + name: "flatten_2/Reshape" + op: "Flatten" + input: "conv6_2_mbox_loc/BiasAdd" +} +node { + name: "conv6_2_mbox_conf/Conv2D" + op: "Conv2D" + input: "conv6_2_h/Relu" + input: "conv6_2_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "conv6_2_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "conv6_2_mbox_conf/Conv2D" + input: "conv6_2_mbox_conf/bias" +} +node { + name: "flatten_8/Reshape" + op: "Flatten" + input: "conv6_2_mbox_conf/BiasAdd" +} +node { + name: "fc7_mbox_loc/Conv2D" + op: "Conv2D" + input: "last_relu" + input: "fc7_mbox_loc/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "fc7_mbox_loc/BiasAdd" + op: "BiasAdd" + input: "fc7_mbox_loc/Conv2D" + input: "fc7_mbox_loc/bias" +} +node { + name: "flatten_1/Reshape" + op: "Flatten" + input: "fc7_mbox_loc/BiasAdd" +} +node { + name: "mbox_loc" + op: "ConcatV2" + input: "flatten/Reshape" + input: "flatten_1/Reshape" + input: "flatten_2/Reshape" + input: "flatten_3/Reshape" + input: "flatten_4/Reshape" + input: "flatten_5/Reshape" + input: "mbox_loc/axis" +} +node { + name: "fc7_mbox_conf/Conv2D" + op: "Conv2D" + input: "last_relu" + input: "fc7_mbox_conf/weights" + attr { + key: "dilations" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } + attr { + key: "padding" + value { + s: "SAME" + } + } + attr { + key: "strides" + value { + list { + i: 1 + i: 1 + i: 1 + i: 1 + } + } + } +} +node { + name: "fc7_mbox_conf/BiasAdd" + op: "BiasAdd" + input: "fc7_mbox_conf/Conv2D" + input: "fc7_mbox_conf/bias" +} +node { + name: "flatten_7/Reshape" + op: "Flatten" + input: "fc7_mbox_conf/BiasAdd" +} +node { + name: "mbox_conf" + op: "ConcatV2" + input: "flatten_6/Reshape" + input: "flatten_7/Reshape" + input: "flatten_8/Reshape" + input: "flatten_9/Reshape" + input: "flatten_10/Reshape" + input: "flatten_11/Reshape" + input: "mbox_conf/axis" +} +node { + name: "mbox_conf_reshape" + op: "Reshape" + input: "mbox_conf" + input: "reshape_before_softmax" +} +node { + name: "mbox_conf_softmax" + op: "Softmax" + input: "mbox_conf_reshape" + attr { + key: "axis" + value { + i: 2 + } + } +} +node { + name: "mbox_conf_flatten" + op: "Flatten" + input: "mbox_conf_softmax" +} +node { + name: "PriorBox_0" + op: "PriorBox" + input: "conv4_3_norm/mul_1" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + } + float_val: 2.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 60 + } + } + attr { + key: "min_size" + value { + i: 30 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 8.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "PriorBox_1" + op: "PriorBox" + input: "last_relu" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 2 + } + } + float_val: 2.0 + float_val: 3.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 111 + } + } + attr { + key: "min_size" + value { + i: 60 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 16.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "PriorBox_2" + op: "PriorBox" + input: "conv6_2_h/Relu" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 2 + } + } + float_val: 2.0 + float_val: 3.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 162 + } + } + attr { + key: "min_size" + value { + i: 111 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 32.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "PriorBox_3" + op: "PriorBox" + input: "conv7_2_h/Relu" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 2 + } + } + float_val: 2.0 + float_val: 3.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 213 + } + } + attr { + key: "min_size" + value { + i: 162 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 64.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "PriorBox_4" + op: "PriorBox" + input: "conv8_2_h/Relu" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + } + float_val: 2.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 264 + } + } + attr { + key: "min_size" + value { + i: 213 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 100.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "PriorBox_5" + op: "PriorBox" + input: "conv9_2_h/Relu" + input: "data" + attr { + key: "aspect_ratio" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + } + float_val: 2.0 + } + } + } + attr { + key: "clip" + value { + b: false + } + } + attr { + key: "flip" + value { + b: true + } + } + attr { + key: "max_size" + value { + i: 315 + } + } + attr { + key: "min_size" + value { + i: 264 + } + } + attr { + key: "offset" + value { + f: 0.5 + } + } + attr { + key: "step" + value { + f: 300.0 + } + } + attr { + key: "variance" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 4 + } + } + float_val: 0.10000000149 + float_val: 0.10000000149 + float_val: 0.20000000298 + float_val: 0.20000000298 + } + } + } +} +node { + name: "mbox_priorbox" + op: "ConcatV2" + input: "PriorBox_0" + input: "PriorBox_1" + input: "PriorBox_2" + input: "PriorBox_3" + input: "PriorBox_4" + input: "PriorBox_5" + input: "mbox_loc/axis" +} +node { + name: "detection_out" + op: "DetectionOutput" + input: "mbox_loc" + input: "mbox_conf_flatten" + input: "mbox_priorbox" + attr { + key: "background_label_id" + value { + i: 0 + } + } + attr { + key: "code_type" + value { + s: "CENTER_SIZE" + } + } + attr { + key: "confidence_threshold" + value { + f: 0.00999999977648 + } + } + attr { + key: "keep_top_k" + value { + i: 200 + } + } + attr { + key: "nms_threshold" + value { + f: 0.449999988079 + } + } + attr { + key: "num_classes" + value { + i: 2 + } + } + attr { + key: "share_location" + value { + b: true + } + } + attr { + key: "top_k" + value { + i: 400 + } + } + attr { + key: "clip" + value { + b: true + } + } +} +node { + name: "reshape_before_softmax" + op: "Const" + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + dim { + size: 3 + } + } + int_val: 0 + int_val: -1 + int_val: 2 + } + } + } +} +library { +} diff --git a/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector_uint8.pb b/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector_uint8.pb new file mode 100644 index 00000000..124bdd1a Binary files /dev/null and b/cpp/OcvSsdFaceDetection/plugin-files/data/opencv_face_detector_uint8.pb differ diff --git a/cpp/OcvSsdFaceDetection/plugin-files/data/shape_predictor_5_face_landmarks.dat b/cpp/OcvSsdFaceDetection/plugin-files/data/shape_predictor_5_face_landmarks.dat new file mode 100644 index 00000000..67878ed3 Binary files /dev/null and b/cpp/OcvSsdFaceDetection/plugin-files/data/shape_predictor_5_face_landmarks.dat differ diff --git a/cpp/OcvSsdFaceDetection/plugin-files/descriptor/descriptor.json b/cpp/OcvSsdFaceDetection/plugin-files/descriptor/descriptor.json new file mode 100644 index 00000000..489891d7 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/plugin-files/descriptor/descriptor.json @@ -0,0 +1,224 @@ +{ + "componentName": "OcvSsdFaceDetection", + "componentVersion": "1.0", + "middlewareVersion": "4.1.0", + "sourceLanguage": "c++", + "batchLibrary": "${MPF_HOME}/plugins/OcvFaceDetection/lib/libmpfOcvSsdFaceDetection.so", + "environmentVariables": [ + { + "name": "LD_LIBRARY_PATH", + "value": "${MPF_HOME}/plugins/OcvSsdFaceDetection/lib:${LD_LIBRARY_PATH}" + } + ], + "algorithm": { + "name": "OCVSSD", + "description": "Detects faces in images and videos using the Open Computer Vision (OCV) library.", + "actionType": "DETECTION", + "requiresCollection": { + "states": [] + }, + "providesCollection": { + "states": [ + "DETECTION", + "DETECTION_FACE", + "DETECTION_FACE_OCVSSD" + ], + "properties": [ + { + "name": "DETECTION_CONFIDENCE_THRESHOLD", + "description": "Minimum detection confidence value [0...1] needed to keep detection and start a track.", + "type": "FLOAT", + "defaultValue": "0.5" + }, + { + "name": "DETECTION_NMS_THRESHOLD", + "description": "Detection non-maximum suppression threshold for removing redundant overlapping bounding boxes.", + "type": "FLOAT", + "defaultValue": "0.5" + }, + { + "name": "DETECTION_INFERENCE_SIZE", + "description": "Max image dimension to use for inferencing e.g. 300 (0 will use original but run slower).", + "type": "INT", + "defaultValue": "-1" + }, + { + "name": "ROTATE_AND_DETECT", + "description": "Perform multiple passes at different image orientations to increase detections found.", + "type": "BOOLEAN", + "defaultValue": "true" + }, + { + "name": "ROTATE_ORIENTATIONS", + "description": "CCW rotations of frame to inference (only multiples of 90 are accepted)", + "type": "STRING", + "defaultValue": " [0, 90, 180, 270] " + }, + { + "name": "MIN_DETECTION_SIZE", + "description": "Minimum bounding box dimension in pixels to keep detection and start a track.", + "type": "INT", + "defaultValue": "46" + }, + { + "name": "DETECTION_FRAME_INTERVAL", + "description": "Number of frames to skip running the detector and rely on just tracking", + "type": "INT", + "defaultValue": "0" + }, + { + "name": "DETECTION_BOUNDING_BOX_SCALE_FACTOR", + "description": "Scale the width and height of the detector bounding box", + "type": "FLOAT", + "defaultValue": "1.0" + }, + { + "name": "TRACKING_MAX_IOU_DIST", + "description": "Maximum intersection over union distance (1.0 - iou) =[0..1] between detections below which detection will be considered part of same track", + "type": "FLOAT", + "defaultValue": "0.5" + }, + { + "name": "TRACKING_MAX_FEATURE_DIST", + "description": "Maximum distance in feature space (1.0 - cos similarity) = [0..1] between detections below which detection will be considered part of same track", + "type": "FLOAT", + "defaultValue": "0.25" + }, + { + "name": "TRACKING_MAX_FRAME_GAP", + "description": "Maximum temporal distance (# of frames) between detection in a track.", + "type": "INT", + "defaultValue": "4" + }, + { + "name": "KF_DISABLED", + "description": "Disables the use of motion prediction via Kalman filtering during tracking.", + "type": "BOOLEAN", + "defaultValue": "false" + }, + { + "name": "KF_RN", + "description": "Bounding box detector noise standard deviations (in pixels) [center_x_noise_stdev, center_y_noise_stddev, width_noise_stdev, height_noise_stddev] for Kalman filter R", + "type": "STRING", + "defaultValue": " [ 6.0,6.0,6.0,6.0 ] " + }, + { + "name": "KF_QN", + "description": "Bounding box motion model acceleration inaccuracies/noise (in pixels/sec^2) [center_x_acceleration_noise_stdev, center_y_acceleration_noise_stdev, width_acceleration_noise_stdev, height_acceleration_noise_stdev ] for Kalman filter Q", + "type": "STRING", + "defaultValue": " [10000.0, 10000.0, 1000.0, 1000.0] " + }, + { + "name": "CUDA_DEVIC_ID", + "description": "CUDA device / GPU to use, -1 to only use CPU", + "type": "INT", + "defaultValue": "-1" + }, + { + "name": "FALLBACK_TO_CPU_WHEN_GPU_PROBLEM", + "description": "if true will fallback to CPU only when GPU fails to initialize", + "type": "BOOLEAN", + "defaultValue": "true" + } + + ] + } + }, + "actions": [ + { + "name": "OCV SSD FACE DETECTION ACTION", + "description": "Executes the OpenCV SSD face detection algorithm using the default parameters.", + "algorithm": "OCVSSD", + "properties": [] + }, + { + "name": "OCV SSD FACE DETECTION (WITH AUTO-ORIENTATION) ACTION", + "description": "Executes the OpenCV SSD face detection algorithm and rotates and/or flips media based on EXIF data or video metadata.", + "algorithm": "FACECV", + "properties": [ + { + "name": "AUTO_ROTATE", + "value": "true" + }, + { + "name": "AUTO_FLIP", + "value": "true" + } + ] + } + ], + "tasks": [ + { + "name": "OCV SSD FACE DETECTION TASK", + "description": "Performs OpenCV face detection.", + "actions": [ + "OCV SSD FACE DETECTION ACTION" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH AUTO-ORIENTATION) TASK", + "description": "Executes the OpenCV SSD face detection algorithm and rotates and/or flips media based on EXIF data or video metadata.", + "actions": [ + "OCV SSD FACE DETECTION (WITH AUTO-ORIENTATION) ACTION" + ] + } + ], + "pipelines": [ + { + "name": "OCV SSD FACE DETECTION PIPELINE", + "description": "Performs OpenCV SSD face detection.", + "tasks": [ + "OCV SSD FACE DETECTION TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH MARKUP) PIPELINE", + "description": "Performs OpenCV SSD face detection and marks up the results.", + "tasks": [ + "OCV SSD FACE DETECTION TASK", + "OCV GENERIC MARKUP TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH MOG MOTION PREPROCESSOR) PIPELINE", + "description": "Performs MOG motion preprocessing and OpenCV SSD face detection.", + "tasks": [ + "MOG MOTION DETECTION PREPROCESSOR TASK", + "OCV SSD FACE DETECTION TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH SUBSENSE MOTION PREPROCESSOR) PIPELINE", + "description": "Performs SuBSENSE motion preprocessing and OpenCV face detection.", + "tasks": [ + "SUBSENSE MOTION DETECTION PREPROCESSOR TASK", + "OCV SSD FACE DETECTION TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH MOG MOTION PREPROCESSOR AND MARKUP) PIPELINE", + "description": "Performs MOG motion preprocessing, OpenCV face detection, and markup.", + "tasks": [ + "MOG MOTION DETECTION PREPROCESSOR TASK", + "OCV SSD FACE DETECTION TASK", + "OCV GENERIC MARKUP TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH SUBSENSE MOTION PREPROCESSOR AND MARKUP) PIPELINE", + "description": "Performs SuBSENSE motion preprocessing, OpenCV face detection, and markup.", + "tasks": [ + "SUBSENSE MOTION DETECTION PREPROCESSOR TASK", + "OCV SSD FACE DETECTION TASK", + "OCV GENERIC MARKUP TASK" + ] + }, + { + "name": "OCV SSD FACE DETECTION (WITH AUTO-ORIENTATION) PIPELINE", + "description": "Executes the OpenCV face detection algorithm and rotates and/or flips media based on EXIF data or video metadata.", + "tasks": [ + "OCV SSD FACE DETECTION (WITH AUTO-ORIENTATION) TASK" + ] + } + ] +} diff --git a/cpp/OcvSsdFaceDetection/sample_ocv_ssd_face_detector.cpp b/cpp/OcvSsdFaceDetection/sample_ocv_ssd_face_detector.cpp new file mode 100644 index 00000000..f80619a3 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/sample_ocv_ssd_face_detector.cpp @@ -0,0 +1,126 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include "OcvSsdFaceDetection.h" + +#include +#include +#include +#include + +#include +#include + +using namespace std; +using namespace MPF; +using namespace COMPONENT; + +//----------------------------------------------------------------------------- +// Process and image +//----------------------------------------------------------------------------- +void processImage(MPFDetectionComponent *detection_engine, int argc, char* argv[]) { + + MPFImageJob job("Testing", argv[1], { }, { }); + vector locations = detection_engine->GetDetections(job); + printf("Number of detections: %i\n", locations.size()); + + for(auto loc:locations){ + printf("[%4i,%4i] (%3i,%3i) conf:%.2f \n", loc.x_left_upper,loc.y_left_upper,loc.width,loc.height,loc.confidence ); + } +} + +//----------------------------------------------------------------------------- +// Process a video +//----------------------------------------------------------------------------- +void processVideo(MPFDetectionComponent *detection_engine, int argc, char* argv[]) { + + // get detection interval if argument is present + int detection_interval = 1; + if (argc > 4) { + detection_interval = stoi(argv[4]); + } + printf("Using detection interval: %i\n", detection_interval); + + map algorithm_properties; + algorithm_properties.insert(pair("FRAME_INTERVAL", to_string(detection_interval))); + + MPFVideoJob job("Testing", argv[1], stoi(argv[2]), stoi(argv[3]), algorithm_properties, { }); + vector tracks = detection_engine->GetDetections(job); + + cout << "Number of video tracks = " << tracks.size() << endl; + for (int i = 0; i < tracks.size(); i++) { + cout << "\nVideo track " << i << "\n" + << " start frame = " << tracks[i].start_frame << "\n" + << " stop frame = " << tracks[i].stop_frame << "\n" + << " number of locations = " << tracks[i].frame_locations.size() << "\n" + << " confidence = " << tracks[i].confidence << endl; + + for (auto it : tracks[i].frame_locations) { + cout << " Image location frame = " << it.first << "\n" + << " x left upper = " << it.second.x_left_upper << "\n" + << " y left upper = " << it.second.y_left_upper << "\n" + << " width = " << it.second.width << "\n" + << " height = " << it.second.height << "\n" + << " confidence = " << it.second.confidence << endl; + } + } +} + +//----------------------------------------------------------------------------- +// Main program to run the OCV face detection in standalone mode. +//----------------------------------------------------------------------------- +int main(int argc, char* argv[]) { + + if (argc < 2 || argc > 5) { + printf("Usage (IMAGE): %s \n", argv[0]); + printf("Usage (VIDEO): %s \n", argv[0]); + return 1; + } + + QCoreApplication *this_app = new QCoreApplication(argc, argv); + string app_dir = (this_app->applicationDirPath()).toStdString(); + delete this_app; + + OcvSsdFaceDetection ocv_ssd_face_detection; + MPFDetectionComponent *detection_engine = &ocv_ssd_face_detection; + detection_engine->SetRunDirectory(app_dir + "/plugin"); + + if (!detection_engine->Init()) { + printf("Failed to initialize.\n"); + return 1; + } + + if (argc == 2) { + processImage(detection_engine, argc, argv); + } else { + processVideo(detection_engine, argc, argv); + } + + if (!detection_engine->Close()) { + printf("Failed to close.\n"); + } + +} \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/test/CMakeLists.txt b/cpp/OcvSsdFaceDetection/test/CMakeLists.txt new file mode 100644 index 00000000..e631519b --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/CMakeLists.txt @@ -0,0 +1,68 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +cmake_minimum_required(VERSION 3.6) +project(ocv-ssd-face-detection-tests) + +set(CMAKE_CXX_STANDARD 11) +find_package(GTest) +if (${GTEST_FOUND}) + #enable_testing() + find_package(Qt4 REQUIRED) + find_package(mpfComponentTestUtils REQUIRED) + + include_directories(..) + add_executable(OcvSsdFaceDetectionTest test_ocv_ssd_face_detection.cpp DetectionComparisonA.cpp) + target_link_libraries(OcvSsdFaceDetectionTest mpfOcvSsdFaceDetection + mpfComponentTestUtils + GTest::GTest + GTest::Main + Qt4::QtCore) + + add_test(NAME OcvSsdFaceDetectionTest COMMAND OcvSsdFaceDetectionTest) + + + # update configs if they change + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + test_ocv_ssd_face_config.ini) + + # allow for 'make check' + add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --verbose) + + # Install config file + file(COPY test_ocv_ssd_face_config.ini DESTINATION config) + + + # Install test images and videos + file(GLOB FACE_TEST_IMAGE_FILES data/test_imgs/*) + file(COPY ${FACE_TEST_IMAGE_FILES} DESTINATION test/test_imgs) + + file(GLOB FACE_TEST_VIDEO_FILES data/test_vids/*) + file(COPY ${FACE_TEST_VIDEO_FILES} DESTINATION test/test_vids) + + # Create directory for test output files + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test/test_output/) +endif() \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.cpp b/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.cpp new file mode 100644 index 00000000..71bad660 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.cpp @@ -0,0 +1,232 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#include +#include + +#include "MPFDetectionComponent.h" +#include "Utils.h" + +#include "DetectionComparisonA.h" + +using std::vector; + +namespace MPF { namespace COMPONENT { namespace DetectionComparisonA { + + namespace { + bool CompareDetections(const MPFImageLocation &query_detection, const MPFImageLocation &target_detection, bool log) { + cv::Rect target_detection_rect = Utils::ImageLocationToCvRect(target_detection); + cv::Rect query_detection_rect = Utils::ImageLocationToCvRect(query_detection); + cv::Rect intersection = target_detection_rect & query_detection_rect; + int intersection_area = intersection.area(); + double target_intersection_area = floor(target_detection_rect.area()) * 0.1; + bool same = intersection_area > target_intersection_area; + if (log && !same) { + printf("\tCalc intersection < min target: %d < %f\n", intersection_area, target_intersection_area); + } + return same; + } + + int CompareTracks(const MPFVideoTrack &query_track, const MPFVideoTrack &target_track) { + int query_track_start_frame = query_track.start_frame; + int target_track_start_frame = target_track.start_frame; + + int query_track_stop_frame = query_track.stop_frame; + int target_track_stop_frame = target_track.stop_frame; + + int loop_start_index = 0; + int query_track_index_modifier = target_track_start_frame - query_track_start_frame; + if (query_track_index_modifier < 0) { + loop_start_index = abs(query_track_index_modifier); + } + + int loop_end_count = -1; + if (query_track_stop_frame < target_track_stop_frame) { + loop_end_count = query_track_stop_frame - target_track_start_frame; + } + else { + loop_end_count = static_cast(target_track.frame_locations.size()); + } + + int matched_detections = 0; + for (int k = loop_start_index; k < loop_end_count; k++) { + const MPFImageLocation &target_track_detection = + target_track.frame_locations.at(target_track_start_frame + k); + auto qtdItr = query_track.frame_locations.find(query_track_start_frame + k + query_track_index_modifier); + if(qtdItr != query_track.frame_locations.end()){ + const MPFImageLocation &query_track_detection = query_track.frame_locations.at(query_track_start_frame + k + query_track_index_modifier); + if (CompareDetections(query_track_detection, target_track_detection, true)) { + matched_detections++; + } + } + } + + return matched_detections; + } + + int FindTrack(const MPFVideoTrack &known_track, const vector &actual_tracks, int frame_diff) { + const MPFImageLocation &first_known_detection = known_track.frame_locations.begin()->second; + + for (unsigned int i = 0; i < actual_tracks.size(); i++) { + const MPFVideoTrack &actual_track = actual_tracks.at(i); + + if (abs(known_track.start_frame - actual_track.start_frame) == frame_diff) { + for (const auto &actual_pair : actual_track.frame_locations) { + + // Weak track match: Only one detection between the tracks needs to overlap. + if (CompareDetections(actual_pair.second, first_known_detection, false)) { + return i; + } + } + } + } + + return -1; + } + + int FindTrack(const MPFVideoTrack &known_track, const vector &actual_tracks) { + for (int i = 0; i < 5; i++) { + int match_track_index = FindTrack(known_track, actual_tracks, i); + if (match_track_index != -1) { + return match_track_index; + } + } + return -1; + } + } // anonymous namespace + + float CompareDetectionOutput(const vector &actual_tracks, + const vector &known_tracks) { + float total_score = 0.0; + int total_known_detections = 0; + + for (unsigned int i = 0; i < known_tracks.size(); ++i) { + total_known_detections += static_cast(known_tracks.at(i).frame_locations.size()); + } + + int matched_detections = 0; + int total_actual_detections = 0; + for (unsigned int i = 0; i < actual_tracks.size(); ++i) { + total_actual_detections += static_cast(actual_tracks.at(i).frame_locations.size()); + } + + float track_count_factor = 1.0; + if (total_actual_detections > total_known_detections) { + printf("There are more actual detections than expected detections: "); + track_count_factor = fabsf(static_cast(total_known_detections) / static_cast(total_actual_detections)); + } + else if (total_actual_detections < total_known_detections) { + printf("There are less actual detections than expected detections: "); + track_count_factor = fabsf(static_cast(total_actual_detections) / static_cast(total_known_detections)); + } + else { + printf("Same number of actual and expected detections: "); + } + + printf("%d actual vs. %d known\n", total_actual_detections, total_known_detections); + + vector known_tracks_copy(known_tracks); + vector actual_tracks_copy(actual_tracks); + while (!known_tracks_copy.empty()) { + const MPFVideoTrack &known_track = known_tracks_copy.front(); + + // Match the known track to as many actual tracks as possible. This is done to address the case where the + // component generates multiple tracks instead of one, maybe due to non-determinism or an OpenCV upgrade. + // These are weak track matches. Only one detection between the known track and actual track need to overlap + // to match. Thus, the number of successfully matched detections can be significantly reduced due to weak + // track matches. One effect is that even if the actual output is exactly the same as the known output, + // this comparison approach can result in a score < 1. + int match_track_index; + do { + match_track_index = FindTrack(known_track, actual_tracks_copy); + if (match_track_index != -1) { + const MPFVideoTrack &match_track = actual_tracks_copy.at(match_track_index); + matched_detections += CompareTracks(match_track, known_track); + actual_tracks_copy.erase(actual_tracks_copy.begin() + match_track_index); + // This break will result in a 1-to-1 matching between known tracks and actual tracks. Uncommenting + // this line can enable a score == 1 that is otherwise not possible due to 1-to-many track matches. + // break; + } + } while(match_track_index != -1); + + known_tracks_copy.erase(known_tracks_copy.begin()); + } + + printf("\t\tMatched detections:\t\t%d\n", matched_detections); + printf("\t\tTotal expected detections:\t%d\n", total_known_detections); + printf("\t\tTrack count factor:\t\t%f\n", track_count_factor); + printf("\t\tCombined:\t\t\t(%d/%d)*%f\n", matched_detections, total_known_detections, track_count_factor); + + total_score = (static_cast(matched_detections) / static_cast(total_known_detections)) * track_count_factor; + printf("\t\tTotal score:\t\t\t%f\n", total_score); + + return total_score; + } + + float CompareDetectionOutput(const vector &actual_detections, + const vector &known_detections) { + float total_score = 1.0; + int total_actual_detections = static_cast(actual_detections.size()); + int matched_detections = 0; + int total_known_detections = static_cast(known_detections.size()); + + float track_count_factor = 1.0; + if (total_actual_detections > total_known_detections) { + printf("There are more actual detections than expected detections: "); + track_count_factor = fabsf(static_cast(total_known_detections) / static_cast(total_actual_detections)); + } + else if (total_actual_detections < total_known_detections) { + printf("There are less actual detections than expected detections: "); + track_count_factor = fabsf(static_cast(total_actual_detections) / static_cast(total_known_detections)); + } + else { + printf("Same number of actual and expected detections: "); + } + + printf("%d actual vs. %d known\n", total_actual_detections, total_known_detections); + float overlapTotal = 0.0; + int overlapCount = 0; + for (int i = 0; i < actual_detections.size(); i++) { + float bestOverlap = 0.0; + cv::Rect actual_detection_rect = Utils::ImageLocationToCvRect(actual_detections.at(i)); + + for (int j = 0; j < known_detections.size(); j++) { + cv::Rect known_detection_rect = Utils::ImageLocationToCvRect(known_detections.at(j)); + cv::Rect intersection = known_detection_rect & actual_detection_rect; + float overlap = (intersection.area() > known_detection_rect.area()) ? + static_cast(known_detection_rect.area()) / static_cast(intersection.area()) : + static_cast(intersection.area()) / static_cast(known_detection_rect.area()); + bestOverlap = (overlap > bestOverlap) ? overlap : bestOverlap; + } + overlapTotal += bestOverlap; + overlapCount++; + } + + total_score = track_count_factor * (overlapTotal / static_cast(overlapCount)); + printf("Total score: %f\n", total_score); + return total_score; + } +}}} diff --git a/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.h b/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.h new file mode 100644 index 00000000..a2a216b6 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/DetectionComparisonA.h @@ -0,0 +1,46 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + + +#ifndef OPENMPF_CPP_COMPONENT_SDK_DETECTIONCOMPARISON_H +#define OPENMPF_CPP_COMPONENT_SDK_DETECTIONCOMPARISON_H + +#include + +#include "MPFDetectionComponent.h" + +namespace MPF { namespace COMPONENT { namespace DetectionComparisonA { + + float CompareDetectionOutput(const std::vector &actual_tracks, + const std::vector &known_tracks); + + + float CompareDetectionOutput(const std::vector &actual_detections, + const std::vector &known_detections); + +}}} + +#endif //OPENMPF_CPP_COMPONENT_SDK_DETECTIONCOMPARISON_H diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/270.png b/cpp/OcvSsdFaceDetection/test/data/test_imgs/270.png new file mode 100644 index 00000000..97e63e53 Binary files /dev/null and b/cpp/OcvSsdFaceDetection/test/data/test_imgs/270.png differ diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/NOTICE b/cpp/OcvSsdFaceDetection/test/data/test_imgs/NOTICE new file mode 100644 index 00000000..508a5161 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/data/test_imgs/NOTICE @@ -0,0 +1,17 @@ +# S001-01-t10_01.jpg, S008-01-t10_01.jpg +# meds_faces_image.png is a combination of scaled and rotated images listed above + +The dataset and other related information can be acquired at http://www.nist.gov/itl/iad/ig/sd32.cfm + +From the Dataset Acknowledgement from the NIST Special Database 32 Multiple Encounter Dataset II (MEDS-II) Data Description Document +"This dataset is being released (as prepared by MITRE Corporation) to support the NIST MultipleBiometric +Evaluation 2010 (MBE). In addition, this dataset is available to any user interested in +biometric research." + + +#270.png is a screen capture of frame 270 of the ../test_vids/new_face_video.avi + +Copyright 2012 Aaron "tango" Tang +Creative Commons Attribution 2.0 Generic License: https://creativecommons.org/licenses/by/2.0/ +https://www.flickr.com/photos/hahatango/7484448042/in/photolist-cpnKDC-9PQvQE-cehi5m-eX6cjM-7XHcXs-8HwEEG-97vd4h-cpo5cS-6fDWiQ-9phRV3-bAPcDX-cpo5ky-7DrHhu-6UNght-8GPWGS-cehi4J-BUk6xN-7DrqHQ-7DrzRy-7eLZPm-6UN5pn-8Hwx5d-88XgZj-xNoj9R-HHwC5c-oiWXVh-oAbTv3-86XZiL-7urFMR-6UShyC-7DnKEP-7DnQU8-7Dryxb-6TU3nb-oNw7jC-6teNT2-6bZoZj-7Drxb5-cpouDu-6UN6Wv-cpokHj-9jEhVN-7DrQ1q-cpouub-7DrPNN-7DoeZ6-8km6iE-7Do4St-8kZ2Ni-83HeW4 +Converted to AVI. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/S001-01-t10_01.jpg b/cpp/OcvSsdFaceDetection/test/data/test_imgs/S001-01-t10_01.jpg new file mode 100644 index 00000000..d42e7476 Binary files /dev/null and b/cpp/OcvSsdFaceDetection/test/data/test_imgs/S001-01-t10_01.jpg differ diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/S008-01-t10_01.jpg b/cpp/OcvSsdFaceDetection/test/data/test_imgs/S008-01-t10_01.jpg new file mode 100644 index 00000000..864426ba Binary files /dev/null and b/cpp/OcvSsdFaceDetection/test/data/test_imgs/S008-01-t10_01.jpg differ diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/meds_faces_image.png b/cpp/OcvSsdFaceDetection/test/data/test_imgs/meds_faces_image.png new file mode 100644 index 00000000..572212be Binary files /dev/null and b/cpp/OcvSsdFaceDetection/test/data/test_imgs/meds_faces_image.png differ diff --git a/cpp/OcvSsdFaceDetection/test/data/test_imgs/ocv_face_known_detections.txt b/cpp/OcvSsdFaceDetection/test/data/test_imgs/ocv_face_known_detections.txt new file mode 100644 index 00000000..9752eaff --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/data/test_imgs/ocv_face_known_detections.txt @@ -0,0 +1,2 @@ +743,886,480,600 +1473,535,480,600 diff --git a/cpp/OcvSsdFaceDetection/test/data/test_vids/NOTICE b/cpp/OcvSsdFaceDetection/test/data/test_vids/NOTICE new file mode 100644 index 00000000..2ee690fc --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/data/test_vids/NOTICE @@ -0,0 +1,5 @@ +# new_face_video.avi +Copyright 2012 Aaron "tango" Tang +Creative Commons Attribution 2.0 Generic License: https://creativecommons.org/licenses/by/2.0/ +https://www.flickr.com/photos/hahatango/7484448042/in/photolist-cpnKDC-9PQvQE-cehi5m-eX6cjM-7XHcXs-8HwEEG-97vd4h-cpo5cS-6fDWiQ-9phRV3-bAPcDX-cpo5ky-7DrHhu-6UNght-8GPWGS-cehi4J-BUk6xN-7DrqHQ-7DrzRy-7eLZPm-6UN5pn-8Hwx5d-88XgZj-xNoj9R-HHwC5c-oiWXVh-oAbTv3-86XZiL-7urFMR-6UShyC-7DnKEP-7DnQU8-7Dryxb-6TU3nb-oNw7jC-6teNT2-6bZoZj-7Drxb5-cpouDu-6UN6Wv-cpokHj-9jEhVN-7DrQ1q-cpouub-7DrPNN-7DoeZ6-8km6iE-7Do4St-8kZ2Ni-83HeW4 +Converted to AVI. \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/test/data/test_vids/new_face_video.avi b/cpp/OcvSsdFaceDetection/test/data/test_vids/new_face_video.avi new file mode 100644 index 00000000..131ea088 Binary files /dev/null and b/cpp/OcvSsdFaceDetection/test/data/test_vids/new_face_video.avi differ diff --git a/cpp/OcvSsdFaceDetection/test/data/test_vids/ocv_face_known_tracks.txt b/cpp/OcvSsdFaceDetection/test/data/test_vids/ocv_face_known_tracks.txt new file mode 100644 index 00000000..ab483d74 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/data/test_vids/ocv_face_known_tracks.txt @@ -0,0 +1,386 @@ +ocv_face_known_tracks.txt +#0 +0 +46 +154,73,191,208 +148,69,183,209 +133,61,187,211 +132,69,181,207 +132,79,184,209 +137,83,182,210 +140,89,177,206 +142,87,177,210 +149,87,179,208 +150,87,183,204 +148,86,186,206 +151,91,186,206 +152,89,185,206 +148,90,190,208 +161,101,182,203 +165,100,180,203 +159,90,184,209 +163,92,181,209 +166,95,178,209 +164,97,180,210 +168,99,178,214 +170,91,178,220 +170,90,178,221 +172,98,181,216 +179,99,180,219 +182,107,178,214 +190,105,181,216 +191,107,181,214 +199,108,176,212 +197,109,176,212 +204,108,172,213 +202,102,176,218 +199,99,176,217 +191,88,177,224 +191,88,178,220 +197,90,180,214 +206,83,178,217 +221,85,177,223 +240,104,179,213 +258,87,179,224 +275,82,182,221 +304,82,186,217 +338,77,185,216 +364,74,204,214 +402,63,223,224 +444,70,194,220 +485,60,154,239 +#1 +42 +71 +0,158,125,225 +0,154,162,218 +17,160,181,208 +51,156,177,209 +88,166,175,208 +113,166,174,209 +120,156,178,210 +157,154,176,212 +205,160,178,214 +226,158,180,217 +238,165,184,217 +235,166,183,220 +240,163,186,224 +243,170,184,223 +231,171,184,220 +236,176,185,222 +243,176,185,225 +235,175,186,224 +234,176,187,220 +236,172,189,223 +232,165,188,223 +231,164,189,221 +236,162,189,225 +240,163,191,229 +258,172,187,230 +278,178,190,227 +307,175,195,238 +378,187,206,246 +457,207,182,249 +514,198,125,260 +#2 +72 +104 +0,126,149,220 +37,150,176,207 +93,140,161,211 +143,130,164,215 +180,120,168,216 +178,98,171,229 +168,74,175,240 +177,57,171,254 +187,43,179,270 +188,41,183,271 +208,45,179,269 +224,51,182,265 +228,50,183,263 +217,56,180,259 +217,55,176,263 +219,59,177,255 +208,58,180,255 +231,53,176,259 +242,60,178,252 +236,67,177,245 +237,65,178,248 +248,70,177,241 +250,72,176,238 +245,73,180,236 +255,72,174,237 +249,70,174,234 +231,73,172,234 +212,77,175,233 +207,73,180,237 +186,70,176,239 +139,78,174,243 +66,77,185,247 +0,62,165,264 +#3 +108 +137 +436,71,203,280 +382,65,237,278 +347,75,229,272 +295,92,216,259 +269,99,204,254 +246,106,206,252 +229,101,206,253 +218,98,210,253 +206,94,209,250 +183,85,218,251 +164,67,207,251 +169,78,202,255 +192,106,194,240 +182,111,194,236 +181,117,190,234 +180,125,188,232 +172,123,188,234 +175,127,186,235 +175,134,186,230 +171,134,186,229 +163,136,188,224 +156,137,188,221 +154,136,184,223 +149,133,186,223 +152,131,183,225 +129,121,188,225 +78,106,190,224 +33,103,199,224 +3,97,198,226 +0,66,131,237 +#4 +134 +149 +492,90,147,182 +462,89,168,181 +427,77,167,193 +351,70,161,183 +318,65,155,186 +289,88,155,175 +264,85,152,172 +253,88,155,174 +253,105,160,169 +254,102,162,178 +252,96,162,182 +261,98,163,182 +267,82,166,197 +263,76,168,195 +262,73,168,200 +263,70,174,200 +#5 +169 +215 +201,2,184,242 +203,11,186,217 +227,0,205,174 +211,7,174,212 +205,4,181,208 +204,4,182,205 +198,3,187,202 +195,4,186,200 +199,5,184,202 +208,7,187,202 +214,1,194,201 +216,0,202,195 +221,0,198,192 +215,0,205,187 +223,0,204,181 +232,0,202,172 +242,0,208,170 +247,0,214,170 +261,0,211,176 +273,0,215,180 +278,0,215,173 +278,0,215,171 +282,0,217,173 +287,0,216,176 +287,0,213,174 +284,0,213,172 +283,0,212,172 +298,0,205,171 +335,0,220,170 +282,0,210,179 +279,0,211,186 +278,0,203,192 +275,0,200,195 +268,0,202,192 +266,0,202,191 +265,0,202,189 +263,0,200,186 +262,0,196,187 +256,0,193,193 +252,0,193,195 +250,0,190,196 +252,0,194,196 +249,0,194,192 +242,0,194,185 +261,0,205,175 +407,0,223,191 +491,0,148,199 +#6 +215 +228 +0,0,163,200 +0,0,224,205 +68,0,239,229 +143,0,231,258 +181,1,229,271 +228,39,231,264 +271,68,220,259 +249,71,225,267 +248,70,224,269 +258,70,224,270 +259,72,223,272 +299,100,223,262 +371,114,242,274 +441,124,198,295 +#7 +232 +255 +175,92,181,236 +216,62,197,247 +255,43,196,259 +275,32,205,277 +265,30,209,288 +259,32,210,288 +274,33,206,286 +273,32,205,291 +269,32,206,299 +285,34,201,289 +284,39,205,286 +278,42,200,285 +279,49,199,281 +290,56,203,275 +299,63,198,266 +301,67,192,265 +299,73,190,258 +288,75,183,258 +259,84,185,248 +239,91,180,240 +211,95,176,239 +150,94,176,237 +77,90,189,238 +1,82,199,248 +#8 +260 +284 +434,84,186,260 +301,75,185,253 +211,88,185,223 +170,104,181,222 +178,129,177,210 +166,144,172,205 +119,155,170,203 +100,157,170,200 +98,174,169,198 +89,179,167,195 +90,179,167,195 +112,183,165,200 +123,184,163,201 +111,182,169,198 +109,178,168,200 +114,171,170,205 +110,160,170,205 +103,156,173,203 +102,154,175,207 +91,152,174,209 +55,152,181,203 +21,142,193,209 +6,135,190,218 +0,126,186,220 +2,113,141,232 +#9 +267 +328 +482,118,157,190 +479,133,158,179 +464,131,160,181 +463,136,160,183 +484,152,155,182 +500,163,139,180 +489,162,150,175 +492,152,147,187 +497,145,142,189 +494,145,145,179 +492,139,147,182 +492,144,147,180 +481,148,156,176 +452,140,160,180 +425,138,160,177 +416,138,154,180 +400,141,160,177 +370,146,153,173 +340,134,149,181 +290,133,156,182 +256,118,158,184 +253,112,160,189 +248,111,158,193 +250,111,159,193 +258,128,159,182 +257,131,160,181 +250,128,160,189 +250,129,162,186 +251,125,160,184 +249,114,160,192 +248,107,159,192 +244,92,160,193 +246,89,161,193 +254,90,168,194 +265,82,169,195 +284,78,175,200 +295,69,178,206 +302,67,179,208 +319,67,177,208 +330,58,179,214 +329,55,181,212 +338,57,187,209 +345,49,189,214 +344,41,186,217 +343,34,185,219 +327,21,184,224 +328,21,186,223 +317,14,195,229 +302,7,204,229 +287,22,194,227 +260,29,201,229 +219,31,207,235 +187,42,211,237 +141,58,217,239 +76,63,232,242 +27,36,247,258 +0,18,234,269 +0,8,181,276 +0,0,173,281 +0,0,176,283 +0,8,165,288 +0,9,169,292 +#10 +445 +468 +0,1,176,335 +0,0,223,350 +0,1,270,349 +0,6,306,347 +16,6,320,345 +72,10,306,340 +115,11,306,345 +150,9,294,341 +180,14,286,329 +201,21,282,318 +221,28,268,313 +219,34,273,316 +225,48,277,310 +235,50,280,312 +237,52,272,303 +245,68,260,289 +270,83,254,281 +296,91,244,285 +319,111,247,267 +328,99,243,260 +348,88,234,259 +372,75,237,277 +398,83,237,267 +428,57,208,289 diff --git a/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_config.ini b/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_config.ini new file mode 100644 index 00000000..ff248b74 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_config.ini @@ -0,0 +1,42 @@ +############################################################################# +# NOTICE # +# # +# This software (or technical data) was produced for the U.S. Government # +# under contract, and is subject to the Rights in Data-General Clause # +# 52.227-14, Alt. IV (DEC 2007). # +# # +# Copyright 2020 The MITRE Corporation. All Rights Reserved. # +############################################################################# + +############################################################################# +# Copyright 2020 The MITRE Corporation # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################# + +OCV_FACE_1_FILE: ./test/test_imgs/S001-01-t10_01.jpg +OCV_FACE_VIDEO_FILE: ./test/test_vids/new_face_video.avi +OCV_FACE_START_FRAME: 0 +OCV_FACE_STOP_FRAME: 511 +OCV_FACE_KNOWN_TRACKS: ./test/test_vids/ocv_face_known_tracks.txt +OCV_FACE_VIDEO_OUTPUT_FILE: ocv_face_found_tracks.avi +OCV_FACE_FOUND_TRACKS: ocv_face_found_tracks.txt +OCV_FACE_FRAME_RATE: 1 +OCV_FACE_COMPARISON_SCORE_VIDEO: 0.6 +OCV_FACE_IMAGE_FILE: ./test/test_imgs/meds_faces_image.png +OCV_FACE_KNOWN_DETECTIONS: ./test/test_imgs/ocv_face_known_detections.txt +OCV_FACE_IMAGE_OUTPUT_FILE: ocv_face_found_detections.png +OCV_FACE_FOUND_DETECTIONS: ocv_face_found_detections.txt +OCV_FACE_COMPARISON_SCORE_IMAGE: 0.2 +OCV_FACE_THUMBNAIL_TEST_FILE_DIR: ./test/test_imgs/ +OCV_FACE_THUMBNAIL_TEST_FILE_00: 270.png \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_detection.cpp b/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_detection.cpp new file mode 100644 index 00000000..992dabe3 --- /dev/null +++ b/cpp/OcvSsdFaceDetection/test/test_ocv_ssd_face_detection.cpp @@ -0,0 +1,482 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +//#include +#include +#include +#include + +#define private public +#include "OcvSsdFaceDetection.h" +#include "DetectionComparisonA.h" +using namespace std; +using namespace MPF::COMPONENT; + +/** *************************************************************************** +* macros for "pretty" gtest messages +**************************************************************************** */ +#define ANSI_TXT_GRN "\033[0;32m" +#define ANSI_TXT_MGT "\033[0;35m" //Magenta +#define ANSI_TXT_DFT "\033[0;0m" //Console default +#define GTEST_BOX "[ ] " +#define GOUT(MSG){ \ + std::cout << GTEST_BOX << MSG << std::endl; \ +} +#define GOUT_MGT(MSG){ \ + std::cout << ANSI_TXT_MGT << GTEST_BOX << MSG << ANSI_TXT_DFT << std::endl; \ +} +#define GOUT_GRN(MSG){ \ + std::cout << ANSI_TXT_GRN << GTEST_BOX << MSG << ANSI_TXT_DFT << std::endl; \ +} + +/** *************************************************************************** +* global variable to hold the file name parameters +**************************************************************************** */ + + QHash GetTestParameters(){ + + QString current_path = QDir::currentPath(); + QHash parameters; + string config_path(current_path.toStdString() + "/config/test_ocv_ssd_face_config.ini"); + int rc = LoadConfig(config_path, parameters); + if(rc == 0){ + parameters["CONFIG_FILE"] = QString::fromStdString(config_path); + }else{ + parameters.clear(); + GOUT("config file failed to load with error:" << rc << " for '" << config_path << "'"); + } + return parameters; +} +/** *************************************************************************** +* get current working directory with minimal error checking +**************************************************************************** */ + +static string GetCurrentWorkingDirectory() { + char cwd[1024]; + if (getcwd(cwd, sizeof(cwd)) != NULL) { + return string(cwd); + }else{ + return ""; + } +} + +/** *************************************************************************** +* Test component opencv version +**************************************************************************** */ +TEST(OcvSsdFaceDetection, OpenCVVersion) { + #if(CV_MAJOR_VERSION > 3) + GOUT("OpenCV Version: 4.x"); + #elif(CV_MAJOR_VERSION > 2) + GOUT("OpenCV Version: 3.x"); + #endif +} + +/** *************************************************************************** +* Test initializing component +**************************************************************************** */ +TEST(OcvSsdFaceDetection, Init) { + + string current_working_dir = GetCurrentWorkingDirectory(); + GOUT("current working dir: " << current_working_dir); + ASSERT_TRUE(current_working_dir != ""); + + + QHash parameters = GetTestParameters(); + GOUT("config file:" << parameters["CONFIG_FILE"].toStdString()); + ASSERT_TRUE(parameters.count() > 1); + + OcvSsdFaceDetection *ocv_ssd_face_detection = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ocv_ssd_face_detection); + + string dir_input(current_working_dir + "/../plugin"); + ocv_ssd_face_detection->SetRunDirectory(dir_input); + string rundir = ocv_ssd_face_detection->GetRunDirectory(); + EXPECT_EQ(dir_input, rundir); + + ASSERT_TRUE(ocv_ssd_face_detection->Init()); + + MPFComponentType comp_type = ocv_ssd_face_detection->GetComponentType(); + ASSERT_TRUE(MPF_DETECTION_COMPONENT == comp_type); + + EXPECT_TRUE(ocv_ssd_face_detection->Close()); + delete ocv_ssd_face_detection; +} + +/** *************************************************************************** +* Test frame preprocessing +**************************************************************************** */ +TEST(OcvSsdFaceDetection, DISABLED_PreProcess) { + string current_working_dir = GetCurrentWorkingDirectory(); + QHash parameters = GetTestParameters(); + + string test_output_dir = current_working_dir + "/test/test_output/"; + + GOUT("Reading parameters for preprocessor video test."); + + int start = parameters["OCV_FACE_START_FRAME"].toInt(); + int stop = parameters["OCV_FACE_STOP_FRAME" ].toInt(); + int rate = parameters["OCV_FACE_FRAME_RATE" ].toInt(); + string inVideoFile = parameters["OCV_PREPROCESS_VIDEO_FILE" ].toStdString(); + string outVideoFile = parameters["OCV_PREPROCESS_VIDEO_OUTPUT_FILE"].toStdString(); + + GOUT("Start:\t" << start); + GOUT("Stop:\t" << stop); + GOUT("Rate:\t" << rate); + GOUT("inVideo:\t" << inVideoFile); + GOUT("outVideo:\t" << outVideoFile); + + // Create an OCV face detection object. + GOUT("\tRunning Preprocessor only"); + OcvSsdFaceDetection *ocv_ssd_face_detection = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ocv_ssd_face_detection); + ocv_ssd_face_detection->SetRunDirectory(current_working_dir + "/../plugin"); + ASSERT_TRUE(ocv_ssd_face_detection->Init()); + + MPFVideoJob videoJob("Testing", inVideoFile, start, stop, { }, { }); + JobConfig cfg(videoJob); + cv::VideoWriter video(outVideoFile,cv::VideoWriter::fourcc('X','2','6','4'),cfg._videocapPtr->GetFrameRate(),cfg._videocapPtr->GetFrameSize()); + + while(cfg.nextFrame()) { + ocv_ssd_face_detection->_normalizeFrame(cfg); + video.write(cfg.bgrFrame); + } + video.release(); + GOUT("\tClosing down detection."); + EXPECT_TRUE(ocv_ssd_face_detection->Close()); + delete ocv_ssd_face_detection; + +} + + +/** *************************************************************************** +* This test checks the confidence of faces detected by the OpenCV low-level +* detection facility, which is used by the OpenCV face detection component. +**************************************************************************** */ +TEST(OcvSsdFaceDetection, VerifyQuality) { + + string current_working_dir = GetCurrentWorkingDirectory(); + string plugins_dir = current_working_dir + "/../plugin/OcvSsdFaceDetection"; + + QHash parameters = GetTestParameters(); + ASSERT_TRUE(parameters.count() > 1); + + + // Create an OCV face detection object. + OcvSsdFaceDetection *ocv_ssd_face_detection = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ocv_ssd_face_detection); + ocv_ssd_face_detection->SetRunDirectory(current_working_dir + "/../plugin"); + ASSERT_TRUE(ocv_ssd_face_detection->Init()); + + // Load test image + string test_image_path = parameters["OCV_FACE_1_FILE"].toStdString(); + if(test_image_path.find_first_of('.') == 0) { + test_image_path = current_working_dir + "/" + test_image_path; + } + // Detect detections and check conf levels + MPFImageJob job1("Testing1", test_image_path, { }, { }); + vector detections = ocv_ssd_face_detection->GetDetections(job1); + ASSERT_TRUE(detections.size() > 0); + GOUT("Detection: " << detections[0]); + ASSERT_TRUE(detections[0].confidence > .9); + detections.clear(); + + // Detect detections and check conf level + MPFImageJob job2("Testing2", test_image_path, {{"MIN_DETECTION_SIZE","500"}}, { }); + detections = ocv_ssd_face_detection->GetDetections(job2); + ASSERT_TRUE(detections.size() == 0); + detections.clear(); + + // Detect detections and check conf levels + MPFImageJob job3("Testing2", test_image_path, {{"MIN_DETECTION_SIZE","48"},{"DETECTION_CONFIDENCE_THRESHOLD","1.1"}}, { }); + detections = ocv_ssd_face_detection->GetDetections(job3); + ASSERT_TRUE(detections.size() == 0); + detections.clear(); + + delete ocv_ssd_face_detection; + + +} + +/** *************************************************************************** +* Test face detection in images +**************************************************************************** */ +TEST(OcvSsdFaceDetection, TestOnKnownImage) { + + string current_working_dir = GetCurrentWorkingDirectory(); + QHash parameters = GetTestParameters(); + + string test_output_dir = current_working_dir + "/test/test_output/"; + string known_image_file = parameters["OCV_FACE_IMAGE_FILE" ].toStdString(); + string known_detections_file = parameters["OCV_FACE_KNOWN_DETECTIONS" ].toStdString(); + string output_image_file = parameters["OCV_FACE_IMAGE_OUTPUT_FILE"].toStdString(); + string output_detections_file = parameters["OCV_FACE_FOUND_DETECTIONS" ].toStdString(); + float comparison_score_threshold = parameters["OCV_FACE_COMPARISON_SCORE_IMAGE"].toFloat(); + + // Create an OCV face detection object. + OcvSsdFaceDetection *ocv_ssd_face_detection = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ocv_ssd_face_detection); + + ocv_ssd_face_detection->SetRunDirectory(current_working_dir + "/../plugin"); + ASSERT_TRUE(ocv_ssd_face_detection->Init()); + + GOUT("Input Known Detections:\t" << known_detections_file); + GOUT("Output Found Detections:\t" << output_detections_file); + GOUT("Input Image:\t" << known_image_file); + GOUT("Output Image:\t" << output_image_file); + GOUT("comparison threshold:\t" << comparison_score_threshold); + + // Load the known detections into memory. + vector known_detections; + ASSERT_TRUE(ReadDetectionsFromFile::ReadImageLocations(known_detections_file, known_detections)); + + + MPFImageJob image_job("Testing", known_image_file, { }, { }); + vector found_detections = ocv_ssd_face_detection->GetDetections(image_job); + EXPECT_FALSE(found_detections.empty()); + + // create output video to view performance + ImageGeneration image_generation; + image_generation.WriteDetectionOutputImage(known_image_file, + found_detections, + test_output_dir + "/" + output_image_file); + + WriteDetectionsToFile::WriteVideoTracks(test_output_dir + "/" + output_detections_file, + found_detections); + + float comparison_score = DetectionComparisonA::CompareDetectionOutput(found_detections, known_detections); + GOUT("Detection comparison score: " << comparison_score); + ASSERT_TRUE(comparison_score > comparison_score_threshold); + + EXPECT_TRUE(ocv_ssd_face_detection->Close()); + delete ocv_ssd_face_detection; +} + + +/** *************************************************************************** +* Test face-recognition with thumbnail images +**************************************************************************** */ + +TEST(OcvSsdFaceDetection, Thumbnails) { + string current_working_dir = GetCurrentWorkingDirectory(); + QHash parameters = GetTestParameters(); + + string test_output_dir = current_working_dir + "/test/test_output/"; + + // Get test image filenames into vector + string test_file_dir = parameters["OCV_FACE_THUMBNAIL_TEST_FILE_DIR"].toStdString(); + GOUT("Input Image Dir: " << test_file_dir); + vector img_file_names; + size_t idx = 0; + while(1){ + stringstream ss; + ss << "OCV_FACE_THUMBNAIL_TEST_FILE_" << setfill('0') << setw(2) << idx; + QString key = QString::fromStdString(ss.str()); + auto img_file_it = parameters.find(key); + if(img_file_it == parameters.end()) break; + img_file_names.push_back(img_file_it.value().toStdString()); + idx++; + } + GOUT("Found " << img_file_names.size() << " test images"); + + // Create an OCV face detection object. + OcvSsdFaceDetection *ssd = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ssd); + + ssd->SetRunDirectory(current_working_dir + "/../plugin"); + ASSERT_TRUE(ssd->Init()); + + TrackPtrList tracks; + for(auto img_file_name:img_file_names){ + string img_file = test_file_dir + img_file_name; + MPFImageLocationVec found_detections; + MPFImageJob job("Testing", img_file, { }, { }); + + JobConfig cfg(job); + //cfg.bgrFrame = cv::imread(img_file); + EXPECT_TRUE(NULL != cfg.bgrFrame.data) << "Could not load:" << img_file; + + // find detections + DetectionLocationPtrVec detections = DetectionLocation::createDetections(cfg); + EXPECT_FALSE(detections.empty()); + + // get landmarks + for(auto &det:detections){ + EXPECT_FALSE(det->getLandmarks().empty()); + } + + // draw landmarks + cv::Mat frame = cfg.bgrFrame.clone(); + for(auto &det:detections){ + det->drawLandmarks(frame,cv::Scalar(255,255,255)); + } + cv::imwrite(test_output_dir + "lm_" + img_file_name,frame); + + // calculate thumbnails and feature vectors + for(auto &det:detections){ + EXPECT_FALSE(det->getFeature().empty()); + } + + // calculate some feature distances + GOUT("feature-magnitude1:" << cv::norm(detections.front()->getFeature(),cv::NORM_L2)); + GOUT("feature-magnitude2:" << cv::norm(detections.back()->getFeature(),cv::NORM_L2)); + //GOUT("self feature dist: " << detections.back()->featureDist(tr)); + //ASSERT_TRUE(detections.front()->featureDist(*detections.front()) < 1e-6); + //GOUT("cross feature dist: " << detections.front()->featureDist(*detections.back())); + + if(tracks.size() == 0){ + for(auto &det:detections){ + tracks.push_back(unique_ptr(new Track(move(det),cfg))); + } + }else{ + //ssd->_assignDetections2Tracks(tracks,detections, ass); + TrackPtrList assignedTracks; + vector av = ssd->_calcAssignmentVector<&DetectionLocation::iouDist>(tracks,detections,cfg.maxIOUDist); + ssd->_assignDetections2Tracks(tracks,detections, av, assignedTracks); + } + float self_fd = tracks.back()->back()->featureDist(*tracks.back()); + GOUT("self feature dist: " << self_fd); + ASSERT_TRUE(self_fd < 1e-6); + float cross_fd = tracks.front()->front()->featureDist(*tracks.back()); + GOUT("cross feature dist: " << cross_fd); + ASSERT_TRUE(cross_fd > 1e-6); + } + + // write out thumbnail image tracks + int t=0; + for(auto &track:tracks){ + for(size_t i=0; i < track->size(); i++){ + EXPECT_FALSE((*track)[i]->getThumbnail().empty()); + stringstream ss; + ss << test_output_dir << "t" << t << "_i"<< i << ".png"; + string out_file = ss.str(); + GOUT("Writing thumbnail: " << out_file); + cv::imwrite(out_file,(*track)[i]->getThumbnail()); + } + t++; + } + + EXPECT_TRUE(ssd->Close()); + delete ssd; +} + +/** *************************************************************************** +* Test face detection and tracking in videos +**************************************************************************** */ +TEST(OcvSsdFaceDetection, TestOnKnownVideo) { + + string current_working_dir = GetCurrentWorkingDirectory(); + QHash parameters = GetTestParameters(); + + string test_output_dir = current_working_dir + "/test/test_output/"; + + GOUT("Reading parameters for video test."); + + int start = parameters["OCV_FACE_START_FRAME"].toInt(); + int stop = parameters["OCV_FACE_STOP_FRAME" ].toInt(); + int rate = parameters["OCV_FACE_FRAME_RATE" ].toInt(); + string inTrackFile = parameters["OCV_FACE_KNOWN_TRACKS" ].toStdString(); + string inVideoFile = parameters["OCV_FACE_VIDEO_FILE" ].toStdString(); + string outTrackFile = parameters["OCV_FACE_FOUND_TRACKS" ].toStdString(); + string outVideoFile = parameters["OCV_FACE_VIDEO_OUTPUT_FILE"].toStdString(); + float comparison_score_threshold = parameters["OCV_FACE_COMPARISON_SCORE_VIDEO"].toFloat(); + + GOUT("Start:\t" << start); + GOUT("Stop:\t" << stop); + GOUT("Rate:\t" << rate); + GOUT("inTrack:\t" << inTrackFile); + GOUT("outTrack:\t" << outTrackFile); + GOUT("inVideo:\t" << inVideoFile); + GOUT("outVideo:\t" << outVideoFile); + GOUT("comparison threshold:\t" << comparison_score_threshold); + + // Create an OCV face detection object. + GOUT("\tCreating OCV Face Detection"); + OcvSsdFaceDetection *ocv_ssd_face_detection = new OcvSsdFaceDetection(); + ASSERT_TRUE(NULL != ocv_ssd_face_detection); + ocv_ssd_face_detection->SetRunDirectory(current_working_dir + "/../plugin"); + ASSERT_TRUE(ocv_ssd_face_detection->Init()); + + // Load the known tracks into memory. + GOUT("\tLoading the known tracks into memory: " << inTrackFile); + vector known_tracks; + ASSERT_TRUE(ReadDetectionsFromFile::ReadVideoTracks(inTrackFile, known_tracks)); + + // create output known video to view ground truth + GOUT("\tWriting ground truth video and test tracks to files."); + VideoGeneration video_generation_gt; + video_generation_gt.WriteTrackOutputVideo(inVideoFile, known_tracks, (test_output_dir + "/ground_truth.avi")); + WriteDetectionsToFile::WriteVideoTracks((test_output_dir + "/ground_truth.txt"), known_tracks); + + // Evaluate the known video file to generate the test tracks. + GOUT("\tRunning the tracker on the video: " << inVideoFile); + MPFVideoJob videoJob("Testing", inVideoFile, start, stop, { }, { }); + auto start_time = chrono::high_resolution_clock::now(); + vector found_tracks = ocv_ssd_face_detection->GetDetections(videoJob); + auto end_time = chrono::high_resolution_clock::now(); + EXPECT_FALSE(found_tracks.empty()); + double time_taken = chrono::duration_cast(end_time - start_time).count(); + time_taken = time_taken * 1e-9; + GOUT("\tVideoJob processing time: " << fixed << setprecision(5) << time_taken << "[sec]"); + + // create output video to view performance + GOUT("\tWriting detected video and test tracks to files."); + VideoGeneration video_generation; + video_generation.WriteTrackOutputVideo(inVideoFile, found_tracks, (test_output_dir + "/" + outVideoFile)); + WriteDetectionsToFile::WriteVideoTracks((test_output_dir + "/" + outTrackFile), found_tracks); + + // Compare the known and test track output. + GOUT("\tComparing the known and test tracks."); + float comparison_score = DetectionComparisonA::CompareDetectionOutput(found_tracks, known_tracks); + GOUT("Tracker comparison score: " << comparison_score); + ASSERT_TRUE(comparison_score > comparison_score_threshold); + + + + // don't forget + GOUT("\tClosing down detection."); + EXPECT_TRUE(ocv_ssd_face_detection->Close()); + delete ocv_ssd_face_detection; +} + +/** ************************************************************************** +* +**************************************************************************** */ +int main(int argc, char **argv){ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/cpp/OcvSsdFaceDetection/types.h b/cpp/OcvSsdFaceDetection/types.h new file mode 100644 index 00000000..d19092da --- /dev/null +++ b/cpp/OcvSsdFaceDetection/types.h @@ -0,0 +1,62 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ + +#ifndef OCVSSDFACEDETECTION_TYPES_H +#define OCVSSDFACEDETECTION_TYPES_H + +#include +#include +#include +#include +#include +#include + +#include "OrientationType.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + class DetectionLocation; + class Track; + + using MPFVideoTrackVec = vector; ///< vector of MPFVideoTracks + using MPFImageLocationVec = vector; ///< vector of MPFImageLocations + using cvMatVec = vector; ///< vector of OpenCV matrices/images + using cvRectVec = vector; ///< vector of OpenCV rectangles + using cvPointVec = vector; ///< vector of OpenCV points + using cvPoint2fVec = vector; ///< vector of OpenCV 2D float points + using cvPoint2fVecVec = vector; ///< vector of vectors of OpenCV 2D float points + using DetectionLocationPtr = unique_ptr; ///< DetectionLocation pointers + using DetectionLocationPtrVec = vector; ///< vector of DetectionLocation pointers + using TrackPtr = unique_ptr; ///< pointer to a track + using TrackPtrList = list; ///< list of track pointers + + } +} + +#endif \ No newline at end of file diff --git a/cpp/OcvSsdFaceDetection/util.cpp b/cpp/OcvSsdFaceDetection/util.cpp new file mode 100644 index 00000000..cb6d177f --- /dev/null +++ b/cpp/OcvSsdFaceDetection/util.cpp @@ -0,0 +1,272 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ +#include "util.h" + +namespace MPF{ + namespace COMPONENT{ + + /** ************************************************************************** + * If a test rectange is within snapDist from frame edges then change another + * rectangel to touch the corresponding frame edges + * + * \param rt rectangle to for snap distance test + * \param rm rectangle to change and return + * \param frameSize frame boundary size to snap to + * \param edgeSnapDist max distance (as fraction of size dimensions) from the edge for a snap + * + * \returns altered version of 'rm' rectangle that is touching the frame edges if snapped + * or the original rectangle 'rm' if no snap happened + * + *************************************************************************** */ + cv::Rect2i snapToEdges(const cv::Rect2i &rt, + const cv::Rect2i &rm, + const cv::Size2i &frameSize, + const float edgeSnapDist=0.0075){ + + cv::Point2i rt_tl = rt.tl(); + cv::Point2i rt_br = rt.br(); + cv::Point2i rm_tl = rm.tl(); + cv::Point2i rm_br = rm.br(); + + int border_x = static_cast(edgeSnapDist * frameSize.width); + int border_y = static_cast(edgeSnapDist * frameSize.height); + + if(rt_tl.x <= border_x){ // near left side of frame + rm_tl.x = 0; + }else if(rt_br.x >= frameSize.width - border_x - 1){ // near right side of frame + rm_br.x = frameSize.width - 1; + } + + if(rt_tl.y <= border_y){ // near top side of frame + rm_tl.y = 0; + }else if(rt_br.y >= frameSize.height - border_y - 1){ // near bottom side of frame + rm_br.y = frameSize.height - 1; + } + + return cv::Rect2i(rm_tl , rm_br); + } + + /** **************************************************************************** + * print out opencv matrix on a single line + * + * \param m matrix to serialize to single line string + * \returns single line string representation of matrix + * + ***************************************************************************** */ + string format(cv::Mat1f m){ + stringstream ss; + ss << "["; + for(int r=0; r(r,c); + if(c!=m.cols-1){ + ss << ", "; + }else if(r!=m.rows-1){ + ss << "; "; + } + } + } + ss << "]"; + return ss.str();; + } + + /** **************************************************************************** + * print out dlib matrix on a single line + * + * \param m matrix to serialize to single line string + * \returns single line string representation of matrix + * + ***************************************************************************** */ + template + string dformat(dlib::matrix m){ + stringstream ss; + ss << "{"; + for(size_t r=0;r(dlib::matrix m); + + /** ************************************************************************** + * Parse a string into a vector + * e.g. [1,2,3,4] + *************************************************************************** */ + template + vector fromString(const string data){ + string::size_type begin = data.find('[') + 1; + string::size_type end = data.find(']', begin); + stringstream ss( data.substr(begin, end - begin) ); + vector ret; + T val; + while(ss >> val){ + ret.push_back(val); + ss.ignore(); // ignore seperating tokens e.g. ',' + } + return ret; + } + // explicit template instantiation + template vector fromString(const string data); + + /** ************************************************************************** + * Parse a string into a opencv matrix + * e.g. [1,2,3,4, 5,6,7,8] + *************************************************************************** */ + cv::Mat fromString(const string data,const int rows,const int cols,const string dt="f"){ + stringstream ss; + ss << "{\"mat\":{\"type_id\":\"opencv-matrix\"" + << ",\"rows\":" << rows + << ",\"cols\":" << cols + << ",\"dt\":\"" << dt << '"' + << ",\"data\":" << data << "}}"; + cv::FileStorage fs(ss.str(), cv::FileStorage::READ | cv::FileStorage::MEMORY | cv::FileStorage::FORMAT_JSON); + cv::Mat mat; + fs["mat"] >> mat; + return mat; + } + + + /** **************************************************************************** + * get MPF properties of various types + * + * \param T data type to cast the result to + * \param p properties map to get property from + * \param k string key to use to retrieve property + * \param def default value to return if key is not found + * + * \return type converted value of property retrived with key or the default + * + ***************************************************************************** */ + template + T get(const Properties &p, const string &k, const T def){ + return DetectionComponentUtils::GetProperty(p,k,def); + } + + template int get (const Properties&, const string&, const int); + template bool get (const Properties&, const string&, const bool); + template float get (const Properties&, const string&, const float); + template string get(const Properties&, const string&, const string); + + + + /** **************************************************************************** + * get configuration from environment variables if not + * provided by job configuration + * + * \param T data type to cast the result to + * \param p properties map to get property from + * \param k string key to use to retrieve property + * \param def default value to return if key is not found + * + * \return type converted value of property retrived with key or the default + * + ***************************************************************************** */ + template + T getEnv(const Properties &p, const string &k, const T def){ + auto iter = p.find(k); + if (iter == p.end()){ + const char* env_p = getenv(k.c_str()); + if(env_p != NULL){ + map envp; + envp.insert(pair(k,string(env_p))); + return DetectionComponentUtils::GetProperty(envp,k,def); + }else{ + return def; + } + } + return DetectionComponentUtils::GetProperty(p,k,def); + } + + template int getEnv (const Properties&, const string&, const int); + template bool getEnv (const Properties&, const string&, const bool); + template float getEnv (const Properties&, const string&, const float); + template string getEnv(const Properties&, const string&, const string); + + + /** ************************************************************************** + * Dump MPFLocation to a stream + *************************************************************************** */ + ostream& operator<< (ostream& os, const MPFImageLocation& l) { + os << "[" << l.x_left_upper << "," << l.y_left_upper << "]-(" + << l.width << "," << l.height << "):" + << l.confidence; + if(l.detection_properties.find("CLASSIFICATION") != l.detection_properties.end()){ + os << "|" << l.detection_properties.at("CLASSIFICATION"); + } + + return os; + } + + /** ************************************************************************** + * Dump MPFTrack to a stream + *************************************************************************** */ + ostream& operator<< (ostream& os, const MPFVideoTrack& t) { + os << t.start_frame << endl; + os << t.stop_frame << endl; + for(auto& p:t.frame_locations){ + os << p.second.x_left_upper << "," << p.second.y_left_upper << "," + << p.second.width << "," << p.second.height << endl; + } + return os; + } + + /** **************************************************************************** + * Dump vectors to a stream + ***************************************************************************** */ + template + ostream& operator<< (ostream& os, const vector& v) { + os << "{"; + size_t last = v.size() - 1; + for(size_t i = 0; i < v.size(); ++i){ + os << v[i]; + if(i != last) os << ", "; + } + os << "}"; + return os; + } + + // Explicit template instantiations + template ostream& operator<< (ostream& os, const vector& v); + template ostream& operator<< (ostream& os, const vector& v); + template ostream& operator<< (ostream& os, const vector& v); + + } +} + +/** ************************************************************************** +* Redefine ocv output for rect +*************************************************************************** */ +std::ostream& cv::operator<< (std::ostream& os, const cv::Rect& r) { + os << "[" << r.x << "," << r.y << "]-(" << r.width << "," << r.height << ")"; + return os; +} diff --git a/cpp/OcvSsdFaceDetection/util.h b/cpp/OcvSsdFaceDetection/util.h new file mode 100644 index 00000000..824a04ab --- /dev/null +++ b/cpp/OcvSsdFaceDetection/util.h @@ -0,0 +1,88 @@ +/****************************************************************************** + * NOTICE * + * * + * This software (or technical data) was produced for the U.S. Government * + * under contract, and is subject to the Rights in Data-General Clause * + * 52.227-14, Alt. IV (DEC 2007). * + * * + * Copyright 2020 The MITRE Corporation. All Rights Reserved. * + ******************************************************************************/ + +/****************************************************************************** + * Copyright 2020 The MITRE Corporation * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ******************************************************************************/ +#ifndef OCVYOLODETECTION_UTIL_H +#define OCVYOLODETECTION_UTIL_H + +#include +#include + +#include "detectionComponentUtils.h" + +#include "types.h" + +namespace MPF{ + namespace COMPONENT{ + + using namespace std; + + #define THROW_EXCEPTION(MSG){ \ + string path(__FILE__); \ + string f(path.substr(path.find_last_of("/\\") + 1)); \ + throw runtime_error(f + "[" + to_string(__LINE__)+"] " + MSG); \ + } ///< exception macro so we can see where in the code it happened + + cv::Rect2i snapToEdges(const cv::Rect2i& rt, + const cv::Rect2i& rm, + const cv::Size2i& frameSize, + const float edgeSnapDist); ///< snap a rectangle to frame edges if close + + inline + float cosDist(const cv::Mat &f1, const cv::Mat &f2){ ///< cosine distance between two unit vectors + return 1.0f - max( 0.0f, min( 1.0f, static_cast(f1.dot(f2)))); + } + + string format(cv::Mat1f m); ///< output cv matrix on single line + + template + string dformat(dlib::matrix m); ///< output dlib matrix on single line + + template + vector fromString(const string data); ///< read vector from string + + cv::Mat fromString(const string data, + const int rows, + const int cols, + const string dt); ///< read opencv matrix from string + + template + T get(const Properties &p, const string &k, const T def); ///< get MPF properties of various types + + template + T getEnv(const Properties &p, const string &k, const T def); ///< get MPF properties of various types with fallback to environment variables + + template + ostream& operator<< (ostream& os, const vector& v); ///< output vector to stream + ostream& operator<< (ostream& os, const MPFImageLocation& l); ///< output MPFImageLocation to stream + ostream& operator<< (ostream& os, const MPFVideoTrack& t); ///< output MPFVideoTrack to stream + + } +} + +namespace cv{ + std::ostream& operator<< (std::ostream& os, const cv::Rect& r); ///< reformat cv::rect output to stream +} + +#endif \ No newline at end of file diff --git a/cpp/SceneChangeDetection/CMakeLists.txt b/cpp/SceneChangeDetection/CMakeLists.txt index 94ffcb3c..76f9c3c0 100644 --- a/cpp/SceneChangeDetection/CMakeLists.txt +++ b/cpp/SceneChangeDetection/CMakeLists.txt @@ -32,7 +32,7 @@ set(CMAKE_CXX_STANDARD 11) include(../ComponentSetup.cmake) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_video opencv_highgui) find_package(mpfComponentInterface REQUIRED) diff --git a/cpp/TesseractOCRTextDetection/CMakeLists.txt b/cpp/TesseractOCRTextDetection/CMakeLists.txt index a4658d3a..4530e718 100755 --- a/cpp/TesseractOCRTextDetection/CMakeLists.txt +++ b/cpp/TesseractOCRTextDetection/CMakeLists.txt @@ -33,7 +33,7 @@ set(PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/plugin/${PROJECT_NAME}) message("Package in ${PACKAGE_DIR}") unset(OpenCV_CONFIG_PATH CACHE) -find_package(OpenCV 3.4.7 EXACT REQUIRED PATHS /opt/opencv-3.4.7 +find_package(OpenCV $ENV{OPENCV_VERSION} EXACT REQUIRED PATHS /opt/opencv-$ENV{OPENCV_VERSION} COMPONENTS opencv_highgui opencv_objdetect opencv_features2d opencv_ml opencv_flann opencv_video)