Merge branch 'feat/add_opencv_color_tracker_example' into 'master'

feat(opencv): add color tracker example

See merge request ae_group/esp-iot-solution!1229

Author: YanKE01

.gitlab/ci/build.yml

@@ -946,6 +946,16 @@ build_example_utilities_xz_decompress_file:
   variables:
     EXAMPLE_DIR: examples/utilities/xz_decompress_file
 
+build_example_vision_opencv_color_tracker:
+  extends:
+    - .build_examples_template
+    - .rules:build:example_vision_opencv_color_tracker
+  parallel:
+    matrix:
+      - IMAGE: espressif/idf:release-v5.4
+  variables:
+    EXAMPLE_DIR: examples/vision/opencv/color_tracker
+
 build_components_audio_adc_mic_test_apps:
   extends:
     - .build_examples_template

.gitlab/ci/deploy.yml

@@ -132,6 +132,7 @@ pack-upload_files:
     - job: "build_example_usb_host_usb_cdc_4g_module: [espressif/idf:release-v5.1]"
     - job: "build_example_usb_host_usb_cdc_basic: [espressif/idf:release-v5.1]"
     - job: "build_example_usb_host_usb_msc_ota: [espressif/idf:release-v5.1]"
+    - job: "build_example_vision_opencv_color_tracker: [espressif/idf:release-v5.4]"
 
   artifacts:
     when: always

.gitlab/ci/rules.yml

@@ -617,6 +617,9 @@
 .patterns-example_utilities_xz_decompress_file: &patterns-example_utilities_xz_decompress_file
   - "examples/utilities/xz_decompress_file/**/*"
 
+.patterns-example_vision_opencv_color_tracker: &patterns-example_vision_opencv_color_tracker
+  - "examples/vision/opencv/color_tracker/**/*"
+
 .patterns-example_gprof_gprof_simple: &patterns-example_gprof_gprof_simple
   - "examples/gprof/gprof_simple/**/*"
 
@@ -1571,6 +1574,16 @@
     - <<: *if-dev-push
       changes: *patterns-example_utilities_xz_decompress_file
 
+.rules:build:example_vision_opencv_color_tracker:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build
+    - <<: *if-trigger-job
+    - <<: *if-dev-push
+      changes: *patterns-build_system
+    - <<: *if-dev-push
+      changes: *patterns-example_vision_opencv_color_tracker
+
 .rules:build:example_elf_loader_elf_loader_example:
   rules:
     - <<: *if-protected

examples/.build-rules.yml

@@ -420,6 +420,10 @@ examples/utilities/xz_decompress_file:
   enable:
     - if: INCLUDE_DEFAULT == 1
 
+examples/vision/opencv/color_tracker:
+  enable:
+    - if: IDF_TARGET in ["esp32p4"] and (IDF_VERSION_MAJOR == 5 and IDF_VERSION_MINOR == 4)
+
 examples/gprof/gprof_simple:
   enable:
     - if: IDF_TARGET in ["esp32","esp32s2","esp32s3","esp32c3"]

examples/vision/opencv/color_tracker/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(color_tracker)

examples/vision/opencv/color_tracker/README.md

@@ -0,0 +1,83 @@
+| Supported Targets | ESP32-P4 |
+| ----------------- | -------- |
+
+# OpenCV Color Tracker
+
+This example is based on the [esp_video](https://components.espressif.com/components/espressif/esp_video/) and [opencv](https://components.espressif.com/components/espressif/opencv/) components, and demonstrates how to display the detect results on an LCD screen.
+
+## How to use the example
+
+## ESP-IDF Required
+
+- This example supports ESP-IDF release/v5.4 and later branches. By default, it runs on ESP-IDF release/v5.4.
+- Please follow the [ESP-IDF Programming Guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32p4/get-started/index.html) to set up the development environment. **We highly recommend** you [Build Your First Project](https://docs.espressif.com/projects/esp-idf/en/latest/esp32p4/get-started/index.html#build-your-first-project) to get familiar with ESP-IDF and make sure the environment is set up correctly.
+
+
+### Prerequisites
+
+* An ESP32-P4-Function-EV-Board.
+* A 7-inch 1024 x 600 LCD screen powered by the [EK79007](https://dl.espressif.com/dl/schematics/display_driver_chip_EK79007AD_datasheet.pdf) IC, accompanied by a 32-pin FPC connection [adapter board](https://dl.espressif.com/dl/schematics/esp32-p4-function-ev-board-lcd-subboard-schematics.pdf) ([LCD Specifications](https://github.com/espressif/esp-dev-kits/blob/master/docs/_static/esp32-p4-function-ev-board/camera_display_datasheet/display_datasheet.pdf)).
+* A MIPI-CSI camera powered by the SC2336 IC, accompanied by a 32-pin FPC connection [adapter board](https://dl.espressif.com/dl/schematics/esp32-p4-function-ev-board-camera-subboard-schematics.pdf) ([Camera Specifications](https://dl.espressif.com/dl/schematics/camera_datasheet.pdf)).
+* A USB-C cable for power supply and programming.
+* Please refer to the following steps for the connection:
+    * **Step 1**. According to the table below, connect the pins on the back of the screen adapter board to the corresponding pins on the development board.
+
+        | Screen Adapter Board | ESP32-P4-Function-EV-Board |
+        | -------------------- | -------------------------- |
+        | 5V (any one)         | 5V (any one)               |
+        | GND (any one)        | GND (any one)              |
+        | PWM                  | GPIO26                     |
+        | LCD_RST              | GPIO27                     |
+
+    * **Step 2**. Connect the FPC of LCD through the `MIPI_DSI` interface.
+    * **Step 3**. Connect the FPC of Camera through the `MIPI_CSI` interface.
+    * **Step 4**. Use a USB-C cable to connect the `USB-UART` port to a PC (Used for power supply and viewing serial output).
+    * **Step 5**. Turn on the power switch of the board.
+
+### Configure the Project
+
+Run `idf.py menuconfig` and navigate to the `Example Configuration` menu, where you can configure the relevant pins and example-related options.
+
+This example supports configuring two HSV ranges, and you can adjust the Primary Color Range and the Secondary Color Range separately to identify the colors you want. By default, the intervals are set for recognizing red.
+
+```
+Example Configuration  --->
+    Primary Color Range  --->
+        (0) Primary Lower H (Hue)
+        (120) Primary Lower S (Saturation)
+        (70) Primary Lower V (Value)
+        (10) Primary Upper H (Hue)
+        (255) Primary Upper S (Saturation)
+        (255) Primary Upper V (Value)
+    Secondary Color Range  --->
+        (170) Secondary Lower H (Hue)
+        (120) Secondary Lower S (Saturation)
+        (70) Secondary Lower V (Value)
+        (180) Secondary Upper H (Hue)
+        (255) Secondary Upper S (Saturation)
+        (255) Secondary Upper V (Value)
+```
+
+As well, in the `Espressif Camera Sensors` Configurations, the camera sensor can be selected.
+
+```
+Component config  --->
+    Espressif Camera Sensors Configurations  --->
+        [*] SC2336  ---->
+            Default format select for MIPI (RAW8 1280x720 30fps, MIPI 2lane 24M input)  --->
+                (X) RAW8 1280x720 30fps, MIPI 2lane 24M input
+```
+
+### Build and Flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output (replace `PORT` with your board's serial port name):
+
+```c
+idf.py -p PORT flash monitor
+```
+
+To exit the serial monitor, type ``Ctrl-]``.
+
+See the [ESP-IDF Getting Started Guide](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/index.html) for full steps to configure and use ESP-IDF to build projects.
+
+![color_tracker](https://dl.espressif.com/AE/esp-iot-solution/cv_color_tracker.gif)

examples/vision/opencv/color_tracker/main/CMakeLists.txt

@@ -0,0 +1,25 @@
+idf_component_register(SRC_DIRS "."
+                    INCLUDE_DIRS "." "include/"
+                    REQUIRES "esp_mm" "esp_timer")
+
+set_source_files_properties(
+    "cv_detect.cpp"
+    PROPERTIES
+        COMPILE_OPTIONS
+        "-Wno-deprecated-enum-enum-conversion"
+    )
+
+set(MODIFY_EXE ${PROJECT_DIR}/modify_opencv.py)
+set(OPENCV_CMAKE_TARGET "${PROJECT_DIR}/managed_components/espressif__opencv/CMakeLists.txt")
+set(OPENCV_CMAKE_SOURCE "CMakeLists.txt.opencv")
+
+add_custom_command(
+    OUTPUT ${OPENCV_CMAKE_TARGET}
+    COMMENT "Updating OpenCV CMakeLists.txt using modify_opencv.py"
+    COMMAND python ${MODIFY_EXE}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    DEPENDS ${OPENCV_CMAKE_SOURCE}
+    VERBATIM)
+
+add_custom_target(update_opencv ALL DEPENDS ${OPENCV_CMAKE_TARGET})
+add_dependencies(app update_opencv)

examples/vision/opencv/color_tracker/main/CMakeLists.txt.opencv

@@ -0,0 +1,187 @@
+set(deps esp_rom vfs lwip pthread newlib)
+
+idf_component_register(
+                       # We need the dummy source file so that the component
+                       # library is not an interface library. This allows to
+                       # get the list of include directories from other components
+                       # via INCLUDE_DIRECTORIES property later on.
+                       SRCS dummy.c
+                       # OpenCV librarires depend on some of the IDF libraries,
+                       # add them here:
+                       PRIV_REQUIRES ${deps})
+
+# Determine compilation flags used for building OpenCV
+# Flags inherited from IDF build system and other IDF components:
+set(idf_include_directories $<TARGET_PROPERTY:idf::${COMPONENT_NAME},INCLUDE_DIRECTORIES>)
+set(includes "-I$<JOIN:${idf_include_directories}, -I>")
+# IDF is built with -D_GNU_SOURCE, but we don't actually implement all _GNU_SOURCE
+# features. Defining _GNU_SOURCE causes OpenCV to try to use them, which fails.
+# Use _DEFAULT_SOURCE instead, it provides enough features to build OpenCV.
+set(extra_defines -D_DEFAULT_SOURCE)
+# Final flags for C and C++:
+set(c_flags "${includes} ${extra_defines}")
+set(cxx_flags "${includes} ${extra_defines}")
+set(cxx_flags "${includes} ${extra_defines} -DCV_INT32_T_IS_LONG_INT=ON -Dalloca=__builtin_alloca")
+
+set(common_flags "-ggdb  -ffunction-sections -fdata-sections")
+
+# Calculate flags for optimizations, assertions, debug info:
+if(COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE)
+   set(assert_flags "-DNDEBUG")
+else()
+   set(assert_flags "-DDEBUG -D_DEBUG")
+endif()
+
+if(CONFIG_IDF_TARGET_ARCH_XTENSA)
+    set(assert_flags "${assert_flags} -mlongcalls -mauto-litpools")
+endif()
+
+if(CONFIG_COMPILER_OPTIMIZATION_DEFAULT)
+    set(opt_c_flags "-Og ${common_flags} ${assert_flags} -Wno-format -Wno-format-security -Wno-undef -Wno-calloc-transposed-args")
+    set(opt_cxx_flags "-Og ${common_flags} ${assert_flags} -Wno-undef -Wno-cast-user-defined -Wno-format -Wno-format-security -Wno-cast-function-type -Wno-deprecated-enum-enum-conversion")
+    set(opt_args -DCMAKE_BUILD_TYPE=Debug
+                 -DCMAKE_C_FLAGS_DEBUG=${opt_c_flags}
+                 -DCMAKE_CXX_FLAGS_DEBUG=${opt_cxx_flags})
+elseif(CONFIG_COMPILER_OPTIMIZATION_SIZE)
+    # Normally we should use MinSizeRel build type for this,
+    # however OpenCV CMake files only handle Debug and Release
+    # (see e.g. opencv/cmake/OpenCVCompilerOptions.cmake).
+    # So we redefine the flags for Release instead.
+    set(opt_c_flags "-Os ${common_flags} ${assert_flags} -Wno-format -Wno-format-security -Wno-undef -Wno-calloc-transposed-args")
+    set(opt_cxx_flags "-Os ${common_flags} ${assert_flags} -Wno-undef -Wno-cast-user-defined -Wno-format -Wno-format-security -Wno-cast-function-type -Wno-deprecated-enum-enum-conversion")
+    set(opt_args -DCMAKE_BUILD_TYPE=Release
+                 -DCMAKE_C_FLAGS_RELEASE=${opt_c_flags}
+                 -DCMAKE_CXX_FLAGS_RELEASE=${opt_cxx_flags})
+elseif(COMPILER_OPTIMIZATION_PERF)
+    # Currently OpenCV fails to compile at -O2 level on Xtensa due to a compiler issue,
+    # details in https://github.com/jcmvbkbc/gcc-xtensa/issues/14.
+    # Compiling at -O3 happens to be okay.
+    set(opt_c_flags "-O3 ${common_flags} ${assert_flags} -Wno-format -Wno-format-security -Wno-undef -Wno-calloc-transposed-args")
+    set(opt_cxx_flags "-O3 ${common_flags} ${assert_flags} -Wno-undef -Wno-cast-user-defined -Wno-format -Wno-format-security -Wno-cast-function-type -Wno-deprecated-enum-enum-conversion")
+    set(opt_args -DCMAKE_BUILD_TYPE=Release
+                 -DCMAKE_C_FLAGS_RELEASE=${opt_c_flags}
+                 -DCMAKE_CXX_FLAGS_RELEASE=${opt_cxx_flags})
+elseif(COMPILER_OPTIMIZATION_NONE)
+    set(opt_c_flags "-O0 ${common_flags} ${assert_flags} -Wno-format -Wno-format-security -Wno-undef -Wno-calloc-transposed-args")
+    set(opt_cxx_flags "-O0 ${common_flags} ${assert_flags} -Wno-undef -Wno-cast-user-defined -Wno-format -Wno-format-security -Wno-cast-function-type -Wno-deprecated-enum-enum-conversion")
+    set(opt_args -DCMAKE_BUILD_TYPE=Debug
+                 -DCMAKE_C_FLAGS_DEBUG=${opt_c_flags}
+                 -DCMAKE_CXX_FLAGS_DEBUG=${opt_cxx_flags})
+else()
+    message(FATAL_ERROR "Unsupported optimization level")
+endif()
+
+# Python to use for OpenCV build
+idf_build_get_property(python PYTHON)
+
+include(ExternalProject)
+
+# Build OpenCV in this directory:
+set(BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv-build)
+
+# List of OpenCV static libraries we expect to see after the build.
+# Obtained manually by looking at ${BINARY_DIR}/install/lib directory.
+# In some cases it's important to have right order of libraries here.
+set(opencv_libraries
+    calib3d flann fuzzy
+    objdetect tracking video videostab features2d
+    surface_matching
+    shape stereo stitching structured_light
+    line_descriptor ml phase_unwrapping photo plot quality rapid
+    imgcodecs imgproc core
+)
+
+set(3rdparty_libraries
+    zlib
+    libopenjp2
+    libjpeg-turbo
+    libpng
+    libtiff
+    libwebp
+)
+
+# List of all static libraries to be produced
+set(all_libraries)
+set(all_targets)
+foreach(libname ${opencv_libraries})
+    set(lib_path ${BINARY_DIR}/install/lib/libopencv_${libname}.a)
+    list(APPEND all_libraries ${lib_path})
+    list(APPEND all_targets opencv_${libname})
+    add_prebuilt_library(opencv_${libname} ${lib_path})
+endforeach()
+foreach(libname ${3rdparty_libraries})
+    set(lib_path ${BINARY_DIR}/install/lib/opencv4/3rdparty/lib${libname}.a)
+    list(APPEND all_libraries ${lib_path})
+    list(APPEND all_targets opencv_${libname})
+    add_prebuilt_library(opencv_${libname} ${lib_path})
+endforeach()
+
+# Add OpenCV as a subproject.
+ExternalProject_Add(opencv_proj
+    SOURCE_DIR ${COMPONENT_DIR}/opencv
+    BINARY_DIR ${BINARY_DIR}
+    BUILD_BYPRODUCTS ${all_libraries}
+    # These two options are set so that Ninja immediately outputs
+    # the subproject build to the terminal. Otherwise it looks like the
+    # build process "hangs" for too long until OpenCV build is complete.
+    USES_TERMINAL_CONFIGURE TRUE
+    USES_TERMINAL_BUILD TRUE
+    # Arguments to pass to OpenCV CMake invocation:
+    CMAKE_ARGS
+        -DCMAKE_C_FLAGS=${c_flags}
+        -DCMAKE_CXX_FLAGS=${cxx_flags}
+        ${opt_args}
+        -DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/install
+        -DENABLE_CCACHE=${CCACHE_ENABLE}
+        -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
+        -DPYTHON_DEFAULT_EXECUTABLE=${python}
+        -DCV_CPU_BASELINE_MODE=${CONFIG_IDF_TARGET}
+        -DCV_DISABLE_OPTIMIZATION=0
+        -DOPENCV_EXTRA_MODULES_PATH=${CMAKE_CURRENT_LIST_DIR}/opencv_contrib/modules
+        -DOPENCV_FORCE_3RDPARTY_BUILD=1
+        -DCMAKE_SYSTEM_PROCESSOR=${CONFIG_IDF_TARGET_ARCH}
+        -DENABLE_PIC=0
+        -DTARGET=${CONFIG_IDF_TARGET}
+        -DWITH_ADE=0
+        -DWITH_JPEG=1
+        -DWITH_OPENCL=0
+        -DWITH_OPENEXR=0
+        -DWITH_OPENGL=0
+        -DWITH_OPENMP=0
+        -DWITH_PNG=1
+        -DWITH_PROTOBUF=0
+        -DWITH_TIFF=1
+        -DWITH_WEBP=1
+        -DBUILD_EXAMPLES=0
+        -DBUILD_PACKAGE=1
+        -DBUILD_PERF_TESTS=0
+        -DBUILD_PNG=1
+        -DBUILD_SHARED_LIBS=0
+        -DBUILD_TESTS=0
+        -DBUILD_ZLIB=1
+        -DBUILD_opencv_apps=0
+        -DBUILD_opencv_bgsegm=0
+        -DBUILD_opencv_datasets=0
+        -DBUILD_opencv_imgcodecs=1
+        -DBUILD_opencv_objdetect=1
+        -DBUILD_opencv_imgproc=1
+        -DBUILD_opencv_tracking=1
+        -DBUILD_opencv_video=1
+        -DBUILD_opencv_videoio=0
+)
+
+# Attach header files to the component library:
+set_target_properties(${COMPONENT_LIB} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${BINARY_DIR}/install/include/opencv4)
+
+# Make sure the subproject is built before the component library:
+add_dependencies(${COMPONENT_LIB} opencv_proj)
+
+
+foreach(opencv_target ${all_targets})
+    # Attach IDF compoenent dependencies to OpenCV libraries
+    foreach(dep ${deps})
+        target_link_libraries(${opencv_target} INTERFACE idf::${dep})
+    endforeach()
+    # Attach OpenCV libraries to the component library
+    target_link_libraries(${COMPONENT_LIB} INTERFACE ${opencv_target})
+endforeach()