samples: tflite-micro: add micro_speech application with OpenAMP on i.MX8MP

samples/modules/tflite-micro/micro_speech/CMakeLists.txt

@@ -0,0 +1,71 @@
+cmake_minimum_required(VERSION 3.20.0)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(micro_speech_openamp)
+
+# These samples use local static initialization. Since Zephyr doesn't support the
+# C++ ABI for thread-safe initialization of local statics and the constructors don't
+# appear to require thread safety, we turn it off in the C++ compiler.
+set(NO_THREADSAFE_STATICS $<TARGET_PROPERTY:compiler-cpp,no_threadsafe_statics>)
+zephyr_compile_options($<$<COMPILE_LANGUAGE:CXX>:${NO_THREADSAFE_STATICS}>)
+
+# Define common paths
+set(OPTIONAL_TFLITE_DIR $ENV{ZEPHYR_BASE}/../optional/modules/lib/tflite-micro/signal)
+
+file(
+        GLOB
+        app_sources
+        src/*
+        src/inference/*
+        src/transport/*
+)
+
+target_include_directories(
+        app
+        PRIVATE
+        src
+        ${ZEPHYR_TFLITE_MICRO_MODULE_DIR}/signal
+        ${ZEPHYR_TFLITE_MICRO_MODULE_DIR}/signal/micro/kernels/
+        ${ZEPHYR_TFLITE_MICRO_MODULE_DIR}/signal/src
+        ${LIBMETAL_INCLUDE_DIR}
+        ${OPENAMP_INCLUDE_DIR}
+        ${PLATFORM_DIR}
+)
+
+target_sources(
+    app
+    PRIVATE
+    ${app_sources}
+
+    # Signal processing
+    ${OPTIONAL_TFLITE_DIR}/src/fft_auto_scale.cc
+    ${OPTIONAL_TFLITE_DIR}/src/max_abs.cc
+    ${OPTIONAL_TFLITE_DIR}/src/msb_32.cc
+    ${OPTIONAL_TFLITE_DIR}/src/energy.cc
+    ${OPTIONAL_TFLITE_DIR}/src/filter_bank.cc
+    ${OPTIONAL_TFLITE_DIR}/src/pcan_argc_fixed.cc
+    ${OPTIONAL_TFLITE_DIR}/src/filter_bank_log.cc
+    ${OPTIONAL_TFLITE_DIR}/src/log.cc
+    ${OPTIONAL_TFLITE_DIR}/src/filter_bank_spectral_subtraction.cc
+
+    # Micro kernels
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/energy.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/fft_auto_scale_common.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/filter_bank_square_root_common.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/filter_bank_spectral_subtraction.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/filter_bank.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/pcan.cc
+    ${OPTIONAL_TFLITE_DIR}/micro/kernels/filter_bank_log.cc
+)
+
+if ("${BOARD_ARCH}" STREQUAL "xtensa")
+        target_sources(
+        app
+        PRIVATE
+
+        # Xtensa kernels
+        ${OPTIONAL_TFLITE_DIR}/micro/kernels/xtensa/fft_auto_scale_kernel.cc
+        ${OPTIONAL_TFLITE_DIR}/micro/kernels/xtensa/filter_bank_square_root.cc
+        ${OPTIONAL_TFLITE_DIR}/micro/kernels/xtensa/xtensa_square_root.S
+        )
+endif()

samples/modules/tflite-micro/micro_speech/README.rst

@@ -0,0 +1,154 @@
+.. zephyr:code-sample:: tflite-micro-speech-openamp
+   :name: Micro Speech OpenAMP
+
+   Recognize speech commands from audio input received on Cortex-A cores and
+   processed on the HiFi4 DSP of the i.MX8M Plus EVK board using TensorFlow Lite
+   for Microcontrollers with a 20KB neural network.
+
+Overview
+********
+
+This sample requires an application running on the Cortex-A cores of the i.MX8M Plus
+to capture audio and send it to the HiFi4 DSP using OpenAMP. The DSP processes
+the audio data and performs inference using TensorFlow Lite Micro that
+detects 2 speech commands ("yes" and "no"), as well as "silence" and "unknown".
+
+.. code-block:: text
+
+   +------------------------- Cortex A (main core) -------------+                            +--------------- HiFi4 DSP (remote core) --------------+
+   |                                                            |                            |                                                      |
+   |  [ALSA/arecord] -> [Linux userspace] -> [/dev/ttyRPMSG*]   |----------> [RPMsg] ------->|  [ring/msgq] -> [frontend] -> [TFLM] -> [output]     |
+   |                                                            |                            |                                                      |
+   +------------------------------------------------------------+                            +------------------------------------------------------+
+
+.. Note::
+    This README and sample have been modified from
+    `the TensorFlow Hello World sample`_,
+    `the OpenAMP using resource table from Zephyr`_ and
+    `the Micro Speech Example from TensorFlow Lite for Microcontrollers`_.
+
+.. _the TensorFlow Hello World sample:
+    https://github.com/tensorflow/tflite-micro-arduino-examples/tree/main/examples/hello_world
+
+.. _the OpenAMP using resource table from Zephyr:
+    https://github.com/zephyrproject-rtos/zephyr/tree/main/samples/subsys/ipc/openamp_rsc_table
+
+.. _the Micro Speech Example from TensorFlow Lite for Microcontrollers:
+    https://github.com/tensorflow/tflite-micro/tree/main/tensorflow/lite/micro/examples/micro_speech
+
+Audio contract
+--------------
+- Sample rate: 16kHz
+- Sample format: S16_LE
+- Frame size (samples per RPMsg payload): 20ms (320 samples or 640 bytes)
+- Endianness: LE
+
+Compatibility
+-------------
+- Validated Platform: i.MX8MP with the HiFi4 DSP core.
+- Porting: It is compatible with other boards, but this requires creating a new board configuration and updating the DTS overlays to match the target hardware.
+
+Building and Running
+********************
+
+West Module Filters
+-------------------
+This sample requires the tflite-micro module.
+
+DSP Firmware
+------------
+
+Add the tflite-micro module to your West manifest and pull it:
+
+.. code-block:: console
+
+    west config manifest.project-filter -- +tflite-micro
+    west update
+
+The sample can be built for the :zephyr:board:`imx8mp_evk/mimx8ml8/adsp` as follows:
+
+.. zephyr-app-commands::
+   :zephyr-app: samples/modules/tflite-micro/micro_speech
+   :host-os: unix
+   :board: imx8mp_evk/mimx8ml8/adsp
+   :goals: run
+   :compact:
+
+Linux Application
+-----------------
+
+The Linux application is not part of the Zephyr repository. It can be found in the `this repository`_.
+
+.. _this repository:
+   https://github.com/thong-phn/linux-app
+
+Sample Output
+*************
+
+Linux Application
+-----
+
+Simulation with a WAV file as input
+
+.. code-block:: console
+
+    root@imx8mpevk:~# ./send default16.wav
+    [L] Using TTY device: /dev/ttyRPMSG0
+    [L] Expect audio frames: 500
+    [L] Consumer:  Consumer thread started
+    [L] Producer:  Producer thread started
+    [L] Producer:  End of file reached
+    [L] Producer:  Producer stopping
+    [L] Consumer:  EOF frame received, stopping
+    [L] Consumer:  EOF marker sent to Zephyr
+    [L] Consumer:  Consumer thread finished
+
+Real-time Recording
+
+.. code-block:: console
+
+    root@imx8mpevk:~# ./record hw:5,0 /dev/ttyRPMSG0
+    [L] Using PCM device: hw:5,0
+    [L] Using TTY device: /dev/ttyRPMSG0
+    [L] PCM device hw:5,0 configured for 16kHz, S16_LE, Mono
+    [L] Consumer:  Consumer thread started
+    [L] Producer:  Producer thread started
+    ^C
+    [L] Ctrl+C detected. Stopping..
+    [L] Producer:  Sending EOF to consumer
+    [L] Producer:  Producer stopping
+    [L] Consumer:  EOF frame received, stopping
+    [L] Consumer:  EOF marker sent to Zephyr
+    [L] Consumer:  Consumer thread finished
+    [L] Application finished.
+
+DSP Firmware
+------------
+
+.. code-block:: console
+
+        [00:00:00.697,000] <inf> micro_speech_openamp: Starting Micro Speech OpenAMP application
+        [00:00:01.231,000] <inf> micro_speech_openamp: Audio processing thread started
+        [00:00:02.321,000] <inf> micro_speech_openamp: Audio processing thread started
+        [00:00:03.591,000] <inf> model_runner: Initializing static interpreters
+        [00:00:03.941,000] <inf> model_runner: Static interpreters initialized successfully
+        [00:00:04.981,000] <inf> model_runner: Detected: yes
+        [00:00:06.102,000] <inf> model_runner: Detected: no
+        [00:00:07.202,000] <inf> model_runner: Detected: silence
+
+Training
+********
+To train your own model for use in this sample, follow the instructions in `this link`_.
+
+.. _this link:
+   https://github.com/tensorflow/tflite-micro/tree/main/tensorflow/lite/micro/examples/micro_speech/train
+
+Limitations
+********
+The basic model uses an inference audio frame size of 1000 ms.
+As a result, there are some limitations:
+
+#. If two commands are spoken within 1000 ms, the second command may not be detected.
+#. If a command lasts longer than 1000 ms, it may be detected as two separate commands.
+
+Potential solution: Retrain the model with a smaller input frame size.

samples/modules/tflite-micro/micro_speech/boards/imx8mp_evk_mimx8ml8_adsp.conf

@@ -0,0 +1,8 @@
+CONFIG_LOG_PRINTK=n
+CONFIG_IPM_IMX_MAX_DATA_SIZE_16=n
+CONFIG_IPM_IMX_MAX_DATA_SIZE_4=y
+CONFIG_OPENAMP_WITH_DCACHE=y
+
+#vendor-specific resource table
+CONFIG_OPENAMP_VENDOR_RSC_TABLE=y
+CONFIG_OPENAMP_VENDOR_RSC_TABLE_FILE="nxp_resource_table.c"

samples/modules/tflite-micro/micro_speech/boards/imx8mp_evk_mimx8ml8_adsp.overlay

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2023 NXP
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/ {
+	chosen {
+		/*
+		 * shared memory reserved for the inter-processor communication
+		 */
+		zephyr,ipc_shm = &dspsram3;
+		zephyr,ipc = &mailbox0;
+	};
+
+	dspsram3: memory@942f0000 {
+		compatible = "mmio-sram";
+		reg = <0x942f0000 0x110000>;
+	};
+};
+
+&mailbox0 {
+	status = "okay";
+};

samples/modules/tflite-micro/micro_speech/prj.conf

@@ -0,0 +1,21 @@
+# tflite
+CONFIG_CPP=y
+CONFIG_STD_CPP17=y
+CONFIG_TENSORFLOW_LITE_MICRO=y
+CONFIG_MAIN_STACK_SIZE=6144 # tflm arena size
+CONFIG_REQUIRES_FLOAT_PRINTF=y
+
+# rpmsg
+CONFIG_KERNEL_BIN_NAME="micro_speech_openamp"
+CONFIG_PRINTK=n
+CONFIG_IPM=y
+CONFIG_HEAP_MEM_POOL_SIZE=5120 # rpmsg buffers
+CONFIG_OPENAMP=y
+CONFIG_OPENAMP_RSC_TABLE_NUM_RPMSG_BUFF=8
+CONFIG_OPENAMP_RSC_TABLE=y
+CONFIG_OPENAMP_MASTER=n
+CONFIG_SHELL=n
+
+# logging
+CONFIG_LOG=y
+CONFIG_LOG_DEFAULT_LEVEL=3

samples/modules/tflite-micro/micro_speech/sample.yaml

@@ -0,0 +1,20 @@
+sample:
+  description: Micro Speech OpenAMP Sample
+  name: micro speech openamp
+common:
+  tags: tensorflow
+  modules:
+    - tflite-micro
+  harness: console
+  harness_config:
+    type: multi_line
+    ordered: false
+    regex:
+      - "Detected: (yes|no|silence|unknown)"
+tests:
+  sample.tensorflow.micro_speech_openamp:
+    platform_allow:
+      - imx8mp_evk/mimx8ml8/adsp
+    integration_platforms:
+      - imx8mp_evk/mimx8ml8/adsp
+    tags: tensorflow, openamp, ipc, audio

samples/modules/tflite-micro/micro_speech/snippets/nxp-openamp-imx8-adsp.conf

@@ -0,0 +1,9 @@
+CONFIG_LOG_PRINTK=n
+CONFIG_IPM_IMX_MAX_DATA_SIZE_16=n
+CONFIG_IPM_IMX_MAX_DATA_SIZE_4=y
+CONFIG_OPENAMP_WITH_DCACHE=y
+CONFIG_SHELL=n
+
+#vendor-specific resource table
+CONFIG_OPENAMP_VENDOR_RSC_TABLE=y
+CONFIG_OPENAMP_VENDOR_RSC_TABLE_FILE="nxp_resource_table.c"

samples/modules/tflite-micro/micro_speech/snippets/nxp-openamp-imx8-adsp.overlay

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2024 NXP
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/ {
+	chosen {
+		/*
+		 * shared memory reserved for the inter-processor communication
+		 */
+		zephyr,ipc_shm = &dspsram3;
+		zephyr,ipc = &mailbox0;
+	};
+
+	dspsram3: memory@942f0000 {
+		compatible = "mmio-sram";
+		reg = <0x942f0000 0x110000>;
+	};
+};
+
+&mailbox0 {
+	status = "okay";
+};

samples/modules/tflite-micro/micro_speech/snippets/snippet.yml

@@ -0,0 +1,6 @@
+name: nxp-openamp-imx8-adsp
+boards:
+  /imx8qxp_mek\/mimx8qx6\/adsp|imx8qm_mek\/mimx8qm6\/adsp/:
+    append:
+      EXTRA_DTC_OVERLAY_FILE: nxp-openamp-imx8-adsp.overlay
+      EXTRA_CONF_FILE: nxp-openamp-imx8-adsp.conf