Refactor: Promote ST40 pipeline out of experimental tree

- relocate the st40 pipeline headers/sources into lib/src/st2110/pipeline and
  export the API from include/st40_pipeline_api.h
- update app meson glue, pipeline samples, and shared sample utilities to point
  at the promoted API
- refresh docs/run + doc/design along with app/sample/README.md to document the
  ST40 pipeline workflow in one place
- wire the meson include lists and new rx length guard so ancillary rx stays
  robust after the move

Author: moleksy

app/meson.build

@@ -25,8 +25,6 @@ libpthread = cc.find_library('pthread', required : true)
 libjson_c = dependency('json-c', required : true)
 libpcap = dependency('pcap', required: true)
 
-app_common_includes = include_directories('..', '../include')
-
 libsdl2 = dependency('sdl2', required: false)
 if libsdl2.found()
   add_global_arguments('-DAPP_HAS_SDL2', language : 'c')
@@ -258,7 +256,6 @@ executable('TxSt30PipelineSample', pipeline_tx_st30_sample_sources,
 executable('TxSt40PipelineSample', pipeline_tx_st40_sample_sources,
   c_args : app_c_args,
   link_args: app_ld_args,
-  include_directories: app_common_includes,
   # asan should be always the first dep
   dependencies: [asan_dep, mtl, libpthread, ws2_32_dep, mman_dep]
 )

app/sample/README.md

@@ -42,6 +42,58 @@ The dir include the simple sample code for how to develop application quickly ba
 ./build/app/RxSt22PipelineSample --p_port 0000:af:01.1 --p_sip 192.168.75.22 --p_rx_ip 239.168.75.22
 ```
 
+### 2.1. ST40 ancillary pipeline samples
+
+`TxSt40PipelineSample` and `RxSt40PipelineSample` live in this directory (see `tx_st40_pipeline_sample.c` / `rx_st40_pipeline_sample.c`) and are built automatically by `./build.sh`. They leverage the same CLI foundation as the other pipeline samples, so familiar options such as `--p_port`, `--udp_port`, `--sessions_cnt`, `--multi_inc_addr`, `--ext_frame`, and destination MAC overrides behave identically.
+
+#### Tx workflow
+
+1. Prepare (or reuse) an ancillary payload. Any binary file works because frames are looped forever:
+   ```bash
+   head -c 4096 /dev/urandom > /tmp/anc_payload.bin
+   ```
+2. Launch the TX sample. The example below keeps every frame below the 255-byte UDW limit returned by `st40p_tx_max_udw_buff_size()` so that each payload fits into a single RFC 8331 RTP packet.
+   ```bash
+   ./build/app/TxSt40PipelineSample \
+		   --p_port 0000:af:01.0 \
+		   --p_tx_ip 239.1.1.1 \
+		   --p_sip 192.168.96.2 \
+		   --udp_port 20000 \
+		   --payload_type 40 \
+		   --sessions_cnt 1 \
+		   --tx_url /tmp/anc_payload.bin \
+		   --ext_frame
+   ```
+
+Helpful switches:
+
+- `--ext_frame` keeps the sample in user-supplied buffer mode, mirroring how most applications integrate the pipeline API.
+- `--multi_inc_addr` increments the destination multicast address per session instead of the UDP port so you can spin up multiple ancillary streams on one NIC queue.
+
+#### Rx workflow
+
+Match the TX parameters on the receiver. Enabling `--rx_dump` writes a human-readable metadata block per frame to `--rx_url` (`foo.txt_0`, `_1`, … if multiple sessions are active):
+
+```bash
+./build/app/RxSt40PipelineSample \
+		--p_port 0000:af:01.0 \
+		--p_rx_ip 239.1.1.1 \
+		--p_sip 192.168.96.2 \
+		--udp_port 20000 \
+		--payload_type 40 \
+		--sessions_cnt 1 \
+		--rx_url /tmp/anc_dump.txt \
+		--rx_dump
+```
+
+The runtime log summarizes the number of metadata entries and UDW bytes received per frame, while the dump files capture DID/SDID/line information followed by the raw payload so you can diff TX/RX easily.
+
+#### Troubleshooting and USDT hooks
+
+- Use `--rx_dump`/`--tx_url` first to confirm that metadata completes the round trip.
+- When `st40_total_pkts` exceeds 1, the payload overflowed the 255-byte UDW buffer; trim the input file or drop the session count on slower links.
+- For end-to-end tracing, follow the ST40P probe examples in `doc/usdt.md` §2.5.6 (they cover `usdt::st40p:*` plus `usdt::sys:log_msg` combinations) instead of keeping scripts here.
+
 ## 3. Forward pipeline samples
 
 [rx_st20p_tx_st22p_fwd.c](fwd/rx_st20p_tx_st22p_fwd.c): A demo application which receive a st20 stream and output as st22 compressed stream with logo rendering.

app/sample/rx_st40_pipeline_sample.c

@@ -2,8 +2,6 @@
  * Copyright(c) 2025 Intel Corporation
  */
 
-#include <mtl/experimental/st40_pipeline_api.h>
-
 #include "sample_util.h"
 
 #define ST40P_SAMPLE_MAX_UDW_SIZE 2048

app/sample/sample_util.h

@@ -10,6 +10,7 @@
 #include <inttypes.h>
 #include <mtl/experimental/st20_combined_api.h>
 #include <mtl/st30_pipeline_api.h>
+#include <mtl/st40_pipeline_api.h>
 #include <mtl/st_convert_api.h>
 #include <mtl/st_pipeline_api.h>
 #include <pthread.h>

app/sample/tx_st40_pipeline_sample.c

@@ -2,8 +2,6 @@
  * Copyright(c) 2025 Intel Corporation
  */
 
-#include <mtl/experimental/st40_pipeline_api.h>
-
 #include "sample_util.h"
 
 #define ST40P_SAMPLE_MAX_UDW_SIZE 255

doc/design.md

@@ -335,6 +335,22 @@ Sample application code can be find in [tx_st20_pipeline_sample.c](../app/sample
 By default, the `st20p_tx_get_frame` and `st20p_rx_get_frame` functions operate in non-blocking mode, which means the function call will immediately return `NULL` if no frame is available.
 To switch to blocking mode, where the call will wait until a frame is ready for application use or one second timeout occurs, you must enable the `ST20P_TX_FLAG_BLOCK_GET` or `ST20P_RX_FLAG_BLOCK_GET` flag respectively during the session creation stage, and application can use `st20p_tx_wake_block`/`st20p_rx_wake_block` to wake up the waiting directly.
 
+#### 6.3.1. Ancillary (ST40) pipeline
+
+The same pipeline abstraction is now available for ancillary-only flows through `st40_pipeline_api.h`. It keeps the zero-copy RFC 8331 payload path but hides scheduler registration, frame queues, and pacing details behind the familiar get/put contract so applications only need to implement:
+
+```c
+st40p_tx_get_frame
+st40p_tx_put_frame
+st40p_rx_get_frame
+st40p_rx_put_frame
+```
+
+- **TX pipeline (`st40p_tx_*`)** wraps the lower-level `st40_tx_*` APIs while honoring redundancy, user timestamps, and user-managed frame buffers (for example when `ST40P_TX_FLAG_EXT_FRAME` is enabled). The helper also exposes the maximum user data word (UDW) size via `st40p_tx_max_udw_buff_size()` so that producers can pack metadata deterministically.
+- **RX pipeline (`st40p_rx_*`)** assembles ancillary packets into user buffers and can optionally dump metadata/UDW buffers directly to files for debugging through `st40p_rx_set_dump`. Each RX callback receives both the metadata blocks and the captured payload, mirroring the TX layout to simplify round-trip validation.
+
+Reference implementations live in `app/sample/tx_st40_pipeline_sample.c` and `app/sample/rx_st40_pipeline_sample.c`. They reuse the shared `sample_util` CLI so you can swap between ST20/22/30/40 pipelines with identical arguments while testing pacing, redundancy, and USDT probes.
+
 ### 6.4. ST22 support
 
 The support for ST 2110-22 JPEG XS can be categorized into two modes: ST 2110-22 raw codestream mode and ST 2110-22 pipeline mode. The pipeline mode leverages an MTL plugin to handle the decoding/encoding between the raw video buffer and the codestream.
@@ -498,21 +514,6 @@ For instance, with a 1080i50 format, you should use the following session parame
 
 For transmission (TX), users can specify whether the current field is the first or second by using the `second_field` flag within the `struct st40_tx_frame_meta`. For reception (RX), RTP passthrough mode is the only supported, it's application's duty to check the if it's the first or second by inspecting the F bits in rfc8331 header.
 
-### 6.18. ST40 pipeline mode
-
-For applications that prefer the simplified pipeline abstraction (similar to `st20p`/`st30p`) MTL now exposes `st40_pipeline_api.h`. Pipeline mode keeps the zero-copy ancillary payload path but bundles common plumbing – scheduler registration, frame queues, tasklet threads, and pacing – behind a single handle so that an app only needs to implement the `get_next_frame`/`notify_frame_done` (TX) or `notify_frame_ready` (RX) callbacks.
-
-- **TX pipeline (`st40p_tx_*`)** – wraps `st40_tx_create` under the hood while honoring flags such as redundant ports, user timestamps, and external frame ownership. The helper also surfaces the UDW/metadata buffers used by ancillary producers.
-- **RX pipeline (`st40p_rx_*`)** – converts incoming ancillary packets into frame callbacks and optionally dumps metadata/UDW buffers directly to user memory.
-
-Reference code lives in `app/sample/tx_st40_pipeline_sample.c` and `app/sample/rx_st40_pipeline_sample.c`. These samples demonstrate how to:
-
-1. Parse the shared sample CLI (`sample_util.c`) to configure multicast, payload-type, and redundant ports.
-2. Register pipeline callbacks that acquire frames, populate UDW/metas, and release them back to MTL.
-3. Enable optional debug paths such as on-disk metadata dumps or user pacing hooks.
-
-Use the pipeline helpers whenever you need to stand up ancillary-only senders/receivers quickly without wiring the lower-level ST40 transport APIs manually.
-
 ## 7. Misc
 
 ### 7.1. Logging

doc/run.md

@@ -364,80 +364,6 @@ packet egresses from the sender.
 --force_numa <id>                    : debug option, force the NIC port numa id instead of reading from PCIE topology
 ```
 
-### 5.4. ST40 pipeline samples
-
-The repository ships standalone ancillary-only apps that exercise the experimental ST40 pipeline helpers. They live under `app/sample/` and are built automatically by `./build.sh`:
-
-- `build/app/TxSt40PipelineSample`
-- `build/app/RxSt40PipelineSample`
-
-Both apps reuse `sample_util.c`, so the CLI syntax mirrors the other sample binaries (`--p_port`, `--udp_port`, `--sessions_cnt`, `--multi_inc_addr`, etc.).
-
-#### 5.4.1. Tx sample workflow
-
-1. Build the binaries and (optionally) prepare a deterministic ANC payload. Any binary file works; the sample loops the buffer forever. For a quick test:
-     ```bash
-     head -c 4096 /dev/urandom > /tmp/anc_payload.bin
-     ```
-2. Launch the TX sample (single-port multicast example shown). The sample pins the UDW payload to the maximum user buffer returned by `st40p_tx_max_udw_buff_size()`—currently 255 bytes—so every frame fits inside one RFC 8331 RTP packet. Larger input files are chunked per frame automatically.
-     ```bash
-     ./build/app/TxSt40PipelineSample \
-             --p_port 0000:af:01.0 \
-             --p_tx_ip 239.1.1.1 \
-             --p_sip 192.168.96.2 \
-             --udp_port 20000 \
-             --payload_type 40 \
-             --sessions_cnt 1 \
-             --tx_url /tmp/anc_payload.bin \
-             --ext_frame
-     ```
-
-Key behaviors:
-
-- `--ext_frame` keeps the sample in "user supplied frame" mode, matching how most applications integrate the pipeline API.
-- When `--multi_inc_addr` is present, each session increments the destination multicast address instead of the UDP port so you can fan out multiple streams on a single port.
-- Destination MAC overrides (`--p_tx_dst_mac`/`--r_tx_dst_mac`) work the same way as the other apps if you need explicit L2 addressing.
-
-#### 5.4.2. Rx sample workflow
-
-Run the RX sample with matching network parameters. Enabling `--rx_dump` writes one human-readable metadata report per frame to `--rx_url` (`foo.txt_0`, `_1`, … when multiple sessions are active):
-
-```bash
-./build/app/RxSt40PipelineSample \
-        --p_port 0000:af:01.0 \
-        --p_rx_ip 239.1.1.1 \
-        --p_sip 192.168.96.2 \
-        --udp_port 20000 \
-        --payload_type 40 \
-        --sessions_cnt 1 \
-        --rx_url /tmp/anc_dump.txt \
-        --rx_dump
-```
-
-The log stream shows per-frame metadata counts plus the number of user data words (UDW) that landed in the buffer. The dump file records DID/SDID/line info followed by the raw payload bytes so you can diff TX/RX easily.
-
-#### 5.4.3. Troubleshooting and USDT hooks
-
-- Use `--rx_dump`/`--tx_url` first to confirm that metadata makes the round trip.
-- When `st40_total_pkts` exceeds 1, it means the payload exceeded 255 bytes; trim the input file or lower `--sessions_cnt` on slow links.
-- Attach `bpftrace` to the USDT probes to see real-time pipeline events:
-    ```bash
-    sudo BPFTRACE_STRLEN=200 bpftrace -e '
-    usdt::st40p:rx_frame_available {
-        printf("%s rx%u: frame %u meta=%u\n", strftime("%H:%M:%S", nsecs), arg0, arg1, arg2);
-    }
-    usdt::st40p:rx_frame_put {
-        printf("%s rx%u: frame %u returned\n", strftime("%H:%M:%S", nsecs), arg0, arg1);
-    }
-    usdt::st40p:rx_frame_dump {
-        printf("%s rx%u: dump meta=%u bytes=%u -> %s\n",
-                     strftime("%H:%M:%S", nsecs), arg0, arg2, arg3, str(arg1));
-    }
-    ' -p $(pidof RxSt40PipelineSample)
-    ```
-- `usdt::sys:log_msg` mirrors the built-in status dump; attach to it when you want the sample to spit out scheduler stats without touching its log level.
-
-These executables are the quickest way to validate port configuration, payload sizing, and USDT instrumentation before embedding the ST40 pipeline API into your production app.
 
 ## 6. Tests
 

include/meson.build

@@ -4,7 +4,7 @@
 mtl_header_files = files('mtl_api.h', 'st_api.h', 'st_convert_api.h', 'st_convert_internal.h',
   'st_pipeline_api.h', 'st20_api.h', 'st30_api.h', 'st40_api.h', 'st41_api.h',
   'mudp_api.h', 'mudp_sockfd_api.h', 'mudp_sockfd_internal.h', 'mtl_lcore_shm_api.h',
-  'mtl_sch_api.h', 'st30_pipeline_api.h')
+  'mtl_sch_api.h', 'st30_pipeline_api.h', 'st40_pipeline_api.h')
 
 if is_windows
   mtl_header_files += files('mudp_win.h')
@@ -13,8 +13,7 @@ endif
 install_headers(mtl_header_files, subdir : meson.project_name())
 
 # experimental features
-mtl_experimental_header_files = files('experimental/st20_combined_api.h',
-  'experimental/st40_pipeline_api.h')
+mtl_experimental_header_files = files('experimental/st20_combined_api.h')
 
 experimental_header_install_path = meson.project_name() + '/experimental'
 

include/experimental/st40_pipeline_api.h include/st40_pipeline_api.hinclude/experimental/st40_pipeline_api.h renamed to include/st40_pipeline_api.h

@@ -3,6 +3,4 @@
 
 sources += files(
 	'st20_redundant_combined_rx.c',
-	'st40_pipeline_tx.c',
-	'st40_pipeline_rx.c',
 )