Tracker service v0.2.0: mqtt support (#923)

Co-authored-by: Tomasz Dorau <tomasz.dorau@intel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: scenescapecicd <sys_scenescape_ci@intel.com>
Co-authored-by: Lukasz Talarczyk <lukasz.talarczyk@intel.com>
Co-authored-by: Dmytro Yermolenko <dmytro.yermolenko@intel.com>

Author: 6 people

.github/resources/.prettierignore

@@ -5,3 +5,10 @@
 **/kubernetes/**/*.yaml
 **/.reuse/templates/template.jinja2
 .venv
+
+# Build directories (CMake/Conan generated)
+**/build/
+**/build-*/
+**/CMakeUserPresets.json
+**/CMakePresets.json
+**/CMakeFiles/

sample_data/docker-compose-dl-streamer-example.yml

@@ -441,8 +441,25 @@ services:
         aliases:
           - tracker.scenescape.intel.com
     user: "10001:10001"
+    depends_on:
+      broker:
+        condition: service_started
     environment:
-      - LOG_LEVEL=${TRACKER_LOG_LEVEL:-info}
+      - TRACKER_LOG_LEVEL=info
+      - TRACKER_MQTT_HOST=broker.scenescape.intel.com
+      - TRACKER_MQTT_PORT=1883
+      - TRACKER_MQTT_INSECURE=false
+      - TRACKER_MQTT_TLS_CA_CERT=/run/secrets/certs/scenescape-ca.pem
+      - TRACKER_MQTT_TLS_VERIFY_SERVER=true
+      # Override host proxy settings - Paho MQTT dont respect no_proxy var, so as a WA
+      # tracker code detects empty vars and unsets them (see mqtt_client.cpp clearEmptyProxyVars)
+      - http_proxy=
+      - https_proxy=
+      - HTTP_PROXY=
+      - HTTPS_PROXY=
+    secrets:
+      - source: root-cert
+        target: certs/scenescape-ca.pem
     restart: always
     mem_limit: ${TRACKER_MEM_LIMIT:-512m}
     # Scale: ~1 CPU per 100 tracked objects. Increase TRACKER_CPUS for larger deployments.

tracker/CMakeLists.txt

@@ -44,9 +44,9 @@ endif()
 # Find Conan-installed packages
 find_package(quill REQUIRED)
 find_package(CLI11 REQUIRED)
-find_package(simdjson REQUIRED)
 find_package(httplib REQUIRED)
 find_package(RapidJSON REQUIRED)
+find_package(PahoMqttCpp REQUIRED)
 
 #####################################################################
 # RobotVision integration
@@ -75,6 +75,9 @@ set(PROJECT_SOURCE_LIST
   ${CMAKE_CURRENT_SOURCE_DIR}/src/config_loader.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/src/healthcheck_server.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/src/healthcheck_command.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/mqtt_client.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/message_handler.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/src/proxy_utils.cpp
 )
 
 add_executable(${PROJECT_NAME}
@@ -105,9 +108,9 @@ target_link_libraries(${PROJECT_NAME}
     RobotVision
     quill::quill
     CLI11::CLI11
-    simdjson::simdjson
     httplib::httplib
     rapidjson
+    PahoMqttCpp::paho-mqttpp3-static
 )
 
 #####################################################################

tracker/Dockerfile

@@ -58,10 +58,11 @@ ARG GIT_COMMIT=unknown
 RUN GIT_COMMIT=${GIT_COMMIT} cmake --build build -j
 
 # Collect all runtime library dependencies for the tracker binary
-RUN mkdir -p /scenescape/runtime-libs
 WORKDIR /scenescape/runtime-libs
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN . /scenescape/build/conanrun.sh && ldd /scenescape/build/tracker \
+# hadolint ignore=SC1091
+RUN mkdir -p /scenescape/runtime-libs \
+    && . /scenescape/build/conanrun.sh && ldd /scenescape/build/tracker \
     | awk '{print $3}' \
     | grep '^/' \
     | xargs -I '{}' cp -v '{}' .

tracker/Makefile

@@ -28,7 +28,8 @@ SRC_FILES := $(wildcard src/*.cpp) $(wildcard inc/*.hpp) $(wildcard test/unit/*.
         build build-debug build-relwithdebinfo run run-debug profile flamegraph \
         test-unit test-unit-coverage test-service clean install-tools install-deps \
         run-image run-image-debug format-cpp lint-cpp format-python lint-python install-hooks \
-        lint-dockerfile lint-trivy lint-all coverage-html coverage-xml coverage-report
+        lint-dockerfile lint-trivy lint-all coverage-html coverage-xml coverage-report \
+        mqtt-subscribe mqtt-publish broker-start broker-stop generate-certs
 
 all: build
 
@@ -81,6 +82,7 @@ install-hooks:
 	@echo "Installing git pre-commit hook..."
 	@mkdir -p ../.git/hooks
 	@echo '#!/bin/bash' > ../.git/hooks/pre-commit
+	@echo 'set -e' >> ../.git/hooks/pre-commit
 	@echo 'make prettier-check' >> ../.git/hooks/pre-commit
 	@echo 'cd tracker && make lint-cpp && make lint-python && make lint-dockerfile' >> ../.git/hooks/pre-commit
 	@chmod +x ../.git/hooks/pre-commit
@@ -173,11 +175,20 @@ build-relwithdebinfo: dependencies-relwithdebinfo
 # Schema and config paths for local development
 TRACKER_SCHEMA_PATH ?= $(CURDIR)/schema/config.schema.json
 TRACKER_CONFIG_PATH ?= $(CURDIR)/config/tracker.json
+
+# MQTT broker connection (auto-detect from test broker, fallback to 1883)
+export TRACKER_MQTT_HOST ?= localhost
+export TRACKER_MQTT_PORT ?= $(shell cd test/service && docker compose port broker 1883 2>/dev/null | cut -d: -f2 || echo 1883)
+
+# Export empty proxy vars - Paho MQTT library fails with proxy env vars,
+# tracker code detects empty vars and unsets them (see mqtt_client.cpp)
 run: build
-	. build/conanrun.sh && ./build/$(NAME) --config $(TRACKER_CONFIG_PATH) --schema $(TRACKER_SCHEMA_PATH)
+	export http_proxy= https_proxy= HTTP_PROXY= HTTPS_PROXY= && \
+		. build/conanrun.sh && ./build/$(NAME) --config $(TRACKER_CONFIG_PATH) --schema $(TRACKER_SCHEMA_PATH)
 
 run-debug: build-debug
-	. build-debug/conanrun.sh && ./build-debug/$(NAME) --config $(TRACKER_CONFIG_PATH) --schema $(TRACKER_SCHEMA_PATH)
+	export http_proxy= https_proxy= HTTP_PROXY= HTTPS_PROXY= && \
+		. build-debug/conanrun.sh && ./build-debug/$(NAME) --config $(TRACKER_CONFIG_PATH) --schema $(TRACKER_SCHEMA_PATH)
 
 # Profile with perf using optimized build with debug symbols.
 # Requires perf permissions. If you see "Error: Failure to open event", run:
@@ -238,6 +249,7 @@ coverage-html:
 		'*/external/*' \
 		'*/third_party/*' \
 		'*/src/main.cpp' \
+		'*/src/mqtt_client.cpp' \
 		--output-file build-debug/coverage/coverage-filtered.info \
 		--rc branch_coverage=1 \
 		--ignore-errors unused \
@@ -267,6 +279,7 @@ coverage-xml:
 		--exclude '.*external.*' \
 		--exclude '.*third_party.*' \
 		--exclude '.*main\.cpp' \
+		--exclude '.*mqtt_client\.cpp' \
 		--xml-pretty \
 		--xml build-debug/coverage/coverage.xml \
 		--exclude-unreachable-branches \
@@ -291,6 +304,7 @@ coverage-report:
 		--exclude '.*external.*' \
 		--exclude '.*third_party.*' \
 		--exclude '.*main\.cpp' \
+		--exclude '.*mqtt_client\.cpp' \
 		--exclude-unreachable-branches \
 		--print-summary \
 		--fail-under-line 90.0 \
@@ -305,7 +319,8 @@ test-service: build-image
 	@test/service/.venv/bin/pip install -q --upgrade pip
 	@test/service/.venv/bin/pip install -q -r test/service/requirements.txt
 	@echo "Running service tests..."
-	cd test/service && .venv/bin/pytest -v --tb=short
+	unset TRACKER_MQTT_PORT TRACKER_MQTT_HOST TRACKER_MQTT_INSECURE && \
+		cd test/service && .venv/bin/pytest -v --tb=short
 
 #####################################################################
 # Docker build targets
@@ -324,8 +339,11 @@ build-image-debug:
 build-image-relwithdebinfo:
 	$(MAKE) build-image TARGET=runtime BUILD_TYPE=RelWithDebInfo IMAGE=scenescape-tracker-relwithdebinfo
 
+# Use host network to reach broker on localhost; empty proxy vars for Paho workaround
 run-image: build-image
-	docker run --rm -it $(IMAGE):$(VERSION)
+	docker run --rm -it --network=host \
+		-e http_proxy= -e https_proxy= -e HTTP_PROXY= -e HTTPS_PROXY= \
+		$(IMAGE):$(VERSION)
 
 run-image-debug: build-image-debug
 	-docker stop tracker-debug 2>/dev/null || true
@@ -339,12 +357,58 @@ stop-image-debug:
 	-docker stop tracker-debug 2>/dev/null || true
 	-docker rm tracker-debug 2>/dev/null || true
 
+#####################################################################
+# MQTT Broker Management
+#####################################################################
+
+# Certificate paths for broker TLS
+CERTS_DIR := $(CURDIR)/test/service/.certs
+ENV_FILE := $(CURDIR)/test/service/.env
+
+generate-certs:
+	@if [ ! -f "$(CERTS_DIR)/ca.crt" ]; then \
+		echo "Generating TLS certificates..."; \
+		python3 -m venv test/service/.venv 2>/dev/null || true; \
+		test/service/.venv/bin/pip install -q cryptography; \
+		cd test/service && .venv/bin/python3 -m utils.generate_certs \
+			$(CERTS_DIR) --env-file $(ENV_FILE); \
+	else \
+		echo "✓ Certificates already exist in $(CERTS_DIR)"; \
+	fi
+
+broker-start: generate-certs
+	@cd test/service && docker compose up -d broker
+	@sleep 1
+	@echo "✓ Broker running on port $$(cd test/service && docker compose port broker 1883 | cut -d: -f2)"
+
+broker-stop:
+	@cd test/service && docker compose down
+	@echo "✓ Broker stopped"
+
+#####################################################################
+# MQTT Manual Testing
+#####################################################################
+
+MQTT_DETECTION_TOPIC := scenescape/data/camera/test-cam
+MQTT_TRACK_TOPIC := scenescape/data/scene/\#
+
+mqtt-subscribe:
+	@echo "Subscribing to tracks on $(TRACKER_MQTT_HOST):$(TRACKER_MQTT_PORT)..."
+	mosquitto_sub -h $(TRACKER_MQTT_HOST) -p $(TRACKER_MQTT_PORT) -t '$(MQTT_TRACK_TOPIC)' -v
+
+mqtt-publish:
+	@echo "Publishing detection to $(TRACKER_MQTT_HOST):$(TRACKER_MQTT_PORT)..."
+	mosquitto_pub -h $(TRACKER_MQTT_HOST) -p $(TRACKER_MQTT_PORT) -t '$(MQTT_DETECTION_TOPIC)' \
+		-m '{"id":"test-cam","timestamp":"$(shell date -u +%Y-%m-%dT%H:%M:%S.000Z)","objects":{"person":[{"bounding_box_px":{"x":100,"y":50,"width":80,"height":200}}]}}'
+	@echo "✓ Detection published"
+
 #####################################################################
 # Clean targets
 #####################################################################
 
 # Override common.mk clean to also remove local build directories
 clean:
-	rm -rf build* test/service/.venv .pytest_cache
+	-@cd test/service && docker compose down 2>/dev/null || true
+	rm -rf build* test/service/.venv test/service/.certs test/service/.env .pytest_cache
 	-docker rmi $(IMAGE):$(VERSION) $(IMAGE):latest 2>/dev/null || true
 	-rm -f $(BUILD_DIR)/$(IMAGE)-*deps.txt $(LOG_FILE) 2>/dev/null || true

tracker/README.md

@@ -24,6 +24,9 @@ make install-tools
 # Coverage tools (optional, for local coverage reports)
 pipx install gcovr
 sudo apt-get install -y lcov
+
+# MQTT client tools (optional, for manual testing)
+sudo apt-get install -y mosquitto-clients
 ```
 
 #### Build

tracker/conanfile.txt

@@ -4,12 +4,12 @@
 [requires]
 quill/11.0.2
 cli11/2.6.0
-simdjson/4.2.2
 cpp-httplib/0.28.0
 rapidjson/cci.20230929
 gtest/1.17.0
 opencv/4.12.0
 eigen/3.4.0
+paho-mqtt-cpp/1.5.3
 
 [generators]
 CMakeDeps

tracker/inc/config_loader.hpp

@@ -4,24 +4,95 @@
 #pragma once
 
 #include <filesystem>
+#include <optional>
 #include <string>
 
 namespace tracker {
 
+/**
+ * @brief TLS certificate settings for secure connections.
+ */
+struct TlsConfig {
+    std::string ca_cert_path;
+    std::string client_cert_path;
+    std::string client_key_path;
+    bool verify_server = true;
+};
+
+/**
+ * @brief MQTT broker connection settings.
+ */
+struct MqttConfig {
+    std::string host;
+    int port;
+    bool insecure = false;
+    std::optional<TlsConfig> tls;
+};
+
+/**
+ * @brief Health check HTTP server settings.
+ */
+struct HealthcheckConfig {
+    int port = 8080;
+};
+
+/**
+ * @brief Tracker service settings.
+ */
+struct TrackerConfig {
+    HealthcheckConfig healthcheck;
+    bool schema_validation = true;
+};
+
+/**
+ * @brief External service connections.
+ */
+struct InfrastructureConfig {
+    MqttConfig mqtt;
+    TrackerConfig tracker;
+};
+
+/**
+ * @brief Logging configuration.
+ */
+struct LoggingConfig {
+    std::string level = "info";
+};
+
+/**
+ * @brief Observability settings.
+ */
+struct ObservabilityConfig {
+    LoggingConfig logging;
+};
+
 /**
  * @brief Service configuration loaded from JSON config file.
  *
  * Values can be overridden by environment variables with TRACKER_ prefix.
  */
 struct ServiceConfig {
-    std::string log_level;
-    int healthcheck_port;
+    InfrastructureConfig infrastructure;
+    ObservabilityConfig observability;
 };
 
 /// JSON Pointer paths (RFC6901) for extracting ServiceConfig values
 namespace json {
-constexpr char LOG_LEVEL[] = "/observability/logging/level";
-constexpr char HEALTHCHECK_PORT[] = "/infrastructure/tracker/healthcheck/port";
+constexpr char OBSERVABILITY_LOGGING_LEVEL[] = "/observability/logging/level";
+constexpr char INFRASTRUCTURE_TRACKER_HEALTHCHECK_PORT[] =
+    "/infrastructure/tracker/healthcheck/port";
+constexpr char INFRASTRUCTURE_TRACKER_SCHEMA_VALIDATION[] =
+    "/infrastructure/tracker/schema_validation";
+constexpr char INFRASTRUCTURE_MQTT_HOST[] = "/infrastructure/mqtt/host";
+constexpr char INFRASTRUCTURE_MQTT_PORT[] = "/infrastructure/mqtt/port";
+constexpr char INFRASTRUCTURE_MQTT_INSECURE[] = "/infrastructure/mqtt/insecure";
+constexpr char INFRASTRUCTURE_MQTT_TLS[] = "/infrastructure/mqtt/tls";
+constexpr char INFRASTRUCTURE_MQTT_TLS_CA_CERT_PATH[] = "/infrastructure/mqtt/tls/ca_cert_path";
+constexpr char INFRASTRUCTURE_MQTT_TLS_CLIENT_CERT_PATH[] =
+    "/infrastructure/mqtt/tls/client_cert_path";
+constexpr char INFRASTRUCTURE_MQTT_TLS_CLIENT_KEY_PATH[] =
+    "/infrastructure/mqtt/tls/client_key_path";
+constexpr char INFRASTRUCTURE_MQTT_TLS_VERIFY_SERVER[] = "/infrastructure/mqtt/tls/verify_server";
 } // namespace json
 
 /**

tracker/inc/env_vars.hpp

@@ -18,4 +18,29 @@ constexpr const char* LOG_LEVEL = "TRACKER_LOG_LEVEL";
 /// Environment variable for overriding healthcheck server port (1024-65535)
 constexpr const char* HEALTHCHECK_PORT = "TRACKER_HEALTHCHECK_PORT";
 
+/// Environment variable for overriding MQTT broker host
+constexpr const char* MQTT_HOST = "TRACKER_MQTT_HOST";
+
+/// Environment variable for overriding MQTT broker port (1-65535)
+constexpr const char* MQTT_PORT = "TRACKER_MQTT_PORT";
+
+/// Environment variable for overriding MQTT insecure mode (true/false)
+constexpr const char* MQTT_INSECURE = "TRACKER_MQTT_INSECURE";
+
+/// Environment variable for overriding MQTT TLS CA certificate path
+constexpr const char* MQTT_TLS_CA_CERT = "TRACKER_MQTT_TLS_CA_CERT";
+
+/// Environment variable for overriding MQTT TLS client certificate path
+constexpr const char* MQTT_TLS_CLIENT_CERT = "TRACKER_MQTT_TLS_CLIENT_CERT";
+
+/// Environment variable for overriding MQTT TLS client key path
+constexpr const char* MQTT_TLS_CLIENT_KEY = "TRACKER_MQTT_TLS_CLIENT_KEY";
+
+/// Environment variable for overriding MQTT TLS server verification (true/false)
+constexpr const char* MQTT_TLS_VERIFY_SERVER = "TRACKER_MQTT_TLS_VERIFY_SERVER";
+
+/// Environment variable for enabling/disabling JSON schema validation for MQTT messages
+/// (true/false)
+constexpr const char* MQTT_SCHEMA_VALIDATION = "TRACKER_MQTT_SCHEMA_VALIDATION";
+
 } // namespace tracker::env

tracker/inc/logger.hpp

@@ -206,6 +206,9 @@ class Logger {
     static void log_warn(const LogEntry& entry);
     static void log_error(const LogEntry& entry);
 
+    // Check if debug logging is enabled (for conditional expensive computations)
+    [[nodiscard]] static bool should_log_debug();
+
 private:
     Logger() = default;
     ~Logger() = default;