Skip to content

Commit 3109d86

Browse files
sbelhaiksbelhaik
authored
ITEP-17695 Test performance for SceneScape with DLStreamer (#171)
Update `tests/Makefile.perf` to execute `scene-performance-full` target with DLStreamer when `DLS=1` Add mqtt-recorder service to docker compose to start it along side with SceneScape services during test Update image used by mqtt-recorder service Update tests runners to support executing performance test with both DLStreamer and Percebro. This allow us to compare results and identify any regression. Fix broken Percebro test, for result comparaison JIRA: [ITEP-17695](https://jira.devtools.intel.com/browse/ITEP-17695) ## 🧪 Testing Scenarios For DLStreamer `make -C tests scene-performance-full DLS=1` Output: ``` Executing: NEX-T10488 Time per msg: 1.171 ms, RATE: 854.040 messages/s, 0 FELL BEHIND messages observed. Reached at least 40 mps! NEX-T10488: PASS MAKE_TARGET: scene-performance-full ``` For Percebro `make -C tests scene-performance-full` Output: ``` mqtt_recorder Waiting for test to start.. 170 mqtt_recorder Waiting for test to start.. 175 mqtt_recorder Waiting for test to start.. 180 --- NEX-T10488: FAIL ``` `make -C tests inference-performance` Output: ``` Achieved 45.375 fps per camera, 90.75 total, threads used: Test CPU load: 1.94 Reached at least 15.0 fps! NEX-T10412: PASS + echo 'MAKE_TARGET: inference-performance' + tee -ia test_data/perf/inference-performance-2025-07-17-09:10:17.log MAKE_TARGET: inference-performance + echo END TEST inference-performance END TEST inference-performance ```
1 parent a8cd18b commit 3109d86

File tree

10 files changed

+579
-42
lines changed

10 files changed

+579
-42
lines changed

‎Makefile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,20 +281,29 @@ setup_tests: build-images
281281

282282
.PHONY: run_tests
283283
run_tests: setup_tests
284+
@if [ "$${DLS}" = "1" ]; then \
285+
$(MAKE) $(DLSTREAMER_SAMPLE_VIDEOS); \
286+
fi
284287
@echo "Running tests..."
285288
@DLS_ARG=""; [ "$${DLS}" = "1" ] && DLS_ARG="DLS=1"; \
286289
$(MAKE) --trace -C tests -j 1 $${DLS_ARG} || (echo "Tests failed" && exit 1)
287290
@echo "DONE ==> Running tests"
288291

289292
.PHONY: run_performance_tests
290293
run_performance_tests:
294+
@if [ "$${DLS}" = "1" ]; then \
295+
$(MAKE) $(DLSTREAMER_SAMPLE_VIDEOS); \
296+
fi
291297
@echo "Running performance tests..."
292298
@DLS_ARG=""; [ "$${DLS}" = "1" ] && DLS_ARG="DLS=1"; \
293299
$(MAKE) -C tests performance_tests -j 1 SUPASS=$(SUPASS) $${DLS_ARG} || (echo "Performance tests failed" && exit 1)
294300
@echo "DONE ==> Running performance tests"
295301

296302
.PHONY: run_stability_tests
297303
run_stability_tests:
304+
@if [ "$${DLS}" = "1" ]; then \
305+
$(MAKE) $(DLSTREAMER_SAMPLE_VIDEOS); \
306+
fi
298307
@echo "Running stability tests..."
299308
@DLS_ARG=""; [ "$${DLS}" = "1" ] && DLS_ARG="DLS=1"
300309
ifeq ($(BUILD_TYPE),DAILY)
@@ -306,6 +315,9 @@ endif
306315

307316
.PHONY: run_basic_acceptance_tests
308317
run_basic_acceptance_tests: setup_tests
318+
@if [ "$${DLS}" = "1" ]; then \
319+
$(MAKE) $(DLSTREAMER_SAMPLE_VIDEOS); \
320+
fi
309321
@echo "Running basic acceptance tests..."
310322
@DLS_ARG=""; [ "$${DLS}" = "1" ] && DLS_ARG="DLS=1"; \
311323
$(MAKE) --trace -C tests basic-acceptance-tests -j 1 SUPASS=$(SUPASS) $${DLS_ARG} || (echo "Basic acceptance tests failed" && exit 1)

‎dlstreamer-pipeline-server/perf-config.json

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
{
2+
"config": {
3+
"logging": {
4+
"C_LOG_LEVEL": "INFO",
5+
"PY_LOG_LEVEL": "INFO"
6+
},
7+
"pipelines": [
8+
{
9+
"name": "apriltag-cam1",
10+
"source": "gstreamer",
11+
"pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam1.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true",
12+
"auto_start": true,
13+
"parameters": {
14+
"type": "object",
15+
"properties": {
16+
"ntp_config": {
17+
"element": {
18+
"name": "timesync",
19+
"property": "kwarg",
20+
"format": "json"
21+
},
22+
"type": "object",
23+
"properties": {
24+
"ntpServer": {
25+
"type": "string"
26+
}
27+
}
28+
},
29+
"camera_config": {
30+
"element": {
31+
"name": "datapublisher",
32+
"property": "kwarg",
33+
"format": "json"
34+
},
35+
"type": "object",
36+
"properties": {
37+
"intrinsics": {
38+
"type": "array",
39+
"items": {
40+
"type": "number",
41+
"description": "Camera intrinsics as [diagonal_fov] or [horizontal_fov, vertical_fov] or [fx,fy,cx,cy] "
42+
}
43+
},
44+
"cameraid": {
45+
"type": "string"
46+
},
47+
"metadatagenpolicy": {
48+
"type": "string",
49+
"description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy"
50+
},
51+
"publish_frame": {
52+
"type": "boolean",
53+
"description": "Publish frame to mqtt"
54+
}
55+
}
56+
}
57+
}
58+
},
59+
"payload": {
60+
"destination": {
61+
"frame": {
62+
"type": "rtsp",
63+
"path": "camera1"
64+
}
65+
},
66+
"parameters": {
67+
"ntp_config": {
68+
"ntpServer": "ntpserv"
69+
},
70+
"camera_config": {
71+
"cameraid": "camera1",
72+
"metadatagenpolicy": "detectionPolicy"
73+
}
74+
}
75+
}
76+
},
77+
{
78+
"name": "apriltag-cam2",
79+
"source": "gstreamer",
80+
"pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam2.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true",
81+
"auto_start": true,
82+
"parameters": {
83+
"type": "object",
84+
"properties": {
85+
"ntp_config": {
86+
"element": {
87+
"name": "timesync",
88+
"property": "kwarg",
89+
"format": "json"
90+
},
91+
"type": "object",
92+
"properties": {
93+
"ntpServer": {
94+
"type": "string"
95+
}
96+
}
97+
},
98+
"camera_config": {
99+
"element": {
100+
"name": "datapublisher",
101+
"property": "kwarg",
102+
"format": "json"
103+
},
104+
"type": "object",
105+
"properties": {
106+
"intrinsics": {
107+
"type": "array",
108+
"items": {
109+
"type": "number",
110+
"description": "Camera intrinsics as [diagonal_fov] or [horizontal_fov, vertical_fov] or [fx,fy,cx,cy] "
111+
}
112+
},
113+
"cameraid": {
114+
"type": "string"
115+
},
116+
"metadatagenpolicy": {
117+
"type": "string",
118+
"description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy"
119+
},
120+
"publish_frame": {
121+
"type": "boolean",
122+
"description": "Publish frame to mqtt"
123+
}
124+
}
125+
}
126+
}
127+
},
128+
"payload": {
129+
"destination": {
130+
"frame": {
131+
"type": "rtsp",
132+
"path": "camera2"
133+
}
134+
},
135+
"parameters": {
136+
"ntp_config": {
137+
"ntpServer": "ntpserv"
138+
},
139+
"camera_config": {
140+
"cameraid": "camera2",
141+
"metadatagenpolicy": "detectionPolicy"
142+
}
143+
}
144+
}
145+
},
146+
{
147+
"name": "apriltag-cam3",
148+
"source": "gstreamer",
149+
"pipeline": "multifilesrc loop=TRUE location=/home/pipeline-server/videos/apriltag-cam3.ts name=source ! decodebin ! videoconvert ! video/x-raw,format=BGR ! gvapython class=PostDecodeTimestampCapture function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=timesync ! gvadetect model=/home/pipeline-server/models/object_detection/person/FP32/person-detection-retail-0013.xml model-proc=/home/pipeline-server/models/object_detection/person/person-detection-retail-0013.json ! gvametaconvert add-tensor-data=true name=metaconvert ! gvapython class=PostInferenceDataPublish function=processFrame module=/home/pipeline-server/user_scripts/gvapython/sscape/sscape_adapter.py name=datapublisher ! gvametapublish name=destination ! appsink sync=true",
150+
"auto_start": true,
151+
"parameters": {
152+
"type": "object",
153+
"properties": {
154+
"ntp_config": {
155+
"element": {
156+
"name": "timesync",
157+
"property": "kwarg",
158+
"format": "json"
159+
},
160+
"type": "object",
161+
"properties": {
162+
"ntpServer": {
163+
"type": "string"
164+
}
165+
}
166+
},
167+
"camera_config": {
168+
"element": {
169+
"name": "datapublisher",
170+
"property": "kwarg",
171+
"format": "json"
172+
},
173+
"type": "object",
174+
"properties": {
175+
"intrinsics": {
176+
"type": "array",
177+
"items": {
178+
"type": "number",
179+
"description": "Camera intrinsics as [diagonal_fov] or [horizontal_fov, vertical_fov] or [fx,fy,cx,cy] "
180+
}
181+
},
182+
"cameraid": {
183+
"type": "string"
184+
},
185+
"metadatagenpolicy": {
186+
"type": "string",
187+
"description": "Meta data generation policy, one of detectionPolicy(default),reidPolicy,classificationPolicy"
188+
},
189+
"publish_frame": {
190+
"type": "boolean",
191+
"description": "Publish frame to mqtt"
192+
}
193+
}
194+
}
195+
}
196+
},
197+
"payload": {
198+
"destination": {
199+
"frame": {
200+
"type": "rtsp",
201+
"path": "camera3"
202+
}
203+
},
204+
"parameters": {
205+
"ntp_config": {
206+
"ntpServer": "ntpserv"
207+
},
208+
"camera_config": {
209+
"cameraid": "camera3",
210+
"metadatagenpolicy": "detectionPolicy"
211+
}
212+
}
213+
}
214+
}
215+
]
216+
}
217+
}

‎sample_data/docker-compose-dl-streamer-example.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ services:
5353
scenescape:
5454
aliases:
5555
- broker.scenescape.intel.com
56-
user: "${UID}:${GID}"
56+
user: "${UID:-1000}:${GID:-1000}"
5757

5858
pgserver:
5959
image: scenescape-manager:${VERSION:-latest}

0 commit comments

Comments
 (0)