Skip to content

Commit 585f137

Browse files
sirknightjsirknightj
authored
kvssink CI enhancements! (#1226)
* Repair unit tests and CI * Add runtime string checks and additional unit tests * Add valgrind * Add reset custom allocators and test timeouts * Adjust test timeouts * Add windows debug dump dir check * Add mac and linux as well * Address comments * Address comments
1 parent b23b129 commit 585f137

File tree

4 files changed

+460
-28
lines changed

4 files changed

+460
-28
lines changed

‎.github/workflows/kvssink.yml‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
name: kvssink checks
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
- master
8+
pull_request:
9+
branches:
10+
- develop
11+
- master
12+
13+
jobs:
14+
unit-tests:
15+
runs-on: ubuntu-latest
16+
timeout-minutes: 30
17+
permissions:
18+
id-token: write
19+
contents: read
20+
strategy:
21+
matrix:
22+
include:
23+
- os: Ubuntu 22.04
24+
image: public.ecr.aws/ubuntu/ubuntu:22.04_stable
25+
- os: Ubuntu 20.04
26+
image: public.ecr.aws/ubuntu/ubuntu:20.04_stable
27+
fail-fast: false
28+
29+
name: kvssink unit tests on ${{ matrix.os }}
30+
steps:
31+
- name: Checkout repository
32+
uses: actions/checkout@v4
33+
34+
- name: Install dependencies
35+
run: |
36+
sudo apt-get update
37+
sudo apt-get install -y automake build-essential cmake git \
38+
gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-bad \
39+
gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly \
40+
gstreamer1.0-tools \
41+
libcurl4-openssl-dev libgstreamer1.0-dev \
42+
libgstreamer-plugins-base1.0-dev liblog4cplus-dev \
43+
libssl-dev pkg-config valgrind
44+
45+
- name: Setup build directory
46+
run: |
47+
mkdir -p build
48+
cd build
49+
50+
- name: Build kvssink unit tests
51+
working-directory: ./build
52+
run: |
53+
cmake .. -DBUILD_GSTREAMER_PLUGIN=ON -DBUILD_TEST=ON -DCMAKE_BUILD_TYPE=Debug -DALIGNED_MEMORY_MODEL=ON
54+
make -j$(nproc)
55+
56+
- name: Configure AWS Credentials
57+
uses: aws-actions/configure-aws-credentials@v4
58+
with:
59+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
60+
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
61+
aws-region: ${{ secrets.AWS_REGION }}
62+
role-duration-seconds: 10800
63+
64+
- name: Run the unit tests
65+
working-directory: ./build
66+
run: |
67+
export GST_PLUGIN_PATH=`pwd`
68+
GST_DEBUG=4 ./tst/gstkvsplugintest
69+
70+
# - name: Run the unit tests with valgrind
71+
# working-directory: ./build
72+
# run: |
73+
# export GST_PLUGIN_PATH=`pwd`
74+
# valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes \
75+
# --verbose --log-file=valgrind-out.txt ./tst/gstkvsplugintest
76+
#
77+
# if grep -q "All heap blocks were freed -- no leaks are possible" valgrind-out.txt; then
78+
# echo "No memory leaks detected"
79+
# else
80+
# echo "Memory leaks detected. Review the valgrind output:"
81+
# cat valgrind-out.txt
82+
# exit 1
83+
# fi
84+
85+
mac-debug-dump-dir:
86+
runs-on: macos-latest
87+
env:
88+
AWS_KVS_LOG_LEVEL: 1
89+
permissions:
90+
id-token: write
91+
contents: read
92+
steps:
93+
- name: Clone repository
94+
uses: actions/checkout@v4
95+
- name: Install dependencies
96+
run: |
97+
brew install gstreamer log4cplus mkvtoolnix
98+
99+
- name: Build kvssink
100+
run: |
101+
mkdir build && cd build
102+
cmake .. -DBUILD_GSTREAMER_PLUGIN=ON -DBUILD_DEPENDENCIES=OFF
103+
make -j$(nproc)
104+
105+
- name: Configure AWS Credentials
106+
uses: aws-actions/configure-aws-credentials@v4
107+
with:
108+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
109+
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
110+
aws-region: ${{ secrets.AWS_REGION }}
111+
role-duration-seconds: 10800
112+
113+
- name: Run kvssink with dump dir
114+
env:
115+
GST_PLUGIN_PATH: ${{ github.workspace }}/build
116+
KVS_DEBUG_DUMP_DATA_FILE_DIR: ${{ github.workspace }}/build/debug_output
117+
working-directory: ./build
118+
run: |
119+
mkdir -p debug_output
120+
gst-launch-1.0 videotestsrc is-live=true num-buffers=450 \
121+
! video/x-raw,framerate=30/1,width=640,height=480 \
122+
! videoconvert ! x264enc tune=zerolatency key-int-max=45 \
123+
! h264parse \
124+
! kvssink stream-name="demo-stream"
125+
126+
- name: Verify MKV dump
127+
working-directory: ./build/debug_output
128+
run: |
129+
mkvfiles=(*.mkv)
130+
if [ ${#mkvfiles[@]} -eq 0 ]; then
131+
echo "No MKV files found in debug_output"
132+
exit 1
133+
fi
134+
135+
for file in "${mkvfiles[@]}"; do
136+
echo "Verifying $file with mkvinfo (verbose and hexdump):"
137+
mkvinfo -v -X "$file"
138+
done
139+
140+
linux-debug-dump-dir:
141+
runs-on: ubuntu-latest
142+
env:
143+
AWS_KVS_LOG_LEVEL: 1
144+
permissions:
145+
id-token: write
146+
contents: read
147+
strategy:
148+
matrix:
149+
include:
150+
- os: Ubuntu 22.04
151+
image: public.ecr.aws/ubuntu/ubuntu:22.04_stable
152+
- os: Ubuntu 20.04
153+
image: public.ecr.aws/ubuntu/ubuntu:20.04_stable
154+
fail-fast: false
155+
steps:
156+
- name: Checkout repository
157+
uses: actions/checkout@v4
158+
159+
- name: Install dependencies
160+
run: |
161+
sudo apt-get update
162+
sudo apt-get install -y automake build-essential cmake git \
163+
gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-bad \
164+
gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly \
165+
gstreamer1.0-tools \
166+
libcurl4-openssl-dev libgstreamer1.0-dev \
167+
libgstreamer-plugins-base1.0-dev liblog4cplus-dev \
168+
libssl-dev pkg-config mkvtoolnix
169+
170+
- name: Setup build directory
171+
run: |
172+
mkdir -p build
173+
cd build
174+
175+
- name: Build kvssink
176+
working-directory: ./build
177+
run: |
178+
cmake .. -DBUILD_GSTREAMER_PLUGIN=ON -DCMAKE_BUILD_TYPE=Debug -DALIGNED_MEMORY_MODEL=ON -DBUILD_DEPENDENCIES=OFF
179+
make -j$(nproc)
180+
181+
- name: Configure AWS Credentials
182+
uses: aws-actions/configure-aws-credentials@v4
183+
with:
184+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
185+
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
186+
aws-region: ${{ secrets.AWS_REGION }}
187+
role-duration-seconds: 10800
188+
189+
- name: Run kvssink with dump dir
190+
env:
191+
GST_PLUGIN_PATH: ${{ github.workspace }}/build
192+
KVS_DEBUG_DUMP_DATA_FILE_DIR: ${{ github.workspace }}/build/debug_output
193+
working-directory: ./build
194+
run: |
195+
mkdir -p debug_output
196+
gst-launch-1.0 videotestsrc is-live=true num-buffers=450 \
197+
! video/x-raw,framerate=30/1,width=640,height=480 \
198+
! videoconvert ! x264enc tune=zerolatency key-int-max=45 \
199+
! h264parse \
200+
! kvssink stream-name="demo-stream"
201+
202+
- name: Verify MKV dump
203+
working-directory: ./build/debug_output
204+
run: |
205+
mkvfiles=(*.mkv)
206+
if [ ${#mkvfiles[@]} -eq 0 ]; then
207+
echo "No MKV files found in debug_output"
208+
exit 1
209+
fi
210+
211+
for file in "${mkvfiles[@]}"; do
212+
echo "Verifying $file with mkvinfo (verbose and hexdump):"
213+
mkvinfo -v -X "$file"
214+
done
215+
216+
windows-debug-dump-dir:
217+
runs-on: windows-2022
218+
env:
219+
AWS_KVS_LOG_LEVEL: 1
220+
permissions:
221+
id-token: write
222+
contents: read
223+
steps:
224+
- name: Clone repository
225+
uses: actions/checkout@v4
226+
- name: Move repository
227+
run: |
228+
mkdir D:\producer
229+
Move-Item -Path "D:\a\amazon-kinesis-video-streams-producer-sdk-cpp\amazon-kinesis-video-streams-producer-sdk-cpp\*" -Destination "D:\producer"
230+
- name: Install dependencies
231+
run: |
232+
choco install nasm strawberryperl mkvtoolnix
233+
choco install gstreamer --version=1.22.8
234+
choco install gstreamer-devel --version=1.22.8
235+
- name: Build repository
236+
run: |
237+
$env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\producer\open-source\local\lib;D:\producer\open-source\local\bin'
238+
git config --system core.longpaths true
239+
cd D:\producer
240+
dir
241+
.github\build_windows.bat
242+
- name: Configure AWS Credentials
243+
uses: aws-actions/configure-aws-credentials@v4
244+
with:
245+
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
246+
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
247+
aws-region: ${{ secrets.AWS_REGION }}
248+
role-duration-seconds: 10800
249+
- name: Run kvssink with dump dir
250+
env:
251+
GST_PLUGIN_PATH: D:\producer\build\
252+
KVS_DEBUG_DUMP_DATA_FILE_DIR: D:\producer\build\debug_output
253+
working-directory: D:\producer\build\
254+
run: |
255+
$env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\producer\open-source\local\lib;D:\producer\open-source\local\bin;D:\gstreamer\1.0\msvc_x86_64\bin'
256+
257+
# Create the debug directory (equivalent to mkdir -p)
258+
New-Item -ItemType Directory -Path "D:\producer\build\debug_output" -Force
259+
260+
# Stream for 15 seconds (450 frames @ 30 fps)
261+
gst-launch-1.0.exe videotestsrc is-live=true num-buffers=450 ! video/x-raw,framerate=30/1,width=640,height=480 ! videoconvert ! x264enc tune=zerolatency key-int-max=45 ! h264parse ! kvssink stream-name="demo-stream"
262+
- name: Verify MKV dump
263+
working-directory: D:\producer\build
264+
run: |
265+
$env:Path += ";C:\Program Files\MKVToolNix"
266+
$mkvFiles = Get-ChildItem -Path "D:\producer\build\debug_output" -Filter *.mkv
267+
268+
if ($mkvFiles.Count -eq 0) {
269+
Write-Error "No MKV files found in D:\producer\build\debug_output"
270+
exit 1
271+
}
272+
273+
# Run mkvinfo on each MKV file
274+
foreach ($file in $mkvFiles) {
275+
Write-Output "Verifying $($file.FullName) with mkvinfo (verbose and hexdump):"
276+
mkvinfo.exe -v -X "$($file.FullName)"
277+
}

‎src/gstreamer/Util/KvsSinkUtil.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@
1717
#define IOT_CONNECTION_TIMEOUT "connection-timeout"
1818
#define IOT_COMPLETION_TIMEOUT "completion-timeout"
1919

20+
#define KVSSINK_THROW_IF_NULL(ptr) \
21+
do { \
22+
if ((ptr) == NULL) { \
23+
throw std::runtime_error(std::string(INTERNAL_CHECK_PREFIX) + " " #ptr " is unexpectedly NULL!"); \
24+
} \
25+
} while (0)
26+
2027
namespace kvs_sink_util{
2128

2229
gboolean gstructToMap(GstStructure *g_struct, std::map<std::string, std::string> *user_map);

‎src/gstreamer/gstkvssink.cpp‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_kvs_sink_debug);
137137

138138
#define MAX_GSTREAMER_MEDIA_TYPE_LEN 16
139139

140+
#define INTERNAL_CHECK_PREFIX "Assertion failed:"
141+
140142
namespace KvsSinkSignals {
141143
guint err_signal_id;
142144
guint ack_signal_id;
@@ -261,6 +263,9 @@ void closed(UINT64 custom_data, STREAM_HANDLE stream_handle, UPLOAD_HANDLE uploa
261263
void kinesis_video_producer_init(GstKvsSink *kvssink)
262264
{
263265
auto data = kvssink->data;
266+
267+
KVSSINK_THROW_IF_NULL(kvssink->stream_name);
268+
264269
unique_ptr<DeviceInfoProvider> device_info_provider(new KvsSinkDeviceInfoProvider(kvssink->storage_size,
265270
kvssink->stop_stream_timeout,
266271
kvssink->service_connection_timeout,
@@ -302,11 +307,11 @@ void kinesis_video_producer_init(GstKvsSink *kvssink)
302307
}
303308

304309
} else {
305-
access_key_str = string(kvssink->access_key);
306-
secret_key_str = string(kvssink->secret_key);
310+
access_key_str = kvssink->access_key ? string(kvssink->access_key) : "";
311+
secret_key_str = kvssink->secret_key ? string(kvssink->secret_key) : "";
307312
}
308313

309-
// Handle session token seperately, since this is optional with long term credentials
314+
// Handle session token separately, since this is optional with long term credentials
310315
if (0 == strcmp(kvssink->session_token, DEFAULT_SESSION_TOKEN)) {
311316
session_token_str = "";
312317
if (nullptr != (session_token = getenv(SESSION_TOKEN_ENV_VAR))) {
@@ -370,6 +375,9 @@ void kinesis_video_producer_init(GstKvsSink *kvssink)
370375
LOG_INFO("Getting URL from env for " << kvssink->stream_name);
371376
control_plane_uri_str = string(control_plane_uri);
372377
}
378+
379+
KVSSINK_THROW_IF_NULL(kvssink->user_agent);
380+
373381
LOG_INFO("User agent string: " << kvssink->user_agent);
374382
data->kinesis_video_producer = KinesisVideoProducer::createSync(std::move(device_info_provider),
375383
std::move(client_callback_provider),
@@ -423,6 +431,14 @@ void create_kinesis_video_stream(GstKvsSink *kvssink) {
423431
break;
424432
}
425433

434+
// StreamDefinition takes in C++ strings, check the gchar* for nullptr before constructing
435+
KVSSINK_THROW_IF_NULL(kvssink->stream_name);
436+
KVSSINK_THROW_IF_NULL(kvssink->content_type);
437+
KVSSINK_THROW_IF_NULL(kvssink->user_agent);
438+
KVSSINK_THROW_IF_NULL(kvssink->kms_key_id);
439+
KVSSINK_THROW_IF_NULL(kvssink->codec_id);
440+
KVSSINK_THROW_IF_NULL(kvssink->track_name);
441+
426442
unique_ptr<StreamDefinition> stream_definition(new StreamDefinition(kvssink->stream_name,
427443
hours(kvssink->retention_period_hours),
428444
p_stream_tags,
@@ -476,10 +492,16 @@ bool kinesis_video_stream_init(GstKvsSink *kvssink, string &err_msg) {
476492
create_kinesis_video_stream(kvssink);
477493
break;
478494
} catch (runtime_error &err) {
495+
ostringstream oss;
496+
oss << "Failed to create stream. Error: " << err.what();
497+
err_msg = oss.str();
498+
499+
// Don't retry if the error is an internal error
500+
if (STRNCMP(INTERNAL_CHECK_PREFIX, err.what(), STRLEN(INTERNAL_CHECK_PREFIX)) == 0) {
501+
return false;
502+
}
503+
479504
if (--retry_count == 0) {
480-
ostringstream oss;
481-
oss << "Failed to create stream. Error: " << err.what();
482-
err_msg = oss.str();
483505
ret = false;
484506
do_retry = false;
485507
} else {
@@ -1570,6 +1592,8 @@ init_track_data(GstKvsSink *kvssink) {
15701592

15711593
g_free(video_content_type);
15721594
g_free(audio_content_type);
1595+
1596+
KVSSINK_THROW_IF_NULL(kvssink->content_type);
15731597
}
15741598

15751599
static GstStateChangeReturn

0 commit comments

Comments
 (0)