Skip to content

Commit fe11ba0

Browse files
skyero-awsskyero-aws
authored
Add github workflow for KCL sample run testing (#290)
1 parent f5762cc commit fe11ba0

File tree

9 files changed

+281
-44
lines changed

9 files changed

+281
-44
lines changed

‎.github/dependabot.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@ updates:
99
directory: "/"
1010
open-pull-requests-limit: 4
1111
schedule:
12-
interval: "daily"
12+
interval: "weekly"
1313
ignore:
1414
- dependency-name: 'awssdk.*'

‎.github/resources/github_workflow.properties‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# The script that abides by the multi-language protocol. This script will
2+
# be executed by the MultiLangDaemon, which will communicate with this script
3+
# over STDIN and STDOUT according to the multi-language protocol.
4+
# Ensure the path to the executable is correct: dotnet <path-to-your-dll>/SampleConsumer.dll
5+
executableName = EXECUTABLE_NAME_PLACEHOLDER
6+
7+
# The name of an Amazon Kinesis stream to process.
8+
streamName = STREAM_NAME_PLACEHOLDER
9+
10+
# Used by the KCL as the name of this application. Will be used as the name
11+
# of an Amazon DynamoDB table which will store the lease and checkpoint
12+
# information for workers with this application name
13+
applicationName = APP_NAME_PLACEHOLDER
14+
15+
# Users can change the credentials provider the KCL will use to retrieve credentials.
16+
# Expected key name (case-sensitive):
17+
# AwsCredentialsProvider / AwsCredentialsProviderDynamoDB / AwsCredentialsProviderCloudWatch
18+
# The DefaultCredentialsProvider checks several other providers, which is
19+
# described here:
20+
# https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
21+
AwsCredentialsProvider = DefaultCredentialsProvider
22+
23+
# Appended to the user agent of the KCL. Does not impact the functionality of the
24+
# KCL in any other way.
25+
processingLanguage = C#
26+
27+
# Valid options at TRIM_HORIZON or LATEST.
28+
# See http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
29+
initialPositionInStream = TRIM_HORIZON
30+
31+
# The KCL defaults to us-east-1
32+
regionName = us-east-1
33+
34+
# Idle time between record reads in milliseconds.
35+
idleTimeBetweenReadsInMillis = 250

‎.github/scripts/clean_up_stream_table.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
# Delete stream
4+
if aws kinesis describe-stream --stream-name $STREAM_NAME &>/dev/null; then
5+
echo "Deleting stream $STREAM_NAME"
6+
for i in {1..10}; do
7+
aws kinesis delete-stream --stream-name $STREAM_NAME && break ||
8+
echo "Stream deletion failed, attempt $i/10. Retrying Stream deletion in $((i * 3))s" && sleep $((i * 3))
9+
done
10+
else
11+
echo "Stream $STREAM_NAME does not exist and does not need to be cleaned up"
12+
fi
13+
14+
# Delete table
15+
delete_table() {
16+
table_name=$1
17+
if aws dynamodb describe-table --table-name $table_name &>/dev/null; then
18+
echo "Deleting table $table_name"
19+
for i in {1..10}; do
20+
aws dynamodb delete-table --table-name $table_name && break ||
21+
echo "Table deletion failed, attempt $i/10. Retrying DynamoDB Table deletion in $((i * 3))s" && sleep $((i * 3))
22+
done
23+
else
24+
echo "Table $table_name does not exist and does not need to be cleaned up"
25+
fi
26+
}
27+
28+
# Delete all tables
29+
for SUFFIX in "" "-CoordinatorState" "-WorkerMetricStats"; do
30+
delete_table "$APP_NAME$SUFFIX"
31+
done

‎.github/scripts/create_stream.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
set -e
3+
4+
for i in {1..10}; do
5+
if aws kinesis create-stream --stream-name $STREAM_NAME --shard-count 1; then
6+
break
7+
else
8+
echo "Stream creation failed, attempt $i/10. Waiting $((i * 3)) seconds..."
9+
sleep $((i * 3))
10+
fi
11+
done
12+
aws kinesis wait stream-exists --stream-name $STREAM_NAME

‎.github/scripts/manipulate_properties.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Build the Bootstrap project and add project references
5+
dotnet add Bootstrap/Bootstrap.csproj reference SampleConsumer/SampleConsumer.csproj
6+
dotnet add Bootstrap/Bootstrap.csproj reference ClientLibrary/ClientLibrary.csproj
7+
dotnet build SampleConsumer/SampleConsumer.csproj -o SampleConsumer/bin
8+
dotnet build Bootstrap/Bootstrap.csproj -o SampleConsumer/bin
9+
10+
SAMPLE_PROPERTIES=".github/resources/github_workflow.properties"
11+
12+
# Manipulate sample.properties file that the KCL application pulls properties from (ex: streamName, applicationName)
13+
# Depending on the OS, different properties need to be changed
14+
if [[ "$RUNNER_OS" == "macOS" ]]; then
15+
sed -i "" "s/STREAM_NAME_PLACEHOLDER/$STREAM_NAME/g" $SAMPLE_PROPERTIES
16+
sed -i "" "s/APP_NAME_PLACEHOLDER/$APP_NAME/g" $SAMPLE_PROPERTIES
17+
sed -i "" "s/EXECUTABLE_NAME_PLACEHOLDER/dotnet bin\/SampleConsumer.dll/g" $SAMPLE_PROPERTIES
18+
sed -i "" "51s/kclnetsample/$STREAM_NAME/g" SampleProducer/SampleProducer.cs
19+
elif [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then
20+
sed -i "s/STREAM_NAME_PLACEHOLDER/$STREAM_NAME/g" $SAMPLE_PROPERTIES
21+
sed -i "s/APP_NAME_PLACEHOLDER/$APP_NAME/g" $SAMPLE_PROPERTIES
22+
sed -i "s/EXECUTABLE_NAME_PLACEHOLDER/dotnet bin\/SampleConsumer.dll/g" $SAMPLE_PROPERTIES
23+
sed -i "51s/kclnetsample/$STREAM_NAME/g" SampleProducer/SampleProducer.cs
24+
else
25+
echo "Unknown OS: $RUNNER_OS"
26+
exit 1
27+
fi
28+
29+
cat $SAMPLE_PROPERTIES

‎.github/scripts/put_words_to_stream.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
set -e
3+
4+
cd SampleProducer
5+
dotnet run
6+
7+
# Get records from stream to verify they exist before continuing
8+
SHARD_ITERATOR=$(aws kinesis get-shard-iterator --stream-name $STREAM_NAME --shard-id shardId-000000000000 --shard-iterator-type TRIM_HORIZON --query 'ShardIterator' --output text)
9+
INITIAL_RECORDS=$(aws kinesis get-records --shard-iterator $SHARD_ITERATOR)
10+
RECORD_COUNT_BEFORE=$(echo $INITIAL_RECORDS | jq '.Records | length')
11+
12+
if [ "$RECORD_COUNT_BEFORE" -eq 0 ]; then
13+
echo "No records found in stream. Test cannot proceed."
14+
exit 1
15+
fi
16+
echo "Found $RECORD_COUNT_BEFORE records in stream before KCL start"

‎.github/scripts/start_kcl.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
set -e
3+
set -o pipefail
4+
5+
SAMPLE_PROPERTIES="../.github/resources/github_workflow.properties"
6+
7+
if [[ "$RUNNER_OS" == "macOS" ]]; then
8+
brew install coreutils
9+
(cd SampleConsumer && gtimeout $RUN_TIME_SECONDS dotnet run --project ../Bootstrap/Bootstrap.csproj --properties $SAMPLE_PROPERTIES --execute 2>&1 | tee ../kcl_output.log) || [ $? -eq 124 ]
10+
elif [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then
11+
(cd SampleConsumer && timeout $RUN_TIME_SECONDS dotnet run --project ../Bootstrap/Bootstrap.csproj --properties $SAMPLE_PROPERTIES --execute 2>&1 | tee ../kcl_output.log) || [ $? -eq 124 ]
12+
else
13+
echo "Unknown OS: $RUNNER_OS"
14+
exit 1
15+
fi
16+
17+
echo "---------ERROR LOGS HERE-------"
18+
grep -i error kcl_output.log || echo "No errors found in logs"

‎.github/scripts/verify_kcl.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
set -e
3+
4+
NUM_LEASES_FOUND=$(aws dynamodb scan --table-name $APP_NAME --select "COUNT" --query "Count" --output text || echo "0")
5+
NUM_CHECKPOINTS_FOUND=$(aws dynamodb scan --table-name $APP_NAME --select "COUNT" --filter-expression "attribute_exists(checkpoint) AND checkpoint <> :trim_horizon" --expression-attribute-values '{":trim_horizon": {"S": "TRIM_HORIZON"}}' --query "Count" --output text || echo "0")
6+
7+
echo "Found $NUM_LEASES_FOUND leases and $NUM_CHECKPOINTS_FOUND non-TRIM-HORIZON checkpoint in DynamoDB"
8+
9+
echo "Printing checkpoint values"
10+
aws dynamodb scan --table-name $APP_NAME --projection-expression "leaseKey,checkpoint" --output json
11+
12+
if [ "$NUM_LEASES_FOUND" -gt 0 ] && [ "$NUM_CHECKPOINTS_FOUND" -gt 0 ]; then
13+
echo "Test passed: Found both leases and non-TRIM_HORIZON checkpoints in DDB (KCL is fully functional)"
14+
exit 0
15+
else
16+
echo "Test failed: KCL not fully functional"
17+
echo "Lease(s) found: $NUM_LEASES_FOUND"
18+
echo "non-TRIM_HORIZON checkpoint(s) found: $NUM_CHECKPOINTS_FOUND"
19+
exit 1
20+
fi

0 commit comments

Comments
 (0)