Soak test helpers (#1484)

Author: edenhill

tests/soak/README.md

@@ -10,4 +10,22 @@ DataDog reporting supported by setting datadog.api_key a and datadog.app_key
 in the soak client configuration file.
 
 
-Use ubuntu-bootstrap.sh in this directory set up the environment (e.g., on ec2).
+There are some convenience script to get you started.
+
+On the host (ec2) where you aim to run the soaktest, do:
+
+$ git clone https://github.com/confluentinc/librdkafka
+$ git clone https://github.com/confluentinc/confluent-kafka-python
+
+# Build librdkafka and python
+$ ~/confluent-kafka-python/tests/soak/build.sh <librdkafka-version> <cfl-python-version>
+
+# Set up config:
+$ cp ~/confluent-kafka-python/tests/soak/ccloud.config.example ~/confluent-kafka-python/ccloud.config
+
+# Start a screen session
+$ screen bash
+
+# Within the screen session, run the soak client
+(screen)$ ~/run.sh
+(screen)$ Ctrl-A d  # to detach

tests/soak/build.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+
+librdkafka_version=$1
+cflpy_version=$2
+
+if [[ -z $cflpy_version ]]; then
+    echo "Usage: $0 <librdkafka_version|tag|branch> <cfl-kafka-python-version|tag|branch>"
+    exit 1
+fi
+
+set -eu
+
+
+
+echo "Building and installing librdkafka $librdkafka_version"
+pushd librdkafka
+sudo make uninstall
+git fetch --tags
+git checkout $librdkafka_version
+./configure --reconfigure
+make clean
+make -j
+sudo make install
+popd
+
+
+echo "Building confluent-kafka-python $cflpy_version"
+set +u
+source venv/bin/activate
+set -u
+pushd confluent-kafka-python
+git fetch --tags
+git checkout $cflpy_version
+python3 setup.py clean -a
+python3 setup.py build
+python3 -m pip install .
+popd
+
+echo ""
+echo "=============================================================================="
+(cd / ; python3 -c 'import confluent_kafka as c; print("python", c.version(), "librdkafka", c.libversion())')
+

tests/soak/ccloud.config.example

@@ -0,0 +1,14 @@
+bootstrap.servers=<add your bootstraps here>
+sasl.mechanisms=PLAIN
+security.protocol=SASL_SSL
+sasl.username=<your ccloud access key>
+sasl.password=<your ccloud secret>
+enable.idempotence=true
+debug=eos,generic,broker,security,consumer
+linger.ms=2
+compression.type=lz4
+# DataDog options/config
+datadog.api_key=<datadog api key>
+datadog.app_key=<datadog app key>
+
+

tests/soak/run.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+
+set -e
+source venv/bin/activate
+
+librdkafka_version=$(python3 -c 'from confluent_kafka import libversion; print(libversion()[0])')
+
+if [[ -z $librdkafka_version ]]; then
+    echo "No librdkafka version found.."
+    exit 1
+fi
+
+if [[ -z $STY ]]; then
+    echo "This script should be run from inside a screen session"
+    exit 1
+fi
+
+set -u
+topic="pysoak-$librdkafka_version"
+logfile="${topic}.log.bz2"
+
+echo "Starting soak client using topic $topic with logs written to $logfile"
+set +x
+time confluent-kafka-python/tests/soak/soakclient.py -t $topic -r 80 -f  confluent-kafka-python/ccloud.config 2>&1 \
+    | tee /dev/stderr | bzip2 > $logfile
+ret=$?
+echo "Python client exited with status $ret"
+exit $ret
+
+