Run soak tests from EC2 (confluentinc#1908)

* Add OTEL config
* Changes to run with EC2
* Remove Kubernetes based soak tests
* Add otel collector installation to the bootstrap script
* Checkout librdkafka and use same order as build.sh for bootstrap.sh
* Use a local librdkafka installation different for every started test
* Use test code from the checked out branch instead of the selected CKPy version
* Bootstrap the same way as when changing version: with `build.sh`
* Migrate to Prometheus remote write
* Remove log collector
* Initialize error counters to zero
* Use test id as client.id
* Script to configure all versions of librdkafka and confluent-kafka-python to soak test
* PEP8 compliance
* Keep last two log files of 50MB max

---------

Co-authored-by: Emanuele Sabellico <[email protected]>

Author: anchitj

tests/soak/Dockerfile

@@ -10,10 +10,24 @@ OpenTelemetry reporting supported through OTLP.
 
 # Installation
 
-TESTID=normal \
-LK_VERSION=v2.2.0 \
-CKPY_VERSION=v2.2.0 \
-CC_BOOSTRAP_SERVERS=_ \
-CC_USERNAME=_ \
-CC_PASSWORD=_ \
-DOCKER_REPOSITORY=_ ./install.sh
+1. Edit `ccloud.config`
+
+2. Edit `otel-config.yaml`
+
+3. the first time:
+```bash
+./bootstrap.sh <python-branch/tag> <librdkafka-branch/tag>
+```
+4. next times:
+```bash
+./build.sh <python-branch/tag> <librdkafka-branch/tag>
+```
+
+5. 
+```bash
+. venv/bin/activate
+
+5. Run some tests
+```bash
+TESTID=<testid> ./run.sh ccloud.config
+```

tests/soak/README.md

@@ -2,58 +2,69 @@
 #
 #
 
-# Bootstrap EC2 instance (Ubuntu 18.04) for soak client use
+# Bootstrap EC2 instance (Ubuntu) for soak client use
 #
 # Usage:
 #  $0 <python-branch/tag> <librdkafka-branch/tag>
 
 set -e
 
 if [[ $# != 2 ]]; then
-    echo "Usage: $0 <python-client-branch/tag> <librdkafka-branch/tag>"
+    echo "Usage: $0 <librdkafka-branch/tag> <python-client-branch/tag>"
     exit 1
 fi
 
-python_branch=$1
-librdkafka_branch=$2
-venv=$PWD/venv
+librdkafka_branch=$1
+python_branch=$2
+otel_collector_version=0.130.0
+otel_collector_package_url="https://github.com/open-telemetry/"\
+"opentelemetry-collector-releases/releases/download/"\
+"v${otel_collector_version}/otelcol-contrib_${otel_collector_version}_linux_amd64.deb"
+
+validate_config() {
+    local errors=0
+
+    if grep -q "FILL_IN_REGION_HERE" otel-config.yaml; then
+        echo "ERROR: AWS region not configured in otel-config.yaml"
+        echo "Please set AWS_REGION in the file"
+        errors=$((errors + 1))
+    fi
+
+    if grep -q "FILL_IN_ROLE_ARN_HERE" otel-config.yaml; then
+        echo "ERROR: AWS Role ARN not configured in otel-config.yaml"
+        echo "Please set AWS_ROLE_ARN in the file"
+        errors=$((errors + 1))
+    fi
+    
+    if grep -q "FILL_IN_REMOTE_WRITE_ENDPOINT_HERE" otel-config.yaml; then
+        echo "ERROR: Prometheus endpoint not configured"
+        echo "Please set PROMETHEUS_ENDPOINT in the file"
+        errors=$((errors + 1))
+    fi
+    
+    if [[ $errors -gt 0 ]]; then
+        echo "Configuration validation failed. Please fix the above errors."
+        exit 1
+    fi
+    
+    echo "Configuration validation passed."
+}
+validate_config
 
 sudo apt update
-sudo apt install -y git curl make gcc g++ zlib1g-dev libssl-dev libzstd-dev \
-    python3-dev python3-pip python3-venv
-
-if [[ ! -d confluent-kafka-python ]]; then
-    git clone https://github.com/confluentinc/confluent-kafka-python
-fi
-
-pushd confluent-kafka-python
-
-git checkout $python_branch
-
-echo "Installing librdkafka $librdkafka_branch"
-tools/bootstrap-librdkafka.sh --require-ssl $librdkafka_branch /usr
-rm -rf tmp-build
-
-# echo "Installing interceptors"
-# tools/install-interceptors.sh
-
-echo "Setting up virtualenv in $venv"
-if [[ ! -d $venv ]]; then
-    python3 -m venv $venv
-fi
-source $venv/bin/activate
-
-pip install -U pip
-
-pip install -v .[soaktest]
-
-popd # ..python
-
-echo "Verifying python client installation"
-python -c "import confluent_kafka; print(confluent_kafka.version(), confluent_kafka.libversion())"
-
-deactivate
+sudo apt install -y git curl wget make gcc g++ zlib1g-dev libssl-dev \
+    libzstd-dev python3-dev python3-pip python3-venv
+wget -O otel_collector_package.deb $otel_collector_package_url
+sudo dpkg -i otel_collector_package.deb
+rm otel_collector_package.deb
+sudo cp otel-config.yaml /etc/otelcol-contrib/config.yaml
+sudo systemctl restart otelcol-contrib
+cp setup_all_versions.py $HOME/setup_all_versions.py
+chmod +x $HOME/setup_all_versions.py
+
+./build.sh $librdkafka_branch $python_branch
 
+venv=$PWD/venv
 echo "All done, activate the virtualenv in $venv before running the client:"
 echo "source $venv/bin/activate"
 

tests/soak/bootstrap.sh

@@ -11,25 +11,50 @@ fi
 
 set -eu
 
+testdir=$PWD
+mkdir -p $testdir/librdkafka-installation
 
+if [[ ! -d confluent-kafka-python ]]; then
+    git clone https://github.com/confluentinc/confluent-kafka-python
+fi
+
+venv=$PWD/venv
+if [[ ! -d $venv ]]; then
+    echo "Setting up virtualenv in $venv"
+    python3 -m venv $venv
+    source $venv/bin/activate
+    pip install -U pip
+    pip install -r $testdir/../../requirements/requirements-soaktest.txt
+    deactivate
+fi
 
 echo "Building and installing librdkafka $librdkafka_version"
+if [[ ! -d librdkafka ]]; then
+    git clone https://github.com/confluentinc/librdkafka.git
+fi
 pushd librdkafka
-sudo make uninstall
 git fetch --tags
 git checkout $librdkafka_version
-./configure --reconfigure
+echo "Configuring librdkafka $librdkafka_version with prefix $testdir/librdkafka-installation"
+./configure --prefix=$testdir/librdkafka-installation
+make uninstall
 make clean
 make -j
-sudo make install
+make install
 popd
 
+export LIBRARY_PATH=$testdir/librdkafka-installation/lib
+export LD_LIBRARY_PATH=$testdir/librdkafka-installation/lib
+export CPLUS_INCLUDE_PATH=$testdir/librdkafka-installation/include
+export C_INCLUDE_PATH=$testdir/librdkafka-installation/include
 
 echo "Building confluent-kafka-python $cflpy_version"
 set +u
 source venv/bin/activate
+python3 -m pip uninstall -y confluent-kafka
 set -u
 pushd confluent-kafka-python
+rm -rf ./build
 git fetch --tags
 git checkout $cflpy_version
 python3 -m pip install .
@@ -38,4 +63,3 @@ popd
 echo ""
 echo "=============================================================================="
 (cd / ; python3 -c 'import confluent_kafka as c; print("python", c.version(), "librdkafka", c.libversion())')
-

tests/soak/build.sh

@@ -0,0 +1,9 @@
+bootstrap.servers=<add your bootstraps here>
+sasl.mechanisms=PLAIN
+security.protocol=SASL_SSL
+sasl.username=<your ccloud access key>
+sasl.password=<your ccloud secret>
+enable.idempotence=true
+debug=eos,generic,broker,security,consumer
+linger.ms=2
+compression.type=lz4