Merge pull request #85 from bbockelm/stress_test

Add stress test (and small fixes)

Author: bbockelm

.github/workflows/test.yml

@@ -29,6 +29,9 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: recursive
+    - uses: actions/setup-go@v5
+      with:
+        go-version: '1.23.5'
     - name: install deps
       run: |
         sudo curl -L https://xrootd.web.cern.ch/repo/RPM-GPG-KEY.txt -o /etc/apt/trusted.gpg.d/xrootd.asc

CMakeLists.txt

@@ -115,6 +115,18 @@ if( BUILD_TESTING )
   target_link_libraries(XrdHTTPServerTesting XrdHTTPServerObj)
   target_include_directories(XrdHTTPServerTesting INTERFACE ${XRootD_INCLUDE_DIRS})
 
+  find_program(GoWrk go-wrk HINTS "$ENV{HOME}/go/bin")
+  if( NOT GoWrk )
+    # Try installing the go-wrk variable to generate a reasonable stress test
+    execute_process( COMMAND go install github.com/bbockelm/go-wrk@92dbe19
+      RESULT_VARIABLE go_install_result )
+    if( go_install_result EQUAL 0 )
+      find_program(GoWrk go-wrk HINTS "$ENV{HOME}/go/bin")
+    else()
+      message(ERROR "Failed to install the go-wrk binary" )
+    endif()
+  endif()
+
   if( NOT XROOTD_PLUGINS_EXTERNAL_GTEST )
     include( FetchContent )
     set( GTEST_URL "${CMAKE_CURRENT_SOURCE_DIR}/googletest-1.15.2.tar.gz" )

src/CurlUtil.cc

@@ -34,7 +34,7 @@
 
 using namespace XrdHTTPServer;
 
-thread_local std::vector<CURL *> HandlerQueue::m_handles;
+thread_local std::stack<CURL *> HandlerQueue::m_handles;
 
 HandlerQueue::HandlerQueue() {
 	int filedes[2];
@@ -85,16 +85,16 @@ CURL *GetHandle(bool verbose) {
 }
 
 CURL *HandlerQueue::GetHandle() {
-	if (m_handles.size()) {
-		auto result = m_handles.back();
-		m_handles.pop_back();
+	if (!m_handles.empty()) {
+		CURL *result = m_handles.top();
+		m_handles.pop();
 		return result;
 	}
 
 	return ::GetHandle(false);
 }
 
-void HandlerQueue::RecycleHandle(CURL *curl) { m_handles.push_back(curl); }
+void HandlerQueue::RecycleHandle(CURL *curl) { m_handles.push(curl); }
 
 void HandlerQueue::Produce(HTTPRequest *handler) {
 	std::unique_lock<std::mutex> lk{m_mutex};
@@ -258,6 +258,7 @@ void CurlWorker::Run() {
 					   << curl_multi_strerror(mres);
 					m_logger.Log(LogMask::Debug, "Run", ss.str().c_str());
 				}
+				m_op_map.erase(curl);
 				op->Fail("E_CURL_LIB",
 						 "Unable to add operation to the curl multi-handle");
 				continue;

src/CurlUtil.hh

@@ -22,8 +22,8 @@
 #include <deque>
 #include <memory>
 #include <mutex>
+#include <stack>
 #include <unordered_map>
-#include <vector>
 
 // Forward dec'ls
 typedef void CURL;
@@ -58,7 +58,7 @@ class HandlerQueue {
 
   private:
 	std::deque<HTTPRequest *> m_ops;
-	thread_local static std::vector<CURL *> m_handles;
+	thread_local static std::stack<CURL *> m_handles;
 	std::condition_variable m_cv;
 	std::mutex m_mutex;
 	const static unsigned m_max_pending_ops{20};

src/S3Commands.cc

@@ -605,7 +605,7 @@ void AmazonS3Head::parseResponse() {
 	size_t current_newline = 0;
 	size_t next_newline = std::string::npos;
 	size_t last_character = headers.size();
-	while (current_newline != std::string::npos &&
+	while (headers.size() && current_newline != std::string::npos &&
 		   current_newline != last_character - 1) {
 		next_newline = headers.find("\r\n", current_newline + 2);
 		line = substring(headers, current_newline + 2, next_newline);

test/CMakeLists.txt

@@ -58,7 +58,7 @@ add_test(NAME HTTP::basic::setup
 set_tests_properties(HTTP::basic::setup
   PROPERTIES
     FIXTURES_SETUP HTTP::basic
-    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR};SOURCE_DIR=${CMAKE_SOURCE_DIR}"
+    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR};SOURCE_DIR=${CMAKE_SOURCE_DIR};XROOTD_BINDIR=${XRootD_DATA_DIR}/../bin"
 )
 
 add_test(NAME HTTP::basic::teardown
@@ -92,7 +92,7 @@ add_test(NAME S3::s3_basic::setup
 set_tests_properties(S3::s3_basic::setup
   PROPERTIES
     FIXTURES_SETUP S3::s3_basic
-    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR};SOURCE_DIR=${CMAKE_SOURCE_DIR}"
+    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR};SOURCE_DIR=${CMAKE_SOURCE_DIR};XROOTD_BINDIR=${XRootD_DATA_DIR}/../bin"
 )
 
 add_test(NAME S3::s3_basic::teardown
@@ -126,3 +126,17 @@ set_tests_properties(S3::s3_basic::stress_test
     ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR}"
     ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}"
 )
+
+#######################################
+# Stress-test using the go-wrk binary #
+#######################################
+
+add_test( NAME S3::s3_basic::gowrk_test
+  COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/s3-gowrk-test.sh" s3_basic )
+
+set_tests_properties( S3::s3_basic::gowrk_test
+  PROPERTIES
+    FIXTURES_REQUIRED S3::s3_basic
+    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR};WRK_BIN=${GoWrk}"
+    ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}"
+)

test/s3-gowrk-test.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+TEST_NAME=$1
+
+if [ -z "$BINARY_DIR" ]; then
+  echo "\$BINARY_DIR environment variable is not set; cannot run test"
+  exit 1
+fi
+if [ ! -d "$BINARY_DIR" ]; then
+  echo "$BINARY_DIR is not a directory; cannot run test"
+  exit 1
+fi
+
+echo "Running $TEST_NAME - go-wrk based stress test"
+
+echo > "$BINARY_DIR/tests/$TEST_NAME/client.log"
+
+if [ ! -f "$BINARY_DIR/tests/$TEST_NAME/setup.sh" ]; then
+  echo "Test environment file $BINARY_DIR/tests/$TEST_NAME/setup.sh does not exist - cannot run test"
+  exit 1
+fi
+. "$BINARY_DIR/tests/$TEST_NAME/setup.sh"
+
+"$WRK_BIN" -c 200 -d 10 -no-vr -T 10000 -f "$PLAYBACK_FILE"

test/s3-setup.sh

@@ -38,7 +38,7 @@ if [ -z "$MC_BIN" ]; then
   exit 1
 fi
 
-XROOTD_BIN="$(command -v xrootd)"
+XROOTD_BIN="$XROOTD_BINDIR/xrootd"
 if [ -z "XROOTD_BIN" ]; then
   echo "xrootd binary not found; cannot run unit test"
   exit 1
@@ -201,6 +201,20 @@ fi
 echo "Hello, World" > "$RUNDIR/hello_world.txt"
 "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket/hello_world.txt
 
+IDX=0
+COUNT=25
+while [ $IDX -ne $COUNT ]; do
+  if ! dd if=/dev/urandom "of=$RUNDIR/test_file" bs=1024 count=1024 2> /dev/null; then
+    echo "Failed to create random file to upload"
+    exit 1
+  fi
+  if ! "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/test_file" "userminio/test-bucket/test_file_$IDX.random" > /dev/null; then
+    echo "Failed to upload random file to S3 instance"
+    exit 1
+  fi
+  IDX=$((IDX+1))
+done
+
 ####
 #    Starting XRootD config with S3 backend
 ####
@@ -274,6 +288,10 @@ while [ -z "$XROOTD_URL" ]; do
   sleep 1
   XROOTD_URL=$(grep "Xrd_ProtLoad: enabling port" "$BINARY_DIR/tests/$TEST_NAME/server.log" | grep 'for protocol XrdHttp' | awk '{print $7}')
   IDX=$(($IDX+1))
+  if ! kill -0 "$XROOTD_PID" 2>/dev/null; then
+    echo "xrootd process (PID $XROOTD_PID) failed to start" >&2
+    exit 1
+  fi
   if [ $IDX -gt 1 ]; then
     echo "Waiting for xrootd to start ($IDX seconds so far) ..."
   fi
@@ -285,8 +303,16 @@ done
 XROOTD_URL="https://$(hostname):$XROOTD_URL/"
 echo "xrootd started at $XROOTD_URL"
 
+IDX=0
+touch "$RUNDIR/playback.txt"
+while [ $IDX -ne $COUNT ]; do
+  echo "$XROOTD_URL/test/test_file_$IDX.random" >> "$RUNDIR/playback.txt"
+  IDX=$((IDX+1))
+done
+
 cat >> "$BINARY_DIR/tests/$TEST_NAME/setup.sh" <<EOF
 XROOTD_PID=$XROOTD_PID
 XROOTD_URL=$XROOTD_URL
 BUCKET_NAME=$BUCKET_NAME
+PLAYBACK_FILE=$RUNDIR/playback.txt
 EOF

test/xrdhttp-setup.sh

@@ -21,8 +21,7 @@ fi
 
 echo "Setting up HTTP server for $TEST_NAME test"
 
-XROOTD_BIN="$(command -v xrootd)"
-
+XROOTD_BIN="$XROOTD_BINDIR/xrootd"
 if [ -z "XROOTD_BIN" ]; then
   echo "xrootd binary not found; cannot run unit test"
   exit 1
@@ -183,6 +182,10 @@ while [ -z "$XROOTD_URL" ]; do
   sleep 1
   XROOTD_URL=$(grep "Xrd_ProtLoad: enabling port" "$BINARY_DIR/tests/$TEST_NAME/server.log" | grep 'for protocol XrdHttp' | awk '{print $7}')
   IDX=$(($IDX+1))
+  if ! kill -0 "$XROOTD_PID" 2>/dev/null; then
+    echo "xrootd process (PID $XROOTD_PID) failed to start" >&2
+    exit 1
+  fi
   if [ $IDX -gt 1 ]; then
     echo "Waiting for xrootd to start ($IDX seconds so far) ..."
   fi