Skip to content

Commit 7fb3b73

Browse files
cfsmp3claude
authored and
canihavesomecoffee
committed
fix: add retry logic to CI VM status and log upload functions
Previously, the CI VM scripts (both Linux and Windows) had no error handling for status POST requests. If the curl request failed due to network issues or server errors, the script would continue without retrying, potentially leaving tests stuck in "testing" state forever. This fix adds: - Retry logic with exponential backoff (3 attempts, 5s -> 10s -> 20s delay) - HTTP status code checking (success = 2xx response) - Curl exit code checking - Detailed logging of retry attempts and failures This addresses the issue where test 7935 completed all 237 tests but the completion status was never reported to the server, leaving the test stuck at 100% progress but "testing" state. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 5d92d0f commit 7fb3b73

File tree

2 files changed

+146
-17
lines changed

2 files changed

+146
-17
lines changed

‎install/ci-vm/ci-linux/ci/runCI‎

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,76 @@ fi
1414

1515
# Functions for re-use in various stages of the test progress
1616

17-
# Post status to the server
17+
# Post status to the server with retry logic
1818
function postStatus {
19-
echo "Posting ${1} - ${2} to the server:" >> "${logFile}"
20-
curl -s -A "${userAgent}" --data "type=progress&status=$1&message=$2" -w "\n" "${reportURL}" >> "${logFile}"
21-
sleep 5
19+
local status="$1"
20+
local message="$2"
21+
local max_retries=3
22+
local retry_delay=5
23+
local attempt=1
24+
25+
echo "Posting ${status} - ${message} to the server:" >> "${logFile}"
26+
27+
while [ $attempt -le $max_retries ]; do
28+
local http_code
29+
http_code=$(curl -s -A "${userAgent}" --data "type=progress&status=${status}&message=${message}" \
30+
-w "%{http_code}" -o /tmp/curl_response.txt "${reportURL}" 2>/dev/null)
31+
local curl_exit=$?
32+
33+
if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
34+
echo "Status posted successfully (HTTP ${http_code})" >> "${logFile}"
35+
cat /tmp/curl_response.txt >> "${logFile}" 2>/dev/null
36+
echo "" >> "${logFile}"
37+
sleep 5
38+
return 0
39+
fi
40+
41+
echo "Attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}"
42+
attempt=$((attempt + 1))
43+
44+
if [ $attempt -le $max_retries ]; then
45+
echo "Retrying in ${retry_delay} seconds..." >> "${logFile}"
46+
sleep $retry_delay
47+
retry_delay=$((retry_delay * 2))
48+
fi
49+
done
50+
51+
echo "ERROR: Failed to post status after ${max_retries} attempts" >> "${logFile}"
52+
return 1
2253
}
2354

24-
# Send the log file to the server so it can be used
55+
# Send the log file to the server with retry logic
2556
function sendLogFile {
57+
local max_retries=3
58+
local retry_delay=5
59+
local attempt=1
60+
2661
echo "Sending log to the server:" >> "${logFile}"
27-
curl -s -A "${userAgent}" --form "type=logupload" --form "file=@${logFile}" -w "\n" "${reportURL}"
28-
sleep 5
62+
63+
while [ $attempt -le $max_retries ]; do
64+
local http_code
65+
http_code=$(curl -s -A "${userAgent}" --form "type=logupload" --form "file=@${logFile}" \
66+
-w "%{http_code}" -o /tmp/curl_log_response.txt "${reportURL}" 2>/dev/null)
67+
local curl_exit=$?
68+
69+
if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
70+
echo "Log uploaded successfully (HTTP ${http_code})" >> "${logFile}"
71+
sleep 5
72+
return 0
73+
fi
74+
75+
echo "Log upload attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}"
76+
attempt=$((attempt + 1))
77+
78+
if [ $attempt -le $max_retries ]; then
79+
echo "Retrying log upload in ${retry_delay} seconds..." >> "${logFile}"
80+
sleep $retry_delay
81+
retry_delay=$((retry_delay * 2))
82+
fi
83+
done
84+
85+
echo "ERROR: Failed to upload log after ${max_retries} attempts" >> "${logFile}"
86+
return 1
2987
}
3088

3189
# Exit script and post abort status

‎install/ci-vm/ci-windows/ci/runCI.bat‎

Lines changed: 81 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,7 @@ if EXIST "%dstDir%\ccextractorwinfull.exe" (
2929
call :executeCommand cd %suiteDstDir%
3030
call :executeCommand "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%"
3131

32-
curl -s -A "%userAgent%" --form "type=logupload" --form "file=@%logFile%" -w "\n" "%reportURL%" >> "%logFile%"
33-
timeout 10
32+
call :sendLogFile
3433

3534
echo Done running tests
3635
call :postStatus "completed" "Ran all tests"
@@ -56,20 +55,92 @@ IF %status% NEQ 0 (
5655
)
5756
EXIT /B 0
5857

59-
rem Post status to the server
58+
rem Post status to the server with retry logic
6059
:postStatus
61-
echo "Posting status %~1 (message: %~2) to the server"
62-
curl -s -A "%userAgent%" --data "type=progress&status=%~1&message=%~2" -w "\n" "%reportURL%" >> "%logFile%"
63-
timeout 10
64-
EXIT /B 0
60+
setlocal EnableDelayedExpansion
61+
set "ps_status=%~1"
62+
set "ps_message=%~2"
63+
set ps_attempt=1
64+
set ps_max_retries=3
65+
set ps_retry_delay=5
66+
67+
:postStatusRetry
68+
echo "Posting status %ps_status% (message: %ps_message%) to the server (attempt !ps_attempt!/%ps_max_retries%)"
69+
echo Posting status %ps_status% - %ps_message% (attempt !ps_attempt!/%ps_max_retries%) >> "%logFile%"
70+
71+
curl -s -A "%userAgent%" --data "type=progress&status=%ps_status%&message=%ps_message%" -w "%%{http_code}" -o "%TEMP%\curl_response.txt" "%reportURL%" > "%TEMP%\http_code.txt" 2>&1
72+
set ps_curl_exit=%ERRORLEVEL%
73+
74+
set /p ps_http_code=<"%TEMP%\http_code.txt"
75+
76+
if %ps_curl_exit% EQU 0 (
77+
if !ps_http_code! GEQ 200 if !ps_http_code! LSS 300 (
78+
echo Status posted successfully (HTTP !ps_http_code!) >> "%logFile%"
79+
type "%TEMP%\curl_response.txt" >> "%logFile%" 2>nul
80+
echo. >> "%logFile%"
81+
timeout 5
82+
endlocal
83+
EXIT /B 0
84+
)
85+
)
86+
87+
echo Attempt !ps_attempt!/%ps_max_retries% failed (curl exit: %ps_curl_exit%, HTTP: !ps_http_code!) >> "%logFile%"
88+
set /A ps_attempt+=1
89+
90+
if !ps_attempt! LEQ %ps_max_retries% (
91+
echo Retrying in %ps_retry_delay% seconds... >> "%logFile%"
92+
timeout %ps_retry_delay%
93+
set /A ps_retry_delay*=2
94+
goto postStatusRetry
95+
)
96+
97+
echo ERROR: Failed to post status after %ps_max_retries% attempts >> "%logFile%"
98+
endlocal
99+
EXIT /B 1
65100

66101
rem Exit script and post abort status
67102
:haltAndCatchFire
68103
echo "Halt and catch fire (reason: %~1)"
69104
echo Post log
70-
curl -s -A "%userAgent%" --form "type=logupload" --form "file=@%logFile%" -w "\n" "%reportURL%" >> "%logFile%"
71-
rem Shut down, but only in 10 seconds, to give the time to finish the post status
72-
timeout 10
105+
call :sendLogFile
73106
call :postStatus "canceled" "%~1"
74107
shutdown -s -t 0
75108
EXIT 0
109+
110+
rem Send log file to server with retry logic
111+
:sendLogFile
112+
setlocal EnableDelayedExpansion
113+
set sl_attempt=1
114+
set sl_max_retries=3
115+
set sl_retry_delay=5
116+
117+
:sendLogFileRetry
118+
echo Sending log to server (attempt !sl_attempt!/%sl_max_retries%) >> "%logFile%"
119+
120+
curl -s -A "%userAgent%" --form "type=logupload" --form "file=@%logFile%" -w "%%{http_code}" -o "%TEMP%\curl_log_response.txt" "%reportURL%" > "%TEMP%\log_http_code.txt" 2>&1
121+
set sl_curl_exit=%ERRORLEVEL%
122+
123+
set /p sl_http_code=<"%TEMP%\log_http_code.txt"
124+
125+
if %sl_curl_exit% EQU 0 (
126+
if !sl_http_code! GEQ 200 if !sl_http_code! LSS 300 (
127+
echo Log uploaded successfully (HTTP !sl_http_code!) >> "%logFile%"
128+
timeout 5
129+
endlocal
130+
EXIT /B 0
131+
)
132+
)
133+
134+
echo Log upload attempt !sl_attempt!/%sl_max_retries% failed (curl exit: %sl_curl_exit%, HTTP: !sl_http_code!) >> "%logFile%"
135+
set /A sl_attempt+=1
136+
137+
if !sl_attempt! LEQ %sl_max_retries% (
138+
echo Retrying log upload in %sl_retry_delay% seconds... >> "%logFile%"
139+
timeout %sl_retry_delay%
140+
set /A sl_retry_delay*=2
141+
goto sendLogFileRetry
142+
)
143+
144+
echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%"
145+
endlocal
146+
EXIT /B 1

0 commit comments

Comments
 (0)