Skip to content
Integration Test

Integration Test #250

Sign in to view logs

Integration Test

Integration Test #250

Workflow file for this run

.github/workflows/integration-test.yml at fc53ada
name: Integration Test
on:
workflow_dispatch:
inputs:
os:
description: "Platform to test"
required: true
type: choice
options:
- all
- windows-latest
- ubuntu-latest
- macos-latest
jobs:
build:
permissions:
contents: read
uses: ./.github/workflows/build.yml
with:
os_list: "${{ inputs.os == 'all' && '[''windows-latest'',''ubuntu-latest'',''macos-latest'']' || format('[{0}{1}{0}]', '''', inputs.os) }}"
cache: true
build-integration-test:
permissions:
contents: read
strategy:
fail-fast: false
matrix:
os: ${{ fromJson(inputs.os == 'all' && '[''windows-latest'',''ubuntu-latest'',''macos-latest'']' || format('[{0}{1}{0}]', '''', inputs.os)) }}
runs-on: ${{ matrix.os }}
steps:
- name: Checkout code
uses: actions/checkout@v5
with:
ref: ${{ github.ref }}
fetch-depth: 0
- name: Cache integration test build output
id: cache-build
uses: actions/cache@v5
with:
path: ./dist
key: build-it-${{ matrix.os }}-go1.25.0-${{ hashFiles('**/*.go', 'go.sum', 'go.mod') }}
- name: Setup go
if: steps.cache-build.outputs.cache-hit != 'true'
uses: actions/setup-go@v6
with:
go-version: "1.25.0"
cache: true
- name: Build integration test binary
if: steps.cache-build.outputs.cache-hit != 'true'
shell: pwsh
id: build
run: ./scripts/build_integration_test.ps1 | % { "binary_path=$_" } >> $env:GITHUB_OUTPUT
- name: Upload integration test binary
uses: actions/upload-artifact@v6
with:
name: it-${{ matrix.os }}
path: ${{ steps.build.outputs.binary_path || './dist/*' }}
set-matrix:
permissions:
contents: read
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.gen.outputs.matrix }}
steps:
- name: Build test matrix include
id: gen
shell: pwsh
env:
OS_INPUT: ${{ inputs.os }}
ORG_ID: ${{ vars.IT_ORG_ID }}
run: |
$winSuccess = @'
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$PS_Results = New-Object -TypeName psobject
# Start of typed powershell
$PS_Results | Add-Member -MemberType NoteProperty -Name \"error\" -Value \"\"
$PS_Results | Add-Member -MemberType NoteProperty -Name \"output\" -Value \"hello world`n\"
# Post results back to Rewst
$postData = $PS_Results | ConvertTo-Json
Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""
'@
$winFailure = @'
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$PS_Results = New-Object -TypeName psobject
# Start of typed powershell
throw \"this is a test error\"
# Post results back to Rewst
$postData = $PS_Results | ConvertTo-Json
Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""
'@
$orgId = $env:ORG_ID
# Output-flood scripts for the bounded-output scenario (sc-106105).
# Deliberately quote-free and backslash-free so they survive the trip
# through the trigger's JSON-quoted commands field unchanged, and
# byte-exact so the produced counts the agent reports are predictable:
# verbose 200 MiB to stdout, then a clean exit
# verbose_stderr a short stdout line, then 200 MiB to stderr
# verbose_hang 200 MiB to stdout, then a hang past the timeout
# All three write 200 MiB - three orders of magnitude past the ceiling
# the scenario configures - so the RSS ceiling asserted while they run
# is far below what buffering the output whole would cost.
$winVerbose = '"$chunk = [string]::new([char]120, 1048576); 1..200 | ForEach-Object { $chunk }"'
$winVerboseStderr = '"Write-Output keep-stdout; $chunk = [string]::new([char]101, 1048576); 1..200 | ForEach-Object { [Console]::Error.Write($chunk) }"'
$winVerboseHang = '"$chunk = [string]::new([char]120, 1048576); 1..200 | ForEach-Object { $chunk }; Start-Sleep -Seconds 60"'
$unixVerbose = '"yes x | head -c 209715200"'
$unixVerboseStderr = '"echo keep-stdout; yes e | head -c 209715200 >&2"'
$unixVerboseHang = '"yes x | head -c 209715200; sleep 60"'
$entries = @(
[ordered]@{
os = 'ubuntu-latest'
binary = 'rewst_agent_config.linux.bin'
it_binary = 'rewst_agent_config.linux.it.bin'
config_dir = '/etc/rewst_remote_agent'
log_file = "/etc/rewst_remote_agent/$orgId/rewst_agent.log"
service = "rewst_remote_agent_$orgId"
success_commands = '"echo \"hello world\""'
failure_commands = '"exit 1"'
timeout_commands = '"sleep 30"'
sweep_commands = '"sleep 180"'
verbose_commands = $unixVerbose
verbose_stderr_commands = $unixVerboseStderr
verbose_hang_commands = $unixVerboseHang
success_patterns = "Command completed`nSending postback`nPostback sent"
no_auto_updates_extra = ''
uninstall_paths = "/etc/rewst_remote_agent/$orgId`n/usr/local/bin/rewst_remote_agent/$orgId`n/tmp/rewst_remote_agent/scripts/$orgId"
service_user = 'rewst-it-svc'
service_username_arg = 'rewst-it-svc'
service_identity = 'rewst-it-svc'
},
[ordered]@{
os = 'windows-latest'
binary = 'rewst_agent_config.win.exe'
it_binary = 'rewst_agent_config.win.it.exe'
config_dir = 'C:\ProgramData\RewstRemoteAgent'
log_file = "C:\ProgramData\RewstRemoteAgent\$orgId\rewst_agent.log"
service = "RewstRemoteAgent_$orgId"
success_commands = $winSuccess
failure_commands = $winFailure
timeout_commands = '"Start-Sleep -Seconds 30"'
sweep_commands = '"Start-Sleep -Seconds 180"'
verbose_commands = $winVerbose
verbose_stderr_commands = $winVerboseStderr
verbose_hang_commands = $winVerboseHang
success_patterns = "Command completed`nSending postback`nPostback already sent"
no_auto_updates_extra = '--disable-agent-postback'
uninstall_paths = "C:\ProgramData\RewstRemoteAgent\$orgId`nC:\Program Files\RewstRemoteAgent\$orgId`nC:\RewstRemoteAgent\scripts\$orgId"
service_user = 'rewst-it-svc'
service_username_arg = '.\rewst-it-svc'
service_identity = '.\rewst-it-svc'
},
[ordered]@{
os = 'macos-latest'
binary = 'rewst_agent_config.mac-os.bin'
it_binary = 'rewst_agent_config.mac-os.it.bin'
config_dir = '/Library/Application Support/rewst_remote_agent'
log_file = "/Library/Application Support/rewst_remote_agent/$orgId/rewst_agent.log"
service = "io.rewst.remote_agent_$orgId"
success_commands = '"echo \"hello world\""'
failure_commands = '"exit 1"'
timeout_commands = '"sleep 30"'
sweep_commands = '"sleep 180"'
verbose_commands = $unixVerbose
verbose_stderr_commands = $unixVerboseStderr
verbose_hang_commands = $unixVerboseHang
success_patterns = "Command completed`nSending postback`nPostback sent"
no_auto_updates_extra = ''
uninstall_paths = "/Library/Application Support/rewst_remote_agent/$orgId`n/usr/local/bin/rewst_remote_agent/$orgId`n`$env:TMPDIR/rewst_remote_agent/scripts/$orgId"
service_user = 'rewst-it-svc'
service_username_arg = 'rewst-it-svc'
service_identity = 'rewst-it-svc'
}
)
if ($env:OS_INPUT -ne 'all') {
$entries = @($entries | Where-Object { $_.os -eq $env:OS_INPUT })
}
$payload = @{ include = $entries }
$json = ConvertTo-Json -InputObject $payload -Depth 10 -Compress
Write-Output $json
"matrix=$json" >> $env:GITHUB_OUTPUT
test:
permissions:
contents: read
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
needs: [build, build-integration-test, set-matrix]
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
runs-on: ${{ matrix.os }}
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Inject PowerShell profile noise
if: matrix.os == 'windows-latest'
id: inject_profile_noise
uses: ./.github/actions/inject-profile-noise
- name: Install agent
id: install_agent
uses: ./.github/actions/install-agent
with:
os: ${{ matrix.os }}
binary: ${{ matrix.binary }}
it_binary: ${{ matrix.it_binary }}
config_url: ${{ vars.IT_CONFIG_URL }}
config_secret: ${{ secrets.IT_CONFIG_SECRET }}
org_id: ${{ vars.IT_ORG_ID }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Check install output for profile noise
if: matrix.os == 'windows-latest'
uses: ./.github/actions/check-profile-noise
with:
install_output_path: ${{ steps.install_agent.outputs.install_output_path }}
noise: ${{ steps.inject_profile_noise.outputs.noise }}
- name: Check service
uses: ./.github/actions/check-service
with:
service: ${{ matrix.service }}
- name: Validate config and get device id
id: config
uses: ./.github/actions/validate-config
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
- name: Check subscribed messages log for topic and QoS
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: Subscribed to messages
subscribed_topic_qos: "true"
wait_seconds: "10"
- name: Send success command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Check logs for successful command
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: ${{ matrix.success_patterns }}
- name: Send failing command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.failure_commands }}
- name: Check logs for error handling
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: Command failed
wait_seconds: "15"
- name: Install integration test binary (old version)
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.it_binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for auto updater to trigger and update
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Updating agent
- name: Verify auto update completed
uses: ./.github/actions/verify-auto-update
with:
log_file: ${{ matrix.log_file }}
- name: Update logging level to debug
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --logging-level debug --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Send test command (debug)
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Check logs for debug level
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: DEBUG
- name: Stop service
uses: ./.github/actions/stop-service
with:
service: ${{ matrix.service }}
- name: Delete log file
uses: ./.github/actions/delete-log
with:
log_file: ${{ matrix.log_file }}
- name: Update agent (no auto updates)
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Check logs for no auto updater
uses: ./.github/actions/assert-log-missing
with:
log_file: ${{ matrix.log_file }}
pattern: Starting auto updater
wait_for_file_seconds: "30"
- name: Send test command (windows postback verification)
if: matrix.os == 'windows-latest'
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Check logs for command received (windows)
if: matrix.os == 'windows-latest'
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: Command completed
- name: Check logs absent postback (windows)
if: matrix.os == 'windows-latest'
uses: ./.github/actions/assert-log-missing
with:
log_file: ${{ matrix.log_file }}
pattern: Postback already sent
- name: Update agent with syslog
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --syslog --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Send test command (syslog)
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Check syslog for command completed
uses: ./.github/actions/assert-syslog-contains
with:
service: ${{ matrix.service }}
pattern: Command completed
# ---- MQTT reconnection scenario ----
# Force a transient broker outage and verify the agent tears down, backs
# off, resubscribes, and resumes command delivery exactly once. Counts are
# captured up front and compared as deltas because the log already contains
# "Subscribed to messages"/"Command completed" lines from earlier cycles
# (the service has restarted several times by now and the log is not reset
# while the service holds it open).
- name: Capture reconnect baseline counts
id: reconnect_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw
$subscribed = ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count
$completed = ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed completed=$completed"
- name: Disrupt broker connection
uses: ./.github/actions/disrupt-connection
with:
mode: block
host: ${{ steps.config.outputs.azure_iot_hub_host }}
- name: Wait for connection lost detection
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Connection lost
max_attempts: "60"
interval_seconds: "2"
- name: Assert connection lost and reconnect/backoff activity
uses: ./.github/actions/assert-log-contains
with:
log_file: ${{ matrix.log_file }}
patterns: |
Connection lost
Reconnecting in
Reconnecting...
wait_seconds: "10"
- name: Restore broker connection
if: always()
uses: ./.github/actions/disrupt-connection
with:
mode: restore
host: ${{ steps.config.outputs.azure_iot_hub_host }}
- name: Wait for fresh subscription after reconnect
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.reconnect_baseline.outputs.subscribed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent resubscribed after reconnect (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent did not resubscribe after reconnect (subscribed count stayed at $baseline)"
exit 1
- name: Send command after reconnect
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Assert command executed exactly once after reconnect
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.reconnect_baseline.outputs.completed }}
run: |
$baseline = [int]$env:BASELINE
$observed = $baseline
# Wait for the post-reconnect command to complete.
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$observed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($observed -gt $baseline) { break }
}
if (($observed - $baseline) -lt 1) {
Write-Error "Post-reconnect command never completed (count stayed at $baseline)"
exit 1
}
# Allow a brief window for any duplicate delivery to surface, then
# require the delta to be exactly one - guarding against re-delivery of
# the buffered message on a non-clean reconnected session.
Start-Sleep -Seconds 10
$content = Get-Content $env:LOG_FILE -Raw
$final = ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count
$delta = $final - $baseline
Write-Output "Post-reconnect 'Command completed' delta: $delta (baseline $baseline, final $final)"
if ($delta -ne 1) {
Write-Error "Expected exactly one post-reconnect 'Command completed', got $delta (duplicate delivery?)"
exit 1
}
Write-Output "Confirmed exactly one post-reconnect command execution"
# ---- Withheld-SUBACK scenario (sc-106107) ----
# The reconnection scenario above severs the connection, which keepalive
# detects and the agent handles as an ordinary reconnect. This one covers
# the harder case it cannot reach: a broker that stays alive - completing
# the TLS handshake, answering CONNECT with CONNACK, and answering every
# PINGREQ - while never sending SUBACK. That is what Azure IoT Hub does when
# it throttles a device, and what a middlebox that half-opens a connection
# produces. Before the fix it left the agent connected but never subscribed,
# silently running no commands and unable to honor a stop; now the subscribe
# wait is bounded and stop-interruptible, so the agent fails cleanly, backs
# off, and recovers on a later cycle once the broker starts acknowledging.
# Counts are compared as deltas because the log already carries lines from
# earlier cycles.
- name: Capture SUBACK-withheld baseline counts
id: suback_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
function Get-PatternCount($text, $needle) {
if ($text) { ([regex]::Matches($text, [regex]::Escape($needle))).Count } else { 0 }
}
$subscribed = Get-PatternCount $content "Subscribed to messages"
$timedOut = Get-PatternCount $content "Failed to subscribe: timed out waiting for broker acknowledgement"
$backoffs = Get-PatternCount $content "Reconnecting in"
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"timed_out=$timedOut" >> $env:GITHUB_OUTPUT
"backoffs=$backoffs" >> $env:GITHUB_OUTPUT
Write-Output "Baseline -> subscribed=$subscribed timed_out=$timedOut backoffs=$backoffs"
# Read off a logged path rather than re-deriving the platform path; see the
# fuller note on the bounded-output scenario's copy of this step.
- name: Resolve the scripts directory (SUBACK scenario)
id: suback_scripts_dir
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$found = if ($content) { [regex]::Matches($content, 'Command saved to[^\r\n]*path=(?<p>\S+)') } else { @() }
if ($found.Count -eq 0) {
Write-Error "No 'Command saved to' line in the log; cannot resolve the scripts directory"
exit 1
}
$dir = Split-Path -Parent $found[$found.Count - 1].Groups['p'].Value.Trim('"')
"path=$dir" >> $env:GITHUB_OUTPUT
Write-Output "Scripts directory: $dir"
- name: Snapshot temp script files before the withheld-SUBACK scenario
uses: ./.github/actions/assert-no-orphaned-scripts
with:
mode: snapshot
scripts_dir: ${{ steps.suback_scripts_dir.outputs.path }}
- name: Start stub broker withholding SUBACK
id: stub_broker
uses: ./.github/actions/stub-broker
with:
mode: start
- name: Point agent at the stub broker
id: stub_host
uses: ./.github/actions/set-broker-host
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
host: ${{ steps.stub_broker.outputs.host }}
# --update restarts the service, so the rewritten broker host takes effect.
# The short subscribe timeout keeps each wedged cycle to ~10s so several
# backoff slots are observable inside the job.
- name: Restart agent against the stub broker
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --mqtt-subscribe-timeout-seconds 10 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Assert bounded subscribe failure and growing backoff
id: assert_suback_withheld
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASE_TIMED_OUT: ${{ steps.suback_baseline.outputs.timed_out }}
BASE_BACKOFFS: ${{ steps.suback_baseline.outputs.backoffs }}
BASE_SUBSCRIBED: ${{ steps.suback_baseline.outputs.subscribed }}
run: |
$baseTimedOut = [int]$env:BASE_TIMED_OUT
$baseBackoffs = [int]$env:BASE_BACKOFFS
$baseSubscribed = [int]$env:BASE_SUBSCRIBED
$needle = "Failed to subscribe: timed out waiting for broker acknowledgement"
# Two wedged cycles plus their backoff waits take roughly 10+2+10+4s.
$content = ""
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$timedOut = if ($content) { ([regex]::Matches($content, [regex]::Escape($needle))).Count } else { 0 }
$backoffs = if ($content) { ([regex]::Matches($content, [regex]::Escape("Reconnecting in"))).Count } else { 0 }
if (($timedOut - $baseTimedOut) -ge 2 -and ($backoffs - $baseBackoffs) -ge 2) { break }
Start-Sleep -Seconds 2
}
$timedOut = if ($content) { ([regex]::Matches($content, [regex]::Escape($needle))).Count } else { 0 }
if (($timedOut - $baseTimedOut) -lt 2) {
Write-Error "Expected at least two bounded subscribe timeouts, got $($timedOut - $baseTimedOut)"
exit 1
}
Write-Output "Bounded subscribe timeouts observed: $($timedOut - $baseTimedOut)"
# The agent must not have subscribed: a withheld SUBACK means no
# subscription, and a "Subscribed to messages" line here would mean the
# stub broker is not the broker the agent reached.
$subscribed = ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count
if ($subscribed -ne $baseSubscribed) {
Write-Error "Agent reported a subscription against a broker that withholds SUBACK"
exit 1
}
# The backoff must not be cleared by a timed-out subscribe: a throttling
# broker needs progressively longer waits, not a tight reconnect loop.
# Go renders a duration as concatenated unit components ("4.1s",
# "1m4s"), so each component is summed rather than parsed as one number.
function ConvertTo-Seconds([string]$value) {
$total = 0.0
foreach ($m in [regex]::Matches($value, '(?<n>[0-9.]+)(?<u>ms|us|ns|h|m|s)')) {
$n = [double]$m.Groups['n'].Value
switch ($m.Groups['u'].Value) {
'ns' { $total += $n / 1e9 }
'us' { $total += $n / 1e6 }
'ms' { $total += $n / 1e3 }
's' { $total += $n }
'm' { $total += $n * 60 }
'h' { $total += $n * 3600 }
}
}
return $total
}
$waits = [regex]::Matches($content, 'Reconnecting in: timeout=(?<t>[0-9.a-z]+)') |
ForEach-Object { ConvertTo-Seconds $_.Groups['t'].Value }
$new = @($waits | Select-Object -Skip $baseBackoffs)
if ($new.Count -lt 2) {
Write-Error "Expected at least two reconnect waits after the subscribe timeouts, got $($new.Count)"
exit 1
}
Write-Output "Reconnect waits after the subscribe timeouts: $($new -join ', ')"
if ($new[1] -le $new[0]) {
Write-Error "Reconnect backoff did not grow ($($new[0]) then $($new[1])); a timed-out subscribe must not clear it"
exit 1
}
Write-Output "Confirmed the reconnect backoff grows after a withheld SUBACK"
# What the agent actually did against the stub is only visible in its own
# log, and the assertion above deliberately reads counts rather than lines.
# Dumping the tail on failure makes a fixture problem (never reached the
# broker at all) distinguishable from a real regression (reached it and
# blocked) without re-running the job.
- name: Dump agent log on withheld-SUBACK failure
if: failure() && steps.assert_suback_withheld.outcome == 'failure'
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
Write-Output "---- last 100 lines of $($env:LOG_FILE) ----"
Get-Content $env:LOG_FILE -Tail 100 -ErrorAction SilentlyContinue
# The original defect made this impossible: with the cycle parked in
# token.Wait() the stop signal was never reached, the platform waited out
# its stop deadline (30s on the Windows SCM) and force-killed the process.
- name: Assert the service stops promptly while wedged on the stub broker
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
SERVICE: ${{ matrix.service }}
run: |
function Get-StopCount {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
if ($content) { ([regex]::Matches($content, [regex]::Escape("Service stopped"))).Count } else { 0 }
}
# Counted here rather than reusing the scenario-wide baseline: the
# --update that pointed the agent at the stub broker restarts the
# service, and that restart logs its own "Service stopped". Measured
# against the older baseline the delta was already satisfied before this
# stop was even issued, so the assertion "passed" in 0.2s without
# observing the stop it exists to test.
$before = Get-StopCount
Write-Output "Service stopped lines before issuing the stop: $before"
$sw = [System.Diagnostics.Stopwatch]::StartNew()
if ($IsWindows) {
sc.exe stop $env:SERVICE | Out-Null
} elseif ($IsLinux) {
sudo systemctl stop $env:SERVICE
} else {
# Bare label, not a "system/<label>" service target. launchctl's
# `stop` is a legacy subcommand that resolves a label in the current
# domain, so the target form is taken as a literal label, matches
# nothing, and exits 3 (ESRCH) without ever signalling the agent.
# This mirrors what the agent's own darwinService.Stop does. `stop`
# leaves the job loaded, so the later start-service can run it again.
sudo launchctl stop $env:SERVICE
}
# launchctl exits non-zero in situations that are not failures here -
# the repo's own stop-service action already guards it with `|| true` -
# and the pwsh shell ends by exiting with $LASTEXITCODE, which failed
# this step on macOS even after the assertion below had passed. Report
# the code and clear it; whether the stop was honored is decided by the
# agent's log, not by the service manager's exit status.
Write-Output "Stop command exit code: $LASTEXITCODE"
$global:LASTEXITCODE = 0
# A clean exit runs the agent's deferred teardown, whose last act is the
# "Service stopped" line; a force-kill past the stop deadline never
# produces one.
$stopped = $false
while ($sw.Elapsed.TotalSeconds -lt 30) {
if ((Get-StopCount) -gt $before) { $stopped = $true; break }
Start-Sleep -Milliseconds 500
}
$sw.Stop()
if (-not $stopped) {
# Both halves matter when this fails: the agent log says whether the
# stop was received and ignored, and the service manager says whether
# it was ever delivered.
Write-Output "---- last 60 lines of the agent log ----"
Get-Content $env:LOG_FILE -Tail 60 -ErrorAction SilentlyContinue
Write-Output "---- service manager state ----"
if ($IsWindows) {
sc.exe query $env:SERVICE
} elseif ($IsLinux) {
sudo systemctl status $env:SERVICE --no-pager
} else {
# print, unlike stop, is a modern subcommand and does take the
# system/<label> service target.
sudo launchctl print "system/$($env:SERVICE)"
}
$global:LASTEXITCODE = 0
Write-Error "Service did not log a clean stop within 30s while wedged on a withholding broker"
exit 1
}
Write-Output "Service stopped cleanly after $([math]::Round($sw.Elapsed.TotalSeconds, 1))s while wedged"
exit 0
- name: Assert no orphaned scripts after the wedged stop
uses: ./.github/actions/assert-no-orphaned-scripts
with:
mode: assert
scripts_dir: ${{ steps.suback_scripts_dir.outputs.path }}
- name: Restart the service against the still-withholding broker
uses: ./.github/actions/start-service
with:
service: ${{ matrix.service }}
- name: Wait for the agent to wedge on the stub broker again
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: "Failed to subscribe: timed out waiting for broker acknowledgement"
max_attempts: "60"
interval_seconds: "2"
# Flipped on the live connection - the stub re-reads its mode per packet -
# so recovery is the agent's own next reconnect cycle, not a restart.
- name: Switch stub broker to acknowledge SUBACK
uses: ./.github/actions/stub-broker
with:
mode: ack
- name: Assert the agent recovers on a later cycle
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.suback_baseline.outputs.subscribed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent subscribed on a later cycle once SUBACK arrived (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent never subscribed after the stub broker started acknowledging"
exit 1
- name: Restore the real broker host
if: always() && steps.stub_host.outputs.previous_host != ''
uses: ./.github/actions/set-broker-host
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
host: ${{ steps.stub_host.outputs.previous_host }}
- name: Stop stub broker
if: always()
uses: ./.github/actions/stub-broker
with:
mode: stop
- name: Restart agent against the real broker
if: always() && steps.stub_host.outputs.previous_host != ''
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --mqtt-subscribe-timeout-seconds 30 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for resubscription to the real broker
if: always() && steps.stub_host.outputs.previous_host != ''
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Subscribed to messages
max_attempts: "60"
interval_seconds: "2"
# ---- Per-command execution timeout scenario (sc-103835) ----
# Configure a short per-command timeout, dispatch a script that sleeps well
# beyond it, and confirm the agent kills the command (logging a timeout) and
# that a subsequent ordinary command still completes - i.e. the worker pool
# recovered instead of staying wedged. Counts are compared as deltas because
# the log already carries lines from earlier cycles.
- name: Enable short command timeout
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --command-timeout-seconds 5 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for fresh subscription after timeout update
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Subscribed to messages
max_attempts: "60"
interval_seconds: "2"
- name: Capture command-timeout baseline counts
id: timeout_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$timedOut = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command timed out"))).Count } else { 0 }
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
"timed_out=$timedOut" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> timed_out=$timedOut completed=$completed"
- name: Send hanging command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.timeout_commands }}
- name: Assert hanging command was killed by the timeout
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.timeout_baseline.outputs.timed_out }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command timed out"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Command timed out as expected (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
}
Write-Error "Hanging command was not killed by the per-command timeout (count stayed at $baseline)"
exit 1
- name: Send command after timeout
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Assert worker pool recovered after timeout
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.timeout_baseline.outputs.completed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Worker pool recovered; a later command completed (count $count > baseline $baseline)"
exit 0
}
}
Write-Error "Worker pool did not process a command after the timeout (count stayed at $baseline)"
exit 1
# ---- SAS token proactive renewal scenario (sc-103969) ----
# The integration test binary is built with a short SAS token lifetime
# (sasTokenLifetimeOverrideStr=90s), so the agent must proactively
# reconnect with a freshly minted token a safety margin BEFORE the token
# expires rather than being forcibly disconnected by the broker on expiry.
# Install the IT binary with auto-updates disabled so it stays the running
# service (otherwise the 30s auto-updater would swap it for the real
# release before renewal fires), then confirm: the renewal fires (distinct
# Info log), the agent re-subscribes with the fresh token, and it is
# graceful - no Error "Connection lost" is produced. Deltas are compared
# because earlier cycles already left "Subscribed to messages"/"Connection
# lost" lines in the log.
- name: Install IT binary with short SAS token lifetime
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.it_binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for fresh subscription after SAS lifetime update
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Subscribed to messages
max_attempts: "60"
interval_seconds: "2"
- name: Capture SAS renewal baseline counts
id: sas_renewal_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$renewed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Renewing SAS token before expiry"))).Count } else { 0 }
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
$lost = if ($content) { ([regex]::Matches($content, [regex]::Escape("Connection lost"))).Count } else { 0 }
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
"renewed=$renewed" >> $env:GITHUB_OUTPUT
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"lost=$lost" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> renewed=$renewed subscribed=$subscribed lost=$lost completed=$completed"
- name: Wait for proactive SAS token renewal
uses: ./.github/actions/wait-for-log-line
with:
log_file: ${{ matrix.log_file }}
pattern: Renewing SAS token before expiry
max_attempts: "60"
interval_seconds: "2"
- name: Assert SAS token renewed and agent resubscribed with fresh token
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
RENEWED_BASELINE: ${{ steps.sas_renewal_baseline.outputs.renewed }}
SUBSCRIBED_BASELINE: ${{ steps.sas_renewal_baseline.outputs.subscribed }}
run: |
$renewedBaseline = [int]$env:RENEWED_BASELINE
$subscribedBaseline = [int]$env:SUBSCRIBED_BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$renewed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Renewing SAS token before expiry"))).Count } else { 0 }
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if (($renewed -gt $renewedBaseline) -and ($subscribed -gt $subscribedBaseline)) {
Write-Output "SAS token renewed (count $renewed > baseline $renewedBaseline) and agent resubscribed (count $subscribed > baseline $subscribedBaseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Proactive SAS renewal did not fire and resubscribe (renewed stayed <= $renewedBaseline or subscribed stayed <= $subscribedBaseline)"
exit 1
- name: Assert SAS renewal was graceful (no forced disconnect)
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
LOST_BASELINE: ${{ steps.sas_renewal_baseline.outputs.lost }}
run: |
$baseline = [int]$env:LOST_BASELINE
$content = Get-Content $env:LOG_FILE -Raw
$lost = ([regex]::Matches($content, [regex]::Escape("Connection lost"))).Count
if ($lost -gt $baseline) {
Write-Error "SAS renewal was not graceful: 'Connection lost' count rose from $baseline to $lost (token expired before renewal fired?)"
exit 1
}
Write-Output "Confirmed graceful renewal: no new 'Connection lost' (count stayed at $baseline)"
- name: Send command after SAS renewal
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Assert command delivery continues after SAS renewal
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.sas_renewal_baseline.outputs.completed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Command delivery continues after renewal (count $count > baseline $baseline)"
exit 0
}
}
Write-Error "No command completed after SAS renewal (count stayed at $baseline)"
exit 1
# ---- Bounded command output scenario (sc-106105) ----
# A command's stdout and stderr used to be buffered whole in memory, so the
# agent's heap tracked however much a script decided to write and a verbose
# one could OOM-kill the service. Configure a deliberately small ceiling,
# flood it by three orders of magnitude, and assert the two properties that
# matter: the agent's RSS stays near the ceiling instead of near the output
# size, and the discarded output is reported rather than silently dropped.
#
# The release binary is installed first with the per-command timeout raised
# well past the floods' runtime, because the earlier timeout scenario left a
# 5s deadline in the config that would kill these commands mid-write, and
# the SAS scenario left the IT binary running with a 90s token lifetime
# whose reconnect would cancel them. --logging-level is deliberately not
# passed: it resets to the default (Info), which keeps the Warn under test
# visible while leaving the debug handler that dumps whole command outputs
# into the log switched off. --disable-agent-postback is likewise not
# passed (unlike the sweep scenario) so the truncated result actually
# traverses the postback path on every platform.
#
# Coverage note: these steps assert on the agent log, which is where the
# byte counts are observable on the endpoint. The same counts are what the
# postback carries as truncated/output_bytes_produced/output_bytes_kept -
# both come from one value computed once per command - but the postback body
# is never logged, so confirming the workflow's received payload stays a
# Rewst-side QA step.
- name: Resolve the scripts directory
id: scripts_dir
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
# Read the directory off a "Command saved to" line rather than
# re-deriving the platform path: the service's temp directory is not the
# runner's, notably root's private TMPDIR on macOS. Many earlier
# scenarios have already logged one.
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$found = if ($content) { [regex]::Matches($content, 'Command saved to[^\r\n]*path=(?<p>\S+)') } else { @() }
if ($found.Count -eq 0) {
Write-Error "No 'Command saved to' line in the log; cannot resolve the scripts directory"
exit 1
}
$path = $found[$found.Count - 1].Groups['p'].Value.Trim('"')
$dir = Split-Path -Parent $path
"path=$dir" >> $env:GITHUB_OUTPUT
Write-Output "Scripts directory: $dir"
- name: Snapshot temp script files before the output floods
uses: ./.github/actions/assert-no-orphaned-scripts
with:
mode: snapshot
scripts_dir: ${{ steps.scripts_dir.outputs.path }}
# Captured before the update below restarts the service: the restart's own
# "Subscribed to messages" line must land after this snapshot for the delta
# wait to mean anything.
- name: Capture bounded-output baseline counts
id: output_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
$truncated = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command output truncated"))).Count } else { 0 }
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"truncated=$truncated" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed truncated=$truncated completed=$completed"
- name: Configure a small output ceiling
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates --max-output-bytes 262144 --command-timeout-seconds 600 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Assert a non-positive output ceiling is rejected
shell: pwsh
env:
BINARY: ${{ matrix.binary }}
ORG_ID: ${{ vars.IT_ORG_ID }}
run: |
# The bound must not be disablable by configuration: a non-positive
# value is refused outright rather than accepted as "no limit" (a
# hand-edited config falls back to the documented default instead).
foreach ($value in @('0', '-1')) {
if ($IsWindows) {
$out = & "./dist/$env:BINARY" --update --org-id $env:ORG_ID --max-output-bytes $value 2>&1 | Out-String
} else {
$out = & sudo "./dist/$env:BINARY" --update --org-id $env:ORG_ID --max-output-bytes $value 2>&1 | Out-String
}
$code = $LASTEXITCODE
if ($code -eq 0) {
Write-Output $out
Write-Error "--max-output-bytes $value was accepted (exit 0); the bound must not be disablable"
exit 1
}
if ($out -notmatch 'invalid max-output-bytes: must be a positive integer') {
Write-Output $out
Write-Error "--max-output-bytes $value was rejected without the expected validation message"
exit 1
}
Write-Output "OK: --max-output-bytes $value rejected (exit $code) with the expected message"
}
# Explicit success: the runner appends "exit $LASTEXITCODE" to pwsh
# steps, and the last thing this step ran was a deliberately failing
# agent invocation, so falling off the end would report that failure as
# the step's own.
exit 0
- name: Wait for fresh subscription after the output ceiling update
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.output_baseline.outputs.subscribed }}
run: |
# A delta, not a bare match: the log already carries many "Subscribed to
# messages" lines, so only a new one proves the restarted agent is
# connected with the new ceiling in effect.
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent subscribed with the new output ceiling (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent did not subscribe after the output ceiling update (count stayed at $baseline)"
exit 1
- name: Send stdout flood command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.verbose_commands }}
- name: Assert the stdout flood was bounded
uses: ./.github/actions/assert-output-bounded
with:
service: ${{ matrix.service }}
log_file: ${{ matrix.log_file }}
baseline_truncations: ${{ steps.output_baseline.outputs.truncated }}
expected_ceiling_bytes: "262144"
# At least the whole ceiling, since stdout alone floods it. The upper
# bound carries 4 KiB of headroom rather than being exact because the
# shell contributes a preamble of its own that is not this scenario's to
# control - on Windows the run deliberately injects a
# PROFILE_NOISE_<guid> marker into the all-users PowerShell profile,
# which every agent-spawned PowerShell echoes. What the bound has to
# catch is a stream that was not capped at all, and that would land
# near the 200 MiB produced, not a few bytes over.
min_kept_bytes: "262144"
max_kept_bytes: "266240"
min_produced_bytes: "209715200"
max_rss_mb: "250"
- name: Assert the flooding command still succeeded
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.output_baseline.outputs.completed }}
run: |
# Being verbose is not a failure: the command runs to completion with the
# excess discarded, so it reports "Command completed", not "Command
# failed" or "Command timed out".
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Flooding command completed normally (count $count > baseline $baseline)"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "The flooding command never completed (count stayed at $baseline): it was killed for being verbose"
exit 1
- name: Capture small-output baseline counts
id: small_output_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$truncated = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command output truncated"))).Count } else { 0 }
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
"truncated=$truncated" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> truncated=$truncated completed=$completed"
- name: Send ordinary command after the flood
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Assert output below the ceiling is untouched and the worker recovered
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
COMPLETED_BASELINE: ${{ steps.small_output_baseline.outputs.completed }}
TRUNCATED_BASELINE: ${{ steps.small_output_baseline.outputs.truncated }}
run: |
# Two things at once: the worker the flood used was released (a later
# command completes), and a command whose output fits under the ceiling
# is not flagged - the bound only engages on the output that exceeds it.
$completedBaseline = [int]$env:COMPLETED_BASELINE
$truncatedBaseline = [int]$env:TRUNCATED_BASELINE
$completed = $completedBaseline
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($completed -gt $completedBaseline) { break }
}
if ($completed -le $completedBaseline) {
Write-Error "No command completed after the flood (count stayed at $completedBaseline): the worker was not released"
exit 1
}
Write-Output "Worker released; a later command completed (count $completed > baseline $completedBaseline)"
$content = Get-Content $env:LOG_FILE -Raw
$truncated = ([regex]::Matches($content, [regex]::Escape("Command output truncated"))).Count
if ($truncated -ne $truncatedBaseline) {
Write-Error "A small-output command was flagged as truncated (count rose from $truncatedBaseline to $truncated)"
exit 1
}
Write-Output "Confirmed small-output command was not truncated (count stayed at $truncatedBaseline)"
- name: Capture stderr-flood baseline counts
id: stderr_flood_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$truncated = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command output truncated"))).Count } else { 0 }
"truncated=$truncated" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> truncated=$truncated"
- name: Send stderr flood command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.verbose_stderr_commands }}
- name: Assert stderr is bounded independently of stdout
uses: ./.github/actions/assert-output-bounded
with:
service: ${{ matrix.service }}
log_file: ${{ matrix.log_file }}
baseline_truncations: ${{ steps.stderr_flood_baseline.outputs.truncated }}
expected_ceiling_bytes: "262144"
# Above the ceiling on purpose: the script floods stderr to its full
# ceiling AND writes a short stdout line, so the kept total can only
# exceed the ceiling if the two streams have separate budgets. A shared
# budget would cap the total at 262144 and starve one of them. The upper
# bound has the same 4 KiB of headroom as the stdout flood, for the same
# reason: the stdout side carries the shell's preamble as well as the
# scenario's own line.
min_kept_bytes: "262145"
max_kept_bytes: "266240"
min_produced_bytes: "209715200"
max_rss_mb: "250"
# Captured before the update below restarts the service, for the same
# reason as the bounded-output baseline.
- name: Capture timeout-composition baseline counts
id: compose_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
$truncated = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command output truncated"))).Count } else { 0 }
$timedOut = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command timed out"))).Count } else { 0 }
$completed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"truncated=$truncated" >> $env:GITHUB_OUTPUT
"timed_out=$timedOut" >> $env:GITHUB_OUTPUT
"completed=$completed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed truncated=$truncated timed_out=$timedOut completed=$completed"
- name: Shorten the command timeout for the composition check
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates --max-output-bytes 262144 --command-timeout-seconds 30 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for fresh subscription after the timeout update
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.compose_baseline.outputs.subscribed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent subscribed with the short timeout (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent did not subscribe after the timeout update (count stayed at $baseline)"
exit 1
- name: Send verbose command that then hangs
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.verbose_hang_commands }}
- name: Assert the hanging command's output was still bounded
uses: ./.github/actions/assert-output-bounded
with:
service: ${{ matrix.service }}
log_file: ${{ matrix.log_file }}
baseline_truncations: ${{ steps.compose_baseline.outputs.truncated }}
expected_ceiling_bytes: "262144"
# Same headroom as the other floods: stdout fills the ceiling on its
# own, and anything the shell adds on either stream is incidental.
min_kept_bytes: "262144"
max_kept_bytes: "266240"
# Lower than the flood's full 200 MiB: the command is killed at its
# deadline, so only the volume written before the kill is counted.
min_produced_bytes: "104857600"
max_rss_mb: "250"
# The warning lands once the deadline kills the process group, and
# cmd.WaitDelay can hold Run for up to 10s more after that.
max_attempts: "120"
- name: Assert the hanging command also timed out and released its worker
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
TIMED_OUT_BASELINE: ${{ steps.compose_baseline.outputs.timed_out }}
run: |
# The two bounds compose: the same command is reported as truncated
# (asserted above) and as timed out, neither masking the other.
$baseline = [int]$env:TIMED_OUT_BASELINE
for ($i = 1; $i -le 30; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command timed out"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Verbose hanging command was killed by the timeout (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Verbose hanging command was not killed by the per-command timeout (count stayed at $baseline)"
exit 1
- name: Send command after the verbose timeout
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.success_commands }}
- name: Assert the worker recovered after the verbose timeout
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.compose_baseline.outputs.completed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
Start-Sleep -Seconds 2
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command completed"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Worker recovered after the verbose timeout (count $count > baseline $baseline)"
exit 0
}
}
Write-Error "No command completed after the verbose timeout (count stayed at $baseline)"
exit 1
- name: Assert the output floods left no orphaned script files
uses: ./.github/actions/assert-no-orphaned-scripts
with:
mode: assert
scripts_dir: ${{ steps.scripts_dir.outputs.path }}
# ---- Stale script file sweep scenario (sc-103967) ----
# A command interrupted by an abrupt kill (force-stop, OOM kill, power
# loss) never runs its deferred temp-file cleanup, so its script file is
# stranded in the scripts directory and nothing used to reclaim it. Verify
# the startup sweep now does: force-kill the service mid-command to strand
# a real script file, age it (plus a control set) into the months-old
# debris a long-lived install accumulates, restart, and confirm only the
# agent's own stale files are removed.
#
# The release binary is reinstalled first, with auto-updates off and the
# per-command timeout raised well past the command's runtime, so nothing
# ends the command before the kill lands - not the 5s timeout from the
# earlier scenario, the 30s auto-updater, or the IT binary's 90s SAS
# lifetime. Each of those tears the command down gracefully, and graceful
# teardown DOES run the cleanup, leaving no stranded file to sweep.
- name: Capture sweep scenario baseline counts
id: sweep_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
$saved = if ($content) { ([regex]::Matches($content, [regex]::Escape("Command saved to"))).Count } else { 0 }
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
"saved=$saved" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed saved=$saved"
- name: Install release binary for the sweep scenario
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --command-timeout-seconds 600 --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for fresh subscription before the sweep scenario
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.sweep_baseline.outputs.subscribed }}
run: |
# A delta, not a bare match: the log already carries many "Subscribed to
# messages" lines from earlier scenarios, so only a new one proves the
# restarted agent is connected and ready to receive the command.
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent subscribed after the sweep-scenario install (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent did not subscribe after the sweep-scenario install (count stayed at $baseline)"
exit 1
- name: Send long-running command
uses: ./.github/actions/send-command
with:
trigger_url: ${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}
device_id: ${{ steps.config.outputs.device_id }}
commands: ${{ matrix.sweep_commands }}
- name: Capture the in-flight script file path
id: inflight_script
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.sweep_baseline.outputs.saved }}
run: |
# The agent logs the temp file it wrote the command to, so read the path
# (and the scripts directory) straight from the log instead of
# re-deriving the platform path - the service's temp directory is not
# the runner's, notably root's private TMPDIR on macOS.
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 30; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$found = if ($content) { [regex]::Matches($content, 'Command saved to[^\r\n]*path=(?<p>\S+)') } else { @() }
if ($found.Count -gt $baseline) {
$path = $found[$found.Count - 1].Groups['p'].Value.Trim('"')
$dir = Split-Path -Parent $path
"path=$path" >> $env:GITHUB_OUTPUT
"scripts_dir=$dir" >> $env:GITHUB_OUTPUT
Write-Output "In-flight script file: $path (scripts directory $dir)"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent never logged a script file for the long-running command (count stayed at $baseline)"
exit 1
- name: Force-kill the agent mid-command
uses: ./.github/actions/kill-service
with:
service: ${{ matrix.service }}
- name: Age the stranded script file and seed sweep control files
uses: ./.github/actions/script-sweep-fixture
with:
mode: seed
scripts_dir: ${{ steps.inflight_script.outputs.scripts_dir }}
orphan_path: ${{ steps.inflight_script.outputs.path }}
- name: Capture sweep log baseline count
id: sweep_log_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$swept = if ($content) { ([regex]::Matches($content, [regex]::Escape("Swept stale script files"))).Count } else { 0 }
"swept=$swept" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> swept=$swept"
- name: Restart agent to trigger the startup sweep
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for and assert the startup sweep was logged
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.sweep_log_baseline.outputs.swept }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$swept = if ($content) { ([regex]::Matches($content, [regex]::Escape("Swept stale script files"))).Count } else { 0 }
if ($swept -gt $baseline) {
Write-Output "Startup sweep logged (count $swept > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Startup sweep was never logged (count stayed at $baseline)"
exit 1
- name: Assert stale script files swept and live files untouched
uses: ./.github/actions/script-sweep-fixture
with:
mode: assert
scripts_dir: ${{ steps.inflight_script.outputs.scripts_dir }}
orphan_path: ${{ steps.inflight_script.outputs.path }}
# ---- Downloaded installer sweep scenario (sc-106111) ----
# Every auto-update downloads a full agent binary and executes it as the
# installer. The installer is spawned detached and the process that could
# delete it afterwards is the one the installer replaces, so the download
# path cannot clean up after itself and nothing else used to - one
# orphaned binary accumulated per update for the lifetime of the install,
# until the volume filled and took future updates and command execution
# with it. Verify the startup sweep now reclaims them.
#
# The fixture is planted rather than produced by a real update cycle: the
# sweep is what this ticket changed, and the download side (where the file
# lands, under what name) is covered by unit tests that assert the two
# halves share one pattern constant. Ageing is the only simulated part, so
# the production 24h threshold stays in play with no test-only override.
#
# Only the org's own updates directory is asserted here. The sweep also
# covers the legacy shared temp directory that older agents downloaded
# into, but that is the *service account's* temp directory - notably root's
# private TMPDIR on macOS, not the runner's - so a fixture planted from
# the runner would land somewhere the service never reads. That half is
# covered by TestExecute_SweepsStaleInstallerFilesOnStartup, which runs
# Execute in-process and can see its own temp directory.
- name: Seed the installer sweep fixture
uses: ./.github/actions/installer-sweep-fixture
with:
mode: seed
updates_dir: ${{ matrix.config_dir }}/${{ vars.IT_ORG_ID }}/updates
- name: Capture installer sweep log baseline count
id: installer_sweep_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$swept = if ($content) { ([regex]::Matches($content, [regex]::Escape("Swept stale installer files"))).Count } else { 0 }
"swept=$swept" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> swept=$swept"
- name: Restart agent to trigger the installer sweep
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for and assert the installer sweep was logged
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.installer_sweep_baseline.outputs.swept }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$swept = if ($content) { ([regex]::Matches($content, [regex]::Escape("Swept stale installer files"))).Count } else { 0 }
if ($swept -gt $baseline) {
Write-Output "Installer sweep logged (count $swept > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Output "---- last 40 log lines ----"
Get-Content $env:LOG_FILE -Tail 40 -ErrorAction SilentlyContinue
Write-Error "Installer sweep was never logged (count stayed at $baseline)"
exit 1
- name: Assert stale installers swept and live files untouched
uses: ./.github/actions/installer-sweep-fixture
with:
mode: assert
updates_dir: ${{ matrix.config_dir }}/${{ vars.IT_ORG_ID }}/updates
# ---- Capped and jittered auto-update retry scenario (sc-106110) ----
# The auto-update retry delay was base * (1 << attempt) with no ceiling and
# no jitter. Unjittered, every agent that failed against the release
# endpoint at the same moment retried at the same instants, so an outage or
# rate limit was sustained by the fleet trying to recover from it; uncapped,
# a large retry count overflowed the shift into a negative duration, which
# made the wait return immediately and spun the loop against the endpoint.
#
# A GitHub outage is not something CI can arrange and an unroutable address
# produces connection errors rather than the HTTP failure the ticket
# describes, so the agent is pointed at the test/stubrelease fixture through
# the override file the integration build honors. The fixture fails every
# request until it is flipped, which puts the agent into retryWithBackoff
# with the shortened build knobs - base 2s, 6 retries, 30s check interval,
# so the cap is a quarter of that interval (7.5s) - and makes a full
# schedule observable in under a minute.
#
# Fleet-wide de-synchronization needs many agents and is covered by unit
# tests over the schedule itself. What one endpoint can show, and what these
# steps assert, is that every slot is strictly positive, bounded by the cap,
# varies rather than repeating, and is never served back to back.
- name: Start stub release endpoint (failing)
id: stub_release
uses: ./.github/actions/stub-release
with:
mode: start
- name: Capture auto-update retry baseline counts
id: retry_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$retrying = if ($content) { ([regex]::Matches($content, [regex]::Escape("Retrying update"))).Count } else { 0 }
$succeeded = if ($content) { ([regex]::Matches($content, [regex]::Escape("Update succeeded on retry"))).Count } else { 0 }
"retrying=$retrying" >> $env:GITHUB_OUTPUT
"succeeded=$succeeded" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> retrying=$retrying succeeded=$succeeded"
# Written before the install, because the updater reads the override when a
# service cycle constructs it - a file written afterwards would not be seen
# until the next restart, and the agent would spend the scenario updating
# itself from the real releases API instead.
- name: Point the agent at the stub release endpoint
uses: ./.github/actions/set-release-url
with:
mode: set
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
url: ${{ steps.stub_release.outputs.url }}
- name: Install integration test binary with auto-updates on
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.it_binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for the update retry schedule to run
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.retry_baseline.outputs.retrying }}
run: |
# The first check fires 30s after the service starts, then the schedule
# is roughly 2s, 4s, 7.5s, 7.5s... Four new lines put two slots at the
# ceiling, which is what the jitter assertion below needs to tell a
# jittered schedule from one that is merely capped, and still leave two
# more slots (~15s) for the recovery flip to land in, so the flip cannot
# race the end of the retry budget.
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 150; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Retrying update"))).Count } else { 0 }
if (($count - $baseline) -ge 4) {
Write-Output "Observed $($count - $baseline) update retries after ~${i}s"
exit 0
}
Start-Sleep -Seconds 1
}
Write-Error "The agent never retried the update against the failing release endpoint"
exit 1
# Flipped mid-sequence - the fixture re-reads its mode per request - so the
# recovery is the agent's own next retry, not a restart.
- name: Serve a valid release payload mid-sequence
uses: ./.github/actions/stub-release
with:
mode: ok
- name: Assert the update succeeds on a retry
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.retry_baseline.outputs.succeeded }}
run: |
# The fixture serves the running agent's own version as tag_name, so a
# recovered check ends at "No updates available" and the retry is seen
# succeeding without downloading or executing anything - the recovery is
# what is under test here, not the installer.
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Update succeeded on retry"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Update succeeded on a retry after ~$($i * 2)s (count $count > baseline $baseline)"
exit 0
}
Start-Sleep -Seconds 2
}
# Separates a real regression from a flip that landed after the budget
# was already spent, which would leave the next check succeeding on its
# first try and logging no retry at all.
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
if ($content -and $content.Contains("All retries exhausted")) {
Write-Output "Note: the log records an exhausted retry budget; the recovery flip may have landed after the last slot"
}
Write-Output "---- last 40 log lines ----"
Get-Content $env:LOG_FILE -Tail 40 -ErrorAction SilentlyContinue
Write-Error "The agent never logged a successful update retry after the endpoint recovered"
exit 1
- name: Assert the retry schedule is capped, jittered and never immediate
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.retry_baseline.outputs.retrying }}
run: |
# The cap the integration build lands on: a quarter of its 30s check
# interval, which is below the absolute 1h ceiling a released build uses.
$cap = 7.5
$baseline = [int]$env:BASELINE
$failures = @()
function ConvertTo-Seconds([string]$value) {
# Go renders a duration as concatenated unit components ("4.1s",
# "1m4s"), and a negative one with a leading '-'. The sign is read
# separately because the overflow this scenario guards against
# produced exactly that negative value, and summing the components
# alone would report it as positive.
$sign = if ($value.StartsWith('-')) { -1.0 } else { 1.0 }
$total = 0.0
foreach ($m in [regex]::Matches($value, '(?<n>[0-9.]+)(?<u>ms|us|ns|h|m|s)')) {
$n = [double]$m.Groups['n'].Value
switch ($m.Groups['u'].Value) {
'ns' { $total += $n / 1e9 }
'us' { $total += $n / 1e6 }
'ms' { $total += $n / 1e3 }
's' { $total += $n }
'm' { $total += $n * 60 }
'h' { $total += $n * 3600 }
}
}
return $sign * $total
}
$lines = @(Get-Content $env:LOG_FILE -ErrorAction SilentlyContinue |
Where-Object { $_ -match 'Retrying update' } |
Select-Object -Skip $baseline)
$slots = @()
foreach ($line in $lines) {
# hclog writes "<timestamp> [INFO] agent_smith: Retrying update: attempt=N backoff=D".
# The timestamp is a fixed-width local time followed by its offset, so
# the first 23 characters parse without worrying about the offset form.
$m = [regex]::Match($line, '^(?<ts>\S{23}).*Retrying update: attempt=(?<attempt>\d+) backoff=(?<backoff>\S+)')
if (-not $m.Success) {
$failures += "could not parse a retry line: $line"
continue
}
$stamp = $null
try {
$stamp = [datetime]::ParseExact(
$m.Groups['ts'].Value,
'yyyy-MM-ddTHH:mm:ss.fff',
[Globalization.CultureInfo]::InvariantCulture)
} catch {
$failures += "could not parse the timestamp of: $line"
}
$slots += [pscustomobject]@{
Attempt = [int]$m.Groups['attempt'].Value
Backoff = ConvertTo-Seconds $m.Groups['backoff'].Value
Time = $stamp
}
}
if ($slots.Count -lt 3) {
Write-Error "Expected at least three retry slots to inspect, got $($slots.Count)"
exit 1
}
# The timestamp table the ticket asks for: every slot, what it waited,
# and how long actually elapsed before the next one arrived.
Write-Output "---- observed update retry schedule ----"
for ($i = 0; $i -lt $slots.Count; $i++) {
$gap = ''
if ($i -lt $slots.Count - 1 -and $slots[$i].Time -and $slots[$i + 1].Time) {
$gap = [math]::Round(($slots[$i + 1].Time - $slots[$i].Time).TotalSeconds, 3)
}
Write-Output ("attempt={0} backoff={1}s elapsed_to_next={2}" -f $slots[$i].Attempt, [math]::Round($slots[$i].Backoff, 3), $gap)
}
foreach ($slot in $slots) {
# A zero or negative slot is the overflow busy-spin: time.After fires
# immediately and the loop issues requests as fast as it can.
if ($slot.Backoff -le 0) {
$failures += "attempt $($slot.Attempt): backoff $($slot.Backoff)s is not strictly positive"
}
# The tolerance covers the millisecond rendering, not the cap itself.
if ($slot.Backoff -gt ($cap + 0.05)) {
$failures += "attempt $($slot.Attempt): backoff $($slot.Backoff)s exceeds the ${cap}s cap"
}
}
# Jitter: a fixed schedule repeats the same values bit for bit.
$distinct = @($slots | ForEach-Object { [math]::Round($_.Backoff, 3) } | Select-Object -Unique)
if ($distinct.Count -lt 2) {
$failures += "every retry slot was $($distinct -join ', ')s - the schedule is not jittered"
}
# Distinct values alone are weak evidence, because the doubling produces
# distinct values on its own until it reaches the ceiling. The slots that
# do reach it are the discriminator: an unjittered schedule pins every
# one of them to exactly the cap, while jitter spreads them across the
# band below it. Skipped when the sequence was cut short by the recovery
# flip before two slots got that far.
$nearCap = @($slots | Where-Object { $_.Backoff -ge ($cap * 0.75) })
if ($nearCap.Count -ge 2) {
$distinctNearCap = @($nearCap | ForEach-Object { [math]::Round($_.Backoff, 3) } | Select-Object -Unique)
if ($distinctNearCap.Count -lt 2) {
$failures += "the capped slots all measured $($distinctNearCap -join ', ')s - the jitter is not being applied"
}
}
# Growth: the schedule still doubles up to the cap rather than being flat.
$maxBackoff = ($slots | Measure-Object -Property Backoff -Maximum).Maximum
if ($maxBackoff -le $slots[0].Backoff) {
$failures += "the schedule never grew (first $($slots[0].Backoff)s, largest ${maxBackoff}s)"
}
# Spacing: consecutive slots of one sequence must be separated by at
# least the wait that was logged. Pairs that are not consecutive attempts
# belong to different retry sequences and are skipped.
for ($i = 0; $i -lt $slots.Count - 1; $i++) {
if ($slots[$i + 1].Attempt -ne $slots[$i].Attempt + 1) { continue }
if (-not $slots[$i].Time -or -not $slots[$i + 1].Time) { continue }
$gap = ($slots[$i + 1].Time - $slots[$i].Time).TotalSeconds
if ($gap -lt 1) {
$failures += "attempts $($slots[$i].Attempt) and $($slots[$i + 1].Attempt) arrived ${gap}s apart - back to back"
}
if ($gap -lt ($slots[$i].Backoff * 0.8)) {
$failures += "attempt $($slots[$i].Attempt) logged a $($slots[$i].Backoff)s backoff but the next retry came after ${gap}s"
}
}
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Output "FAIL: $_" }
Write-Error "the update retry schedule is not capped, jittered and spaced as expected"
exit 1
}
Write-Output "Retry schedule bounded by ${cap}s, jittered across $($distinct.Count) distinct values, and never served back to back"
exit 0
- name: Fail the release endpoint again for the stop check
uses: ./.github/actions/stub-release
with:
mode: fail
- name: Capture stop-during-backoff baseline counts
id: retry_stop_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$retrying = if ($content) { ([regex]::Matches($content, [regex]::Escape("Retrying update"))).Count } else { 0 }
$stopped = if ($content) { ([regex]::Matches($content, [regex]::Escape("Auto updater stopped"))).Count } else { 0 }
"retrying=$retrying" >> $env:GITHUB_OUTPUT
"stopped=$stopped" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> retrying=$retrying stopped=$stopped"
- name: Wait for a fresh backoff to be entered
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.retry_stop_baseline.outputs.retrying }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 90; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Retrying update"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent is in a retry backoff again after ~${i}s"
exit 0
}
Start-Sleep -Seconds 1
}
Write-Error "The agent never entered a retry backoff after the endpoint was failed again"
exit 1
- name: Assert a service stop during a backoff is prompt
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
SERVICE: ${{ matrix.service }}
BASELINE: ${{ steps.retry_stop_baseline.outputs.stopped }}
run: |
# The agent has just entered a backoff with most of its 6-slot budget
# left (~30s of waiting at the 7.5s cap). A stop that has to wait out
# the pending slot - or worse, the whole budget - delays every caller of
# Stop(): the installer, an upgrade, and an uninstall.
$baseline = [int]$env:BASELINE
$sw = [System.Diagnostics.Stopwatch]::StartNew()
if ($IsWindows) {
sc.exe stop $env:SERVICE | Out-Null
} elseif ($IsLinux) {
sudo systemctl stop $env:SERVICE
} else {
# Bare label, not "system/<label>": stop is a legacy launchctl
# subcommand that resolves a label in the current domain, and a
# service-target form is read as a literal label that matches nothing.
sudo launchctl stop $env:SERVICE
}
# The service managers disagree about exit status for a stop that was
# delivered; the verdict here comes from the agent's own log.
$global:LASTEXITCODE = 0
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Auto updater stopped"))).Count } else { 0 }
if ($count -gt $baseline) {
$sw.Stop()
$elapsed = [math]::Round($sw.Elapsed.TotalSeconds, 1)
if ($sw.Elapsed.TotalSeconds -gt 20) {
Write-Error "The auto updater stopped only after ${elapsed}s, long enough to have waited out its retry schedule"
exit 1
}
Write-Output "Auto updater stopped ${elapsed}s after the stop was issued, mid-backoff"
exit 0
}
Start-Sleep -Milliseconds 500
}
$sw.Stop()
Write-Output "---- last 40 log lines ----"
Get-Content $env:LOG_FILE -Tail 40 -ErrorAction SilentlyContinue
Write-Error "The auto updater never logged a stop after the service was stopped mid-backoff"
exit 1
# The restore steps run under always() so a failed assertion above cannot
# leave the rest of the job running an agent that points at a stub release
# endpoint (or leave the service stopped for the scenarios that follow).
- name: Capture restore baseline counts
if: always()
id: retry_restore_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed"
- name: Clear the release url override
if: always()
uses: ./.github/actions/set-release-url
with:
mode: clear
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
- name: Stop stub release endpoint
if: always()
uses: ./.github/actions/stub-release
with:
mode: stop
- name: Reinstall the release binary after the retry scenario
if: always()
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates ${{ matrix.no_auto_updates_extra }} --github-token ${{ secrets.GITHUB_TOKEN }}
- name: Wait for resubscription after the retry scenario
if: always()
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.retry_restore_baseline.outputs.subscribed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Agent resubscribed after the retry scenario (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Agent never resubscribed after the retry scenario"
exit 1
# ---- Wedged Windows service stop scenario (sc-106108) ----
# windowsService.Stop() polled the service state every 250ms in an
# unbounded loop, so a service that never reached Stopped hung its caller
# forever - and every caller is blocking: the unattended auto-updater, the
# installer, and uninstall. The symptom was a device that went offline
# during a routine update with no error logged anywhere.
#
# The wedge is produced by pointing the agent's own service registration at
# the test/wedgedservice fixture, which accepts the stop control, reports
# StopPending, and never stops (see the action for why killing or
# suspending the agent will not do). The installed agent executable and
# config file are left alone, so they can be hashed before and after to
# prove an aborted update did not touch them.
#
# Windows only: the Unix service implementations have no such polling loop.
- name: Capture installed file hashes before the wedge
if: matrix.os == 'windows-latest'
id: wedge_hashes
shell: pwsh
env:
ORG_ID: ${{ vars.IT_ORG_ID }}
CONFIG_DIR: ${{ matrix.config_dir }}
run: |
$ErrorActionPreference = 'Stop'
$exe = Join-Path $env:PROGRAMFILES "RewstRemoteAgent\$($env:ORG_ID)\agent_smith.win.exe"
$config = Join-Path $env:CONFIG_DIR "$($env:ORG_ID)\config.json"
foreach ($path in @($exe, $config)) {
if (-not (Test-Path $path)) { throw "expected installed file is missing: $path" }
}
$exeHash = (Get-FileHash -Path $exe -Algorithm SHA256).Hash
$configHash = (Get-FileHash -Path $config -Algorithm SHA256).Hash
"exe_path=$exe" >> $env:GITHUB_OUTPUT
"config_path=$config" >> $env:GITHUB_OUTPUT
"exe_hash=$exeHash" >> $env:GITHUB_OUTPUT
"config_hash=$configHash" >> $env:GITHUB_OUTPUT
Write-Output "agent executable $exe sha256=$exeHash"
Write-Output "config file $config sha256=$configHash"
- name: Wedge the agent service in StopPending
if: matrix.os == 'windows-latest'
uses: ./.github/actions/wedged-service
with:
mode: wedge
service: ${{ matrix.service }}
- name: Assert the update aborts on a wedged stop
if: matrix.os == 'windows-latest'
shell: pwsh
env:
IT_BINARY: ${{ matrix.it_binary }}
ORG_ID: ${{ vars.IT_ORG_ID }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# The integration binary is the invoker because its stop deadline is
# overridden to 25s (see scripts/build_integration_test.ps1). The
# released default is five minutes; that the constant is what governs a
# production build is covered by unit tests rather than by keeping a
# runner idle for five minutes here.
$sw = [System.Diagnostics.Stopwatch]::StartNew()
$out = & "./dist/$($env:IT_BINARY)" --update --org-id $env:ORG_ID `
--github-token $env:GH_TOKEN 2>&1 | Out-String
$sw.Stop()
# The updater reports an aborted update through its log, not its exit
# status, and this step's verdict comes from the log below.
$global:LASTEXITCODE = 0
$elapsed = [math]::Round($sw.Elapsed.TotalSeconds, 1)
Write-Output "---- updater output after ${elapsed}s ----"
Write-Output $out
$failures = @()
# Returning at all is the headline: before the fix this call never came
# back. The floor proves it waited out its deadline rather than failing
# fast for an unrelated reason; the ceiling is far above the 25s
# deadline yet far below an unbounded hang.
if ($sw.Elapsed.TotalSeconds -lt 20) {
$failures += "update returned after ${elapsed}s, before its 25s stop deadline could elapse"
}
if ($sw.Elapsed.TotalSeconds -gt 180) {
$failures += "update took ${elapsed}s, far past its 25s stop deadline"
}
# The log has to name the service, the deadline and the last observed
# state, since that is the only account of why an unattended update
# aborted.
foreach ($needle in @(
'Failed to stop service',
'did not stop within',
'last observed state StopPending',
'Update aborted; existing installation left untouched'
)) {
if (-not $out.Contains($needle)) { $failures += "missing from the log: '$needle'" }
}
# And it must not have carried on as if the stop had succeeded.
foreach ($absent in @(
'Configuration successfully updated',
'Agent installed to',
'Service started'
)) {
if ($out.Contains($absent)) { $failures += "update proceeded past the failed stop: '$absent'" }
}
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Output "FAIL: $_" }
Write-Error "the update did not abort cleanly on a wedged stop"
exit 1
}
Write-Output "Update aborted on the wedged stop after ${elapsed}s, as expected"
exit 0
- name: Assert the aborted update left the installation untouched
if: matrix.os == 'windows-latest'
shell: pwsh
env:
EXE_PATH: ${{ steps.wedge_hashes.outputs.exe_path }}
CONFIG_PATH: ${{ steps.wedge_hashes.outputs.config_path }}
EXE_HASH: ${{ steps.wedge_hashes.outputs.exe_hash }}
CONFIG_HASH: ${{ steps.wedge_hashes.outputs.config_hash }}
run: |
# The whole point of aborting: the wedged process may still hold the
# executable open, so a half-written binary or a rewritten config would
# leave the endpoint unrecoverable without hands on it.
$ErrorActionPreference = 'Stop'
$failures = @()
foreach ($pair in @(
@{ Path = $env:EXE_PATH; Expected = $env:EXE_HASH; Label = 'agent executable' },
@{ Path = $env:CONFIG_PATH; Expected = $env:CONFIG_HASH; Label = 'config file' }
)) {
if (-not (Test-Path $pair.Path)) {
$failures += "$($pair.Label) was removed: $($pair.Path)"
continue
}
$actual = (Get-FileHash -Path $pair.Path -Algorithm SHA256).Hash
if ($actual -ne $pair.Expected) {
$failures += "$($pair.Label) changed: $($pair.Expected) -> $actual"
} else {
Write-Output "$($pair.Label) unchanged (sha256=$actual)"
}
}
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Output "FAIL: $_" }
Write-Error "the aborted update modified the installation"
exit 1
}
exit 0
# The failed stop above left the service in StopPending, and the agent
# treats only Running as active - so an uninstall issued now would skip the
# stop entirely instead of exercising it. Start a fresh wedged run so the
# uninstall meets a Running service, as an operator would.
- name: Re-wedge the service for the uninstall check
if: matrix.os == 'windows-latest'
uses: ./.github/actions/wedged-service
with:
mode: rewedge
service: ${{ matrix.service }}
- name: Assert the uninstall aborts on a wedged stop
if: matrix.os == 'windows-latest'
shell: pwsh
env:
IT_BINARY: ${{ matrix.it_binary }}
ORG_ID: ${{ vars.IT_ORG_ID }}
SERVICE: ${{ matrix.service }}
EXE_PATH: ${{ steps.wedge_hashes.outputs.exe_path }}
CONFIG_PATH: ${{ steps.wedge_hashes.outputs.config_path }}
EXE_HASH: ${{ steps.wedge_hashes.outputs.exe_hash }}
CONFIG_HASH: ${{ steps.wedge_hashes.outputs.config_hash }}
UNINSTALL_PATHS: ${{ matrix.uninstall_paths }}
run: |
$sw = [System.Diagnostics.Stopwatch]::StartNew()
$out = & "./dist/$($env:IT_BINARY)" --uninstall --org-id $env:ORG_ID 2>&1 | Out-String
$sw.Stop()
$global:LASTEXITCODE = 0
$elapsed = [math]::Round($sw.Elapsed.TotalSeconds, 1)
Write-Output "---- uninstaller output after ${elapsed}s ----"
Write-Output $out
$failures = @()
if ($sw.Elapsed.TotalSeconds -lt 20) {
$failures += "uninstall returned after ${elapsed}s, before its 25s stop deadline could elapse"
}
if ($sw.Elapsed.TotalSeconds -gt 180) {
$failures += "uninstall took ${elapsed}s, far past its 25s stop deadline"
}
foreach ($needle in @(
'Failed to stop service',
'did not stop within',
'Uninstall aborted; nothing was removed'
)) {
if (-not $out.Contains($needle)) { $failures += "missing from the log: '$needle'" }
}
foreach ($absent in @('Service deleted', 'Directory deleted')) {
if ($out.Contains($absent)) { $failures += "uninstall proceeded past the failed stop: '$absent'" }
}
# Fully intact or fully removed, never mixed: the registration, the
# installed directories, and both hashed files must all still be there.
if (-not (Get-Service -Name $env:SERVICE -ErrorAction SilentlyContinue)) {
$failures += "service registration $($env:SERVICE) was deleted"
}
foreach ($path in ($env:UNINSTALL_PATHS -split "`n" | ForEach-Object { $_.Trim() } |
Where-Object { $_ })) {
if (-not (Test-Path $path)) { $failures += "installed directory was removed: $path" }
}
foreach ($pair in @(
@{ Path = $env:EXE_PATH; Expected = $env:EXE_HASH; Label = 'agent executable' },
@{ Path = $env:CONFIG_PATH; Expected = $env:CONFIG_HASH; Label = 'config file' }
)) {
if (-not (Test-Path $pair.Path)) {
$failures += "$($pair.Label) was removed: $($pair.Path)"
} elseif ((Get-FileHash -Path $pair.Path -Algorithm SHA256).Hash -ne $pair.Expected) {
$failures += "$($pair.Label) changed during the aborted uninstall"
}
}
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Output "FAIL: $_" }
Write-Error "the uninstall did not abort cleanly on a wedged stop"
exit 1
}
Write-Output "Uninstall aborted on the wedged stop after ${elapsed}s with nothing removed"
exit 0
# Captured before the restore restarts the real agent, so the delta below
# can only come from the recovered service.
- name: Capture recovery baseline counts
if: matrix.os == 'windows-latest'
id: wedge_recovery_baseline
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
run: |
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$subscribed = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
"subscribed=$subscribed" >> $env:GITHUB_OUTPUT
Write-Output "Baseline counts -> subscribed=$subscribed"
# always(): a failed assertion above must not leave the runner's service
# registration pointing at the fixture, or the rest of the job (and the
# final uninstall) would be testing the fixture instead of the agent.
- name: Restore the agent service after the wedge
if: always() && matrix.os == 'windows-latest'
uses: ./.github/actions/wedged-service
with:
mode: restore
service: ${{ matrix.service }}
- name: Assert the recovered endpoint comes back online
if: matrix.os == 'windows-latest'
shell: pwsh
env:
LOG_FILE: ${{ matrix.log_file }}
BASELINE: ${{ steps.wedge_recovery_baseline.outputs.subscribed }}
run: |
$baseline = [int]$env:BASELINE
for ($i = 1; $i -le 60; $i++) {
$content = Get-Content $env:LOG_FILE -Raw -ErrorAction SilentlyContinue
$count = if ($content) { ([regex]::Matches($content, [regex]::Escape("Subscribed to messages"))).Count } else { 0 }
if ($count -gt $baseline) {
Write-Output "Recovered agent subscribed again (count $count > baseline $baseline) after ~$($i * 2)s"
exit 0
}
Start-Sleep -Seconds 2
}
Write-Error "Recovered agent never resubscribed (count stayed at $baseline)"
exit 1
- name: Assert a subsequent update succeeds after recovery
if: matrix.os == 'windows-latest'
shell: pwsh
env:
BINARY: ${{ matrix.binary }}
ORG_ID: ${{ vars.IT_ORG_ID }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NO_AUTO_UPDATES_EXTRA: ${{ matrix.no_auto_updates_extra }}
run: |
# The bound must only catch a wedge: once the endpoint is recovered, an
# ordinary update stops the service, replaces the binary and starts it
# again with no behavioral change.
$out = & "./dist/$($env:BINARY)" --update --org-id $env:ORG_ID --no-auto-updates `
$env:NO_AUTO_UPDATES_EXTRA --github-token $env:GH_TOKEN 2>&1 | Out-String
$global:LASTEXITCODE = 0
Write-Output $out
$failures = @()
foreach ($needle in @('Agent installed to', 'Service started')) {
if (-not $out.Contains($needle)) { $failures += "missing from the log: '$needle'" }
}
if ($out.Contains('Failed to stop service')) {
$failures += "a healthy stop was reported as a failure"
}
if ($failures.Count -gt 0) {
$failures | ForEach-Object { Write-Output "FAIL: $_" }
Write-Error "the post-recovery update did not complete normally"
exit 1
}
Write-Output "Post-recovery update completed normally"
exit 0
- name: Uninstall agent
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --uninstall --org-id ${{ vars.IT_ORG_ID }}
- name: Check deleted directories
uses: ./.github/actions/assert-uninstalled
with:
paths: ${{ matrix.uninstall_paths }}
test-service-user:
permissions:
contents: read
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
needs: [build, build-integration-test, set-matrix]
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
runs-on: ${{ matrix.os }}
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Provision service account
id: provision
uses: ./.github/actions/provision-service-user
with:
username: ${{ matrix.service_user }}
- name: Install agent as service account
uses: ./.github/actions/install-agent
with:
os: ${{ matrix.os }}
binary: ${{ matrix.binary }}
it_binary: ${{ matrix.it_binary }}
config_url: ${{ vars.IT_CONFIG_URL }}
config_secret: ${{ secrets.IT_CONFIG_SECRET }}
org_id: ${{ vars.IT_ORG_ID }}
github_token: ${{ secrets.GITHUB_TOKEN }}
service_username: ${{ matrix.service_username_arg }}
service_password: ${{ steps.provision.outputs.password }}
- name: Assert service runs as account (config flow)
uses: ./.github/actions/assert-service-identity
with:
service: ${{ matrix.service }}
expected_identity: ${{ matrix.service_identity }}
- name: Validate config and get device id
uses: ./.github/actions/validate-config
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
- name: Assert password absent from config (config flow)
uses: ./.github/actions/assert-config-secret-absent
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
secret: ${{ steps.provision.outputs.password }}
- name: Re-register service via update flow (Unix)
if: runner.os != 'Windows'
shell: bash
env:
ORG_ID: ${{ vars.IT_ORG_ID }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SERVICE_USERNAME: ${{ matrix.service_username_arg }}
run: |
sudo "./dist/${{ matrix.binary }}" \
--update \
--org-id "$ORG_ID" \
--github-token "$GH_TOKEN" \
--service-username "$SERVICE_USERNAME"
- name: Re-register service via update flow (Windows)
if: runner.os == 'Windows'
shell: pwsh
env:
ORG_ID: ${{ vars.IT_ORG_ID }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SERVICE_USERNAME: ${{ matrix.service_username_arg }}
SERVICE_PASSWORD: ${{ steps.provision.outputs.password }}
run: |
& "./dist/${{ matrix.binary }}" `
--update `
--org-id $env:ORG_ID `
--github-token $env:GH_TOKEN `
--service-username $env:SERVICE_USERNAME `
--service-password $env:SERVICE_PASSWORD
- name: Assert service runs as account (update flow)
uses: ./.github/actions/assert-service-identity
with:
service: ${{ matrix.service }}
expected_identity: ${{ matrix.service_identity }}
- name: Assert password absent from config (update flow)
uses: ./.github/actions/assert-config-secret-absent
with:
config_dir: ${{ matrix.config_dir }}
org_id: ${{ vars.IT_ORG_ID }}
secret: ${{ steps.provision.outputs.password }}
- name: Uninstall agent
uses: ./.github/actions/run-agent
with:
binary: ${{ matrix.binary }}
args: --uninstall --org-id ${{ vars.IT_ORG_ID }}
- name: Check deleted directories
uses: ./.github/actions/assert-uninstalled
with:
paths: ${{ matrix.uninstall_paths }}
- name: Remove service account
if: always()
uses: ./.github/actions/remove-service-user
with:
username: ${{ matrix.service_user }}