integration-test.yml

name: Integration Test

on:
  workflow_dispatch:
    inputs:
      os:
        description: "Platform to test"
        required: true
        type: choice
        options:
          - all
          - windows-latest
          - ubuntu-latest
          - macos-latest

jobs:
  build:
    permissions:
      contents: read
    uses: ./.github/workflows/build.yml
    with:
      os_list: "${{ inputs.os == 'all' && '[''windows-latest'',''ubuntu-latest'',''macos-latest'']' || format('[{0}{1}{0}]', '''', inputs.os) }}"
      cache: true

  build-integration-test:
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJson(inputs.os == 'all' && '[''windows-latest'',''ubuntu-latest'',''macos-latest'']' || format('[{0}{1}{0}]', '''', inputs.os)) }}
    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          ref: ${{ github.ref }}
          fetch-depth: 0

      - name: Cache integration test build output
        id: cache-build
        uses: actions/cache@v5
        with:
          path: ./dist
          key: build-it-${{ matrix.os }}-go1.24.0-${{ hashFiles('**/*.go', 'go.sum', 'go.mod') }}

      - name: Setup go
        if: steps.cache-build.outputs.cache-hit != 'true'
        uses: actions/setup-go@v6
        with:
          go-version: "1.24.0"
          cache: true

      - name: Build integration test binary
        if: steps.cache-build.outputs.cache-hit != 'true'
        shell: pwsh
        id: build
        run: ./scripts/build_integration_test.ps1 | % { "binary_path=$_" } >> $env:GITHUB_OUTPUT

      - name: Upload integration test binary
        uses: actions/upload-artifact@v6
        with:
          name: it-${{ matrix.os }}
          path: ${{ steps.build.outputs.binary_path || './dist/*' }}

  test-ubuntu-latest:
    permissions:
      contents: read
    needs: [build, build-integration-test]
    if: inputs.os == 'ubuntu-latest' || inputs.os == 'all'
    runs-on: ubuntu-latest
    steps:
      - name: Download compiled binary
        uses: actions/download-artifact@v8
        with:
          name: ubuntu-latest
          path: ./dist

      - name: Download integration test binary
        uses: actions/download-artifact@v8
        with:
          name: it-ubuntu-latest
          path: ./dist

      - name: Make binaries executable
        run: |
          chmod +x ./dist/rewst_agent_config.linux.bin
          chmod +x ./dist/rewst_agent_config.linux.it.bin

      - name: Install agent
        id: install_agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.bin --config-url ${{ vars.IT_CONFIG_URL }} --config-secret ${{ secrets.IT_CONFIG_SECRET }} --org-id ${{ vars.IT_ORG_ID }}
          echo "service=rewst_remote_agent_${{ vars.IT_ORG_ID }}" >> $GITHUB_OUTPUT

      - name: Check service
        shell: bash
        run: |
          systemctl status ${{ steps.install_agent.outputs.service }}

      - name: Get device id
        id: get_device_id
        shell: bash
        run: |
          echo "value=$(cat /etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/config.json | jq -r '.device_id')" >> "$GITHUB_OUTPUT"

      - name: Validate config.json fields
        shell: bash
        run: |
          config="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/config.json"
          failed=false

          for field in device_id rewst_org_id rewst_engine_host shared_access_key azure_iot_hub_host; do
            val=$(jq -r ".$field" "$config")
            if [ -z "$val" ] || [ "$val" = "null" ]; then
              echo "ERROR: Config field '$field' is missing or empty"
              failed=true
            else
              echo "OK: $field = $val"
            fi
          done

          org_id=$(jq -r '.rewst_org_id' "$config")
          if [ "$org_id" != "${{ vars.IT_ORG_ID }}" ]; then
            echo "ERROR: rewst_org_id '$org_id' does not match expected '${{ vars.IT_ORG_ID }}'"
            failed=true
          fi

          if [ "$failed" = true ]; then exit 1; fi
          echo "All config.json fields validated"

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check logs
        shell: bash
        run: |
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          for pattern in "Command completed" "Sending postback" "Postback sent"; do
            if ! echo "$logContent" | grep -qF "$pattern"; then
              echo "ERROR: Expected log line not found: $pattern"
              exit 1
            fi
          done
          echo "All expected log lines found"

      - name: Send failing command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="exit 1"'

      - name: Check logs for error handling
        shell: bash
        run: |
          sleep 15
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          if ! echo "$logContent" | grep -qF "Command failed"; then
            echo "ERROR: Expected 'Command failed' not found in logs"
            exit 1
          fi
          echo "Error handling verified: failing command error was captured"

      - name: Install integration test binary (old version)
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.it.bin --update --org-id ${{ vars.IT_ORG_ID }}
          echo "Installed integration test binary (version 0.0.0-it)"

      - name: Wait for auto updater to trigger and update
        shell: bash
        run: |
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          echo "Waiting for auto updater to detect newer version and update..."
          for i in $(seq 1 60); do
            logContent=$(cat "$logFile" 2>/dev/null || true)
            if echo "$logContent" | grep -qF "Updating agent"; then
              echo "Auto update triggered after ${i}s"
              break
            fi
            sleep 2
          done

      - name: Verify auto update completed
        shell: bash
        run: |
          sleep 15
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          for pattern in "Checking for updates" "Updating agent"; do
            if ! echo "$logContent" | grep -qF "$pattern"; then
              echo "ERROR: Expected auto update log line not found: $pattern"
              exit 1
            fi
          done

          if echo "$logContent" | grep -qF "version=0.0.0-it"; then
            lastVersion=$(echo "$logContent" | grep "Agent Smith started" | tail -1)
            if echo "$lastVersion" | grep -qF "version=0.0.0-it"; then
              echo "ERROR: Agent is still running the integration test version after update"
              exit 1
            fi
          fi
          echo "Auto updater integration test passed"

      - name: Update logging level to debug
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.bin --update --org-id ${{ vars.IT_ORG_ID }} --logging-level debug
          echo "Logging level successfully updated to debug"

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check logs for debug level
        shell: bash
        run: |
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          if ! echo "$logContent" | grep -qF "DEBUG"; then
            echo "ERROR: Expected log line not found: DEBUG"
            exit 1
          fi
          echo "All expected log lines found"

      - name: Stop service
        shell: bash
        run: |
          sudo systemctl stop ${{ steps.install_agent.outputs.service }}

      - name: Delete log file
        shell: bash
        run: |
          sudo rm -f "/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"

      - name: Update agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.bin --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates

      - name: Check logs for no auto updater
        shell: bash
        run: |
          logFile="/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          for i in $(seq 1 30); do
            [ -f "$logFile" ] && break
            echo "Waiting for log file to be created... ($i/30)"
            sleep 1
          done
          logContent=$(cat "$logFile")
          echo "$logContent"

          if echo "$logContent" | grep -qF "Starting auto updater"; then
            echo "ERROR: Unexpected log line found: Starting auto updater"
            exit 1
          fi
          echo "Confirmed: no 'Starting auto updater' log line found"

      - name: Update agent with syslog
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.bin --update --org-id ${{ vars.IT_ORG_ID }} --syslog

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check syslog for command completed
        shell: bash
        run: |
          sleep 15
          entries=$(journalctl -t "rewst_remote_agent_${{ vars.IT_ORG_ID }}" --no-pager 2>/dev/null || sudo grep "rewst_remote_agent_${{ vars.IT_ORG_ID }}" /var/log/syslog 2>/dev/null || sudo grep "rewst_remote_agent_${{ vars.IT_ORG_ID }}" /var/log/messages 2>/dev/null || true)
          echo "$entries"

          if ! echo "$entries" | grep -qF "Command completed"; then
            echo "ERROR: Expected 'Command completed' entry not found in syslog"
            exit 1
          fi
          echo "Found 'Command completed' in syslog"

      - name: Uninstall agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.linux.bin --uninstall --org-id ${{ vars.IT_ORG_ID }}

      - name: Check deleted directories
        shell: pwsh
        run: |
          if (Test-Path -Path "/etc/rewst_remote_agent/${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "/usr/local/bin/rewst_remote_agent/${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "/tmp/rewst_remote_agent/scripts/${{ vars.IT_ORG_ID }}") { exit 1 }
          echo "All directories have been deleted"

  test-windows-latest:
    permissions:
      contents: read
    needs: [build, build-integration-test]
    if: inputs.os == 'windows-latest' || inputs.os == 'all'
    runs-on: windows-latest
    steps:
      - name: Download compiled binary
        uses: actions/download-artifact@v8
        with:
          name: windows-latest
          path: ./dist

      - name: Download integration test binary
        uses: actions/download-artifact@v8
        with:
          name: it-windows-latest
          path: ./dist

      - name: Install agent
        id: install_agent
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.exe --config-url ${{ vars.IT_CONFIG_URL }} --config-secret ${{ secrets.IT_CONFIG_SECRET }} --org-id ${{ vars.IT_ORG_ID }}
          Write-Output "service=RewstRemoteAgent_${{ vars.IT_ORG_ID }}" >> $env:GITHUB_OUTPUT

      - name: Check service
        shell: pwsh
        run: |
          sc.exe query ${{ steps.install_agent.outputs.service }}

      - name: Get device id
        id: get_device_id
        shell: pwsh
        run: |
          (Get-Content C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\config.json | ConvertFrom-Json).device_id | % { "value=$_" } >> $env:GITHUB_OUTPUT

      - name: Validate config.json fields
        shell: pwsh
        run: |
          $config = Get-Content C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\config.json | ConvertFrom-Json

          $requiredFields = @(
            @{ Name = "device_id";          Value = $config.device_id },
            @{ Name = "rewst_org_id";       Value = $config.rewst_org_id },
            @{ Name = "rewst_engine_host";  Value = $config.rewst_engine_host },
            @{ Name = "shared_access_key";  Value = $config.shared_access_key },
            @{ Name = "azure_iot_hub_host"; Value = $config.azure_iot_hub_host }
          )

          $failed = $false
          foreach ($field in $requiredFields) {
            if ([string]::IsNullOrWhiteSpace($field.Value)) {
              Write-Error "Config field '$($field.Name)' is missing or empty"
              $failed = $true
            } else {
              Write-Output "OK: $($field.Name) = $($field.Value)"
            }
          }

          if ($config.rewst_org_id -ne "${{ vars.IT_ORG_ID }}") {
            Write-Error "rewst_org_id '$($config.rewst_org_id)' does not match expected '${{ vars.IT_ORG_ID }}'"
            $failed = $true
          }

          if ($failed) { exit 1 }
          Write-Output "All config.json fields validated"

      - name: Send test command
        shell: pwsh
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' `
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' `
          --form 'commands="[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

          $PS_Results = New-Object -TypeName psobject

          # Start of typed powershell
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"error\" -Value \"\"
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"output\" -Value \"hello world`n\"

          # Post results back to Rewst
          $postData = $PS_Results | ConvertTo-Json
          Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""'

      - name: Check logs
        shell: pwsh
        run: |
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          $expectedPatterns = @(
            "Command completed",
            "Sending postback",
            "Postback already sent"
          )

          foreach ($pattern in $expectedPatterns) {
            if ($logContent -notmatch [regex]::Escape($pattern)) {
              Write-Error "Expected log line not found: $pattern"
              exit 1
            }
          }
          Write-Output "All expected log lines found"

      - name: Send failing command
        shell: pwsh
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' `
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' `
          --form 'commands="[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

          $PS_Results = New-Object -TypeName psobject

          # Start of typed powershell
          throw \"this is a test error\"

          # Post results back to Rewst
          $postData = $PS_Results | ConvertTo-Json
          Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""'

      - name: Check logs for error handling
        shell: pwsh
        run: |
          Start-Sleep -Seconds 15
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          if ($logContent -notmatch "Command failed") {
            Write-Error "Expected error message 'Command failed' not found in logs"
            exit 1
          }
          Write-Output "Error handling verified: failing command error was captured"

      - name: Install integration test binary (old version)
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.it.exe --update --org-id ${{ vars.IT_ORG_ID }}
          Write-Output "Installed integration test binary (version 0.0.0-it)"

      - name: Wait for auto updater to trigger and update
        shell: pwsh
        run: |
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          Write-Output "Waiting for auto updater to detect newer version and update..."
          for ($i = 1; $i -le 60; $i++) {
            $logContent = Get-Content $logFile -Raw -ErrorAction SilentlyContinue
            if ($logContent -match [regex]::Escape("Updating agent")) {
              Write-Output "Auto update triggered after ${i}s"
              break
            }
            Start-Sleep -Seconds 2
          }

      - name: Verify auto update completed
        shell: pwsh
        run: |
          Start-Sleep -Seconds 15
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          foreach ($pattern in @("Checking for updates", "Updating agent")) {
            if ($logContent -notmatch [regex]::Escape($pattern)) {
              Write-Error "Expected auto update log line not found: $pattern"
              exit 1
            }
          }

          $lastStartLine = ($logContent -split "`n" | Where-Object { $_ -match "Agent Smith started" } | Select-Object -Last 1)
          if ($lastStartLine -match [regex]::Escape("version=0.0.0-it")) {
            Write-Error "Agent is still running the integration test version after update"
            exit 1
          }
          Write-Output "Auto updater integration test passed"

      - name: Update logging level to debug
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.exe --update --org-id ${{ vars.IT_ORG_ID }} --logging-level debug
          Write-Output "Logging level successfully updated to debug"

      - name: Send test command
        shell: pwsh
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' `
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' `
          --form 'commands="[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

          $PS_Results = New-Object -TypeName psobject

          # Start of typed powershell
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"error\" -Value \"\"
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"output\" -Value \"hello world`n\"

          # Post results back to Rewst
          $postData = $PS_Results | ConvertTo-Json
          Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""'

      - name: Check logs
        shell: pwsh
        run: |
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          if ($logContent -notmatch [regex]::Escape("DEBUG")) {
            Write-Error "Expected log line not found: DEBUG"
          }
          Write-Output "All expected log lines found"

      - name: Stop service
        shell: pwsh
        run: |
          sc.exe stop ${{ steps.install_agent.outputs.service }}

      - name: Delete log file
        shell: pwsh
        run: |
          Remove-Item -Force "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"

      - name: Update agent
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.exe --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates --disable-agent-postback

      - name: Check logs for no auto updater
        shell: pwsh
        run: |
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          for ($i = 1; $i -le 30; $i++) {
            if (Test-Path $logFile) { break }
            Write-Output "Waiting for log file to be created... ($i/30)"
            Start-Sleep -Seconds 1
          }
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          if ($logContent -match [regex]::Escape("Starting auto updater")) {
            Write-Error "Unexpected log line found: Starting auto updater"
            exit 1
          }
          Write-Output "Confirmed: no 'Starting auto updater' log line found"

      - name: Send test command
        shell: pwsh
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' `
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' `
          --form 'commands="[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

          $PS_Results = New-Object -TypeName psobject

          # Start of typed powershell
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"error\" -Value \"\"
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"output\" -Value \"hello world`n\"

          # Post results back to Rewst
          $postData = $PS_Results | ConvertTo-Json
          Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""'

      - name: Check logs for command received
        shell: pwsh
        run: |
          $logFile = "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}\rewst_agent.log"
          $logContent = Get-Content $logFile -Raw
          Write-Output $logContent

          if ($logContent -notmatch [regex]::Escape("Command completed")) {
            Write-Error "Expected log line not found: Command completed"
            exit 1
          }

          if ($logContent -match [regex]::Escape("Postback already sent")) {
            Write-Error "Unexpected log line found: Postback already sent"
            exit 1
          }
          Write-Output "All log checks passed"

      - name: Update agent with syslog
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.exe --update --org-id ${{ vars.IT_ORG_ID }} --syslog

      - name: Send test command
        shell: pwsh
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' `
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' `
          --form 'commands="[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

          $PS_Results = New-Object -TypeName psobject

          # Start of typed powershell
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"error\" -Value \"\"
          $PS_Results | Add-Member -MemberType NoteProperty -Name \"output\" -Value \"hello world`n\"

          # Post results back to Rewst
          $postData = $PS_Results | ConvertTo-Json
          Invoke-RestMethod -Method \"Post\" -Uri $post_url -Body $postData -ContentType \"application/json\""'

      - name: Check Windows Event Log for command completed
        shell: pwsh
        run: |
          $events = Get-WinEvent -LogName Application -FilterXPath "*[System[Provider[@Name='RewstRemoteAgent_${{ vars.IT_ORG_ID }}']]]"
          Write-Output "Found $($events.Count) event(s)"
          $events | ForEach-Object { Write-Output $_.Message }

          $matched = $events | Where-Object { $_.Message -match "Command completed" }
          if (-not $matched) {
            Write-Error "Expected 'Command completed' entry not found in Windows Event Log"
            exit 1
          }
          Write-Output "Found 'Command completed' in Windows Event Log"

      - name: Uninstall agent
        shell: pwsh
        run: |
          ./dist/rewst_agent_config.win.exe --uninstall --org-id ${{ vars.IT_ORG_ID }}

      - name: Check deleted directories
        shell: pwsh
        run: |
          if (Test-Path -Path "C:\ProgramData\RewstRemoteAgent\${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "C:\Program Files\RewstRemoteAgent\${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "C:\RewstRemoteAgent\scripts\${{ vars.IT_ORG_ID }}") { exit 1 }
          echo "All directories have been deleted"

  test-macos-latest:
    permissions:
      contents: read
    needs: [build, build-integration-test]
    if: inputs.os == 'macos-latest' || inputs.os == 'all'
    runs-on: macos-latest
    steps:
      - name: Download compiled binary
        uses: actions/download-artifact@v8
        with:
          name: macos-latest
          path: ./dist

      - name: Download integration test binary
        uses: actions/download-artifact@v8
        with:
          name: it-macos-latest
          path: ./dist

      - name: Make binaries executable
        run: |
          chmod +x ./dist/rewst_agent_config.mac-os.bin
          chmod +x ./dist/rewst_agent_config.mac-os.it.bin

      - name: Install agent
        id: install_agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.bin --config-url ${{ vars.IT_CONFIG_URL }} --config-secret ${{ secrets.IT_CONFIG_SECRET }} --org-id ${{ vars.IT_ORG_ID }}
          echo "service=io.rewst.remote_agent_${{ vars.IT_ORG_ID }}" >> $GITHUB_OUTPUT

      - name: Check service
        shell: bash
        run: |
          sudo launchctl print system/${{ steps.install_agent.outputs.service }}

      - name: Get device id
        id: get_device_id
        shell: bash
        run: |
          echo "value=$(cat "/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/config.json" | jq -r '.device_id')" >> "$GITHUB_OUTPUT"

      - name: Validate config.json fields
        shell: bash
        run: |
          config="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/config.json"
          failed=false

          for field in device_id rewst_org_id rewst_engine_host shared_access_key azure_iot_hub_host; do
            val=$(jq -r ".$field" "$config")
            if [ -z "$val" ] || [ "$val" = "null" ]; then
              echo "ERROR: Config field '$field' is missing or empty"
              failed=true
            else
              echo "OK: $field = $val"
            fi
          done

          org_id=$(jq -r '.rewst_org_id' "$config")
          if [ "$org_id" != "${{ vars.IT_ORG_ID }}" ]; then
            echo "ERROR: rewst_org_id '$org_id' does not match expected '${{ vars.IT_ORG_ID }}'"
            failed=true
          fi

          if [ "$failed" = true ]; then exit 1; fi
          echo "All config.json fields validated"

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check logs
        shell: bash
        run: |
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          for pattern in "Command completed" "Sending postback" "Postback sent"; do
            if ! echo "$logContent" | grep -qF "$pattern"; then
              echo "ERROR: Expected log line not found: $pattern"
              exit 1
            fi
          done
          echo "All expected log lines found"

      - name: Send failing command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="exit 1"'

      - name: Check logs for error handling
        shell: bash
        run: |
          sleep 15
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          if ! echo "$logContent" | grep -qF "Command failed"; then
            echo "ERROR: Expected 'Command failed' not found in logs"
            exit 1
          fi
          echo "Error handling verified: failing command error was captured"

      - name: Install integration test binary (old version)
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.it.bin --update --org-id ${{ vars.IT_ORG_ID }}
          echo "Installed integration test binary (version 0.0.0-it)"

      - name: Wait for auto updater to trigger and update
        shell: bash
        run: |
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          echo "Waiting for auto updater to detect newer version and update..."
          for i in $(seq 1 60); do
            logContent=$(cat "$logFile" 2>/dev/null || true)
            if echo "$logContent" | grep -qF "Updating agent"; then
              echo "Auto update triggered after ${i}s"
              break
            fi
            sleep 2
          done

      - name: Verify auto update completed
        shell: bash
        run: |
          sleep 15
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          for pattern in "Checking for updates" "Updating agent"; do
            if ! echo "$logContent" | grep -qF "$pattern"; then
              echo "ERROR: Expected auto update log line not found: $pattern"
              exit 1
            fi
          done

          lastVersion=$(echo "$logContent" | grep "Agent Smith started" | tail -1)
          if echo "$lastVersion" | grep -qF "version=0.0.0-it"; then
            echo "ERROR: Agent is still running the integration test version after update"
            exit 1
          fi
          echo "Auto updater integration test passed"

      - name: Update logging level to debug
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.bin --update --org-id ${{ vars.IT_ORG_ID }} --logging-level debug
          echo "Logging level successfully updated to debug"

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check logs for debug level
        shell: bash
        run: |
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          logContent=$(cat "$logFile")
          echo "$logContent"

          if ! echo "$logContent" | grep -qF "DEBUG"; then
            echo "ERROR: Expected log line not found: DEBUG"
            exit 1
          fi
          echo "All expected log lines found"

      - name: Stop service
        shell: bash
        run: |
          sudo launchctl stop system/${{ steps.install_agent.outputs.service }} || true

      - name: Delete log file
        shell: bash
        run: |
          sudo rm -f "/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"

      - name: Update agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.bin --update --org-id ${{ vars.IT_ORG_ID }} --no-auto-updates

      - name: Check logs for no auto updater
        shell: bash
        run: |
          logFile="/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}/rewst_agent.log"
          for i in $(seq 1 30); do
            [ -f "$logFile" ] && break
            echo "Waiting for log file to be created... ($i/30)"
            sleep 1
          done
          logContent=$(cat "$logFile")
          echo "$logContent"

          if echo "$logContent" | grep -qF "Starting auto updater"; then
            echo "ERROR: Unexpected log line found: Starting auto updater"
            exit 1
          fi
          echo "Confirmed: no 'Starting auto updater' log line found"

      - name: Update agent with syslog
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.bin --update --org-id ${{ vars.IT_ORG_ID }} --syslog

      - name: Send test command
        shell: bash
        run: |
          curl --location '${{ vars.IT_SEND_COMMAND_TRIGGER_URL }}' \
          --form 'device_id="${{ steps.get_device_id.outputs.value }}"' \
          --form 'commands="echo \"hello world\""'

      - name: Check syslog for command completed
        shell: bash
        run: |
          sleep 15
          entries=$(log show --predicate 'processImagePath contains "logger"' --info --last 5m 2>/dev/null || true)
          echo "$entries"

          if ! echo "$entries" | grep -qF "Command completed"; then
            echo "ERROR: Expected 'Command completed' entry not found in syslog"
            exit 1
          fi
          echo "Found 'Command completed' in syslog"

      - name: Uninstall agent
        shell: bash
        run: |
          sudo ./dist/rewst_agent_config.mac-os.bin --uninstall --org-id ${{ vars.IT_ORG_ID }}

      - name: Check deleted directories
        shell: pwsh
        run: |
          if (Test-Path -Path "/Library/Application Support/rewst_remote_agent/${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "/usr/local/bin/rewst_remote_agent/${{ vars.IT_ORG_ID }}") { exit 1 }
          if (Test-Path -Path "$env:TMPDIR/rewst_remote_agent/scripts/${{ vars.IT_ORG_ID }}") { exit 1 }
          echo "All directories have been deleted"