Nightly #96
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| on: | |
| schedule: | |
| # 02:17 UTC daily. Off the hour to avoid GitHub's on-the-hour scheduling lag. | |
| - cron: "17 2 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| # Checkout only. Issues are filed with CICDBOT_TOKEN, not the workflow token. | |
| contents: read | |
| concurrency: | |
| group: nightly | |
| cancel-in-progress: false | |
| jobs: | |
| integration: | |
| name: "Integration: ${{ matrix.name }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # The per-test guard variable is exported per shard inside the check step. | |
| # All API credentials are provided here as the union of every adapter's | |
| # needs; only the targeted adapter's test actually runs in a given shard | |
| # (gated by its SORTIE_*_TEST variable), so unused keys are inert. | |
| env: | |
| SORTIE_JIRA_ENDPOINT: ${{ secrets.SORTIE_JIRA_ENDPOINT }} | |
| SORTIE_JIRA_API_KEY: ${{ secrets.SORTIE_JIRA_API_KEY }} | |
| SORTIE_JIRA_PROJECT: ${{ secrets.SORTIE_JIRA_PROJECT }} | |
| SORTIE_LINEAR_API_KEY: ${{ secrets.SORTIE_LINEAR_API_KEY }} | |
| SORTIE_LINEAR_TEAM_KEY: ${{ vars.SORTIE_LINEAR_TEAM_KEY }} | |
| SORTIE_GITHUB_TOKEN: ${{ secrets.SORTIE_GITHUB_TOKEN }} | |
| SORTIE_GITHUB_PROJECT: sortie-ai/sortie-test | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} | |
| KIRO_API_KEY: ${{ secrets.KIRO_API_KEY }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Claude Code | |
| adapter: claude | |
| kind: agent | |
| guard: SORTIE_CLAUDE_TEST | |
| run_filter: Integration | |
| test_path: ./internal/agent/claude/... | |
| timeout: 300s | |
| node: true | |
| install: npm install -g @anthropic-ai/claude-code | |
| version_cmd: claude --version | |
| source: "npm: @anthropic-ai/claude-code (unpinned, installs latest)" | |
| - name: OpenAI Codex | |
| adapter: codex | |
| kind: agent | |
| guard: SORTIE_CODEX_TEST | |
| run_filter: Integration | |
| test_path: ./internal/agent/codex/... | |
| timeout: 300s | |
| node: true | |
| install: npm install -g @openai/codex | |
| version_cmd: codex --version | |
| source: "npm: @openai/codex (unpinned, installs latest)" | |
| - name: GitHub Copilot CLI | |
| adapter: copilot | |
| kind: agent | |
| guard: SORTIE_COPILOT_TEST | |
| run_filter: Integration | |
| test_path: ./internal/agent/copilot/... | |
| timeout: 300s | |
| node: true | |
| install: npm install -g @github/copilot | |
| version_cmd: copilot --version | |
| source: "npm: @github/copilot (unpinned, installs latest)" | |
| - name: OpenCode | |
| adapter: opencode | |
| kind: agent | |
| guard: SORTIE_OPENCODE_TEST | |
| run_filter: Integration | |
| test_path: ./internal/agent/opencode/... | |
| timeout: 300s | |
| node: true | |
| install: npm install -g opencode-ai | |
| version_cmd: opencode --version | |
| source: "npm: opencode-ai (unpinned, installs latest)" | |
| - name: Kiro | |
| adapter: kiro | |
| kind: agent | |
| guard: SORTIE_KIRO_TEST | |
| run_filter: Integration | |
| test_path: ./internal/agent/kiro/... | |
| timeout: 300s | |
| node: false | |
| install: curl -fsSL https://cli.kiro.dev/install | bash | |
| version_cmd: kiro-cli --version | |
| source: "curl https://cli.kiro.dev/install (unpinned, installs latest)" | |
| - name: Jira | |
| adapter: jira | |
| kind: tracker | |
| guard: SORTIE_JIRA_TEST | |
| run_filter: Integration | |
| test_path: ./internal/tracker/jira/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: "" | |
| source: "Jira Cloud REST API (remote service, no pinned client)" | |
| - name: Linear | |
| adapter: linear | |
| kind: tracker | |
| guard: SORTIE_LINEAR_TEST | |
| run_filter: Integration | |
| test_path: ./internal/tracker/linear/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: "" | |
| source: "Linear GraphQL API (remote service, no pinned client)" | |
| - name: Gitea | |
| adapter: gitea | |
| kind: tracker | |
| guard: SORTIE_GITEA_TEST | |
| run_filter: Integration | |
| test_path: ./internal/scm/gitea/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: printenv GITEA_IMAGE | |
| source: "docker.gitea.com/gitea:1.27.0-rootless (pinned container image)" | |
| container_image: docker.gitea.com/gitea:1.27.0-rootless | |
| provision_script: scripts/gitea-integration-provision.sh | |
| - name: GitLab | |
| adapter: gitlab | |
| kind: tracker | |
| guard: SORTIE_GITLAB_TEST | |
| run_filter: Integration | |
| test_path: ./internal/scm/gitlab/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: printenv GITLAB_IMAGE | |
| source: "gitlab/gitlab-ce:19.2.1-ce.0 (pinned container image)" | |
| container_image: gitlab/gitlab-ce:19.2.1-ce.0 | |
| provision_script: scripts/gitlab-integration-provision.sh | |
| - name: GitLab.com | |
| adapter: gitlab-saas | |
| kind: tracker | |
| guard: SORTIE_GITLAB_TEST | |
| run_filter: Integration | |
| test_path: ./internal/scm/gitlab/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: "" | |
| source: "GitLab.com REST API v4 (remote service, no pinned version)" | |
| hosted: true | |
| - name: GitHub | |
| adapter: github | |
| kind: SCM | |
| guard: SORTIE_GITHUB_TEST | |
| run_filter: Integration | |
| test_path: ./internal/scm/github/... | |
| timeout: "" | |
| node: false | |
| install: "" | |
| version_cmd: "" | |
| source: "GitHub REST API (remote service, no pinned client)" | |
| - name: GitHub orchestrator E2E | |
| adapter: github-e2e | |
| kind: orchestrator E2E | |
| guard: SORTIE_GITHUB_E2E | |
| run_filter: TestGitHubIntegration | |
| test_path: ./internal/orchestrator/... | |
| timeout: 120s | |
| node: false | |
| install: "" | |
| version_cmd: "" | |
| source: "GitHub REST API (remote service, no pinned client)" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Set up Node.js | |
| if: ${{ matrix.node }} | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: "24" | |
| - name: Publish hosted GitLab coordinates | |
| if: ${{ matrix.hosted }} | |
| env: | |
| HOSTED_TOKEN: ${{ secrets.SORTIE_GITLAB_TOKEN }} | |
| HOSTED_ENDPOINT: ${{ vars.SORTIE_GITLAB_ENDPOINT }} | |
| HOSTED_PROJECT: ${{ vars.SORTIE_GITLAB_PROJECT }} | |
| run: | | |
| if [ -z "${HOSTED_TOKEN}" ]; then | |
| echo "::warning::SORTIE_GITLAB_TOKEN secret is not configured" | |
| fi | |
| if [ -z "${HOSTED_ENDPOINT}" ]; then | |
| echo "::warning::SORTIE_GITLAB_ENDPOINT variable is not configured" | |
| fi | |
| if [ -z "${HOSTED_PROJECT}" ]; then | |
| echo "::warning::SORTIE_GITLAB_PROJECT variable is not configured" | |
| fi | |
| if [ -n "${HOSTED_TOKEN}" ]; then | |
| echo "::add-mask::${HOSTED_TOKEN}" | |
| fi | |
| { | |
| echo "SORTIE_GITLAB_ENDPOINT=${HOSTED_ENDPOINT}" | |
| echo "SORTIE_GITLAB_TOKEN=${HOSTED_TOKEN}" | |
| echo "SORTIE_GITLAB_PROJECT=${HOSTED_PROJECT}" | |
| } >> "$GITHUB_ENV" | |
| exit 0 | |
| - name: Boot and provision the containerized instance | |
| if: ${{ matrix.provision_script != '' }} | |
| run: ${{ matrix.provision_script }} "${{ matrix.container_image }}" | |
| - name: Run integration check | |
| id: check | |
| env: | |
| ADAPTER: ${{ matrix.adapter }} | |
| GUARD: ${{ matrix.guard }} | |
| INSTALL_CMD: ${{ matrix.install }} | |
| VERSION_CMD: ${{ matrix.version_cmd }} | |
| RUN_FILTER: ${{ matrix.run_filter }} | |
| TEST_PATH: ${{ matrix.test_path }} | |
| TIMEOUT: ${{ matrix.timeout }} | |
| # Consumed only by the copilot adapter's test, which reads GH_TOKEN. | |
| # The issue-reporting steps override GH_TOKEN with CICDBOT_TOKEN. | |
| GH_TOKEN: ${{ secrets.SORTIE_COPILOT_TOKEN }} | |
| run: | | |
| # The runner's default bash uses -e; disable it so a failing test is | |
| # captured and reported instead of aborting the step. | |
| set +e | |
| if [ -n "${INSTALL_CMD}" ]; then | |
| echo "::group::Install ${ADAPTER} CLI" | |
| bash -c "${INSTALL_CMD}" | |
| install_status=$? | |
| echo "::endgroup::" | |
| if [ "${install_status}" -ne 0 ]; then | |
| echo "ADAPTER_VERSION=install failed (exit ${install_status})" >> "$GITHUB_ENV" | |
| echo "outcome=failure" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| # Record the version actually under test for the issue body. | |
| VER="remote service (no pinned client)" | |
| if [ -n "${VERSION_CMD}" ]; then | |
| VER="$(bash -c "${VERSION_CMD}" 2>&1 | head -n1)" | |
| if [ -z "${VER}" ]; then VER="unknown"; fi | |
| fi | |
| echo "ADAPTER_VERSION=${VER}" >> "$GITHUB_ENV" | |
| # Enable only the targeted adapter's integration tests. | |
| export "${GUARD}=1" | |
| TIMEOUT_FLAG=() | |
| if [ -n "${TIMEOUT}" ]; then TIMEOUT_FLAG=(-timeout "${TIMEOUT}"); fi | |
| go test -json -race -count=1 "${TIMEOUT_FLAG[@]}" -run "${RUN_FILTER}" "${TEST_PATH}" 2>&1 | tee gotest.json | |
| status=${PIPESTATUS[0]} | |
| if [ "${status}" -ne 0 ]; then | |
| echo "outcome=failure" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "outcome=success" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: File or update failure issue | |
| if: steps.check.outputs.outcome == 'failure' | |
| env: | |
| GH_TOKEN: ${{ secrets.CICDBOT_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| ADAPTER: ${{ matrix.adapter }} | |
| ADAPTER_NAME: ${{ matrix.name }} | |
| KIND: ${{ matrix.kind }} | |
| SOURCE: ${{ matrix.source }} | |
| run: | | |
| # Backticks in the printf formats below are literal Markdown for the | |
| # issue body, not command substitution. | |
| # shellcheck disable=SC2016 | |
| set -euo pipefail | |
| export TITLE="Nightly integration failure: ${ADAPTER_NAME}" | |
| FAILED="" | |
| LOG="Install step failed before tests ran; no test output captured." | |
| if [ -f gotest.json ]; then | |
| FAILED="$(jq -rR 'fromjson? | select(.Action == "fail" and (.Test != null)) | .Test' gotest.json 2>/dev/null | sort -u)" | |
| LOG="$(jq -jR 'fromjson? | select(.Action == "output") | .Output' gotest.json 2>/dev/null | tail -c 6000)" | |
| if [ -z "${LOG}" ]; then LOG="$(tail -c 6000 gotest.json)"; fi | |
| fi | |
| if [ -n "${FAILED}" ]; then | |
| FAILED_MD="$(printf '%s\n' "${FAILED}" | sed 's/^/- `/; s/$/`/')" | |
| else | |
| FAILED_MD="_No test-level failures parsed (build error, panic, timeout, or install failure). See the log excerpt._" | |
| fi | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| NOW="$(date -u '+%Y-%m-%d %H:%M UTC')" | |
| { | |
| printf 'Nightly integration tests failed for `%s`.\n\n' "${ADAPTER}" | |
| printf '| Field | Value |\n|---|---|\n' | |
| printf '| Adapter | `%s` (%s) |\n' "${ADAPTER}" "${KIND}" | |
| printf '| Tested version | %s |\n' "${ADAPTER_VERSION}" | |
| printf '| Install source | %s |\n' "${SOURCE}" | |
| printf '| Run | %s |\n' "${RUN_URL}" | |
| printf '| Commit | `%s` |\n' "${GITHUB_SHA}" | |
| printf '| Date | %s |\n\n' "${NOW}" | |
| printf '### Failing tests\n\n%s\n\n' "${FAILED_MD}" | |
| printf '### Log excerpt\n\n```\n%s\n```\n\n' "${LOG}" | |
| printf '%s\n' "_Filed by the nightly integration monitor. It comments on recurrence and closes automatically when this adapter passes again._" | |
| } > body.md | |
| NUM="$(gh issue list --state open --limit 100 --search "in:title \"${TITLE}\"" --json number,title \ | |
| | jq -r --arg t "${TITLE}" '[.[] | select(.title == $t) | .number] | (first // empty)')" | |
| if [ -n "${NUM}" ]; then | |
| gh issue comment "${NUM}" --body-file body.md | |
| echo "Updated existing issue #${NUM}" | |
| else | |
| gh label create ci-nightly --color D73A4A --description "Nightly integration test failures" --force >/dev/null 2>&1 || true | |
| gh issue create --title "${TITLE}" --label ci-nightly --body-file body.md | |
| fi | |
| - name: Close resolved issue | |
| if: steps.check.outputs.outcome == 'success' | |
| env: | |
| GH_TOKEN: ${{ secrets.CICDBOT_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| ADAPTER: ${{ matrix.adapter }} | |
| ADAPTER_NAME: ${{ matrix.name }} | |
| run: | | |
| set -euo pipefail | |
| export TITLE="Nightly integration failure: ${ADAPTER_NAME}" | |
| NUM="$(gh issue list --state open --limit 100 --search "in:title \"${TITLE}\"" --json number,title \ | |
| | jq -r --arg t "${TITLE}" '[.[] | select(.title == $t) | .number] | (first // empty)')" | |
| if [ -n "${NUM}" ]; then | |
| RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| NOW="$(date -u '+%Y-%m-%d %H:%M UTC')" | |
| gh issue comment "${NUM}" --body "Recovered: \`${ADAPTER}\` integration tests passed on ${NOW}. Closing. Run: ${RUN_URL}" | |
| gh issue close "${NUM}" | |
| echo "Closed issue #${NUM} (recovered)" | |
| fi | |
| - name: Fail job on integration failure | |
| if: steps.check.outputs.outcome == 'failure' | |
| env: | |
| ADAPTER: ${{ matrix.adapter }} | |
| run: | | |
| echo "::error::Integration check failed for ${ADAPTER}" | |
| exit 1 | |
| hook-stress: | |
| name: Windows hook stress | |
| runs-on: windows-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Go | |
| uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Run the hook stress harness | |
| env: | |
| SORTIE_HOOKSTRESS_TEST: "1" | |
| run: go test -v -count=1 -timeout 15m ./internal/workspace/... |