feat(tools): structured shell output envelope and per-path read sandb… #3159
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| security-events: write | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: "-D warnings" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| detect-changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| outputs: | |
| docs-only: ${{ steps.classify.outputs.docs-only }} | |
| specs-only: ${{ steps.classify.outputs.specs-only }} | |
| run-full-ci: ${{ steps.classify.outputs.run-full-ci }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Filter changed paths | |
| uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| with: | |
| filters: | | |
| docs: | |
| - 'book/src/**' | |
| - 'docs/**' | |
| - '**.md' | |
| - '.local/testing/**' | |
| specs: | |
| - 'specs/**' | |
| code: | |
| - 'crates/**' | |
| - 'src/**' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| - 'build.rs' | |
| workflows: | |
| - '.github/workflows/**' | |
| - '.cargo/**' | |
| shell: | |
| - 'install/**' | |
| - name: Classify changes | |
| id: classify | |
| run: | | |
| DOCS="${{ steps.filter.outputs.docs }}" | |
| SPECS="${{ steps.filter.outputs.specs }}" | |
| CODE="${{ steps.filter.outputs.code }}" | |
| WORKFLOWS="${{ steps.filter.outputs.workflows }}" | |
| # docs-only: docs or specs changed, nothing that requires building | |
| if [[ "$CODE" == "false" && "$WORKFLOWS" == "false" && ("$DOCS" == "true" || "$SPECS" == "true") ]]; then | |
| echo "docs-only=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "docs-only=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # specs-only: specs changed, no code or workflow changes | |
| if [[ "$SPECS" == "true" && "$CODE" == "false" && "$WORKFLOWS" == "false" && "$DOCS" == "false" ]]; then | |
| echo "specs-only=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "specs-only=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| # run-full-ci: any code, workflow, or shell change triggers full CI | |
| if [[ "$CODE" == "true" || "$WORKFLOWS" == "true" ]]; then | |
| echo "run-full-ci=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run-full-ci=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| cla: | |
| name: CLA Check | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1 | |
| with: | |
| path-to-signatures: cla-signatures.json | |
| path-to-document: https://github.com/bug-ops/zeph/blob/main/.github/CLA.md | |
| branch: cla-signatures | |
| allowlist: bug-ops,dependabot[bot],github-actions[bot] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| lint-shellcheck: | |
| name: Lint (shellcheck) | |
| needs: detect-changes | |
| # Run when shell scripts changed, or when doing full CI (workflows changed) | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Run shellcheck | |
| uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 | |
| with: | |
| scandir: install | |
| lint-fmt: | |
| name: Lint (fmt) | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@5b842231ba77f5c045dba54ac5560fed2db780e2 # nightly | |
| with: | |
| toolchain: nightly | |
| components: rustfmt | |
| - name: Check formatting | |
| run: cargo +nightly fmt --check | |
| lint-clippy: | |
| name: Lint (clippy) | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-targets: "false" | |
| shared-key: "ci" | |
| - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 | |
| - name: Clippy | |
| run: cargo clippy --profile ci --workspace --features full -- -D warnings | |
| build-tests: | |
| name: Build Tests (${{ matrix.os }}) | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-targets: "false" | |
| shared-key: "ci" | |
| - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 | |
| - uses: taiki-e/install-action@5ce196a31930e2795a1b2185c2f79b55c0d57733 # nextest | |
| - name: Build and archive tests | |
| run: cargo nextest archive --config-file .github/nextest.toml --cargo-profile ci --workspace --features full --lib --bins --tests --archive-file nextest-archive.tar.zst | |
| - name: Upload test archive | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: nextest-archive-${{ matrix.os }} | |
| path: nextest-archive.tar.zst | |
| retention-days: 1 | |
| - name: Build and archive zeph-db postgres integration tests | |
| if: matrix.os == 'ubuntu-latest' | |
| run: cargo nextest archive --config-file .github/nextest.toml --cargo-profile ci -p zeph-db --no-default-features --features test-utils --tests --archive-file nextest-archive-zeph-db-postgres.tar.zst | |
| - name: Upload zeph-db postgres test archive | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: nextest-archive-zeph-db-postgres | |
| path: nextest-archive-zeph-db-postgres.tar.zst | |
| retention-days: 1 | |
| - name: Upload binary | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: zeph-binary | |
| path: target/ci/zeph | |
| retention-days: 1 | |
| test: | |
| name: "Test (shard ${{ matrix.partition }})" | |
| needs: [detect-changes, lint-fmt, lint-clippy, build-tests] | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| partition: ["1/5", "2/5", "3/5", "4/5", "5/5"] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: taiki-e/install-action@5ce196a31930e2795a1b2185c2f79b55c0d57733 # nextest | |
| - name: Download test archive | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: nextest-archive-ubuntu-latest | |
| - name: "Run tests (shard ${{ matrix.partition }})" | |
| run: | | |
| cargo nextest run \ | |
| --config-file .github/nextest.toml \ | |
| --archive-file nextest-archive.tar.zst \ | |
| --workspace-remap . \ | |
| --profile ci-partition \ | |
| --partition hash:${{ matrix.partition }} | |
| integration: | |
| name: Integration Tests | |
| needs: [detect-changes, lint-fmt, lint-clippy, build-tests] | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: taiki-e/install-action@5ce196a31930e2795a1b2185c2f79b55c0d57733 # nextest | |
| - name: Download test archive | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: nextest-archive-ubuntu-latest | |
| - name: Run integration tests (testcontainers) | |
| run: cargo nextest run --config-file .github/nextest.toml --archive-file nextest-archive.tar.zst --workspace-remap . --profile ci -E 'binary(~integration)' | |
| - name: Download zeph-db postgres test archive | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: nextest-archive-zeph-db-postgres | |
| - name: Run postgres integration tests (testcontainers) | |
| run: cargo nextest run --config-file .github/nextest.toml --archive-file nextest-archive-zeph-db-postgres.tar.zst --workspace-remap . --profile ci --run-ignored ignored-only | |
| - name: Run Qdrant integration tests (testcontainers) | |
| run: cargo nextest run --config-file .github/nextest.toml --archive-file nextest-archive.tar.zst --workspace-remap . --profile ci --run-ignored ignored-only -E 'binary(qdrant_integration)' | |
| coverage: | |
| name: Coverage | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.detect-changes.outputs.run-full-ci == 'true' | |
| needs: [detect-changes, lint-fmt, lint-clippy] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # coverage uses -C instrument-coverage which produces different artifacts than | |
| # normal builds; sccache still helps for unchanged crates between main pushes | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-targets: "false" | |
| shared-key: "coverage" | |
| - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 | |
| - uses: taiki-e/install-action@2f388922703f86d338d8d1e86e4cedef98b9e790 # cargo-llvm-cov | |
| - uses: taiki-e/install-action@5ce196a31930e2795a1b2185c2f79b55c0d57733 # nextest | |
| - name: Generate coverage | |
| run: cargo llvm-cov nextest --config-file .github/nextest.toml --cargo-profile ci --workspace --features full --lib --bins --lcov --output-path lcov.info | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: lcov.info | |
| fail_ci_if_error: false | |
| docker-build-and-scan: | |
| name: Docker Build and Security Scan | |
| needs: [detect-changes, lint-fmt, lint-clippy, build-tests] | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download binary | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: zeph-binary | |
| path: target/ci | |
| - name: Prepare binaries for Docker | |
| run: | | |
| mkdir -p binaries | |
| cp target/ci/zeph binaries/zeph-amd64 | |
| cp target/ci/zeph binaries/zeph-arm64 | |
| - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 | |
| - name: Build Docker image | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| load: true | |
| tags: zeph:local | |
| push: false | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@e368e328979b113139d6f9068e03accaed98a518 # 0.34.1 | |
| with: | |
| image-ref: zeph:local | |
| format: sarif | |
| output: trivy-results.sarif | |
| severity: CRITICAL,HIGH | |
| ignore-unfixed: true | |
| exit-code: '1' | |
| limit-severities-for-sarif: true | |
| - name: Upload Trivy results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4 | |
| if: always() | |
| with: | |
| sarif_file: trivy-results.sarif | |
| bundle-check: | |
| name: Bundle Check (${{ matrix.bundle }}) | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| bundle: [desktop, ide, server, chat, ml] | |
| include: | |
| - bundle: ml | |
| allow_failure: true | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-targets: "false" | |
| shared-key: "bundle-check" | |
| - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 | |
| - name: Check bundle | |
| continue-on-error: ${{ matrix.allow_failure == true }} | |
| run: cargo check --features ${{ matrix.bundle }} | |
| build-postgres: | |
| name: Build (postgres) | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.run-full-ci == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 | |
| with: | |
| cache-targets: "false" | |
| shared-key: "ci-postgres" | |
| - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 | |
| - name: Check postgres build (zeph-db) | |
| run: cargo check -p zeph-db --no-default-features --features postgres | |
| validate-specs: | |
| name: Validate Specs | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.specs-only == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Check spec structure | |
| run: | | |
| echo "Spec files found:" | |
| find specs -name "*.md" -type f | sort | |
| echo "Total: $(find specs -name "*.md" -type f | wc -l)" | |
| # Verify README index exists | |
| test -f specs/README.md || (echo "::error::specs/README.md index is missing" && exit 1) | |
| ci-status: | |
| name: CI Status | |
| if: always() | |
| needs: [cla, lint-shellcheck, lint-fmt, lint-clippy, build-tests, test, integration, coverage, docker-build-and-scan, bundle-check, validate-specs, build-postgres] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| steps: | |
| - name: Check all jobs | |
| run: | | |
| results=( | |
| "${{ needs.cla.result }}" | |
| "${{ needs.lint-shellcheck.result }}" | |
| "${{ needs.lint-fmt.result }}" | |
| "${{ needs.lint-clippy.result }}" | |
| "${{ needs.build-tests.result }}" | |
| "${{ needs.test.result }}" | |
| "${{ needs.integration.result }}" | |
| "${{ needs.coverage.result }}" | |
| "${{ needs.docker-build-and-scan.result }}" | |
| "${{ needs.bundle-check.result }}" | |
| "${{ needs.validate-specs.result }}" | |
| "${{ needs.build-postgres.result }}" | |
| ) | |
| for r in "${results[@]}"; do | |
| if [[ "$r" != "success" && "$r" != "skipped" ]]; then | |
| echo "::error::One or more jobs failed or were cancelled" | |
| exit 1 | |
| fi | |
| done | |
| echo "All jobs passed" |