From 74196fce263981a6e302624e6991fe5b4c22d366 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 16:33:15 -0700 Subject: [PATCH 1/6] feat: add backward compatibility tests for run.yaml Add automated testing to catch breaking changes to StackRunConfig before they reach users. - Schema validation: pytest-based test validates old run.yaml files parse with current StackRunConfig - Integration tests: run full test suite with old ci-tests run.yaml to catch runtime semantic issues - Dual baseline: test against both main (blocks PRs) and latest release (informational) - Breaking change acknowledgment: require '!:' in PR title or 'BREAKING CHANGE:' in commit message to merge breaking changes Resolves #3311 Related to #3237 --- .github/workflows/README.md | 1 + .github/workflows/backward-compat.yml | 384 +++++++++++++++++++++++ tests/backward_compat/test_run_config.py | 52 +++ 3 files changed, 437 insertions(+) create mode 100644 .github/workflows/backward-compat.yml create mode 100644 tests/backward_compat/test_run_config.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 00a8f54ac1..ef6a8bb3c0 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,6 +4,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Name | File | Purpose | | ---- | ---- | ------- | +| Backward Compatibility Check | [backward-compat.yml](backward-compat.yml) | Check backward compatibility for run.yaml configs | | Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md | | API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. | | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script | diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml new file mode 100644 index 0000000000..857120af7e --- /dev/null +++ b/.github/workflows/backward-compat.yml @@ -0,0 +1,384 @@ +name: Backward Compatibility Check + +run-name: Check backward compatibility for run.yaml configs + +on: + pull_request: + branches: [main] + paths: + - 'src/llama_stack/core/datatypes.py' + - 'src/llama_stack/providers/datatypes.py' + - 'src/llama_stack/distributions/**/run.yaml' + - 'tests/backward_compat/**' + - '.github/workflows/backward-compat.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-main-compatibility: + name: Check Compatibility with main + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 # Need full history to access main branch + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Extract run.yaml files from main branch + id: extract_configs + run: | + # Get list of run.yaml paths from main + git fetch origin main + CONFIG_PATHS=$(git ls-tree -r --name-only origin/main | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "No run.yaml files found in main branch" + exit 1 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/main_configs + echo "Extracting configs from main branch:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + # Extract filename for storage + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show origin/main:"$config_path" > "/tmp/main_configs/${filename}.yaml" + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/main_configs/*.yaml | wc -l) config files" + + - name: Test all configs from main + id: test_configs + continue-on-error: true + run: | + # Run pytest once with all configs parameterized + if COMPAT_TEST_CONFIGS_DIR=/tmp/main_configs uv run pytest tests/backward_compat/test_run_config.py -v; then + echo "failed=false" >> $GITHUB_OUTPUT + else + echo "failed=true" >> $GITHUB_OUTPUT + exit 1 + fi + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_configs.outputs.failed == 'true' + run: | + echo "Breaking changes detected. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate results + if: always() + run: | + FAILED="${{ steps.test_configs.outputs.failed }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Breaking changes detected but acknowledged" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml." + echo "The changes have been properly acknowledged." + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Breaking changes detected without acknowledgment" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml" + echo "that will break existing user configurations." + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Add the 'breaking-change' label to this PR" + echo " 3. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-main: + name: Run Integration Tests with main Config + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Extract ci-tests run.yaml from main + run: | + git fetch origin main + git show origin/main:src/llama_stack/distributions/ci-tests/run.yaml > /tmp/main-ci-tests-run.yaml + echo "Extracted ci-tests run.yaml from main branch" + + - name: Setup test environment + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with main config + id: test_integration + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/main-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_integration.outcome == 'failure' + run: | + echo "Integration tests failed. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate integration test results + if: always() + run: | + TEST_FAILED="${{ steps.test_integration.outcome == 'failure' }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$TEST_FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Integration tests failed with main config but acknowledged" + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Integration tests failed with main config without acknowledgment" + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-release: + name: Run Integration Tests with Latest Release (Informational) + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Get latest release + id: get_release + run: | + # Get the latest release from GitHub + LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract ci-tests run.yaml from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_config + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + if git show "$RELEASE_TAG:src/llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG" + echo "has_config=true" >> $GITHUB_OUTPUT + else + echo "::warning::ci-tests/run.yaml not found in release $RELEASE_TAG" + echo "has_config=false" >> $GITHUB_OUTPUT + fi + + - name: Setup test environment + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with release config + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/release-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Report results + if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + echo "::warning::Integration tests against release $RELEASE_TAG completed" + echo "::warning::This is informational only - does not block merge" + + check-schema-release-compatibility: + name: Check Schema Compatibility with Latest Release (Informational) + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Get latest release + id: get_release + run: | + # Get the latest release from GitHub + LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract configs from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_release_configs + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Get run.yaml files from the release + CONFIG_PATHS=$(git ls-tree -r --name-only "$RELEASE_TAG" | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "::warning::No run.yaml files found in release $RELEASE_TAG" + echo "has_configs=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/release_configs + echo "Extracting configs from release $RELEASE_TAG:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show "$RELEASE_TAG:$config_path" > "/tmp/release_configs/${filename}.yaml" 2>/dev/null || true + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/release_configs/*.yaml 2>/dev/null | wc -l) config files" + echo "has_configs=true" >> $GITHUB_OUTPUT + + - name: Test against release configs + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Run pytest with all release configs + if COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short; then + echo "✓ All configs from release $RELEASE_TAG are compatible" + else + echo "::warning::Some configs from release $RELEASE_TAG are incompatible" + echo "::warning::This is informational only - does not block merge" + fi diff --git a/tests/backward_compat/test_run_config.py b/tests/backward_compat/test_run_config.py new file mode 100644 index 0000000000..13aac85e4b --- /dev/null +++ b/tests/backward_compat/test_run_config.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +""" +Backward compatibility test for run.yaml files. + +This test ensures that changes to StackRunConfig don't break +existing run.yaml files from previous versions. +""" + +import os +from pathlib import Path + +import pytest +import yaml + +from llama_stack.core.datatypes import StackRunConfig + + +def get_test_configs(): + configs_dir = os.environ.get("COMPAT_TEST_CONFIGS_DIR") + if configs_dir: + # CI mode: test configs extracted from main/release + config_dir = Path(configs_dir) + if not config_dir.exists(): + pytest.skip(f"Config directory not found: {configs_dir}") + + config_files = sorted(config_dir.glob("*.yaml")) + if not config_files: + pytest.skip(f"No .yaml files found in {configs_dir}") + + return config_files + else: + # Local mode: test current distribution configs + repo_root = Path(__file__).parent.parent.parent + config_files = sorted((repo_root / "src" / "llama_stack" / "distributions").glob("*/run.yaml")) + + if not config_files: + pytest.skip("No run.yaml files found in distributions/") + + return config_files + + +@pytest.mark.parametrize("config_file", get_test_configs(), ids=lambda p: p.stem) +def test_load_run_config(config_file): + with open(config_file) as f: + config_data = yaml.safe_load(f) + + StackRunConfig.model_validate(config_data) From e3765a6cea36fe26fdd02d81fbbd5331b632b593 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 16:44:11 -0700 Subject: [PATCH 2/6] fix: handle path changes between releases (src/ prefix) --- .github/workflows/backward-compat.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml index 857120af7e..fc27acb62b 100644 --- a/.github/workflows/backward-compat.yml +++ b/.github/workflows/backward-compat.yml @@ -260,8 +260,12 @@ jobs: run: | RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + # Try with src/ prefix first (newer releases), then without (older releases) if git show "$RELEASE_TAG:src/llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then - echo "Extracted ci-tests run.yaml from release $RELEASE_TAG" + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG (src/ path)" + echo "has_config=true" >> $GITHUB_OUTPUT + elif git show "$RELEASE_TAG:llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG (old path)" echo "has_config=true" >> $GITHUB_OUTPUT else echo "::warning::ci-tests/run.yaml not found in release $RELEASE_TAG" @@ -342,8 +346,8 @@ jobs: run: | RELEASE_TAG="${{ steps.get_release.outputs.tag }}" - # Get run.yaml files from the release - CONFIG_PATHS=$(git ls-tree -r --name-only "$RELEASE_TAG" | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + # Get run.yaml files from the release (try both src/ and old path) + CONFIG_PATHS=$(git ls-tree -r --name-only "$RELEASE_TAG" | grep "llama_stack/distributions/.*/run.yaml$" || true) if [ -z "$CONFIG_PATHS" ]; then echo "::warning::No run.yaml files found in release $RELEASE_TAG" From fa11791c327c0466d90bfa1beb13df4087957ce1 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 19:32:40 -0700 Subject: [PATCH 3/6] fix --- .github/workflows/backward-compat.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml index fc27acb62b..87186660e6 100644 --- a/.github/workflows/backward-compat.yml +++ b/.github/workflows/backward-compat.yml @@ -284,6 +284,7 @@ jobs: - name: Run integration tests with release config if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + continue-on-error: true uses: ./.github/actions/run-and-record-tests with: stack-config: /tmp/release-ci-tests-run.yaml From 709376484dd5a27173f63f5173f74cf82e3f22be Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 21:27:45 -0700 Subject: [PATCH 4/6] yell louder? --- .github/workflows/backward-compat.yml | 28 +++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml index 87186660e6..8094ce9af0 100644 --- a/.github/workflows/backward-compat.yml +++ b/.github/workflows/backward-compat.yml @@ -283,6 +283,7 @@ jobs: inference-mode: 'replay' - name: Run integration tests with release config + id: test_release if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' continue-on-error: true uses: ./.github/actions/run-and-record-tests @@ -296,8 +297,20 @@ jobs: if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' run: | RELEASE_TAG="${{ steps.get_release.outputs.tag }}" - echo "::warning::Integration tests against release $RELEASE_TAG completed" - echo "::warning::This is informational only - does not block merge" + TEST_OUTCOME="${{ steps.test_release.outcome }}" + + if [[ "$TEST_OUTCOME" == "failure" ]]; then + echo "" + echo "::error::❌ FAILED: Integration tests against release $RELEASE_TAG FAILED" + echo "::error::⚠️ This PR may break compatibility with the latest release" + echo "::error::⚠️ Users on release $RELEASE_TAG may not be able to upgrade" + echo "" + echo "::warning::This is informational only - does not block merge" + echo "::warning::Consider whether this breaking change is acceptable for users" + else + echo "::notice::✅ Integration tests against release $RELEASE_TAG passed" + echo "::notice::This PR is compatible with the latest release" + fi check-schema-release-compatibility: name: Check Schema Compatibility with Latest Release (Informational) @@ -382,8 +395,15 @@ jobs: # Run pytest with all release configs if COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short; then - echo "✓ All configs from release $RELEASE_TAG are compatible" + echo "" + echo "::notice::✅ All configs from release $RELEASE_TAG are compatible" + echo "::notice::This PR maintains backward compatibility with the latest release" else - echo "::warning::Some configs from release $RELEASE_TAG are incompatible" + echo "" + echo "::error::❌ FAILED: Schema incompatibility detected with release $RELEASE_TAG" + echo "::error::⚠️ This PR breaks backward compatibility with existing run.yaml configs" + echo "::error::⚠️ Users on release $RELEASE_TAG will not be able to upgrade" + echo "" echo "::warning::This is informational only - does not block merge" + echo "::warning::Consider whether this breaking change is acceptable for users" fi From 8d6b01a4457d66167d74d6d4fe40df5fd73c5491 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 21:33:54 -0700 Subject: [PATCH 5/6] try summary --- .github/workflows/backward-compat.yml | 60 ++++++++++++++++++++------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml index 8094ce9af0..ae9ee1d2e1 100644 --- a/.github/workflows/backward-compat.yml +++ b/.github/workflows/backward-compat.yml @@ -300,16 +300,30 @@ jobs: TEST_OUTCOME="${{ steps.test_release.outcome }}" if [[ "$TEST_OUTCOME" == "failure" ]]; then - echo "" - echo "::error::❌ FAILED: Integration tests against release $RELEASE_TAG FAILED" + echo "::error::❌ Integration tests against release $RELEASE_TAG FAILED" echo "::error::⚠️ This PR may break compatibility with the latest release" - echo "::error::⚠️ Users on release $RELEASE_TAG may not be able to upgrade" - echo "" - echo "::warning::This is informational only - does not block merge" - echo "::warning::Consider whether this breaking change is acceptable for users" + + # Write to job summary for high visibility + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users. + EOF else - echo "::notice::✅ Integration tests against release $RELEASE_TAG passed" - echo "::notice::This PR is compatible with the latest release" + cat >> $GITHUB_STEP_SUMMARY <> $GITHUB_STEP_SUMMARY <> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users. + EOF fi From 8b19bd0903e6567c45a18c7f77b4b22ad471712b Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 21:45:23 -0700 Subject: [PATCH 6/6] more robust escalation --- .github/workflows/backward-compat.yml | 187 ++++++++++++++++++++++---- 1 file changed, 162 insertions(+), 25 deletions(-) diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml index ae9ee1d2e1..72d2b0c278 100644 --- a/.github/workflows/backward-compat.yml +++ b/.github/workflows/backward-compat.yml @@ -282,8 +282,8 @@ jobs: suite: 'base' inference-mode: 'replay' - - name: Run integration tests with release config - id: test_release + - name: Run integration tests with release config (PR branch) + id: test_release_pr if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' continue-on-error: true uses: ./.github/actions/run-and-record-tests @@ -293,31 +293,98 @@ jobs: inference-mode: 'replay' suite: 'base' - - name: Report results + - name: Checkout main branch to test baseline + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + run: | + git checkout origin/main + + - name: Setup test environment for main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with release config (main branch) + id: test_release_main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/release-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Report results and post PR comment if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' run: | RELEASE_TAG="${{ steps.get_release.outputs.tag }}" - TEST_OUTCOME="${{ steps.test_release.outcome }}" + PR_OUTCOME="${{ steps.test_release_pr.outcome }}" + MAIN_OUTCOME="${{ steps.test_release_main.outcome }}" + + if [[ "$PR_OUTCOME" == "failure" && "$MAIN_OUTCOME" == "success" ]]; then + # NEW breaking change - PR fails but main passes + echo "::error::🚨 This PR introduces a NEW breaking change!" + + # Check if we already posted a comment (to avoid spam on every push) + EXISTING_COMMENT=$(gh pr view ${{ github.event.pull_request.number }} --json comments --jq '.comments[] | select(.body | contains("🚨 New Breaking Change Detected") and contains("Integration tests")) | .id' | head -1) + + if [[ -z "$EXISTING_COMMENT" ]]; then + gh pr comment ${{ github.event.pull_request.number }} --body "## 🚨 New Breaking Change Detected - if [[ "$TEST_OUTCOME" == "failure" ]]; then - echo "::error::❌ Integration tests against release $RELEASE_TAG FAILED" - echo "::error::⚠️ This PR may break compatibility with the latest release" + **Integration tests against release \`$RELEASE_TAG\` are now failing** + + ⚠️ This PR introduces a breaking change that affects compatibility with the latest release. + + - Users on release \`$RELEASE_TAG\` may not be able to upgrade + - Existing configurations may break + + The tests pass on \`main\` but fail with this PR's changes. + + > **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users." + else + echo "Comment already exists, skipping to avoid spam" + fi - # Write to job summary for high visibility cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. > Consider whether this breaking change is acceptable for users. EOF + + elif [[ "$PR_OUTCOME" == "failure" ]]; then + # Existing breaking change - both PR and main fail + echo "::warning::Breaking change already exists in main branch" + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only. + EOF + else + # Success - tests pass cat >> $GITHUB_STEP_SUMMARY </dev/null | wc -l) config files" echo "has_configs=true" >> $GITHUB_OUTPUT - - name: Test against release configs + - name: Test against release configs (PR branch) + id: test_schema_pr if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + continue-on-error: true run: | RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short - # Run pytest with all release configs - if COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short; then - echo "::notice::✅ All configs from release $RELEASE_TAG are compatible" + - name: Checkout main branch to test baseline + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + git checkout origin/main - cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users." + else + echo "Comment already exists, skipping to avoid spam" + fi cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. > Consider whether this breaking change is acceptable for users. EOF + + elif [[ "$PR_OUTCOME" == "failure" ]]; then + # Existing breaking change - both PR and main fail + echo "::warning::Schema breaking change already exists in main branch" + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only. + EOF + + else + # Success - tests pass + cat >> $GITHUB_STEP_SUMMARY <