ntsh

Signed-off-by: Derek Higgins <[email protected]>

Author: derekhiggins

.github/actions/run-and-record-tests/action.yml

@@ -52,16 +52,16 @@ runs:
           git add tests/integration/recordings/
 
           if [ "${{ inputs.run-vision-tests }}" == "true" ]; then
-            git commit -m "Recordings update from CI (vision)"
+            git commit -m "Recordings update from CI (vision) (${{ inputs.provider }})"
           else
-            git commit -m "Recordings update from CI"
+            git commit -m "Recordings update from CI (${{ inputs.provider }})"
           fi
 
           git fetch origin ${{ github.event.pull_request.head.ref }}
           git rebase origin/${{ github.event.pull_request.head.ref }}
           echo "Rebased successfully"
           git push origin HEAD:${{ github.event.pull_request.head.ref }}
-          echo "Pushed successfully"
+          echo "Pushed successfully "
         else
           echo "No recording changes"
         fi
@@ -70,8 +70,8 @@ runs:
       if: ${{ always() }}
       shell: bash
       run: |
-        sudo docker logs ollama > ollama-${{ inputs.inference-mode }}.log || true
-        sudo docker logs vllm > vllm-${{ inputs.inference-mode }}.log || true
+        sudo docker logs ollama > ollama-${{ inputs.inference-mode }}.log 2>&1 || true
+        sudo docker logs vllm > vllm-${{ inputs.inference-mode }}.log 2>&1 || true
 
     - name: Upload logs
       if: ${{ always() }}

.github/workflows/record-integration-tests.yml

@@ -15,12 +15,6 @@ on:
       - '.github/actions/setup-ollama/action.yml'
       - '.github/actions/setup-test-environment/action.yml'
       - '.github/actions/run-and-record-tests/action.yml'
-  workflow_dispatch:
-    inputs:
-      test-provider:
-        description: 'Test against a specific provider'
-        type: string
-        default: 'ollama'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -31,9 +25,6 @@ jobs:
     if: contains(github.event.pull_request.labels.*.name, 're-record-tests') ||
       contains(github.event.pull_request.labels.*.name, 're-record-vision-tests')
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        provider: [ollama, vllm]
     outputs:
       test-types: ${{ steps.generate-test-types.outputs.test-types }}
       matrix-modes: ${{ steps.generate-test-types.outputs.matrix-modes }}
@@ -45,21 +36,11 @@ jobs:
       - name: Generate test types
         id: generate-test-types
         run: |
-          if [ ${{ matrix.provider }} == "vllm" ]; then
-            echo "test-types=[\"inference\"]" >> $GITHUB_OUTPUT
-          elif
-            # Get test directories dynamically, excluding non-test directories
-            TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
-              grep -Ev "^(__pycache__|fixtures|test_cases|recordings|post_training)$" |
-              sort | jq -R -s -c 'split("\n")[:-1]')
-            echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
-          fi
-
           labels=$(gh pr view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
           echo "labels=$labels"
 
           modes_array=()
-          if [[ $labels == *"re-record-vision-tests"* ]] && [[ ${{ matrix.provider }} == "ollama" ]]; then
+          if [[ $labels == *"re-record-vision-tests"* ]]; then
             modes_array+=("vision")
           fi
           if [[ $labels == *"re-record-tests"* ]]; then
@@ -89,6 +70,10 @@ jobs:
       fail-fast: false
       matrix:
         mode: ${{ fromJSON(needs.discover-tests.outputs.matrix-modes) }}
+        provider: [ollama, vllm]
+        exclude:
+          - mode: vision
+            provider: vllm
 
     steps:
       - name: Checkout repository
@@ -97,20 +82,33 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           fetch-depth: 0
 
+      - name: Generate test types
+        id: generate-test-types
+        run: |
+          if [ ${{ matrix.provider }} == "vllm" ]; then
+            echo "test-types=[\"inference\"]" >> $GITHUB_OUTPUT
+          else
+            # Get test directories dynamically, excluding non-test directories
+            TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
+            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|post_training)$" |
+            sort | jq -R -s -c 'split("\n")[:-1]')
+            echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
+          fi
+
       - name: Setup test environment
         uses: ./.github/actions/setup-test-environment
         with:
           python-version: "3.12"  # Use single Python version for recording
           client-version: "latest"
-          provider: ${{ inputs.test-provider || 'ollama' }}
+          provider: ${{ matrix.provider }}
           run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}
           inference-mode: 'record'
 
       - name: Run and record tests
         uses: ./.github/actions/run-and-record-tests
         with:
-          test-types: ${{ needs.discover-tests.outputs.test-types }}
+          test-types: ${{ steps.generate-test-types.outputs.test-types }}
           stack-config: 'server:ci-tests'  # recording must be done with server since more tests are run
-          provider: ${{ inputs.test-provider || 'ollama' }}
+          provider: ${{ matrix.provider }}
           inference-mode: 'record'
           run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}