Skip to content

Commit 08dad78

Browse files
Robdel12Robdel12
committed
🐛 Fix Vitest SDK status values and add SDK E2E tests to CI
Fixes #179 - Vitest client was checking for 'passed'/'failed' status values but PR #169 changed the TDD handler to return 'match'/'diff'. **Vitest SDK fix:** - Changed status checks from 'passed' to 'match' - Changed status checks from 'failed' to 'diff' - Added handling for 'baseline-updated' status **CI improvements:** - Added `test-sdk-e2e` job that always runs (not conditional on path changes) - Tests Core JS Client, Vitest, Ember, and Ruby SDKs end-to-end - Removed redundant `test-site` job (now part of test-sdk-e2e) - Shared Playwright browser cache across SDK tests This ensures CLI changes that break SDK integrations are caught before merge.
1 parent 83918d1 commit 08dad78

File tree

2 files changed

+113
-64
lines changed

2 files changed

+113
-64
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 105 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -60,56 +60,6 @@ jobs:
6060
- name: Run type tests
6161
run: npm run test:types
6262

63-
test-site:
64-
runs-on: blacksmith-4vcpu-ubuntu-2404
65-
timeout-minutes: 8
66-
needs: lint
67-
68-
steps:
69-
- uses: actions/checkout@v4
70-
71-
- name: Use Node.js 22
72-
uses: actions/setup-node@v4
73-
with:
74-
node-version: 22
75-
cache: 'npm'
76-
77-
- name: Install CLI dependencies
78-
run: npm ci
79-
80-
- name: Build CLI
81-
run: npm run build
82-
83-
- name: Install test site dependencies
84-
working-directory: ./test-site
85-
run: npm ci
86-
87-
- name: Get installed Playwright version
88-
working-directory: ./test-site
89-
id: playwright-version
90-
run: echo "version=$(npx playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT
91-
92-
- name: Cache Playwright browsers
93-
uses: actions/cache@v4
94-
id: playwright-cache
95-
with:
96-
path: ~/.cache/ms-playwright
97-
key: playwright-browsers-${{ steps.playwright-version.outputs.version }}-firefox
98-
99-
- name: Install Playwright browsers
100-
if: steps.playwright-cache.outputs.cache-hit != 'true'
101-
working-directory: ./test-site
102-
run: npx playwright install firefox --with-deps
103-
104-
- name: Test Vizzly integration
105-
working-directory: ./test-site
106-
run: node ../bin/vizzly.js run "npm test"
107-
env:
108-
CI: true
109-
VIZZLY_TOKEN: ${{ secrets.VIZZLY_TOKEN }}
110-
VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }}
111-
VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
112-
11363
test-reporter:
11464
runs-on: blacksmith-4vcpu-ubuntu-2404
11565
timeout-minutes: 8
@@ -408,6 +358,109 @@ jobs:
408358
working-directory: ./clients/static-site
409359
run: npm run build
410360

361+
# SDK E2E Integration Tests
362+
# Always runs to catch CLI changes that break SDK integrations
363+
# Tests the full flow: test runner -> SDK -> CLI server -> visual comparison
364+
test-sdk-e2e:
365+
runs-on: blacksmith-4vcpu-ubuntu-2404
366+
timeout-minutes: 12
367+
needs: lint
368+
369+
steps:
370+
- uses: actions/checkout@v4
371+
372+
- name: Use Node.js 22
373+
uses: actions/setup-node@v4
374+
with:
375+
node-version: 22
376+
cache: 'npm'
377+
378+
- name: Install CLI dependencies
379+
run: npm ci
380+
381+
- name: Build CLI
382+
run: npm run build
383+
384+
# Shared Playwright setup (used by multiple SDKs)
385+
- name: Get Playwright version
386+
id: playwright-version
387+
run: echo "version=$(npx playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT
388+
389+
- name: Cache Playwright browsers
390+
uses: actions/cache@v4
391+
id: playwright-cache
392+
with:
393+
path: ~/.cache/ms-playwright
394+
key: playwright-${{ steps.playwright-version.outputs.version }}-chromium-firefox
395+
396+
- name: Install Playwright browsers
397+
if: steps.playwright-cache.outputs.cache-hit != 'true'
398+
run: npx playwright install chromium firefox --with-deps
399+
400+
# Core JS Client SDK (vizzlyScreenshot function)
401+
- name: Install test-site dependencies
402+
working-directory: ./test-site
403+
run: npm ci
404+
405+
- name: Run Core JS Client E2E tests
406+
working-directory: ./test-site
407+
run: node ../bin/vizzly.js run "npm test"
408+
env:
409+
CI: true
410+
VIZZLY_TOKEN: ${{ secrets.VIZZLY_TOKEN }}
411+
VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }}
412+
VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
413+
414+
# Vitest SDK (toMatchScreenshot matcher)
415+
- name: Install Vitest client dependencies
416+
working-directory: ./clients/vitest
417+
run: npm install
418+
419+
- name: Run Vitest SDK E2E tests
420+
working-directory: ./clients/vitest
421+
run: ../../bin/vizzly.js run "npm run test:e2e"
422+
env:
423+
CI: true
424+
VIZZLY_TOKEN: ${{ secrets.VIZZLY_VITEST_CLIENT_TOKEN }}
425+
VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }}
426+
VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
427+
428+
# Ember SDK (vizzlyScreenshot helper with Testem + Playwright)
429+
- name: Install Ember client dependencies
430+
working-directory: ./clients/ember
431+
run: npm install
432+
433+
- name: Run Ember SDK E2E tests
434+
working-directory: ./clients/ember
435+
run: |
436+
cd test-app
437+
npm install
438+
npm run build -- --mode development
439+
../../../bin/vizzly.js run "npx testem ci --file testem.cjs"
440+
env:
441+
CI: true
442+
VIZZLY_TOKEN: ${{ secrets.VIZZLY_EMBER_CLIENT_TOKEN }}
443+
VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }}
444+
VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
445+
446+
# Ruby SDK (Vizzly.screenshot with TDD server)
447+
- name: Set up Ruby
448+
uses: ruby/setup-ruby@v1
449+
with:
450+
ruby-version: '3.3'
451+
452+
- name: Install Ruby dependencies
453+
working-directory: ./clients/ruby
454+
run: |
455+
gem install bundler
456+
bundle install
457+
458+
- name: Run Ruby SDK integration tests
459+
working-directory: ./clients/ruby
460+
run: VIZZLY_INTEGRATION=1 ruby -I lib test/integration_test.rb
461+
env:
462+
CI: true
463+
411464
test-vitest-client:
412465
runs-on: blacksmith-4vcpu-ubuntu-2404
413466
timeout-minutes: 8
@@ -463,16 +516,6 @@ jobs:
463516
env:
464517
CI: true
465518

466-
- name: Run Vitest client E2E tests
467-
if: matrix.node-version == 22
468-
working-directory: ./clients/vitest
469-
run: ../../bin/vizzly.js run "npm run test:e2e"
470-
env:
471-
CI: true
472-
VIZZLY_TOKEN: ${{ secrets.VIZZLY_VITEST_CLIENT_TOKEN }}
473-
VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }}
474-
VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
475-
476519
test-swift-client:
477520
runs-on: macos-latest
478521
timeout-minutes: 8
@@ -569,13 +612,13 @@ jobs:
569612

570613
ci-check:
571614
runs-on: ubuntu-latest
572-
needs: [lint, test, test-site, test-reporter, test-tui, changes-ruby, test-ruby-client, changes-storybook, test-storybook-client, changes-static-site, test-static-site-client, changes-vitest, test-vitest-client, changes-swift, test-swift-client, changes-ember, test-ember-client]
615+
needs: [lint, test, test-reporter, test-tui, test-sdk-e2e, changes-ruby, test-ruby-client, changes-storybook, test-storybook-client, changes-static-site, test-static-site-client, changes-vitest, test-vitest-client, changes-swift, test-swift-client, changes-ember, test-ember-client]
573616
if: always()
574617
steps:
575618
- name: Check if all jobs passed
576619
run: |
577620
# Required jobs that always run
578-
if [[ "${{ needs.lint.result }}" == "failure" || "${{ needs.test.result }}" == "failure" || "${{ needs.test-site.result }}" == "failure" || "${{ needs.test-reporter.result }}" == "failure" || "${{ needs.test-tui.result }}" == "failure" ]]; then
621+
if [[ "${{ needs.lint.result }}" == "failure" || "${{ needs.test.result }}" == "failure" || "${{ needs.test-reporter.result }}" == "failure" || "${{ needs.test-tui.result }}" == "failure" || "${{ needs.test-sdk-e2e.result }}" == "failure" ]]; then
579622
echo "One or more required jobs failed"
580623
exit 1
581624
fi

‎clients/vitest/src/setup.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,18 @@ async function toMatchScreenshot(element, name, options = {}) {
9393
message: () =>
9494
`New screenshot baseline created: ${screenshotName}. View at http://localhost:47392/dashboard`,
9595
};
96-
} else if (comparisonStatus === 'passed') {
96+
} else if (comparisonStatus === 'match') {
9797
return {
9898
pass: true,
9999
message: () => '',
100100
};
101-
} else if (comparisonStatus === 'failed') {
101+
} else if (comparisonStatus === 'baseline-updated') {
102+
return {
103+
pass: true,
104+
message: () =>
105+
`Baseline updated: ${screenshotName}. View at http://localhost:47392/dashboard`,
106+
};
107+
} else if (comparisonStatus === 'diff') {
102108
let diffPercent = comparison.diffPercentage
103109
? comparison.diffPercentage.toFixed(2)
104110
: '0.00';

0 commit comments

Comments
 (0)