🧪 Fix SDK E2E tests to work with vizzly tdd run #745

Workflow file for this run

	name: CI

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]

	jobs:
	lint:
	runs-on: ubuntu-latest
	timeout-minutes: 8
	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install dependencies
	run: npm ci

	- name: Run linter
	run: npm run lint

	- name: Check formatting
	run: npm run format:check

	test:
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	strategy:
	matrix:
	node-version: [22, 24]

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}
	cache: 'npm'

	- name: Install dependencies
	run: npm ci

	- name: Run tests
	run: npm test
	env:
	CI: true

	- name: Build
	run: npm run build

	- name: Run type tests
	run: npm run test:types

	test-reporter:
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install dependencies
	run: npm ci

	- name: Build
	run: npm run build

	- name: Get installed Playwright version
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-browsers-${{ steps.playwright-version.outputs.version }}-firefox

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	run: npx playwright install firefox --with-deps

	- name: Run reporter visual tests
	run: npm run test:reporter:visual
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_REPORTER_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	test-tui:
	runs-on: ubuntu-latest
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Setup tui-driver
	uses: vizzly-testing/tui-driver@main

	- name: Install dependencies
	run: npm ci

	- name: Build
	run: npm run build

	- name: Run TUI visual tests
	run: node bin/vizzly.js run "npm run test:tui"
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_TUI_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	changes-ruby:
	runs-on: ubuntu-latest
	outputs:
	ruby: ${{ steps.filter.outputs.ruby }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	ruby:
	- 'clients/ruby/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-ruby-client.yml'

	test-ruby-client:
	runs-on: ubuntu-latest
	timeout-minutes: 8
	needs: [lint, changes-ruby]
	if: needs.changes-ruby.outputs.ruby == 'true'

	strategy:
	matrix:
	ruby-version: ['3.0', '3.1', '3.2', '3.3']

	steps:
	- uses: actions/checkout@v4

	- name: Set up Ruby ${{ matrix.ruby-version }}
	uses: ruby/setup-ruby@v1
	with:
	ruby-version: ${{ matrix.ruby-version }}

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Ruby dependencies
	working-directory: ./clients/ruby
	run: \|
	gem install bundler
	bundle install

	- name: Run RuboCop
	working-directory: ./clients/ruby
	run: bundle exec rubocop

	- name: Run Ruby unit tests
	working-directory: ./clients/ruby
	run: ruby -I lib test/vizzly_test.rb

	- name: Run Ruby integration tests
	working-directory: ./clients/ruby
	run: VIZZLY_INTEGRATION=1 ruby -I lib test/integration_test.rb
	env:
	CI: true

	changes-storybook:
	runs-on: ubuntu-latest
	outputs:
	storybook: ${{ steps.filter.outputs.storybook }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	storybook:
	- 'clients/storybook/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-storybook-client.yml'

	changes-static-site:
	runs-on: ubuntu-latest
	outputs:
	static-site: ${{ steps.filter.outputs.static-site }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	static-site:
	- 'clients/static-site/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-static-site-client.yml'

	changes-vitest:
	runs-on: ubuntu-latest
	outputs:
	vitest: ${{ steps.filter.outputs.vitest }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	vitest:
	- 'clients/vitest/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-vitest-client.yml'

	changes-swift:
	runs-on: ubuntu-latest
	outputs:
	swift: ${{ steps.filter.outputs.swift }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	swift:
	- 'clients/swift/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-swift-client.yml'

	changes-ember:
	runs-on: ubuntu-latest
	outputs:
	ember: ${{ steps.filter.outputs.ember }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	ember:
	- 'clients/ember/**'
	- '.github/workflows/ci.yml'
	- '.github/workflows/release-ember-client.yml'

	test-storybook-client:
	runs-on: ubuntu-latest
	timeout-minutes: 8
	needs: [lint, changes-storybook]
	if: needs.changes-storybook.outputs.storybook == 'true'

	strategy:
	matrix:
	node-version: [22, 24]

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Storybook client dependencies
	working-directory: ./clients/storybook
	run: npm install

	- name: Run linter
	working-directory: ./clients/storybook
	run: npm run lint

	- name: Run tests
	working-directory: ./clients/storybook
	run: npm test
	env:
	CI: true

	- name: Build package
	working-directory: ./clients/storybook
	run: npm run build

	test-static-site-client:
	runs-on: ubuntu-latest
	timeout-minutes: 8
	needs: [lint, changes-static-site]
	if: needs.changes-static-site.outputs.static-site == 'true'

	strategy:
	matrix:
	node-version: [22, 24]

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install static-site client dependencies
	working-directory: ./clients/static-site
	run: npm install

	- name: Run linter
	working-directory: ./clients/static-site
	run: npm run lint

	- name: Run tests
	working-directory: ./clients/static-site
	run: npm test
	env:
	CI: true

	- name: Build package
	working-directory: ./clients/static-site
	run: npm run build

	# SDK E2E Integration Tests
	# Always run to catch CLI changes that break SDK integrations
	# Each SDK has its own job for isolation, parallelism, and clear failure diagnosis

	e2e-core-js-client:
	name: E2E - Core JS Client
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install test-site dependencies
	working-directory: ./test-site
	run: npm ci

	- name: Get Playwright version
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-${{ steps.playwright-version.outputs.version }}-firefox

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	working-directory: ./test-site
	run: npx playwright install firefox --with-deps

	- name: Run E2E tests (TDD mode)
	working-directory: ./test-site
	run: node ../bin/vizzly.js tdd run "npm test"
	env:
	CI: true

	- name: Run E2E tests (Cloud mode)
	working-directory: ./test-site
	run: node ../bin/vizzly.js run "npm test"
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	e2e-vitest-sdk:
	name: E2E - Vitest SDK
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Vitest client dependencies
	working-directory: ./clients/vitest
	run: npm install

	- name: Get Playwright version
	working-directory: ./clients/vitest
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-${{ steps.playwright-version.outputs.version }}-chromium

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	working-directory: ./clients/vitest
	run: npx playwright install chromium --with-deps

	- name: Run E2E tests (TDD mode)
	working-directory: ./clients/vitest
	run: ../../bin/vizzly.js tdd run "npm run test:e2e"
	env:
	CI: true

	- name: Run E2E tests (Cloud mode)
	working-directory: ./clients/vitest
	run: ../../bin/vizzly.js run "npm run test:e2e"
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_VITEST_CLIENT_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	e2e-ember-sdk:
	name: E2E - Ember SDK
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Ember client dependencies
	working-directory: ./clients/ember
	run: npm install

	- name: Get Playwright version
	working-directory: ./clients/ember
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-${{ steps.playwright-version.outputs.version }}-chromium

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	working-directory: ./clients/ember
	run: npx playwright install chromium --with-deps

	- name: Build Ember test app
	working-directory: ./clients/ember/test-app
	run: \|
	npm install
	npm run build -- --mode development

	- name: Run E2E tests (TDD mode)
	working-directory: ./clients/ember/test-app
	run: ../../../bin/vizzly.js tdd run "npx testem ci --file testem.cjs"
	env:
	CI: true

	- name: Run E2E tests (Cloud mode)
	working-directory: ./clients/ember/test-app
	run: ../../../bin/vizzly.js run "npx testem ci --file testem.cjs"
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_EMBER_CLIENT_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	e2e-ruby-sdk:
	name: E2E - Ruby SDK
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: lint

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22
	cache: 'npm'

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Set up Ruby
	uses: ruby/setup-ruby@v1
	with:
	ruby-version: '3.3'

	- name: Install Ruby dependencies
	working-directory: ./clients/ruby
	run: \|
	gem install bundler
	bundle install

	- name: Run integration tests (TDD mode)
	working-directory: ./clients/ruby
	run: ../../bin/vizzly.js tdd run "VIZZLY_INTEGRATION=1 ruby -I lib test/integration_test.rb"
	env:
	CI: true

	test-vitest-client:
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: [lint, changes-vitest]
	if: needs.changes-vitest.outputs.vitest == 'true'

	strategy:
	matrix:
	node-version: [22, 24]

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node-version }}

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Vitest client dependencies
	working-directory: ./clients/vitest
	run: npm install

	- name: Get installed Playwright version
	working-directory: ./clients/vitest
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-browsers-${{ steps.playwright-version.outputs.version }}-chromium

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	working-directory: ./clients/vitest
	run: npx playwright install chromium --with-deps

	- name: Run Vitest client linter
	working-directory: ./clients/vitest
	run: npm run lint

	- name: Run Vitest client unit tests
	working-directory: ./clients/vitest
	run: npm run test:unit
	env:
	CI: true

	test-swift-client:
	runs-on: macos-latest
	timeout-minutes: 8
	needs: [lint, changes-swift]
	if: needs.changes-swift.outputs.swift == 'true'

	strategy:
	matrix:
	xcode-version: ['16.2', '16.4']

	steps:
	- uses: actions/checkout@v4

	- name: Select Xcode version
	run: sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode-version }}.app

	- name: Build Swift package
	working-directory: ./clients/swift
	run: swift build

	- name: Run Swift tests
	working-directory: ./clients/swift
	run: swift test

	test-ember-client:
	runs-on: blacksmith-4vcpu-ubuntu-2404
	timeout-minutes: 8
	needs: [lint, changes-ember]
	if: needs.changes-ember.outputs.ember == 'true'

	steps:
	- uses: actions/checkout@v4

	- name: Use Node.js 22
	uses: actions/setup-node@v4
	with:
	node-version: 22

	- name: Install CLI dependencies
	run: npm ci

	- name: Build CLI
	run: npm run build

	- name: Install Ember client dependencies
	working-directory: ./clients/ember
	run: npm install

	- name: Get installed Playwright version
	working-directory: ./clients/ember
	id: playwright-version
	run: echo "version=$(npx playwright --version \| grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" >> $GITHUB_OUTPUT

	- name: Cache Playwright browsers
	uses: actions/cache@v4
	id: playwright-cache
	with:
	path: ~/.cache/ms-playwright
	key: playwright-browsers-${{ steps.playwright-version.outputs.version }}-chromium

	- name: Install Playwright browsers
	if: steps.playwright-cache.outputs.cache-hit != 'true'
	working-directory: ./clients/ember
	run: npx playwright install chromium --with-deps

	- name: Run linter
	working-directory: ./clients/ember
	run: npm run lint

	- name: Run unit tests
	working-directory: ./clients/ember
	run: npm test
	env:
	CI: true

	- name: Run integration tests
	working-directory: ./clients/ember
	run: npm run test:integration
	env:
	CI: true

	- name: Run Ember E2E visual tests
	working-directory: ./clients/ember
	run: \|
	cd test-app
	npm install
	npm run build -- --mode development
	../../../bin/vizzly.js run "npx testem ci --file testem.cjs"
	env:
	CI: true
	VIZZLY_TOKEN: ${{ secrets.VIZZLY_EMBER_CLIENT_TOKEN }}
	VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message \|\| github.event.head_commit.message }}
	VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}

	ci-check:
	runs-on: ubuntu-latest
	needs: [lint, test, test-reporter, test-tui, e2e-core-js-client, e2e-vitest-sdk, e2e-ember-sdk, e2e-ruby-sdk, changes-ruby, test-ruby-client, changes-storybook, test-storybook-client, changes-static-site, test-static-site-client, changes-vitest, test-vitest-client, changes-swift, test-swift-client, changes-ember, test-ember-client]
	if: always()
	steps:
	- name: Check if all jobs passed
	run: \|
	# Required jobs that always run
	if [[ "${{ needs.lint.result }}" == "failure" \|\| "${{ needs.test.result }}" == "failure" \|\| "${{ needs.test-reporter.result }}" == "failure" \|\| "${{ needs.test-tui.result }}" == "failure" ]]; then
	echo "One or more required jobs failed"
	exit 1
	fi

	# SDK E2E jobs (always run)
	if [[ "${{ needs.e2e-core-js-client.result }}" == "failure" \|\| "${{ needs.e2e-vitest-sdk.result }}" == "failure" \|\| "${{ needs.e2e-ember-sdk.result }}" == "failure" \|\| "${{ needs.e2e-ruby-sdk.result }}" == "failure" ]]; then
	echo "One or more SDK E2E jobs failed"
	exit 1
	fi

	# Conditional jobs - only check if they ran
	if [[ "${{ needs.changes-ruby.outputs.ruby }}" == "true" && "${{ needs.test-ruby-client.result }}" == "failure" ]]; then
	echo "Ruby client tests failed"
	exit 1
	fi

	if [[ "${{ needs.changes-storybook.outputs.storybook }}" == "true" && "${{ needs.test-storybook-client.result }}" == "failure" ]]; then
	echo "Storybook client tests failed"
	exit 1
	fi

	if [[ "${{ needs.changes-static-site.outputs.static-site }}" == "true" && "${{ needs.test-static-site-client.result }}" == "failure" ]]; then
	echo "Static site client tests failed"
	exit 1
	fi

	if [[ "${{ needs.changes-vitest.outputs.vitest }}" == "true" && "${{ needs.test-vitest-client.result }}" == "failure" ]]; then
	echo "Vitest client tests failed"
	exit 1
	fi

	if [[ "${{ needs.changes-swift.outputs.swift }}" == "true" && "${{ needs.test-swift-client.result }}" == "failure" ]]; then
	echo "Swift client tests failed"
	exit 1
	fi

	if [[ "${{ needs.changes-ember.outputs.ember }}" == "true" && "${{ needs.test-ember-client.result }}" == "failure" ]]; then
	echo "Ember client tests failed"
	exit 1
	fi

	echo "All jobs passed"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

🧪 Fix SDK E2E tests to work with vizzly tdd run #745

Workflow file

🧪 Fix SDK E2E tests to work with vizzly tdd run #745

Uh oh!

Workflow file for this run