diff --git a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts index e30898f365a3..91567c330eec 100644 --- a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts +++ b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts @@ -1,8 +1,10 @@ import type { AsyncExpectationResult, MatcherState } from '@vitest/expect' +import type { VisualRegressionArtifact } from '@vitest/runner' import type { ScreenshotMatcherOptions } from '../../../../context' import type { ScreenshotMatcherArguments, ScreenshotMatcherOutput } from '../../../shared/screenshotMatcher/types' import type { Locator } from '../locators' -import { getBrowserState, getWorkerState } from '../../utils' +import { recordArtifact } from '@vitest/runner' +import { getBrowserState } from '../../utils' import { convertToSelector } from '../tester-utils' const counters = new Map([]) @@ -19,13 +21,11 @@ export default async function toMatchScreenshot( throw new Error('\'toMatchScreenshot\' cannot be used with "not"') } - const currentTest = getWorkerState().current - - if (currentTest === undefined || this.currentTestName === undefined) { + if (this.task === undefined || this.currentTestName === undefined) { throw new Error('\'toMatchScreenshot\' cannot be used without test context') } - const counterName = `${currentTest.result?.repeatCount ?? 0}${this.testPath}${this.currentTestName}` + const counterName = `${this.task.result?.repeatCount ?? 0}${this.testPath}${this.currentTestName}` let counter = counters.get(counterName) if (counter === undefined) { @@ -66,24 +66,29 @@ export default async function toMatchScreenshot( ] satisfies ScreenshotMatcherArguments, ) - if (result.pass === false && 'context' in currentTest) { - const { annotate } = currentTest.context - - const annotations: ReturnType[] = [] + if (result.pass === false) { + const attachments: VisualRegressionArtifact['attachments'] = [] if (result.reference) { - annotations.push(annotate('Reference screenshot', { path: result.reference })) + attachments.push({ name: 'reference', ...result.reference }) } if (result.actual) { - annotations.push(annotate('Actual screenshot', { path: result.actual })) + attachments.push({ name: 'actual', ...result.actual }) } if (result.diff) { - annotations.push(annotate('Diff', { path: result.diff })) + attachments.push({ name: 'diff', path: result.diff }) } - await Promise.all(annotations) + if (attachments.length > 0) { + await recordArtifact(this.task, { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: result.message, + attachments, + }) + } } return { @@ -96,14 +101,15 @@ export default async function toMatchScreenshot( '', result.message, result.reference - ? `\nReference screenshot:\n ${this.utils.EXPECTED_COLOR(result.reference)}` + ? `\nReference screenshot:\n ${this.utils.EXPECTED_COLOR(result.reference.path)}` : null, result.actual - ? `\nActual screenshot:\n ${this.utils.RECEIVED_COLOR(result.actual)}` + ? `\nActual screenshot:\n ${this.utils.RECEIVED_COLOR(result.actual.path)}` : null, result.diff ? this.utils.DIM_COLOR(`\nDiff image:\n ${result.diff}`) : null, + '', ] .filter(element => element !== null) .join('\n'), diff --git a/packages/browser/src/node/commands/screenshotMatcher/index.ts b/packages/browser/src/node/commands/screenshotMatcher/index.ts index 200ed96a3bae..1d35054d4aa9 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/index.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/index.ts @@ -55,7 +55,7 @@ export const screenshotMatcher: BrowserCommand< if (value === null || value.actual === null) { return { pass: false, - reference: referenceFile && paths.reference, + reference: referenceFile && { path: paths.reference, width: reference!.metadata.width, height: reference!.metadata.height }, actual: null, diff: null, message: `Could not capture a stable screenshot within ${timeout}ms.`, @@ -80,7 +80,8 @@ export const screenshotMatcher: BrowserCommand< if (updateSnapshot !== 'all') { return { pass: false, - reference: referencePath, + // we use `actual`'s metadata because that's the screenshot we saved + reference: { path: referencePath, width: value.actual.metadata.width, height: value.actual.metadata.height }, actual: null, diff: null, message: `No existing reference screenshot found${ @@ -143,8 +144,8 @@ export const screenshotMatcher: BrowserCommand< // - fail return { pass: false, - reference: paths.reference, - actual: paths.diffs.actual, + reference: { path: paths.reference, width: reference.metadata.width, height: reference.metadata.height }, + actual: { path: paths.diffs.actual, width: value.actual.metadata.width, height: value.actual.metadata.height }, diff: finalResult.diff && paths.diffs.diff, message: `Screenshot does not match the stored reference.${ finalResult.message === null diff --git a/packages/browser/src/shared/screenshotMatcher/types.ts b/packages/browser/src/shared/screenshotMatcher/types.ts index 1bc124de30b7..2af6c2e02238 100644 --- a/packages/browser/src/shared/screenshotMatcher/types.ts +++ b/packages/browser/src/shared/screenshotMatcher/types.ts @@ -12,11 +12,13 @@ export type ScreenshotMatcherArguments< }, ] +interface ScreenshotData { path: string; width: number; height: number } + export type ScreenshotMatcherOutput = Promise< { pass: false - reference: string | null - actual: string | null + reference: ScreenshotData | null + actual: ScreenshotData | null diff: string | null message: string } diff --git a/packages/runner/src/types.ts b/packages/runner/src/types.ts index eaccf81481b8..84e2379eef8f 100644 --- a/packages/runner/src/types.ts +++ b/packages/runner/src/types.ts @@ -54,4 +54,5 @@ export type { TestFunction, TestOptions, Use, + VisualRegressionArtifact, } from './types/tasks' diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index b726e0829d2c..3ae8f199f6b8 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -780,6 +780,24 @@ export interface TestAnnotationArtifact extends TestArtifactBase { annotation: TestAnnotation } +type VisualRegressionArtifactAttachment = TestAttachment & ({ + name: 'reference' | 'actual' + width: number + height: number +} | { name: 'diff' }) + +/** + * @experimental + * + * Artifact type for visual regressions. + */ +export interface VisualRegressionArtifact extends TestArtifactBase { + type: 'internal:toMatchScreenshot' + kind: 'visual-regression' + message: string + attachments: VisualRegressionArtifactAttachment[] +} + /** * @experimental * @advanced @@ -861,4 +879,4 @@ export interface TestArtifactRegistry {} * * This type automatically includes all artifacts registered via {@link TestArtifactRegistry}. */ -export type TestArtifact = TestAnnotationArtifact | TestArtifactRegistry[keyof TestArtifactRegistry] +export type TestArtifact = TestAnnotationArtifact | VisualRegressionArtifact | TestArtifactRegistry[keyof TestArtifactRegistry] diff --git a/packages/ui/client/components/artifacts/ArtifactTemplate.vue b/packages/ui/client/components/artifacts/ArtifactTemplate.vue new file mode 100644 index 000000000000..0b5ab742e643 --- /dev/null +++ b/packages/ui/client/components/artifacts/ArtifactTemplate.vue @@ -0,0 +1,17 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/SmallTabs.spec.ts b/packages/ui/client/components/artifacts/visual-regression/SmallTabs.spec.ts new file mode 100644 index 000000000000..84fa5faa6ee2 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/SmallTabs.spec.ts @@ -0,0 +1,94 @@ +import { faker } from '@faker-js/faker' +import { describe, expect, it } from 'vitest' +import { userEvent } from 'vitest/browser' +import { defineComponent, h } from 'vue' +import { page, render } from '~/test' +import SmallTabs from './SmallTabs.vue' +import SmallTabsPane from './SmallTabsPane.vue' + +function createSmallTabs(children: number) { + return defineComponent({ + setup: () => + () => + h( + SmallTabs, + null, + { + default: () => Array.from({ length: children }, () => h( + SmallTabsPane, + { title: faker.lorem.word() }, + () => faker.lorem.words(2), + )), + }, + ), + }) +} + +describe('SmallTabs', () => { + it('has accessible elements', async () => { + render(createSmallTabs(2)) + + // a tablist with two elements inside + const tablist = page.getByRole('tablist') + const tabs = tablist.getByRole('tab') + const firstTab = tabs.first() + const secondTab = tabs.last() + + await expect.element(tablist).toBeInTheDocument() + expect(tabs.all()).toHaveLength(2) + + await expect.element(firstTab).toHaveAttribute('aria-selected', 'true') + await expect.element(secondTab).toHaveAttribute('aria-selected', 'false') + + // two tab panels, with one hidden + const panels = page.getByRole('tabpanel', { includeHidden: true }) + const firstPanel = panels.first() + const secondPanel = panels.last() + + expect(panels.all()).toHaveLength(2) + + await expect.element(firstPanel).not.toHaveAttribute('hidden') + await expect.element(secondPanel).toHaveAttribute('hidden') + + // panels should be labelled by their tab button + await expect.element(firstPanel).toHaveAttribute( + 'aria-labelledby', + firstTab.element().getAttribute('id'), + ) + await expect.element(secondPanel).toHaveAttribute( + 'aria-labelledby', + secondTab.element().getAttribute('id'), + ) + + await expect.element(firstTab).toHaveAttribute( + 'aria-controls', + firstPanel.element().getAttribute('id'), + ) + await expect.element(secondTab).toHaveAttribute( + 'aria-controls', + secondPanel.element().getAttribute('id'), + ) + }) + + it('opens one panel at a time', async () => { + const tabsLimit = 5 + + render(createSmallTabs(tabsLimit)) + + const tabs = page.getByRole('tablist').getByRole('tab') + const panels = page.getByRole('tabpanel', { includeHidden: true }) + + for (let tabIndex = 0; tabIndex < tabsLimit; tabIndex += 1) { + const activeTab = tabs.nth(tabIndex) + const activePanel = panels.nth(tabIndex) + + await userEvent.click(activeTab) + await expect.element( + tabs.and(page.getByRole('tab', { selected: true })), + ).toBe(activeTab.element()) + await expect.element( + page.getByRole('tabpanel'), + ).toBe(activePanel.element()) + } + }) +}) diff --git a/packages/ui/client/components/artifacts/visual-regression/SmallTabs.vue b/packages/ui/client/components/artifacts/visual-regression/SmallTabs.vue new file mode 100644 index 000000000000..346cb8311444 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/SmallTabs.vue @@ -0,0 +1,62 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/SmallTabsPane.vue b/packages/ui/client/components/artifacts/visual-regression/SmallTabsPane.vue new file mode 100644 index 000000000000..3477af76a0b5 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/SmallTabsPane.vue @@ -0,0 +1,39 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegression.spec.ts b/packages/ui/client/components/artifacts/visual-regression/VisualRegression.spec.ts new file mode 100644 index 000000000000..70d81fc54b9d --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegression.spec.ts @@ -0,0 +1,160 @@ +import type { VisualRegressionArtifact } from '@vitest/runner' +import { faker } from '@faker-js/faker' +import { describe, expect, it } from 'vitest' +import { userEvent } from 'vitest/browser' +import { page, render } from '~/test' +import VisualRegression from './VisualRegression.vue' + +const diff = { + name: 'diff', + path: '/__diff.png', +} as const + +const reference = { + name: 'reference', + path: '/__reference.png', + width: 500, + height: 200, +} as const + +const actual = { + name: 'actual', + path: '/__actual.png', + width: 500, + height: 200, +} as const + +describe('VisualRegression', () => { + it('renders content with no attachments', async () => { + const messageContent = faker.lorem.words(5) + + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: messageContent, + attachments: [], + } satisfies VisualRegressionArtifact, + }, + }) + + const article = page.getByRole('article') + + await expect.element(article).toBeInTheDocument() + await expect.element(article.getByRole('heading')) + .toHaveTextContent('Visual Regression') + await expect.element(article.getByRole('paragraph')) + .toHaveTextContent(messageContent) + await expect.element(article.getByRole('tablist')).toHaveTextContent('') + }) + + it('renders diff tab', async () => { + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: faker.lorem.words(5), + attachments: [diff], + } satisfies VisualRegressionArtifact, + }, + }) + + await expect.element(page.getByRole('tablist').getByRole('tab')) + .toHaveTextContent('Diff') + await expect.element(page.getByRole('tabpanel').getByRole('img')) + .toBeInTheDocument() + }) + + it('renders reference tab', async () => { + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: faker.lorem.words(5), + attachments: [reference], + } satisfies VisualRegressionArtifact, + }, + }) + + await expect.element(page.getByRole('tablist').getByRole('tab')) + .toHaveTextContent('Reference') + await expect.element(page.getByRole('tabpanel').getByRole('img')) + .toBeInTheDocument() + }) + + it('renders actual tab', async () => { + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: faker.lorem.words(5), + attachments: [actual], + } satisfies VisualRegressionArtifact, + }, + }) + + await expect.element(page.getByRole('tablist').getByRole('tab')) + .toHaveTextContent('Actual') + await expect.element(page.getByRole('tabpanel').getByRole('img')) + .toBeInTheDocument() + }) + + it('renders reference, actual, and slider tabs', async () => { + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: faker.lorem.words(5), + attachments: [reference, actual], + } satisfies VisualRegressionArtifact, + }, + }) + + const tablist = page.getByRole('tablist') + const tabs = tablist.getByRole('tab') + + await expect.element(tablist).toBeInTheDocument() + + expect(tabs.all()).toHaveLength(3) + await expect.element(tabs.nth(0)).toHaveTextContent('Reference') + await expect.element(tabs.nth(1)).toHaveTextContent('Actual') + await expect.element(tabs.nth(2)).toHaveTextContent('Slider') + + await userEvent.click(tabs.nth(2)) + + await expect.element( + page.getByLabelText( + 'Image comparison slider showing reference and actual screenshots', + ), + ).toBeInTheDocument() + }) + + it('renders diff, reference, actual, and slider tabs', async () => { + render(VisualRegression, { + props: { + regression: { + type: 'internal:toMatchScreenshot', + kind: 'visual-regression', + message: faker.lorem.words(5), + attachments: [diff, reference, actual], + } satisfies VisualRegressionArtifact, + }, + }) + + const tablist = page.getByRole('tablist') + const tabs = tablist.getByRole('tab') + + await expect.element(tablist).toBeInTheDocument() + + expect(tabs.all()).toHaveLength(4) + await expect.element(tabs.nth(0)).toHaveTextContent('Diff') + await expect.element(tabs.nth(1)).toHaveTextContent('Reference') + await expect.element(tabs.nth(2)).toHaveTextContent('Actual') + await expect.element(tabs.nth(3)).toHaveTextContent('Slider') + }) +}) diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegression.vue b/packages/ui/client/components/artifacts/visual-regression/VisualRegression.vue new file mode 100644 index 000000000000..77cca7289eda --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegression.vue @@ -0,0 +1,62 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImage.vue b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImage.vue new file mode 100644 index 000000000000..b238497a6060 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImage.vue @@ -0,0 +1,26 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImageContainer.vue b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImageContainer.vue new file mode 100644 index 000000000000..1dcadc41ad12 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionImageContainer.vue @@ -0,0 +1,14 @@ + + + diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.spec.ts b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.spec.ts new file mode 100644 index 000000000000..7872fc84dc98 --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.spec.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from 'vitest' +import { locators, userEvent } from 'vitest/browser' +import { page, render } from '~/test' +import VisualRegressionSlider from './VisualRegressionSlider.vue' + +locators.extend({ + // @ts-expect-error this gets used only in the first block which tests elements removed from the accessibility tree + getByCSS(selector: string) { + return selector + }, +}) + +const reference = { + path: '/__reference.png', + width: 500, + height: 200, +} + +const actual = { + path: '/__actual.png', + width: 500, + height: 200, +} + +const containerLabel = 'Image comparison slider showing reference and actual screenshots' +const inputLabel = 'Adjust slider to compare reference and actual images' + +describe('VisualRegressionSlider', () => { + it('renders both images with correct URLs', async () => { + render(VisualRegressionSlider, { + props: { + reference, + actual, + }, + }) + + // !!! images are hidden/removed from the accessibility tree, so we fall back to CSS selectors + + // both image tags should be rendered and contain the path + // @ts-expect-error extended only for this block + const images = page.getByCSS('img') + const referenceImage = images.first() + const actualImage = images.last() + + expect(images.all()).toHaveLength(2) + + await expect.element(referenceImage).toHaveProperty( + 'src', + expect.stringContaining(encodeURIComponent(reference.path)), + ) + await expect.element(actualImage).toHaveProperty( + 'src', + expect.stringContaining(encodeURIComponent(actual.path)), + ) + + // images should be wrapped in (hidden) containers with role presentation + // @ts-expect-error extended only for this block + const referenceContainer = page.getByCSS('div:has(> img)').first() + // @ts-expect-error extended only for this block + const actualContainer = page.getByCSS('div:has(> img)').last() + + await expect.element(referenceContainer).toHaveAttribute('aria-hidden', 'true') + await expect.element(referenceContainer).toHaveAttribute('role', 'presentation') + + await expect.element(actualContainer).toHaveAttribute('aria-hidden', 'true') + await expect.element(actualContainer).toHaveAttribute('role', 'presentation') + }) + + it('has accessible descriptions', async () => { + render(VisualRegressionSlider, { + props: { + reference, + actual, + }, + }) + + const container = page.getByLabelText(containerLabel) + const input = page.getByLabelText(inputLabel) + const status = page.getByRole( + 'status', + { hasText: 'Showing 50% reference, 50% actual' }, + ) + + // container with accessible label should exist and contain input and status + await expect.element(container).toBeInTheDocument() + await expect.element(container).toContainElement(input) + await expect.element(container).toContainElement(status) + // split is an implementation detail, but in absence of VRT it's a best-effort approach to check it's working + await expect.element(container).toHaveStyle('--split: 50%') + + // input with label should be an input and have an id + await expect.element(input).toBeInstanceOf(HTMLInputElement) + await expect.element(input).toHaveAttribute('id') + + // status element should exist and be connected to the input + await expect.element(status).toBeInTheDocument() + await expect.element(status).toHaveAttribute( + 'for', + input.element().getAttribute('id'), + ) + }) + + it('has slider boundaries', async () => { + render(VisualRegressionSlider, { + props: { + reference, + actual, + }, + }) + + const input = page.getByLabelText(inputLabel) + + await expect.element(input).toHaveAttribute('min', '0') + await expect.element(input).toHaveAttribute('max', '100') + await expect.element(input).toHaveAttribute('step', '0.1') + }) + + it('updates split percentage on slider movement', async () => { + render(VisualRegressionSlider, { + props: { + reference, + actual, + }, + }) + + const container = page.getByLabelText(containerLabel) + const input = page.getByLabelText(inputLabel) + const status = page.getByRole('status') + + // initial state should be 50 + await expect.element(input).toHaveValue('50') + await expect.element(container).toHaveStyle('--split: 50%') + + // keyboard interaction + await userEvent.click(status) + await userEvent.tab() + await userEvent.keyboard('{ArrowRight>10/}') + await userEvent.click(status) + + await expect.element(input).toHaveValue('51') + await expect.element(container).toHaveStyle('--split: 51%') + await expect.element(status).toHaveTextContent('Showing 51% reference, 49% actual') + + // mouse interaction + await userEvent.click( + input, + { + position: { + x: 1, + y: input.element().getBoundingClientRect().height / 2, + }, + }, + ) + await userEvent.click(status) + + await expect.element(input).toHaveValue('0') + await expect.element(status).toHaveTextContent('Showing 0% reference, 100% actual') + await expect.element(container).toHaveStyle('--split: 0%') + }) +}) diff --git a/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.vue b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.vue new file mode 100644 index 000000000000..76101543d17c --- /dev/null +++ b/packages/ui/client/components/artifacts/visual-regression/VisualRegressionSlider.vue @@ -0,0 +1,74 @@ + + + diff --git a/packages/ui/client/components/views/ViewTestReport.vue b/packages/ui/client/components/views/ViewTestReport.vue index c20043bb5338..13530a71f840 100644 --- a/packages/ui/client/components/views/ViewTestReport.vue +++ b/packages/ui/client/components/views/ViewTestReport.vue @@ -1,15 +1,16 @@