diff --git a/.github/workflows/test-preview.yml b/.github/workflows/test-preview.yml new file mode 100644 index 00000000000..fbeaa953a32 --- /dev/null +++ b/.github/workflows/test-preview.yml @@ -0,0 +1,68 @@ +name: Playwright + Argos Tests + +on: + # Trigger on deployment event + deployment_status: + workflow_dispatch: + # Trigger when a label is added to a PR + pull_request: + types: + - labeled + +jobs: + test: + # Run tests only if the deployment is successful + if: | + ( + github.event_name == 'deployment_status' && + github.event.deployment_status.state == 'success' + ) || ( + github.event_name == 'pull_request' && + github.event.action == 'labeled' && + github.event.label.name == 'visual-test' + ) + runs-on: ubuntu-latest + + steps: + + # Step 1: Check out the repository + - name: Check out docs repo + uses: actions/checkout@v4 + with: + repository: ClickHouse/clickhouse-docs + path: ./ + + # Step 2: Log url + - name: Log BASE_URL + run: echo "BASE_URL=${{ github.event.deployment_status.environment_url }}" + env: + BASE_URL: ${{ github.event.deployment_status.environment_url }} + + # Step 3: Setup node + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.18.0' + cache: 'yarn' + + # Step 4: Install dependencies and build the site (needed for sitemap.xml) + - name: Install and build + run: | + export NODE_OPTIONS="--max_old_space_size=4096" + npm install -g yarn + yarn install + + # Step 3: Run Playwright tests in Docker + - name: Run Playwright tests + run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \ + -e BASE_URL=${{ github.event.deployment_status.environment_url }} \ + -e ARGOS_BRANCH="${{ github.event.deployment_status.environment == 'Production' && 'main' || github.ref_name }}" \ + -e CI=true \ + -e NUM_WORKERS=4 \ + -e ARGOS_TOKEN=${{ secrets.ARGOS_TOKEN }} \ + mcr.microsoft.com/playwright:v1.49.1-noble \ + sh -c "git config --global --add safe.directory /workspace && npm exec -- playwright test --workers 4" diff --git a/.gitignore b/.gitignore index a997a074bca..0e5e9dc4c63 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ FormatFactorySettings.h Settings.cpp .vscode + +.vscode +test-results/** +tests/screenshot.spec.ts-snapshots/** diff --git a/package.json b/package.json index 0bb7dc9ed0e..f114c853314 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,10 @@ "write-heading-ids": "docusaurus write-heading-ids" }, "dependencies": { - "@docusaurus/core": "2.3.1", - "@docusaurus/plugin-client-redirects": "2.3.1", - "@docusaurus/preset-classic": "2.3.1", - "@docusaurus/theme-mermaid": "2.3.1", + "@docusaurus/core": "2.4.3", + "@docusaurus/plugin-client-redirects": "2.4.3", + "@docusaurus/preset-classic": "2.4.3", + "@docusaurus/theme-mermaid": "2.4.3", "@mdx-js/react": "^1.6.22", "@radix-ui/react-navigation-menu": "^1.1.4", "axios": "^1.7.9", @@ -44,7 +44,11 @@ "sass": "^1.82.0" }, "devDependencies": { - "@docusaurus/module-type-aliases": "3.6.3" + "@argos-ci/cli": "^2.5.3", + "@argos-ci/playwright": "^3.9.4", + "@docusaurus/module-type-aliases": "3.6.3", + "@playwright/test": "^1.49.1", + "cheerio": "^1.0.0" }, "browserslist": { "production": [ @@ -59,6 +63,6 @@ ] }, "engines": { - "node": ">=16.14" + "node": ">=20.18" } } diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000000..fe23cf06f7a --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,49 @@ +import {devices} from '@playwright/test'; +import type {PlaywrightTestConfig} from '@playwright/test'; + +const isCI = !!process.env.CI; // Check if running in CI +const baseURL = isCI ? process.env.BASE_URL : "http://localhost:3000"; + + +const config: PlaywrightTestConfig = { + fullyParallel: true, + webServer: { + port: 3000, + command: 'yarn docusaurus serve', + }, + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + }, + }, + ], + reporter: [ + // Use "dot" reporter on CI, "list" otherwise (Playwright default). + process.env.CI ? ["dot"] : ["list"], + // Add Argos reporter. + [ + "@argos-ci/playwright/reporter", + { + // Upload to Argos on CI only. + uploadToArgos: isCI, + + // Set your Argos token. + token: process.env.ARGOS_TOKEN, + }, + ], + ], + timeout: 1800000, + use: { + // On CI, we will set `BASE_URL` from Vercel preview URL + baseURL: baseURL, + extraHTTPHeaders: { + // Hide Vercel Toolbar in tests + "x-vercel-skip-toolbar": "0", + }, + }, + +}; + +export default config; diff --git a/tests/screenshot.css b/tests/screenshot.css new file mode 100644 index 00000000000..0619db30bef --- /dev/null +++ b/tests/screenshot.css @@ -0,0 +1,35 @@ +/* +We need to hide some elements in Argos/Playwright screenshots +Those elements are source of flakiness due to nondeterministic rendering +They don't consistently render exactly the same across CI runs + */ + +/******* DOCUSAURUS GLOBAL / THEME *******/ + +/* Iframes can load lazily */ +iframe, +/* Avatar images can be flaky due to using external sources: GitHub/Unavatar */ +.avatar__photo, +/* Gifs load lazily and are animated */ +img[src$='.gif'], +/* Algolia Keyboard shortcuts appear with a little delay */ +.DocSearch-Button-Keys > kbd, +/* The live playground preview can often display dates/counters */ +[class*='playgroundPreview'] { + visibility: hidden; +} + +/* +Different docs last-update dates can alter layout +"visibility: hidden" is not enough + */ +.theme-last-updated { + display: none; +} + +/* +Mermaid diagrams are rendered client-side and produce layout shifts + */ +.docusaurus-mermaid-container { + display: none; +} diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts new file mode 100644 index 00000000000..2daea9ad0f3 --- /dev/null +++ b/tests/screenshot.spec.ts @@ -0,0 +1,88 @@ +import * as fs from 'fs'; +import { argosScreenshot } from '@argos-ci/playwright'; +import { test } from '@playwright/test'; +import axios from 'axios'; +import { extractSitemapPathnames, pathnameToArgosName } from './utils'; + +// Constants +const siteUrl = process.env.CI ? process.env.BASE_URL : 'http://localhost:3000'; +const sitemapUrl = `${siteUrl}/docs/sitemap.xml`; +const stylesheetPath = './tests/screenshot.css'; +const stylesheet = fs.readFileSync(stylesheetPath).toString(); +const NUM_WORKERS = parseInt(process.env.NUM_WORKERS || '1', 10); + +// Wait for hydration, requires Docusaurus v2.4.3+ +// Docusaurus adds a once hydrated +function waitForDocusaurusHydration() { + return document.documentElement.dataset.hasHydrated === 'true'; +} + +// Iterate over the number of workers +for (let workerIndex = 0; workerIndex < NUM_WORKERS; workerIndex++) { + test.describe(`Docusaurus site screenshots batch ${workerIndex}`, async () => { + let pathnames: string[] = []; + + test.beforeAll(async () => { + // Fetch the sitemap dynamically + try { + const response = await axios.get(sitemapUrl); + const sitemapContent = response.data; + pathnames = extractSitemapPathnames(sitemapContent).filter((pathname) => + pathname.startsWith('/docs/en') // currently test en only + ); + } catch (error) { + console.error(`Failed to fetch sitemap: ${error.message}`); + throw error; + } + }); + + test('Generate and run screenshot tests', async ({ page, browser }) => { + const timeout = 30000; // 30 seconds timeout for navigation + const workerPaths = pathnames.filter((_, index) => index % NUM_WORKERS === workerIndex); + console.log(`${workerPaths.length} paths to test`); + + for (let i = 0; i < workerPaths.length; i++) { + const pathname = workerPaths[i]; + console.log(`Processing ${pathname}`); + const url = siteUrl + pathname; + + try { + // Recreate the browser context every 10 pages to reduce memory usage + if (i > 0 && i % 10 === 0) { + console.log(`Restarting browser context after processing ${i} pages.`); + await page.context().close(); + const newContext = await browser.newContext(); + page = await newContext.newPage(); + } + await page.setViewportSize({ width: 1920, height: 1080 }); // Set the viewport size + await page.goto(url, { timeout }); + + // Check for meta redirect + const metaRedirect = await page.$('meta[http-equiv="refresh"]'); + if (metaRedirect) { + console.warn(`Skipping ${pathname} due to meta redirect.`); + continue; + } + + // Wait for hydration with a timeout + try { + await page.waitForFunction(waitForDocusaurusHydration, { timeout }); + } catch (error) { + console.warn(`Skipping ${pathname} due to missing hydration.`); + continue; + } + + // Add custom stylesheet for screenshots + await page.addStyleTag({ content: stylesheet }); + + // Take a screenshot + + await argosScreenshot(page, pathnameToArgosName(pathname)); + console.log(`Screenshot captured for ${pathname}`); + } catch (error) { + console.error(`Failed to process ${pathname}: ${error.message}`); + } + } + }); + }); +} diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 00000000000..3425d106c1d --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,16 @@ +import * as cheerio from "cheerio"; +import * as fs from "fs"; + +export function extractSitemapPathnames(sitemap: string): string[] { + const $ = cheerio.load(sitemap, { xmlMode: true }); + const urls: string[] = []; + $("loc").each(function handleLoc() { + urls.push($(this).text()); + }); + return urls.map((url) => new URL(url).pathname); +} + +// Converts a pathname to a decent screenshot name +export function pathnameToArgosName(pathname: string): string { + return pathname.replace(/^\/|\/$/g, "") || "index"; +}