Merge pull request #5350 from JetBrains/e2e-tc-reporter

chore: improve production test runner on TeamCity

Author: zoobestik

.gitignore

@@ -50,4 +50,5 @@ generated
 /data/page_views_map.json
 test-results
 
-public/data
+public/data
+test/storage-state.json

.teamcity/tests/buildTypes/E2EProductionTest.kt

@@ -48,6 +48,7 @@ object E2EProductionTest : BuildType({
 
     features {
         notifications {
+            branchFilter = "+:master"
             enabled = !isProjectPlayground()
             notifierSettings = slackNotifier {
                 connection = "PROJECT_EXT_486"

package.json

@@ -107,6 +107,7 @@
         "next-optimized-images": "3.0.0-canary.10",
         "next-transpile-modules": "^10.0.1",
         "playwright": "1.57.0",
+        "playwright-teamcity-reporter": "1.0.5",
         "postcss-custom-media": "10.0.0",
         "postcss-import": "15.1.0",
         "prettier": "2.6.2",

playwright.config.ts

@@ -2,17 +2,28 @@ import { defineConfig, devices } from '@playwright/test';
 
 const MAX_DIFF_PIXEL_RATIO = 0.025 as const;
 
+const isDevelopment = !process.env.CI;
+
+const reporter = isDevelopment ? 'list' : 'playwright-teamcity-reporter';
+const retries = isDevelopment ? 0 : 2;
+const timeout = isDevelopment ? 10000 : 5000;
+
+const forbidOnly = !isDevelopment;
+
 export default defineConfig({
-    forbidOnly: !!process.env.CI,
-    retries: process.env.CI ? 2 : 0,
-    reporter: process.env.CI ? 'dot' : 'list',
+    globalSetup: require.resolve('./test/global-setup.ts'),
+    forbidOnly,
+    retries,
+    reporter,
     snapshotDir: 'test/snapshots',
     expect: {
+        timeout,
         toMatchSnapshot: { maxDiffPixelRatio: MAX_DIFF_PIXEL_RATIO },
         toHaveScreenshot: { maxDiffPixelRatio: MAX_DIFF_PIXEL_RATIO }
     },
     use: {
-        baseURL: process.env.BASE_URL || 'http://localhost:9000',
+        baseURL: process.env.BASE_URL || 'http://localhost:3000',
+        storageState: 'test/storage-state.json',
         trace: 'off',
         screenshot: 'only-on-failure',
         video: 'retain-on-failure'

test/e2e/teach/courses.spec.ts

@@ -1,14 +1,12 @@
 import { expect, test } from '@playwright/test';
 import { CoursesPage } from '../../page/teach/courses-page';
-import { closeExternalBanners } from '../utils';
 import { testSelector } from '../../utils';
 import { checkTeachCta, checkTeachMap, checkTeachNav } from './utils';
 
 test.describe('Courses page appearance and functionality', async () => {
     test.beforeEach(async ({ page, context, baseURL }) => {
         const coursesPage = new CoursesPage(page);
         await coursesPage.init();
-        await closeExternalBanners(context, page, baseURL);
     });
 
     test('Should load the courses page correctly', async ({ page }) => {

test/e2e/teach/education.spec.ts

@@ -1,14 +1,12 @@
 import { expect, test } from '@playwright/test';
 import { TeachPage } from '../../page/teach/education';
-import { closeExternalBanners } from '../utils';
 import { testSelector } from '../../utils';
 import { checkTeachCta, checkTeachMap, checkTeachNav, MAILTO_LINK, MATERIALS_LINK, SIGNUP_LINK } from './utils';
 
 test.describe('Education landing page content and interactions', async () => {
-    test.beforeEach(async ({ context, page, baseURL }) => {
+    test.beforeEach(async ({ page }) => {
         const teachPage = new TeachPage(page);
         await teachPage.init();
-        await closeExternalBanners(context, page, baseURL);
     });
 
     test('Should load the education page correctly', async ({ page }) => {

test/e2e/teach/why.spec.ts

@@ -1,6 +1,5 @@
 import { expect, test } from '@playwright/test';
 import { WhyTeachPage } from '../../page/teach/why-page';
-import { closeExternalBanners } from '../utils';
 import { checkTeachCta, checkTeachNav } from './utils';
 import { testSelector } from '../../utils';
 
@@ -21,10 +20,9 @@ function toId(label: typeof LIST_OF_SECTION[number][0]) {
 }
 
 test.describe('Why Teach Kotlin page appearance and functionality', async () => {
-    test.beforeEach(async ({ page, context, baseURL }) => {
+    test.beforeEach(async ({ page }) => {
         const whyTeachPage = new WhyTeachPage(page);
         await whyTeachPage.init();
-        await closeExternalBanners(context, page, baseURL);
     });
 
     test('Should load the Why Teach Kotlin page correctly', async ({ page }) => {

test/e2e/utils.ts

@@ -1,5 +1,5 @@
-import { BrowserContext, ElementHandle, expect, Page, test } from '@playwright/test';
-import { closeCookiesConsentBanner, isSkipScreenshot } from '../utils';
+import { ElementHandle, expect, Page, test } from '@playwright/test';
+import { isSkipScreenshot } from '../utils';
 import { PageAssertionsToHaveScreenshotOptions } from 'playwright/types/test';
 
 export async function getElementScreenshotWithPadding(page: Page, element: ElementHandle, padding: number): Promise<Buffer | undefined> {
@@ -18,16 +18,6 @@ export async function getElementScreenshotWithPadding(page: Page, element: Eleme
     }
 }
 
-export async function closeExternalBanners(context: BrowserContext, page: Page, baseUrl: string) {
-    if (baseUrl.startsWith('https://kotlinlang.org/')) {
-        await closeCookiesConsentBanner(context, baseUrl);
-    } else {
-        await page.frameLocator('#webpack-dev-server-client-overlay')
-            .locator('[aria-label="Dismiss"]')
-            .click();
-    }
-}
-
 export function pageWrapperMask(page: Page) {
     return [
         page.locator('header[data-test="header"]'),

test/global-setup.ts

@@ -0,0 +1,56 @@
+import { join } from 'node:path';
+import { writeFile } from 'node:fs/promises';
+import { chromium, FullConfig } from '@playwright/test';
+import { isProduction } from './utils';
+
+export default async function globalSetup(config: FullConfig) {
+    const storageStatePath = join(__dirname, 'storage-state.json');
+    await writeFile(storageStatePath, '{}', 'utf-8');
+
+    const project = config.projects[0];
+    console.log(`[Global Setup] Processing project ${project.name}`);
+    const baseURL = project.use?.baseURL;
+
+    if (isProduction(baseURL)) {
+        await closeProductionElements(baseURL, storageStatePath);
+    }
+}
+
+async function closeProductionElements(baseURL: string, storageStatePath: string) {
+    console.log(`[Global Setup] Starting cookie banner setup for ${baseURL}`);
+
+    const browser = await chromium.launch();
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    try {
+        await page.goto(baseURL, { waitUntil: 'domcontentloaded' });
+
+        try {
+            const acceptButton = page.getByRole('button', { name: 'Accept All' });
+            await acceptButton.waitFor({ state: 'visible', timeout: 5000 });
+            await acceptButton.click();
+
+            await page.waitForTimeout(1000);
+        } catch (error) {
+            console.log('[Global Setup] Cookie banner not found - continuing');
+        }
+
+        const closeBanner = page.locator('#optly-banner_close');
+
+        if (await closeBanner.count() > 0) {
+            console.log('[Global Setup] Closing "purple" banner');
+            await closeBanner.click();
+            await page.waitForSelector('#optly-banner_close', { state: 'hidden' });
+        }
+
+        await context.storageState({ path: storageStatePath });
+        console.log(`[Global Setup] Storage state saved to ${storageStatePath}`);
+    } catch (error) {
+        console.error('[Global Setup] Error during setup:', error);
+        throw error;
+    } finally {
+        await context.close();
+        await browser.close();
+    }
+}

test/production/api-navigation.spec.ts

@@ -3,8 +3,6 @@ import { expect, Page, test } from '@playwright/test';
 test.describe('Api navigation', () => {
     test.beforeEach(async ({ page }) => {
         await page.goto('/');
-        await page.waitForSelector('button.ch2-btn.ch2-btn-primary');
-        await page.click('button.ch2-btn.ch2-btn-primary');
         const navbar = page.locator('[data-test="header"]');
         const apiButton = navbar.getByText('API', { exact: true });
         await expect(apiButton).toBeVisible();