[8.18] [scout] unique runId for reporting, disabled failed test reporter locally (#209507) (#210087)

kibanamachine · dmlemeshko · web-flow · commit 443bfc75379c · 2025-02-06T20:42:25.000+01:00
# Backport This will backport the following commits from `main` to `8.18`: - [[scout] unique runId for reporting, disabled failed test reporter locally (#209507)](#209507)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>"}},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/209507","number":209507,"mergeCommit":{"message":"[scout] unique runId for reporting, disabled failed test reporter locally (#209507)\n\n## Summary\r\n\r\nThis PR makes few changes to scout reporter:\r\n\r\nRecently I found out that Playwright load configuration file _multiple\r\ntimes_:\r\n- first time after you run `npx playwright test -c ...`\r\n- on every worker start\r\n\r\nlog before:\r\n\r\n```\r\n[main][~/github/kibana]$ npx playwright test --config x-pack/platform/plugins/shared/maps/ui_tests/playwright.config.ts\r\n[createPlaywrightConfig] called with runId: 18f35f735a10155c\r\n\r\nRunning 1 test using 1 worker\r\n\r\n[createPlaywrightConfig] called with runId: 2633b4e4c20afa15\r\n[chromium] › full_screen_mode.spec.ts:28:9 › Maps › Full screen mode @svlSecurity @svlOblt @svlSearch @ess\r\n```\r\n\r\nWith our current logic unique `runId` will be generated on each\r\nconfiguration load, meaning for parallel run we will report failures in\r\ndifferent directories instead of the same one.\r\n\r\nPlaywright doesn't expose any unique identifier for the run, so we have\r\ndo something similar described in\r\n\r\nhttps://github.com/microsoft/playwright/issues/28941#issuecomment-1888008102\r\n\r\nlog after fix:\r\n\r\n```\r\n[main][~/github/kibana]$ npx playwright test --config x-pack/platform/plugins/shared/maps/ui_tests/playwright.config.ts\r\n[createPlaywrightConfig] called with runId: 310a576f32d3b8a5\r\n\r\nRunning 1 test using 1 worker\r\n\r\n[createPlaywrightConfig] called with runId: 310a576f32d3b8a5\r\n[chromium] › full_screen_mode.spec.ts:28:9 › Maps › Full screen mode @svlSecurity @svlOblt @svlSearch @ess\r\n```\r\n\r\nWe also had a chat with @dolaru and agreed that Scout reporters to be\r\ndisabled for local test run. Few reasons:\r\n- Scout custom reporting targets CI execution: events-based reporter was\r\nalready disabled\r\n- Failed test reporter purpose is to provide html boilerplate to be\r\nannotated in pipeline build\r\n- When you run tests with IDE playwright plugin it provides its own\r\nreporter / history, should be enough.","sha":"6635fe501c307d567afa71a346fc91a7675e3cc5"}}]}] BACKPORT--> Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts
@@ -26,5 +26,7 @@ export const scoutPlaywrightReporter = (
 export const scoutFailedTestsReporter = (
   options?: ScoutPlaywrightReporterOptions
 ): ReporterDescription => {
-  return ['@kbn/scout-reporting/src/reporting/playwright/failed_test', options];
+  return SCOUT_REPORTER_ENABLED
+    ? ['@kbn/scout-reporting/src/reporting/playwright/failed_test', options]
+    : ['null'];
 };
diff --git a/packages/kbn-scout/src/playwright/config/create_config.test.ts b/packages/kbn-scout/src/playwright/config/create_config.test.ts
@@ -7,26 +7,35 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { SCOUT_REPORTER_ENABLED, SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
+import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
+import { scoutPlaywrightReporter, scoutFailedTestsReporter } from '@kbn/scout-reporting';
 import { createPlaywrightConfig } from './create_config';
 import { VALID_CONFIG_MARKER } from '../types';
 import { generateTestRunId } from '@kbn/scout-reporting';
 
 jest.mock('@kbn/scout-reporting', () => ({
   ...jest.requireActual('@kbn/scout-reporting'),
   generateTestRunId: jest.fn(),
+  scoutPlaywrightReporter: jest.fn(),
+  scoutFailedTestsReporter: jest.fn(),
 }));
 
 describe('createPlaywrightConfig', () => {
+  const mockedRunId = 'mocked-run-id';
   const mockGenerateTestRunId = generateTestRunId as jest.Mock;
+  const mockedScoutPlaywrightReporter = scoutPlaywrightReporter as jest.Mock;
+  const mockedScoutFailedTestsReporter = scoutFailedTestsReporter as jest.Mock;
 
   beforeEach(() => {
     jest.clearAllMocks();
+    delete process.env.TEST_RUN_ID;
   });
 
   it('should return a valid default Playwright configuration', () => {
-    const testRunId = 'test-run-id';
-    mockGenerateTestRunId.mockImplementationOnce(() => testRunId);
+    mockGenerateTestRunId.mockImplementationOnce(() => mockedRunId);
+    // Scout reporters are disabled by default
+    mockedScoutPlaywrightReporter.mockReturnValueOnce(['null']);
+    mockedScoutFailedTestsReporter.mockReturnValueOnce(['null']);
 
     const testDir = './my_tests';
     const config = createPlaywrightConfig({ testDir });
@@ -48,28 +57,63 @@ describe('createPlaywrightConfig', () => {
     expect(config.reporter).toEqual([
       ['html', { open: 'never', outputFolder: './output/reports' }],
       ['json', { outputFile: './output/reports/test-results.json' }],
-      SCOUT_REPORTER_ENABLED
-        ? [
-            '@kbn/scout-reporting/src/reporting/playwright/events',
-            { name: 'scout-playwright', runId: testRunId },
-          ]
-        : ['null'],
-      [
-        '@kbn/scout-reporting/src/reporting/playwright/failed_test',
-        { name: 'scout-playwright-failed-tests', runId: testRunId },
-      ],
+      ['null'],
+      ['null'],
     ]);
     expect(config.timeout).toBe(60000);
     expect(config.expect?.timeout).toBe(10000);
     expect(config.outputDir).toBe('./output/test-artifacts');
     expect(config.projects![0].name).toEqual('chromium');
   });
 
+  it('should return a Playwright configuration with Scout reporters', () => {
+    mockGenerateTestRunId.mockImplementationOnce(() => mockedRunId);
+    mockedScoutPlaywrightReporter.mockReturnValueOnce([
+      '@kbn/scout-reporting/src/reporting/playwright/events',
+      { name: 'scout-playwright', runId: mockedRunId },
+    ]);
+    mockedScoutFailedTestsReporter.mockReturnValueOnce([
+      '@kbn/scout-reporting/src/reporting/playwright/failed_test',
+      { name: 'scout-playwright-failed-tests', runId: mockedRunId },
+    ]);
+
+    const testDir = './my_tests';
+    const config = createPlaywrightConfig({ testDir });
+
+    expect(mockGenerateTestRunId).toHaveBeenCalledTimes(1);
+    expect(config.reporter).toEqual([
+      ['html', { open: 'never', outputFolder: './output/reports' }],
+      ['json', { outputFile: './output/reports/test-results.json' }],
+      [
+        '@kbn/scout-reporting/src/reporting/playwright/events',
+        { name: 'scout-playwright', runId: mockedRunId },
+      ],
+      [
+        '@kbn/scout-reporting/src/reporting/playwright/failed_test',
+        { name: 'scout-playwright-failed-tests', runId: mockedRunId },
+      ],
+    ]);
+  });
+
   it(`should override 'workers' count in Playwright configuration`, () => {
     const testDir = './my_tests';
     const workers = 2;
 
     const config = createPlaywrightConfig({ testDir, workers });
     expect(config.workers).toBe(workers);
   });
+
+  it('should generate and cache runId in process.env.TEST_RUN_ID', () => {
+    mockGenerateTestRunId.mockReturnValue(mockedRunId);
+
+    // First call to create config
+    createPlaywrightConfig({ testDir: 'tests' });
+    expect(process.env.TEST_RUN_ID).toBe(mockedRunId);
+
+    // Second call (should use the cached value)
+    createPlaywrightConfig({ testDir: 'tests' });
+
+    expect(generateTestRunId).toHaveBeenCalledTimes(1);
+    expect(process.env.TEST_RUN_ID).toBe(mockedRunId);
+  });
 });
diff --git a/packages/kbn-scout/src/playwright/config/create_config.ts b/packages/kbn-scout/src/playwright/config/create_config.ts
@@ -17,7 +17,15 @@ import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info';
 import { ScoutPlaywrightOptions, ScoutTestOptions, VALID_CONFIG_MARKER } from '../types';
 
 export function createPlaywrightConfig(options: ScoutPlaywrightOptions): PlaywrightTestConfig {
-  const runId = generateTestRunId();
+  /**
+   * Playwright loads the config file multiple times, so we need to generate a unique run id
+   * and store it in the environment to be used across all config function calls.
+   */
+  let runId = process.env.TEST_RUN_ID;
+  if (!runId) {
+    runId = generateTestRunId();
+    process.env.TEST_RUN_ID = runId;
+  }
 
   return defineConfig<ScoutTestOptions>({
     testDir: options.testDir,
