feat(test runner): config.tag for a global tag (#37846)

Author: dgozman

docs/src/test-api/class-fullconfig.md

@@ -112,6 +112,12 @@ Base directory for all relative paths used in the reporters.
 
 See [`property: TestConfig.shard`].
 
+## property: FullConfig.tags
+* since: v1.57
+- type: <[Array]<[string]>>
+
+Resolved global tags. See [`property: TestConfig.tag`].
+
 ## property: FullConfig.updateSnapshots
 * since: v1.10
 - type: <[UpdateSnapshots]<"all"|"changed"|"missing"|"none">>

docs/src/test-api/class-testconfig.md

@@ -535,6 +535,25 @@ export default defineConfig({
 ```
 
 
+## property: TestConfig.tag
+* since: v1.57
+- type: ?<[string]|[Array]<[string]>>
+
+Tag or tags prepended to each test in the report. Useful for tagging your test run to differentiate between [CI environments](../test-sharding.md#merging-reports-from-multiple-environments).
+
+Note that each tag must start with `@` symbol. Learn more about [tagging](../test-annotations.md#tag-tests).
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  tag: process.env.CI_ENVIRONMENT_NAME,  // for example "@APIv2"
+});
+```
+
+
 ## property: TestConfig.testDir
 * since: v1.10
 - type: ?<[string]>

docs/src/test-reporters-js.md

@@ -263,16 +263,50 @@ Blob reports contain all the details about the test run and can be used later to
 npx playwright test --reporter=blob
 ```
 
-By default, the report is written into the `blob-report` directory in the package.json directory or current working directory (if no package.json is found). The report file name looks like `report-<hash>.zip` or `report-<hash>-<shard_number>.zip` when [sharding](./test-sharding.md) is used. The hash is an optional value computed from `--grep`, `--grepInverted`, `--project` and file filters passed as command line arguments. The hash guarantees that running Playwright with different command line options will produce different but stable between runs report names. The output file name can be overridden in the configuration file or pass as `'PLAYWRIGHT_BLOB_OUTPUT_FILE'` environment variable.
+By default, the report is written into the `blob-report` directory in the package.json directory or current working directory (if no package.json is found).
+
+The report file name looks like `report-<hash>.zip` or `report-<hash>-<shard_number>.zip` when [sharding](./test-sharding.md) is used. The hash is an optional value computed from `--grep`, `--grepInverted`, `--project`, [`property: TestConfig.tag`] and file filters passed as command line arguments. The hash guarantees that running Playwright with different command line options will produce different but stable between runs report names. The output file name can be overridden in the configuration file or passed as `'PLAYWRIGHT_BLOB_OUTPUT_FILE'` environment variable.
+
+<Tabs
+  groupId="blob-report"
+  defaultValue="shards"
+  values={[
+    {label: 'Shards', value: 'shards'},
+    {label: 'Environments', value: 'environments'},
+  ]
+}>
+
+<TabItem value="shards">
+
+When using blob report to merge multiple shards, you don't have to pass any options.
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: 'blob',
+});
+```
+
+</TabItem>
+
+<TabItem value="environments">
+
+When running tests in different environments, you might want to use [`property: TestConfig.tag`] to add a global tag corresponding to the environment. This tag will bring clarity to the merged report, and it will be used to produce a unique blob report name.
 
 ```js title="playwright.config.ts"
 import { defineConfig } from '@playwright/test';
 
 export default defineConfig({
-  reporter: [['blob', { outputFile: `./blob-report/report-${os.platform()}.zip` }]],
+  reporter: 'blob',
+  tag: process.env.CI_ENVIRONMENT_NAME,  // for example "@APIv2" or "@linux"
 });
 ```
 
+</TabItem>
+
+</Tabs>
+
 Blob report supports following configuration options and environment variables:
 
 | Environment Variable Name | Reporter Config Option| Description | Default

docs/src/test-sharding-js.md

@@ -58,7 +58,7 @@ export default defineConfig({
 });
 ```
 
-Blob report contains information about all the tests that were run and their results as well as all test attachments such as traces and screenshot diffs. Blob reports can be merged and converted to any other Playwright report. By default, blob report will be generated into `blob-report` directory.
+Blob report contains information about all the tests that were run and their results as well as all test attachments such as traces and screenshot diffs. Blob reports can be merged and converted to any other Playwright report. By default, blob report will be generated into `blob-report` directory. You can learn about [blob report options here](./test-reporters.md#blob-reporter).
 
 To merge reports from multiple shards, put the blob report files into a single directory, for example `all-blob-reports`. Blob report names contain shard number, so they will not clash.
 
@@ -164,6 +164,22 @@ You can now see the reports have been merged and a combined HTML report is avail
 <img width="875" alt="image" src="https://github.com/microsoft/playwright/assets/9798949/b69dac59-fc19-4b98-8f49-814b1c29ca02" />
 
 
+## Merging reports from multiple environments
+
+If you want to run the same tests in multiple environments, as opposed to shard your tests onto multiple machines, you need to differentiate these enviroments.
+
+In this case, it is useful to specify the [`property: TestConfig.tag`] property, to tag all tests with the environment name. This tag will be automatically picked up by the blob report and later on by the merge tool.
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: process.env.CI ? 'blob' : 'html',
+  tag: process.env.CI_ENVIRONMENT_NAME,  // for example "@APIv2"
+});
+```
+
+
 ## Merge-reports CLI
 
 `npx playwright merge-reports path/to/blob-reports-dir` reads all blob reports from the passed directory and merges them into a single report.

packages/playwright/src/common/config.ts

@@ -89,6 +89,12 @@ export class FullConfigInternal {
     // so that plugins such as gitCommitInfoPlugin can populate metadata once.
     userConfig.metadata = userConfig.metadata || {};
 
+    const globalTags = Array.isArray(userConfig.tag) ? userConfig.tag : (userConfig.tag ? [userConfig.tag] : []);
+    for (const tag of globalTags) {
+      if (tag[0] !== '@')
+        throw new Error(`Tag must start with "@" symbol, got "${tag}" instead.`);
+    }
+
     this.config = {
       configFile: resolvedConfigFile,
       rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
@@ -107,6 +113,7 @@ export class FullConfigInternal {
       quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false),
       projects: [],
       shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null),
+      tags: globalTags,
       updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'),
       updateSourceMethod: takeFirst(configCLIOverrides.updateSourceMethod, userConfig.updateSourceMethod, 'patch'),
       version: require('../../package.json').version,

packages/playwright/src/common/testLoader.ts

@@ -25,19 +25,21 @@ import { requireOrImport } from '../transform/transform';
 import { filterStackTrace } from '../util';
 
 import type { TestError } from '../../types/testReporter';
+import type { FullConfigInternal } from './config';
 
 export const defaultTimeout = 30000;
 
 // To allow multiple loaders in the same process without clearing require cache,
 // we make these maps global.
 const cachedFileSuites = new Map<string, Suite>();
 
-export async function loadTestFile(file: string, rootDir: string, testErrors?: TestError[]): Promise<Suite> {
+export async function loadTestFile(file: string, config: FullConfigInternal, testErrors?: TestError[]): Promise<Suite> {
   if (cachedFileSuites.has(file))
     return cachedFileSuites.get(file)!;
-  const suite = new Suite(path.relative(rootDir, file) || path.basename(file), 'file');
+  const suite = new Suite(path.relative(config.config.rootDir, file) || path.basename(file), 'file');
   suite._requireFile = file;
   suite.location = { file, line: 0, column: 0 };
+  suite._tags = [...config.config.tags];
 
   setCurrentlyLoadingFileSuite(suite);
   if (!isWorkerProcess()) {

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -25,7 +25,10 @@ export type JsonStackFrame = { file: string, line: number, column: number };
 
 export type JsonStdIOType = 'stdout' | 'stderr';
 
-export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown'>;
+export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown'> & {
+  // optional for backwards compatibility
+  tags?: reporterTypes.FullConfig['tags'],
+};
 
 export type JsonPattern = {
   s?: string;
@@ -748,6 +751,7 @@ export const baseFullConfig: reporterTypes.FullConfig = {
   rootDir: '',
   quiet: false,
   shard: null,
+  tags: [],
   updateSnapshots: 'missing',
   updateSourceMethod: 'patch',
   version: '',

packages/playwright/src/loader/loaderMain.ts

@@ -44,7 +44,7 @@ export class LoaderMain extends ProcessRunner {
   async loadTestFile(params: { file: string }) {
     const testErrors: TestError[] = [];
     const config = await this._config();
-    const fileSuite = await loadTestFile(params.file, config.config.rootDir, testErrors);
+    const fileSuite = await loadTestFile(params.file, config, testErrors);
     this._poolBuilder.buildPools(fileSuite);
     return { fileSuite: fileSuite._deepSerialize(), testErrors };
   }

packages/playwright/src/reporters/merge.ts

@@ -38,6 +38,7 @@ type ReportData = {
   eventPatchers: JsonEventPatchers;
   reportFile: string;
   metadata: BlobReportMetadata;
+  tags: string[];
 };
 
 export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {
@@ -76,13 +77,15 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
   };
 
   await dispatchEvents(eventData.prologue);
-  for (const { reportFile, eventPatchers, metadata } of eventData.reports) {
+  for (const { reportFile, eventPatchers, metadata, tags } of eventData.reports) {
     const reportJsonl = await fs.promises.readFile(reportFile);
     const events = parseTestEvents(reportJsonl);
     new JsonStringInternalizer(stringPool).traverse(events);
     eventPatchers.patchers.push(new AttachmentPathPatcher(dir));
     if (metadata.name)
       eventPatchers.patchers.push(new GlobalErrorPatcher(metadata.name));
+    if (tags.length)
+      eventPatchers.patchers.push(new GlobalErrorPatcher(tags.join(' ')));
     eventPatchers.patchEvents(events);
     await dispatchEvents(events);
   }
@@ -219,20 +222,24 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
       eventPatchers.patchers.push(new PathSeparatorPatcher(metadata.pathSeparator));
     eventPatchers.patchEvents(parsedEvents);
 
+    let tags: string[] = [];
     for (const event of parsedEvents) {
-      if (event.method === 'onConfigure')
+      if (event.method === 'onConfigure') {
         configureEvents.push(event);
-      else if (event.method === 'onProject')
+        tags = event.params.config.tags || [];
+      } else if (event.method === 'onProject') {
         projectEvents.push(event);
-      else if (event.method === 'onEnd')
+      } else if (event.method === 'onEnd') {
         endEvents.push(event);
+      }
     }
 
     // Save information about the reports to stream their test events later.
     reports.push({
       eventPatchers,
       reportFile: localPath,
       metadata,
+      tags,
     });
   }
 

packages/playwright/src/reporters/teleEmitter.ts

@@ -172,6 +172,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       workers: config.workers,
       globalSetup: config.globalSetup,
       globalTeardown: config.globalTeardown,
+      tags: config.tags,
     };
   }