tests(e2e): Various fixes to visual testing (#6569)

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6569
Reviewed-by: 0ko <0ko@noreply.codeberg.org>

Author: 0ko

tests/e2e/actions.test.e2e.ts

@@ -71,4 +71,5 @@ test('workflow dispatch box not available for unauthenticated users', async ({pa
   await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0');
 
   await expect(page.locator('body')).not.toContainText(workflow_trigger_notification_text);
+  await save_visual(page);
 });

tests/e2e/clipboard-copy.test.e2e.ts

@@ -8,7 +8,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {test} from './utils_e2e.ts';
+import {save_visual, test} from './utils_e2e.ts';
 
 test('copy src file path to clipboard', async ({page}, workerInfo) => {
   test.skip(['Mobile Safari', 'webkit'].includes(workerInfo.project.name), 'Apple clipboard API addon - starting at just $499!');
@@ -19,6 +19,7 @@ test('copy src file path to clipboard', async ({page}, workerInfo) => {
   await page.click('[data-clipboard-text]');
   const clipboardText = await page.evaluate(() => navigator.clipboard.readText());
   expect(clipboardText).toContain('README.md');
+  await save_visual(page);
 });
 
 test('copy diff file path to clipboard', async ({page}, workerInfo) => {
@@ -30,4 +31,6 @@ test('copy diff file path to clipboard', async ({page}, workerInfo) => {
   await page.click('[data-clipboard-text]');
   const clipboardText = await page.evaluate(() => navigator.clipboard.readText());
   expect(clipboardText).toContain('README.md');
+  await expect(page.getByText('Copied')).toBeVisible();
+  await save_visual(page);
 });

tests/e2e/dashboard-ci-status.test.e2e.ts

@@ -3,7 +3,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {save_visual, test} from './utils_e2e.ts';
+import {test} from './utils_e2e.ts';
 
 test.use({user: 'user2'});
 
@@ -23,5 +23,6 @@ test('Correct link and tooltip', async ({page}, testInfo) => {
   const repoStatus = page.locator('.dashboard-repos .repo-owner-name-list > li:nth-child(1) > a:nth-child(2)');
   await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000});
   await expect(repoStatus).toHaveAttribute('data-tooltip-content', /^(Error|Failure)$/);
-  await save_visual(page);
+  // ToDo: Ensure stable screenshot of dashboard. Known to be flaky: https://code.forgejo.org/forgejo/visual-browser-testing/commit/206d4cfb7a4af6d8d7043026cdd4d63708798b2a
+  // await save_visual(page);
 });

tests/e2e/e2e_test.go

@@ -82,6 +82,7 @@ func TestE2e(t *testing.T) {
 
 	runArgs := []string{"npx", "playwright", "test"}
 
+	_, testVisual := os.LookupEnv("VISUAL_TEST")
 	// To update snapshot outputs
 	if _, set := os.LookupEnv("ACCEPT_VISUAL"); set {
 		runArgs = append(runArgs, "--update-snapshots")
@@ -105,6 +106,10 @@ func TestE2e(t *testing.T) {
 			onForgejoRun(t, func(*testing.T, *url.URL) {
 				defer DeclareGitRepos(t)()
 				thisTest := runArgs
+				// when all tests are run, use unique artifacts directories per test to preserve artifacts from other tests
+				if testVisual {
+					thisTest = append(thisTest, "--output=tests/e2e/test-artifacts/"+testname)
+				}
 				thisTest = append(thisTest, path)
 				cmd := exec.Command(runArgs[0], thisTest...)
 				cmd.Env = os.Environ()
@@ -114,7 +119,7 @@ func TestE2e(t *testing.T) {
 				cmd.Stderr = os.Stderr
 
 				err := cmd.Run()
-				if err != nil {
+				if err != nil && !testVisual {
 					log.Fatal("Playwright Failed: %s", err)
 				}
 			})

tests/e2e/example.test.e2e.ts

@@ -5,7 +5,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {test} from './utils_e2e.ts';
+import {save_visual, test} from './utils_e2e.ts';
 
 test('Load Homepage', async ({page}) => {
   const response = await page.goto('/');
@@ -26,6 +26,7 @@ test('Register Form', async ({page}, workerInfo) => {
   expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`);
   await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible();
   await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!');
+  await save_visual(page);
 });
 
 // eslint-disable-next-line playwright/no-skipped-test

tests/e2e/explore.test.e2e.ts

@@ -7,7 +7,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {test} from './utils_e2e.ts';
+import {save_visual, test} from './utils_e2e.ts';
 
 test('Explore view taborder', async ({page}) => {
   await page.goto('/explore/repos');
@@ -42,4 +42,5 @@ test('Explore view taborder', async ({page}) => {
     }
   }
   expect(res).toBe(exp);
+  await save_visual(page);
 });

tests/e2e/markup.test.e2e.ts

@@ -3,7 +3,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {test} from './utils_e2e.ts';
+import {save_visual, test} from './utils_e2e.ts';
 
 test('markup with #xyz-mode-only', async ({page}) => {
   const response = await page.goto('/user2/repo1/issues/1');
@@ -13,4 +13,5 @@ test('markup with #xyz-mode-only', async ({page}) => {
   await expect(comment).toBeVisible();
   await expect(comment.locator('[src$="#gh-light-mode-only"]')).toBeVisible();
   await expect(comment.locator('[src$="#gh-dark-mode-only"]')).toBeHidden();
+  await save_visual(page);
 });

tests/e2e/repo-code.test.e2e.ts

@@ -49,6 +49,7 @@ test('Line Range Selection', async ({page}) => {
   // out-of-bounds end line
   await page.goto(`${filePath}#L1-L100`);
   await assertSelectedLines(page, ['1', '2', '3']);
+  await save_visual(page);
 });
 
 test('Readable diff', async ({page}, workerInfo) => {
@@ -75,6 +76,7 @@ test('Readable diff', async ({page}, workerInfo) => {
       await expect(page.getByText(thisDiff.added, {exact: true})).toHaveCSS('background-color', 'rgb(134, 239, 172)');
     }
   }
+  await save_visual(page);
 });
 
 test.describe('As authenticated user', () => {

tests/e2e/repo-commitgraph.test.e2e.ts

@@ -5,7 +5,7 @@
 // @watch end
 
 import {expect} from '@playwright/test';
-import {test} from './utils_e2e.ts';
+import {save_visual, test} from './utils_e2e.ts';
 
 test('Commit graph overflow', async ({page}) => {
   await page.goto('/user2/diff-test/graph');
@@ -28,4 +28,5 @@ test('Switch branch', async ({page}) => {
   await expect(page.locator('#loading-indicator')).toBeHidden();
   await expect(page.locator('#rel-container')).toBeVisible();
   await expect(page.locator('#rev-container')).toBeVisible();
+  await save_visual(page);
 });

tests/e2e/repo-migrate.test.e2e.ts

@@ -21,7 +21,6 @@ test('Migration Progress Page', async ({page, browser}, workerInfo) => {
   await form.locator('button.primary').click({timeout: 5000});
   await expect(page).toHaveURL('user2/invalidrepo');
   await save_visual(page);
-  // page screenshot of unauthenticatedPage is checked automatically after the test
 
   const ctx = await test_context(browser);
   const unauthenticatedPage = await ctx.newPage();
@@ -37,4 +36,6 @@ test('Migration Progress Page', async ({page, browser}, workerInfo) => {
   await save_visual(page);
   await deleteModal.getByRole('button', {name: 'Delete repository'}).click();
   await expect(page).toHaveURL('/');
+  // checked last to preserve the order of screenshots from first run
+  await save_visual(unauthenticatedPage);
 });