Update the way we run UI-tests (#1054)

* Update the way we run UI-tests

This uses a logic closer to other projects (ipympl, bqplot,
ipydatagrid). And run ui-tests with the dark theme.

* Install browser

* Increase timeout

* Update Playwright Snapshots

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: martinRenou

.github/workflows/update_galata_references.yml

@@ -47,6 +47,7 @@ jobs:
       - name: Install JS deps
         run: |
           jlpm
+          jlpm playwright install chromium
           jlpm build
         working-directory: js
 

ui-tests/jupyter_server_config.py

@@ -5,4 +5,3 @@
 c.ServerApp.open_browser = False
 c.LabApp.open_browser = False
 c.LabApp.expose_app_in_browser = True
-c.ServerApp.notebook_dir = './notebooks' 

ui-tests/package.json

@@ -15,6 +15,7 @@
   "license": "MIT",
   "dependencies": {
     "@jupyterlab/galata": "^4.3.5",
+    "klaw-sync": "^6.0.0",
     "rimraf": "^3.0.2"
   }
 }

ui-tests/playwright.config.js

@@ -5,6 +5,7 @@ module.exports = {
   expect: {
     toMatchSnapshot: { threshold: 0.33 },
   },
-  preserveOutput: 'failures-only', 
-  retries: 0
-};
+  preserveOutput: 'failures-only',
+  retries: 0,
+  timeout: 600000,
+};

ui-tests/tests/ipyleaflet.test.ts

@@ -1,49 +1,85 @@
-import { galata, IJupyterLabPageFixture, test } from "@jupyterlab/galata";
-import { expect } from "@playwright/test";
-
-async function renderMap(fileName: string, page: IJupyterLabPageFixture) {
-  const fullName = `./${fileName}.ipynb`;
-  await page.notebook.openByPath(fullName);
-  await page.notebook.activate(fullName);
-  await page.notebook.run();
-  await page.notebook.waitForRun();
-  const maps = await page.$("div.leaflet-container");
-  await new Promise((_) => setTimeout(_, 1000));
-
-  // Move the mouse to the center of the map
-  const bb = await maps?.boundingBox();
-  await page.mouse.move(bb!.x + bb!.width / 2, bb!.y + bb!.height / 2);
-
-  expect(await maps!.screenshot()).toMatchSnapshot({
-    name: `${fileName}.png`,
-  });
+import { IJupyterLabPageFixture, test } from '@jupyterlab/galata';
+import { expect } from '@playwright/test';
+import * as path from 'path';
+const klaw = require('klaw-sync');
+
+
+const filterUpdateNotebooks = item => {
+  const basename = path.basename(item.path);
+  return basename.includes('_update');
 }
 
-const notebookList = [
-  "DivIcon",
-  "DrawControl",
-  "FullScreenControl",
-  "Icon",
-  "LayersControl",
-  "LegendControl",
-  "MagnifyingGlass",
-  "Marker",
-  "Polyline",
-  "Popup",
-  "ScaleControl",
-  "SplitMapControl",
-  "TileLayer",
-  "WidgetControl",
-  "ZoomControl",
-];
-
-test.describe("ipyleaflet Visual Regression", () => {
-  test.beforeEach(async ({ page }) => {
-    page.setViewportSize({ width: 1920, height: 1080 });
-  });
-  for (const name of notebookList) {
-    test(`Render ${name}`, async ({ page }) => {
-      await renderMap(name, page);
+const testCellOutputs = async (page: IJupyterLabPageFixture, tmpPath: string, theme: 'JupyterLab Light' | 'JupyterLab Dark') => {
+  const paths = klaw(path.resolve(__dirname, '../notebooks'), {filter: item => !filterUpdateNotebooks(item), nodir: true});
+  const notebooks = paths.map(item => path.basename(item.path));
+
+  const contextPrefix = theme == 'JupyterLab Light' ? 'light_' : 'dark_';
+  page.theme.setTheme(theme);
+
+  for (const notebook of notebooks) {
+    let results = [];
+
+    await page.notebook.openByPath(`${tmpPath}/${notebook}`);
+    await page.notebook.activate(notebook);
+
+    let numCellImages = 0;
+
+    const getCaptureImageName = (contextPrefix: string, notebook: string, id: number): string => {
+      return `${contextPrefix}-${notebook}-cell-${id}.png`;
+    };
+
+    await page.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        const cell = await page.notebook.getCellOutput(cellIndex);
+        if (cell) {
+          const map = await cell.$("div.leaflet-container");
+
+          if (map) {
+            await new Promise((_) => setTimeout(_, 1000));
+
+            // Move the mouse to the center of the map
+            const bb = await map.boundingBox();
+            if (bb) {
+              await page.mouse.move(bb.x + bb.width / 2, bb.y + bb.height / 2);
+            }
+          }
+
+          results.push(await cell.screenshot());
+          numCellImages++;
+        }
+      }
     });
+
+    await page.notebook.save();
+
+    for (let c = 0; c < numCellImages; ++c) {
+      expect(results[c]).toMatchSnapshot(getCaptureImageName(contextPrefix, notebook, c));
+    }
+
+    await page.notebook.close(true);
   }
+}
+
+test.describe('ipyleaflet Visual Regression', () => {
+  test.beforeEach(async ({ page, tmpPath }) => {
+    await page.contents.uploadDirectory(
+      path.resolve(__dirname, '../notebooks'),
+      tmpPath
+    );
+    await page.filebrowser.openDirectory(tmpPath);
+  });
+
+  test('Light theme: Check ipyleaflet renders', async ({
+    page,
+    tmpPath,
+  }) => {
+    await testCellOutputs(page, tmpPath, 'JupyterLab Light');
+  });
+
+  test('Dark theme: Check ipyleaflet renders', async ({
+    page,
+    tmpPath,
+  }) => {
+    await testCellOutputs(page, tmpPath, 'JupyterLab Dark');
+  });
 });