Add ui tests (#171)

* add ui tests

* Execute ui test only on linux

* Restore running test against JLab prerelease
To many selectors and screenshots are wrong otherwise

* Change notebook directory to an empty folder

* Force viewport & remove slow motion
Pin playwright

* Fixe embedme

* Close filebrowser before taking screenshot
Fix other tests

* Add test on server-extension
Remove slow down

* Ensure homogeneous ui test config

* Homogeneous timeout

* Fix integration test on server-extension

* Fix JSDoc warnings

* Fix README

* More robust tests

* Update README.md

Author: fcollonval

.github/workflows/main.yml

@@ -64,6 +64,8 @@ jobs:
           diff ../hello-world/.eslintignore .eslintignore
           diff ../hello-world/.eslintrc.js .eslintrc.js
           diff ../hello-world/.gitignore .gitignore
+          diff ../hello-world/ui-tests/playwright.config.ts ./ui-tests/playwright.config.ts
+          diff ../hello-world/ui-tests/README.md ./ui-tests/README.md
         shell: bash
       - name: Install node
         if: steps.filter.outputs.extension == 'true'
@@ -103,13 +105,34 @@ jobs:
       - name: Build and check by extension
         if: steps.filter.outputs.extension == 'true'
         run: |
-          pip install . -vvv
+          pip install . -v
           jupyter labextension list 2>&1 | tee labextension.list
           cat labextension.list | grep -ie "@jupyterlab-examples/*.*OK"
           python -m jupyterlab.browser_check
           pip uninstall -y $(python setup.py --name)
         shell: bash
 
+      - name: Integration tests
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux')
+        run: |
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env down || true
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env pull -q || true
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env build
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env run --rm e2e
+      - name: Upload UI Test artifacts
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux') && always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ui-test-output
+          path: |
+            ${{ matrix.example }}/ui-tests/test-results
+      - name: Stop containers
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux') && always()
+        run: |
+          # Print jupyterlab logs before removing the containers using the container name set in docker-compose file
+          docker logs jupyterlab
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env down
+
   build_serverextension:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -147,6 +170,8 @@ jobs:
           diff hello-world/tsconfig.json server-extension/tsconfig.json
           diff hello-world/.eslintignore server-extension/.eslintignore
           diff hello-world/.eslintrc.js server-extension/.eslintrc.js
+          diff hello-world/ui-tests/playwright.config.ts server-extension/ui-tests/playwright.config.ts
+          diff hello-world/ui-tests/README.md server-extension/ui-tests/README.md
         shell: bash
       - name: Install Python
         if: steps.filter.outputs.extension == 'true'
@@ -218,6 +243,29 @@ jobs:
           cat labextension.list | grep -ie "@jupyterlab-examples/server-extension.*OK"
           python -m jupyterlab.browser_check
 
+      - name: Integration tests
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux')
+        run: |
+          cd server-extension
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env down || true
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env pull -q || true
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env build
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env run --rm e2e
+      - name: Upload UI Test artifacts
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux') && always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: ui-test-output
+          path: |
+            server-extension/ui-tests/test-results
+      - name: Stop containers
+        if: steps.filter.outputs.extension == 'true' && startsWith(runner.os, 'Linux') && always()
+        run: |
+          cd server-extension
+          # Print jupyterlab logs before removing the containers using the container name set in docker-compose file
+          docker logs jupyterlab
+          docker-compose -f ../end-to-end-tests/docker-compose.yml --env-file ./ui-tests/.env down
+
   build_all:
     runs-on: ${{ matrix.os }}
     strategy:

.gitignore

@@ -13,3 +13,4 @@ __pycache__
 
 static
 labextension
+**/test-results/

command-palette/.eslintignore

@@ -2,3 +2,4 @@ node_modules
 dist
 coverage
 **/*.d.ts
+ui-tests

command-palette/style/index.css

@@ -0,0 +1 @@
+@import url('base.css');

command-palette/ui-tests/.env

@@ -0,0 +1,2 @@
+EXT_FOLDER=command-palette/jupyterlab_examples_command_palette
+EXT_NAME=command-palette

command-palette/ui-tests/README.md

@@ -0,0 +1,114 @@
+# Test
+
+The test will produce a video to help debugging and check what happened.
+
+To execute integration tests, you can two options:
+
+- use docker-compose (cons: needs to know and use docker) - this is a more reliable solution.
+- run tests locally (cons: will interact with your JupyterLab user settings)
+
+## Test on docker
+
+1. Compile the extension:
+
+```
+jlpm install
+jlpm run build:prod
+```
+
+2. Execute the docker stack (`extension-path` needs to be set accordingly):
+
+```
+docker-compose -f ./end-to-end-tests/docker-compose.yml --env-file ./extension-path/ui-tests/.env build
+docker-compose -f ./end-to-end-tests/docker-compose.yml --env-file ./extension-path/ui-tests/.env run --rm e2e
+docker-compose -f ./end-to-end-tests/docker-compose.yml --env-file ./extension-path/ui-tests/.env down
+```
+
+## Test locally
+
+1. Compile the extension:
+
+```
+jlpm install
+jlpm run build:prod
+```
+
+2. Start JupyterLab _with the extension installed_ without any token or password
+
+```
+jupyter lab --ServerApp.token= --ServerApp.password=
+```
+
+3. Execute in another console the [Playwright](https://playwright.dev/docs/test-intro) tests:
+
+```
+cd ui-tests
+jlpm install
+npx playwright test
+```
+
+# Create tests
+
+To create tests, the easiest way is to use the code generator tool of playwright:
+
+1. Compile the extension:
+
+```
+jlpm install
+jlpm run build:prod
+```
+
+2. Start JupyterLab _with the extension installed_ without any token or password:
+
+**Using docker**
+
+```
+docker-compose -f ./end-to-end-tests/docker-compose.yml --env-file ./extension-path/ui-tests/.env run --rm -p 8888:8888 lab
+```
+
+**Using local installation**
+
+```
+jupyter lab --ServerApp.token= --ServerApp.password=
+```
+
+3. Launch the code generator tool:
+
+```
+cd ui-tests
+jlpm install
+npx playwright codegen localhost:8888
+```
+
+# Debug tests
+
+To debug tests, a good way is to use the inspector tool of playwright:
+
+1. Compile the extension:
+
+```
+jlpm install
+jlpm run build:prod
+```
+
+2. Start JupyterLab _with the extension installed_ without any token or password:
+
+**Using docker**
+
+```
+docker-compose -f ./end-to-end-tests/docker-compose.yml --env-file ./extension-path/ui-tests/.env run --rm -p 8888:8888 lab
+```
+
+**Using local installation**
+
+```
+jupyter lab --ServerApp.token= --ServerApp.password=
+```
+
+3. Launch the code generator tool:
+
+```
+cd ui-tests
+jlpm install
+PWDEBUG=1 npx playwright test
+```

command-palette/ui-tests/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@jupyterlab-examples/command-palette-tests",
+  "version": "0.1.0",
+  "description": "Integration test for command-palette example",
+  "repository": "https://github.com/jupyterlab/extension-examples",
+  "author": "Project Jupyter Contributors",
+  "license": "BSD-3-Clause",
+  "private": true,
+  "devDependencies": {
+    "@playwright/test": "1.12.3",
+    "typescript": "~4.1.3"
+  }
+}

command-palette/ui-tests/playwright.config.ts

@@ -0,0 +1,18 @@
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  timeout: 60000,
+  use: {
+    // Browser options
+    // headless: false,
+    // slowMo: 500,
+
+    // Context options
+    viewport: { width: 1280, height: 720 },
+
+    // Artifacts
+    video: 'on',
+  },
+};
+
+export default config;

command-palette/ui-tests/tests/command-palette.spec.ts

@@ -0,0 +1,35 @@
+import { test, expect } from '@playwright/test';
+
+const TARGET_URL = process.env.TARGET_URL ?? 'http://localhost:8888';
+
+test('should emit console message when called from palette', async ({
+  page,
+}) => {
+  const logs: string[] = [];
+
+  page.on('console', (message) => {
+    logs.push(message.text());
+  });
+
+  await page.goto(`${TARGET_URL}/lab`);
+  await page.waitForSelector('#jupyterlab-splash', { state: 'detached' });
+  await page.waitForSelector('div[role="main"] >> text=Launcher');
+
+  // Click text=View
+  await page.click('text=View');
+  // Click text=Activate Command Palette
+  await page.click('text=Activate Command Palette');
+  // Fill [aria-label="Command Palette Section"] [placeholder="SEARCH"]
+  await page.fill(
+    '[aria-label="Command Palette Section"] [placeholder="SEARCH"]',
+    'Execute'
+  );
+  // Click text=Execute jlab-examples:command-palette Command
+  await page.click('text=Execute jlab-examples:command-palette Command');
+
+  expect(
+    logs.filter((s) =>
+      s.startsWith('jlab-examples:command-palette has been called from palette')
+    )
+  ).toHaveLength(1);
+});

commands/.eslintignore

@@ -2,3 +2,4 @@ node_modules
 dist
 coverage
 **/*.d.ts
+ui-tests