Merge pull request #320 from fractal-analytics-platform/playwright

Setup of end to end tests and coverage

Author: tcompa

.github/workflows/coverage.yaml

@@ -0,0 +1,52 @@
+name: Coverage
+
+on:
+  push:
+    branches: ['main']
+  pull_request:
+    branches: ['main']
+
+jobs:
+  unit_tests:
+    uses: ./.github/workflows/unit_tests.yaml
+
+  end_to_end_tests:
+    uses: ./.github/workflows/end_to_end_tests.yaml
+
+  coverage:
+    name: Compute coverage
+    runs-on: ubuntu-22.04
+    needs: [unit_tests, end_to_end_tests]
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Set up node
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+          cache: npm
+
+      - name: Install nyc
+        run: npm install -g nyc
+
+      - name: Create coverage-all folder
+        run: mkdir coverage-all
+
+      - name: Download unit coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: unit-coverage
+          path: coverage-all
+
+      - name: Download playwright coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: playwright-coverage
+          path: coverage-all
+
+      - name: Code coverage report
+        uses: tj-actions/[email protected]
+        with:
+          coverage-command: "nyc report --temp-dir ./coverage-all --reporter=text --exclude-after-remap false"

.github/workflows/end_to_end_tests.yaml

@@ -1,15 +1,11 @@
 name: End-to-end tests
 
-on:
-  push:
-    branches: ['main']
-  pull_request:
-    branches: ['main']
+on: [workflow_call]
 
 jobs:
   end_to_end_tests:
     name: 'Node ${{ matrix.node-version }}'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 10
 
     strategy:
@@ -29,5 +25,20 @@ jobs:
       - name: Install dependencies
         run: npm install
 
-      - name: Run playwright tests (not implemented)
-        run: echo "Not implemented"
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      # Fix for Node 18 issue:
+      # See: https://github.com/node-fetch/node-fetch/issues/1624
+      - name: Set fractal-server host for Node 18
+        if: matrix.node-version == '18'
+        run: echo "FRACTAL_SERVER_HOST=http://127.0.0.1:8000" >> $GITHUB_ENV
+
+      - name: Run Playwright tests
+        run: npx playwright test
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: playwright-coverage
+          path: coverage-playwright/
+          retention-days: 30

.github/workflows/github_release.yml renamed to .github/workflows/github_release.yaml

@@ -1,15 +1,11 @@
 name: Unit tests
 
-on:
-  push:
-    branches: ['main']
-  pull_request:
-    branches: ['main']
+on: [workflow_call]
 
 jobs:
   unit_tests:
     name: 'Node ${{ matrix.node-version }}'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 10
 
     strategy:
@@ -30,4 +26,10 @@ jobs:
         run: npm install
 
       - name: Run vitest tests
-        run: npm run test
+        run: npx vitest run --coverage.enabled --coverage.provider=istanbul --coverage.include=src --coverage.reporter=json --coverage.all
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: unit-coverage
+          path: coverage-unit/
+          retention-days: 30

.github/workflows/project-management.yml renamed to .github/workflows/project-management.yaml

@@ -9,4 +9,6 @@ node_modules
 .idea
 *.bak
 .*.swp
-/tests/.auth
+/tests/.auth
+coverage*/
+.nyc_output

.github/workflows/unit_tests.yaml

@@ -0,0 +1,3 @@
+{
+  "all": true
+}

.gitignore

@@ -1,5 +1,9 @@
 *Note: Numbers like (\#123) point to closed Pull Requests on the fractal-web repository.*
 
+# Unreleased
+
+* Added end to end tests and coverage configuration (\#320).
+
 # 0.6.1
 
 This release works best with fractal-server >=1.3.12 (since it makes use of the

.nycrc

@@ -2,14 +2,40 @@
 
 ## Unit tests
 
-Unit tests are performed via [vitest](https://vitest.dev), via the `test` script defined in `package.json`. Current tests are in the `__tests__` folder.
-
+Unit tests are performed via [vitest](https://vitest.dev), via the `test` script defined in `package.json`.
 
 ## End-to-end testing
 
-This is not yet implemented in an automated way (tracked in
-https://github.com/fractal-analytics-platform/fractal-web/issues/8), and for
-the moment it needs to be done manually.
+E2E tests are done using playwright. They can be executed using the following command:
+
+```
+npx playwright test
+```
+
+To print Svelte webserver log set the environment variable `DEBUG=pw:webserver`.
+
+To execute the tests seeing the browser add the `--headed` flag or the `--debug` flag if you need to watch them step by step.
+
+## Coverage
+
+Coverage for the unit tests:
+
+```
+npx vitest run --coverage.enabled --coverage.provider=istanbul --coverage.include=src --coverage.reporter=json --coverage.all
+```
+
+Warning: coverage takes some time using the option `--coverage.all`.
+
+Coverage for the playwright tests is automatically collected when running `npx playwright test`.
+
+To generate a merged html report:
+
+```
+mkdir coverage-all
+cp coverage-playwright/* coverage-all
+cp coverage-unit/* coverage-all
+npx nyc report --report-dir ./coverage-html --temp-dir ./coverage-all --reporter=html --exclude-after-remap false
+```
 
 ### Local `fractal-server` instance