Add Playwright tests, remove websockify from image

Author: manics

.github/workflows/test.yaml

@@ -40,25 +40,14 @@ jobs:
         run: |
           docker build --progress=plain --build-arg vncserver=${{ matrix.vncserver }} -t test .
 
-      - name: (inside container) websockify --help
-        run: |
-          docker run test websockify --help
-
       - name: (inside container) vncserver -help
         run: |
           # -help flag is not available for TurboVNC, but it emits the -help
           # equivalent information anyhow if passed -help, but also errors. Due
           # to this, we fallback to use the errorcode of vncsrever -list.
           docker run test bash -c "vncserver -help || vncserver -list > /dev/null"
 
-      - name: Install websocat, a test dependency"
-        run: |
-          wget -q https://github.com/vi/websocat/releases/download/v1.12.0/websocat.x86_64-unknown-linux-musl \
-              -O /usr/local/bin/websocat
-          chmod +x /usr/local/bin/websocat
-
       - name: Test vncserver
-        if: always()
         run: |
           container_id=$(docker run -d -it -p 5901:5901 test vncserver -xstartup /opt/install/jupyter_remote_desktop_proxy/share/xstartup -verbose -fg -geometry 1680x1050 -SecurityTypes None -rfbport 5901)
           sleep 1
@@ -79,71 +68,24 @@ jobs:
 
           docker stop $container_id > /dev/null
           if [ "$TEST_OK" == "false" ]; then
-              echo "One or more tests failed!"
+              echo "Test failed!"
               exit 1
           fi
 
-      - name: Test websockify'ed vncserver
-        if: always()
+      - name: Install playwright
         run: |
-          container_id=$(docker run -d -it -p 5901:5901 test websockify --verbose --log-file=/tmp/websockify.log --heartbeat=30 5901 -- vncserver -xstartup /opt/install/jupyter_remote_desktop_proxy/share/xstartup -verbose -fg -geometry 1680x1050 -SecurityTypes None -rfbport 5901)
-          sleep 1
+          python -mpip install -r dev-requirements.txt
+          python -mplaywright install --with-deps
 
-          echo "::group::Install websocat, a test dependency"
-          docker exec --user root $container_id bash -c '
-              wget -q https://github.com/vi/websocat/releases/download/v1.12.0/websocat.x86_64-unknown-linux-musl \
-                  -O /usr/local/bin/websocat
-              chmod +x /usr/local/bin/websocat
-          '
-          echo "::endgroup::"
-
-          docker exec -it $container_id websocat --binary --one-message --exit-on-eof "ws://localhost:5901/" 2>&1 | tee -a /dev/stderr | \
-              grep --quiet RFB && echo "Passed test" || { echo "Failed test" && TEST_OK=false; }
-
-          echo "::group::websockify logs"
-          docker exec $container_id bash -c "cat /tmp/websockify.log"
-          echo "::endgroup::"
-
-          echo "::group::vncserver logs"
-          docker exec $container_id bash -c 'cat ~/.vnc/*.log'
-          echo "::endgroup::"
-
-          docker stop $container_id > /dev/null
-          if [ "$TEST_OK" == "false" ]; then
-              echo "One or more tests failed!"
-              exit 1
-          fi
-
-      - name: Test project's proxy to websockify'ed vncserver
-        if: always()
+      - name: Playwright browser test
         run: |
           container_id=$(docker run -d -it -p 8888:8888 -e JUPYTER_TOKEN=secret test)
           sleep 3
+          export CONTAINER_ID=$container_id
+          export JUPYTER_HOST=http://localhost:8888
+          export JUPYTER_TOKEN=secret
 
-          curl --silent --fail 'http://localhost:8888/desktop/?token=secret' | grep --quiet 'Jupyter Remote Desktop Proxy' && echo "Passed get index.html test" || { echo "Failed get index.html test" && TEST_OK=false; }
-          curl --silent --fail 'http://localhost:8888/desktop/static/dist/viewer.js?token=secret' > /dev/null && echo "Passed get viewer.js test" || { echo "Failed get viewer.js test" && TEST_OK=false; }
-
-          # The first attempt often fails, but the second always(?) succeeds.
-          #
-          # This could be related to jupyter-server-proxy's issue
-          # https://github.com/jupyterhub/jupyter-server-proxy/issues/459
-          # because the client/proxy websocket handshake completes before the
-          # proxy/server handshake. This issue is tracked for this project by
-          # https://github.com/jupyterhub/jupyter-remote-desktop-proxy/issues/105.
-          #
-          websocat --binary --one-message --exit-on-eof 'ws://localhost:8888/desktop-websockify/?token=secret' 2>&1 \
-            | tee -a /dev/stderr \
-            | grep --quiet RFB \
-           && echo "Passed initial websocket test" \
-           || { \
-                  echo "Failed initial websocket test" \
-               && sleep 1 \
-               && websocat --binary --one-message --exit-on-eof 'ws://localhost:8888/desktop-websockify/?token=secret' 2>&1 \
-                    | tee -a /dev/stderr \
-                    | grep --quiet RFB \
-                  && echo "Passed second websocket test" \
-                  || { echo "Failed second websocket test" && TEST_OK=false; } \
-              }
+          python -mpytest -vs
 
           echo "::group::jupyter_server logs"
           docker logs $container_id
@@ -159,6 +101,3 @@ jobs:
               echo "One or more tests failed!"
               exit 1
           fi
-
-      # TODO: Check VNC desktop works, e.g. by comparing Playwright screenshots
-      # https://playwright.dev/docs/test-snapshots

.gitignore

@@ -134,3 +134,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Additional ignores
+screenshots/

Dockerfile

@@ -5,6 +5,7 @@ USER root
 RUN apt-get -y -qq update \
  && apt-get -y -qq install \
         dbus-x11 \
+        xclip \
         xfce4 \
         xfce4-panel \
         xfce4-session \
@@ -55,5 +56,4 @@ RUN . /opt/conda/bin/activate && \
 
 COPY --chown=$NB_UID:$NB_GID . /opt/install
 RUN . /opt/conda/bin/activate && \
-    pip install -e /opt/install && \
-    jupyter server extension enable jupyter_remote_desktop_proxy
+    pip install /opt/install

dev-requirements.txt

@@ -0,0 +1,3 @@
+pillow==10.3.0
+playwright==1.44.0
+pytest==8.2.2

environment.yml

@@ -4,5 +4,3 @@ dependencies:
   - jupyter-server-proxy>=4.3.0
   - jupyterhub-singleuser
   - pip
-  # TODO: remove when test.yaml is updated
-  - websockify

tests/conftest.py

@@ -0,0 +1,18 @@
+from os import getenv
+
+import pytest
+from playwright.sync_api import sync_playwright
+
+HEADLESS = getenv("HEADLESS", "1").lower() == "1"
+
+
+@pytest.fixture()
+def browser():
+    # browser_type in ["chromium", "firefox", "webkit"]
+    with sync_playwright() as playwright:
+        browser = playwright.firefox.launch(headless=HEADLESS)
+        context = browser.new_context()
+        page = context.new_page()
+        yield page
+        context.clear_cookies()
+        browser.close()

tests/reference/desktop.png

@@ -0,0 +1,77 @@
+from os import getenv
+from pathlib import Path
+from shutil import which
+from subprocess import check_output
+from uuid import uuid4
+
+from PIL import Image, ImageChops
+from playwright.sync_api import expect
+
+HERE = Path(__file__).absolute().parent
+
+CONTAINER_ID = getenv("CONTAINER_ID", "test")
+JUPYTER_HOST = getenv("JUPYTER_HOST", "http://localhost:8888")
+JUPYTER_TOKEN = getenv("JUPYTER_TOKEN", "secret")
+
+
+def compare_screenshot(test_image, threshold=1):
+    # Compare images by calculating the mean absolute difference
+    # Images must be the same size
+    # threshold: Average difference per pixel, this depends on the image type
+    # e.g. for 24 bit images (8 bit RGB pixels) threshold=1 means a maximum
+    # difference of 1 bit per pixel per channel
+    reference = Image.open(HERE / "reference" / "desktop.png")
+    test = Image.open(test_image)
+
+    # Absolute difference
+    # Convert to RGB, alpha channel breaks ImageChops
+    diff = ImageChops.difference(reference.convert("RGB"), test.convert("RGB"))
+    diff_data = diff.getdata()
+
+    m = sum(sum(px) for px in diff_data) / diff_data.size[0] / diff_data.size[1]
+    assert m < threshold
+
+
+# To debug this set environment variable HEADLESS=0
+def test_desktop(browser):
+    page = browser
+    page.goto(f"{JUPYTER_HOST}/lab?token={JUPYTER_TOKEN}")
+    page.wait_for_url(f"{JUPYTER_HOST}/lab")
+
+    # JupyterLab extension icon
+    expect(page.get_by_text("Desktop [↗]")).to_be_visible()
+    with page.expect_popup() as page1_info:
+        page.get_by_text("Desktop [↗]").click()
+    page1 = page1_info.value
+    page1.wait_for_url(f"{JUPYTER_HOST}/desktop/")
+
+    expect(page1.get_by_text("Status: Connected")).to_be_visible()
+    expect(page1.locator("canvas")).to_be_visible()
+
+    # Screenshot the desktop element only
+    # May take a few seconds to load
+    page1.wait_for_timeout(5000)
+    # Use a nontemp folder so we can check it manually if necessary
+    screenshot = "screenshots/desktop.png"
+    page1.locator("canvas").screenshot(path=screenshot)
+
+    # Open clipboard, enter random text, close clipboard
+    clipboard_text = str(uuid4())
+    page1.get_by_role("link", name="Remote Clipboard").click()
+    page1.wait_for_selector("#clipboard-text")
+    page1.locator("#clipboard-text").click()
+    page1.locator("#clipboard-text").fill(clipboard_text)
+    page1.get_by_role("link", name="Remote Clipboard").click()
+
+    # Exec into container to check clipboard contents
+    for engine in ["docker", "podman"]:
+        if which(engine):
+            break
+    else:
+        raise RuntimeError("Container engine not found")
+    clipboard = check_output(
+        [engine, "exec", "-eDISPLAY=:1", CONTAINER_ID, "xclip", "-o"]
+    )
+    assert clipboard.decode() == clipboard_text
+
+    compare_screenshot(screenshot)