Add some integration tests

samuelhwilliams · samuelhwilliams · commit 14fa758eb318 · 2020-07-03T15:41:05.000+01:00
At the moment we have no tests, which is bad. We provide a number of
examples that are intended to help users see working examples in order
to grab what they need and get working. It would be nice to
simultaneously validate that our examples are functioning as well as
provide the first bits of assurance that Eel itself is working, so the
first integration tests will run the example code and make assertions
as needed against those. This feels like the most bang-for-buck per test
at the moment.

We also set a default timeout of 5 seconds on all integration tests for
the time being, as (locally, at least) they shouldn't take too long. On
GitHub CI, we bump this to 30 seconds because things seem to run quite a
bit slower there. But we still don't want them to stall and get stuck
forever if we do something wrong. 30 seconds should be more than
generous - but we could bump it if needed.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -24,4 +24,4 @@ jobs:
       - name: Setup test execution environment.
         run: pip3 install -r requirements-meta.txt
       - name: Run tox tests
-        run: tox
+        run: tox -- --durations=0 --timeout=30
diff --git a/examples/01 - hello_world/hello.py b/examples/01 - hello_world/hello.py
@@ -10,4 +10,4 @@ def say_hello_py(x):
 say_hello_py('Python World!')
 eel.say_hello_js('Python World!')   # Call a Javascript function
 
-eel.start('hello.html', size=(300, 200))    # Start
+eel.start('hello.html', size=(300, 200))  # Start
diff --git a/examples/02 - callbacks/web/callbacks.html b/examples/02 - callbacks/web/callbacks.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+ <!DOCTYPE html>
 <html>
     <head>
         <title>Callbacks Demo</title>
@@ -27,4 +27,4 @@
     <body>
         Callbacks demo
     </body>
-</html>
+</html>
diff --git a/examples/06 - jinja_templates/hello.py b/examples/06 - jinja_templates/hello.py
@@ -1,7 +1,13 @@
+import random
+
 import eel
 
 eel.init('web')                     # Give folder containing web files
 
+@eel.expose
+def py_random():
+    return random.random()
+
 @eel.expose                         # Expose this function to Javascript
 def say_hello_py(x):
     print('Hello from %s' % x)
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -1 +1,6 @@
+.[jinja2]
+
+psutil==5.7.0
 pytest==5.4.3
+pytest-timeout==1.4.1
+selenium==3.141.0
diff --git a/requirements.txt b/requirements.txt
@@ -3,5 +3,4 @@ bottle-websocket==0.2.9
 gevent==1.3.6
 gevent-websocket==0.10.1
 greenlet==0.4.15
-pkg-resources==0.0.0
 whichcraft==0.4.1
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,37 @@
+import os
+from unittest import mock
+
+import pytest
+from selenium import webdriver
+from selenium.webdriver import DesiredCapabilities
+
+
+@pytest.fixture
+def driver():
+    TEST_BROWSER = os.environ.get("TEST_BROWSER", "chrome").lower()
+
+    if TEST_BROWSER == "chrome":
+        options = webdriver.ChromeOptions()
+        options.headless = True
+        capabilities = DesiredCapabilities.CHROME
+        capabilities['goog:loggingPrefs'] = {"browser": "ALL"}
+
+        driver = webdriver.Chrome(options=options, desired_capabilities=capabilities, service_log_path=os.path.devnull)
+
+    # Firefox doesn't currently supported pulling JavaScript console logs, which we currently scan to affirm that
+    # JS/Python can communicate in some places. So for now, we can't really use firefox/geckodriver during testing.
+    # This may be added in the future: https://github.com/mozilla/geckodriver/issues/284
+    
+    # elif TEST_BROWSER == "firefox":
+    #     options = webdriver.FirefoxOptions()
+    #     options.headless = True
+    #     capabilities = DesiredCapabilities.FIREFOX
+    #     capabilities['loggingPrefs'] = {"browser": "ALL"}
+    #
+    #     driver = webdriver.Firefox(options=options, capabilities=capabilities, service_log_path=os.path.devnull)
+
+    else:
+        raise ValueError(f"Unsupported browser for testing: {TEST_BROWSER}")
+
+    with mock.patch("eel.browsers.open"):
+        yield driver
diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py
@@ -0,0 +1,61 @@
+import os
+from tempfile import TemporaryDirectory, NamedTemporaryFile
+
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions
+from selenium.webdriver.support.wait import WebDriverWait
+
+from tests.utils import get_eel_server, get_console_logs
+
+
+def test_01_hello_world(driver):
+    with get_eel_server('examples/01 - hello_world/hello.py', 'hello.html') as eel_url:
+        driver.get(eel_url)
+        assert driver.title == "Hello, World!"
+
+        console_logs = get_console_logs(driver, minimum_logs=2)
+        assert "Hello from Javascript World!" in console_logs[0]['message']
+        assert "Hello from Python World!" in console_logs[1]['message']
+
+
+def test_02_callbacks(driver):
+    with get_eel_server('examples/02 - callbacks/callbacks.py', 'callbacks.html') as eel_url:
+        driver.get(eel_url)
+        assert driver.title == "Callbacks Demo"
+
+        console_logs = get_console_logs(driver, minimum_logs=1)
+        assert "Got this from Python:" in console_logs[0]['message']
+        assert "callbacks.html" in console_logs[0]['message']
+
+
+def test_03_callbacks(driver):
+    with get_eel_server('examples/03 - sync_callbacks/sync_callbacks.py', 'sync_callbacks.html') as eel_url:
+        driver.get(eel_url)
+        assert driver.title == "Synchronous callbacks"
+
+        console_logs = get_console_logs(driver, minimum_logs=1)
+        assert "Got this from Python:" in console_logs[0]['message']
+        assert "callbacks.html" in console_logs[0]['message']
+
+
+def test_04_file_access(driver: webdriver.Remote):
+    with get_eel_server('examples/04 - file_access/file_access.py', 'file_access.html') as eel_url:
+        driver.get(eel_url)
+        assert driver.title == "Eel Demo"
+
+        with TemporaryDirectory() as temp_dir, NamedTemporaryFile(dir=temp_dir) as temp_file:
+            driver.find_element_by_id('input-box').clear()
+            driver.find_element_by_id('input-box').send_keys(temp_dir)
+            driver.find_element_by_css_selector('button').click()
+
+            assert driver.find_element_by_id('file-name').text == os.path.basename(temp_file.name)
+
+
+def test_06_jinja_templates(driver: webdriver.Remote):
+    with get_eel_server('examples/06 - jinja_templates/hello.py', 'templates/hello.html') as eel_url:
+        driver.get(eel_url)
+        assert driver.title == "Hello, World!"
+
+        driver.find_element_by_css_selector('a').click()
+        WebDriverWait(driver, 2.0).until(expected_conditions.presence_of_element_located((By.XPATH, '//h1[text()="This is page 2"]')))
diff --git a/tests/utils.py b/tests/utils.py
@@ -0,0 +1,62 @@
+import contextlib
+import os
+import subprocess
+import tempfile
+import time
+
+import psutil
+
+
+def get_process_listening_port(proc):
+    psutil_proc = psutil.Process(proc.pid)
+    while not any(conn.status == 'LISTEN' for conn in psutil_proc.connections()):
+        time.sleep(0.01)
+
+    conn = next(filter(lambda conn: conn.status == 'LISTEN', psutil_proc.connections()))
+    return conn.laddr.port
+
+
+@contextlib.contextmanager
+def get_eel_server(example_py, start_html):
+    """Run an Eel example with the mode/port overridden so that no browser is launched and a random port is assigned"""
+    test = None
+
+    try:
+        with tempfile.NamedTemporaryFile(mode='w', dir=os.path.dirname(example_py), delete=False) as test:
+            # We want to run the examples unmodified to keep the test as realistic as possible, but all of the examples
+            # want to launch browsers, which won't be supported in CI. The below script will configure eel to open on
+            # a random port and not open a browser, before importing the Python example file - which will then
+            # do the rest of the set up and start the eel server. This is definitely hacky, and means we can't
+            # test mode/port settings for examples ... but this is OK for now.
+            test.write(f"""
+import eel
+
+eel._start_args['mode'] = None
+eel._start_args['port'] = 0
+
+import {os.path.splitext(os.path.basename(example_py))[0]}
+""")
+
+        proc = subprocess.Popen(['python', test.name], cwd=os.path.dirname(example_py))
+        eel_port = get_process_listening_port(proc)
+
+        yield f"http://localhost:{eel_port}/{start_html}"
+
+        proc.terminate()
+
+    finally:
+        if test:
+            try:
+                os.unlink(test.name)
+            except FileNotFoundError:
+                pass
+
+
+def get_console_logs(driver, minimum_logs=0):
+    console_logs = driver.get_log('browser')
+
+    while len(console_logs) < minimum_logs:
+        console_logs += driver.get_log('browser')
+        time.sleep(0.1)
+
+    return console_logs
diff --git a/tox.ini b/tox.ini
@@ -1,6 +1,9 @@
 [tox]
 envlist = py36,py37,py38
 
+[pytest]
+timeout = 5
+
 [gh-actions]
 python =
     3.6: py36
@@ -9,4 +12,8 @@ python =
 
 [testenv]
 deps = -r requirements-test.txt
-commands = pytest {posargs}
+commands =
+  # this ugly hack is here because:
+  # https://github.com/tox-dev/tox/issues/149
+  pip install -q -r {toxinidir}/requirements-test.txt
+  {envpython} -m pytest {posargs}
