Skip to content

Commit bd835bb

Browse files
authored
Merge pull request #320 from rstudio/e2e-tests
End-to-end testing infrastructure
2 parents 6dd624a + 567285f commit bd835bb

File tree

16 files changed

+601
-7
lines changed

16 files changed

+601
-7
lines changed

.github/workflows/pytest.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ jobs:
7676
run: |
7777
make lint
7878
79+
- name: Run e2e tests
80+
if: steps.install.outcome == 'success' && (success() || failure())
81+
run: |
82+
make e2e
83+
7984
deploy:
8085
name: "Deploy to PyPI"
8186
runs-on: ubuntu-latest

Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: clean clean-test clean-pyc clean-build docs help
1+
.PHONY: help clean clean-test clean-pyc clean-build docs help lint test e2e test-all
22
.DEFAULT_GOAL := help
33

44
define BROWSER_PYSCRIPT
@@ -64,6 +64,9 @@ test: ## run tests quickly with the default Python
6464
python3 tests/asyncio_prevent.py
6565
pytest
6666

67+
e2e: ## run e2e tests with playwright
68+
tox
69+
6770
test-all: ## run tests on every Python version with tox
6871
tox
6972

e2e/README.md

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# End-to-end tests
2+
3+
This directory contains end-to-end (i.e. browser based) tests for Shiny for Python.
4+
5+
The Python files directly in this subdirectory are for Pytest fixtures and helper code
6+
to make test writing easier. (Eventually this logic may move to the `shiny` package
7+
itself or its own dedicated package, so that Shiny app authors can set up their own e2e
8+
tests against their apps.)
9+
10+
The actual tests are in subdirectories. Each subdirectory contains one or more Pytest
11+
files (`test_*.py`) containing [Playwright](https://playwright.dev/python/) assertions,
12+
and optionally, a single app (`app.py`) that the assertions test against. (The app is
13+
optional, because the tests may also be for apps in the `../examples` or `../shiny/examples` directory.)
14+
15+
## Running tests
16+
17+
The following commands can be run from the repo root:
18+
19+
```sh
20+
# Run all e2e tests
21+
make e2e
22+
# Another way to run all e2e tests
23+
tox
24+
25+
# Run just the tests in e2e/async/
26+
tox e2e/async
27+
28+
# Run just the tests in e2e/async/, in headed mode
29+
tox -- --headed e2e/async
30+
```
31+
32+
## Shiny app fixtures
33+
34+
Playwright for Python launches and controls (headless or headful) browsers, but does not
35+
know how to start/stop Shiny app processes. Instead, we have our own [Pytest
36+
fixtures](https://docs.pytest.org/en/latest/explanation/fixtures.html) for
37+
Shiny app process management.
38+
39+
### Testing a local `app.py`
40+
41+
The `local_app: ShinyAppProc` fixture launches the `app.py` in the current directory
42+
as the calling `test_*.py` file.
43+
44+
```python
45+
46+
import re
47+
from playwright.sync_api import Page, expect
48+
from conftest import ShinyAppProc
49+
50+
51+
def test_airmass(page: Page, local_app: ShinyAppProc):
52+
page.goto(local_app.url)
53+
plot = page.locator("#plot")
54+
expect(plot).to_have_class(re.compile(r"\bshiny-bound-output\b"))
55+
```
56+
57+
### Testing an app from `../examples`
58+
59+
It's also possible to test apps that live in the `../examples` directory: at the test
60+
module level, create a fixture with the help of `conftest.create_example_fixture`, and
61+
use it from test funcs.
62+
63+
```python
64+
import re
65+
66+
from playwright.sync_api import Page, expect
67+
68+
from conftest import ShinyAppProc, create_example_fixture
69+
70+
airmass_app = create_example_fixture("airmass")
71+
72+
73+
def test_airmass(page: Page, airmass_app: ShinyAppProc):
74+
page.goto(airmass_app.url)
75+
plot = page.locator("#plot")
76+
expect(plot).to_have_class(re.compile(r"\bshiny-bound-output\b"))
77+
```

e2e/async/app.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import asyncio
2+
import hashlib
3+
import time
4+
5+
import shiny as s
6+
from shiny import ui, reactive
7+
8+
9+
def calc(value: str) -> str:
10+
# Simluate this taking a long time
11+
time.sleep(1)
12+
m = hashlib.sha256()
13+
m.update(value.encode("utf-8"))
14+
return m.digest().hex()
15+
16+
17+
app_ui = ui.page_fluid(
18+
ui.input_text_area(
19+
"value", "Value to sha256sum", value="The quick brown fox", rows=5, width="100%"
20+
),
21+
ui.p(ui.input_action_button("go", "Calculate"), class_="mb-3"),
22+
ui.output_text_verbatim("hash_output"),
23+
)
24+
25+
26+
def server(input: s.Inputs, output: s.Outputs, session: s.Session):
27+
@output()
28+
@s.render.text()
29+
@reactive.event(input.go)
30+
async def hash_output():
31+
content = await hash_result()
32+
return content
33+
34+
@reactive.Calc()
35+
async def hash_result() -> str:
36+
with ui.Progress() as p:
37+
p.set(message="Calculating...")
38+
39+
value = input.value()
40+
return await asyncio.get_running_loop().run_in_executor(None, calc, value)
41+
42+
43+
app = s.App(app_ui, server)

e2e/async/test_async.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# See https://github.com/microsoft/playwright-python/issues/1532
2+
# pyright: reportUnknownMemberType=false
3+
4+
from playwright.sync_api import Page, expect
5+
from conftest import ShinyAppProc
6+
7+
8+
def test_async_app(page: Page, local_app: ShinyAppProc) -> None:
9+
page.goto(local_app.url)
10+
11+
textarea = page.locator("textarea#value")
12+
textarea.fill("Hello\nGoodbye")
13+
14+
btn = page.locator("#go")
15+
btn.click()
16+
17+
progress = page.locator("#shiny-notification-panel")
18+
expect(progress).to_be_visible()
19+
expect(progress).to_contain_text("Calculating...")
20+
21+
output = page.locator("#hash_output")
22+
expect(output).to_have_text(
23+
"2e220fb9d401bf832115305b9ae0277e7b8b1a9368c6526e450acd255e0ec0c2", timeout=2000
24+
)
25+
26+
expect(progress).to_be_hidden()

e2e/conftest.py

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
from __future__ import annotations
2+
3+
import datetime
4+
import logging
5+
import random
6+
import socket
7+
import subprocess
8+
import sys
9+
import threading
10+
from pathlib import PurePath
11+
from time import sleep
12+
from types import TracebackType
13+
from typing import IO, Callable, Generator, List, Optional, TextIO, Type, Union
14+
15+
import pytest
16+
17+
__all__ = (
18+
"ShinyAppProc",
19+
"create_app_fixture",
20+
"create_doc_example_fixture",
21+
"create_example_fixture",
22+
"local_app",
23+
"run_shiny_app",
24+
)
25+
26+
here = PurePath(__file__).parent
27+
28+
29+
def random_port():
30+
while True:
31+
port = random.randint(1024, 49151)
32+
with socket.socket() as s:
33+
try:
34+
s.bind(("127.0.0.1", port))
35+
return port
36+
except Exception:
37+
# Let's just assume that port was in use; try again
38+
continue
39+
40+
41+
class OutputStream:
42+
"""Designed to wrap an IO[str] and accumulate the output using a bg thread
43+
44+
Also allows for blocking waits for particular lines."""
45+
46+
def __init__(self, io: IO[str], desc: Optional[str] = None):
47+
self._io = io
48+
self._closed = False
49+
self._lines: List[str] = []
50+
self._cond = threading.Condition()
51+
self._thread = threading.Thread(
52+
group=None, target=self._run, daemon=True, name=desc
53+
)
54+
55+
self._thread.start()
56+
57+
def _run(self):
58+
"""Pump lines into self._lines in a tight loop."""
59+
60+
try:
61+
while not self._io.closed:
62+
try:
63+
line = self._io.readline()
64+
except ValueError:
65+
# This is raised when the stream is closed
66+
break
67+
if line is None:
68+
break
69+
if line != "":
70+
with self._cond:
71+
self._lines.append(line)
72+
self._cond.notify_all()
73+
finally:
74+
# If we got here, we're finished reading self._io and need to signal any
75+
# waiters that we're done and they'll never hear from us again.
76+
with self._cond:
77+
self._closed = True
78+
self._cond.notify_all()
79+
80+
def wait_for(self, predicate: Callable[[str], bool], timeoutSecs: float) -> bool:
81+
timeoutAt = datetime.datetime.now() + datetime.timedelta(seconds=timeoutSecs)
82+
pos = 0
83+
with self._cond:
84+
while True:
85+
while pos < len(self._lines):
86+
if predicate(self._lines[pos]):
87+
return True
88+
pos += 1
89+
if self._closed:
90+
return False
91+
else:
92+
remaining = (timeoutAt - datetime.datetime.now()).total_seconds()
93+
if remaining < 0 or not self._cond.wait(timeout=remaining):
94+
# Timed out
95+
raise TimeoutError(
96+
"Timeout while waiting for Shiny app to become ready"
97+
)
98+
99+
def __str__(self):
100+
with self._cond:
101+
return "".join(self._lines)
102+
103+
104+
def dummyio() -> TextIO:
105+
io = TextIO()
106+
io.close()
107+
return io
108+
109+
110+
class ShinyAppProc:
111+
def __init__(self, proc: subprocess.Popen[str], port: int):
112+
self.proc = proc
113+
self.port = port
114+
self.url = f"http://127.0.0.1:{port}/"
115+
self.stdout = OutputStream(proc.stdout or dummyio())
116+
self.stderr = OutputStream(proc.stderr or dummyio())
117+
threading.Thread(group=None, target=self._run, daemon=True).start()
118+
119+
def _run(self) -> None:
120+
self.proc.wait()
121+
if self.proc.stdout is not None:
122+
self.proc.stdout.close()
123+
if self.proc.stderr is not None:
124+
self.proc.stderr.close()
125+
126+
def close(self) -> None:
127+
sleep(0.5)
128+
self.proc.terminate()
129+
130+
def __enter__(self) -> ShinyAppProc:
131+
return self
132+
133+
def __exit__(
134+
self,
135+
exc_type: Optional[Type[BaseException]],
136+
exc_value: Optional[BaseException],
137+
traceback: Optional[TracebackType],
138+
):
139+
self.close()
140+
141+
def wait_until_ready(self, timeoutSecs: float) -> None:
142+
if self.stderr.wait_for(
143+
lambda line: "Uvicorn running on" in line, timeoutSecs=timeoutSecs
144+
):
145+
return
146+
else:
147+
raise RuntimeError("Shiny app exited without ever becoming ready")
148+
149+
150+
def run_shiny_app(
151+
app_file: Union[str, PurePath],
152+
*,
153+
port: int = 0,
154+
cwd: Optional[str] = None,
155+
wait_for_start: bool = True,
156+
timeout_secs: float = 10,
157+
bufsize: int = 64 * 1024,
158+
) -> ShinyAppProc:
159+
if port == 0:
160+
port = random_port()
161+
162+
child = subprocess.Popen(
163+
[sys.executable, "-m", "shiny", "run", "--port", str(port), str(app_file)],
164+
bufsize=bufsize,
165+
executable=sys.executable,
166+
stdout=subprocess.PIPE,
167+
stderr=subprocess.PIPE,
168+
cwd=cwd,
169+
encoding="utf-8",
170+
)
171+
172+
# TODO: Detect early exit
173+
174+
sa = ShinyAppProc(child, port)
175+
if wait_for_start:
176+
sa.wait_until_ready(timeout_secs)
177+
return sa
178+
179+
180+
def create_app_fixture(app: Union[PurePath, str], scope: str = "module"):
181+
def fixture_func():
182+
sa = run_shiny_app(app, wait_for_start=False)
183+
try:
184+
with sa:
185+
sa.wait_until_ready(30)
186+
yield sa
187+
finally:
188+
logging.warning("Application output:\n" + str(sa.stderr))
189+
190+
return pytest.fixture(
191+
scope=scope, # type: ignore
192+
)(fixture_func)
193+
194+
195+
def create_example_fixture(example_name: str, scope: str = "module"):
196+
"""Used to create app fixtures from apps in py-shiny/examples"""
197+
return create_app_fixture(here / "../examples" / example_name / "app.py", scope)
198+
199+
200+
def create_doc_example_fixture(example_name: str, scope: str = "module"):
201+
"""Used to create app fixtures from apps in py-shiny/shiny/examples"""
202+
return create_app_fixture(
203+
here / "../shiny/examples" / example_name / "app.py", scope
204+
)
205+
206+
207+
@pytest.fixture(scope="module")
208+
def local_app(request: pytest.FixtureRequest) -> Generator[ShinyAppProc, None, None]:
209+
sa = run_shiny_app(PurePath(request.path).parent / "app.py", wait_for_start=False)
210+
try:
211+
with sa:
212+
sa.wait_until_ready(30)
213+
yield sa
214+
finally:
215+
logging.warning("Application output:\n" + str(sa.stderr))

0 commit comments

Comments
 (0)