test(templates): test py-shiny templates (#1998)

Author: karangattu

.github/workflows/pytest.yaml

@@ -193,6 +193,17 @@ jobs:
         working-directory: examples/brownian/shinymediapipe
         run: |
           npm ci
+      - name: Checkout py-shiny-templates repository
+        uses: actions/checkout@v4
+        with:
+          repository: posit-dev/py-shiny-templates
+          path: py-shiny-templates
+
+      - name: Install py-shiny-templates dependencies
+        if: matrix.python-version != '3.9'
+        # Scikit-learn 1.7.0+ requires Python 3.10+
+        run: |
+          make ci-install-py-shiny-templates-deps
 
       - name: Run example app tests
         timeout-minutes: 60

Makefile

@@ -223,6 +223,8 @@ install-deps: FORCE ## install dependencies
 	pip install -e ".[dev,test]" --upgrade
 ci-install-deps: FORCE
 	uv pip install -e ".[dev,test]"
+ci-install-py-shiny-templates-deps: FORCE
+	uv pip install -r py-shiny-templates/requirements.txt
 
 install-docs: FORCE
 	pip install -e ".[dev,test,doc]"

pyproject.toml

@@ -113,11 +113,7 @@ dev = [
     "anthropic",
     "google-generativeai;python_version>='3.9'",
     "langchain_core",
-    # langsmith (needed for langchain_core) versions >= 0.3
-    # (up to at least 0.3.2 as of 2025-01-29)
-    # cause an `argparse.ArgumentError` when running `pytest`.
-    # https://github.com/posit-dev/py-shiny/issues/1829
-    "langsmith<0.3",
+    "langsmith>=0.3.4",
     "openai",
     "ollama",
     "chatlas>=0.6.1",

tests/playwright/examples/example_apps.py

@@ -4,7 +4,7 @@
 from pathlib import PurePath
 from typing import Literal
 
-from playwright.sync_api import ConsoleMessage, Page
+from playwright.sync_api import ConsoleMessage, Page, expect
 
 from shiny.run import ShinyAppProc, run_shiny_app
 
@@ -91,11 +91,20 @@ def get_apps(path: str) -> typing.List[str]:
     "FutureWarning: use_inf_as_na option is deprecated",
     "pd.option_context('mode.use_inf_as_na",  # continutation of line above,
     "RuntimeWarning: invalid value encountered in dot",  # some groups didn't have enough data points to create a meaningful line
+    "DatetimeIndex.format is deprecated and will be removed in a future version. Convert using index.astype(str) or index.map(formatter) instead.",  # cufflinks package is using this method
+    "Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`",  # cufflinks package is using this method
 ]
 app_allow_js_errors: typing.Dict[str, typing.List[str]] = {
     "examples/brownian": ["Failed to acquire camera feed:"],
 }
 
+# Check for Shiny output errors, except for known exception cases
+app_allow_output_error = [
+    "shiny/api-examples/SafeException/app-express.py",
+    "shiny/api-examples/SafeException/app-core.py",
+    "examples/global_pyplot/app.py",
+]
+
 
 # Altered from `shinytest2:::app_wait_for_idle()`
 # https://github.com/rstudio/shinytest2/blob/b8fdce681597e9610fc078aa6e376134c404f3bd/R/app-driver-wait.R
@@ -251,3 +260,8 @@ def on_console_msg(msg: ConsoleMessage) -> None:
             + " had JavaScript console errors!\n"
             + "* ".join(console_errors)
         )
+
+        if ex_app_path not in app_allow_output_error:
+            # Ensure there are no output errors present
+            error_locator = page.locator(".shiny-output-error")
+            expect(error_locator).to_have_count(0)

tests/playwright/examples/test_external_templates.py

@@ -0,0 +1,24 @@
+import sys
+from pathlib import Path
+
+import pytest
+from conftest import here_root
+from example_apps import get_apps, reruns, reruns_delay, validate_example
+from playwright.sync_api import Page
+
+if not Path(here_root / "py-shiny-templates").exists():
+    pytest.skip(
+        "./py-shiny-templates dir is not available. Skipping test.",
+        allow_module_level=True,
+    )
+
+
+@pytest.mark.skipif(
+    sys.version_info[:2] == (3, 9),
+    reason="Skipping test for Python 3.9 since scikit-learn pinned version is only supported on Python 3.10+",
+)
+@pytest.mark.only_browser("chromium")
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
+@pytest.mark.parametrize("ex_app_path", get_apps("py-shiny-templates"))
+def test_external_templates(page: Page, ex_app_path: str) -> None:
+    validate_example(page, ex_app_path)