Remove implicit frontend testing from testing.run_app_in_cloud (#16741)

Co-authored-by: Ethan Harris <[email protected]>

Author: 2 people

docs/source-app/examples/file_server/app.py

@@ -230,7 +230,7 @@ def test_file_server():
 def test_file_server_in_cloud():
     # You need to provide the directory containing the app file.
     app_dir = "docs/source-app/examples/file_server"
-    with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn):
+    with run_app_in_cloud(app_dir) as (admin_page, view_page, get_logs_fn, name):
         """# 1. `admin_page` and `view_page` are playwright Page Objects.
 
         # Check out https://playwright.dev/python/ doc to learn more.

src/lightning_app/CHANGELOG.md

@@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [1.9.4] - 2023-02-28
 
-No changes.
+### Removed
+
+- Removed implicit ui testing with `testing.run_app_in_cloud` in favor of headless login and app selection ([#16741](https://github.com/Lightning-AI/lightning/pull/16741))
 
 
 ## [1.9.3] - 2023-02-21

src/lightning_app/runners/cloud.py

@@ -367,6 +367,10 @@ def dispatch(
                 click.launch(
                     self._get_app_url(project, run_instance, "logs" if run.is_headless else "web-ui", needs_credits)
                 )
+
+            if bool(int(os.getenv("LIGHTING_TESTING", "0"))):
+                print(f"APP_LOGS_URL: {self._get_app_url(project, run_instance, 'logs')}")
+
         except ApiException as e:
             logger.error(e.body)
             sys.exit(1)

src/lightning_app/testing/testing.py

@@ -36,7 +36,7 @@
 
 from lightning_app import LightningApp, LightningFlow
 from lightning_app.cli.lightning_cli import run_app
-from lightning_app.core.constants import LIGHTNING_CLOUD_PROJECT_ID
+from lightning_app.core import constants
 from lightning_app.runners.multiprocess import MultiProcessRuntime
 from lightning_app.testing.config import _Config
 from lightning_app.utilities.app_logs import _app_logs_reader
@@ -50,7 +50,6 @@
 from lightning_app.utilities.proxies import ProxyWorkRun
 
 if _is_playwright_available():
-    import playwright
     from playwright.sync_api import HttpCredentials, sync_playwright
 
 
@@ -248,8 +247,8 @@ def run_app_in_cloud(
 ) -> Generator:
     """This utility is used to automate testing e2e application with lightning.ai."""
     # 1. Validate the provide app_folder is correct.
-    if not os.path.exists(os.path.join(app_folder, "app.py")):
-        raise Exception("The app folder should contain an app.py file.")
+    if not os.path.exists(os.path.join(app_folder, app_name)):
+        raise Exception(f"The app folder should contain an {app_name} file.")
     if app_folder.endswith("/"):
         app_folder = app_folder[:-1]
 
@@ -315,6 +314,14 @@ def run_app_in_cloud(
             process = Popen((cmd + extra_args), cwd=tmpdir, env=env_copy, stdout=stdout, stderr=sys.stderr)
             process.wait()
 
+        # Fallback URL to prevent failures in case we don't get the admin URL
+        admin_url = _Config.url
+        with open(stdout_path) as fo:
+            for line in fo.readlines():
+                if line.startswith("APP_LOGS_URL: "):
+                    admin_url = line.replace("APP_LOGS_URL: ", "")
+                    break
+
         if is_editable_mode:
             # Added to ensure the current code is properly uploaded.
             # Otherwise, it could result in un-tested PRs.
@@ -341,11 +348,20 @@ def run_app_in_cloud(
             record_video_dir=os.path.join(_Config.video_location, TEST_APP_NAME),
             record_har_path=_Config.har_location,
         )
+
+        client = LightningClient()
+        project_id = _get_project(client).project_id
+
+        app = _fetch_app_by_name(client, project_id, name)
+        app_id = app.id
+        print(f"The Lightning App ID is: {app.id}")  # useful for Grafana
+
+        if debug:
+            process = Process(target=_print_logs, kwargs={"app_id": app_id})
+            process.start()
+
         admin_page = context.new_page()
-        print(f"The Lightning App Token is: {token}")
-        print(f"The Lightning App user key is: {_Config.key}")
-        print(f"The Lightning App user id is: {_Config.id}")
-        admin_page.goto(_Config.url)
+        admin_page.goto(admin_url)
         admin_page.evaluate(
             """data => {
             window.localStorage.setItem('gridUserId', data[0]);
@@ -355,59 +371,14 @@ def run_app_in_cloud(
         """,
             [_Config.id, _Config.key, token],
         )
-        if LIGHTNING_CLOUD_PROJECT_ID:
+        if constants.LIGHTNING_CLOUD_PROJECT_ID:
             admin_page.evaluate(
                 """data => {
                 window.localStorage.setItem('gridDefaultProjectIdOverride', JSON.stringify(data[0]));
             }
             """,
-                [LIGHTNING_CLOUD_PROJECT_ID],
+                [constants.LIGHTNING_CLOUD_PROJECT_ID],
             )
-        admin_page.goto(f"{_Config.url}/{_Config.username}/apps", timeout=60 * 1000)
-
-        # Closing the Complete your profile dialog
-        try:
-            dialog = admin_page.locator("text=Complete your profile")
-            dialog.wait_for(timeout=10 * 1000, state="visible")
-            print("'Complete your profile' dialog visible, closing it.")
-            admin_page.locator('input[name="firstName"]').fill("first")
-            admin_page.locator('input[name="lastName"]').fill("last")
-            admin_page.locator('input[name="email"]').fill("[email protected]")
-            admin_page.locator('input[name="organization"]').fill("Lightning AI")
-            button = admin_page.locator('button:has-text("Confirm")')
-            button.wait_for(timeout=3 * 1000)
-            button.click()
-        except playwright._impl._api_types.TimeoutError:
-            print("'Complete your profile' dialog not visible, skipping.")
-
-        # Closing the Create Project dialog.
-        try:
-            project_dialog = admin_page.locator("text=Create a project")
-            project_dialog.wait_for(timeout=10 * 1000, state="visible")
-            print("'Create Project' dialog visible, closing it.")
-            project_name_input = admin_page.locator('input[type="text"]')
-            project_name_input.fill("Default Project")
-            button = admin_page.locator('button:has-text("Continue")')
-            button.wait_for(timeout=3 * 1000)
-            button.click()
-        except playwright._impl._api_types.TimeoutError:
-            print("'Create Project' dialog not visible, skipping.")
-
-        admin_page.locator(f'[data-cy="{name}"]').click()
-
-        app_url = admin_page.url
-        admin_page.goto(app_url + "/logs")
-
-        client = LightningClient()
-        project_id = _get_project(client).project_id
-
-        app = _fetch_app_by_name(client, project_id, name)
-        app_id = app.id
-        print(f"The Lightning App ID is: {app.id}")  # useful for Grafana
-
-        if debug:
-            process = Process(target=_print_logs, kwargs={"app_id": app_id})
-            process.start()
 
         view_page = context.new_page()
         i = 1

tests/integrations_app/public/test_commands_and_api.py

@@ -12,7 +12,7 @@
 @pytest.mark.cloud
 def test_commands_and_api_example_cloud() -> None:
     with run_app_in_cloud(os.path.join(_PATH_EXAMPLES, "app_commands_and_api")) as (
-        admin_page,
+        _,
         view_page,
         fetch_logs,
         app_name,

tests/integrations_app/public/test_v0_app.py

@@ -67,11 +67,7 @@ def test_v0_app_example_byoc_cloud() -> None:
     with run_app_in_cloud(
         os.path.join(_PATH_EXAMPLES, "app_v0"),
         extra_args=["--cluster-id", os.environ.get("LIGHTNING_BYOC_CLUSTER_ID")],
-    ) as (
-        _,
-        view_page,
-        fetch_logs,
-    ):
+    ) as (_, view_page, fetch_logs, app_name):
         run_v0_app(fetch_logs, view_page)