add (opt-in) pabot, clean up ff

Author: bollwyvl

.gitignore

@@ -110,6 +110,7 @@ node_modules
 *.tgz
 
 atest/output/
+.pabotsuitenames
 junit.xml
 coverage/
 .vscode/

atest/Keywords.resource

@@ -112,24 +112,16 @@ Wait For Splash
 
 Open JupyterLab
     Set Environment Variable    MOZ_HEADLESS    ${HEADLESS}
-    ${firefox} =    Get Firefox Binary
     ${geckodriver} =    Which    geckodriver
     Should Not Be Equal As Strings    ${geckodriver}    None
     ...    geckodriver not found, do you need to install firefox-geckodriver?
     ${service args} =    Create List    --log    debug
     Create WebDriver    Firefox
     ...    executable_path=${geckodriver}
-    ...    firefox_binary=${firefox}
     ...    service_log_path=${GECKODRIVER LOG}
     ...    service_args=${service args}
     Wait Until Keyword Succeeds    3x    5s    Wait For Splash
 
-Get Firefox Binary
-    [Documentation]    Get Firefox path from the environment... or hope for the best
-    ${from which} =    Which    firefox
-    ${firefox} =    Set Variable If    "%{FIREFOX_BINARY}"    %{FIREFOX_BINARY}    ${from which}
-    [Return]    ${firefox}
-
 Close JupyterLab
     Close All Browsers
 

scripts/atest.py

@@ -1,39 +1,45 @@
 """ Run acceptance tests with robot framework
 """
 # pylint: disable=broad-except
+import json
+import multiprocessing
 import os
 import platform
 import shutil
 import sys
 import time
-from os.path import join
 from pathlib import Path
 
 import robot
+from pabot import pabot
+
+OS = platform.system()
+PY = "".join(map(str, sys.version_info[:2]))
 
 ROOT = Path(__file__).parent.parent.resolve()
 ATEST = ROOT / "atest"
 OUT = ATEST / "output"
 
-OS = platform.system()
-PY = "".join(map(str, sys.version_info[:2]))
+ATEST_RETRIES = json.loads(os.environ.get("ATEST_RETRIES", "0"))
+ATEST_PROCESSES = json.loads(os.environ.get("ATEST_PROCESSES", "1"))
+
+if not ATEST_PROCESSES:
+    # each test incurs a jupyter server, a browser, and a bunch of language
+    # servers and kernels, so be a bit more conservative than the default
+    # $CPU_COUNT + 1
+    ATEST_PROCESSES = max(int(multiprocessing.cpu_count() / 2), 1) + 1
 
 OS_PY_ARGS = {
     # notebook and ipykernel releases do not yet support python 3.8 on windows
     # ("Windows", "38"): ["--include", "not-supported", "--runemptysuite"]
-    # TODO: restore when we figure out win36 vs jedi on windows
-    ("Windows", "36"): ["--exclude", "feature:completion", "--runemptysuite"]
 }
 
 NON_CRITICAL = [
     # TODO: restore when yaml-language-server supports both config and...
     # everything else: https://github.com/jupyter-lsp/jupyterlab-lsp/pull/245
     ["language:yaml", "feature:config"],
-    # TODO: restore when we figure out win36 vs jedi on windows
-    ["language:python", "py:36", "os:windows"],
 ]
 
-
 # because we use diagnostics as a litmus for "working", revert to behavior
 # from before https://github.com/bash-lsp/bash-language-server/pull/269
 os.environ["HIGHLIGHT_PARSING_ERRORS"] = "true"
@@ -51,24 +57,6 @@ def get_stem(attempt, extra_args):
 def atest(attempt, extra_args):
     """perform a single attempt of the acceptance tests"""
 
-    # TODO: investigate whether this is still required vs geckodriver 0.28
-    if "FIREFOX_BINARY" not in os.environ:
-        os.environ["FIREFOX_BINARY"] = shutil.which("firefox")
-
-        prefix = os.environ.get("CONDA_PREFIX")
-
-        if prefix:
-            app_dir = join(prefix, "bin", "FirefoxApp")
-            os.environ["FIREFOX_BINARY"] = {
-                "Windows": join(prefix, "Library", "bin", "firefox.exe"),
-                "Linux": join(app_dir, "firefox"),
-                "Darwin": join(app_dir, "Contents", "MacOS", "firefox"),
-            }[OS]
-
-    print("Will use firefox at", os.environ["FIREFOX_BINARY"])
-
-    assert os.path.exists(os.environ["FIREFOX_BINARY"])
-
     extra_args += OS_PY_ARGS.get((OS, PY), [])
 
     stem = get_stem(attempt, extra_args)
@@ -88,41 +76,49 @@ def atest(attempt, extra_args):
         f"{OS}{PY}",
         "--outputdir",
         out_dir,
-        "--output",
-        OUT / f"{stem}.robot.xml",
-        "--log",
-        OUT / f"{stem}.log.html",
-        "--report",
-        OUT / f"{stem}.report.html",
-        "--xunit",
-        OUT / f"{stem}.xunit.xml",
         "--variable",
         f"OS:{OS}",
         "--variable",
         f"PY:{PY}",
         # don't ever test our examples
         "--exclude",
         "atest:example",
-        # random ensures there's not inter-test coupling
-        "--randomize",
-        "all",
         *(extra_args or []),
         ATEST,
     ]
 
     print("Robot Arguments\n", " ".join(["robot"] + list(map(str, args))))
 
-    os.chdir(ATEST)
-
     if out_dir.exists():
         print("trying to clean out {}".format(out_dir))
         try:
             shutil.rmtree(out_dir)
         except Exception as err:
             print("Error deleting {}, hopefully harmless: {}".format(out_dir, err))
 
+    os.chdir(ATEST)
+
+    str_args = list(map(str, args))
+
     try:
-        robot.run_cli(list(map(str, args)))
+        if "--dryrun" in extra_args or ATEST_PROCESSES == 1:
+            robot.run_cli(
+                [
+                    # random ensures there's not inter-test coupling
+                    "--randomize",
+                    "all",
+                    *str_args,
+                ]
+            )
+        else:
+            pabot.main(
+                [
+                    *("--processes", f"{ATEST_PROCESSES}"),
+                    *("--artifacts", "png,log"),
+                    "--artifactsinsubfolders",
+                    *str_args,
+                ]
+            )
         return 0
     except SystemExit as err:
         return err.code
@@ -133,7 +129,7 @@ def attempt_atest_with_retries(*extra_args):
     attempt = 0
     error_count = -1
 
-    retries = int(os.environ.get("ATEST_RETRIES") or "0")
+    retries = ATEST_RETRIES
 
     while error_count != 0 and attempt <= retries:
         attempt += 1

scripts/lint.py

@@ -65,12 +65,21 @@
 def lint():
     """get that linty fresh feeling"""
 
+    def call_with_print(args):
+        str_args = [a for a in args if isinstance(a, str)]
+        paths = [a for a in args if isinstance(a, Path)]
+        print(f"{len(paths)} paths:", " ".join(str_args))
+        return_code = call(list(map(str, args)))
+        if return_code:
+            print("\n...", f"ERROR {return_code}", *str_args, "\n")
+        return return_code
+
     return max(
         map(
-            call,
+            call_with_print,
             [
                 ["isort", *ALL_PY],
-                ["black", *ALL_PY],
+                ["black", "--quiet", *ALL_PY],
                 ["flake8", *ALL_PY],
                 *[
                     # see https://github.com/python/mypy/issues/4008