Merge pull request #150 from bollwyvl/test-retries-and-38

More Robot Test Robustness

Author: krassowski

README.md

@@ -63,7 +63,7 @@ Advanced static-analysis autocompletion without a running kernel
 Either:
 
 - JupyterLab >=1.1.4,<1.2
-- JupyterLab >=1.2.3,<1.3.0a0
+- JupyterLab >=1.2.4,<1.3.0a0
 - Python 3.5+
 - nodejs 8+
 

atest/00_Smoke.robot

@@ -5,10 +5,6 @@ Resource          Keywords.robot
 *** Test Cases ***
 Lab Version
     Capture Page Screenshot    00-smoke.png
-    ${script} =    Get Element Attribute    id:jupyter-config-data    innerHTML
-    ${config} =    Evaluate    __import__("json").loads("""${script}""")
-    Set Global Variable    ${PAGE CONFIG}    ${config}
-    Set Global Variable    ${LAB VERSION}    ${config["appVersion"]}
 
 Root URI
     [Documentation]    the rootUri should be set in the page config

atest/Keywords.robot

@@ -28,6 +28,10 @@ Setup Server and Browser
     ...    stderr=STDOUT
     Set Global Variable    ${SERVER}    ${server}
     Open JupyterLab
+    ${script} =    Get Element Attribute    id:jupyter-config-data    innerHTML
+    ${config} =    Evaluate    __import__("json").loads("""${script}""")
+    Set Global Variable    ${PAGE CONFIG}    ${config}
+    Set Global Variable    ${LAB VERSION}    ${config["appVersion"]}
 
 Setup Suite For Screenshots
     [Arguments]    ${folder}

ci/job.combine.yml

@@ -6,6 +6,7 @@ parameters:
   pythons:
     - ThreeSix
     - ThreeSeven
+    - ThreeEight
   install_robot: conda install -yc conda-forge robotframework
 
 jobs:

ci/job.test.yml

@@ -15,7 +15,10 @@ parameters:
       lab: '>=1.1,<1.2.0a0'
     - name: ThreeSeven
       spec: '>=3.7,<3.8.0a0'
-      lab: '>=1.2.3,<1.3.0a0'
+      lab: '>=1.2.4,<1.3.0a0'
+    - name: ThreeEight
+      spec: '>=3.8,<3.9.0a0'
+      lab: '>=1.2.4,<1.3.0a0'
   env_update: conda env update -n jupyterlab-lsp --file env-test.yml --quiet
   lab_ext: jupyter labextension install --no-build $(THIRD_PARTY_LABEXTENSIONS) $(FIRST_PARTY_LABEXTENSIONS)
 
@@ -39,13 +42,6 @@ jobs:
               - script: conda info && conda list -n jupyterlab-lsp
                 displayName: list conda packages and info
 
-              # TODO: determine how can bring this back to get more robust installs (see #115)
-              # - task: CacheBeta@0
-              #   inputs:
-              #     key: yarn | $(Agent.OS) | yarn.lock
-              #     path: $(YARN_CACHE_FOLDER)
-              #   displayName: restore cached yarn packages
-
               - script: ${{ platform.activate }} jupyterlab-lsp && jlpm || jlpm || jlpm
                 displayName: install npm dependencies
 

environment.yml

@@ -7,7 +7,7 @@ channels:
 dependencies:
   # runtime dependencies
   - python >=3.7,<3.8.0a0
-  - jupyterlab >=1.2.3,<1.3.0a0
+  - jupyterlab >=1.2.4,<1.3.0a0
   - notebook >=4.3.1
   # build dependencies
   - nodejs

requirements-lab.txt

@@ -1,3 +1,3 @@
 # the version of jupyterlab
 -r requirements.txt
-jupyterlab >=1.2.3,<1.3.0a0
+jupyterlab >=1.2.4,<1.3.0a0

scripts/atest.py

@@ -5,6 +5,7 @@
 import platform
 import shutil
 import sys
+import time
 from pathlib import Path
 
 import robot
@@ -16,11 +17,33 @@
 OS = platform.system()
 PY = "".join(map(str, sys.version_info[:2]))
 
+OS_PY_ARGS = {
+    # notebook and ipykernel releases do not yet support python 3.8 on windows
+    ("Windows", "38"): ["--include", "not-supported", "--runemptysuite"]
+}
+
+
+def get_stem(attempt, extra_args):
+    stem = "_".join([OS, PY, str(attempt)]).replace(".", "_").lower()
+
+    if "--dryrun" in extra_args:
+        stem = f"dry_run_{stem}"
+
+    return stem
+
 
 def atest(attempt, extra_args):
     """ perform a single attempt of the acceptance tests
     """
-    stem = "_".join([OS, PY, str(attempt)]).replace(".", "_").lower()
+    extra_args += OS_PY_ARGS.get((OS, PY), [])
+
+    stem = get_stem(attempt, extra_args)
+
+    if attempt != 1:
+        previous = OUT / f"{get_stem(attempt - 1, extra_args)}.robot.xml"
+        if previous.exists():
+            extra_args += ["--rerunfailed", str(previous)]
+
     out_dir = OUT / stem
 
     args = [
@@ -40,6 +63,8 @@ def atest(attempt, extra_args):
         f"OS:{OS}",
         "--variable",
         f"PY:{PY}",
+        "--randomize",
+        "all",
         *(extra_args or []),
         ATEST,
     ]
@@ -71,7 +96,9 @@ def attempt_atest_with_retries(*extra_args):
     while error_count != 0 and attempt <= retries:
         attempt += 1
         print("attempt {} of {}...".format(attempt, retries + 1))
-        error_count = atest(attempt=attempt, extra_args=extra_args)
+        start_time = time.time()
+        error_count = atest(attempt=attempt, extra_args=list(extra_args))
+        print(error_count, "errors in", int(time.time() - start_time), "seconds")
 
     return error_count
 

scripts/combine.py

@@ -24,7 +24,10 @@ def combine_robot_reports():
         *OUT.glob("*.robot.xml"),
     ]
 
-    return rebot_cli(args, exit=False)
+    return_code = rebot_cli(args, exit=False)
+    print("rebot returned", return_code)
+
+    return 0
 
 
 if __name__ == "__main__":

scripts/utest.py

@@ -1,9 +1,20 @@
 """ run python unit tests with pytest
 """
+import platform
 import sys
 
 import pytest
 
+OS = platform.system()
+PY = "".join(map(str, sys.version_info[:2]))
+
+OS_PY_ARGS = {
+    # notebook and ipykernel releases do not yet support python 3.8 on windows
+    ("Windows", "38"): ["-k", "not serverextension"]
+}
+
+DEFAULT_ARGS = ["--cov-fail-under=100"]
+
 
 def run_tests():
     """ actually run the tests
@@ -18,8 +29,8 @@ def run_tests():
         "-p",
         "no:warnings",
         "--flake8",
-        "--cov-fail-under=100",
         "-vv",
+        *OS_PY_ARGS.get((OS, PY), DEFAULT_ARGS),
     ] + list(sys.argv[1:])
 
     return pytest.main(args)