PyTest test runner for Basilisp (#195)

* PyTest test runner for Basilisp [WIP]

* Functioning test runner! 🎉

* Fix linter silencing comment

* Fix six at 1.10.0 for coverage tox env

* Reset collected tests for each file

* Test the test runner

* Slightly update the tests; still broken

* Stop removing basilisp.core in random tests

* Return a map from test-runners

Author: chrisrink10

Pipfile

@@ -7,6 +7,7 @@ name = "pypi"
 twine = "*"
 tox = "*"
 tox-pyenv = "*"
+pytest-pycharm = "*"
 
 [packages]
 astor = "*"
@@ -16,6 +17,7 @@ pyfunctional = "*"
 pyrsistent = "*"
 python-dateutil = "*"
 basilisp = {editable = true, path = "."}
+pytest = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -105,6 +105,7 @@ def run(self):
 
     entry_points={
         'console_scripts': ['basilisp=basilisp.cli:cli'],
+        'pytest11': ['basilisp_test_runner=basilisp.testrunner']
     },
     install_requires=REQUIRED,
     extras_require=EXTRAS,

setup.py

@@ -5,6 +5,7 @@
 import types
 
 import click
+import pytest
 
 import basilisp.compiler as compiler
 import basilisp.lang.runtime as runtime
@@ -103,5 +104,12 @@ def run(file_or_code, code, in_ns):
         print(compiler.lrepr(eval_file(file_or_code, ctx, ns.module)))
 
 
+@cli.command(short_help='run tests in a Basilisp project')
+@click.argument('args', nargs=-1)
+def test(args):
+    """Run tests in a Basilisp project."""
+    pytest.main(args=list(args))
+
+
 if __name__ == "__main__":
     cli()

src/basilisp/cli.py

@@ -0,0 +1,76 @@
+(ns basilisp.test)
+
+(def ^:private collected-tests
+  (atom []))
+
+(def ^:private current-ns
+  (atom nil))
+
+(defn ^:private add-test!
+  "Add the test named by test-var to the test suite for ns."
+  [ns test-var]
+  (swap! collected-tests conj test-var))
+
+(defmacro is
+  "Assert that expr is true. Must appear inside of a deftest form."
+  ([expr]
+   `(is ~expr (str "Test failure: " ~expr)))
+  ([expr msg]
+   (let [line-no (:basilisp.reader/line (meta &form))
+
+         eq-sym `=
+
+         deconstruct? (and (list? expr) (= eq-sym (first expr)))
+
+         expected (if deconstruct?
+                    (second expr)
+                    expr)
+         actual   (if deconstruct?
+                    (nth expr 2)
+                    expr)]
+     `(let [computed# ~expr
+            actual#   ~actual
+            expected# ~expected]
+        (when-not computed#
+          ;; Collect test failures in the `failures` atom created
+          ;; by `deftest`.
+          (swap! ~'failures
+                 conj
+                 [~msg
+                  {:test-name    ~'test-name
+                   :test-section ~'test-section
+                   :expr         (quote ~expr)
+                   :line         ~line-no
+                   :actual       actual#
+                   :expected     expected#}]))))))
+
+(defmacro testing
+  "Wrapper for test cases to provide additional messaging and context
+  around the test or group of tests contained inside. Must appear inside
+  of a deftest form."
+  [msg & body]
+  `(let [~'test-section (if ~'test-section
+                          (str ~'test-section " :: " ~msg)
+                          ~msg)]
+     ~@body))
+
+(defmacro deftest
+  "Define a new test function. Assertions can be made with the is macro.
+  Group tests with the testing macro.
+
+  Tests defined by deftest will be run by default by the PyTest test
+  runner using Basilisp's builtin PyTest hook."
+  [name-sym & body]
+  (let [test-name-sym (with-meta name-sym {:test true})
+        test-name-str (name test-name-sym)]
+    `(do
+       (defn ~test-name-sym
+         []
+         (let [~'test-name    ~test-name-str
+               ~'test-section nil
+               ~'failures     (atom [])]
+           ~@body
+           {:failures (deref ~'failures)}))
+
+       (add-test! *ns* (var ~test-name-sym))
+       (reset! current-ns *ns*))))

src/basilisp/test.lpy

@@ -0,0 +1,148 @@
+import importlib
+from typing import Optional, Callable
+
+import pytest
+
+import basilisp.lang.exception as lexc
+import basilisp.lang.keyword as kw
+import basilisp.lang.map as lmap
+import basilisp.lang.runtime as runtime
+import basilisp.lang.symbol as sym
+import basilisp.lang.vector as vec
+import basilisp.main as basilisp
+from basilisp.lang.util import lrepr
+from basilisp.util import Maybe
+
+basilisp.init()
+importlib.import_module('basilisp.test')
+
+_COLLECTED_TESTS_SYM = sym.symbol('collected-tests', ns='basilisp.test')
+_CURRENT_NS_SYM = sym.symbol('current-ns', ns='basilisp.test')
+
+
+def pytest_collect_file(parent, path):
+    """Primary PyTest hook to identify Basilisp test files."""
+    if path.ext == ".lpy":
+        if path.basename.startswith("test_") or path.purebasename.endswith("_test"):
+            return BasilispFile(path, parent)
+    return None
+
+
+def _collected_tests() -> Optional[vec.Vector]:
+    """Fetch the collected tests for the namespace ns from
+    basilisp.test/collected-tests atom. If no tests are found, return
+    None."""
+    var = Maybe(runtime.Var.find(_COLLECTED_TESTS_SYM)).or_else_raise(
+        lambda: runtime.RuntimeException(f"Unable to find test Var {_COLLECTED_TESTS_SYM}."))
+    return var.value.deref()
+
+
+def _current_ns() -> str:
+    """Fetch the current namespace from basilisp.test/current-ns."""
+    var = Maybe(runtime.Var.find(_CURRENT_NS_SYM)).or_else_raise(
+        lambda: runtime.RuntimeException(f"Unable to find test Var {_CURRENT_NS_SYM}."))
+    ns = var.value.deref()
+    return ns.name
+
+
+def _reset_collected_tests() -> None:
+    """Reset the collected tests."""
+    var = Maybe(runtime.Var.find(_COLLECTED_TESTS_SYM)).or_else_raise(
+        lambda: runtime.RuntimeException(f"Unable to find test Var {_COLLECTED_TESTS_SYM}."))
+    return var.value.reset(vec.Vector.empty())
+
+
+TestFunction = Callable[[], Optional[vec.Vector]]
+
+
+class BasilispFile(pytest.File):
+    """Files represent a test module in Python or a test namespace in Basilisp."""
+    def collect(self):
+        """Collect all of the tests in the namespace (module) given.
+
+        Basilisp's test runner imports the namespace which will (as a side
+        effect) collect all of the test functions in a namespace (represented
+        by `deftest` forms in Basilisp) into an atom in `basilisp.test`.
+        BasilispFile.collect fetches those test functions and generates
+        BasilispTestItems for PyTest to run the tests."""
+        _reset_collected_tests()
+        filename = self.fspath.basename
+        self.fspath.pyimport()
+        ns = _current_ns()
+        tests = _collected_tests()
+        for test in tests:
+            f: TestFunction = test.value
+            yield BasilispTestItem(test.name.name, self, f, ns, filename)
+
+
+_ACTUAL_KW = kw.keyword('actual')
+_EXPECTED_KW = kw.keyword('expected')
+_FAILURES_KW = kw.keyword('failures')
+_LINE_KW = kw.keyword('line')
+_EXPR_KW = kw.keyword('expr')
+_TEST_SECTION_KW = kw.keyword('test-section')
+
+
+class BasilispTestItem(pytest.Item):
+    """Test items correspond to a single `deftest` form in a Basilisp test.
+
+    `deftest` forms run each `is` assertion and collect all failures in an
+    atom, reporting their results as a vector of failures when each test
+    concludes.
+
+    The BasilispTestItem collects all the failures and returns a report
+    to PyTest to show to the end-user."""
+
+    def __init__(self, name: str,  # pylint: disable=too-many-arguments
+                 parent: BasilispFile,
+                 run_test: TestFunction,
+                 namespace: str,
+                 filename: str) -> None:
+        super(BasilispTestItem, self).__init__(name, parent)
+        self._run_test = run_test
+        self._namespace = namespace
+        self._filename = filename
+
+    def runtest(self):
+        """Run the tests associated with this test item.
+
+        If any tests fail, raise an ExceptionInfo exception with the
+        test failures. PyTest will invoke self.repr_failure to display
+        the failures to the user."""
+        results: lmap.Map = self._run_test()
+        failures: Optional[vec.Vector] = results.entry(_FAILURES_KW)
+        if runtime.to_seq(failures):
+            raise lexc.ExceptionInfo("Test failures", lmap.map(results))
+
+    def repr_failure(self, excinfo):
+        """Representation function called when self.runtest() raises an
+        exception."""
+        if isinstance(excinfo.value, lexc.ExceptionInfo):
+            exc = excinfo.value
+            failures = exc.data.entry(_FAILURES_KW)
+            messages = []
+
+            for failure in failures:
+                msg: str = failure.entry(0)
+                details: lmap.Map = failure.entry(1)
+
+                actual = details.entry(_ACTUAL_KW)
+                expected = details.entry(_EXPECTED_KW)
+
+                test_section = details.entry(_TEST_SECTION_KW)
+                line = details.entry(_LINE_KW)
+                section_msg = Maybe(test_section).map(lambda s: f" {s} :: ").or_else_get("")
+
+                messages.append("\n".join([
+                    f"FAIL in ({self.name}) ({self._filename}:{line})",
+                    f"    {section_msg}{msg}",
+                    "",
+                    f"    expected: {lrepr(expected)}",
+                    f"      actual: {lrepr(actual)}"
+                ]))
+
+            return "\n\n".join(messages)
+        return None
+
+    def reportinfo(self):
+        return self.fspath, 0, self.name

src/basilisp/testrunner.py

@@ -0,0 +1 @@
+pytest_plugins = ["pytester"]

tests/__init__.py

@@ -0,0 +1,23 @@
+(ns basilisp.string-test
+  (:require
+   [basilisp.string :as str]
+   [basilisp.test :refer [deftest is]]))
+
+(deftest alpha?-test
+  (is (not (str/alpha? "")))
+  (is (not (str/alpha? "?")))
+  (is (not (str/alpha? "1")))
+  (is (str/alpha? "abcdef")))
+
+(deftest alphanumeric?-test
+  (is (not (str/alphanumeric? "")))
+  (is (not (str/alphanumeric? "?")))
+  (is (str/alphanumeric? "1"))
+  (is (str/alphanumeric? "abcdef")))
+
+(deftest digits?-test
+  (is (not (str/digits? "")))
+  (is (not (str/digits? "?")))
+  (is (not (str/digits? "abcdef")))
+  (is (str/digits? "1"))
+  (is (str/digits? "1375234723984")))