Improve compiler exception and syntax error formatting at the REPL (#869)

Fixes #870

---------

Co-authored-by: Christopher Rink <[email protected]>

Author: chrisrink10

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  * Added support for `*flush-on-newline*` to flush the `prn` and `println` output stream after the last newline (#865)
  * Added support for binding destructuring in `for` bindings (#774)
  * Added `==` as an alias to `=` (#859)
+ * Added custom exception formatting for `basilisp.lang.compiler.exception.CompilerException` and `basilisp.lang.reader.SyntaxError` to show more useful details to users on errors (#870)
 
 ### Changed
  * Cause exceptions arising from compilation issues during macroexpansion will no longer be nested for each level of macroexpansion (#852)

src/basilisp/cli.py

@@ -4,7 +4,6 @@
 import os
 import sys
 import textwrap
-import traceback
 import types
 from pathlib import Path
 from typing import Any, Callable, Optional, Sequence, Type
@@ -15,6 +14,7 @@
 from basilisp.lang import runtime as runtime
 from basilisp.lang import symbol as sym
 from basilisp.lang import vector as vec
+from basilisp.lang.exception import print_exception
 from basilisp.prompt import get_prompter
 
 CLI_INPUT_FILE_PATH = "<CLI Input>"
@@ -449,17 +449,15 @@ def repl(
                 prompter.print(runtime.lrepr(result))
                 repl_module.mark_repl_result(result)
             except reader.SyntaxError as e:
-                traceback.print_exception(reader.SyntaxError, e, e.__traceback__)
+                print_exception(e, reader.SyntaxError, e.__traceback__)
                 repl_module.mark_exception(e)
                 continue
             except compiler.CompilerException as e:
-                traceback.print_exception(
-                    compiler.CompilerException, e, e.__traceback__
-                )
+                print_exception(e, compiler.CompilerException, e.__traceback__)
                 repl_module.mark_exception(e)
                 continue
             except Exception as e:  # pylint: disable=broad-exception-caught
-                traceback.print_exception(Exception, e, e.__traceback__)
+                print_exception(e, Exception, e.__traceback__)
                 repl_module.mark_exception(e)
                 continue
 

src/basilisp/core.lpy

@@ -4171,7 +4171,7 @@
   ([]
    (.write *out* os/linesep)
    (when *flush-on-newline*
-       (.flush *out*))
+     (.flush *out*))
    nil)
   ([x]
    (let [stdout *out*]
@@ -4402,11 +4402,12 @@
   (with-open [f (python/open path ** :mode "r")]
     (load-reader f)))
 
-(defn- resolve-load-path [path]
+(defn- resolve-load-path
   "Resolve load ``path`` relative to the current namespace, or, if it
   begins with \"/\", relative to the syspath, and return it.
 
- Throw a python/FileNotFoundError if the ``path`` cannot be resolved."
+  Throw a ``python/FileNotFoundError`` if the ``path`` cannot be resolved."
+  [path]
   (let [path-rel (if (.startswith path "/")
                    path
                    (let [path-parent (-> (name *ns*)

src/basilisp/lang/compiler/exception.py

@@ -1,12 +1,15 @@
 import ast
+import os
 from enum import Enum
-from typing import Any, Dict, Optional, Union
+from types import TracebackType
+from typing import Any, Dict, List, Optional, Type, Union
 
 import attr
 
 from basilisp.lang import keyword as kw
 from basilisp.lang import map as lmap
 from basilisp.lang.compiler.nodes import Node
+from basilisp.lang.exception import format_exception
 from basilisp.lang.interfaces import IExceptionInfo, IMeta, IPersistentMap, ISeq
 from basilisp.lang.obj import lrepr
 from basilisp.lang.reader import (
@@ -15,6 +18,7 @@
     READER_END_LINE_KW,
     READER_LINE_KW,
 )
+from basilisp.lang.source import format_source_context
 from basilisp.lang.typing import LispForm
 
 _FILE = kw.keyword("file")
@@ -56,16 +60,15 @@ def __bool__(self):
 class CompilerException(IExceptionInfo):
     msg: str
     phase: CompilerPhase
-    filename: Optional[str] = None
+    filename: str
     form: Union[LispForm, None, ISeq] = None
     lisp_ast: Optional[Node] = None
     py_ast: Optional[ast.AST] = None
 
     @property
     def data(self) -> IPersistentMap:
         d: Dict[kw.Keyword, Any] = {_PHASE: self.phase.value}
-        if self.filename is not None:
-            d[_FILE] = self.filename
+        d[_FILE] = self.filename
         loc = None
         if self.form is not None:
             d[_FORM] = self.form
@@ -104,3 +107,57 @@ def data(self) -> IPersistentMap:
 
     def __str__(self):
         return f"{self.msg} {lrepr(self.data)}"
+
+
+@format_exception.register(CompilerException)
+def format_compiler_exception(  # pylint: disable=too-many-branches,unused-argument
+    e: CompilerException,
+    tp: Optional[Type[Exception]] = None,
+    tb: Optional[TracebackType] = None,
+) -> List[str]:
+    """Format a compiler exception as a list of newline-terminated strings."""
+    context_exc: Optional[BaseException] = e.__cause__
+
+    lines = [os.linesep]
+    if context_exc is not None:
+        lines.append(f"  exception: {type(context_exc)} from {type(e)}{os.linesep}")
+    else:
+        lines.append(f"  exception: {type(e)}{os.linesep}")
+    lines.append(f"      phase: {e.phase.value}{os.linesep}")
+    if context_exc is None:
+        lines.append(f"    message: {e.msg}{os.linesep}")
+    elif e.phase in {CompilerPhase.MACROEXPANSION, CompilerPhase.INLINING}:
+        if isinstance(context_exc, CompilerException):
+            lines.append(f"    message: {e.msg}: {context_exc.msg}{os.linesep}")
+        else:
+            lines.append(f"    message: {e.msg}: {context_exc}{os.linesep}")
+    else:
+        lines.append(f"    message: {e.msg}: {context_exc}{os.linesep}")
+    if e.form is not None:
+        lines.append(f"       form: {e.form!r}{os.linesep}")
+
+    d = e.data
+    line = d.val_at(_LINE)
+    end_line = d.val_at(_END_LINE)
+    if line is not None and end_line is not None and line != end_line:
+        line_nums = f"{line}-{end_line}"
+    elif line is not None:
+        line_nums = str(line)
+    else:
+        line_nums = ""
+
+    lines.append(
+        f"   location: {e.filename}:{line_nums or 'NO_SOURCE_LINE'}{os.linesep}"
+    )
+
+    # Print context source lines around the error. Use the current exception to
+    # derive source lines, but use the inner cause exception to place a marker
+    # around the error.
+    if line is not None and (
+        context_lines := format_source_context(e.filename, line, end_line=end_line)
+    ):
+        lines.append(f"    context:{os.linesep}")
+        lines.append(os.linesep)
+        lines.extend(context_lines)
+
+    return lines

src/basilisp/lang/exception.py

@@ -1,3 +1,9 @@
+import functools
+import sys
+import traceback
+from types import TracebackType
+from typing import List, Optional, Type
+
 import attr
 
 from basilisp.lang.interfaces import IExceptionInfo, IPersistentMap
@@ -16,3 +22,37 @@ def __repr__(self):
 
     def __str__(self):
         return f"{self.message} {lrepr(self.data)}"
+
+
+@functools.singledispatch
+def format_exception(
+    e: Optional[BaseException],
+    tp: Optional[Type[BaseException]] = None,
+    tb: Optional[TracebackType] = None,
+) -> List[str]:
+    """Format an exception into something readable, returning a list of newline
+    terminated strings.
+
+    For the majority of Python exceptions, this will just be the result from calling
+    `traceback.format_exception`. For Basilisp specific compilation errors, a custom
+    output will be returned."""
+    if isinstance(e, BaseException):
+        if tp is None:
+            tp = type(e)
+        if tb is None:
+            tb = e.__traceback__
+    return traceback.format_exception(tp, e, tb)
+
+
+def print_exception(
+    e: Optional[BaseException],
+    tp: Optional[Type[BaseException]] = None,
+    tb: Optional[TracebackType] = None,
+) -> None:
+    """Print the given exception `e` using Basilisp's own exception formatting.
+
+    For the majority of exception types, this should be identical to the base Python
+    traceback formatting. `basilisp.lang.compiler.CompilerException` and
+    `basilisp.lang.reader.SyntaxError` have special handling to print useful information
+    on exceptions."""
+    print("".join(format_exception(e, tp, tb)), file=sys.stderr)

src/basilisp/lang/reader.py

@@ -5,11 +5,13 @@
 import decimal
 import functools
 import io
+import os
 import re
 import uuid
 from datetime import datetime
 from fractions import Fraction
 from itertools import chain
+from types import TracebackType
 from typing import (
     Any,
     Callable,
@@ -24,6 +26,7 @@
     Sequence,
     Set,
     Tuple,
+    Type,
     TypeVar,
     Union,
     cast,
@@ -39,6 +42,7 @@
 from basilisp.lang import symbol as sym
 from basilisp.lang import util as langutil
 from basilisp.lang import vector as vec
+from basilisp.lang.exception import format_exception
 from basilisp.lang.interfaces import (
     ILispObject,
     ILookup,
@@ -60,6 +64,7 @@
     get_current_ns,
     lrepr,
 )
+from basilisp.lang.source import format_source_context
 from basilisp.lang.typing import IterableLispForm, LispForm, ReaderForm
 from basilisp.lang.util import munge
 from basilisp.util import Maybe, partition
@@ -152,6 +157,53 @@ def __str__(self):
             return f"{self.message} ({details})"
 
 
+@format_exception.register(SyntaxError)
+def format_syntax_error(  # pylint: disable=unused-argument
+    e: SyntaxError,
+    tp: Optional[Type[Exception]] = None,
+    tb: Optional[TracebackType] = None,
+) -> List[str]:
+    context_exc: Optional[BaseException] = e.__cause__
+
+    lines = [os.linesep]
+    if context_exc is not None:
+        lines.append(f"  exception: {type(context_exc)} from {type(e)}{os.linesep}")
+    else:
+        lines.append(f"  exception: {type(e)}{os.linesep}")
+    if context_exc is None:
+        lines.append(f"    message: {e.message}{os.linesep}")
+    else:
+        lines.append(f"    message: {e.message}: {context_exc}{os.linesep}")
+
+    if e.line is not None and e.col:
+        line_num = f"{e.line}:{e.col}"
+    elif e.line is not None:
+        line_num = str(e.line)
+    else:
+        line_num = ""
+
+    if e.filename is not None:
+        lines.append(
+            f"   location: {e.filename}:{line_num or 'NO_SOURCE_LINE'}{os.linesep}"
+        )
+    elif line_num:
+        lines.append(f"       line: {line_num}{os.linesep}")
+
+    # Print context source lines around the error. Use the current exception to
+    # derive source lines, but use the inner cause exception to place a marker
+    # around the error.
+    if (
+        e.filename is not None
+        and e.line is not None
+        and (context_lines := format_source_context(e.filename, e.line))
+    ):
+        lines.append(f"    context:{os.linesep}")
+        lines.append(os.linesep)
+        lines.extend(context_lines)
+
+    return lines
+
+
 class UnexpectedEOFError(SyntaxError):
     """Syntax Error type raised when the reader encounters an unexpected EOF
     reading a form.

src/basilisp/lang/source.py

@@ -0,0 +1,82 @@
+import linecache
+import os
+from typing import List, Optional
+
+try:
+    import pygments.formatters
+    import pygments.lexers
+    import pygments.styles
+except ImportError:  # pragma: no cover
+
+    def _format_source(s: str) -> str:
+        return f"{s}{os.linesep}"
+
+else:
+
+    def _get_formatter_name() -> Optional[str]:  # pragma: no cover
+        """Get the Pygments formatter name for formatting the source code by
+        inspecting various environment variables set by terminals.
+
+        If `BASILISP_NO_COLOR` is set to a truthy value, use no formatting."""
+        if os.environ.get("BASILISP_NO_COLOR", "false").lower() in {"1", "true"}:
+            return None
+        elif os.environ.get("COLORTERM", "") in {"truecolor", "24bit"}:
+            return "terminal16m"
+        elif "256" in os.environ.get("TERM", ""):
+            return "terminal256"
+        else:
+            return "terminal"
+
+    def _format_source(s: str) -> str:  # pragma: no cover
+        """Format source code for terminal output."""
+        if (formatter_name := _get_formatter_name()) is None:
+            return f"{s}{os.linesep}"
+        return pygments.highlight(
+            s,
+            lexer=pygments.lexers.get_lexer_by_name("clojure"),
+            formatter=pygments.formatters.get_formatter_by_name(
+                formatter_name, style=pygments.styles.get_style_by_name("emacs")
+            ),
+        )
+
+
+def format_source_context(
+    filename: str,
+    line: int,
+    end_line: Optional[int] = None,
+    num_context_lines: int = 5,
+    show_cause_marker: bool = True,
+) -> List[str]:
+    """Format source code context with line numbers and identifiers for the affected
+    line(s)."""
+    assert num_context_lines >= 0
+
+    lines = []
+
+    if not filename.startswith("<") and not filename.endswith(">"):
+        cause_range: Optional[range]
+        if not show_cause_marker:
+            cause_range = None
+        elif end_line is not None and end_line != line:
+            cause_range = range(line, end_line)
+        else:
+            cause_range = range(line, line + 1)
+
+        if source_lines := linecache.getlines(filename):
+            start = max(0, line - num_context_lines)
+            end = min((end_line or line) + num_context_lines, len(source_lines))
+            num_justify = max(len(str(start)), len(str(end))) + 1
+            for n, source_line in zip(range(start, end), source_lines[start:end]):
+                if cause_range is None:
+                    line_marker = " "
+                elif n + 1 in cause_range:
+                    line_marker = " > "
+                else:
+                    line_marker = "   "
+
+                line_num = str(n + 1).rjust(num_justify)
+                lines.append(
+                    f"{line_num}{line_marker}| {_format_source(source_line.rstrip())}"
+                )
+
+    return lines