Fix transient macro namespace resolution (#438)

* Fix transient namespace resolution

* Formatting

* Add a test for referred names as well

* Additional test cases (which are currently broken)

* Plz fix

* Resolve some remaining issues

* Satisfy PyLint

Author: chrisrink10

src/basilisp/core/template.lpy

@@ -1,4 +1,6 @@
-(ns basilisp.core.template)
+(ns basilisp.core.template
+  (:require
+   [basilisp.walk :refer [postwalk-replace]]))
 
 (defmacro do-template
   "Given a template expression expr and bindings, produce a do expression with
@@ -23,6 +25,6 @@
                         (as-> arg-group $
                           (interleave argv $)
                           (apply hash-map $)
-                          (replace $ expr)))]
+                          (postwalk-replace $ expr)))]
     `(do
        ~@(map template-expr arg-groups))) )

src/basilisp/lang/compiler/analyzer.py

@@ -281,6 +281,7 @@ class AnalyzerContext:
         "_filename",
         "_func_ctx",
         "_is_quoted",
+        "_macro_ns",
         "_opts",
         "_recur_points",
         "_should_macroexpand",
@@ -298,6 +299,7 @@ def __init__(
         self._filename = Maybe(filename).or_else_get(DEFAULT_COMPILER_FILE_PATH)
         self._func_ctx: Deque[bool] = collections.deque([])
         self._is_quoted: Deque[bool] = collections.deque([])
+        self._macro_ns: Deque[Optional[runtime.Namespace]] = collections.deque([])
         self._opts = (
             Maybe(opts).map(lmap.map).or_else_get(lmap.Map.empty())  # type: ignore
         )
@@ -348,6 +350,35 @@ def quoted(self):
         yield
         self._is_quoted.pop()
 
+    @property
+    def current_macro_ns(self) -> Optional[runtime.Namespace]:
+        """Return the current transient namespace available during macroexpansion.
+
+        If None, the analyzer should only use the current namespace for symbol
+        resolution."""
+        try:
+            return self._macro_ns[-1]
+        except IndexError:
+            return None
+
+    @contextlib.contextmanager
+    def macro_ns(self, ns: Optional[runtime.Namespace]):
+        """Set the transient namespace which is available to the analyer during a
+        macroexpansion phase.
+
+        If set to None, prohibit the analyzer from using another namespace for symbol
+        resolution.
+
+        During macroexpansion, new forms referenced from the macro namespace would
+        be unavailable to the namespace containing the original macro invocation.
+        The macro namespace is a temporary override pointing to the namespace of the
+        macro definition which can be used to resolve these transient references."""
+        self._macro_ns.append(ns)
+        try:
+            yield
+        finally:
+            self._macro_ns.pop()
+
     @property
     def should_allow_unresolved_symbols(self) -> bool:
         """If True, the analyzer will allow unresolved symbols. This is primarily
@@ -1639,7 +1670,10 @@ def _invoke_ast(ctx: AnalyzerContext, form: Union[llist.List, ISeq]) -> Node:
                 try:
                     macro_env = ctx.symbol_table.as_env_map()
                     expanded = fn.var.value(macro_env, form, *form.rest)
-                    expanded_ast = _analyze_form(ctx, expanded)
+                    with ctx.macro_ns(
+                        fn.var.ns if fn.var.ns is not ctx.current_ns else None
+                    ):
+                        expanded_ast = _analyze_form(ctx, expanded)
 
                     # Verify that macroexpanded code also does not have any
                     # non-tail recur forms
@@ -2085,34 +2119,60 @@ def _resolve_nested_symbol(ctx: AnalyzerContext, form: sym.Symbol) -> HostField:
     )
 
 
-def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches
-    ctx: AnalyzerContext, form: sym.Symbol
-) -> Union[Const, HostField, MaybeClass, MaybeHostForm, VarRef]:
-    """Resolve a namespaced symbol into a Python name or Basilisp Var."""
+def __fuzzy_resolve_namespace_reference(
+    ctx: AnalyzerContext, which_ns: runtime.Namespace, form: sym.Symbol
+) -> Optional[VarRef]:
+    """Resolve a symbol within `which_ns` based on any namespaces required or otherwise
+    referenced within `which_ns` (e.g. by a :refer).
+
+    When a required or resolved symbol is read by the reader in the context of a syntax
+    quote, the reader will fully resolve the symbol, so a symbol like `set/union` would be
+    expanded to `basilisp.set/union`. However, the namespace still does not maintain a
+    direct mapping of the symbol `basilisp.set` to the namespace it names, since the
+    namespace was required as `[basilisp.set :as set]`.
+
+    During macroexpansion, the Analyzer needs to resolve these transitive requirements,
+    so we 'fuzzy' resolve against any namespaces known to the current macro namespace."""
     assert form.ns is not None
+    ns_name = form.ns
+
+    def resolve_ns_reference(
+        ns_map: Mapping[str, runtime.Namespace]
+    ) -> Optional[VarRef]:
+        match: Optional[runtime.Namespace] = ns_map.get(ns_name)
+        if match is not None:
+            v = match.find(sym.symbol(form.name))
+            if v is not None:
+                return VarRef(form=form, var=v, env=ctx.get_node_env())
+        return None
 
-    if form.ns == ctx.current_ns.name:
-        v = ctx.current_ns.find(sym.symbol(form.name))
-        if v is not None:
-            return VarRef(form=form, var=v, env=ctx.get_node_env())
-    elif form.ns == _BUILTINS_NS:
-        class_ = munge(form.name, allow_builtins=True)
-        target = getattr(builtins, class_, None)
-        if target is None:
-            raise AnalyzerException(
-                f"cannot resolve builtin function '{class_}'", form=form
-            )
-        return MaybeClass(
-            form=form, class_=class_, target=target, env=ctx.get_node_env()
-        )
+    # Try to match a required namespace
+    required_namespaces = {ns.name: ns for ns in which_ns.aliases.values()}
+    match = resolve_ns_reference(required_namespaces)
+    if match is not None:
+        return match
 
-    if "." in form.name and form.name != _DOUBLE_DOT_MACRO_NAME:
-        raise AnalyzerException(
-            "symbol names may not contain the '.' operator", form=form
-        )
+    # Try to match a referred namespace
+    referred_namespaces = {
+        ns.name: ns for ns in {var.ns for var in which_ns.refers.values()}
+    }
+    return resolve_ns_reference(referred_namespaces)
+
+
+def __resolve_namespaced_symbol_in_ns(  # pylint: disable=too-many-branches
+    ctx: AnalyzerContext,
+    which_ns: runtime.Namespace,
+    form: sym.Symbol,
+    allow_fuzzy_macroexpansion_matching: bool = False,
+) -> Optional[Union[MaybeHostForm, VarRef]]:
+    """Resolve the symbol `form` in the context of the Namespace `which_ns`. If
+    `allow_fuzzy_macroexpansion_matching` is True and no match is made on existing
+    imports, import aliases, or namespace aliases, then attempt to match the
+    namespace portion"""
+    assert form.ns is not None
 
     ns_sym = sym.symbol(form.ns)
-    if ns_sym in ctx.current_ns.imports or ns_sym in ctx.current_ns.import_aliases:
+    if ns_sym in which_ns.imports or ns_sym in which_ns.import_aliases:
         # We still import Basilisp code, so we'll want to make sure
         # that the symbol isn't referring to a Basilisp Var first
         v = Var.find(form)
@@ -2123,8 +2183,8 @@ def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches
         # We don't need this for actually generating the link later, but
         # we _do_ need it for fetching a reference to the module to check
         # for membership.
-        if ns_sym in ctx.current_ns.import_aliases:
-            ns = ctx.current_ns.import_aliases[ns_sym]
+        if ns_sym in which_ns.import_aliases:
+            ns = which_ns.import_aliases[ns_sym]
             assert ns is not None
             ns_name = ns.name
         else:
@@ -2161,16 +2221,58 @@ def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches
             target=vars(ns_module)[safe_name],
             env=ctx.get_node_env(),
         )
-    elif ns_sym in ctx.current_ns.aliases:
-        aliased_ns: runtime.Namespace = ctx.current_ns.aliases[ns_sym]
+    elif ns_sym in which_ns.aliases:
+        aliased_ns: runtime.Namespace = which_ns.aliases[ns_sym]
         v = Var.find(sym.symbol(form.name, ns=aliased_ns.name))
         if v is None:
             raise AnalyzerException(
                 f"unable to resolve symbol '{sym.symbol(form.name, ns_sym.name)}' in this context",
                 form=form,
             )
         return VarRef(form=form, var=v, env=ctx.get_node_env())
-    elif "." in form.ns:
+    elif allow_fuzzy_macroexpansion_matching:
+        return __fuzzy_resolve_namespace_reference(ctx, which_ns, form)
+
+    return None
+
+
+def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches
+    ctx: AnalyzerContext, form: sym.Symbol
+) -> Union[Const, HostField, MaybeClass, MaybeHostForm, VarRef]:
+    """Resolve a namespaced symbol into a Python name or Basilisp Var."""
+    assert form.ns is not None
+
+    if form.ns == ctx.current_ns.name:
+        v = ctx.current_ns.find(sym.symbol(form.name))
+        if v is not None:
+            return VarRef(form=form, var=v, env=ctx.get_node_env())
+    elif form.ns == _BUILTINS_NS:
+        class_ = munge(form.name, allow_builtins=True)
+        target = getattr(builtins, class_, None)
+        if target is None:
+            raise AnalyzerException(
+                f"cannot resolve builtin function '{class_}'", form=form
+            )
+        return MaybeClass(
+            form=form, class_=class_, target=target, env=ctx.get_node_env()
+        )
+
+    if "." in form.name and form.name != _DOUBLE_DOT_MACRO_NAME:
+        raise AnalyzerException(
+            "symbol names may not contain the '.' operator", form=form
+        )
+
+    resolved = __resolve_namespaced_symbol_in_ns(ctx, ctx.current_ns, form)
+    if resolved is not None:
+        return resolved
+    elif ctx.current_macro_ns is not None:
+        resolved = __resolve_namespaced_symbol_in_ns(
+            ctx, ctx.current_macro_ns, form, allow_fuzzy_macroexpansion_matching=True
+        )
+        if resolved is not None:
+            return resolved
+
+    if "." in form.ns:
         return _resolve_nested_symbol(ctx, form)
     elif ctx.should_allow_unresolved_symbols:
         return _const_node(ctx, form)
@@ -2192,6 +2294,12 @@ def __resolve_bare_symbol(
     if v is not None:
         return VarRef(form=form, var=v, env=ctx.get_node_env())
 
+    # Look up the symbol in the current macro namespace, if one
+    if ctx.current_macro_ns is not None:
+        v = ctx.current_macro_ns.find(form)
+        if v is not None:
+            return VarRef(form=form, var=v, env=ctx.get_node_env())
+
     if "." in form.name:
         raise AnalyzerException(
             "symbol names may not contain the '.' operator", form=form

src/basilisp/lang/interfaces.py

@@ -223,6 +223,9 @@ def __eq__(self, other):
                 return False
         return True
 
+    def __hash__(self):
+        return hash(tuple(self))
+
     def __iter__(self):
         o = self
         if o:

src/basilisp/test.lpy

@@ -2,7 +2,7 @@
   (:import
    inspect)
   (:require
-   [basilisp.core.template :refer [do-template]]))
+   [basilisp.core.template :as template]))
 
 (def ^:private collected-tests
   (atom []))
@@ -115,7 +115,7 @@
 (defmacro are
   "Assert that expr is true. Must appear inside of a deftest form."
   [argv expr & args]
-  `(do-template ~argv (is ~expr) ~@args))
+  `(template/do-template ~argv (is ~expr) ~@args))
 
 (defmacro testing
   "Wrapper for test cases to provide additional messaging and context

src/basilisp/testrunner.py

@@ -153,7 +153,7 @@ def runtest(self):
         if runtime.to_seq(failures):
             raise TestFailuresInfo("Test failures", lmap.map(results))
 
-    def repr_failure(self, excinfo):
+    def repr_failure(self, excinfo, style=None):  # pylint: disable=unused-argument
         """Representation function called when self.runtest() raises an
         exception."""
         if isinstance(excinfo.value, TestFailuresInfo):

tests/basilisp/compiler_test.py

@@ -2877,6 +2877,112 @@ def test_aliased_macro_symbol_resolution(self, ns: runtime.Namespace):
         finally:
             runtime.Namespace.remove(other_ns_name)
 
+    def test_cross_ns_macro_symbol_resolution(self, ns: runtime.Namespace):
+        """Ensure that a macro symbol, `a`, delegating to another macro, named
+        by the symbol `b`, in a namespace directly required by `a`'s namespace
+        (and which will not be required by downstream namespaces) is still
+        properly resolved when used by the final consumer."""
+        current_ns: runtime.Namespace = ns
+        other_ns_name = sym.symbol("other.ns")
+        third_ns_name = sym.symbol("third.ns")
+        try:
+            other_ns = runtime.Namespace.get_or_create(other_ns_name)
+            current_ns.add_alias(other_ns_name, other_ns)
+
+            third_ns = runtime.Namespace.get_or_create(third_ns_name)
+            other_ns.add_alias(third_ns_name, third_ns)
+
+            with runtime.ns_bindings(third_ns_name.name):
+                lcompile(
+                    "(def ^:macro t (fn* [&env &form v] `(name ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            with runtime.ns_bindings(other_ns_name.name):
+                lcompile(
+                    "(def ^:macro o (fn* [&env &form v] `(third.ns/t ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            with runtime.ns_bindings(current_ns.name):
+                assert "z" == lcompile(
+                    "(other.ns/o :z)", resolver=runtime.resolve_alias
+                )
+        finally:
+            runtime.Namespace.remove(other_ns_name)
+            runtime.Namespace.remove(third_ns_name)
+
+    def test_cross_ns_macro_symbol_resolution_with_aliases(self, ns: runtime.Namespace):
+        """Ensure that `a` macro symbol, a, delegating to another macro, named
+        by the symbol `b`, which is referenced by `a`'s namespace (and which will
+        not be referred by downstream namespaces) is still properly resolved
+        when used by the final consumer."""
+        current_ns: runtime.Namespace = ns
+        other_ns_name = sym.symbol("other.ns")
+        third_ns_name = sym.symbol("third.ns")
+        try:
+            other_ns = runtime.Namespace.get_or_create(other_ns_name)
+            current_ns.add_alias(other_ns_name, other_ns)
+
+            third_ns = runtime.Namespace.get_or_create(third_ns_name)
+            other_ns.add_alias(sym.symbol("third"), third_ns)
+
+            with runtime.ns_bindings(third_ns_name.name):
+                lcompile(
+                    "(def ^:macro t (fn* [&env &form v] `(name ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            with runtime.ns_bindings(other_ns_name.name):
+                lcompile(
+                    "(def ^:macro o (fn* [&env &form v] `(third/t ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            with runtime.ns_bindings(current_ns.name):
+                assert "z" == lcompile(
+                    "(other.ns/o :z)", resolver=runtime.resolve_alias
+                )
+        finally:
+            runtime.Namespace.remove(other_ns_name)
+            runtime.Namespace.remove(third_ns_name)
+
+    def test_cross_ns_macro_symbol_resolution_with_refers(self, ns: runtime.Namespace):
+        """Ensure that a macro symbol, `a`, delegating to another macro, named
+        by the symbol `b`, which is referred by `a`'s namespace (and which will
+        not be referred by downstream namespaces) is still properly resolved
+        when used by the final consumer."""
+        current_ns: runtime.Namespace = ns
+        other_ns_name = sym.symbol("other.ns")
+        third_ns_name = sym.symbol("third.ns")
+        try:
+            other_ns = runtime.Namespace.get_or_create(other_ns_name)
+            current_ns.add_alias(other_ns_name, other_ns)
+
+            third_ns = runtime.Namespace.get_or_create(third_ns_name)
+
+            with runtime.ns_bindings(third_ns_name.name):
+                lcompile(
+                    "(def ^:macro t (fn* [&env &form v] `(name ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            other_ns.add_refer(sym.symbol("t"), third_ns.find(sym.symbol("t")))
+
+            with runtime.ns_bindings(other_ns_name.name):
+                lcompile(
+                    "(def ^:macro o (fn* [&env &form v] `(t ~v)))",
+                    resolver=runtime.resolve_alias,
+                )
+
+            with runtime.ns_bindings(current_ns.name):
+                assert "z" == lcompile(
+                    "(other.ns/o :z)", resolver=runtime.resolve_alias
+                )
+        finally:
+            runtime.Namespace.remove(other_ns_name)
+            runtime.Namespace.remove(third_ns_name)
+
 
 class TestWarnOnVarIndirection:
     @pytest.fixture