Pass macro environment as the first argument to macro functions (#339)

* Pass macro environment as the first argument to macro functions

* Formatting

Author: chrisrink10

src/basilisp/core/__init__.lpy

@@ -61,15 +61,15 @@
        (.with-meta o meta)))
 
 (def ^:macro ^:redef let
-  (fn* let [&form & decl]
+  (fn* let [&env &form & decl]
        (cons 'let* decl)))
 
 (def ^:macro ^:redef loop
-  (fn* loop [&form & decl]
+  (fn* loop [&env &form & decl]
        (cons 'loop* decl)))
 
 (def ^:macro ^:redef fn
-  (fn* fn [&form & decl]
+  (fn* fn [&env &form & decl]
        (with-meta
          (cons 'fn* decl)
          (meta &form))))
@@ -264,7 +264,7 @@
     :doc      "Define a new function with an optional docstring."
     :arglists '([name & body] [name doc & body])}
   defn
-  (fn defn [&form name & body]
+  (fn defn [&env &form name & body]
     (if (symbol? name)
       nil  ;; Do nothing!
       (throw (ex-info "First argument to defn must be a symbol"
@@ -382,7 +382,7 @@
     :doc   "Define a new macro like defn. Macro functions are available to the
             compiler during macroexpansion."}
   defmacro
-  (fn defmacro [&form name & body]
+  (fn defmacro [&env &form name & body]
     (let [body   (concat body)
           doc    (if (string? (first body))
                    (first body)
@@ -408,7 +408,10 @@
           add-implicit-args (fn [body]
                               (cons
                                (if (vector? (first body))
-                                 (apply vector (cons '&form (first body)))
+                                 (apply vector
+                                        (cons '&env
+                                              (cons '&form
+                                                    (first body))))
                                  (throw
                                   (ex-info "Expected an argument vector"
                                            {:found (first body)})))
@@ -2245,3 +2248,29 @@
   (let [[bindings body] (loop-with-destructuring bindings body)]
     `(loop* ~bindings
         ~@body)))
+
+;;;;;;;;;;;;;;;;;;;;;;;
+;; Interop Functions ;;
+;;;;;;;;;;;;;;;;;;;;;;;
+
+(defn lisp->py
+  "Recursively convert Basilisp data structures into Python data structures.
+
+  Callers can specify a keyword argument :keyword-fn, which names a function
+  which is called for each keyword value in the input structure to return a
+  new value. By default :keyword-fn is the function `name`."
+  ([o]
+   (basilisp.lang.runtime/to-py o))
+  ([o & {:keys [keyword-fn] :or {keyword-fn name}}]
+   (basilisp.lang.runtime/to-py o keyword-fn)))
+
+(defn py->lisp
+  "Recursively convert Python data structures into Basilisp data structures.
+
+  Callers can specify a keyword argument :keywordize-keys, which defaults to
+  true. If :keywordize-keys is true, then all string keys in Python dicts will
+  be converted into keywords in the final return value."
+  ([o]
+   (basilisp.lang.runtime/to-lisp o))
+  ([o & {:keys [keywordize-keys] :or {keywordize-keys true}}]
+   (basilisp.lang.runtime/to-lisp o keywordize-keys)))

src/basilisp/lang/compiler/nodes.py

@@ -19,7 +19,7 @@
 import basilisp.lang.set as lset
 import basilisp.lang.symbol as sym
 import basilisp.lang.vector as vec
-from basilisp.lang.runtime import Namespace, Var
+from basilisp.lang.runtime import Namespace, Var, to_lisp
 from basilisp.lang.typing import LispForm, ReaderForm as ReaderLispForm, SpecialForm
 from basilisp.lang.util import munge
 
@@ -120,6 +120,9 @@ def top_level(self) -> bool:
     def env(self) -> "NodeEnv":
         raise NotImplementedError()
 
+    def to_map(self) -> lmap.Map:
+        return to_lisp(attr.asdict(self))
+
     def assoc(self, **kwargs):
         return attr.evolve(self, **kwargs)
 

src/basilisp/lang/compiler/parser.py

@@ -134,11 +134,18 @@ class RecurPoint:
 
 @attr.s(auto_attribs=True, frozen=True, slots=True)
 class SymbolTableEntry:
-    context: LocalType
-    symbol: sym.Symbol
+    binding: Binding
     used: bool = False
     warn_if_unused: bool = True
 
+    @property
+    def symbol(self) -> sym.Symbol:
+        return self.binding.form
+
+    @property
+    def context(self) -> LocalType:
+        return self.binding.local
+
 
 # pylint: disable=unsupported-membership-test,unsupported-delete-operation,unsupported-assignment-operation
 @attr.s(auto_attribs=True, slots=True)
@@ -149,14 +156,16 @@ class SymbolTable:
     _children: Dict[str, "SymbolTable"] = attr.ib(factory=dict)
 
     def new_symbol(
-        self, s: sym.Symbol, ctx: LocalType, warn_if_unused: bool = True
+        self, s: sym.Symbol, binding: Binding, warn_if_unused: bool = True
     ) -> "SymbolTable":
+        assert s == binding.form, "Binding symbol must match passed symbol"
+
         if s in self._table:
             self._table[s] = attr.evolve(
-                self._table[s], context=ctx, symbol=s, warn_if_unused=warn_if_unused
+                self._table[s], binding=binding, warn_if_unused=warn_if_unused
             )
         else:
-            self._table[s] = SymbolTableEntry(ctx, s, warn_if_unused=warn_if_unused)
+            self._table[s] = SymbolTableEntry(binding, warn_if_unused=warn_if_unused)
         return self
 
     def find_symbol(self, s: sym.Symbol) -> Optional[SymbolTableEntry]:
@@ -228,6 +237,16 @@ def new_frame(self, name, warn_on_unused_names):
             new_frame._warn_unused_names()
         self.pop_frame(name)
 
+    def _as_env_map(self) -> Dict[sym.Symbol, lmap.Map]:
+        locals_ = {} if self._parent is None else self._parent._as_env_map()
+        locals_.update({k: v.binding.to_map() for k, v in self._table.items()})
+        return locals_
+
+    def as_env_map(self) -> lmap.Map:
+        """Return a map of symbols to the local binding objects in the
+        local symbol table as of this call."""
+        return lmap.map(self._as_env_map())
+
 
 class ParserContext:
     __slots__ = ("_filename", "_is_quoted", "_opts", "_recur_points", "_st")
@@ -304,7 +323,7 @@ def symbol_table(self) -> SymbolTable:
     def put_new_symbol(  # pylint: disable=too-many-arguments
         self,
         s: sym.Symbol,
-        sym_ctx: LocalType,
+        binding: Binding,
         warn_on_shadowed_name: bool = True,
         warn_on_shadowed_var: bool = True,
         warn_if_unused: bool = True,
@@ -336,7 +355,7 @@ def put_new_symbol(  # pylint: disable=too-many-arguments
                 logger.warning(f"name '{s}' shadows def'ed Var from outer scope")
         if s.meta is not None and s.meta.entry(SYM_NO_WARN_WHEN_UNUSED_META_KEY, None):
             warn_if_unused = False
-        st.new_symbol(s, sym_ctx, warn_if_unused=warn_if_unused)
+        st.new_symbol(s, binding, warn_if_unused=warn_if_unused)
 
     @contextlib.contextmanager
     def new_symbol_table(self, name):
@@ -548,7 +567,7 @@ def _do_ast(ctx: ParserContext, form: lseq.Seq) -> Do:
     )
 
 
-def __fn_method_ast(  # pylint: disable=too-many-branches
+def __fn_method_ast(  # pylint: disable=too-many-branches,too-many-locals
     ctx: ParserContext, form: lseq.Seq, fnname: Optional[sym.Symbol] = None
 ) -> FnMethod:
     with ctx.new_symbol_table("fn-method"):
@@ -571,18 +590,16 @@ def __fn_method_ast(  # pylint: disable=too-many-branches
                 vargs_idx = i
                 break
 
-            param_nodes.append(
-                Binding(
-                    form=s,
-                    name=s.name,
-                    local=LocalType.ARG,
-                    arg_id=i,
-                    is_variadic=False,
-                    env=ctx.get_node_env(),
-                )
+            binding = Binding(
+                form=s,
+                name=s.name,
+                local=LocalType.ARG,
+                arg_id=i,
+                is_variadic=False,
+                env=ctx.get_node_env(),
             )
-
-            ctx.put_new_symbol(s, LocalType.ARG)
+            param_nodes.append(binding)
+            ctx.put_new_symbol(s, binding)
 
         if has_vargs:
             try:
@@ -593,18 +610,16 @@ def __fn_method_ast(  # pylint: disable=too-many-branches
                         "function rest parameter name must be a symbol", form=vargs_sym
                     )
 
-                param_nodes.append(
-                    Binding(
-                        form=vargs_sym,
-                        name=vargs_sym.name,
-                        local=LocalType.ARG,
-                        arg_id=vargs_idx + 1,
-                        is_variadic=True,
-                        env=ctx.get_node_env(),
-                    )
+                binding = Binding(
+                    form=vargs_sym,
+                    name=vargs_sym.name,
+                    local=LocalType.ARG,
+                    arg_id=vargs_idx + 1,
+                    is_variadic=True,
+                    env=ctx.get_node_env(),
                 )
-
-                ctx.put_new_symbol(vargs_sym, LocalType.ARG)
+                param_nodes.append(binding)
+                ctx.put_new_symbol(vargs_sym, binding)
             except IndexError:
                 raise ParserException(
                     "Expected variadic argument name after '&'", form=params
@@ -658,10 +673,11 @@ def _fn_ast(  # pylint: disable=too-many-branches  # noqa: MC0001
             )
 
         if isinstance(name, sym.Symbol):
-            ctx.put_new_symbol(name, LocalType.FN, warn_if_unused=False)
             name_node: Optional[Binding] = Binding(
                 form=name, name=name.name, local=LocalType.FN, env=ctx.get_node_env()
             )
+            assert name_node is not None
+            ctx.put_new_symbol(name, name_node, warn_if_unused=False)
             idx += 1
         elif isinstance(name, (llist.List, vec.Vector)):
             name = None
@@ -944,7 +960,8 @@ def _invoke_ast(ctx: ParserContext, form: Union[llist.List, lseq.Seq]) -> Node:
     if fn.op == NodeOp.VAR and isinstance(fn, VarRef):
         if _is_macro(fn.var):
             try:
-                expanded = fn.var.value(form, *form.rest)
+                macro_env = ctx.symbol_table.as_env_map()
+                expanded = fn.var.value(macro_env, form, *form.rest)
                 expanded_ast = _parse_ast(ctx, expanded)
 
                 # Verify that macroexpanded code also does not have any
@@ -997,18 +1014,16 @@ def _let_ast(ctx: ParserContext, form: lseq.Seq) -> Let:
             if not isinstance(name, sym.Symbol):
                 raise ParserException("let binding name must be a symbol", form=name)
 
-            binding_nodes.append(
-                Binding(
-                    form=name,
-                    name=name.name,
-                    local=LocalType.LET,
-                    init=_parse_ast(ctx, value),
-                    children=vec.v(INIT),
-                    env=ctx.get_node_env(),
-                )
+            binding = Binding(
+                form=name,
+                name=name.name,
+                local=LocalType.LET,
+                init=_parse_ast(ctx, value),
+                children=vec.v(INIT),
+                env=ctx.get_node_env(),
             )
-
-            ctx.put_new_symbol(name, LocalType.LET)
+            binding_nodes.append(binding)
+            ctx.put_new_symbol(name, binding)
 
         let_body = runtime.nthrest(form, 2)
         *statements, ret = map(partial(_parse_ast, ctx), let_body)
@@ -1050,17 +1065,15 @@ def _loop_ast(ctx: ParserContext, form: lseq.Seq) -> Loop:
             if not isinstance(name, sym.Symbol):
                 raise ParserException("loop binding name must be a symbol", form=name)
 
-            binding_nodes.append(
-                Binding(
-                    form=name,
-                    name=name.name,
-                    local=LocalType.LOOP,
-                    init=_parse_ast(ctx, value),
-                    env=ctx.get_node_env(),
-                )
+            binding = Binding(
+                form=name,
+                name=name.name,
+                local=LocalType.LOOP,
+                init=_parse_ast(ctx, value),
+                env=ctx.get_node_env(),
             )
-
-            ctx.put_new_symbol(name, LocalType.LOOP)
+            binding_nodes.append(binding)
+            ctx.put_new_symbol(name, binding)
 
         with ctx.new_recur_point(loop_id, binding_nodes):
             loop_body = runtime.nthrest(form, 2)
@@ -1222,19 +1235,20 @@ def _catch_ast(ctx: ParserContext, form: lseq.Seq) -> Catch:
         raise ParserException("catch local must be a symbol", form=local_name)
 
     with ctx.new_symbol_table("catch"):
-        ctx.put_new_symbol(local_name, LocalType.CATCH)
+        catch_binding = Binding(
+            form=local_name,
+            name=local_name.name,
+            local=LocalType.CATCH,
+            env=ctx.get_node_env(),
+        )
+        ctx.put_new_symbol(local_name, catch_binding)
 
         catch_body = runtime.nthrest(form, 3)
         *catch_statements, catch_ret = map(partial(_parse_ast, ctx), catch_body)
         return Catch(
             form=form,
             class_=catch_cls,
-            local=Binding(
-                form=local_name,
-                name=local_name.name,
-                local=LocalType.CATCH,
-                env=ctx.get_node_env(),
-            ),
+            local=catch_binding,
             body=Do(
                 form=catch_body,
                 statements=vec.vector(catch_statements),