Support letfn Special Form (#473)

* Support `letfn` Special Form

* Stop the spinning

* Fix everything

* letfn* correctly

* Lint it out

* Docstrings

* Letfn sequential destructuring tests

* letfn associative destructuring tests

Author: chrisrink10

.circleci/config.yml

@@ -18,6 +18,7 @@ jobs:
           shell: /bin/bash -leo pipefail
           environment:
             TOX_NUM_CORES: 2
+            TOX_PARALLEL_NO_SPINNER: 1
             TOX_SHOW_OUTPUT: "True"
             TOX_SKIP_ENV: pypy3
           command: |

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  * Added support for Shebang-style line comments (#469)
  * Added multiline REPL support using `prompt-toolkit` (#467)
  * Added node syntactic location (statement or expression) to Basilisp AST nodes emitted by the analyzer (#463)
+ * Added `letfn` special form (#473)
 
 ### Changed
  * Change the default user namespace to `basilisp.user` (#466)

src/basilisp/core.lpy

@@ -3980,6 +3980,22 @@
                  (mapcat destructure))]
      ~@body))
 
+(defmacro letfn
+  "Let form specifically for function definitions.
+
+  Functions are defined as bindings:
+
+    [(plus-two [x] (+ (plus-one x) 1))
+     (plus-one [x] (+ x 1))]
+
+  Functions defined in `letfn` bindings may refer to each other regardless
+  of their order of definition."
+  [bindings & body]
+  `(letfn* [~@(mapcat (fn [fn-body]
+                        [(first fn-body) (cons `fn fn-body)])
+                      bindings)]
+           ~@body))
+
 (defn ^:private loop-with-destructuring
   "Take a loop definition (an binding vector and 0 or more body
   expressions) whose binding vector may or may not require destructuring

src/basilisp/lang/compiler/analyzer.py

@@ -57,6 +57,7 @@
     SYM_DYNAMIC_META_KEY,
     SYM_MACRO_META_KEY,
     SYM_MUTABLE_META_KEY,
+    SYM_NO_WARN_ON_SHADOW_META_KEY,
     SYM_NO_WARN_WHEN_UNUSED_META_KEY,
     SYM_PROPERTY_META_KEY,
     SYM_STATICMETHOD_META_KEY,
@@ -438,6 +439,13 @@ def put_new_symbol(  # pylint: disable=too-many-arguments
         previously checked case, so this is a simple way of disabling these
         warnings for those cases.
 
+        There are cases where undesired warnings may be triggered non-locally,
+        so the Python keyword arguments cannot be used to suppress unwanted
+        warnings. For these cases, symbols may include the `:no-warn-on-shadow`
+        metadata key to indicate that warnings for shadowing names from outer
+        scopes should be suppressed. It is not currently possible to suppress
+        Var shadowing warnings at the symbol level.
+
         If WARN_ON_SHADOWED_NAME compiler option is active and the
         warn_on_shadowed_name keyword argument is True, then a warning will be
         emitted if a local name is shadowed by another local name. Note that
@@ -447,7 +455,16 @@ def put_new_symbol(  # pylint: disable=too-many-arguments
         warn_on_shadowed_var keyword argument is True, then a warning will be
         emitted if a named var is shadowed by a local name."""
         st = self.symbol_table
-        if warn_on_shadowed_name and self.warn_on_shadowed_name:
+        no_warn_on_shadow = (
+            Maybe(s.meta)
+            .map(lambda m: m.val_at(SYM_NO_WARN_ON_SHADOW_META_KEY, False))
+            .or_else_get(False)
+        )
+        if (
+            not no_warn_on_shadow
+            and warn_on_shadowed_name
+            and self.warn_on_shadowed_name
+        ):
             if st.find_symbol(s) is not None:
                 logger.warning(f"name '{s}' shadows name from outer scope")
         if (
@@ -1840,6 +1857,126 @@ def _let_ast(ctx: AnalyzerContext, form: ISeq) -> Let:
         )
 
 
+def __letfn_fn_body(ctx: AnalyzerContext, form: ISeq) -> Fn:
+    """Produce an `Fn` node for a `letfn*` special form.
+
+    `letfn*` forms use `let*`-like bindings. Each function binding name is
+    added to the symbol table as a forward declaration before analyzing the
+    function body. The function bodies are defined as
+
+        (fn* name
+          [...]
+          ...)
+
+    When the `name` is added to the symbol table for the function, a warning
+    will be produced because it will previously have been defined in the
+    `letfn*` binding scope. This function adds `:no-warn-on-shadow` metadata to
+    the function name symbol to disable the compiler warning."""
+    fn_sym = form.first
+
+    fn_name = runtime.nth(form, 1)
+    if not isinstance(fn_name, sym.Symbol):
+        raise AnalyzerException("letfn function name must be a symbol", form=fn_name)
+
+    fn_rest = runtime.nthrest(form, 2)
+
+    fn_body = _analyze_form(
+        ctx,
+        fn_rest.cons(
+            fn_name.with_meta(
+                (fn_name.meta or lmap.Map.empty()).assoc(
+                    SYM_NO_WARN_ON_SHADOW_META_KEY, True
+                )
+            )
+        ).cons(fn_sym),
+    )
+
+    if not isinstance(fn_body, Fn):
+        raise AnalyzerException(
+            "letfn bindings must be functions", form=form, lisp_ast=fn_body
+        )
+
+    return fn_body
+
+
+def _letfn_ast(  # pylint: disable=too-many-locals
+    ctx: AnalyzerContext, form: ISeq
+) -> LetFn:
+    assert form.first == SpecialForm.LETFN
+    nelems = count(form)
+
+    if nelems < 2:
+        raise AnalyzerException(
+            "letfn forms must have bindings vector and 0 or more body forms", form=form
+        )
+
+    bindings = runtime.nth(form, 1)
+    if not isinstance(bindings, vec.Vector):
+        raise AnalyzerException("letfn bindings must be a vector", form=bindings)
+    elif len(bindings) % 2 != 0:
+        raise AnalyzerException(
+            "letfn bindings must appear in name-value pairs", form=bindings
+        )
+
+    with ctx.new_symbol_table("letfn"):
+        # Generate empty Binding nodes to put into the symbol table
+        # as forward declarations. All functions in letfn* forms may
+        # refer to all other functions regardless of order of definition.
+        empty_binding_nodes = []
+        for name, value in partition(bindings, 2):
+            if not isinstance(name, sym.Symbol):
+                raise AnalyzerException(
+                    "letfn binding name must be a symbol", form=name
+                )
+
+            if not isinstance(value, llist.List):
+                raise AnalyzerException(
+                    "letfn binding value must be a list", form=value
+                )
+
+            binding = Binding(
+                form=name,
+                name=name.name,
+                local=LocalType.LETFN,
+                init=_const_node(ctx, None),
+                children=vec.v(INIT),
+                env=ctx.get_node_env(),
+            )
+            empty_binding_nodes.append((name, value, binding))
+            ctx.put_new_symbol(
+                name, binding,
+            )
+
+        # Once we've generated all of the filler Binding nodes, analyze the
+        # function bodies and replace the Binding nodes with full nodes.
+        binding_nodes = []
+        for fn_name, fn_def, binding in empty_binding_nodes:
+            fn_body = __letfn_fn_body(ctx, fn_def)
+            new_binding = binding.assoc(init=fn_body)
+            binding_nodes.append(new_binding)
+            ctx.put_new_symbol(
+                fn_name,
+                new_binding,
+                warn_on_shadowed_name=False,
+                warn_on_shadowed_var=False,
+            )
+
+        letfn_body = runtime.nthrest(form, 2)
+        stmts, ret = _body_ast(ctx, letfn_body)
+        return LetFn(
+            form=form,
+            bindings=vec.vector(binding_nodes),
+            body=Do(
+                form=letfn_body,
+                statements=vec.vector(stmts),
+                ret=ret,
+                is_body=True,
+                env=ctx.get_node_env(),
+            ),
+            env=ctx.get_node_env(pos=ctx.syntax_position),
+        )
+
+
 def _loop_ast(ctx: AnalyzerContext, form: ISeq) -> Loop:
     assert form.first == SpecialForm.LOOP
     nelems = count(form)
@@ -2185,6 +2322,7 @@ def _var_ast(ctx: AnalyzerContext, form: ISeq) -> VarRef:
     SpecialForm.IMPORT: _import_ast,
     SpecialForm.INTEROP_CALL: _host_interop_ast,
     SpecialForm.LET: _let_ast,
+    SpecialForm.LETFN: _letfn_ast,
     SpecialForm.LOOP: _loop_ast,
     SpecialForm.QUOTE: _quote_ast,
     SpecialForm.RECUR: _recur_ast,

src/basilisp/lang/compiler/constants.py

@@ -15,6 +15,7 @@ class SpecialForm:
     INTEROP_CALL = sym.symbol(".")
     INTEROP_PROP = sym.symbol(".-")
     LET = sym.symbol("let*")
+    LETFN = sym.symbol("letfn*")
     LOOP = sym.symbol("loop*")
     QUOTE = sym.symbol("quote")
     RECUR = sym.symbol("recur")
@@ -141,6 +142,7 @@ class SpecialForm:
 SYM_MACRO_META_KEY = kw.keyword("macro")
 SYM_MUTABLE_META_KEY = kw.keyword("mutable")
 SYM_NO_WARN_ON_REDEF_META_KEY = kw.keyword("no-warn-on-redef")
+SYM_NO_WARN_ON_SHADOW_META_KEY = kw.keyword("no-warn-on-shadow")
 SYM_NO_WARN_WHEN_UNUSED_META_KEY = kw.keyword("no-warn-when-unused")
 SYM_REDEF_META_KEY = kw.keyword("redef")
 SYM_STATICMETHOD_META_KEY = kw.keyword("staticmethod")

src/basilisp/lang/compiler/generator.py

@@ -64,6 +64,7 @@
     Import,
     Invoke,
     Let,
+    LetFn,
     Local,
     LocalType,
     Loop,
@@ -113,6 +114,7 @@
 _IF_RESULT_PREFIX = "if_result"
 _IF_TEST_PREFIX = "if_test"
 _LET_RESULT_PREFIX = "let_result"
+_LETFN_RESULT_PREFIX = "letfn_result"
 _LOOP_RESULT_PREFIX = "loop_result"
 _MULTI_ARITY_ARG_NAME = "multi_arity_args"
 _SET_BANG_TEMP_PREFIX = "set_bang_val"
@@ -1607,6 +1609,53 @@ def _let_to_py_ast(ctx: GeneratorContext, node: Let) -> GeneratedPyAST:
             return GeneratedPyAST(node=_noop_node(), dependencies=let_body_ast)
 
 
+@_with_ast_loc_deps
+def _letfn_to_py_ast(ctx: GeneratorContext, node: LetFn) -> GeneratedPyAST:
+    """Return a Python AST Node for a `letfn*` expression."""
+    assert node.op == NodeOp.LETFN
+
+    with ctx.new_symbol_table("letfn"):
+        binding_names = []
+        for binding in node.bindings:
+            binding_name = genname(munge(binding.name))
+            ctx.symbol_table.new_symbol(
+                sym.symbol(binding.name), binding_name, LocalType.LET
+            )
+            binding_names.append((binding_name, binding))
+
+        letfn_body_ast: List[ast.AST] = []
+        for binding_name, binding in binding_names:
+            init_node = binding.init
+            assert init_node is not None
+            init_ast = gen_py_ast(ctx, init_node)
+            letfn_body_ast.extend(init_ast.dependencies)
+            letfn_body_ast.append(
+                ast.Assign(
+                    targets=[ast.Name(id=binding_name, ctx=ast.Store())],
+                    value=init_ast.node,
+                )
+            )
+
+        letfn_result_name = genname(_LETFN_RESULT_PREFIX)
+        body_ast = _synthetic_do_to_py_ast(ctx, node.body)
+        letfn_body_ast.extend(map(statementize, body_ast.dependencies))
+
+        if node.env.pos == NodeSyntacticPosition.EXPR:
+            letfn_body_ast.append(
+                ast.Assign(
+                    targets=[ast.Name(id=letfn_result_name, ctx=ast.Store())],
+                    value=body_ast.node,
+                )
+            )
+            return GeneratedPyAST(
+                node=ast.Name(id=letfn_result_name, ctx=ast.Load()),
+                dependencies=letfn_body_ast,
+            )
+        else:
+            letfn_body_ast.append(body_ast.node)
+            return GeneratedPyAST(node=_noop_node(), dependencies=letfn_body_ast)
+
+
 @_with_ast_loc_deps
 def _loop_to_py_ast(ctx: GeneratorContext, node: Loop) -> GeneratedPyAST:
     """Return a Python AST Node for a `loop*` expression."""
@@ -2659,7 +2708,7 @@ def _const_node_to_py_ast(ctx: GeneratorContext, lisp_ast: Const) -> GeneratedPy
     NodeOp.IMPORT: _import_to_py_ast,
     NodeOp.INVOKE: _invoke_to_py_ast,
     NodeOp.LET: _let_to_py_ast,
-    NodeOp.LETFN: None,  # type: ignore
+    NodeOp.LETFN: _letfn_to_py_ast,
     NodeOp.LOCAL: _local_sym_to_py_ast,
     NodeOp.LOOP: _loop_to_py_ast,
     NodeOp.MAP: _map_to_py_ast,

src/basilisp/lang/compiler/nodes.py

@@ -855,6 +855,7 @@ class WithMeta(Node[LispForm]):
     Import,
     Invoke,
     Let,
+    LetFn,
     Loop,
     Quote,
     Recur,

src/basilisp/lang/runtime.py

@@ -86,6 +86,7 @@
 _INTEROP_CALL = sym.symbol(".")
 _INTEROP_PROP = sym.symbol(".-")
 _LET = sym.symbol("let*")
+_LETFN = sym.symbol("letfn*")
 _LOOP = sym.symbol("loop*")
 _QUOTE = sym.symbol("quote")
 _RECUR = sym.symbol("recur")
@@ -106,6 +107,7 @@
     _INTEROP_CALL,
     _INTEROP_PROP,
     _LET,
+    _LETFN,
     _LOOP,
     _QUOTE,
     _RECUR,