Re-working import logic and symbol table (#68)

* Re-working import logic and symbol table [WIP]

* Use namespace aliases, mappings, and imports for canonical reference

* Fix all typechecking errors

* Remove unused symbol table contexts

* Remove commented code. Leave some nice comments. 🎉

* Allow individual vars to be specified as dynamic

* Minor changes after rebase

Author: chrisrink10

basilisp/compiler.py

@@ -58,19 +58,43 @@ def get_filename(self, fullname: str) -> str:
         return spec.loader_state.filename
 
     def create_module(self, spec: importlib.machinery.ModuleSpec):
-        mod = super().create_module(spec)
-        self._cache[spec.name] = {"module": mod, "spec": spec}
+        mod = types.ModuleType(spec.name)
+        mod.__loader__ = spec.loader
+        mod.__package__ = spec.parent
+        mod.__spec__ = spec
+        self._cache[spec.name] = {"spec": spec}
         return mod
 
-    def get_code(self, fullname: str) -> types.CodeType:
-        runtime.set_current_ns(fullname)
+    def exec_module(self, module):
+        """Compile the Basilisp module into Python code.
+
+        Basilisp is fundamentally a form-at-a-time compilation, meaning that
+        each form in a module may require code compiled from an earlier form, so
+        we incrementally compile a Python module by evaluating a single top-level
+        form at a time and inserting the resulting AST nodes into the Pyton module."""
+        fullname = module.__name__
         cached = self._cache[fullname]
+        cached["module"] = module
         spec = cached["spec"]
         filename = spec.loader_state["filename"]
+
+        # During the bootstrapping process, the 'basilisp.core namespace is created with
+        # a blank module. If we do not replace the module here with the module we are
+        # generating, then we will not be able to use advanced compilation features such
+        # as direct Python variable access to functions and other def'ed values.
+        ns: runtime.Namespace = runtime.set_current_ns(fullname).value
+        ns.module = module
+
         forms = reader.read_file(filename)
-        bytecode = compiler.compile_module_bytecode(forms, compiler.CompilerContext(), filename)
+        compiler.compile_module(forms, compiler.CompilerContext(), module, filename)
+
+        # Because we want to (by default) add 'basilisp.core into every namespace by default,
+        # we want to make sure we don't try to add 'basilisp.core into itself, causing a
+        # circular import error.
+        #
+        # Later on, we can probably remove this and just use the 'ns macro to auto-refer
+        # all 'basilisp.core values into the current namespace.
         runtime.Namespace.add_default_import(fullname)
-        return bytecode
 
 
 def hook_imports():

basilisp/importer.py

@@ -1,5 +1,6 @@
 import itertools
 import threading
+import types
 from typing import Optional
 
 from functional import seq
@@ -19,6 +20,22 @@
 _PRINT_GENERATED_PY_VAR_NAME = '*print-generated-python*'
 
 
+def _new_module(name: str, doc=None) -> types.ModuleType:
+    """Create a new empty Basilisp Python module.
+
+    Modules are created for each Namespace when it is created."""
+    mod = types.ModuleType(name, doc=doc)
+    mod.__loader__ = None
+    mod.__package__ = None
+    mod.__spec__ = None
+    mod.__basilisp_bootstrapped__ = False  # type: ignore
+    return mod
+
+
+class RuntimeException(Exception):
+    pass
+
+
 class Var:
     __slots__ = ('_name', '_ns', '_root', '_dynamic', '_is_bound', '_tl', '_meta')
 
@@ -118,14 +135,14 @@ def intern_unbound(ns: sym.Symbol,
         return var
 
     @staticmethod
-    def find_in_ns(ns_sym: sym.Symbol, name_sym: sym.Symbol) -> "Var":
+    def find_in_ns(ns_sym: sym.Symbol, name_sym: sym.Symbol) -> "Optional[Var]":
         """Return the value current bound to the name `name_sym` in the namespace
         specified by `ns_sym`."""
         ns = Namespace.get_or_create(ns_sym)
         return ns.find(name_sym)
 
     @staticmethod
-    def find(ns_qualified_sym: sym.Symbol) -> "Var":
+    def find(ns_qualified_sym: sym.Symbol) -> "Optional[Var]":
         """Return the value currently bound to the name in the namespace specified
         by `ns_qualified_sym`."""
         ns = Maybe(ns_qualified_sym.ns).or_else_raise(
@@ -175,8 +192,9 @@ class Namespace:
 
     __slots__ = ('_name', '_module', '_mappings', '_refers', '_aliases', '_imports')
 
-    def __init__(self, name: sym.Symbol) -> None:
+    def __init__(self, name: sym.Symbol, module: types.ModuleType = None) -> None:
         self._name = name
+        self._module = Maybe(module).or_else(lambda: _new_module(name._as_python_sym))
         self._mappings: atom.Atom = atom.Atom(pmap())
         self._aliases: atom.Atom = atom.Atom(pmap())
         self._imports: atom.Atom = atom.Atom(pset(Namespace.DEFAULT_IMPORTS.deref()))
@@ -194,6 +212,18 @@ def add_default_import(cls, module: str):
     def name(self) -> str:
         return self._name.name
 
+    @property
+    def module(self):
+        return self._module
+
+    @module.setter
+    def module(self, m: types.ModuleType):
+        """Override the Python module for this Namespace.
+
+        This should only be done by basilisp.importer code to make sure the
+        correct module is generated for `basilisp.core`."""
+        self._module = m
+
     @property
     def aliases(self) -> PMap:
         return self._aliases.deref()
@@ -239,7 +269,7 @@ def _intern(m: PMap, sym: sym.Symbol, new_var: Var,
             return m.set(sym, new_var)
         return m
 
-    def find(self, sym: sym.Symbol) -> Var:
+    def find(self, sym: sym.Symbol) -> Optional[Var]:
         """Find Vars mapped by the given Symbol input or None if no Vars are
         mapped by that Symbol."""
         return self.mappings.get(sym, None)
@@ -273,25 +303,26 @@ def __import_core_mappings(ns_cache: PMap,
     @staticmethod
     def __get_or_create(ns_cache: PMap,
                         name: sym.Symbol,
+                        module: types.ModuleType = None,
                         core_ns_name=_CORE_NS) -> PMap:
         """Private swap function used by `get_or_create` to atomically swap
         the new namespace map into the global cache."""
         ns = ns_cache.get(name, None)
         if ns is not None:
             return ns_cache
-        new_ns = Namespace(name)
+        new_ns = Namespace(name, module=module)
         if name.name != core_ns_name:
             Namespace.__import_core_mappings(
                 ns_cache, new_ns, core_ns_name=core_ns_name)
         return ns_cache.set(name, new_ns)
 
     @classmethod
-    def get_or_create(cls, name: sym.Symbol) -> "Namespace":
+    def get_or_create(cls, name: sym.Symbol, module: types.ModuleType = None) -> "Namespace":
         """Get the namespace bound to the symbol `name` in the global namespace
         cache, creating it if it does not exist.
 
         Return the namespace."""
-        return cls._NAMESPACES.swap(Namespace.__get_or_create, name)[name]
+        return cls._NAMESPACES.swap(Namespace.__get_or_create, name, module=module)[name]
 
     @classmethod
     def remove(cls, name: sym.Symbol) -> Optional["Namespace"]:
@@ -412,12 +443,14 @@ def init_ns_var(which_ns: str = _CORE_NS,
 
 
 def set_current_ns(ns_name: str,
+                   module: types.ModuleType = None,
                    ns_var_name: str = _NS_VAR_NAME,
                    ns_var_ns: str = _NS_VAR_NS) -> Var:
     """Set the value of the dynamic variable `*ns*` in the current thread."""
     symbol = sym.Symbol(ns_name)
-    ns = Namespace.get_or_create(symbol)
-    ns_var = Var.find(sym.Symbol(ns_var_name, ns=ns_var_ns))
+    ns = Namespace.get_or_create(symbol, module=module)
+    ns_var = Maybe(Var.find(sym.Symbol(ns_var_name, ns=ns_var_ns))) \
+        .or_else_raise(lambda: RuntimeException(f"Dynamic Var {sym.Symbol(ns_var_name, ns=ns_var_ns)} not bound!"))
     ns_var.push_bindings(ns)
     return ns_var
 
@@ -426,7 +459,9 @@ def get_current_ns(ns_var_name: str = _NS_VAR_NAME,
                    ns_var_ns: str = _NS_VAR_NS) -> Namespace:
     """Get the value of the dynamic variable `*ns*` in the current thread."""
     ns_sym = sym.Symbol(ns_var_name, ns=ns_var_ns)
-    ns: Namespace = Var.find(ns_sym).value
+    ns: Namespace = Maybe(Var.find(ns_sym)) \
+        .map(lambda v: v.value) \
+        .or_else_raise(lambda: RuntimeException(f"Dynamic Var {ns_sym} not bound!"))
     return ns
 
 
@@ -451,7 +486,9 @@ def print_generated_python(var_name: str = _PRINT_GENERATED_PY_VAR_NAME,
                            core_ns_name: str = _CORE_NS) -> bool:
     """Return the value of the `*print-generated-python*` dynamic variable."""
     ns_sym = sym.Symbol(var_name, ns=core_ns_name)
-    return Var.find(ns_sym).value
+    return Maybe(Var.find(ns_sym)) \
+        .map(lambda v: v.value) \
+        .or_else_raise(lambda: RuntimeException(f"Dynamic Var {ns_sym} not bound!"))
 
 
 def bootstrap(ns_var_name: str = _NS_VAR_NAME,
@@ -460,13 +497,15 @@ def bootstrap(ns_var_name: str = _NS_VAR_NAME,
     express with the very minimal lisp environment."""
     core_ns_sym = sym.symbol(core_ns_name)
     ns_var_sym = sym.symbol(ns_var_name, ns=core_ns_name)
-    __NS = Var.find(ns_var_sym)
+    __NS = Maybe(Var.find(ns_var_sym)) \
+        .or_else_raise(lambda: RuntimeException(f"Dynamic Var {ns_var_sym} not bound!"))
 
     def set_BANG_(var_sym: sym.Symbol, expr):
         ns = Maybe(var_sym.ns).or_else(lambda: __NS.value.name)
         name = var_sym.name
 
-        v = Var.find(sym.symbol(name, ns=ns))
+        v = Maybe(Var.find(sym.symbol(name, ns=ns))) \
+            .or_else_raise(lambda: RuntimeException(f"Var {ns_var_sym} not bound!"))
         v.value = expr
         return expr
 

basilisp/lang/runtime.py

@@ -1,6 +1,7 @@
 from typing import Optional
 
 from basilisp.lang.meta import Meta
+from basilisp.lang.util import munge
 
 
 class Symbol(Meta):
@@ -27,6 +28,12 @@ def with_meta(self, meta) -> "Symbol":
         new_meta = meta if self._meta is None else self._meta.update(meta)
         return Symbol(self._name, self._ns, meta=new_meta)
 
+    @property
+    def _as_python_sym(self) -> str:
+        if self.ns is not None:
+            return f"{munge(self.ns)}.{munge(self.name)}"
+        return f"{munge(self.name)}"
+
     def __str__(self):
         if self._ns is not None:
             return "{ns}/{name}".format(ns=self._ns, name=self._name)

basilisp/lang/symbol.py

@@ -47,7 +47,7 @@ def lrepr(f) -> str:
 }
 
 
-def munge(s: str, allow_builtins: bool = True) -> str:
+def munge(s: str, allow_builtins: bool = False) -> str:
     """Replace characters which are not valid in Python symbols
     with valid replacement strings."""
     new_str = []

basilisp/lang/util.py

@@ -2,35 +2,37 @@
 # noinspection PyUnresolvedReferences
 import readline
 import traceback
+import types
 
 import basilisp.compiler as compiler
 import basilisp.importer as importer
 import basilisp.lang.runtime as runtime
 import basilisp.reader as reader
 
 
-def eval_file(filename: str, ctx: compiler.CompilerContext):
+def eval_file(filename: str, ctx: compiler.CompilerContext, module: types.ModuleType):
     """Evaluate a file with the given name into a Python module AST node."""
     last = None
     for form in reader.read_file(filename, resolver=runtime.resolve_alias):
-        last = compiler.compile_and_exec_form(form, ctx=ctx)
+        last = compiler.compile_and_exec_form(form, ctx, module, filename)
     return last
 
 
-def eval_str(s: str, ctx: compiler.CompilerContext):
+def eval_str(s: str, ctx: compiler.CompilerContext, module: types.ModuleType):
     """Evaluate the forms in a string into a Python module AST node."""
     last = None
     for form in reader.read_str(s, resolver=runtime.resolve_alias):
-        last = compiler.compile_and_exec_form(form, ctx=ctx)
+        last = compiler.compile_and_exec_form(form, ctx, module, source_filename='REPL Input')
     return last
 
 
 def repl(default_ns=runtime._REPL_DEFAULT_NS):
     ctx = compiler.CompilerContext()
     ns_var = runtime.set_current_ns(default_ns)
     while True:
+        ns: runtime.Namespace = ns_var.value
         try:
-            lsrc = input(f'{ns_var.value.name}=> ')
+            lsrc = input(f'{ns.name}=> ')
         except EOFError:
             break
         except KeyboardInterrupt:
@@ -41,7 +43,7 @@ def repl(default_ns=runtime._REPL_DEFAULT_NS):
             continue
 
         try:
-            print(compiler.lrepr(eval_str(lsrc, ctx)))
+            print(compiler.lrepr(eval_str(lsrc, ctx, ns.module)))
         except reader.SyntaxError as e:
             traceback.print_exception(reader.SyntaxError, e, e.__traceback__)
             continue