Fix/namespace aliasing behavior 2

CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+ * Added the `--emit-generated-python` CLI argument to control whether generated Python code strings are stored by the runtime for each compiled namespace (#1045)
+
+### Changed
+ * The compiler will issue a warning when adding any alias that might conflict with any other alias (#1045)
 
 ## [v0.2.2]
 ### Added

docs/runtime.rst

@@ -111,6 +111,31 @@ This is roughly analogous to the Java classpath in Clojure.
 These values may be set manually, but are more often configured by some project management tool such as Poetry or defined in your Python virtualenv.
 These values may also be set via :ref:`cli` arguments.
 
+.. _namespace_imports:
+
+Namespace Imports
+^^^^^^^^^^^^^^^^^
+
+Basilisp compiles Lisp code into Python code in Python modules exactly the same way the Python compiler does.
+The Python code compiled by the Basilisp compiler expects certain features to be available at runtime beyond the standard Python builtins.
+To support this, the Python modules compiled by Basilisp automatically import a number of modules both from the Basilisp runtime, the Python standard library, and Basilisp's Python dependencies.
+In Basilisp modules (particularly :lpy:ns:`basilisp.core`) you may find references to such modules without any corresponding :lpy:form:`import`.
+
+The modules imported by default are given below:
+
+- ``attr`` (from the `attrs <https://www.attrs.org/en/stable/>`_ project)
+- :external:py:mod:`builtins` (Basilisp users should prefer the ``python`` namespace for calling :ref:`python_builtins`)
+- :external:py:mod:`functools`
+- :external:py:mod:`io`
+- :external:py:mod:`importlib`
+- :external:py:mod:`operator`
+- :external:py:mod:`sys`
+- The majority of the modules in ``basilisp.lang.*``
+
+.. warning::
+
+   Using any of these names (particularly the Python standard library module names) as an alias for a required namespace or imported Python module will trigger a warning.
+
 .. _vars:
 
 Vars

src/basilisp/cli.py

@@ -307,6 +307,20 @@ def _add_debug_arg_group(parser: argparse.ArgumentParser) -> None:
             "(env: BASILISP_LOGGING_LEVEL; default: WARNING)"
         ),
     )
+    group.add_argument(
+        "--emit-generated-python",
+        action=_set_envvar_action(
+            "BASILISP_EMIT_GENERATED_PYTHON", parent=argparse._StoreAction
+        ),
+        nargs="?",
+        const=True,
+        type=_to_bool,
+        help=(
+            "if true, store generated Python code in `*generated-python*` dynamic "
+            "Vars within each namespace (env: BASILISP_EMIT_GENERATED_PYTHON; "
+            "default: true)"
+        ),
+    )
 
 
 def _add_import_arg_group(parser: argparse.ArgumentParser) -> None:

src/basilisp/core.lpy

@@ -6,8 +6,6 @@
          datetime
          decimal
          fractions
-         functools
-         importlib
          importlib.util
          math
          multiprocessing
@@ -20,10 +18,6 @@
          [time :as py-time]
          uuid)
 
-(import* attr)
-
-(import* basilisp.lang.multifn)
-
 (def ^{:doc      "Create a list from the arguments."
        :arglists '([& args])}
   list

src/basilisp/lang/compiler/__init__.py

@@ -139,7 +139,7 @@ def _emit_ast_string(
     # TODO: eventually, this default should become "false" but during this
     #       period of heavy development, having it set to "true" by default
     #       is tremendously useful
-    if os.getenv("BASILISP_EMIT_GENERATED_PYTHON", "true") != "true":
+    if os.getenv("BASILISP_EMIT_GENERATED_PYTHON", "true").lower() != "true":
         return
 
     if runtime.print_generated_python():

src/basilisp/lang/compiler/analyzer.py

@@ -3470,6 +3470,7 @@ def __resolve_namespaced_symbol_in_ns(
         if safe_name in vars(ns_module):
             return MaybeHostForm(
                 form=form,
+                class_original=ns_sym.name,
                 class_=munge(ns_sym.name),
                 field=safe_name,
                 target=vars(ns_module)[safe_name],
@@ -3485,6 +3486,7 @@ def __resolve_namespaced_symbol_in_ns(
         # don't need to care if this is an import or an alias.
         return MaybeHostForm(
             form=form,
+            class_original=ns_sym.name,
             class_=munge(ns_sym.name),
             field=safe_name,
             target=vars(ns_module)[safe_name],
@@ -3584,6 +3586,7 @@ def __resolve_namespaced_symbol(  # pylint: disable=too-many-branches  # noqa: M
             ctx.symbol_table.mark_used(maybe_import_or_require_sym)
             return MaybeHostForm(
                 form=form,
+                class_original=form.ns,
                 class_=munge(form.ns),
                 field=munge(form.name),
                 target=None,

src/basilisp/lang/compiler/constants.py

@@ -1,5 +1,6 @@
 from basilisp.lang import keyword as kw
 from basilisp.lang import symbol as sym
+from basilisp.lang.util import genname
 
 
 class SpecialForm:
@@ -32,6 +33,8 @@ class SpecialForm:
 
 DEFAULT_COMPILER_FILE_PATH = "NO_SOURCE_PATH"
 
+OPERATOR_ALIAS = genname("operator")
+
 SYM_ABSTRACT_META_KEY = kw.keyword("abstract")
 SYM_ABSTRACT_MEMBERS_META_KEY = kw.keyword("abstract-members")
 SYM_ASYNC_META_KEY = kw.keyword("async")

src/basilisp/lang/compiler/generator.py

@@ -1,9 +1,11 @@
 # pylint: disable=too-many-lines
 
 import ast
+import base64
 import collections
 import contextlib
 import functools
+import hashlib
 import logging
 import re
 import uuid
@@ -46,6 +48,7 @@
 from basilisp.lang.compiler.constants import (
     DEFAULT_COMPILER_FILE_PATH,
     INTERFACE_KW,
+    OPERATOR_ALIAS,
     REST_KW,
     SYM_DYNAMIC_META_KEY,
     SYM_REDEF_META_KEY,
@@ -684,6 +687,13 @@ def _var_ns_as_python_sym(name: str) -> str:
 #######################
 
 
+_ATTR_ALIAS = genname("attr")
+_BUILTINS_ALIAS = genname("builtins")
+_FUNCTOOLS_ALIAS = genname("functools")
+_IMPORTLIB_ALIAS = genname("importlib")
+_IO_ALIAS = genname("io")
+_SYS_ALIAS = genname("sys")
+
 _ATOM_ALIAS = genname("atom")
 _COMPILER_ALIAS = genname("compiler")
 _CORE_ALIAS = genname("core")
@@ -706,9 +716,17 @@ def _var_ns_as_python_sym(name: str) -> str:
 _VEC_ALIAS = genname("vec")
 _VOLATILE_ALIAS = genname("volatile")
 _VAR_ALIAS = genname("Var")
+_UNION_ALIAS = genname("Union")
 _UTIL_ALIAS = genname("langutil")
 
 _MODULE_ALIASES = {
+    "attr": _ATTR_ALIAS,
+    "builtins": _BUILTINS_ALIAS,
+    "functools": _FUNCTOOLS_ALIAS,
+    "importlib": _IMPORTLIB_ALIAS,
+    "io": _IO_ALIAS,
+    "operator": OPERATOR_ALIAS,
+    "sys": _SYS_ALIAS,
     "basilisp.lang.atom": _ATOM_ALIAS,
     "basilisp.lang.compiler": _COMPILER_ALIAS,
     "basilisp.core": _CORE_ALIAS,
@@ -732,6 +750,9 @@ def _var_ns_as_python_sym(name: str) -> str:
     "basilisp.lang.volatile": _VOLATILE_ALIAS,
     "basilisp.lang.util": _UTIL_ALIAS,
 }
+assert set(_MODULE_ALIASES.keys()).issuperset(
+    map(lambda s: s.name, runtime.Namespace.DEFAULT_IMPORTS)
+), "All default Namespace imports should have generator aliases"
 
 _NS_VAR_VALUE = f"{_NS_VAR}.value"
 
@@ -753,16 +774,16 @@ def _var_ns_as_python_sym(name: str) -> str:
 _INTERN_VAR_FN_NAME = _load_attr(f"{_VAR_ALIAS}.intern")
 _INTERN_UNBOUND_VAR_FN_NAME = _load_attr(f"{_VAR_ALIAS}.intern_unbound")
 _FIND_VAR_FN_NAME = _load_attr(f"{_VAR_ALIAS}.find_safe")
-_ATTR_CLASS_DECORATOR_NAME = _load_attr("attr.define")
-_ATTR_FROZEN_DECORATOR_NAME = _load_attr("attr.frozen")
-_ATTRIB_FIELD_FN_NAME = _load_attr("attr.field")
+_ATTR_CLASS_DECORATOR_NAME = _load_attr(f"{_ATTR_ALIAS}.define")
+_ATTR_FROZEN_DECORATOR_NAME = _load_attr(f"{_ATTR_ALIAS}.frozen")
+_ATTRIB_FIELD_FN_NAME = _load_attr(f"{_ATTR_ALIAS}.field")
 _COERCE_SEQ_FN_NAME = _load_attr(f"{_RUNTIME_ALIAS}.to_seq")
 _BASILISP_FN_FN_NAME = _load_attr(f"{_RUNTIME_ALIAS}._basilisp_fn")
 _FN_WITH_ATTRS_FN_NAME = _load_attr(f"{_RUNTIME_ALIAS}._with_attrs")
 _BASILISP_TYPE_FN_NAME = _load_attr(f"{_RUNTIME_ALIAS}._basilisp_type")
 _BASILISP_WITH_META_INTERFACE_NAME = _load_attr(f"{_INTERFACES_ALIAS}.IWithMeta")
-_BUILTINS_IMPORT_FN_NAME = _load_attr("builtins.__import__")
-_IMPORTLIB_IMPORT_MODULE_FN_NAME = _load_attr("importlib.import_module")
+_BUILTINS_IMPORT_FN_NAME = _load_attr(f"{_BUILTINS_ALIAS}.__import__")
+_IMPORTLIB_IMPORT_MODULE_FN_NAME = _load_attr(f"{_IMPORTLIB_ALIAS}.import_module")
 _LISP_FN_APPLY_KWARGS_FN_NAME = _load_attr(f"{_RUNTIME_ALIAS}._lisp_fn_apply_kwargs")
 _LISP_FN_COLLECT_KWARGS_FN_NAME = _load_attr(
     f"{_RUNTIME_ALIAS}._lisp_fn_collect_kwargs"
@@ -1964,7 +1985,7 @@ def fn(*args):
             ret_ann_deps.extend(ret_ann.dependencies)
         ret_ann_ast = (
             ast.Subscript(
-                value=ast.Name(id="Union", ctx=ast.Load()),
+                value=ast.Name(id=_UNION_ALIAS, ctx=ast.Load()),
                 slice=ast_index(ast.Tuple(elts=ret_ann_asts, ctx=ast.Load())),
                 ctx=ast.Load(),
             )
@@ -2256,6 +2277,34 @@ def _if_to_py_ast(ctx: GeneratorContext, node: If) -> GeneratedPyAST[ast.expr]:
     )
 
 
+_IMPORT_HASH_TRANSLATE_TABLE = str.maketrans({"=": "", "+": "", "/": ""})
+
+
+@functools.lru_cache
+def _import_hash(s: str) -> str:
+    """Generate a short, consistent, hash which can be appended to imported module
+    names to effectively separate them from objects of the same name defined in the
+    module.
+
+    Aliases in Clojure exist in a separate "namespace" from interned values, but
+    Basilisp generates Python modules (which are essentially just a single shared
+    namespace), so it is possible that imported module names could clash with `def`'ed
+    names.
+
+    Below, we generate a truncated URL-safe Base64 representation of the MD5 hash of
+    the input string (typically the first '.' delimited component of the potentially
+    qualified module name), removing any '-' characters since those are not safe for
+    Python identifiers.
+
+    The hash doesn't need to be cryptographically secure, but it does need to be
+    consistent across sessions such that when cached namespace modules are reloaded,
+    the new session can find objects generated by the session which generated the
+    cache file. Since we are not concerned with being able to round-trip this data,
+    destructive modifications are not an issue."""
+    digest = hashlib.md5(s.encode()).digest()
+    return base64.b64encode(digest).decode().translate(_IMPORT_HASH_TRANSLATE_TABLE)[:6]
+
+
 @_with_ast_loc_deps
 def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast.expr]:
     """Return a Python AST node for a Basilisp `import*` expression."""
@@ -2272,9 +2321,13 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast
         #   (import* collections collections.abc)
         if alias.alias is not None:
             py_import_alias = munge(alias.alias)
+            py_import_alias = f"{py_import_alias}_{_import_hash(py_import_alias)}"
+            full_import_name = py_import_alias
             import_func = _IMPORTLIB_IMPORT_MODULE_FN_NAME
         else:
-            py_import_alias = safe_name.split(".", maxsplit=1)[0]
+            py_import_alias, *submodules = safe_name.split(".", maxsplit=1)
+            py_import_alias = f"{py_import_alias}_{_import_hash(py_import_alias)}"
+            full_import_name = ".".join([py_import_alias, *submodules])
             import_func = _BUILTINS_IMPORT_FN_NAME
 
         ctx.symbol_table.context_boundary.new_symbol(
@@ -2307,6 +2360,11 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast
                                 keywords=[],
                             ),
                             last,
+                            ast.Call(
+                                func=_NEW_SYM_FN_NAME,
+                                args=[ast.Constant(full_import_name)],
+                                keywords=[],
+                            ),
                         ],
                         (
                             [
@@ -3261,14 +3319,20 @@ def _maybe_class_to_py_ast(
 
 @_with_ast_loc
 def _maybe_host_form_to_py_ast(
-    _: GeneratorContext, node: MaybeHostForm
+    ctx: GeneratorContext, node: MaybeHostForm
 ) -> GeneratedPyAST[ast.expr]:
     """Generate a Python AST node for accessing a potential Python module
     variable name with a namespace."""
     assert node.op == NodeOp.MAYBE_HOST_FORM
-    return GeneratedPyAST(
-        node=_load_attr(f"{_MODULE_ALIASES.get(node.class_, node.class_)}.{node.field}")
-    )
+    if (mod_name := _MODULE_ALIASES.get(node.class_)) is None:
+        current_ns = ctx.current_ns
+        mod_or_class = sym.symbol(node.class_original)
+        alias = current_ns.import_aliases.val_at(mod_or_class)
+        if (mod := current_ns.import_names.val_at(alias or mod_or_class)) is not None:
+            mod_name = mod.name
+    if mod_name is None:
+        mod_name = f"{node.class_}_{_import_hash(node.class_)}"
+    return GeneratedPyAST(node=_load_attr(f"{mod_name}.{node.field}"))
 
 
 #########################
@@ -3922,7 +3986,7 @@ def _from_module_imports() -> Iterable[ast.ImportFrom]:
         ast.ImportFrom(
             module="typing",
             names=[
-                ast.alias(name="Union", asname=None),
+                ast.alias(name="Union", asname=_UNION_ALIAS),
             ],
             level=0,
         ),

src/basilisp/lang/compiler/nodes.py

@@ -727,6 +727,7 @@ class MaybeClass(Node[sym.Symbol]):
 @attr.frozen
 class MaybeHostForm(Node[sym.Symbol]):
     form: sym.Symbol
+    class_original: str
     class_: str
     field: str
     target: Any

src/basilisp/lang/compiler/optimizer.py

@@ -4,6 +4,7 @@
 from contextlib import contextmanager
 from typing import Deque, Iterable, List, Optional, Set
 
+from basilisp.lang.compiler.constants import OPERATOR_ALIAS
 from basilisp.lang.compiler.utils import ast_FunctionDef, ast_index
 
 
@@ -43,7 +44,7 @@ def _optimize_operator_call_attr(  # pylint: disable=too-many-return-statements
     Using Python operators directly will allow for more direct bytecode to be
     emitted by the Python compiler and take advantage of any additional performance
     improvements in future versions of Python."""
-    if isinstance(fn.value, ast.Name) and fn.value.id == "operator":
+    if isinstance(fn.value, ast.Name) and fn.value.id == OPERATOR_ALIAS:
         binop = {
             "add": ast.Add,
             "and_": ast.BitAnd,