diff --git a/CHANGELOG.md b/CHANGELOG.md index de6965d5..9a45f425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +* Added support for referring imported Python names as by `from ... import ...` (#1154) ## [v0.3.8] ### Added @@ -18,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Fix an issue where consecutive reader comment forms would not be ignored (#1207) ## [v0.3.7] -### Fixed +### Fixed * Fix a regression introduced in #1176 where the testrunner couldn't handle relative paths in `sys.path`, causing `basilisp test` to fail when no arugments were provided (#1204) * Fix a bug where `basilisp.process/exec` could deadlock reading process output if that output exceeded the buffer size (#1202) * Fix `basilisp boostrap` issue on MS-Windows where the boostrap file loaded too early, before Basilisp was in `sys.path` (#1208) diff --git a/docs/pyinterop.rst b/docs/pyinterop.rst index b96b38b6..09de1554 100644 --- a/docs/pyinterop.rst +++ b/docs/pyinterop.rst @@ -49,13 +49,39 @@ Submodules will be available under the full, dot-separated name. To avoid name clashes from the above, you may alias imports (as in native Python code) using the same syntax as ``require``. Both top-level modules and submodules may be aliased: ``(import [module.sub :as sm])``. -Note that none of the other convenience features or flags from :lpy:fn:`require` are available, so you will not be able to, say, refer unqualified module members into the current Namespace. .. code-block:: (import [os.path :as path]) (path/exists "test.txt") ;;=> false +As with Basilisp ``refers`` (and as in Python), it is possible to refer individual module members by name into the current namespace using the ``:refer`` option. +It is also possible to refer all module members into the namespace using ``:refer :all``. + +.. code-block:: + + (import [math :refer [sqrt pi]]) + pi ;; 3.141592653589793 + + (import [statistics :refer :all]) + mean ;; + +.. warning:: + + Basilisp refers names into the current module in different conceptual namespaces and resolves names against those namespaces in order of precedence, preferring Basilisp members first. + Referred Python module members may not resolve if other names take precedence within the current namespace context. + + .. code-block:: + + (import [datetime :as dt :refer :all]) + + ;; This name using the module alias directly will guarantee we are referencing + ;; the module member `datetime.time` (a class) + dt/time ;; + + ;; ...whereas this reference prefers the `basilisp.core` function `time` + time ;; + .. note:: Users should generally prefer to use the :lpy:fn:`ns` macro for importing modules into their namespace, rather than using the :lpy:fn:`import` form directly. @@ -65,14 +91,9 @@ Note that none of the other convenience features or flags from :lpy:fn:`require` (ns myproject.ns (:import [os.path :as path])) -.. warning:: - - Unlike in Python, imported module names and aliases cannot be referred to directly in Basilisp code. - Module and Namespace names are resolved separately from local names and will not resolve as unqualified names. - .. seealso:: - :lpy:form:`import`, :lpy:fn:`import`, :lpy:fn:`ns-imports`, :lpy:fn:`ns-map` + :lpy:form:`import`, :lpy:fn:`import`, :lpy:fn:`ns-imports`, :lpy:fn:`ns-import-refers`, :lpy:fn:`ns-map` .. seealso:: diff --git a/src/basilisp/core.lpy b/src/basilisp/core.lpy index ce481d15..5f4e1b41 100644 --- a/src/basilisp/core.lpy +++ b/src/basilisp/core.lpy @@ -4915,6 +4915,11 @@ [ns] (.-imports (the-ns ns))) +(defn ^:inline ns-import-refers + "Return a set of Python module members which are referred in the current namespace." + [ns] + (.-import-refers (the-ns ns))) + (defn ^:inline ns-interns "Return a map of symbols to Vars which are interned in the current namespace." [ns] @@ -4950,14 +4955,26 @@ (defn ns-map "Return a map of all the mapped symbols in the namespace. - Includes the return values of :lpy:fn:`ns-interns` and :lpy:fn:`ns-refers` in one - map." + Includes the return values of :lpy:fn:`ns-interns`, :lpy:fn:`ns-refers`, and + :lpy:fn:`ns-imports` in one map. + + .. note:: + + It is possible that the same symbol is mapped to an interned name, a referred + name, an imported name, or an import-referred name. They are stored in separate + collections in each namespace and Basilisp symbol resolution rules determine the + precedence. This function returns the combined set of all mapped symbols in a + single map and therefore symbols defined in multiple maps will only show one + value. Inspecting the contents of each of the component maps will show all + mappings, including those that were overwritten by the merge operation in this + function." ([] (ns-map *ns*)) ([ns] (let [resolved-ns (the-ns ns)] (merge - (ns-interns resolved-ns) - (ns-refers resolved-ns))))) + (ns-imports resolved-ns) + (ns-refers resolved-ns) + (ns-interns resolved-ns))))) (defn ^:inline ns-resolve "Return the Var which will be resolved by the symbol in the given namespace." @@ -4974,7 +4991,13 @@ "Import Python modules by name. Modules may be specified either as symbols naming the full module path or as a - vector taking the form ``[full.module.path :as alias]``\\. + vector taking the form ``[full.module.path & opts]``\\. The ``opts`` should be + pairs of a keyword and a value from below, similar to :lpy:fn:`import`: + + - ``:as name`` which will alias the imported module to the symbol name + - ``:refer [& syms]`` which will refer module members in the local namespace + directly + - ``:refer :all`` which will refer all module members from the namespace directly Note that unlike in Python, ``import`` ed Python module names are always hoisted to the current Namespace, so imported names will be available within a Namespace even diff --git a/src/basilisp/lang/compiler/analyzer.py b/src/basilisp/lang/compiler/analyzer.py index 9eb2e01b..e52a5180 100644 --- a/src/basilisp/lang/compiler/analyzer.py +++ b/src/basilisp/lang/compiler/analyzer.py @@ -175,9 +175,11 @@ FINALLY = kw.keyword("finally") # Constants used in analyzing +ALL = kw.keyword("all") AS = kw.keyword("as") IMPLEMENTS = kw.keyword("implements") INTERFACE = kw.keyword("interface") +REFER = kw.keyword("refer") STAR_STAR = sym.symbol("**") _DOUBLE_DOT_MACRO_NAME = ".." _BUILTINS_NS = "python" @@ -2571,8 +2573,8 @@ def _do_warn_on_import_or_require_name_clash( def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import: assert form.first == SpecialForm.IMPORT - aliases = [] - for f in form.rest: + aliases, refers, refer_all = [], [], False + for f in form.rest: # pylint: disable=too-many-nested-blocks if isinstance(f, sym.Symbol): module_name = f module_alias = None @@ -2588,40 +2590,85 @@ def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import: symbol_table=ctx.symbol_table.context_boundary, ) elif isinstance(f, vec.PersistentVector): - if len(f) != 3: + if len(f) < 1: raise ctx.AnalyzerException( - "import alias must take the form: [module :as alias]", form=f + "import alias must take the form: [module :as alias :refer [...]]", + form=f, ) module_name = f.val_at(0) # type: ignore[assignment] if not isinstance(module_name, sym.Symbol): raise ctx.AnalyzerException( "Python module name must be a symbol", form=f ) - if not AS == f.val_at(1): + + try: + opts = lmap.hash_map(*f[1:]) + except IndexError: raise ctx.AnalyzerException( - "expected :as alias for Python import", form=f - ) - module_alias_sym = f.val_at(2) - if not isinstance(module_alias_sym, sym.Symbol): + "Expected options: ':as alias' or ':refer [...]'", form=f + ) from None + + if not {AS, REFER}.issuperset(set(opts.keys())): raise ctx.AnalyzerException( - "Python module alias must be a symbol", form=f + f"Unexpected import options: {lset.set(opts.keys())}", form=f ) - module_alias = module_alias_sym.name - if "." in module_alias: - raise ctx.AnalyzerException( - "Python module alias must not contain '.'", form=f + + if (module_alias_sym := opts.val_at(AS)) is not None: + if not isinstance(module_alias_sym, sym.Symbol): + raise ctx.AnalyzerException( + "Python module alias must be a symbol", form=f + ) + module_alias = module_alias_sym.name + if "." in module_alias: + raise ctx.AnalyzerException( + "Python module alias must not contain '.'", form=f + ) + + ctx.put_new_symbol( + module_alias_sym, + Binding( + form=module_alias_sym, + name=module_alias, + local=LocalType.IMPORT, + env=ctx.get_node_env(), + ), + symbol_table=ctx.symbol_table.context_boundary, ) + else: + module_alias = module_name.name - ctx.put_new_symbol( - module_alias_sym, - Binding( - form=module_alias_sym, - name=module_alias, - local=LocalType.IMPORT, - env=ctx.get_node_env(), - ), - symbol_table=ctx.symbol_table.context_boundary, - ) + if (module_refers := opts.val_at(REFER)) is not None: + if ALL == module_refers: + refer_all = True + else: + if not isinstance(module_refers, vec.PersistentVector): + raise ctx.AnalyzerException( + "Python module refers must be a vector of symbols", + form=module_refers, + ) + + if len(module_refers) == 0: + raise ctx.AnalyzerException( + "Must refer at least one name", form=module_refers + ) + + for refer in module_refers: + if not isinstance(refer, sym.Symbol): + raise ctx.AnalyzerException( + "Python module refer name must be a symbol", form=refer + ) + refers.append(refer.name) + + ctx.put_new_symbol( + refer, + Binding( + form=refer, + name=refer.name, + local=LocalType.IMPORT, + env=ctx.get_node_env(), + ), + symbol_table=ctx.symbol_table.context_boundary, + ) else: raise ctx.AnalyzerException("symbol or vector expected for import*", form=f) @@ -2643,6 +2690,8 @@ def _import_ast(form: ISeq, ctx: AnalyzerContext) -> Import: return Import( form=form, aliases=aliases, + refers=refers, + refer_all=refer_all, env=ctx.get_node_env(pos=ctx.syntax_position), ) @@ -3738,7 +3787,7 @@ def __resolve_namespaced_symbol( # pylint: disable=too-many-branches def __resolve_bare_symbol( ctx: AnalyzerContext, form: sym.Symbol -) -> Union[Const, MaybeClass, VarRef]: +) -> Union[Const, HostField, MaybeClass, VarRef]: """Resolve a non-namespaced symbol into a Python name or a local Basilisp Var.""" assert form.ns is None @@ -3767,6 +3816,25 @@ def __resolve_bare_symbol( env=ctx.get_node_env(pos=ctx.syntax_position), ) + maybe_import_refer_module = current_ns.get_import_refer(form) + if maybe_import_refer_module is not None: + refer_module = current_ns.get_import(maybe_import_refer_module) + # For referred imports, we want to generate a fully qualified reference + # to the object, so we don't have to pollute the module with more names. + # The user won't know the difference. + return HostField( + form=form, + field=munge(form.name), + target=MaybeClass( + form=maybe_import_refer_module, + class_=munge(maybe_import_refer_module.name), + target=refer_module, + env=ctx.get_node_env(pos=ctx.syntax_position), + ), + is_assignable=False, + env=ctx.get_node_env(pos=ctx.syntax_position), + ) + # Allow users to resolve imported module names directly maybe_import = current_ns.get_import(form) if maybe_import is not None: diff --git a/src/basilisp/lang/compiler/generator.py b/src/basilisp/lang/compiler/generator.py index ce7b919d..6581041e 100644 --- a/src/basilisp/lang/compiler/generator.py +++ b/src/basilisp/lang/compiler/generator.py @@ -2409,6 +2409,62 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast ), ) ) + + refers: Optional[ast.expr] = None + if node.refer_all: + key, val = genname("k"), genname("v") + refers = ast.DictComp( + key=ast.Call( + func=_NEW_SYM_FN_NAME, + args=[ast.Name(id=key, ctx=ast.Load())], + keywords=[], + ), + value=ast.Name(id=val, ctx=ast.Load()), + generators=[ + ast.comprehension( + target=ast.Tuple( + elts=[ + ast.Name(id=key, ctx=ast.Store()), + ast.Name(id=val, ctx=ast.Store()), + ], + ctx=ast.Store(), + ), + iter=ast.Call( + func=ast.Attribute( + value=ast.Call( + func=ast.Name(id="vars", ctx=ast.Load()), + args=[ast.Name(id=py_import_alias, ctx=ast.Load())], + keywords=[], + ), + attr="items", + ctx=ast.Load(), + ), + args=[], + keywords=[], + ), + ifs=[], + is_async=0, + ) + ], + ) + elif node.refers: + refer_keys: list[Optional[ast.expr]] = [] + refer_vals: list[ast.expr] = [] + for refer in node.refers: + refer_keys.append( + ast.Call( + func=_NEW_SYM_FN_NAME, args=[ast.Constant(refer)], keywords=[] + ) + ) + refer_vals.append( + ast.Attribute( + value=ast.Name(id=py_import_alias, ctx=ast.Load()), + attr=refer, + ctx=ast.Load(), + ) + ) + refers = ast.Dict(keys=refer_keys, values=refer_vals) + last = ast.Name(id=py_import_alias, ctx=ast.Load()) deps.append( @@ -2437,7 +2493,11 @@ def _import_to_py_ast(ctx: GeneratorContext, node: Import) -> GeneratedPyAST[ast ), ) ), - keywords=[], + keywords=( + [ast.keyword(arg="refers", value=refers)] + if refers is not None + else [] + ), ) ) diff --git a/src/basilisp/lang/compiler/nodes.py b/src/basilisp/lang/compiler/nodes.py index aee5a44e..8d14ab18 100644 --- a/src/basilisp/lang/compiler/nodes.py +++ b/src/basilisp/lang/compiler/nodes.py @@ -612,6 +612,8 @@ class If(Node[SpecialForm]): class Import(Node[SpecialForm]): form: SpecialForm aliases: Iterable["ImportAlias"] + refers: Iterable[str] + refer_all: bool env: NodeEnv = attr.field(hash=False) children: Sequence[kw.Keyword] = vec.EMPTY op: NodeOp = NodeOp.IMPORT diff --git a/src/basilisp/lang/runtime.py b/src/basilisp/lang/runtime.py index 189513b9..9630a0b2 100644 --- a/src/basilisp/lang/runtime.py +++ b/src/basilisp/lang/runtime.py @@ -21,6 +21,8 @@ from fractions import Fraction from typing import AbstractSet, Any, Callable, NoReturn, Optional, TypeVar, Union, cast +import attr + from basilisp.lang import keyword as kw from basilisp.lang import list as llist from basilisp.lang import map as lmap @@ -483,13 +485,23 @@ def pop_bindings(self) -> Frame: _THREAD_BINDINGS = _ThreadBindings() +@attr.frozen +class ImportRefer: + module_name: sym.Symbol + value: Any + + AliasMap = lmap.PersistentMap[sym.Symbol, sym.Symbol] +ImportReferMap = lmap.PersistentMap[sym.Symbol, Any] Module = Union[BasilispModule, types.ModuleType] ModuleMap = lmap.PersistentMap[sym.Symbol, Module] NamespaceMap = lmap.PersistentMap[sym.Symbol, "Namespace"] VarMap = lmap.PersistentMap[sym.Symbol, Var] +_PRIVATE_NAME_PATTERN = re.compile(r"(_\w*|__\w+__)") + + class Namespace(ReferenceBase): """Namespaces serve as organizational units in Basilisp code, just as they do in Clojure code. @@ -576,6 +588,7 @@ class Namespace(ReferenceBase): "_aliases", "_imports", "_import_aliases", + "_import_refers", ) def __init__( @@ -597,6 +610,7 @@ def __init__( ) ) self._import_aliases: AliasMap = lmap.EMPTY + self._import_refers: lmap.PersistentMap[sym.Symbol, ImportRefer] = lmap.EMPTY self._interns: VarMap = lmap.EMPTY self._refers: VarMap = lmap.EMPTY @@ -637,6 +651,12 @@ def import_aliases(self) -> AliasMap: with self._lock: return self._import_aliases + @property + def import_refers(self) -> ImportReferMap: + """A mapping of a symbolic alias and a Python object from an imported module.""" + with self._lock: + return lmap.map({name: v.value for name, v in self._import_refers.items()}) + @property def interns(self) -> VarMap: """A mapping between a symbolic name and a Var. The Var may point to @@ -765,17 +785,45 @@ def find(self, sym: sym.Symbol) -> Optional[Var]: return self._refers.val_at(sym, None) return v - def add_import(self, sym: sym.Symbol, module: Module, *aliases: sym.Symbol) -> None: - """Add the Symbol as an imported Symbol in this Namespace. If aliases are given, - the aliases will be applied to the""" + def add_import( + self, + sym: sym.Symbol, + module: Module, + *aliases: sym.Symbol, + refers: Optional[dict[sym.Symbol, Any]] = None, + ) -> None: + """Add the Symbol as an imported Symbol in this Namespace. + + If aliases are given, the aliases will be associated to the module as well. + + If a dictionary of refers is provided, add the referred names as import refers. + """ with self._lock: self._imports = self._imports.assoc(sym, module) + + if refers: + final_refers = self._import_refers + for s, v in refers.items(): + # Filter out dunder names and private names + if _PRIVATE_NAME_PATTERN.fullmatch(s.name) is not None: + logger.debug(f"Ignoring import refer for {sym} member {s}") + continue + final_refers = final_refers.assoc(s, ImportRefer(sym, v)) + self._import_refers = final_refers + if aliases: m = self._import_aliases for alias in aliases: m = m.assoc(alias, sym) self._import_aliases = m + def get_import_refer(self, sym: sym.Symbol) -> Optional[sym.Symbol]: + """Get the Python module member name referred by Symbol or None if it does not + exist.""" + with self._lock: + refer = self._import_refers.val_at(sym, None) + return refer.module_name if refer is not None else None + def get_import(self, sym: sym.Symbol) -> Optional[BasilispModule]: """Return the module if a module named by sym has been imported into this Namespace, None otherwise. @@ -963,13 +1011,16 @@ def is_match(entry: tuple[sym.Symbol, Var]) -> bool: ) def __complete_refers(self, value: str) -> Iterable[str]: - """Return an iterable of possible completions matching the given - prefix from the list of referred Vars.""" + """Return an iterable of possible completions matching the given prefix from + the list of referred Vars and referred Python module members.""" return map( lambda entry: f"{entry[0].name}", filter( Namespace.__completion_matcher(value), - ((s, v) for s, v in self.refers.items()), + itertools.chain( + ((s, v) for s, v in self.refers.items()), + ((s, v) for s, v in self.import_refers.items()), + ), ), ) diff --git a/tests/basilisp/compiler_test.py b/tests/basilisp/compiler_test.py index 996871fc..b6b1397d 100644 --- a/tests/basilisp/compiler_test.py +++ b/tests/basilisp/compiler_test.py @@ -3474,6 +3474,7 @@ def test_import_alias_may_not_contain_dot(self, lcompile: CompileFn): @pytest.mark.parametrize( "code", [ + "(import* [])", "(import* [:time :as py-time])", "(import* [time py-time])", "(import* [time :as :py-time])", @@ -3486,6 +3487,51 @@ def test_import_aliased_module_format(self, lcompile: CompileFn, code: str): with pytest.raises(compiler.CompilerException): lcompile(code) + @pytest.mark.parametrize( + "code", + [ + "(import* [math :refer ()])", + "(import* [math :refer {}])", + "(import* [math :refer #{}])", + ], + ) + def test_import_refer_name_collection_must_be_a_vector( + self, lcompile: CompileFn, code: str + ): + with pytest.raises(compiler.CompilerException): + lcompile(code) + + def test_import_refer_names_must_have_one_name(self, lcompile: CompileFn): + with pytest.raises(compiler.CompilerException): + lcompile("(import* [math :refer []])") + + @pytest.mark.parametrize( + "code", ["(import* [math :refer [:sqrt]])", '(import* [math :refer ["sqrt"]])'] + ) + def test_import_refer_names_must_by_symbols(self, lcompile: CompileFn, code: str): + with pytest.raises(compiler.CompilerException): + lcompile(code) + + def test_import_refer_multiple_names(self, lcompile: CompileFn): + import math + + lcompile("(import* [math :refer [pi sqrt]])") + refers = lcompile("[pi sqrt]") + assert refers[0] == math.pi + assert refers[1] == math.sqrt + + def test_import_refer_all(self, lcompile: CompileFn, ns: runtime.Namespace): + import math + + lcompile("(import* [math :refer :all])") + + import_refers = {name: val for name, val in ns.import_refers.items()} + assert import_refers == { + sym.symbol(name): val + for name, val in vars(math).items() + if not runtime._PRIVATE_NAME_PATTERN.fullmatch(name) + } + def test_import_module_must_exist(self, lcompile: CompileFn): with pytest.raises(ImportError): lcompile("(import* real.fake.module)") diff --git a/tests/basilisp/namespace_test.py b/tests/basilisp/namespace_test.py index ec487b78..c9496e59 100644 --- a/tests/basilisp/namespace_test.py +++ b/tests/basilisp/namespace_test.py @@ -327,7 +327,12 @@ def ns(self) -> Namespace: time_sym = sym.symbol("time") time_alias = sym.symbol("py-time") - ns.add_import(time_sym, __import__("time"), time_alias) + ns.add_import( + time_sym, + __import__("time"), + time_alias, + refers={sym.symbol("sleep"): __import__("time").sleep}, + ) core_ns = Namespace(sym.symbol("basilisp.core")) map_alias = sym.symbol("map") @@ -341,6 +346,7 @@ def test_ns_completion(self, ns: Namespace): assert {"str/", "string?", "str"} == set(ns.complete("st")) assert {"map"} == set(ns.complete("m")) assert {"map"} == set(ns.complete("ma")) + assert {"sleep"} == set(ns.complete("sl")) def test_import_and_alias(self, ns: Namespace): assert {"time/"} == set(ns.complete("ti"))