diff --git a/formal/name-resolution.md b/formal/name-resolution.md new file mode 100644 index 0000000..941bc42 --- /dev/null +++ b/formal/name-resolution.md @@ -0,0 +1,11 @@ +# Python Name Resolution Rules + +This is intended to be a fuller version of the rules found in the [Python Language Reference](https://docs.python.org/3.10/reference/executionmodel.html#resolution-of-names) section 4.2.2 + +1. + +2. + +* +* +* diff --git a/formal/namespaces.py b/formal/namespaces.py new file mode 100644 index 0000000..bcaf06f --- /dev/null +++ b/formal/namespaces.py @@ -0,0 +1,344 @@ +""" Emulation for the runtime concept of a namespace. + +Namespaces form a tree, starting with RootNamespace, which has a RootScope. +Under that is a GlobalNamespace, which has a GlobalScope. + +There are two ways to build the tree up to this point. +1. Make a RootScope, make a RootNamespace with this scope. + Call main namespace.add_module(optional name, key, etc.). + This makes a GlobalScope and returns the new GlobalNamespace. +2. Make a GlobalScope, which will create a RootScope as a parent. + Call GlobalNamespace(this scope, optional key). + +""" + +from __future__ import annotations +from typing import * +from abc import * + +from scopes import * + +NsT = TypeVar('NsT', bound='Namespace') +ValT = TypeVar('ValT') +BuildT = Callable[[NsT, ScopeT, RefT, Iterator[ScopeT]], None] + +def null_builder(space: NsT, scope: ScopeT, ref: RefT, nested: Iterator[ScopeT]): pass + +class Namespace(Generic[RefT, ValT]): + """ Abstract base class. + Able to bind, rebind, or unbind identifiers. + Associated with a Scope object. + Part of a tree, where every Namespace other than the RootNamespace has a parent. + A Namespace doesn't necessarily keep track of its children. The client has the + option of making an index of some or all of the children it creates. + + Namespaces are related to Scopes, which are regions of a python source. + A Namespace has an associated Scope object. + Scopes also form a tree, and the structure of the Namespace tree matches that of + the Scope tree. That is, + + RootNamespace RootScope + ^ ^ + ... ... + Namespace --> Scope + ^ ^ + | parent | parent + Namespace --> Scope + + Variables and Bindings. + + Every Variable (or "var") is an occurrence of an identifier in the python source. + It is is found in some particular Scope. + As the program runs, the value of that Var can be asigned to it or + removed from it. In the latter case, the Var is "unbound". + + In the scopes.py module, the concept of "binding scope" is discussed. For any var which + appears in a "current scope", the binding scope is some enclosing Scope, possibly the + same as the current scope. The Namespace associated with the binding scope is known as + the "binding namespace", and this namespace keeps track of the current value, if any, + of that var. + + The Var is always Local in the binding scope. + + Now, a Binding is simply a container which is either "bound" or "unbound". + If it is bound, it has a value (any python object). + (Note, an unbound Binding is NOT the same as a having value of None). + The attribute Binding.value can be gotten, set, or deleted. If it is unbound, then + get and delete will raise AttributeError. bool(Binding) is True if bound, False if unbound. + + The Binding for the Var is stored in its binding namespace. This is a runtime concept, not compile time. + + A Namespace contains a Binding for every Var which is local to its own Scope. + + The procedure for finding the binding namespace for a var from the current namespace is to go up the + parent chain until finding the namespace whose scope is the binding scope. See this diagram: + + binding namespace --> binding scope + ^ ^ + | 0 or more parents | binding_scope(Var) + current namespace --> current scope + + The current Namespace can get the value (or raise an exception) for a Var known to the + current scope by delegating this operation to the binding namespace. + This step is implemented differently in different types of namespaces (corresponding to + different types of scopes). + + In a closed namespace (i.e. a function): + The value, or the exception, is obtained from the Binding for the Var. + If the Binding is bound, return its value. + Else the exception will be UnboundLocalError if the current namespace is the binding namespace, + otherwise it will be NameError. + + In an open namespace (i.e. a class): + This will always be the current namespace as well, because python's scope resolution + prevents it from being the binding namespace for something else. + If the Binding is bound, return the value. + Otherwise, the operation is delegated to global Namespace. + + In the global namespace (i.e. a module): + If the Binding is bound, return the value. + Otherwise, the operation is delegated to the main namespace. + + In the main namespace (i.e. the entire program): + There is a mapping of builtin names to their values, taken from the program-wide builtins module. + An alternate mapping can be provided to the main namespace constructor. + This mapping is read-only and the values are always bound. + If there a Binding for the Var, then it returns its value. + Otherwise, it raises NameError. + + The current namespace sets the value of a Var, or unbinds the Var, by delegating this operation + to the binding namespace. It is never further delegated to a different namespace. + In the binding namespace, the value of the binding for the Var is bound or unbound, respectively. + + Building a Namespace tree: + + This is a recursive operation, starting from a RootNamespace. It uses a builder function, which + will be provided by the client in constructing the GlobalNamespace. The same builder can be used + for all branches of the tree, but the client may also specify a different builder for a given branch. + + The builder function is called to process a namespace. For convenience, it is also given: + the reference object contained in the scope, and + the nested scopes. These are in the form of an iterator, so that the builder can get + the nested scopes one at a time without needing a 'for' loop. + An example would be a builderwhich traverses a syntax tree for the scope. + Whenever it visits something which creates a nested scope, it can get the corresponding + Scope object (assuming that the Scopes tree was build in the same order, + such as by traversing the same syntax tree). + + The Namespace.nest() method does the recursive build of a nested namespace. It is given: + one of the nested scopes (or the iterator from which it gets the next scope), + an optional key for indexing the new nested namespace in the current namespace, and + an optional builder to use instead of self. + The nested namespace is created, and the builder.build() method is called for it immediately. + This is in contrast to building Scopes, where the build of a nested scope is delayed. + + """ + scope: Scope + parent: Namespace | None + vars: Mapping[str, Binding[ValT]] + # Optional place where client can find nested namespaces by name or some other key. + nested: Mapping[object, Namespace] + scope_class: ClassVar[Type[Scope]] + builder: BuildT + global_ns: GlobalNamespace | None + + def __init__(self, scope: Scope, parent: Namespace | None, *, + build: BuildT = null_builder, key: object = None): + assert isinstance(scope, self.scope_class) + self.scope = scope + self.parent = parent + if parent: + assert scope.parent is parent.scope + if key is not None: + parent.nested[key] = self + self.global_ns = parent.global_ns + else: + assert scope.parent is None + self.global_ns = None + + # Create bindings for local names in the scope. + self.vars = {} + for var in scope.vars: + self.vars[var] = Binding() + self.builder = build + self.nested = {} + + def build(self): + """ Builds entire tree, using the builder recursively. """ + self.builder(self, self.scope, self.scope.ref, iter(self.scope.nested)) + + # Methods called by the builder... + + def load(self, var: str) -> ValT: + binding_ns = self._binding_namespace(var) + binding = binding_ns._load_binding(var) + if binding: return binding.value + if self is binding_ns: + raise UnboundLocalError(f"local variable '{var}' referenced before assignment") + else: + raise NameError(f"name '{var}' is not defined") + + def has(self, var: str) -> bool: + """ True if there is a Binding for Var and the Binding is bound. """ + return bool(self._binding_namespace(var)._load_binding(var)) + + def store(self, var: str, value: ValT) -> None: + self._binding_namespace(var).vars[var].bind(value) + + # Same as store(), except in ComprehensionNamespace + store_walrus = store + + def delete(self, var: str) -> None: + if self.has(var): + self._binding_namespace(var).vars[var].unbind() + return + # This will raise the appropriate exception. + self.load(var) + + def nest(self, nested: ScopeT | Iterator[ScopeT], **kwds) -> NsT: + """ Create a nested Namespace using a nested Scope. """ + if not isinstance(nested, Scope): + nested = next(nested) + newspace = self._nest_scope(nested, **kwds) + newspace.builder(newspace, nested, nested.ref, iter(nested.nested)) + return newspace + + # Helper methods... + + def _binding_namespace(self, var: str) -> Namespace: + scope: Scope = self.scope.vars[var] + while True: + if scope is self.scope: return self + self = self.parent + + @abstractmethod + def _load_binding(self, var: str) -> Binding | None: + """ Find the Binding, if any, containing the value of Var. + self is a binding namespace for Var, but the result might not always + be the binding stored here. + The main namespace is not a binding namespace, and is handled differently. + """ + ... + + def _nest_scope(self, scope: ScopeT, build: BuildT = None, **kwds) -> NsT: + """ Create a nested Namespace for given Scope. """ + cls: Type[NsT] = scope_to_ns.get(type(scope)) + if not cls: + raise TypeError("Scope must be a standard Scope subclass, not 'type(scope).__name__'") + return cls(scope, self, build=build or self.builder, **kwds) + + +class RootNamespace(Namespace): + """ The environment for a program and its modules. + Includes bindings for the builtins module. + """ + scope_class = RootScope + + def __init__(self, scope: RootScope = None): + super().__init__(scope or RootScope(), None) + + def _load_binding(self, var: str) -> Binding | None: + """ Find the Binding, if any, containing the value of Var. + self is not a binding namespace. There might or might not be a Binding for var. + """ + return self.vars.get(var) + + def add_module(self, name: str = '', key: object = None) -> GlobalNamespace: + """ Create a nested GlobalNamespace, using a new GlobalScope. """ + return self._nest_scope(GlobalScope(name, parent=self.scope), key=key) + +class GlobalNamespace(Namespace): + scope_class = GlobalScope + + def __init__(self, scope: GlobalScope | None, parent: RootNamespace = None, **kwds): + if not scope: + scope = GlobalScope() + super().__init__(scope, parent or RootNamespace(scope.parent), **kwds) + self.global_ns = self + + def _load_binding(self, var: str) -> Binding | None: + """ Find the Binding, if any, containing the value of Var. + """ + binding = self.vars.get(var) + if binding: return binding # Binding exists and is bound. + # Else try the main namespace. + return self.parent._load_binding(var) + +class ClassNamespace(Namespace): + scope_class = ClassScope + + def _load_binding(self, var: str) -> Binding | None: + """ Find the Binding, if any, containing the value of var. + """ + binding = self.vars[var] + if binding: return binding + # Else try the global namespace. + return self.global_ns._load_binding(var) + +class FunctionNamespace(Namespace): + scope_class = FunctionScope + + def _load_binding(self, var: str) -> Binding | None: + """ Find the Binding, if any, containing the value of var. + """ + return self.vars.get(var) + +class LambdaNamespace(FunctionNamespace): + scope_class = LambdaScope + +class ComprehensionNamespace(FunctionNamespace): + scope_class = ComprehensionScope + + def store_walrus(self, var: str, value: ValT) -> None: + self.parent.store(var, value) + +scope_to_ns: Mapping[Scope, Namespace] = {} + +for Ns in ( + RootNamespace, + GlobalNamespace, + ClassNamespace, + FunctionNamespace, + LambdaNamespace, + ComprehensionNamespace, + ): + scope_to_ns[Ns.scope_class] = Ns + +class Binding(Generic[ValT]): + """ The current value (if any) of a Var in a Namespace. + """ + # The value attribute only exists if the Binding is bound. + value: ValT + + class Unbound: pass + _unbound: Final = Unbound() # Sentinel for constructor or bind(). + + def __init__(self, value: ValT | Unbound = _unbound): + if value is not self._unbound: + self.value = value + + def __bool__ (self) -> bool: + return hasattr(self, 'value') + + def bind(self, value: ValT | Unbound = _unbound): + if value is not self._unbound: + self.value = value + else: + self.unbind() + + def unbind(self): + del self.value + + def __repr__(self) -> str: + if self: + return repr(self.value) + else: + return '' + +x = RootNamespace() + +y = x.add_module('foo', key=42) + +z = GlobalNamespace(GlobalScope('bar'), key=43) + +x diff --git a/formal/scopes.py b/formal/scopes.py index 4147098..c929c5a 100644 --- a/formal/scopes.py +++ b/formal/scopes.py @@ -16,188 +16,503 @@ from __future__ import annotations +import sys +from typing import * +from typing_extensions import Self, TypeAlias +from abc import * +from enum import * + +__all__ = ( + 'Scope', + 'RootScope', + 'GlobalScope', + 'ClassScope', + 'FunctionScope', + 'LambdaScope', + 'ComprehensionScope', + 'ScopeT', + 'RefT', + ) +""" +Everything about scopes: + +Every Scope corresponds to a module, a function, or a class. + A function includes function defs, lambdas, and comprehensions. + +Scopes form a tree, starting with a GlobalScope object. Scope.nested is a list of the subtree Scopes. + +A Scope has variables. A variable (or 'var') is a name which is found in the scope. + In the global scope, it may also be a var which is declared global in a nested scope. + +It provides static status of various Var names seen in the scope. + +A scope has a "ref" attribute, which is an arbitrary object supplied to the constructor. +This can be used by a builder function to build the scope and its nested scopes. +The nested scopes will have their own "ref" attributes. + +Static status of a var. + This involves a "binding scope" for the var, which is the same or an enclosing Scope. + Status is one of the following, and may evolve while the scope is being examined: + Unknown. The var does not appear yet. + Local. A value is bound somewhere in the scope, including: + assignment or reassignment, + deletion, + a walrus operator in a nested comprehension. + This is only if var is not already declared nonlocal or global. + The binding scope is the current scope + Nonlocal. The binding scope is a specific enclosing closed scope where the var is Local. + Global. The binding scope is the global scope. + Top. This is used instead of Local and Global in the global scope for a var that is bound there. + Used. The var appears in this scope but it not yet Local, Nonlocal, or Global. + The binding scope is None. The purpose of this status is to prevent further + nonlocal or global declarations. + Unknown will change to one of the others the first time the var is seen in the scope. + Used will change to Local the first time the var is used in a binding situation. + Used will change to Nonlocal or Global otherwise, some time after all uses of the var + in the current scope have been reported. + +Building the Scope: This consists of: + 1. Note all names in the code for the scope, + except for nested function and class defs: + Do not include the body of the def, or any names (such as function + arguments) which are known in the context of the nested scope. + Create a scope object for the def. This will be treated as an assignment + of the def's name in the current scope. + Do include other names appearing in the def statement other than the var, + such as function default values, base classes, etc. + 2. Do the entire build (recursively) on all the nested scopes. + A walrus in a nested comprehension can change a var from Used to Local. + If the binding scope is needed by some nested scope, then this can + change the var from Used to Nonlocal or Global. + 3. Resolve all remaining Used vars by finding the binding scopes for them, making them + either Nonlocal or Global. + + This done with the Scope.build() method. + + It uses a builder function, which is stored in the Scope.builder attribute. + This is provided to the global scope constructor. + It is inherited by a nested scope, unless a different builder is provided. + + The function of the builder is to report any use of any var in the Scope which exists. + It excludes any var which was reported to the parent Scope earlier. + For functions, classes, and comprehensions, + Report the nesting of a new scope. For function and class defs, include the name, + which will become bound in the current scope. + Report vars occurring in the following: + function argument defaults, + class bases, + class definition arguments, + decorators. + in comprehensions, all walrus operators at any level of nesting. + If current scope is a class, this is a SyntaxError. + +Operations on a Scope performed by its builder: + Note, all enclosing scopes will have been completely built by this time. + load(var). Just reports the fact that this var appears. + add_global(var). Declares the var to be in the global scope. + If earlier loaded in a non-global scope, this is an error. + Also adds a load(var) to the global scope. + add_nonlocal(var). Declares the var to be in an enclosing closed scope. + If earlier loaded, or the enclosing scope is not found, this is an error. + store(var). Notes the fact that the var has been assigned, reassigned or deleted. + store_walrus(var): Records an assignment via := operator contained somewhere in an + immediately nested comprehension. Same as store(), except SyntaxError in + certain cases. + + nest(...). Notes a nested Scope. Creates a Scope for it and stores the name as well. + Arguments provided: + A subclass of Scope to be used. + A ref object. + An optional name, for function and class defs only. + An optional builder for the new Scope, defaults to the builder of the current scope. + +After the Scope's builder is complete, these operations are valid: + + binding_scope(var). Returns the binding scope for the var, if any. + If one exists for the var, that scope is returned. + Else the var is Used or Unknown. The scope is found by looking at enclosing scopes. + If found, then this is stored in this scope, which makes it Nonlocal or Global. + If Global, then load(var) is performed on the global scope as well. + If not found, returns None. + + These are the binding_scope resolution rules: + 1. If the var is Local, returns itself. + 2. The global scope returns itself. + 3. An open scope returns the global scope. + 4. A closed scope returns its nonlocal_scope() if there is one, + otherwise returns the global scope. + + nonlocal_scope(var). Looks for a closed scope which will match a nonlocal declaration + in a nested scope, using the parent chain. + + These are the nonlocal_scope rules: + 1. An open scope returns its parent.nonlocal_scope(). + 2. The global scope returns None. + 3. A closed scope returns itself if var is Local, + otherwise returns its parent.nonlocal_scope(). + +""" -class Scope: - scope_name: str - parent: Scope | None - uses: set[str] - locals: set[str] - nonlocals: set[str] - globals: set[str] - - def __init__(self, scope_name: str, parent: Scope | None): - self.scope_name = scope_name - self.parent = parent - self.uses = set() - self.locals = set() - self.nonlocals = set() - self.globals = set() - # locals, nonlocals and globals are all disjunct - - def __repr__(self) -> str: - return f"{self.__class__.__qualname__}({self.scope_name!r})" - - def store(self, name: str) -> None: - if name in self.locals or name in self.nonlocals or name in self.globals: - return - self.locals.add(name) - - def load(self, name: str) -> None: - self.uses.add(name) - - def add_nonlocal(self, name: str) -> None: - if name in self.uses: - raise SyntaxError("name used prior to nonlocal declaration") - if name in self.locals: - raise SyntaxError("name assigned before nonlocal declaration") - if name in self.globals: - raise SyntaxError("name is global and nonlocal") - self.nonlocals.add(name) - - def add_global(self, name: str) -> None: - if name in self.uses: - raise SyntaxError("name used prior to global declaration") - if name in self.locals: - raise SyntaxError("name assigned before global declaration") - if name in self.nonlocals: - raise SyntaxError("name is nonlocal and global") - self.globals.add(name) - - def global_scope(self) -> GlobalScope: - # GlobalScope overrides this - assert self.parent is not None - return self.parent.global_scope() - - def enclosing_closed_scope(self) -> ClosedScope | None: - if self.parent is None: - return None - elif isinstance(self.parent, ClosedScope): - return self.parent - else: - return self.parent.enclosing_closed_scope() - - def lookup(self, name: str) -> Scope | None: - # Implemented differently in OpenScope, GlobalScope and ClosedScope - raise NotImplementedError - - -class OpenScope(Scope): - def lookup(self, name: str) -> Scope | None: - if name in self.locals: - return self - else: - s = self.enclosing_closed_scope() - if s is not None: - return s.lookup(name) - else: - return self.global_scope() - - -class GlobalScope(OpenScope): - parent: None # Must be None - - def __init__(self): - super().__init__("", None) - - def global_scope(self) -> GlobalScope: - return self - - def lookup(self, name: str) -> Scope | None: - if name in self.locals: - return self - else: - return None - - def add_nonlocal(self, name: str) -> None: - raise SyntaxError("nonlocal declaration not allowed at module level") - - def add_global(self, name: str) -> None: - return self.store(name) - - -# For modules, exec and eval -class ToplevelScope(OpenScope): - parent: GlobalScope # Cannot be None - - def __init__(self, parent: GlobalScope): - super().__init__("", parent) +RefT = TypeVar('RefT') +ScopeT: TypeAlias = 'Scope[RefT]' +BuildT = Callable[[ScopeT, RefT], None] + +def null_builder(s: ScopeT, r: RefT): pass + +class VarStatus(Enum): + Unknown = 0 # var does not appear at all + Used = auto() # var appears in the scope but has no static scope + Local = auto() # var is in current scope, which is not the global scope + Nonlocal = auto() # var is in some scope other than current or global + Global = auto() # var is in global scope, which is not the current scope + Top = auto() # var is in the global scope which is also the current scope + # is_local and is_global are both true + + def __bool__(self): return bool(self.value) + @property + def is_used(self): return self is self.Used + @property + def is_local(self): return self in (self.Local, self.Top) + @property + def is_nonlocal(self): return self is self.Nonlocal + @property + def is_global(self): return self in (self.Global, self.Top) + @property + def is_top(self): return self is self.Top + +class Scope(Generic[RefT]): + scope_name: str + parent: Self | None + global_scope: GlobalScope[RefT] = None + + # Mapping of variable names to their binding scopes, for every var that appears in this scope. + # The location may be self, or some ancestor scope. It is determined at compile time. + # The var is Local in its binding scope, which means that in that scope, the var is mapped to itself. + # The binding scope may temporarily be unknown, but this is eventually resolved by the time the + # entire scope has been built. + vars: Mapping[str, ScopeT | None] + + ref: RefT | None + builder: BuildT + nested: List[ScopeT] + + # True if this Scope, or any ancestor, is an "in" clause of a comprehension. + # This makes the walrus operator illegal, as well as in all nested Scopes. + no_walrus: bool = False + + is_master: ClassVar[bool] = False + is_global: ClassVar[bool] = False + is_class: ClassVar[bool] = False + is_function: ClassVar[bool] = False + is_comp: ClassVar[bool] = False + + @abstractmethod + def __init__(self, scope_name: str, parent: Scope | None, *, + ref: RefT = None, build: BuildT = null_builder, + no_walrus: bool = False): + self.scope_name = scope_name + self.parent = parent + self.global_scope = parent and parent.global_scope + self.ref = ref + self.builder = build + self.vars = dict() + self.nested = [] + self.vars = dict() + if no_walrus or (parent and parent.no_walrus): self.no_walrus = True + + def __repr__(self) -> str: + return f"{self.__class__.__qualname__}({self.scope_name!r})" + + def qualname(self, varname: str = '', *, sep: str = '.') -> str: + """ Fully qualified name of this scope, or given variable name in this scope. + Optional separator to replace '.'. + Global scope is part of this name only if it has its own name. + """ + names = list(self.scope_names) + if varname: names.append(varname) + return sep.join(names) + + @property + def scope_names(self) -> Iterator[str]: + """ Iterator for names of self and enclosed scopes, from globals to self. + Global scope is part of this only if it has its own name. + """ + yield from self.parent.scope_names + yield self.scope_name + + def status(self, var: str) -> VarStatus: + try: scope = self.vars[var] + except KeyError: return VarStatus.Unknown + if not scope: return VarStatus.Used + if self is self.global_scope: return VarStatus.Top + if scope is self: return VarStatus.Local + if scope is self.global_scope: return VarStatus.Global + return VarStatus.Nonlocal + + def load(self, var: str) -> Scope: + """ Change from Unknown to Used, otherwise no change. Returns static scope. """ + return self.vars.setdefault(var, None) + + def store(self, var: str) -> Scope: + """ Marks the var as being stored in this, or some enclosing Scope. + In case of global scope, marks the var there too. + """ + # Change from Unknown to Used, and get static scope. + scope = self.load(var) + if not scope: + # Change from Used to Local (or Top) + self.vars[var] = self + return self + if self.status(var) is VarStatus.Global: + scope.store(var) + return scope + + def store_walrus(self, var: str) -> Scope: + """ store_walrus() is same as store(), except: + 1. In a ClassScope, it is implemented separately as a SyntaxError. + 2. Anywhere in a comprehension "in" clause, it is a SyntaxError. + """ + if self.no_walrus: + raise SyntaxError('assignment expression cannot be used in a comprehension iterable expression') + return self.store(var) + + def add_nonlocal(self, var: str) -> Scope: + """ Change from Unknown to Nonlocal. Return new static scope. + SyntaxError if nonlocal scope not found, or if var is not already Nonlocal. + GlobalScope overrides this method. + """ + status = self.status(var) + if not status: + # Name Unknown. Change to Nonlocal, or error if nonlocal scope not found. + scope = self.nonlocal_scope(var) + if not scope: + raise SyntaxError(f"no binding for nonlocal '{var}' found") + self.vars[var] = scope + return scope + # Only Nonlocal is valid. + if status.is_nonlocal: + return self.vars[var] + if status.is_global: + # Global is an error. + raise SyntaxError(f"var '{var}' is nonlocal and global") + else: + # Used is an error. + raise SyntaxError("var '{var}' is used prior to nonlocal declaration") + + def add_global(self, var: str) -> Scope: + """ Change from Unknown to Global or Top. Return new static scope. + Error if static scope is not global. + """ + status = self.status(var) + if not status: + # Name Unknown. Change to Global + scope = self.vars[var] = self.global_scope + return scope + # Only Global is valid. + if status.is_global: + return self.global_scope + if status.is_nonlocal: + # Nonlocal is an error. + raise SyntaxError(f"var '{var}' is nonlocal and global") + else: + # Used is an error. + raise SyntaxError("var '{var}' used prior to global declaration") + + def nest(self, cls: Type[Scope], name: str = None, *, + ref: RefT = None, build: BuildT | None = None) -> Self: + """ Report a nested scope. Create the Scope object. + Report the name as assigned in the current scope, except for + Lambda and Comprehension, which are anonymous. + Optional keyword to provide builder, otherwise uses current builder. + """ + res = cls(name, self, ref=ref, build=build or self.builder) + self.nested.append(res) + return res + + # Methods after build is complete... + + def binding_scope(self, var: str) -> Scope | None: + """ Tries to find the static scope, setting it if not already known. + Only valid after all the above static methods have been called. + """ + # Implemented differently in NestedScope, GlobalScope. + raise NotImplementedError + + def nonlocal_scope(self, var) -> ClosedScope | None: + """ Try to find a nonlocal scope for a var in some enclosed scope. + """ + # Implemented differently in OpenScope, GlobalScope and ClosedScope + raise NotImplementedError + + def build(self): + """ Builds entire tree statically, using the builder recursively. + Nested scopes are built by the same or their own individual builders. + """ + # Phase 1, implemented by the builder. + self.builder(self, self.ref) + # Phase 2, build all the nested scopes. + for nested in self.nested: + nested.build() + # Phase 3, resolve all remaining Used names. + for var, scope in self.vars.items(): + if scope: continue + self.binding_scope(var) + +class RootScope(Scope): + """ Container for all the modules in a program. + Will be created for a GlobalScope's parent if one is not provided to it. + """ + is_master: ClassVar[bool] = True + + modules: Mapping[str, GlobalScope] + + def __init__(self, **kwds): + super().__init__('', None, **kwds) + self.modules = {} + + def add_module(self, var: str = '', **kwds) -> GlobalScope: + self.nest(GlobalScope, var, **kwds) + +class GlobalScope(Scope): + parent: RootScope | None + + is_global: ClassVar[bool] = True + + def __init__(self, var: str = '', *, parent: RootScope = None, **kwds): + super().__init__(var, parent or RootScope(), **kwds) + self.global_scope = self + if var: + self.parent.modules[var] = self + + def add_nonlocal(self, var: str) -> None: + raise SyntaxError("nonlocal declaration not allowed at module level") + + def add_global(self, var: str) -> None: + return self.store(var) + + def binding_scope(self, var: str) -> Scope | None: + """ Get the static scope for this var. + It is always self, and the var is made Local. + """ + return self.store(var) + + def nonlocal_scope(self, var) -> None: + return None + + @property + def scope_names(self) -> Iterator[str]: + return [] + + +class NestedScope(Scope): + """ Any Scope other than GlobalScope. Subclasses are OpenScope and ClosedScope. + """ + def __init__(self, *args, **kwds): + super().__init__(*args, **kwds) + self.parent.store(self.scope_name) + + def binding_scope(self, var: str) -> Scope | None: + """ Find the static scope, setting it not already known. + Only valid after all other static methods have been called. + """ + # Change Unknown -> Used. Get static scope or None if Used. + scope = self.load(var) + if scope: + return scope + # Used. Will be in a nonlocal scope, else in globals. + scope = self.parent.nonlocal_scope(var) + if not scope: scope = self.global_scope + self.vars[var] = scope + return scope + +class OpenScope(NestedScope): + + def nonlocal_scope(self, var) -> ClosedScope | None: + return self.parent.nonlocal_scope(var) + +# For modules, exec and eval. Provides a module name, otherwise unnecessary (??) +class ToplevelScope(Scope): + parent: GlobalScope # Cannot be None + + def __init__(self, parent: GlobalScope): + super().__init__("", parent) class ClassScope(OpenScope): - parent: Scope # Cannot be None - - def __init__(self, name: str, parent: Scope): - super().__init__(name, parent) - parent.store(name) - - -class ClosedScope(Scope): - parent: Scope # Cannot be None - - def lookup(self, name: str) -> Scope | None: - if name in self.locals: - return self - elif name in self.globals: - return self.global_scope() - else: - res: Scope | None = None - p: Scope | None = self.enclosing_closed_scope() - if p is None: - res = None - else: - res = p.lookup(name) - if name in self.nonlocals and not isinstance(res, ClosedScope): - # res could be None or GlobalScope - raise SyntaxError(f"nonlocal name {name!r} not found") - else: - return res + parent: Scope # Cannot be None + is_class: ClassVar[bool] = True + + def store_walrus(self, var: str) -> Scope: + """ Reports the var as being used as lvalue in := operator in a directly nested comprehension. + This is a syntax error. + """ + raise SyntaxError('assignment expression within a comprehension cannot be used in a class body') + +class ClosedScope(NestedScope): + parent: Scope # Cannot be None + + def nonlocal_scope(self, var) -> ClosedScope | None: + """ Change Unknown to Used. + Return static scope if Local or Nonlocal, None if Global, go to parent if Used. + """ + scope = self.load(var) + if scope is self.global_scope: + return None + if scope: + return scope + return self.parent.nonlocal_scope(var) class FunctionScope(ClosedScope): - def __init__(self, name: str, parent: Scope): - super().__init__(name, parent) - parent.store(name) + is_function: ClassVar[bool] = True class LambdaScope(FunctionScope): - pass + pass -class ComprehensionScope(ClosedScope): - pass - +class ComprehensionScope(FunctionScope): + is_comp: ClassVar[bool] = True def test(): - # Set up a sample program - # class C: - # def foo(self, a = blah): - # global x - # x = a - - globals = GlobalScope() - c = ClassScope("C", globals) - foo = FunctionScope("foo", c) - foo.store("self") - foo.store("a") - foo.add_global("x") - - assert foo.lookup("C") is globals - assert c.lookup("C") is globals - - assert foo.lookup("foo") is None - assert c.lookup("foo") is c - - assert foo.lookup("self") is foo - assert c.lookup("self") is None - - assert foo.lookup("a") is foo - assert c.lookup("a") is None - - assert foo.lookup("blah") is None - assert c.lookup("blah") is None - - assert foo.lookup("x") is globals - assert c.lookup("x") is None + # Set up a sample program + # class C: + # def foo(self, a = blah): + # global x + # x = a + + c = None + foo = None + def foo_build(scope: Scope, ref): + scope.store("self") + scope.store("a") + scope.add_global("x") + def c_build(scope: Scope, ref): + nonlocal foo + foo = scope.nest(FunctionScope, "foo", build=foo_build) + def globals_build(scope: Scope, ref): + nonlocal c + c = scope.nest(ClassScope, "C", build=c_build) + globals = GlobalScope(build=globals_build) + globals.build() + + assert foo.binding_scope("C") is globals + assert c.binding_scope("C") is globals + + assert foo.binding_scope("foo") is globals + assert c.binding_scope("foo") is c + + assert foo.binding_scope("self") is foo + assert c.binding_scope("self") is globals + + assert foo.binding_scope("a") is foo + assert c.binding_scope("a") is globals + + assert foo.binding_scope("blah") is globals + assert c.binding_scope("blah") is globals + + assert foo.binding_scope("x") is globals + assert c.binding_scope("x") is globals if __name__ == "__main__": - test() + test() diff --git a/formal/scopes_test.py b/formal/scopes_test.py new file mode 100644 index 0000000..c2e0a41 --- /dev/null +++ b/formal/scopes_test.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import sys, os +from contextlib import contextmanager +from io import StringIO +from typing import NamedTuple, Iterable, Iterator + +from enum import * +import attrs + +from scopes import( + Scope, + FunctionScope, + ClassScope, + GlobalScope, + ComprehensionScope, + BuildT as ScopeBuildT, + ) + +from namespaces import( + BuildT as NsBuildT, + GlobalNamespace, + ) + + +# Parameters at the current level, will be saved/restored by various context managers. +level = 0 + +@contextmanager +def indent(lev: int): + # Set the current level during the context. + global level + oldlevel = level + level = lev + yield + level = oldlevel + +# Writing the output file... +out = StringIO() +lineno = 1 + +def write(s: str): + breaklines = [ 23 ] + print(' ' * level + s, file=out) + global lineno + lineno += len(s.split('\n')) + + +class VarMode(Enum): + """ Defines how a scope uses the variable 'x'. + Except for LocalNoCapt, makes no restrictions on nested classes. + """ + Unused = 'none' + Used = 'use' + Nonlocal = 'nloc' # x is declared nonlocal in the scope. If this is not allowed, + # the Scope will be left as Unused, but the genenrated code will + # verify that the nonlocal declaration is a syntax error. + Global = 'glob' + Local = 'loc' # x will be bound in this scope by an assignment. + LocalNoCapt = 'ncap' # same, but nested scopes may not capture x. + # used as a mode for a nested scope, nested.mode = Local and nested.nocapt = True. + + @classmethod + def nocapt_modes(cls, nocapt: bool = True) -> Iterable[VarMode]: + """ Gives all the possible modes, based on whether "no captures" is in effect, + which defaults to True. + """ + if nocapt: + yield cls.Unused + yield cls.Global + yield cls.Local + else: + yield from cls.__members__.values() + + @property + def is_loc(self) -> bool: + return self in (self.Local, self.LocalNoCapt) + + def name_sfx(self, name: str = '') -> str: + """ Suffix for a scope name, optionally added to givenname. """ + if self is self.Unused: return name + return f'{name}_{self.value}' + +@attrs.define(frozen=True) +class ScopeParams: + """ Everything the builder needs to know about building a scope, + other than attributes of the scope itself. + """ + level: int # How deep in the scope tree. Global scope is level 0. + depth: int # How much deeper to make nested scopes. 0 means no nested. + mode: VarMode # How the varible 'x' will be used in this scope. + nocapt: bool = False # If true, restricts nested scopes to those that don't capture + # 'x' from this scope. + is_class: bool = False + + def nest(self, mode: VarMode, is_class: bool = False) -> ScopeParams: + """ New object to go with a nested scope. """ + nocapt = self.nocapt + if mode is mode.LocalNoCapt: + nocapt = True + return attrs.evolve(self, + level=self.level + 1, + depth=self.depth - 1, + mode=mode, + nocapt=nocapt, + is_class=is_class, + ) + + def nested_params(self) -> Iterable[ScopeParams]: + """ Characterizes all of the set of nested scopes to generate. + Tuple of (is class, var type). + At the bottom nesting level, the iterator is empty. + """ + if self.depth: + for is_class in True, False: + for mode in self.mode.nocapt_modes(self.nocapt): + yield self.nest(mode, is_class) + +""" + Functions to initialize given Scope and create nested Scopes. Does not include operating on + the nested scopes + + Special functions for a function, or list comprehension, that stores a value in its parent. +""" + +def build_scope(scope: Scope, ref: ScopeParams): + """ Define the static properties of the scope and create (but do not build) + the nested scopes. + """ + # Set the scope's status. It is currently Unknown + mode = ref.mode + if mode is mode.Unused: pass + if mode is mode.Used: + scope.load('x') + if mode is mode.Nonlocal: + try: scope.add_nonlocal('x') + # If nonlocal is not allowed, then leave this scope empty. + except SyntaxError: return + if mode.is_loc: + scope.store('x') + if mode is mode.Global: + scope.add_global('x') + + # Make nested function and class scopes. May be called twice. + def makesubs(suffix: str = ''): + for nested_ref in ref.nested_params(): + cls = (FunctionScope, ClassScope)[nested_ref.is_class] + name = makename(nested_ref.is_class, ref.level, nested_ref.mode, suffix) + scope.nest(cls, name, ref=nested_ref) + + makesubs() + + if mode.is_loc: + # This scope does some binding operations, then makes a second set of nested scopes. + + # It needs a nested scope which will assign to x in the current scope. + # We can use a Comprehension if possible, otherwise a Function. + # It has a [x := ...] to assign to x, before the second set of nested scopes. + if sys.version_info < (3, 8) or scope.is_class: + scope.nest(FunctionScope, ref=ref.nest(VarMode.Unused), build=build_set_in_parent_scope) + else: + scope.nest(ComprehensionScope, ref=ref.nest(VarMode.Unused), build=build_comp_scope) + + makesubs('2') + +def build_comp_scope(scope: ComprehensionScope, ref: ScopeParams): + """ Define the static properties of the scope. No nested scopes. + """ + scope.store_walrus('x') + +def build_set_in_parent_scope(scope: FunctionScope, ref: ScopeParams): + """ Define the static properties of the scope. No nested scopes. + """ + scope.parent.store('x') + +""" + Functions to write code based on given Namespace, including nested Namespaces. + + Special functions for a function, or list comprehension, that stores a value in its parent. +""" + +def build_ns(space: NsT, scope: Scope, ref: ScopeParams, nested: Iterator[ScopeT]): + """ Write the python test file for this scope and nested scopes, in the correct order. + This method is called recursively to generate nested scopes, therefore it cannot keep + state in this Builder object. + """ + def maketest(): + try: value = space.load('x') + except NameError: value = None + write(f'try: test(x, {value!r}, {lineno})') + write(f'except NameError: test(None, {value!r}, {lineno})') + + mode = ref.mode + objname = scope.qualname(sep="_") + # 1. Write the function/class definition line. + if scope.is_class: + write(f'class {objname}:') + elif scope.is_function: + write(f'def {objname}():') + + def writebody(): + """ Write the body of the scope. This is a separate function in order to bail out early. """ + nonlocal mode + with indent(ref.level): + # 2. Write initial setup of the variable. + if mode is mode.Unused: + if ref.depth == 0: + write('pass') + if mode is mode.Used: + pass + if mode is mode.Nonlocal: + status = scope.status('x') + if status is status.Nonlocal: + write(f'nonlocal x') + else: + # No nonlocal binding. There are no enclosed scopes, but generate a test. + write('# No enclosed binding exists.') + write('try: compile("nonlocal x", "exec")') + write(f'except: test(None, None, {lineno})') + write(f'else: error("Enclosed binding exists", {lineno})') + n = list(nested) + assert not n + return + + if mode.is_loc: + pass + if mode is mode.Global: + write(f'global x') + + if mode is not mode.Unused: + maketest() + + # 3. Write recursively the first, or only, set of nested scopes. + for _ in ref.nested_params(): + space.nest(nested) + #next(nested).build() + + if mode.is_loc: + # 4. Write the modifications of the variable. + + value = scope.qualname('x') + write(f'x = "{value}"') + space.store('x', value) + maketest() + + write('del x') + space.delete('x') + maketest() + + # Set it again. This time use a nested scope. + # It might be a list comprehension with a walrus. Or it might be a function. + nest = next(nested) + if nest.is_comp: + build = build_comp_ns + else: + build = build_set_parent_ns + space.nest(nest, build=build) + + maketest() + + ... + # 5. Write recursively the second set of nested scopes. + for _ in ref.nested_params(): + space.nest(nested) + writebody() + + # 6. Call the function, if a function. + if scope.is_function: + write(f'{objname}()') + +def build_comp_ns(space: ComprehensionNamespace, scope: ComprehensionScope, ref: ScopeParams, nested: Iterator[ScopeT]): + """ Write code to store value in parent, using a walrus in a list comprehension. There are no nested scopes. + """ + value: str = scope.parent.qualname('x') + write(f'[x := _ for _ in ["{value}"]]') + space.store_walrus('x', value) + +def build_set_parent_ns(space: FunctionNamespace, scope: FuncionScope, ref: ScopeParams, nested: Iterator[ScopeT]): + """ Write code to store value in parent, without a walrus in a list comprehension. There are no nested scopes. + """ + value: str = scope.parent.qualname('x') + # Before assignment expressions were introduced, or when parent is a class, + # we need an explicit function to perform the store. + # The method depends on whether the parent scope is a function, class, or global. + write('def listcomp():') + with indent(ref.level): + if scope.parent.is_global: + write('global x; ' f'x = "{value}"') + if scope.parent.is_function: + write('nonlocal x; ' f'x = "{value}"') + if scope.parent.is_class: + # For a class, it is necessary to find its stack frame and store in its locals. + write(f'inspect.stack()[1].frame.f_locals["x"] = "{value}"') + write('listcomp()') + space.parent.store('x', value) + +def makename(is_class: bool, level: int, mode: VarMode, suffix: str = ''): + "Name for a nested scope, given the class of the scope and the current nesting level." + name = 'aA'[is_class] + name = chr(ord(name) + level) + name += suffix + return mode.name_sfx(name) + +def gen(depth): + """ Main function to write most of the output. """ + + scope = GlobalScope(ref=ScopeParams(0, depth, VarMode.Local), build=build_scope) + scope.build() + ns = GlobalNamespace(scope, key=43, build=build_ns) + ns.build() + +print('Creating file "test.py"... ', end='', flush=True) + +write( +'''from __future__ import annotations +import inspect +ntests = 0 +def test(value: str | None, comp: str | None, lineno: int): + if value != comp: + raise ValueError(f'Line {lineno}: expected {comp!r}, got {value!r}.', lineno) from None + global ntests + ntests += 1 + if ntests % 1000 == 0: + print(f'{ntests:5d}') + +def error(msg: str, lineno: int): + raise ValueError(f'Line {lineno}: {msg}.', lineno) from None + +print('done') +print('Running tests. ') +''') +gen(4) + +with open(f'{sys.path[0]}/test.py', 'w') as f: + f.write(out.getvalue()) +with open(f'{sys.path[0]}/test.py.txt', 'w') as f: + f.write(out.getvalue()) +print('done') +print('Importing file "test.py"... ', end='', flush=True) + +try: import test +except ValueError as exc: + print() + msg, lineno = exc.args + lines = out.getvalue().splitlines() + print(*lines[max(lineno - 11, 0):lineno], sep='\n') + print('---- ' + msg) + print(*lines[lineno: lineno + 10], sep='\n') +else: + print(' done') + print(f'All {test.ntests} tests passed.')