Support artificially abstract base classes (#565)

Add support for "artificially abstract" base classes. The Python ecosystem is much less strict with its use of `abc.ABC` to interfaces than Java (which has a native `interface` construct), so even in where a type may be _in practice_ an interface or ABC, the compiler would not permit you to declare such types as supertypes because they do not inherit from `abc.ABC`. In these cases, users can mark the type as  abstract with the `:abstract` metadata key to force the compiler to accept that it is abstract.

In this case, the compiler will attempt to verify that any extra methods present on the `deftype`, `defrecord`, or `reify` are present on any one of the artificially abstract base classes. This will allow users to override existing class members, but not introduce new ones without a true interface.

Author: chrisrink10

CHANGELOG.md

@@ -15,7 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  * Added support for multi-arity methods on `definterface` (#538)
  * Added support for Protocols (#460)
  * Added support for Volatiles (#460)
- * Add JSON encoder and decoder in `basilisp.json` namespace (#484)
+ * Added JSON encoder and decoder in `basilisp.json` namespace (#484)
+ * Added support for generically diffing Basilisp data structures in `basilisp.data` namespace (#555)
+ * Added support for artificially abstract bases classes in `deftype`, `defrecord`, and `reify` types (#565)
 
 ### Changed
  * Basilisp set and map types are now backed by the HAMT provided by `immutables` (#557)

src/basilisp/lang/compiler/analyzer.py

@@ -1,6 +1,7 @@
 import builtins
 import collections
 import contextlib
+import inspect
 import logging
 import re
 import sys
@@ -49,6 +50,7 @@
     LINE_KW,
     NAME_KW,
     NS_KW,
+    SYM_ABSTRACT_META_KEY,
     SYM_ASYNC_META_KEY,
     SYM_CLASSMETHOD_META_KEY,
     SYM_DEFAULT_META_KEY,
@@ -623,6 +625,7 @@ def has_meta_prop(o: Union[IMeta, Var]) -> bool:
     return has_meta_prop
 
 
+_is_artificially_abstract = _meta_getter(SYM_ABSTRACT_META_KEY)
 _is_async = _meta_getter(SYM_ASYNC_META_KEY)
 _is_mutable = _meta_getter(SYM_MUTABLE_META_KEY)
 _is_py_classmethod = _meta_getter(SYM_CLASSMETHOD_META_KEY)
@@ -1512,24 +1515,57 @@ def __deftype_or_reify_impls(  # pylint: disable=too-many-branches,too-many-loca
 _var_is_protocol = _meta_getter(VAR_IS_PROTOCOL_META_KEY)
 
 
+def __is_deftype_member(mem) -> bool:
+    """Return True if `mem` names a valid `deftype*` member."""
+    return (
+        inspect.isfunction(mem)
+        or isinstance(mem, (property, staticmethod))
+        or inspect.ismethod(mem)
+    )
+
+
+def __is_reify_member(mem) -> bool:
+    """Return True if `mem` names a valid `reify*` member."""
+    return inspect.isfunction(mem) or isinstance(mem, property)
+
+
 def __deftype_and_reify_impls_are_all_abstract(  # pylint: disable=too-many-branches,too-many-locals
     special_form: sym.Symbol,
     fields: Iterable[str],
     interfaces: Iterable[DefTypeBase],
     members: Iterable[DefTypeMember],
-) -> bool:
-    """Return True if all `deftype*` or `reify*` super-types can be verified abstract
-    statically. Return False otherwise.
+) -> Tuple[bool, lset.Set[DefTypeBase]]:
+    """Return a tuple of two items indicating the abstractness of the `deftype*` or
+    `reify*` super-types. The first element is a boolean value which, if True,
+    indicates that all bases have been statically verified abstract. If False, that
+    value indicates at least one base could not be statically verified. The second
+    element is the set of all super-types which have been marked as artificially
+    abstract.
 
     In certain cases, such as in macro definitions and potentially inside of
     functions, the compiler will be unable to resolve the named super-type as an
     object during compilation and these checks will need to be deferred to runtime.
     In these cases, the compiler will wrap the emitted class in a decorator that
     performs the checks when the class is compiled by the Python compiler.
 
+    The Python ecosystem is much less strict with its use of `abc.ABC` to define
+    interfaces than Java (which has a native `interface` construct), so even in cases
+    where a type may be _in practice_ an interface or ABC, the compiler would not
+    permit you to declare such types as supertypes because they do not themselves
+    inherit from `abc.ABC`. In these cases, users can mark the type as artificially
+    abstract with the `:abstract` metadata key.
+
     For normal compile-time errors, an `AnalyzerException` will be raised."""
     assert special_form in {SpecialForm.DEFTYPE, SpecialForm.REIFY}
 
+    unverifiably_abstract = set()
+    artificially_abstract: Set[DefTypeBase] = set()
+    artificially_abstract_base_members: Set[str] = set()
+    is_member = {
+        SpecialForm.DEFTYPE: __is_deftype_member,
+        SpecialForm.REIFY: __is_reify_member,
+    }[special_form]
+
     field_names = frozenset(fields)
     member_names = frozenset(deftype_or_reify_python_member_names(members))
     all_member_names = field_names.union(member_names)
@@ -1547,7 +1583,10 @@ def __deftype_and_reify_impls_are_all_abstract(  # pylint: disable=too-many-bran
                     f"{special_form} interface Var '{interface.form}' is not bound"
                     "and cannot be checked for abstractness; deferring to runtime",
                 )
-                return False
+                unverifiably_abstract.add(interface)
+                if _is_artificially_abstract(interface.form):
+                    artificially_abstract.add(interface)
+                continue
 
             # Protocols are defined as maps, with the interface being simply a member
             # of the map, denoted by the keyword `:interface`.
@@ -1561,54 +1600,72 @@ def __deftype_and_reify_impls_are_all_abstract(  # pylint: disable=too-many-bran
         if interface_type is object:
             continue
 
-        if not is_abstract(interface_type):
-            raise AnalyzerException(
-                f"{special_form} interface must be Python abstract class or object",
-                form=interface.form,
-                lisp_ast=interface,
+        if is_abstract(interface_type):
+            interface_names: FrozenSet[str] = interface_type.__abstractmethods__
+            interface_property_names: FrozenSet[str] = frozenset(
+                method
+                for method in interface_names
+                if isinstance(getattr(interface_type, method), property)
             )
+            interface_method_names = interface_names - interface_property_names
+            if not interface_method_names.issubset(member_names):
+                missing_methods = ", ".join(interface_method_names - member_names)
+                raise AnalyzerException(
+                    f"{special_form} definition missing interface members for "
+                    f"interface {interface.form}: {missing_methods}",
+                    form=interface.form,
+                    lisp_ast=interface,
+                )
+            elif not interface_property_names.issubset(all_member_names):
+                missing_fields = ", ".join(interface_property_names - field_names)
+                raise AnalyzerException(
+                    f"{special_form} definition missing interface properties for "
+                    f"interface {interface.form}: {missing_fields}",
+                    form=interface.form,
+                    lisp_ast=interface,
+                )
 
-        interface_names: FrozenSet[str] = interface_type.__abstractmethods__
-        interface_property_names: FrozenSet[str] = frozenset(
-            method
-            for method in interface_names
-            if isinstance(getattr(interface_type, method), property)
-        )
-        interface_method_names = interface_names - interface_property_names
-        if not interface_method_names.issubset(member_names):
-            missing_methods = ", ".join(interface_method_names - member_names)
-            raise AnalyzerException(
-                f"{special_form} definition missing interface members for "
-                f"interface {interface.form}: {missing_methods}",
-                form=interface.form,
-                lisp_ast=interface,
+            all_interface_methods.update(interface_names)
+        elif _is_artificially_abstract(interface.form):
+            # Given that artificially abstract bases aren't real `abc.ABC`s and do
+            # not annotate their `abstractmethod`s, we can't assert right now that
+            # any the type will satisfy the artificially abstract base. However,
+            # we can collect any defined methods into a set for artificial bases
+            # and assert that any extra methods are included in that set below.
+            artificially_abstract.add(interface)
+            artificially_abstract_base_members.update(
+                map(
+                    lambda v: v[0],
+                    inspect.getmembers(interface_type, predicate=is_member),
+                )
             )
-        elif not interface_property_names.issubset(all_member_names):
-            missing_fields = ", ".join(interface_property_names - field_names)
+        else:
             raise AnalyzerException(
-                f"{special_form} definition missing interface properties for "
-                f"interface {interface.form}: {missing_fields}",
+                f"{special_form} interface must be Python abstract class or object",
                 form=interface.form,
                 lisp_ast=interface,
             )
 
-        all_interface_methods.update(interface_names)
-
-    extra_methods = member_names - all_interface_methods - OBJECT_DUNDER_METHODS
-    if extra_methods:
-        extra_method_str = ", ".join(extra_methods)
-        raise AnalyzerException(
-            f"{special_form} definition for interface includes members not "
-            f"part of defined interfaces: {extra_method_str}"
-        )
+    # We cannot compute if there are extra methods defined if there are any
+    # unverifiably abstract bases, so we just skip this check.
+    if not unverifiably_abstract:
+        extra_methods = member_names - all_interface_methods - OBJECT_DUNDER_METHODS
+        if extra_methods and not extra_methods.issubset(
+            artificially_abstract_base_members
+        ):
+            extra_method_str = ", ".join(extra_methods)
+            raise AnalyzerException(
+                f"{special_form} definition for interface includes members not "
+                f"part of defined interfaces: {extra_method_str}"
+            )
 
-    return True
+    return not unverifiably_abstract, lset.set(artificially_abstract)
 
 
 __DEFTYPE_DEFAULT_SENTINEL = object()
 
 
-def _deftype_ast(  # pylint: disable=too-many-branches
+def _deftype_ast(  # pylint: disable=too-many-branches,too-many-locals
     ctx: AnalyzerContext, form: ISeq
 ) -> DefType:
     assert form.first == SpecialForm.DEFTYPE
@@ -1684,7 +1741,10 @@ def _deftype_ast(  # pylint: disable=too-many-branches
         interfaces, members = __deftype_or_reify_impls(
             ctx, runtime.nthrest(form, 3), SpecialForm.DEFTYPE
         )
-        verified_abstract = __deftype_and_reify_impls_are_all_abstract(
+        (
+            verified_abstract,
+            artificially_abstract,
+        ) = __deftype_and_reify_impls_are_all_abstract(
             SpecialForm.DEFTYPE, map(lambda f: f.name, fields), interfaces, members
         )
         return DefType(
@@ -1694,6 +1754,7 @@ def _deftype_ast(  # pylint: disable=too-many-branches
             fields=vec.vector(param_nodes),
             members=vec.vector(members),
             verified_abstract=verified_abstract,
+            artificially_abstract=artificially_abstract,
             is_frozen=is_frozen,
             env=ctx.get_node_env(pos=ctx.syntax_position),
         )
@@ -2552,14 +2613,18 @@ def _reify_ast(ctx: AnalyzerContext, form: ISeq) -> Reify:
         interfaces, members = __deftype_or_reify_impls(
             ctx, runtime.nthrest(form, 1), SpecialForm.REIFY
         )
-        verified_abstract = __deftype_and_reify_impls_are_all_abstract(
+        (
+            verified_abstract,
+            artificially_abstract,
+        ) = __deftype_and_reify_impls_are_all_abstract(
             SpecialForm.REIFY, (), interfaces, members
         )
         return Reify(
             form=form,
             interfaces=vec.vector(interfaces),
             members=vec.vector(members),
             verified_abstract=verified_abstract,
+            artificially_abstract=artificially_abstract,
             env=ctx.get_node_env(pos=ctx.syntax_position),
         )
 

src/basilisp/lang/compiler/constants.py

@@ -31,6 +31,7 @@ class SpecialForm:
 
 DEFAULT_COMPILER_FILE_PATH = "NO_SOURCE_PATH"
 
+SYM_ABSTRACT_META_KEY = kw.keyword("abstract")
 SYM_ASYNC_META_KEY = kw.keyword("async")
 SYM_KWARGS_META_KEY = kw.keyword("kwargs")
 SYM_PRIVATE_META_KEY = kw.keyword("private")

src/basilisp/lang/compiler/generator.py

@@ -52,6 +52,7 @@
     ConstType,
     Def,
     DefType,
+    DefTypeBase,
     DefTypeClassMethod,
     DefTypeMember,
     DefTypeMethod,
@@ -397,6 +398,7 @@ def _class_ast(  # pylint: disable=too-many-arguments
     fields: Iterable[str] = (),
     members: Iterable[str] = (),
     verified_abstract: bool = False,
+    artificially_abstract_bases: Iterable[ast.AST] = (),
     is_frozen: bool = True,
     use_slots: bool = _ATTR_SLOTS_ON,
 ) -> ast.ClassDef:
@@ -426,6 +428,10 @@ def _class_ast(  # pylint: disable=too-many-arguments
                                 arg="interfaces",
                                 value=ast.Tuple(elts=list(bases), ctx=ast.Load()),
                             ),
+                            ast.keyword(
+                                arg="artificially_abstract_bases",
+                                value=ast.Set(elts=list(artificially_abstract_bases)),
+                            ),
                             ast.keyword(
                                 arg="members",
                                 value=ast.Tuple(
@@ -1268,13 +1274,15 @@ def __deftype_member_to_py_ast(
 
 
 def __deftype_or_reify_bases_to_py_ast(
-    ctx: GeneratorContext, node: Union[DefType, Reify]
+    ctx: GeneratorContext,
+    node: Union[DefType, Reify],
+    interfaces: Iterable[DefTypeBase],
 ) -> List[ast.AST]:
     """Return a list of AST nodes for the base classes for a `deftype*` or `reify*`."""
     assert node.op in {NodeOp.DEFTYPE, NodeOp.REIFY}
 
     bases: List[ast.AST] = []
-    for base in node.interfaces:
+    for base in interfaces:
         base_node = gen_py_ast(ctx, base)
         assert (
             count(base_node.dependencies) == 0
@@ -1316,7 +1324,10 @@ def _deftype_to_py_ast(  # pylint: disable=too-many-branches,too-many-locals
     type_name = munge(node.name)
     ctx.symbol_table.new_symbol(sym.symbol(node.name), type_name, LocalType.DEFTYPE)
 
-    bases = __deftype_or_reify_bases_to_py_ast(ctx, node)
+    bases = __deftype_or_reify_bases_to_py_ast(ctx, node, node.interfaces)
+    artificially_abstract_bases = __deftype_or_reify_bases_to_py_ast(
+        ctx, node, node.artificially_abstract
+    )
 
     with ctx.new_symbol_table(node.name):
         fields = []
@@ -1363,6 +1374,7 @@ def _deftype_to_py_ast(  # pylint: disable=too-many-branches,too-many-locals
                             fields=fields,
                             members=node.python_member_names,
                             verified_abstract=node.verified_abstract,
+                            artificially_abstract_bases=artificially_abstract_bases,
                             is_frozen=node.is_frozen,
                             use_slots=True,
                         ),
@@ -2374,8 +2386,11 @@ def _reify_to_py_ast(
 
     bases: List[ast.AST] = [
         _BASILISP_WITH_META_INTERFACE_NAME,
-        *__deftype_or_reify_bases_to_py_ast(ctx, node),
+        *__deftype_or_reify_bases_to_py_ast(ctx, node, node.interfaces),
     ]
+    artificially_abstract_bases = __deftype_or_reify_bases_to_py_ast(
+        ctx, node, node.artificially_abstract
+    )
     type_name = munge(genname("ReifiedType"))
 
     with ctx.new_symbol_table("reify"):
@@ -2454,6 +2469,7 @@ def _reify_to_py_ast(
                                 ["meta", "with_meta"], node.python_member_names
                             ),
                             verified_abstract=node.verified_abstract,
+                            artificially_abstract_bases=artificially_abstract_bases,
                             is_frozen=True,
                             use_slots=_ATTR_SLOTS_ON,
                         )

src/basilisp/lang/compiler/nodes.py

@@ -20,7 +20,7 @@
 import basilisp.lang.set as lset
 import basilisp.lang.symbol as sym
 import basilisp.lang.vector as vec
-from basilisp.lang.interfaces import IPersistentMap, IPersistentVector
+from basilisp.lang.interfaces import IPersistentMap, IPersistentSet, IPersistentVector
 from basilisp.lang.runtime import Namespace, Var, to_lisp
 from basilisp.lang.typing import LispForm, ReaderForm as ReaderLispForm, SpecialForm
 from basilisp.lang.util import munge
@@ -405,6 +405,7 @@ class DefType(Node[SpecialForm]):
     members: Iterable["DefTypeMember"]
     env: NodeEnv
     verified_abstract: bool = False
+    artificially_abstract: IPersistentSet[DefTypeBase] = lset.Set.empty()
     is_frozen: bool = True
     meta: NodeMeta = None
     children: Sequence[kw.Keyword] = vec.v(FIELDS, MEMBERS)
@@ -802,6 +803,7 @@ class Reify(Node[SpecialForm]):
     members: Iterable["DefTypeMember"]
     env: NodeEnv
     verified_abstract: bool = False
+    artificially_abstract: IPersistentSet[DefTypeBase] = lset.Set.empty()
     meta: NodeMeta = None
     children: Sequence[kw.Keyword] = vec.v(MEMBERS)
     op: NodeOp = NodeOp.REIFY