UDTs: complete support (#367)

* expand list_types, class and unit test to unsupported items

* integration test for unsupported UDTs

* add (now-failing) test for short-form in collection columns with prim.types

* enhance special test to guard against longform in collection column schema

* Final handling of null/omitted/incomplete UDTs in responses, with unit testing

* add maps-as-lists to the unit test for incomplete udts

* one more representation for empty-dict udts

* integration tests for nulls/partial UDTs on tables

* integration test for UDT-related filtering

* add async udt-filtering IT

* builder interface for CreateTypeDefinition

* adapt 'test_table_userdefinedtypes_[a]sync to null/partial udt final specs

Author: hemidactylus

astrapy/data/database.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, overload
 
@@ -1844,10 +1845,18 @@ def list_types(
             )
         else:
             logger.info("finished listTypes")
-            return [
-                ListTypeDescriptor._from_dict(type_json)
-                for type_json in lt_response["status"]["types"]
-            ]
+            all_types_json = lt_response["status"]["types"]
+            lt_descriptors: list[ListTypeDescriptor] = []
+            for type_json in all_types_json:
+                if ListTypeDescriptor._is_valid_dict(type_json):
+                    lt_descriptors.append(ListTypeDescriptor._from_dict(type_json))
+                else:
+                    warnings.warn(
+                        "Unexpected item encountered while reading the response of "
+                        "listTypes. the offending item will be skipped from the "
+                        f"return value of `list_types`. Its value is: '{type_json}'."
+                    )
+            return lt_descriptors
 
     def alter_type(
         self,
@@ -4001,10 +4010,18 @@ async def list_types(
             )
         else:
             logger.info("finished listTypes")
-            return [
-                ListTypeDescriptor._from_dict(type_json)
-                for type_json in lt_response["status"]["types"]
-            ]
+            all_types_json = lt_response["status"]["types"]
+            lt_descriptors: list[ListTypeDescriptor] = []
+            for type_json in all_types_json:
+                if ListTypeDescriptor._is_valid_dict(type_json):
+                    lt_descriptors.append(ListTypeDescriptor._from_dict(type_json))
+                else:
+                    warnings.warn(
+                        "Unexpected item encountered while reading the response of "
+                        "listTypes. the offending item will be skipped from the "
+                        f"return value of `list_types`. Its value is: '{type_json}'."
+                    )
+            return lt_descriptors
 
     async def alter_type(
         self,

astrapy/data/info/table_descriptor/type_creation.py

@@ -17,7 +17,11 @@
 from dataclasses import dataclass
 from typing import Any
 
-from astrapy.data.info.table_descriptor.table_columns import TableColumnTypeDescriptor
+from astrapy.data.info.table_descriptor.table_columns import (
+    TableColumnTypeDescriptor,
+    TableScalarColumnTypeDescriptor,
+)
+from astrapy.data.utils.table_types import ColumnType
 from astrapy.utils.parsing import _warn_residual_keys
 
 
@@ -30,9 +34,9 @@ class CreateTypeDefinition:
 
     See the Data API specifications for detailed specification and allowed values.
 
-    Instances of this object can be created either by passing a fully-formed
-    definition to the class constructor, or coercing an appropriately-shaped
-    plain dictionary into this class.
+    Instances of this object can be created in three ways: using a fluent interface,
+    passing a fully-formed definition to the class constructor, or coercing an
+    appropriately-shaped plain dictionary into this class.
 
     Attributes:
         fields: a map from field names to their type definition object. This follows
@@ -42,6 +46,14 @@ class CreateTypeDefinition:
             >>> from astrapy.info import CreateTypeDefinition
             >>> from astrapy.info import ColumnType, TableScalarColumnTypeDescriptor
             >>>
+            >>> type_definition_0 = (
+            ...     CreateTypeDefinition.builder()
+            ...     .add_field("tagline", ColumnType.TEXT)
+            ...     .add_field("score", ColumnType.INT)
+            ...     .add_field("height", "float")  # plain strings accepted for field types
+            ...     .build()
+            ... )
+            >>>
             >>> type_definition_1 = CreateTypeDefinition(fields={
             ...     "tagline": TableScalarColumnTypeDescriptor(ColumnType.TEXT),
             ...     "score": TableScalarColumnTypeDescriptor(ColumnType.INT),
@@ -65,6 +77,8 @@ class CreateTypeDefinition:
             ...     },
             ... }
             >>> type_definition_3_mixed = CreateTypeDefinition.coerce(fields_dict_3)
+            >>> type_definition_0 == type_definition_1
+            True
             >>> type_definition_1 == type_definition_2
             True
             >>> type_definition_2 == type_definition_3_mixed
@@ -122,3 +136,68 @@ def coerce(
             return raw_input
         else:
             return cls._from_dict(raw_input)
+
+    @staticmethod
+    def builder() -> CreateTypeDefinition:
+        """
+        Create an "empty" builder for constructing a type definition through
+        a fluent interface. The resulting object has no fields yet: those are to
+        be added progressively with the `add_field` method.
+
+        Being a "type without fields", the type definition returned by this method
+        cannot be directly used to create a type.
+
+        See the class docstring for a full example on using the fluent interface.
+
+        Returns:
+            a CreateTypeDefinition formally describing a type without fields.
+        """
+
+        return CreateTypeDefinition(fields={})
+
+    def add_field(
+        self, field_name: str, field_type: str | ColumnType
+    ) -> CreateTypeDefinition:
+        """
+        Return a new type definition object with an added field
+        of a scalar type (i.e. not a list, set or other composite type).
+        This method is for use within the fluent interface for progressively
+        building the final type definition.
+
+        See the class docstring for a full example on using the fluent interface.
+
+        Args:
+            field_name: the name of the new field to add to the type.
+            field_type: a string, or a `ColumnType` value, defining
+                the scalar type for the field.
+
+        Returns:
+            a CreateTypeDefinition obtained by adding (or replacing) the desired
+            field to this type definition.
+        """
+
+        return CreateTypeDefinition(
+            fields={
+                **self.fields,
+                **{
+                    field_name: TableScalarColumnTypeDescriptor(
+                        column_type=ColumnType.coerce(field_type)
+                    )
+                },
+            },
+        )
+
+    def build(self) -> CreateTypeDefinition:
+        """
+        The final step in the fluent (builder) interface. Calling this method
+        finalizes the definition that has been built so far and makes it into a
+        type definition ready for use e.g. with the database's `create_type` method.
+
+        See the class docstring for a full example on using the fluent interface.
+
+        Returns:
+            a CreateTypeDefinition obtained by finalizing the definition being
+                built so far.
+        """
+
+        return self

astrapy/data/info/table_descriptor/type_listing.py

@@ -14,13 +14,15 @@
 
 from __future__ import annotations
 
-import warnings
 from dataclasses import dataclass
 from typing import Any
 
 from astrapy.data.info.table_descriptor.table_columns import TableAPISupportDescriptor
 from astrapy.data.info.table_descriptor.type_creation import CreateTypeDefinition
-from astrapy.data.utils.table_types import TableUDTColumnType
+from astrapy.data.utils.table_types import (
+    TableUDTColumnType,
+    TableUnsupportedColumnType,
+)
 from astrapy.utils.parsing import _warn_residual_keys
 
 
@@ -29,47 +31,78 @@ class ListTypeDescriptor:
     """
     A structure describing a user-defined type (UDT) stored on the database.
 
-    This description provides all information, including the UDT name as found
-    on the database and possibly a sub-object detailing the allowed operations
-    with the UDT. `ListTypeDescriptor` objects are used for the return
-    type of the database `list_types` method.
+    This object is used for the items returned by the database `list_types` method.
+    `ListTypeDescriptor` expresses all information received by the Data API, including
+    (when provided) the UDT name as found on the database, the UDT name and possibly a
+    sub-object detailing the allowed operations with the UDT.
+
+    This object must be able to describe any item returned from the Data API:
+    this means it can describe "unsupported" UDTs as well (i.e. those which have been
+    created outside of the Data API). Unsupported UDTs lack some attributes compared
+    to the fully-supported ones.
 
     Attributes:
+        udt_type: a value of either the TableUDTColumnType or
+            the TableUnsupportedColumnType enum, depending on the UDT support status.
         udt_name: the name of the UDT as is stored in the database (and in a keyspace).
         definition: the definition of the type, i.e. its fields and their types.
         api_support: a structure detailing what operations the type supports.
     """
 
-    udt_name: str
-    definition: CreateTypeDefinition
+    udt_type: TableUDTColumnType | TableUnsupportedColumnType
+    udt_name: str | None
+    definition: CreateTypeDefinition | None
     api_support: TableAPISupportDescriptor | None
 
     def __init__(
         self,
         *,
-        udt_name: str,
-        definition: CreateTypeDefinition,
+        udt_type: TableUDTColumnType | TableUnsupportedColumnType,
+        udt_name: str | None,
+        definition: CreateTypeDefinition | None,
         api_support: TableAPISupportDescriptor | None,
     ) -> None:
+        self.udt_type = udt_type
         self.udt_name = udt_name
         self.definition = definition
         self.api_support = api_support
 
     def __repr__(self) -> str:
-        return f"{self.__class__.__name__}({self.udt_name}: {self.definition})"
+        if isinstance(self.udt_type, TableUnsupportedColumnType):
+            return f"{self.__class__.__name__}({self.udt_type.value})"
+        else:
+            return f"{self.__class__.__name__}({self.udt_name}: {self.definition})"
+
+    @staticmethod
+    def _is_valid_dict(raw_dict: dict[str, Any]) -> bool:
+        """
+        Assess whether a dictionary can be converted into a ListTypeDescriptor.
+
+        This can be used by e.g. the database `list_types` method to filter
+        offending responses and issue warnings if needed.
+
+        Returns:
+            True if and only if the dict is valid, otherwise False.
+        """
+
+        return all(fld in raw_dict for fld in {"type", "apiSupport"})
 
     def as_dict(self) -> dict[str, Any]:
         """Recast this object into a dictionary."""
 
         return {
-            "type": TableUDTColumnType.USERDEFINED.value,
-            "udtName": self.udt_name,
-            "definition": self.definition.as_dict(),
-            **(
-                {"apiSupport": self.api_support.as_dict()}
+            k: v
+            for k, v in {
+                "type": self.udt_type.value,
+                "udtName": self.udt_name,
+                "definition": self.definition.as_dict()
+                if self.definition is not None
+                else None,
+                "apiSupport": self.api_support.as_dict()
                 if self.api_support is not None
-                else {}
-            ),
+                else None,
+            }.items()
+            if v is not None
         }
 
     @classmethod
@@ -82,16 +115,17 @@ def _from_dict(cls, raw_dict: dict[str, Any]) -> ListTypeDescriptor:
         _warn_residual_keys(
             cls, raw_dict, {"type", "udtName", "definition", "apiSupport"}
         )
-        if "type" in raw_dict:
-            if raw_dict["type"] != TableUDTColumnType.USERDEFINED.value:
-                warnings.warn(
-                    "Unexpected 'type' found in a UDT description from the Data API: "
-                    f"{repr(raw_dict['type'])} "
-                    f"(for user-defined type '{raw_dict.get('udtName')}')."
-                )
+        _udt_type: TableUDTColumnType | TableUnsupportedColumnType
+        if raw_dict["type"] in TableUDTColumnType:
+            _udt_type = TableUDTColumnType.coerce(raw_dict["type"])
+        else:
+            _udt_type = TableUnsupportedColumnType.coerce(raw_dict["type"])
         return ListTypeDescriptor(
-            udt_name=raw_dict["udtName"],
-            definition=CreateTypeDefinition._from_dict(raw_dict["definition"]),
+            udt_type=_udt_type,
+            udt_name=raw_dict.get("udtName"),
+            definition=CreateTypeDefinition._from_dict(raw_dict["definition"])
+            if "definition" in raw_dict
+            else None,
             api_support=TableAPISupportDescriptor._from_dict(raw_dict["apiSupport"])
             if "apiSupport" in raw_dict
             else None,