fix: fix definition issues in record type (#2)

* fix(test): set static typing compatible to python 3.10

Signed-off-by: Cesar Berrospi Ramis <[email protected]>

* style: enforce some pre-commit hooks on tests

Enforce pre-commit hooks black, isort, autoflake, and mypy on test modules.

Signed-off-by: Cesar Berrospi Ramis <[email protected]>

* fix(rec): fix definition issues in attribute, predicate, subject

Remove duplicate generic types across base and predicate modules.
Create an identifier class for subject names.
Remove unnecessary type variables in attribute model.

Signed-off-by: Cesar Berrospi Ramis <[email protected]>

* docs: refer to Docling data objects

Signed-off-by: Cesar Berrospi Ramis <[email protected]>

---------

Signed-off-by: Cesar Berrospi Ramis <[email protected]>

Author: ceberam

.pre-commit-config.yaml

@@ -4,31 +4,31 @@ repos:
     hooks:
       - id: black
         name: Black
-        entry: poetry run black docling_core
+        entry: poetry run black docling_core test
         pass_filenames: false
         language: system
         files: '\.py$'
   - repo: local
     hooks:
       - id: isort
         name: isort
-        entry: poetry run isort docling_core
+        entry: poetry run isort docling_core test
         pass_filenames: false
         language: system
         files: '\.py$'
   - repo: local
     hooks:
       - id: autoflake
         name: autoflake
-        entry: poetry run autoflake docling_core
+        entry: poetry run autoflake docling_core test
         pass_filenames: false
         language: system
         files: '\.py$'
   - repo: local
     hooks:
       - id: mypy
         name: MyPy
-        entry: poetry run mypy docling_core
+        entry: poetry run mypy docling_core test
         pass_filenames: false
         language: system
         files: '\.py$'

CONTRIBUTING.md

@@ -183,7 +183,7 @@ Note: Checks like `Black` and `isort` will _fail_ if they modify files. This is
 
 ## Documentation
 
-We use [JSON Schema for Humans](https://github.com/coveooss/json-schema-for-humans) to generate Markdown pages documenting the JSON schema of the Deep Search objects.
+We use [JSON Schema for Humans](https://github.com/coveooss/json-schema-for-humans) to generate Markdown pages documenting the JSON schema of the Docling objects.
 
 The documentation pages are stored in [docs](./docs/) folder and are updated at every commit, as part of the `pre-commit` check hooks.
 To generate the documentation on-demand, run:

docling_core/types/base.py

@@ -39,6 +39,10 @@
 PredicateKeyTypeT = TypeVar("PredicateKeyTypeT", bound=str)
 ProvenanceTypeT = TypeVar("ProvenanceTypeT", bound=str)
 CollectionNameTypeT = TypeVar("CollectionNameTypeT", bound=str)
+Coordinates = Annotated[
+    list[float],
+    Field(min_length=2, max_length=2, json_schema_extra=es_field(type="geo_point")),
+]
 T = TypeVar("T", bound=Hashable)
 
 UniqueList = Annotated[
@@ -61,7 +65,7 @@
 
 
 class Identifier(AliasModel, Generic[IdentifierTypeT], extra="forbid"):
-    """Unique identifier of a Deep Search data object."""
+    """Unique identifier of a Docling data object."""
 
     type_: IdentifierTypeT = Field(
         alias="type",
@@ -81,7 +85,7 @@ class Identifier(AliasModel, Generic[IdentifierTypeT], extra="forbid"):
         alias="_name",
         title="_Name",
         description=(
-            "A unique identifier of the data object across Deep Search, consisting of "
+            "A unique identifier of the data object across Docling, consisting of "
             "the concatenation of type and value in lower case, separated by hash "
             "(#)."
         ),
@@ -118,7 +122,7 @@ class Log(AliasModel, extra="forbid"):
         json_schema_extra=es_field(type="keyword", ignore_above=8191),
     )
     agent: StrictStr = Field(
-        description="The Deep Search agent that performed the task, e.g., CCS or CXS.",
+        description="The Docling agent that performed the task, e.g., CCS or CXS.",
         json_schema_extra=es_field(type="keyword", ignore_above=8191),
     )
     type_: StrictStr = Field(
@@ -138,7 +142,7 @@ class Log(AliasModel, extra="forbid"):
 
 
 class FileInfoObject(AliasModel):
-    """Filing information for any data object to be stored in a Deep Search database."""
+    """Filing information for any data object to be stored in a Docling database."""
 
     filename: StrictStr = Field(
         description="The name of a persistent object that created this data object",
@@ -156,15 +160,15 @@ class FileInfoObject(AliasModel):
     document_hash: StrictStr = Field(
         description=(
             "A unique identifier of this data object within a collection of a "
-            "Deep Search database"
+            "Docling database"
         ),
         alias="document-hash",
         json_schema_extra=es_field(type="keyword", ignore_above=8191),
     )
 
 
 class CollectionTypeEnum(str, Enum):
-    """Enumeration of valid Deep Search collection types."""
+    """Enumeration of valid Docling collection types."""
 
     generic = "Generic"
     document = "Document"

docling_core/types/doc/document.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: MIT
 #
 
-"""Models for the Deep Search Document data type."""
+"""Models for the Docling Document data type."""
 
 from datetime import datetime
 from typing import Generic, Optional, Union
@@ -352,7 +352,7 @@ class ExportedCCSDocument(
         CollectionNameTypeT,
     ],
 ):
-    """Document model for Deep Search."""
+    """Document model for Docling."""
 
     obj_type: StrictStr = Field(
         "pdf-document",

docling_core/types/rec/attribute.py

@@ -6,7 +6,7 @@
 """Define the model Attribute."""
 from typing import Generic, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
 from typing_extensions import Annotated
 
 from docling_core.search.mapping import es_field
@@ -16,23 +16,20 @@
     PredicateKeyTypeT,
     PredicateValueTypeT,
     ProvenanceTypeT,
-    SubjectNameTypeT,
-    SubjectTypeT,
 )
 from docling_core.types.rec.base import ProvenanceItem
 from docling_core.types.rec.predicate import Predicate
+from docling_core.utils.alias import AliasModel
 
 
 class Attribute(
-    BaseModel,
+    AliasModel,
     Generic[
         IdentifierTypeT,
         PredicateValueTypeT,
         PredicateKeyNameT,
         PredicateKeyTypeT,
         ProvenanceTypeT,
-        SubjectTypeT,
-        SubjectNameTypeT,
     ],
     extra="forbid",
 ):

docling_core/types/rec/predicate.py

@@ -5,7 +5,7 @@
 
 """Define the model Predicate."""
 from datetime import datetime
-from typing import Annotated, Generic, Optional, TypeVar
+from typing import Annotated, Generic, Optional
 
 from pydantic import (
     BaseModel,
@@ -17,16 +17,14 @@
 )
 
 from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    Coordinates,
+    PredicateKeyNameT,
+    PredicateKeyTypeT,
+    PredicateValueTypeT,
+)
 from docling_core.utils.alias import AliasModel
 
-PredicateValueTypeT = TypeVar("PredicateValueTypeT", bound=str)
-PredicateKeyNameT = TypeVar("PredicateKeyNameT", bound=str)
-PredicateKeyTypeT = TypeVar("PredicateKeyTypeT", bound=str)
-Coordinates = Annotated[
-    list[float],
-    Field(min_length=2, max_length=2, json_schema_extra=es_field(type="geo_point")),
-]
-
 
 class NumericalValue(BaseModel, extra="forbid"):
     """Model for numerical values."""
@@ -117,7 +115,7 @@ class PredicateValue(AliasModel, Generic[PredicateValueTypeT], extra="forbid"):
 
 
 class Predicate(
-    BaseModel,
+    AliasModel,
     Generic[PredicateValueTypeT, PredicateKeyNameT, PredicateKeyTypeT],
     extra="forbid",
 ):

docling_core/types/rec/record.py

@@ -80,8 +80,6 @@ class Record(
                 PredicateKeyNameT,
                 PredicateKeyTypeT,
                 ProvenanceTypeT,
-                SubjectTypeT,
-                SubjectNameTypeT,
             ]
         ]
     ] = None

docling_core/types/rec/subject.py

@@ -19,6 +19,10 @@
 from docling_core.utils.alias import AliasModel
 
 
+class SubjectNameIdentifier(Identifier[SubjectNameTypeT], Generic[SubjectNameTypeT]):
+    """Identifier of subject names.""" ""
+
+
 class Subject(
     AliasModel,
     Generic[IdentifierTypeT, SubjectTypeT, SubjectNameTypeT],
@@ -53,7 +57,7 @@ class Subject(
         ),
         json_schema_extra=es_field(type="keyword", ignore_above=8191),
     )
-    names: list[Identifier[SubjectNameTypeT]] = Field(
+    names: list[SubjectNameIdentifier[SubjectNameTypeT]] = Field(
         description=(
             "List of given names for this subject. They may not be unique across "
             "different subjects."

docling_core/utils/ds_generate_docs.py

@@ -44,7 +44,7 @@ def _prepare_directory(folder: str, clean: bool = False) -> None:
 
 
 def generate_collection_jsonschema(folder: str):
-    """Generate the JSON schema of Deep Search collections and export them to a folder.
+    """Generate the JSON schema of Docling collections and export them to a folder.
 
     Args:
         folder: The name of the directory.
@@ -58,7 +58,7 @@ def generate_collection_jsonschema(folder: str):
 
 
 def generate_collection_html(folder: str):
-    """Generate HTML pages documenting the data model of Deep Search collections.
+    """Generate HTML pages documenting the data model of Docling collections.
 
     The JSON schemas files need to be in a folder and the generated HTML pages will be
     written in the same folder.
@@ -79,7 +79,7 @@ def generate_collection_html(folder: str):
 
 
 def generate_collection_markdown(folder: str):
-    """Generate Markdown pages documenting the data model of Deep Search collections.
+    """Generate Markdown pages documenting the data model of Docling collections.
 
     The JSON schemas files need to be in a folder and the generated markdown pages will
     be written in the same folder.
@@ -101,7 +101,7 @@ def generate_collection_markdown(folder: str):
 
 
 def main() -> None:
-    """Generate the JSON Schema of Deep Search collections and export documentation."""
+    """Generate the JSON Schema of Docling collections and export documentation."""
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
         "directory",

docs/Document.json

@@ -781,7 +781,7 @@
           "x-es-type": "keyword"
         },
         "document-hash": {
-          "description": "A unique identifier of this data object within a collection of a Deep Search database",
+          "description": "A unique identifier of this data object within a collection of a Docling database",
           "title": "Document-Hash",
           "type": "string",
           "x-es-ignore_above": 8191,
@@ -1086,7 +1086,7 @@
     },
     "Identifier": {
       "additionalProperties": false,
-      "description": "Unique identifier of a Deep Search data object.",
+      "description": "Unique identifier of a Docling data object.",
       "properties": {
         "type": {
           "description": "A string representing a collection or database that contains this data object.",
@@ -1103,7 +1103,7 @@
           "x-es-type": "keyword"
         },
         "_name": {
-          "description": "A unique identifier of the data object across Deep Search, consisting of the concatenation of type and value in lower case, separated by hash (#).",
+          "description": "A unique identifier of the data object across Docling, consisting of the concatenation of type and value in lower case, separated by hash (#).",
           "pattern": "^.+#.+$",
           "title": "_Name",
           "type": "string",
@@ -1139,7 +1139,7 @@
           "x-es-type": "keyword"
         },
         "agent": {
-          "description": "The Deep Search agent that performed the task, e.g., CCS or CXS.",
+          "description": "The Docling agent that performed the task, e.g., CCS or CXS.",
           "title": "Agent",
           "type": "string",
           "x-es-ignore_above": 8191,
@@ -1725,7 +1725,7 @@
       "type": "object"
     }
   },
-  "description": "Document model for Deep Search.",
+  "description": "Document model for Docling.",
   "properties": {
     "_name": {
       "title": " Name",