Merge branch 'main' into examples

Author: badmonster0

examples/manuals_llm_extraction/pyproject.toml

@@ -3,7 +3,7 @@ name = "manuals-llm-extraction"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.79", "marker-pdf>=1.5.2"]
+dependencies = ["cocoindex>=0.1.79", "marker-pdf>=1.8.5"]
 
 [tool.setuptools]
 packages = []

examples/paper_metadata/pyproject.toml

@@ -6,7 +6,7 @@ requires-python = ">=3.11"
 dependencies = [
     "cocoindex[embeddings]>=0.1.83",
     "pypdf>=5.7.0",
-    "marker-pdf>=1.5.2",
+    "marker-pdf>=1.8.5",
 ]
 
 [tool.setuptools]

examples/pdf_embedding/pyproject.toml

@@ -6,7 +6,7 @@ requires-python = ">=3.11"
 dependencies = [
     "cocoindex[embeddings]>=0.1.79",
     "python-dotenv>=1.0.1",
-    "marker-pdf>=1.5.2",
+    "marker-pdf>=1.8.5",
     "psycopg[binary,pool]",
     "jinja2>=3.1.6",
 ]

python/cocoindex/convert.py

@@ -28,7 +28,6 @@
     encode_enriched_type,
     is_namedtuple_type,
     is_numpy_number_type,
-    is_struct_type,
 )
 
 
@@ -88,38 +87,25 @@ def encode_struct_list(value: Any) -> Any:
 
             return encode_struct_list
 
-    if isinstance(variant, AnalyzedDictType):
-        if not variant.value_type:
-            return lambda value: value
+        # Otherwise it's a vector, falling into basic type in the engine.
 
+    if isinstance(variant, AnalyzedDictType):
         value_type_info = analyze_type_info(variant.value_type)
-        if isinstance(value_type_info.variant, AnalyzedStructType):
-
-            def encode_struct_dict(value: Any) -> Any:
-                if not isinstance(value, dict):
-                    return value
-                if not value:
-                    return []
-
-                sample_key, sample_val = next(iter(value.items()))
-                key_type, val_type = type(sample_key), type(sample_val)
-
-                # Handle KTable case
-                if value and is_struct_type(val_type):
-                    key_encoder = (
-                        make_engine_value_encoder(analyze_type_info(key_type))
-                        if is_struct_type(key_type)
-                        else make_engine_value_encoder(ANY_TYPE_INFO)
-                    )
-                    value_encoder = make_engine_value_encoder(
-                        analyze_type_info(val_type)
-                    )
-                    return [
-                        [key_encoder(k)] + value_encoder(v) for k, v in value.items()
-                    ]
-                return {key_encoder(k): value_encoder(v) for k, v in value.items()}
+        if not isinstance(value_type_info.variant, AnalyzedStructType):
+            raise ValueError(
+                f"Value type for dict is required to be a struct (e.g. dataclass or NamedTuple), got {variant.value_type}. "
+                f"If you want a free-formed dict, use `cocoindex.Json` instead."
+            )
 
-            return encode_struct_dict
+        key_encoder = make_engine_value_encoder(analyze_type_info(variant.key_type))
+        value_encoder = make_engine_value_encoder(analyze_type_info(variant.value_type))
+
+        def encode_struct_dict(value: Any) -> Any:
+            if not value:
+                return []
+            return [[key_encoder(k)] + value_encoder(v) for k, v in value.items()]
+
+        return encode_struct_dict
 
     if isinstance(variant, AnalyzedStructType):
         struct_type = variant.struct_type
@@ -132,8 +118,8 @@ def encode_struct_dict(value: Any) -> Any:
             field_names = [f.name for f in fields]
 
             def encode_dataclass(value: Any) -> Any:
-                if not dataclasses.is_dataclass(value):
-                    return value
+                if value is None:
+                    return None
                 return [
                     encoder(getattr(value, name))
                     for encoder, name in zip(field_encoders, field_names)
@@ -154,8 +140,8 @@ def encode_dataclass(value: Any) -> Any:
             ]
 
             def encode_namedtuple(value: Any) -> Any:
-                if not is_namedtuple_type(type(value)):
-                    return value
+                if value is None:
+                    return None
                 return [
                     encoder(getattr(value, name))
                     for encoder, name in zip(field_encoders, field_names)