Merge branch 'main' into patches

Author: lemorage

README.md

@@ -18,7 +18,7 @@
 </div>
 
 <div align="center">
-<!--     <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a> -->
+    <a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </div>
 
 

docs/docs/core/data_types.mdx

@@ -19,7 +19,7 @@ You don't need to spell out any data type explicitly when you define the flow.
 All you need to do is to make sure the data passed to functions and targets are compatible with them.
 
 Each type in CocoIndex type system is mapped to one or multiple types in Python.
-When you define a [custom function](/docs/core/custom_function), you need to annotate the data types of arguments and return values.
+When you define a [custom function](/docs/custom_ops/custom_functions), you need to annotate the data types of arguments and return values.
 
 *   When you pass a Python value to the engine (e.g. return values of a custom function), a specific type annotation is required.
     The type annotation needs to be specific in describing the target data type, as it provides the ground truth of the data type in the flow.

docs/docs/core/custom_function.mdx docs/docs/custom_ops/custom_functions.mdxdocs/docs/core/custom_function.mdx renamed to docs/docs/custom_ops/custom_functions.mdx

@@ -57,6 +57,10 @@ const config: Config = {
             from: '/core/initialization',
             to: '/core/settings',
           },
+          {
+            from: '/core/custom_function',
+            to: '/custom_ops/custom_functions',
+          },
           {
             from: '/ops/storages',
             to: '/ops/targets',

docs/docusaurus.config.ts

@@ -31,7 +31,6 @@ const sidebars: SidebarsConfig = {
         'core/settings',
         'core/flow_methods',
         'core/cli',
-        'core/custom_function',
       ],
     },
     {
@@ -44,6 +43,14 @@ const sidebars: SidebarsConfig = {
         'ops/targets',
       ],
     },
+    {
+      type: 'category',
+      label: 'Custom Operations',
+      collapsed: false,
+      items: [
+        'custom_ops/custom_functions',
+      ],
+    },
     {
       type: 'category',
       label: 'AI Support',

docs/sidebars.ts

@@ -30,8 +30,12 @@ features = ["pyo3/extension-module"]
 
 [project.optional-dependencies]
 dev = ["pytest", "ruff", "mypy", "pre-commit"]
+
 embeddings = ["sentence-transformers>=3.3.1"]
-all = ["cocoindex[embeddings]"]
+
+# We need to repeat the dependency above to make it available for the `all` feature.
+# Indirect dependencies such as "cocoindex[embeddings]" will not work for local development.
+all = ["sentence-transformers>=3.3.1"]
 
 [tool.mypy]
 python_version = "3.11"

pyproject.toml

@@ -2,6 +2,8 @@
 Utilities to convert between Python and engine values.
 """
 
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import inspect
@@ -28,6 +30,24 @@
     is_struct_type,
 )
 
+class ChildFieldPath:
+    """Context manager to append a field to field_path on enter and pop it on exit."""
+
+    _field_path: list[str]
+    _field_name: str
+
+    def __init__(self, field_path: list[str], field_name: str):
+        self._field_path: list[str] = field_path
+        self._field_name = field_name
+
+    def __enter__(self) -> ChildFieldPath:
+        self._field_path.append(self._field_name)
+        return self
+
+    def __exit__(self, _exc_type: Any, _exc_val: Any, _exc_tb: Any) -> None:
+        self._field_path.pop()
+
+
 _CONVERTIBLE_KINDS = {
     ("Float32", "Float64"),
     ("LocalDateTime", "OffsetDateTime"),
@@ -48,7 +68,6 @@ def _encode_engine_value_core(
     type_variant: AnalyzedTypeInfo | None = None,
 ) -> Any:
     """Core encoding logic for converting Python values to engine values."""
-
     if dataclasses.is_dataclass(value):
         fields = dataclasses.fields(value)
         return [
@@ -200,66 +219,65 @@ def make_engine_value_decoder(
         )
 
     if src_type_kind == "Struct":
-        return _make_engine_struct_value_decoder(
+        return make_engine_struct_decoder(
             field_path,
             src_type["fields"],
             dst_type_info,
         )
 
     if src_type_kind in TABLE_TYPES:
-        field_path.append("[*]")
-        engine_fields_schema = src_type["row"]["fields"]
+        with ChildFieldPath(field_path, "[*]"):
+            engine_fields_schema = src_type["row"]["fields"]
 
-        if src_type_kind == "LTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ltable_to_list_dict_decoder(
-                    field_path, engine_fields_schema
-                )
-            if not isinstance(dst_type_variant, AnalyzedListType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a list type expected"
+            if src_type_kind == "LTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    return _make_engine_ltable_to_list_dict_decoder(
+                        field_path, engine_fields_schema
+                    )
+                if not isinstance(dst_type_variant, AnalyzedListType):
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a list type expected"
+                    )
+                row_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema,
+                    analyze_type_info(dst_type_variant.elem_type),
                 )
-            row_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema,
-                analyze_type_info(dst_type_variant.elem_type),
-            )
 
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return [row_decoder(v) for v in value]
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return [row_decoder(v) for v in value]
 
-        elif src_type_kind == "KTable":
-            if isinstance(dst_type_variant, AnalyzedAnyType):
-                return _make_engine_ktable_to_dict_dict_decoder(
-                    field_path, engine_fields_schema
+            elif src_type_kind == "KTable":
+                if isinstance(dst_type_variant, AnalyzedAnyType):
+                    return _make_engine_ktable_to_dict_dict_decoder(
+                        field_path, engine_fields_schema
+                    )
+                if not isinstance(dst_type_variant, AnalyzedDictType):
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"declared `{dst_type_info.core_type}`, a dict type expected"
+                    )
+
+                key_field_schema = engine_fields_schema[0]
+                field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
+                key_decoder = make_engine_value_decoder(
+                    field_path, key_field_schema["type"], dst_type_variant.key_type
                 )
-            if not isinstance(dst_type_variant, AnalyzedDictType):
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"declared `{dst_type_info.core_type}`, a dict type expected"
+                field_path.pop()
+                value_decoder = make_engine_struct_decoder(
+                    field_path,
+                    engine_fields_schema[1:],
+                    analyze_type_info(dst_type_variant.value_type),
                 )
 
-            key_field_schema = engine_fields_schema[0]
-            field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-            key_decoder = make_engine_value_decoder(
-                field_path, key_field_schema["type"], dst_type_variant.key_type
-            )
-            field_path.pop()
-            value_decoder = _make_engine_struct_value_decoder(
-                field_path,
-                engine_fields_schema[1:],
-                analyze_type_info(dst_type_variant.value_type),
-            )
-
-            def decode(value: Any) -> Any | None:
-                if value is None:
-                    return None
-                return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
+                def decode(value: Any) -> Any | None:
+                    if value is None:
+                        return None
+                    return {key_decoder(v[0]): value_decoder(v[1:]) for v in value}
 
-        field_path.pop()
         return decode
 
     if src_type_kind == "Union":
@@ -274,22 +292,22 @@ def decode(value: Any) -> Any | None:
         src_type_variants = src_type["types"]
         decoders = []
         for i, src_type_variant in enumerate(src_type_variants):
-            src_field_path = field_path + [f"[{i}]"]
-            decoder = None
-            for dst_type_variant in dst_type_variants:
-                try:
-                    decoder = make_engine_value_decoder(
-                        src_field_path, src_type_variant, dst_type_variant
+            with ChildFieldPath(field_path, f"[{i}]"):
+                decoder = None
+                for dst_type_variant in dst_type_variants:
+                    try:
+                        decoder = make_engine_value_decoder(
+                            field_path, src_type_variant, dst_type_variant
+                        )
+                        break
+                    except ValueError:
+                        pass
+                if decoder is None:
+                    raise ValueError(
+                        f"Type mismatch for `{''.join(field_path)}`: "
+                        f"cannot find matched target type for source type variant {src_type_variant}"
                     )
-                    break
-                except ValueError:
-                    pass
-            if decoder is None:
-                raise ValueError(
-                    f"Type mismatch for `{''.join(field_path)}`: "
-                    f"cannot find matched target type for source type variant {src_type_variant}"
-                )
-            decoders.append(decoder)
+                decoders.append(decoder)
         return lambda value: decoders[value[0]](value[1])
 
     if isinstance(dst_type_variant, AnalyzedAnyType):
@@ -368,7 +386,7 @@ def decode_scalar(value: Any) -> Any | None:
     return lambda value: value
 
 
-def _make_engine_struct_value_decoder(
+def make_engine_struct_decoder(
     field_path: list[str],
     src_fields: list[dict[str, Any]],
     dst_type_info: AnalyzedTypeInfo,
@@ -426,25 +444,24 @@ def make_closure_for_value(
         name: str, param: inspect.Parameter
     ) -> Callable[[list[Any]], Any]:
         src_idx = src_name_to_idx.get(name)
-        if src_idx is not None:
-            field_path.append(f".{name}")
-            field_decoder = make_engine_value_decoder(
-                field_path, src_fields[src_idx]["type"], param.annotation
-            )
-            field_path.pop()
-            return (
-                lambda values: field_decoder(values[src_idx])
-                if len(values) > src_idx
-                else param.default
-            )
+        with ChildFieldPath(field_path, f".{name}"):
+            if src_idx is not None:
+                field_decoder = make_engine_value_decoder(
+                    field_path, src_fields[src_idx]["type"], param.annotation
+                )
+                return (
+                    lambda values: field_decoder(values[src_idx])
+                    if len(values) > src_idx
+                    else param.default
+                )
 
-        default_value = param.default
-        if default_value is inspect.Parameter.empty:
-            raise ValueError(
-                f"Field without default value is missing in input: {''.join(field_path)}"
-            )
+            default_value = param.default
+            if default_value is inspect.Parameter.empty:
+                raise ValueError(
+                    f"Field without default value is missing in input: {''.join(field_path)}"
+                )
 
-        return lambda _: default_value
+            return lambda _: default_value
 
     field_value_decoder = [
         make_closure_for_value(name, param) for (name, param) in parameters.items()
@@ -464,13 +481,12 @@ def _make_engine_struct_to_dict_decoder(
     field_decoders = []
     for i, field_schema in enumerate(src_fields):
         field_name = field_schema["name"]
-        field_path.append(f".{field_name}")
-        field_decoder = make_engine_value_decoder(
-            field_path,
-            field_schema["type"],
-            Any,  # Use Any for recursive decoding
-        )
-        field_path.pop()
+        with ChildFieldPath(field_path, f".{field_name}"):
+            field_decoder = make_engine_value_decoder(
+                field_path,
+                field_schema["type"],
+                Any,  # Use Any for recursive decoding
+            )
         field_decoders.append((field_name, field_decoder))
 
     def decode_to_dict(values: list[Any] | None) -> dict[str, Any] | None:
@@ -527,9 +543,10 @@ def _make_engine_ktable_to_dict_dict_decoder(
     value_fields_schema = src_fields[1:]
 
     # Create decoders
-    field_path.append(f".{key_field_schema.get('name', KEY_FIELD_NAME)}")
-    key_decoder = make_engine_value_decoder(field_path, key_field_schema["type"], Any)
-    field_path.pop()
+    with ChildFieldPath(field_path, f".{key_field_schema.get('name', KEY_FIELD_NAME)}"):
+        key_decoder = make_engine_value_decoder(
+            field_path, key_field_schema["type"], Any
+        )
 
     value_decoder = _make_engine_struct_to_dict_decoder(field_path, value_fields_schema)