feat:Support geo data type for insert, query and search (#2925) (#3043)

issue: #2924

Support geometry types for pymilvus

This pull request adds support for the `GEOMETRY` data type throughout
the bulk writing and search result handling code in PyMilvus. The main
changes ensure that geometry data is correctly validated, encoded,
decoded, and handled in all relevant utility and helper functions.
Geometry values are stored as bytes (with string values encoded as
UTF-8), and decoding is performed when reading results.

**Support for GEOMETRY data type**

* Added validation logic for `DataType.GEOMETRY` in `constants.py`,
allowing geometry data to be either `str` or `bytes`.
* Implemented encoding of geometry values to bytes in
`pack_field_value_to_field_data` and `entity_to_field_data`, handling
both string (WKT) and bytes input.

[[1]](diffhunk://#diff-0c1494647658a30477e2d780c6bc64f56ef49b14c79027428e3cd70bb491c3c5R449-R469)
[[2]](diffhunk://#diff-0c1494647658a30477e2d780c6bc64f56ef49b14c79027428e3cd70bb491c3c5R560-R573)

**Geometry data decoding and assignment**

* Updated assignment and append logic in `entity_helper.py` to decode
geometry bytes back to strings when possible, both in batch assignment
and single row extraction.

[[1]](diffhunk://#diff-0c1494647658a30477e2d780c6bc64f56ef49b14c79027428e3cd70bb491c3c5R760-R772)
[[2]](diffhunk://#diff-0c1494647658a30477e2d780c6bc64f56ef49b14c79027428e3cd70bb491c3c5R863-R878)

**Search result integration**

* Added `GEOMETRY` to supported types in search result parsing and
implemented decoding logic for geometry fields in search result
extraction and field data retrieval.

[[1]](diffhunk://#diff-76d170285437d4f74831de67c329c80a2e84cfa1b65375229198810a3e65b2e6R58)
[[2]](diffhunk://#diff-76d170285437d4f74831de67c329c80a2e84cfa1b65375229198810a3e65b2e6R352-R368)
[[3]](diffhunk://#diff-76d170285437d4f74831de67c329c80a2e84cfa1b65375229198810a3e65b2e6R474-R475)

**Utility function updates**

* Updated `len_of` utility to support length calculation for geometry
data fields.

---------

Signed-off-by: Yinwei Li <[email protected]>
Signed-off-by: Cai Zhang <[email protected]>
Co-authored-by: ZhuXi <[email protected]>

Author: xiaocai2333

pymilvus/bulk_writer/constants.py

@@ -52,6 +52,7 @@
     DataType.DOUBLE.name: lambda x: isinstance(x, float),
     DataType.VARCHAR.name: lambda x, max_len: isinstance(x, str) and len(x) <= max_len,
     DataType.JSON.name: lambda x: isinstance(x, (str, list, dict)),
+    DataType.GEOMETRY.name: lambda x: isinstance(x, str),
     DataType.FLOAT_VECTOR.name: lambda x, dim: float_vector_validator(x, dim),
     DataType.BINARY_VECTOR.name: lambda x, dim: binary_vector_validator(x, dim),
     DataType.FLOAT16_VECTOR.name: lambda x, dim: float16_vector_validator(x, dim, False),

pymilvus/client/entity_helper.py

@@ -499,6 +499,19 @@ def pack_field_value_to_field_data(
                 % (field_name, "varchar", type(field_value))
                 + f" Detail: {e!s}"
             ) from e
+    elif field_type == DataType.GEOMETRY:
+        try:
+            if field_value is None:
+                field_data.scalars.geometry_wkt_data.data.extend([])
+            else:
+                field_data.scalars.geometry_wkt_data.data.append(
+                    convert_to_str_array(field_value, field_info, CHECK_STR_ARRAY)
+                )
+        except (TypeError, ValueError) as e:
+            raise DataNotMatchException(
+                message=ExceptionsMessage.FieldDataInconsistent
+                % (field_name, "geometry", type(field_value))
+            ) from e
     elif field_type == DataType.JSON:
         try:
             if field_value is None:
@@ -589,6 +602,10 @@ def entity_to_field_data(entity: Dict, field_info: Any, num_rows: int) -> schema
             field_data.scalars.array_data.data.extend(
                 entity_to_array_arr(entity_values, field_info)
             )
+        elif entity_type == DataType.GEOMETRY:
+            field_data.scalars.geometry_wkt_data.data.extend(
+                entity_to_str_arr(entity_values, field_info, CHECK_STR_ARRAY)
+            )
         else:
             raise ParamError(message=f"Unsupported data type: {entity_type}")
     except (TypeError, ValueError) as e:
@@ -775,6 +792,11 @@ def assign_scalar(data: List[Any]) -> None:
         assign_scalar(data)
         return False
 
+    if field_data.type == DataType.GEOMETRY:
+        data = field_data.scalars.geometry_wkt_data.data
+        assign_scalar(data)
+        return False
+
     if field_data.type == DataType.JSON:
         return True
 
@@ -894,6 +916,18 @@ def check_append(field_data: Any, row_data: Dict):
             row_data[field_data.field_name] = field_data.scalars.string_data.data[index]
             return
 
+        if (
+            field_data.type == DataType.GEOMETRY
+            and len(field_data.scalars.geometry_wkt_data.data) >= index
+        ):
+            if len(field_data.valid_data) > 0 and field_data.valid_data[index] is False:
+                entity_row_data[field_data.field_name] = None
+                return
+            entity_row_data[field_data.field_name] = field_data.scalars.geometry_wkt_data.data[
+                index
+            ]
+            return
+
         if field_data.type == DataType.JSON and len(field_data.scalars.json_data.data) >= index:
             if len(field_data.valid_data) > 0 and field_data.valid_data[index] is False:
                 row_data[field_data.field_name] = None

pymilvus/client/search_result.py

@@ -56,6 +56,7 @@ def __init__(
                 DataType.FLOAT,
                 DataType.DOUBLE,
                 DataType.VARCHAR,
+                DataType.GEOMETRY,
             ]:
                 if has_valid:
                     [
@@ -402,6 +403,15 @@ def _get_fields_by_range(
                 )
                 continue
 
+            if dtype == DataType.GEOMETRY:
+                field2data[name] = (
+                    apply_valid_data(
+                        scalars.geometry_wkt_data.data[start:end], field.valid_data, start, end
+                    ),
+                    field_meta,
+                )
+                continue
+
             if dtype == DataType.JSON:
                 res = apply_valid_data(
                     scalars.json_data.data[start:end], field.valid_data, start, end
@@ -521,6 +531,8 @@ def get_field_data(field_data: FieldData):
         return field_data.scalars.double_data.data
     if field_data.type == DataType.VARCHAR:
         return field_data.scalars.string_data.data
+    if field_data.type == DataType.GEOMETRY:
+        return field_data.scalars.geometry_wkt_data.data
     if field_data.type == DataType.JSON:
         return field_data.scalars.json_data.data
     if field_data.type == DataType.ARRAY:

pymilvus/client/utils.py

@@ -175,6 +175,9 @@ def len_of(field_data: Any) -> int:
         if field_data.scalars.HasField("array_data"):
             return len(field_data.scalars.array_data.data)
 
+        if field_data.scalars.HasField("geometry_wkt_data"):
+            return len(field_data.scalars.geometry_wkt_data.data)
+
         raise MilvusException(message="Unsupported scalar type")
 
     if field_data.HasField("vectors"):