feat: Add field-level descriptions to FieldSchema

- Add description field to Rust FieldSchema struct (Arc<str>)
- Update Python FieldSchema to include description field
- Extract Pydantic field descriptions during schema creation
- Update JSON schema builder to include field descriptions
- Add comprehensive tests for field description functionality

Resolves cocoindex-io#1074

Author: belloibrahv

python/cocoindex/tests/test_convert.py

@@ -1715,3 +1715,81 @@ class MixedStruct:
     order = OrderPydantic(order_id="O1", name="item1", price=10.0)
     mixed = MixedStruct(name="test", pydantic_order=order)
     validate_full_roundtrip(mixed, MixedStruct)
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_field_descriptions() -> None:
+    """Test that Pydantic field descriptions are extracted and included in schema."""
+    from pydantic import BaseModel, Field
+
+    class UserWithDescriptions(BaseModel):
+        """A user model with field descriptions."""
+        name: str = Field(description="The user's full name")
+        age: int = Field(description="The user's age in years", ge=0, le=150)
+        email: str = Field(description="The user's email address")
+        is_active: bool = Field(description="Whether the user account is active", default=True)
+
+    # Test that field descriptions are extracted
+    encoded_schema = dump_engine_object(UserWithDescriptions)
+    
+    # Check that the schema contains field descriptions
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+    
+    # Find fields by name and check descriptions
+    field_descriptions = {field["name"]: field.get("description") for field in fields}
+    
+    assert field_descriptions["name"] == "The user's full name"
+    assert field_descriptions["age"] == "The user's age in years"
+    assert field_descriptions["email"] == "The user's email address"
+    assert field_descriptions["is_active"] == "Whether the user account is active"
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_field_descriptions_without_field() -> None:
+    """Test that Pydantic models without field descriptions work correctly."""
+    from pydantic import BaseModel
+
+    class UserWithoutDescriptions(BaseModel):
+        """A user model without field descriptions."""
+        name: str
+        age: int
+        email: str
+
+    # Test that the schema works without descriptions
+    encoded_schema = dump_engine_object(UserWithoutDescriptions)
+    
+    # Check that the schema contains fields but no descriptions
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+    
+    # Verify no descriptions are present
+    for field in fields:
+        assert "description" not in field or field["description"] is None
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_mixed_descriptions() -> None:
+    """Test Pydantic model with some fields having descriptions and others not."""
+    from pydantic import BaseModel, Field
+
+    class MixedDescriptions(BaseModel):
+        """A model with mixed field descriptions."""
+        name: str = Field(description="The name field")
+        age: int  # No description
+        email: str = Field(description="The email field")
+        active: bool  # No description
+
+    # Test that only fields with descriptions have them in the schema
+    encoded_schema = dump_engine_object(MixedDescriptions)
+    
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+    
+    # Find fields by name and check descriptions
+    field_descriptions = {field["name"]: field.get("description") for field in fields}
+    
+    assert field_descriptions["name"] == "The name field"
+    assert field_descriptions["age"] is None
+    assert field_descriptions["email"] == "The email field"
+    assert field_descriptions["active"] is None

python/cocoindex/typing.py

@@ -359,7 +359,7 @@ def _encode_struct_schema(
 ) -> tuple[dict[str, Any], int | None]:
     fields = []
 
-    def add_field(name: str, analyzed_type: AnalyzedTypeInfo) -> None:
+    def add_field(name: str, analyzed_type: AnalyzedTypeInfo, description: str | None = None) -> None:
         try:
             type_info = encode_enriched_type_info(analyzed_type)
         except ValueError as e:
@@ -369,6 +369,8 @@ def add_field(name: str, analyzed_type: AnalyzedTypeInfo) -> None:
             )
             raise
         type_info["name"] = name
+        if description is not None:
+            type_info["description"] = description
         fields.append(type_info)
 
     def add_fields_from_struct(struct_type: type) -> None:
@@ -384,7 +386,9 @@ def add_fields_from_struct(struct_type: type) -> None:
                 for name, field_info in struct_type.model_fields.items():  # type: ignore[attr-defined]
                     # Get the annotation from the field info
                     field_type = field_info.annotation
-                    add_field(name, analyze_type_info(field_type))
+                    # Extract description from Pydantic field info
+                    description = getattr(field_info, 'description', None)
+                    add_field(name, analyze_type_info(field_type), description)
             else:
                 raise ValueError(f"Invalid Pydantic model: {struct_type}")
         else:
@@ -624,14 +628,21 @@ def encode(self) -> dict[str, Any]:
 class FieldSchema:
     name: str
     value_type: EnrichedValueType
+    description: str | None = None
 
     @staticmethod
     def decode(obj: dict[str, Any]) -> "FieldSchema":
-        return FieldSchema(name=obj["name"], value_type=EnrichedValueType.decode(obj))
+        return FieldSchema(
+            name=obj["name"], 
+            value_type=EnrichedValueType.decode(obj),
+            description=obj.get("description")
+        )
 
     def encode(self) -> dict[str, Any]:
         result = self.value_type.encode()
         result["name"] = self.name
+        if self.description is not None:
+            result["description"] = self.description
         return result
 
 

src/base/json_schema.rs

@@ -219,6 +219,10 @@ impl JsonSchemaBuilder {
                             *instance_type = SingleOrVec::Vec(types);
                         }
                     }
+                    // Set field description if available
+                    if let Some(description) = &f.description {
+                        self.set_description(&mut schema, description, field_path.prepend(&f.name));
+                    }
                     (f.name.to_string(), schema.into())
                 })
                 .collect(),

src/base/schema.rs

@@ -325,20 +325,38 @@ pub struct FieldSchema<DataType = ValueType> {
 
     #[serde(flatten)]
     pub value_type: EnrichedValueType<DataType>,
+
+    /// Optional description for the field.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub description: Option<Arc<str>>,
 }
 
 impl FieldSchema {
     pub fn new(name: impl ToString, value_type: EnrichedValueType) -> Self {
         Self {
             name: name.to_string(),
             value_type,
+            description: None,
+        }
+    }
+
+    pub fn new_with_description(
+        name: impl ToString,
+        value_type: EnrichedValueType,
+        description: Option<impl ToString>,
+    ) -> Self {
+        Self {
+            name: name.to_string(),
+            value_type,
+            description: description.map(|d| d.to_string().into()),
         }
     }
 
     pub fn without_attrs(&self) -> Self {
         Self {
             name: self.name.clone(),
             value_type: self.value_type.without_attrs(),
+            description: None,
         }
     }
 }
@@ -351,6 +369,7 @@ impl<DataType> FieldSchema<DataType> {
         Ok(Self {
             name: field.name.clone(),
             value_type: EnrichedValueType::from_alternative(&field.value_type)?,
+            description: field.description.clone(),
         })
     }
 }