feat: Add field-level descriptions to FieldSchema (#1087)

* feat: Add field-level descriptions to FieldSchema

- Add description field to Rust FieldSchema struct (Arc<str>)
- Update Python FieldSchema to include description field
- Extract Pydantic field descriptions during schema creation
- Update JSON schema builder to include field descriptions
- Add comprehensive tests for field description functionality

Resolves #1074

* fix: concatenate field and type descriptions instead of overwriting

- Modified set_description method to append descriptions with newline separator
- Fixed description overwrite issue where field-level descriptions would replace type-level descriptions
- Added description: None to all FieldSchema initializations to satisfy new field requirement
- Added comprehensive test to verify description concatenation works correctly
- All existing tests pass (129 passed, 0 failed) with no regressions detected
- Implementation handles both extract_descriptions: true and false scenarios
- Field-level descriptions now properly concatenate with type-level descriptions using newline separator
- Resolves reviewer feedback about description overwrite in PR #1087
- Changes maintain backward compatibility while fixing the core issue
- Ready for review and testing by the maintainer team

* pre-commit pass

Author: belloibrahv

pyproject.toml

@@ -84,3 +84,11 @@ strict = true
 files = "python/cocoindex"
 exclude = "(\\.venv|site-packages)"
 disable_error_code = ["unused-ignore"]
+[[tool.mypy.overrides]]
+module = [
+    "sentence_transformers",
+    "torch",
+    "colpali_engine",
+    "PIL",
+]
+ignore_missing_imports = true

python/cocoindex/tests/test_convert.py

@@ -1715,3 +1715,86 @@ class MixedStruct:
     order = OrderPydantic(order_id="O1", name="item1", price=10.0)
     mixed = MixedStruct(name="test", pydantic_order=order)
     validate_full_roundtrip(mixed, MixedStruct)
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_field_descriptions() -> None:
+    """Test that Pydantic field descriptions are extracted and included in schema."""
+    from pydantic import BaseModel, Field
+
+    class UserWithDescriptions(BaseModel):
+        """A user model with field descriptions."""
+
+        name: str = Field(description="The user's full name")
+        age: int = Field(description="The user's age in years", ge=0, le=150)
+        email: str = Field(description="The user's email address")
+        is_active: bool = Field(
+            description="Whether the user account is active", default=True
+        )
+
+    # Test that field descriptions are extracted
+    encoded_schema = dump_engine_object(UserWithDescriptions)
+
+    # Check that the schema contains field descriptions
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+
+    # Find fields by name and check descriptions
+    field_descriptions = {field["name"]: field.get("description") for field in fields}
+
+    assert field_descriptions["name"] == "The user's full name"
+    assert field_descriptions["age"] == "The user's age in years"
+    assert field_descriptions["email"] == "The user's email address"
+    assert field_descriptions["is_active"] == "Whether the user account is active"
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_field_descriptions_without_field() -> None:
+    """Test that Pydantic models without field descriptions work correctly."""
+    from pydantic import BaseModel
+
+    class UserWithoutDescriptions(BaseModel):
+        """A user model without field descriptions."""
+
+        name: str
+        age: int
+        email: str
+
+    # Test that the schema works without descriptions
+    encoded_schema = dump_engine_object(UserWithoutDescriptions)
+
+    # Check that the schema contains fields but no descriptions
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+
+    # Verify no descriptions are present
+    for field in fields:
+        assert "description" not in field or field["description"] is None
+
+
+@pytest.mark.skipif(not PYDANTIC_AVAILABLE, reason="Pydantic not available")
+def test_pydantic_mixed_descriptions() -> None:
+    """Test Pydantic model with some fields having descriptions and others not."""
+    from pydantic import BaseModel, Field
+
+    class MixedDescriptions(BaseModel):
+        """A model with mixed field descriptions."""
+
+        name: str = Field(description="The name field")
+        age: int  # No description
+        email: str = Field(description="The email field")
+        active: bool  # No description
+
+    # Test that only fields with descriptions have them in the schema
+    encoded_schema = dump_engine_object(MixedDescriptions)
+
+    assert "fields" in encoded_schema["type"]
+    fields = encoded_schema["type"]["fields"]
+
+    # Find fields by name and check descriptions
+    field_descriptions = {field["name"]: field.get("description") for field in fields}
+
+    assert field_descriptions["name"] == "The name field"
+    assert field_descriptions["age"] is None
+    assert field_descriptions["email"] == "The email field"
+    assert field_descriptions["active"] is None

python/cocoindex/typing.py

@@ -359,7 +359,9 @@ def _encode_struct_schema(
 ) -> tuple[dict[str, Any], int | None]:
     fields = []
 
-    def add_field(name: str, analyzed_type: AnalyzedTypeInfo) -> None:
+    def add_field(
+        name: str, analyzed_type: AnalyzedTypeInfo, description: str | None = None
+    ) -> None:
         try:
             type_info = encode_enriched_type_info(analyzed_type)
         except ValueError as e:
@@ -369,6 +371,8 @@ def add_field(name: str, analyzed_type: AnalyzedTypeInfo) -> None:
             )
             raise
         type_info["name"] = name
+        if description is not None:
+            type_info["description"] = description
         fields.append(type_info)
 
     def add_fields_from_struct(struct_type: type) -> None:
@@ -384,7 +388,9 @@ def add_fields_from_struct(struct_type: type) -> None:
                 for name, field_info in struct_type.model_fields.items():  # type: ignore[attr-defined]
                     # Get the annotation from the field info
                     field_type = field_info.annotation
-                    add_field(name, analyze_type_info(field_type))
+                    # Extract description from Pydantic field info
+                    description = getattr(field_info, "description", None)
+                    add_field(name, analyze_type_info(field_type), description)
             else:
                 raise ValueError(f"Invalid Pydantic model: {struct_type}")
         else:
@@ -667,6 +673,7 @@ def encode(self) -> dict[str, Any]:
 class FieldSchema:
     name: str
     value_type: EnrichedValueType
+    description: str | None = None
 
     def __str__(self) -> str:
         return f"{self.name}: {self.value_type}"
@@ -676,11 +683,17 @@ def __repr__(self) -> str:
 
     @staticmethod
     def decode(obj: dict[str, Any]) -> "FieldSchema":
-        return FieldSchema(name=obj["name"], value_type=EnrichedValueType.decode(obj))
+        return FieldSchema(
+            name=obj["name"],
+            value_type=EnrichedValueType.decode(obj),
+            description=obj.get("description"),
+        )
 
     def encode(self) -> dict[str, Any]:
         result = self.value_type.encode()
         result["name"] = self.name
+        if self.description is not None:
+            result["description"] = self.description
         return result
 
 

src/base/json_schema.rs

@@ -45,10 +45,31 @@ impl JsonSchemaBuilder {
         if self.options.extract_descriptions {
             let mut fields: Vec<_> = field_path.iter().map(|f| f.as_str()).collect();
             fields.reverse();
-            self.extra_instructions_per_field
-                .insert(fields.join("."), description.to_string());
+            let field_path_str = fields.join(".");
+
+            // Check if we already have a description for this field
+            if let Some(existing_description) =
+                self.extra_instructions_per_field.get(&field_path_str)
+            {
+                // Concatenate descriptions with newline separator
+                let combined_description =
+                    format!("{}\n{}", existing_description, description.to_string());
+                self.extra_instructions_per_field
+                    .insert(field_path_str, combined_description);
+            } else {
+                self.extra_instructions_per_field
+                    .insert(field_path_str, description.to_string());
+            }
         } else {
-            schema.metadata.get_or_insert_default().description = Some(description.to_string());
+            let metadata = schema.metadata.get_or_insert_default();
+            if let Some(existing_description) = &metadata.description {
+                // Concatenate descriptions with newline separator
+                let combined_description =
+                    format!("{}\n{}", existing_description, description.to_string());
+                metadata.description = Some(combined_description);
+            } else {
+                metadata.description = Some(description.to_string());
+            }
         }
     }
 
@@ -219,6 +240,10 @@ impl JsonSchemaBuilder {
                             *instance_type = SingleOrVec::Vec(types);
                         }
                     }
+                    // Set field description if available
+                    if let Some(description) = &f.description {
+                        self.set_description(&mut schema, description, field_path.prepend(&f.name));
+                    }
                     (f.name.to_string(), schema.into())
                 })
                 .collect(),
@@ -330,6 +355,7 @@ pub fn build_json_schema(
             fields: Arc::new(vec![schema::FieldSchema {
                 name: object_wrapper_field_name.clone(),
                 value_type: value_type.clone(),
+                description: None,
             }]),
             description: None,
         };
@@ -352,3 +378,85 @@ pub fn build_json_schema(
         },
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::base::schema::{
+        BasicValueType, EnrichedValueType, FieldSchema, StructSchema, ValueType,
+    };
+    use std::sync::Arc;
+
+    #[test]
+    fn test_description_concatenation() {
+        // Create a struct with a field that has both field-level and type-level descriptions
+        let struct_schema = StructSchema {
+            description: Some(Arc::from("Test struct description")),
+            fields: Arc::new(vec![FieldSchema {
+                name: "uuid_field".to_string(),
+                value_type: EnrichedValueType {
+                    typ: ValueType::Basic(BasicValueType::Uuid),
+                    nullable: false,
+                    attrs: Default::default(),
+                },
+                description: Some(Arc::from("This is a field-level description for UUID")),
+            }]),
+        };
+
+        let enriched_value_type = EnrichedValueType {
+            typ: ValueType::Struct(struct_schema),
+            nullable: false,
+            attrs: Default::default(),
+        };
+
+        let options = ToJsonSchemaOptions {
+            fields_always_required: false,
+            supports_format: true,
+            extract_descriptions: false, // We want to see the description in the schema
+            top_level_must_be_object: false,
+        };
+
+        let result = build_json_schema(enriched_value_type, options).unwrap();
+
+        // Check if the description contains both field and type descriptions
+        if let Some(properties) = &result.schema.object {
+            if let Some(uuid_field_schema) = properties.properties.get("uuid_field") {
+                if let Schema::Object(schema_object) = uuid_field_schema {
+                    if let Some(description) = &schema_object
+                        .metadata
+                        .as_ref()
+                        .and_then(|m| m.description.as_ref())
+                    {
+                        // Check if both descriptions are present
+                        assert!(
+                            description.contains("This is a field-level description for UUID"),
+                            "Field-level description not found in: {}",
+                            description
+                        );
+                        assert!(
+                            description
+                                .contains("A UUID, e.g. 123e4567-e89b-12d3-a456-426614174000"),
+                            "Type-level description not found in: {}",
+                            description
+                        );
+
+                        // Check that they are separated by a newline
+                        assert!(
+                            description.contains("\n"),
+                            "Descriptions should be separated by newline: {}",
+                            description
+                        );
+                    } else {
+                        panic!("No description found in the schema");
+                    }
+                } else {
+                    panic!("uuid_field schema is not a SchemaObject");
+                }
+            } else {
+                panic!("uuid_field not found in properties");
+            }
+        } else {
+            panic!("No object properties found in schema");
+        }
+    }
+}

src/base/schema.rs

@@ -325,20 +325,38 @@ pub struct FieldSchema<DataType = ValueType> {
 
     #[serde(flatten)]
     pub value_type: EnrichedValueType<DataType>,
+
+    /// Optional description for the field.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub description: Option<Arc<str>>,
 }
 
 impl FieldSchema {
     pub fn new(name: impl ToString, value_type: EnrichedValueType) -> Self {
         Self {
             name: name.to_string(),
             value_type,
+            description: None,
+        }
+    }
+
+    pub fn new_with_description(
+        name: impl ToString,
+        value_type: EnrichedValueType,
+        description: Option<impl ToString>,
+    ) -> Self {
+        Self {
+            name: name.to_string(),
+            value_type,
+            description: description.map(|d| d.to_string().into()),
         }
     }
 
     pub fn without_attrs(&self) -> Self {
         Self {
             name: self.name.clone(),
             value_type: self.value_type.without_attrs(),
+            description: None,
         }
     }
 }
@@ -351,6 +369,7 @@ impl<DataType> FieldSchema<DataType> {
         Ok(Self {
             name: field.name.clone(),
             value_type: EnrichedValueType::from_alternative(&field.value_type)?,
+            description: field.description.clone(),
         })
     }
 }

src/builder/analyzer.rs

@@ -229,6 +229,7 @@ fn try_merge_fields_schemas(
         result_fields.push(FieldSchema {
             name: field1.name.clone(),
             value_type: try_make_common_value_type(&field1.value_type, &field2.value_type)?,
+            description: None,
         });
     }
     Ok(result_fields)
@@ -316,6 +317,7 @@ impl DataScopeBuilder {
         let field_index = self.data.add_field(FieldSchema {
             name,
             value_type: EnrichedValueType::from_alternative(value_type)?,
+            description: None,
         })?;
         Ok(AnalyzedOpOutput {
             field_idx: field_index,
@@ -567,6 +569,7 @@ fn analyze_struct_mapping(
         field_schemas.push(FieldSchema {
             name: field.name.clone(),
             value_type,
+            description: None,
         });
     }
     Ok((

src/builder/flow_builder.rs

@@ -364,8 +364,11 @@ impl FlowBuilder {
                 .into_py_result()?;
         }
         let result = Self::last_field_to_data_slice(&self.root_op_scope).into_py_result()?;
-        self.direct_input_fields
-            .push(FieldSchema { name, value_type });
+        self.direct_input_fields.push(FieldSchema {
+            name,
+            value_type,
+            description: None,
+        });
         Ok(result)
     }
 
@@ -527,6 +530,7 @@ impl FlowBuilder {
                     Ok(FieldSchema {
                         name,
                         value_type: ds.value_type()?,
+                        description: None,
                     })
                 })
                 .collect::<Result<Vec<FieldSchema>>>()