fix: concatenate field and type descriptions instead of overwriting

- Modified set_description method to append descriptions with newline separator
- Fixed description overwrite issue where field-level descriptions would replace type-level descriptions
- Added description: None to all FieldSchema initializations to satisfy new field requirement
- Added test to verify description concatenation works correctly
- All existing tests pass (129 passed, 0 failed)

Resolves reviewer feedback about description overwrite in PR cocoindex-io#1087

Author: belloibrahv

src/base/json_schema.rs

@@ -45,10 +45,27 @@ impl JsonSchemaBuilder {
         if self.options.extract_descriptions {
             let mut fields: Vec<_> = field_path.iter().map(|f| f.as_str()).collect();
             fields.reverse();
-            self.extra_instructions_per_field
-                .insert(fields.join("."), description.to_string());
+            let field_path_str = fields.join(".");
+            
+            // Check if we already have a description for this field
+            if let Some(existing_description) = self.extra_instructions_per_field.get(&field_path_str) {
+                // Concatenate descriptions with newline separator
+                let combined_description = format!("{}\n{}", existing_description, description.to_string());
+                self.extra_instructions_per_field
+                    .insert(field_path_str, combined_description);
+            } else {
+                self.extra_instructions_per_field
+                    .insert(field_path_str, description.to_string());
+            }
         } else {
-            schema.metadata.get_or_insert_default().description = Some(description.to_string());
+            let metadata = schema.metadata.get_or_insert_default();
+            if let Some(existing_description) = &metadata.description {
+                // Concatenate descriptions with newline separator
+                let combined_description = format!("{}\n{}", existing_description, description.to_string());
+                metadata.description = Some(combined_description);
+            } else {
+                metadata.description = Some(description.to_string());
+            }
         }
     }
 
@@ -334,6 +351,7 @@ pub fn build_json_schema(
             fields: Arc::new(vec![schema::FieldSchema {
                 name: object_wrapper_field_name.clone(),
                 value_type: value_type.clone(),
+                description: None,
             }]),
             description: None,
         };
@@ -356,3 +374,71 @@ pub fn build_json_schema(
         },
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::base::schema::{BasicValueType, EnrichedValueType, FieldSchema, StructSchema, ValueType};
+    use std::sync::Arc;
+
+    #[test]
+    fn test_description_concatenation() {
+        // Create a struct with a field that has both field-level and type-level descriptions
+        let struct_schema = StructSchema {
+            description: Some(Arc::from("Test struct description")),
+            fields: Arc::new(vec![
+                FieldSchema {
+                    name: "uuid_field".to_string(),
+                    value_type: EnrichedValueType {
+                        typ: ValueType::Basic(BasicValueType::Uuid),
+                        nullable: false,
+                        attrs: Default::default(),
+                    },
+                    description: Some(Arc::from("This is a field-level description for UUID")),
+                },
+            ]),
+        };
+
+        let enriched_value_type = EnrichedValueType {
+            typ: ValueType::Struct(struct_schema),
+            nullable: false,
+            attrs: Default::default(),
+        };
+
+        let options = ToJsonSchemaOptions {
+            fields_always_required: false,
+            supports_format: true,
+            extract_descriptions: false, // We want to see the description in the schema
+            top_level_must_be_object: false,
+        };
+
+        let result = build_json_schema(enriched_value_type, options).unwrap();
+        
+        // Check if the description contains both field and type descriptions
+        if let Some(properties) = &result.schema.object {
+            if let Some(uuid_field_schema) = properties.properties.get("uuid_field") {
+                if let Schema::Object(schema_object) = uuid_field_schema {
+                    if let Some(description) = &schema_object.metadata.as_ref().and_then(|m| m.description.as_ref()) {
+                        // Check if both descriptions are present
+                        assert!(description.contains("This is a field-level description for UUID"), 
+                            "Field-level description not found in: {}", description);
+                        assert!(description.contains("A UUID, e.g. 123e4567-e89b-12d3-a456-426614174000"), 
+                            "Type-level description not found in: {}", description);
+                        
+                        // Check that they are separated by a newline
+                        assert!(description.contains("\n"), 
+                            "Descriptions should be separated by newline: {}", description);
+                    } else {
+                        panic!("No description found in the schema");
+                    }
+                } else {
+                    panic!("uuid_field schema is not a SchemaObject");
+                }
+            } else {
+                panic!("uuid_field not found in properties");
+            }
+        } else {
+            panic!("No object properties found in schema");
+        }
+    }
+}

src/builder/analyzer.rs

@@ -229,6 +229,7 @@ fn try_merge_fields_schemas(
         result_fields.push(FieldSchema {
             name: field1.name.clone(),
             value_type: try_make_common_value_type(&field1.value_type, &field2.value_type)?,
+            description: None,
         });
     }
     Ok(result_fields)
@@ -316,6 +317,7 @@ impl DataScopeBuilder {
         let field_index = self.data.add_field(FieldSchema {
             name,
             value_type: EnrichedValueType::from_alternative(value_type)?,
+            description: None,
         })?;
         Ok(AnalyzedOpOutput {
             field_idx: field_index,
@@ -567,6 +569,7 @@ fn analyze_struct_mapping(
         field_schemas.push(FieldSchema {
             name: field.name.clone(),
             value_type,
+            description: None,
         });
     }
     Ok((

src/builder/flow_builder.rs

@@ -365,7 +365,7 @@ impl FlowBuilder {
         }
         let result = Self::last_field_to_data_slice(&self.root_op_scope).into_py_result()?;
         self.direct_input_fields
-            .push(FieldSchema { name, value_type });
+            .push(FieldSchema { name, value_type, description: None });
         Ok(result)
     }
 
@@ -527,6 +527,7 @@ impl FlowBuilder {
                     Ok(FieldSchema {
                         name,
                         value_type: ds.value_type()?,
+                        description: None,
                     })
                 })
                 .collect::<Result<Vec<FieldSchema>>>()

src/py/convert.rs

@@ -459,6 +459,7 @@ mod tests {
                         nullable: false,
                         attrs: Default::default(),
                     },
+                    description: None,
                 },
                 schema::FieldSchema {
                     name: "b".to_string(),
@@ -467,6 +468,7 @@ mod tests {
                         nullable: false,
                         attrs: Default::default(),
                     },
+                    description: None,
                 },
             ]),
         };
@@ -496,6 +498,7 @@ mod tests {
                         nullable: false,
                         attrs: Default::default(),
                     },
+                    description: None,
                 },
                 schema::FieldSchema {
                     name: "data_col_1".to_string(),
@@ -504,6 +507,7 @@ mod tests {
                         nullable: false,
                         attrs: Default::default(),
                     },
+                    description: None,
                 },
                 schema::FieldSchema {
                     name: "data_col_2".to_string(),
@@ -512,6 +516,7 @@ mod tests {
                         nullable: false,
                         attrs: Default::default(),
                     },
+                    description: None,
                 },
             ]),
         });