Implement nested value retrieval and metadata matching in StepBasedSearchStrategy

This commit adds two new methods to the StepBasedSearchStrategy class: _get_nested_value for extracting values from nested dictionaries using a dot-separated path, and _metadata_matches for checking if a memory object matches a given metadata filter. Additionally, the filtering logic in _filter_memories is updated to utilize the new _metadata_matches method. The scoring calculations in the memory processing logic are also enhanced to avoid division by zero and improve step weight application. Test suite updates include adjustments to metadata filters for improved validation coverage.

Author: csmangum

memory/search/strategies/step.py

@@ -314,6 +314,67 @@ def _get_memory_step(self, memory: Dict[str, Any]) -> Optional[int]:
 
         return None
 
+    def _get_nested_value(self, obj: Dict[str, Any], path: str) -> Any:
+        """Get value from a nested dictionary using a dot-separated path.
+        
+        Args:
+            obj: Dictionary to extract value from
+            path: Dot-separated path to the value (e.g., "content.metadata.importance")
+            
+        Returns:
+            Value found at the path, or None if not found
+        """
+        if not path or not isinstance(obj, dict):
+            return None
+            
+        parts = path.split('.')
+        current = obj
+        
+        for part in parts:
+            if not isinstance(current, dict) or part not in current:
+                return None
+            current = current[part]
+            
+        return current
+
+    def _metadata_matches(self, memory: Dict[str, Any], metadata_filter: Dict[str, Any]) -> bool:
+        """Check if a memory matches the metadata filter.
+        
+        Args:
+            memory: Memory object to check
+            metadata_filter: Metadata filter to apply
+            
+        Returns:
+            True if memory matches filter, False otherwise
+        """
+        for key, filter_value in metadata_filter.items():
+            memory_value = self._get_nested_value(memory, key)
+            if memory_value is None:
+                # Try to get from non-nested metadata
+                memory_value = memory.get("metadata", {}).get(key)
+                
+            # No matching value found
+            if memory_value is None:
+                logger.debug(f"No value found for key {key} in memory")
+                return False
+                
+            # Handle list/array values - check if filter_value is a subset of memory_value
+            if isinstance(filter_value, list):
+                if not isinstance(memory_value, list):
+                    # Convert to list if memory_value is a single value
+                    memory_value = [memory_value]
+                
+                # Check if all items in filter_value are in memory_value
+                if not all(item in memory_value for item in filter_value):
+                    logger.debug(f"List match failed for {key}: filter={filter_value}, memory={memory_value}")
+                    return False
+            # For other types, do a direct comparison
+            elif memory_value != filter_value:
+                logger.debug(f"Value mismatch for {key}: filter={filter_value}, memory={memory_value}")
+                return False
+                
+        return True
+
     def _filter_memories(
         self,
         memories: List[Dict[str, Any]],
@@ -364,11 +425,8 @@ def _filter_memories(
                 continue
 
             # Apply metadata filter
-            if metadata_filter:
-                memory_metadata = memory.get("metadata", {})
-                if not all(
-                    memory_metadata.get(k) == v for k, v in metadata_filter.items()
-                ):
+            if metadata_filter and len(metadata_filter) > 0:
+                if not self._metadata_matches(memory, metadata_filter):
                     continue
 
             filtered.append(memory)
@@ -412,13 +470,24 @@ def _score_memories(
                 # Normalize step distance (closer = higher score)
                 # Adjust max_distance based on your simulation scale
                 max_distance = step_params.get("step_range", 100) * 2
-                normalized_distance = min(step_distance / max_distance, 1.0)
+                # Avoid division by zero
+                if max_distance > 0:
+                    normalized_distance = min(step_distance / max_distance, 1.0)
+                else:
+                    normalized_distance = 1.0 if step_distance > 0 else 0.0
 
-                # Higher score for closer steps
+                # Higher score for closer steps (1.0 for exact match, decreasing as distance increases)
                 step_score = 1.0 - normalized_distance
 
-                # Apply step weight
-                step_score = step_score * step_weight
+                # Apply step weight (higher weight emphasizes proximity more)
+                step_score = pow(step_score, 1.0 / max(step_weight, 0.001))
+
+                # Log the scoring calculations for debugging
+                logger.debug(
+                    f"Memory {memory.get('id', 'unknown')}: step={memory_step}, "
+                    f"ref={reference_step}, distance={step_distance}, "
+                    f"normalized_distance={normalized_distance}, score={step_score}"
+                )
 
             # Create a copy of the memory to avoid modifying the original
             memory_copy = memory.copy()

validation/search/step/step_test_suite.py

@@ -62,6 +62,7 @@ def run_basic_tests(self) -> None:
                 "test-agent-step-search-stm-1",
                 "test-agent-step-search-stm-2"
             ],
+            tier="stm",
             memory_checksum_map=self.memory_checksum_map,
         )
 
@@ -153,7 +154,10 @@ def run_advanced_tests(self) -> None:
                 "test-agent-step-search-ltm-1",
                 "test-agent-step-search-ltm-2"
             ],
-            metadata_filter={"content.metadata.type": "system", "content.metadata.importance": "high"},
+            metadata_filter={
+                "content.metadata.type": "state",
+                "content.metadata.importance": "high"
+            },
             memory_checksum_map=self.memory_checksum_map,
         )
 
@@ -264,7 +268,8 @@ def run_edge_case_tests(self) -> None:
             {"start_step": 100, "end_step": 200},
             expected_memory_ids=[
                 "test-agent-step-search-stm-1",
-                "test-agent-step-search-stm-2"
+                "test-agent-step-search-stm-2",
+                "test-agent-step-search-im-1"
             ],
             metadata_filter={},
             memory_checksum_map=self.memory_checksum_map,