Release v3.10.15

Author: MervinPraison

docker/Dockerfile.chat

@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=3.10.14" \
+    "praisonai>=3.10.15" \
     "praisonai[chat]" \
     "embedchain[github,youtube]"
 

docker/Dockerfile.dev

@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=3.10.14" \
+    "praisonai>=3.10.15" \
     "praisonai[ui]" \
     "praisonai[chat]" \
     "praisonai[realtime]" \

docker/Dockerfile.ui

@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
 # Install Python packages (using latest versions)
 RUN pip install --no-cache-dir \
     praisonai_tools \
-    "praisonai>=3.10.14" \
+    "praisonai>=3.10.15" \
     "praisonai[ui]" \
     "praisonai[crewai]"
 

src/praisonai-agents/praisonaiagents/config/__init__.py

@@ -29,6 +29,7 @@
     "MemoryConfig",
     "KnowledgeConfig",
     "PlanningConfig",
+    "MultiAgentPlanningConfig",
     "ReflectionConfig",
     "GuardrailConfig",
     "WebConfig",
@@ -91,6 +92,7 @@
     "MemoryConfig": "feature_configs",
     "KnowledgeConfig": "feature_configs",
     "PlanningConfig": "feature_configs",
+    "MultiAgentPlanningConfig": "feature_configs",
     "ReflectionConfig": "feature_configs",
     "GuardrailConfig": "feature_configs",
     "WebConfig": "feature_configs",

src/praisonai-agents/praisonaiagents/context/instrumentation.py

@@ -46,7 +46,11 @@ class ContextMetrics:
     # Token tracking
     tokens_processed: int = 0
     tokens_saved: int = 0
+    tokens_saved_by_summarization: int = 0  # Tokens saved by LLM summarization
+    tokens_saved_by_truncation: int = 0  # Tokens saved by truncation
     compactions_triggered: int = 0
+    tool_outputs_summarized: int = 0  # Count of tool outputs summarized
+    tool_outputs_truncated: int = 0  # Count of tool outputs truncated
 
     # Cache stats
     cache_hits: int = 0
@@ -76,8 +80,14 @@ def to_dict(self) -> Dict[str, Any]:
             "tokens": {
                 "processed": self.tokens_processed,
                 "saved": self.tokens_saved,
+                "saved_by_summarization": self.tokens_saved_by_summarization,
+                "saved_by_truncation": self.tokens_saved_by_truncation,
                 "compactions": self.compactions_triggered,
             },
+            "tool_outputs": {
+                "summarized": self.tool_outputs_summarized,
+                "truncated": self.tool_outputs_truncated,
+            },
             "cache": {
                 "hits": self.cache_hits,
                 "misses": self.cache_misses,

src/praisonai-agents/praisonaiagents/context/manager.py

@@ -92,6 +92,67 @@ def clear(self) -> None:
             self._stats = {"duplicates_prevented": 0, "tokens_saved": 0}
 
 
+def deduplicate_topics(topics: list, key: str = "title", similarity_threshold: float = 0.8) -> list:
+    """
+    Programmatic deduplication of topics/items before agent processing.
+    
+    This helps prevent duplicate content from being passed to downstream agents,
+    reducing token waste and improving quality.
+    
+    Args:
+        topics: List of topic dicts or strings
+        key: Key to use for comparison if topics are dicts (default: "title")
+        similarity_threshold: Similarity threshold for fuzzy matching (0.0-1.0)
+        
+    Returns:
+        Deduplicated list of topics
+    """
+    if not topics:
+        return topics
+    
+    seen_hashes = set()
+    seen_normalized = set()
+    unique_topics = []
+    
+    for topic in topics:
+        # Get the content to compare
+        if isinstance(topic, dict):
+            content = str(topic.get(key, topic.get("content", str(topic))))
+        else:
+            content = str(topic)
+        
+        # Normalize for comparison
+        normalized = content.lower().strip()
+        # Remove common words for better matching
+        normalized = " ".join(w for w in normalized.split() if len(w) > 3)
+        
+        # Check exact hash match
+        content_hash = hashlib.md5(normalized.encode()).hexdigest()
+        if content_hash in seen_hashes:
+            continue
+        
+        # Check fuzzy match using simple word overlap
+        is_duplicate = False
+        for seen in seen_normalized:
+            # Calculate Jaccard similarity
+            words1 = set(normalized.split())
+            words2 = set(seen.split())
+            if words1 and words2:
+                intersection = len(words1 & words2)
+                union = len(words1 | words2)
+                similarity = intersection / union if union > 0 else 0
+                if similarity >= similarity_threshold:
+                    is_duplicate = True
+                    break
+        
+        if not is_duplicate:
+            seen_hashes.add(content_hash)
+            seen_normalized.add(normalized)
+            unique_topics.append(topic)
+    
+    return unique_topics
+
+
 class EstimationMode(str, Enum):
     """Token estimation modes."""
     HEURISTIC = "heuristic"
@@ -256,6 +317,10 @@ class ManagerConfig:
     # LLM-powered summarization
     llm_summarize: bool = False  # Enable LLM-powered summarization
 
+    # Smart tool output summarization
+    smart_tool_summarize: bool = True  # Summarize large tool outputs using LLM before truncating
+    tool_summarize_limits: Dict[str, int] = field(default_factory=dict)  # Per-tool min_chars_to_summarize
+    
     # Estimation
     estimation_mode: EstimationMode = EstimationMode.HEURISTIC
     log_estimation_mismatch: bool = False
@@ -306,6 +371,8 @@ def to_dict(self) -> Dict[str, Any]:
             "prune_after_tokens": self.prune_after_tokens,
             "keep_recent_turns": self.keep_recent_turns,
             "llm_summarize": self.llm_summarize,
+            "smart_tool_summarize": self.smart_tool_summarize,
+            "tool_summarize_limits": self.tool_summarize_limits,
             "source": self.source,
         }
         return result
@@ -617,6 +684,8 @@ def _optimize_with_benefit_check(
             preserve_recent=self.config.keep_recent_turns,
             protected_tools=self.config.protected_tools,
             llm_summarize_fn=self._llm_summarize_fn if self.config.llm_summarize else None,
+            smart_tool_summarize=self.config.smart_tool_summarize,
+            tool_summarize_limits=self.config.tool_summarize_limits,
         )
 
         # Try optimization

src/praisonai-agents/praisonaiagents/context/models.py

@@ -214,6 +214,9 @@ class OptimizationResult:
     messages_removed: int = 0
     messages_tagged: int = 0
     tool_outputs_pruned: int = 0
+    tool_outputs_summarized: int = 0  # Count of tool outputs summarized via LLM
+    tokens_saved_by_summarization: int = 0  # Tokens saved specifically by LLM summarization
+    tokens_saved_by_truncation: int = 0  # Tokens saved specifically by truncation
     summary_added: bool = False
 
     @property
@@ -286,6 +289,9 @@ class ContextConfig:
     # LLM-powered summarization
     llm_summarize: bool = False  # Enable LLM-powered summarization (uses agent's LLM)
 
+    # Smart tool output summarization (summarize before truncating)
+    smart_tool_summarize: bool = True  # Summarize large tool outputs using LLM before truncating
+    
     # Session tracking (Agno pattern)
     session_tracking: bool = False     # Enable goal/plan/progress tracking
     track_summary: bool = True         # Auto-extract conversation summary

src/praisonai-agents/praisonaiagents/context/optimizer.py

@@ -395,6 +395,109 @@ def _create_summary(self, messages: List[Dict[str, Any]]) -> str:
         return ""
 
 
+class SummarizeToolOutputsOptimizer(BaseOptimizer):
+    """
+    Summarize large tool outputs using LLM before truncation.
+    
+    This optimizer specifically targets tool role messages with large content,
+    using an LLM to create intelligent summaries that preserve key information.
+    Falls back to keeping original content if LLM is unavailable or fails.
+    """
+    
+    def __init__(
+        self,
+        llm_summarize_fn: Optional[callable] = None,
+        max_output_tokens: int = 1000,
+        min_chars_to_summarize: int = 2000,
+        preserve_recent: int = 2,
+        tool_summarize_limits: Optional[Dict[str, int]] = None,
+    ):
+        """
+        Initialize tool output summarizer.
+        
+        Args:
+            llm_summarize_fn: Function(content, max_tokens) -> summary string
+            max_output_tokens: Target token count for summarized output
+            min_chars_to_summarize: Default minimum chars before summarization triggers
+            preserve_recent: Number of recent tool outputs to preserve intact
+            tool_summarize_limits: Per-tool min_chars_to_summarize limits {tool_name: min_chars}
+        """
+        self.llm_summarize_fn = llm_summarize_fn
+        self.max_output_tokens = max_output_tokens
+        self.min_chars_to_summarize = min_chars_to_summarize
+        self.preserve_recent = preserve_recent
+        self.tool_summarize_limits = tool_summarize_limits or {}
+    
+    def optimize(
+        self,
+        messages: List[Dict[str, Any]],
+        target_tokens: int,
+        ledger: Optional[ContextLedger] = None,
+    ) -> tuple:
+        original_tokens = estimate_messages_tokens(messages)
+        
+        # If already under budget or no LLM function, return as-is
+        if original_tokens <= target_tokens or not self.llm_summarize_fn:
+            return messages, OptimizationResult(
+                original_tokens=original_tokens,
+                optimized_tokens=original_tokens,
+                tokens_saved=0,
+                strategy_used=OptimizerStrategy.SMART,
+            )
+        
+        result = []
+        summarized_count = 0
+        
+        # Find tool messages and their indices
+        tool_indices = [i for i, m in enumerate(messages) if m.get("role") == "tool"]
+        
+        # Preserve recent tool outputs (only if there are more than preserve_recent tools)
+        if tool_indices and self.preserve_recent > 0 and len(tool_indices) > self.preserve_recent:
+            recent_tool_indices = set(tool_indices[-self.preserve_recent:])
+        else:
+            recent_tool_indices = set()  # Summarize all if few tools or preserve_recent=0
+        
+        for i, msg in enumerate(messages):
+            role = msg.get("role", "")
+            content = msg.get("content", "")
+            
+            # Only process tool messages with large content
+            if role == "tool" and i not in recent_tool_indices:
+                # Get per-tool limit or use default
+                tool_name = msg.get("name", "")
+                min_chars = self.tool_summarize_limits.get(tool_name, self.min_chars_to_summarize)
+                if isinstance(content, str) and len(content) >= min_chars:
+                    # Try to summarize
+                    try:
+                        summary = self.llm_summarize_fn(content, self.max_output_tokens)
+                        if summary and len(summary) < len(content):
+                            summarized_msg = msg.copy()
+                            summarized_msg["content"] = summary
+                            summarized_msg["_summarized"] = True
+                            summarized_msg["_original_length"] = len(content)
+                            result.append(summarized_msg)
+                            summarized_count += 1
+                            continue
+                    except Exception:
+                        # Fallback to original on error
+                        pass
+            
+            result.append(msg)
+        
+        optimized_tokens = estimate_messages_tokens(result)
+        
+        tokens_saved = original_tokens - optimized_tokens
+        
+        return result, OptimizationResult(
+            original_tokens=original_tokens,
+            optimized_tokens=optimized_tokens,
+            tokens_saved=tokens_saved,
+            strategy_used=OptimizerStrategy.SMART,
+            tool_outputs_summarized=summarized_count,
+            tokens_saved_by_summarization=tokens_saved,  # All savings from summarization
+        )
+
+
 class LLMSummarizeOptimizer(SummarizeOptimizer):
     """
     LLM-powered summarization optimizer.
@@ -475,9 +578,10 @@ class SmartOptimizer(BaseOptimizer):
     Smart optimization combining multiple strategies.
     
     Applies strategies in order:
-    1. Prune tool outputs
-    2. Sliding window
-    3. Summarize if still over
+    1. Summarize tool outputs (if LLM available and smart_tool_summarize=True)
+    2. Prune tool outputs (fallback truncation)
+    3. Sliding window
+    4. Summarize conversation if still over
     """
 
     def __init__(
@@ -486,11 +590,21 @@ def __init__(
         protected_tools: Optional[List[str]] = None,
         tool_limits: Optional[Dict[str, int]] = None,
         llm_summarize_fn: Optional[callable] = None,
+        smart_tool_summarize: bool = True,
+        tool_summarize_limits: Optional[Dict[str, int]] = None,
     ):
         self.preserve_recent = preserve_recent
         self.protected_tools = protected_tools or []
         self.tool_limits = tool_limits or {}
+        self.smart_tool_summarize = smart_tool_summarize
+        self.tool_summarize_limits = tool_summarize_limits or {}
 
+        # Tool output summarization (LLM-powered, applied first when available)
+        self._summarize_tools = SummarizeToolOutputsOptimizer(
+            llm_summarize_fn=llm_summarize_fn if smart_tool_summarize else None,
+            preserve_recent=preserve_recent,
+            tool_summarize_limits=tool_summarize_limits,
+        )
         self._prune = PruneToolsOptimizer(
             preserve_recent=preserve_recent,
             protected_tools=protected_tools,
@@ -518,19 +632,43 @@ def optimize(
                 strategy_used=OptimizerStrategy.SMART,
             )
 
-        # Step 1: Prune tool outputs
-        result, prune_result = self._prune.optimize(messages, target_tokens, ledger)
+        # Step 1: Summarize tool outputs (LLM-powered, if available)
+        tool_summarized_count = 0
+        tokens_saved_by_summarization = 0
+        if self._summarize_tools.llm_summarize_fn:
+            result, tool_summary_result = self._summarize_tools.optimize(messages, target_tokens, ledger)
+            tool_summarized_count = tool_summary_result.tool_outputs_summarized
+            tokens_saved_by_summarization = tool_summary_result.tokens_saved_by_summarization
+            
+            if estimate_messages_tokens(result) <= target_tokens:
+                return result, OptimizationResult(
+                    original_tokens=original_tokens,
+                    optimized_tokens=tool_summary_result.optimized_tokens,
+                    tokens_saved=original_tokens - tool_summary_result.optimized_tokens,
+                    strategy_used=OptimizerStrategy.SMART,
+                    tool_outputs_summarized=tool_summarized_count,
+                    tokens_saved_by_summarization=tokens_saved_by_summarization,
+                )
+        else:
+            result = messages
+        
+        # Step 2: Prune tool outputs (fallback truncation)
+        result, prune_result = self._prune.optimize(result, target_tokens, ledger)
+        tokens_saved_by_truncation = prune_result.tokens_saved
 
         if estimate_messages_tokens(result) <= target_tokens:
             return result, OptimizationResult(
                 original_tokens=original_tokens,
                 optimized_tokens=prune_result.optimized_tokens,
                 tokens_saved=original_tokens - prune_result.optimized_tokens,
                 strategy_used=OptimizerStrategy.SMART,
+                tool_outputs_summarized=tool_summarized_count,
                 tool_outputs_pruned=prune_result.tool_outputs_pruned,
+                tokens_saved_by_summarization=tokens_saved_by_summarization,
+                tokens_saved_by_truncation=tokens_saved_by_truncation,
             )
 
-        # Step 2: Sliding window
+        # Step 3: Sliding window
         result, window_result = self._window.optimize(result, target_tokens, ledger)
 
         if estimate_messages_tokens(result) <= target_tokens:
@@ -543,7 +681,7 @@ def optimize(
                 messages_removed=window_result.messages_removed,
             )
 
-        # Step 3: Summarize
+        # Step 4: Summarize conversation
         result, summary_result = self._summarize.optimize(result, target_tokens, ledger)
 
         optimized_tokens = estimate_messages_tokens(result)
@@ -553,7 +691,10 @@ def optimize(
             optimized_tokens=optimized_tokens,
             tokens_saved=original_tokens - optimized_tokens,
             strategy_used=OptimizerStrategy.SMART,
+            tool_outputs_summarized=tool_summarized_count,
             tool_outputs_pruned=prune_result.tool_outputs_pruned,
+            tokens_saved_by_summarization=tokens_saved_by_summarization,
+            tokens_saved_by_truncation=tokens_saved_by_truncation,
             messages_removed=window_result.messages_removed + summary_result.messages_removed,
             summary_added=summary_result.summary_added,
         )