[ROB-1734] Robusta runbook support + better runbook usage

holmes/config.py

@@ -223,10 +223,9 @@ def __get_cluster_name() -> Optional[str]:
 
         return None
 
-    @staticmethod
-    def get_runbook_catalog() -> Optional[RunbookCatalog]:
+    def get_runbook_catalog(self) -> Optional[RunbookCatalog]:
         # TODO(mainred): besides the built-in runbooks, we need to allow the user to bring their own runbooks
-        runbook_catalog = load_runbook_catalog()
+        runbook_catalog = load_runbook_catalog(dal=self.dal)
         return runbook_catalog
 
     def create_console_tool_executor(

holmes/core/conversations.py

@@ -1,7 +1,6 @@
 from typing import Dict, List, Optional
 
 import sentry_sdk
-
 from holmes.config import Config
 from holmes.core.models import (
     ToolCallConversationResult,
@@ -10,9 +9,10 @@
 )
 from holmes.plugins.prompts import load_and_render_prompt
 from holmes.core.tool_calling_llm import ToolCallingLLM
+from holmes.plugins.runbooks import RunbookCatalog
 from holmes.utils.global_instructions import (
     Instructions,
-    add_global_instructions_to_user_prompt,
+    add_runbooks_to_user_prompt,
 )
 
 DEFAULT_TOOL_SIZE = 10000
@@ -64,6 +64,7 @@ def build_issue_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """
     This function generates a list of messages for issue conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -120,8 +121,10 @@ def build_issue_chat_messages(
     tools_for_investigation = issue_chat_request.investigation_result.tools
 
     if not conversation_history or len(conversation_history) == 0:
-        user_prompt = add_global_instructions_to_user_prompt(
-            user_prompt, global_instructions
+        user_prompt = add_runbooks_to_user_prompt(
+            user_prompt=user_prompt,
+            runbook_catalog=runbooks,
+            global_instructions=global_instructions,
         )
 
         number_of_tools_for_investigation = len(tools_for_investigation)  # type: ignore
@@ -134,6 +137,7 @@ def build_issue_chat_messages(
                     "issue": issue_chat_request.issue_type,
                     "toolsets": ai.tool_executor.toolsets,
                     "cluster_name": config.cluster_name,
+                    "runbooks_enabled": True if runbooks else False,
                 },
             )
             messages = [
@@ -154,6 +158,7 @@ def build_issue_chat_messages(
             "issue": issue_chat_request.issue_type,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_without_tools = load_and_render_prompt(
             template_path, template_context_without_tools
@@ -187,6 +192,7 @@ def build_issue_chat_messages(
             "issue": issue_chat_request.issue_type,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_with_truncated_tools = load_and_render_prompt(
             template_path, truncated_template_context
@@ -202,8 +208,10 @@ def build_issue_chat_messages(
             },
         ]
 
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
     )
 
     conversation_history.append(
@@ -228,6 +236,7 @@ def build_issue_chat_messages(
         "issue": issue_chat_request.issue_type,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_without_tools = load_and_render_prompt(
         template_path, template_context_without_tools
@@ -251,6 +260,7 @@ def build_issue_chat_messages(
         "issue": issue_chat_request.issue_type,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_with_truncated_tools = load_and_render_prompt(
         template_path, template_context
@@ -267,6 +277,7 @@ def add_or_update_system_prompt(
     ai: ToolCallingLLM,
     config: Config,
     additional_system_prompt: Optional[str] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """Either add the system prompt or replace an existing system prompt.
     As a 'defensive' measure, this code will only replace an existing system prompt if it is the
@@ -278,10 +289,10 @@ def add_or_update_system_prompt(
     context = {
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
 
     system_prompt = load_and_render_prompt(template_path, context)
-
     if additional_system_prompt:
         system_prompt = system_prompt + "\n" + additional_system_prompt
 
@@ -311,6 +322,7 @@ def build_chat_messages(
     config: Config,
     global_instructions: Optional[Instructions] = None,
     additional_system_prompt: Optional[str] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ) -> List[dict]:
     """
     This function generates a list of messages for general chat conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -370,9 +382,15 @@ def build_chat_messages(
         ai=ai,
         config=config,
         additional_system_prompt=additional_system_prompt,
+        runbooks=runbooks,
+    )
+
+    ask = add_runbooks_to_user_prompt(
+        user_prompt=ask,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
     )
 
-    ask = add_global_instructions_to_user_prompt(ask, global_instructions)
     conversation_history.append(  # type: ignore
         {
             "role": "user",
@@ -404,6 +422,7 @@ def build_workload_health_chat_messages(
     ai: ToolCallingLLM,
     config: Config,
     global_instructions: Optional[Instructions] = None,
+    runbooks: Optional[RunbookCatalog] = None,
 ):
     """
     This function generates a list of messages for workload health conversation and ensures that the message sequence adheres to the model's context window limitations
@@ -462,8 +481,10 @@ def build_workload_health_chat_messages(
     resource = workload_health_chat_request.resource
 
     if not conversation_history or len(conversation_history) == 0:
-        user_prompt = add_global_instructions_to_user_prompt(
-            user_prompt, global_instructions
+        user_prompt = add_runbooks_to_user_prompt(
+            user_prompt=user_prompt,
+            runbook_catalog=runbooks,
+            global_instructions=global_instructions,
         )
 
         number_of_tools_for_workload = len(tools_for_workload)  # type: ignore
@@ -476,6 +497,7 @@ def build_workload_health_chat_messages(
                     "resource": resource,
                     "toolsets": ai.tool_executor.toolsets,
                     "cluster_name": config.cluster_name,
+                    "runbooks_enabled": True if runbooks else False,
                 },
             )
             messages = [
@@ -496,6 +518,7 @@ def build_workload_health_chat_messages(
             "resource": resource,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_without_tools = load_and_render_prompt(
             template_path, template_context_without_tools
@@ -529,6 +552,7 @@ def build_workload_health_chat_messages(
             "resource": resource,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbooks else False,
         }
         system_prompt_with_truncated_tools = load_and_render_prompt(
             template_path, truncated_template_context
@@ -544,8 +568,10 @@ def build_workload_health_chat_messages(
             },
         ]
 
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbooks,
+        global_instructions=global_instructions,
     )
 
     conversation_history.append(
@@ -570,6 +596,7 @@ def build_workload_health_chat_messages(
         "resource": resource,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_without_tools = load_and_render_prompt(
         template_path, template_context_without_tools
@@ -593,6 +620,7 @@ def build_workload_health_chat_messages(
         "resource": resource,
         "toolsets": ai.tool_executor.toolsets,
         "cluster_name": config.cluster_name,
+        "runbooks_enabled": True if runbooks else False,
     }
     system_prompt_with_truncated_tools = load_and_render_prompt(
         template_path, template_context

holmes/core/investigation.py

@@ -1,14 +1,16 @@
 import logging
 from typing import Optional
 
+
 from holmes.common.env_vars import HOLMES_POST_PROCESSING_PROMPT
 from holmes.config import Config
 from holmes.core.investigation_structured_output import process_response_into_sections
 from holmes.core.issue import Issue
 from holmes.core.models import InvestigateRequest, InvestigationResult
 from holmes.core.supabase_dal import SupabaseDal
 from holmes.core.tracing import DummySpan, SpanType
-from holmes.utils.global_instructions import add_global_instructions_to_user_prompt
+from holmes.plugins.runbooks import RunbookCatalog
+from holmes.utils.global_instructions import add_runbooks_to_user_prompt
 
 from holmes.core.investigation_structured_output import (
     DEFAULT_SECTIONS,
@@ -25,6 +27,7 @@ def investigate_issues(
     config: Config,
     model: Optional[str] = None,
     trace_span=DummySpan(),
+    runbooks: Optional[RunbookCatalog] = None,
 ) -> InvestigationResult:
     context = dal.get_issue_data(investigate_request.context.get("robusta_issue_id"))
 
@@ -60,6 +63,7 @@ def investigate_issues(
         global_instructions=global_instructions,
         sections=investigate_request.sections,
         trace_span=trace_span,
+        runbooks=runbooks,
     )
 
     (text_response, sections) = process_response_into_sections(investigation.result)
@@ -95,18 +99,11 @@ def get_investigation_context(
         raw=raw_data,
     )
 
-    runbooks = ai.runbook_manager.get_instructions_for_issue(issue)
+    issue_instructions = ai.runbook_manager.get_instructions_for_issue(issue)
 
-    instructions = dal.get_resource_instructions(
+    resource_instructions = dal.get_resource_instructions(
         "alert", investigate_request.context.get("issue_type")
     )
-    if instructions is not None and instructions.instructions:
-        runbooks.extend(instructions.instructions)
-    if instructions is not None and len(instructions.documents) > 0:
-        docPrompts = []
-        for document in instructions.documents:
-            docPrompts.append(f"* fetch information from this URL: {document.url}\n")
-        runbooks.extend(docPrompts)
 
     # This section is about setting vars to request the LLM to return structured output.
     # It does not mean that Holmes will not return structured sections for investigation as it is
@@ -131,6 +128,7 @@ def get_investigation_context(
     else:
         logging.info("Structured output is disabled for this request")
 
+    runbook_catalog = config.get_runbook_catalog()
     system_prompt = load_and_render_prompt(
         investigate_request.prompt_template,
         {
@@ -139,21 +137,20 @@ def get_investigation_context(
             "structured_output": request_structured_output_from_llm,
             "toolsets": ai.tool_executor.toolsets,
             "cluster_name": config.cluster_name,
+            "runbooks_enabled": True if runbook_catalog else False,
         },
     )
-
     user_prompt = ""
-    if runbooks:
-        for runbook_str in runbooks:
-            user_prompt += f"* {runbook_str}\n"
-
-        user_prompt = f'My instructions to check \n"""{user_prompt}"""'
 
     global_instructions = dal.get_global_instructions_for_account()
-    user_prompt = add_global_instructions_to_user_prompt(
-        user_prompt, global_instructions
+    user_prompt = add_runbooks_to_user_prompt(
+        user_prompt=user_prompt,
+        runbook_catalog=runbook_catalog,
+        global_instructions=global_instructions,
+        issue_instructions=issue_instructions,
+        resource_instructions=resource_instructions,
     )
 
-    user_prompt = f"{user_prompt}\n This is context from the issue {issue.raw}"
+    user_prompt = f"{user_prompt}\n #This is context from the issue:\n{issue.raw}"
 
-    return ai, system_prompt, user_prompt, response_format, sections, runbooks
+    return ai, system_prompt, user_prompt, response_format, sections, issue_instructions

holmes/core/prompt.py

@@ -57,7 +57,7 @@ def build_initial_ask_messages(
     system_prompt_template = "builtin://generic_ask.jinja2"
     template_context = {
         "toolsets": tool_executor.toolsets,
-        "runbooks": runbooks or {},
+        "runbooks_enabled": True if runbooks else False,
         "system_prompt_additions": system_prompt_additions or "",
     }
     system_prompt_rendered = load_and_render_prompt(