{AKS} Bump holmesgpt and add feedback slash command (Azure#9261)

Author: mainred

src/aks-agent/HISTORY.rst

@@ -12,6 +12,16 @@ To release a new version, please select a new version number (usually plus 1 to
 Pending
 +++++++
 
+1.0.0b5
++++++++
+* Bump holmesgpt to 0.15.0 - Enhanced AI debugging experience and bug fixes
+  * Added TODO list feature to allows holmes to reliably answers questions it wasn't able to answer before due to early-stopping
+  * Fixed mcp server http connection fails when using socks proxy by adding the missing socks dependency
+  * Fixed gpt-5 temperature bug by upgrading litellm and dropping non-1 values for temperature
+  * Improved the installation time by removing unnecessary dependencies and move test dependencies to dev dependency group
+* Added Feedback slash command Feature to allow users to provide feedback on their experience with the agent performance
+* Disable prometheus toolset loading by default to workaround the libbz2-dev missing issue in Azure CLI python environment.
+
 1.0.0b4
 +++++++
 * Fix the --aks-mcp flag to allow true/false values.

src/aks-agent/azext_aks_agent/_consts.py

@@ -3,11 +3,24 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-# aks agent constants
+# Constants to customized holmesgpt
 CONST_AGENT_CONFIG_PATH_DIR_ENV_KEY = "HOLMES_CONFIGPATH_DIR"
 CONST_AGENT_NAME = "AKS AGENT"
 CONST_AGENT_NAME_ENV_KEY = "AGENT_NAME"
 CONST_AGENT_CONFIG_FILE_NAME = "aksAgent.yaml"
+CONST_PRIVACY_NOTICE_BANNER_ENV_KEY = "PRIVACY_NOTICE_BANNER"
+# Privacy Notice Banner displayed in the format of rich.Console
+CONST_PRIVACY_NOTICE_BANNER = (
+    "When you send Microsoft this feedback, you agree we may combine this information, which might include other "
+    "diagnostic data, to help improve Microsoft products and services. Processing of feedback data is governed by "
+    "the Microsoft Products and Services Data Protection Addendum between your organization and Microsoft, and the "
+    "feedback you submit is considered Personal Data under that addendum. "
+    "Privacy Statement: https://go.microsoft.com/fwlink/?LinkId=521839"
+)
+# Holmesgpt leverages prometheus_api_client for prometheus toolsets and introduces bz2 library.
+# Before libbz2-dev is bundled into azure cli python by https://github.com/Azure/azure-cli/pull/32163,
+# we ignore loading prometheus toolset to avoid loading error of bz2 module.
+CONST_DISABLE_PROMETHEUS_TOOLSET_ENV_KEY = "DISABLE_PROMETHEUS_TOOLSET"
 
 # MCP Integration Constants (ported from previous change)
 CONST_MCP_BINARY_NAME = "aks-mcp"

src/aks-agent/azext_aks_agent/_params.py

@@ -6,12 +6,10 @@
 # pylint: disable=too-many-statements,too-many-lines
 import os.path
 
-from azure.cli.core.api import get_config_dir
-from azure.cli.core.commands.parameters import get_three_state_flag
-
 from azext_aks_agent._consts import CONST_AGENT_CONFIG_FILE_NAME
-
 from azext_aks_agent._validators import validate_agent_config_file
+from azure.cli.core.api import get_config_dir
+from azure.cli.core.commands.parameters import get_three_state_flag
 
 
 def load_arguments(self, _):
@@ -37,7 +35,7 @@ def load_arguments(self, _):
         c.argument(
             "max_steps",
             type=int,
-            default=10,
+            default=40,
             required=False,
             help="Maximum number of steps the LLM can take to investigate the issue.",
         )

src/aks-agent/azext_aks_agent/agent/agent.py

@@ -11,14 +11,25 @@
     CONST_AGENT_CONFIG_PATH_DIR_ENV_KEY,
     CONST_AGENT_NAME,
     CONST_AGENT_NAME_ENV_KEY,
+    CONST_DISABLE_PROMETHEUS_TOOLSET_ENV_KEY,
+    CONST_PRIVACY_NOTICE_BANNER,
+    CONST_PRIVACY_NOTICE_BANNER_ENV_KEY,
 )
 from azure.cli.core.api import get_config_dir
 from azure.cli.core.commands.client_factory import get_subscription_id
 from knack.util import CLIError
 
+from .error_handler import MCPError
 from .prompt import AKS_CONTEXT_PROMPT_MCP, AKS_CONTEXT_PROMPT_TRADITIONAL
 from .telemetry import CLITelemetryClient
-from .error_handler import MCPError
+
+
+# NOTE(mainred): environment variables to disable prometheus toolset loading should be set before importing holmes.
+def customize_holmesgpt():
+    os.environ[CONST_DISABLE_PROMETHEUS_TOOLSET_ENV_KEY] = "true"
+    os.environ[CONST_AGENT_CONFIG_PATH_DIR_ENV_KEY] = get_config_dir()
+    os.environ[CONST_AGENT_NAME_ENV_KEY] = CONST_AGENT_NAME
+    os.environ[CONST_PRIVACY_NOTICE_BANNER_ENV_KEY] = CONST_PRIVACY_NOTICE_BANNER
 
 
 # NOTE(mainred): holmes leverage the log handler RichHandler to provide colorful, readable and well-formatted logs
@@ -151,21 +162,19 @@ def aks_agent(
     :type use_aks_mcp: bool
     """
 
-    with CLITelemetryClient():
+    with CLITelemetryClient() as telemetry:
         if sys.version_info < (3, 10):
             raise CLIError(
                 "Please upgrade the python version to 3.10 or above to use aks agent."
             )
+        # customizing holmesgpt should called before importing holmes
+        customize_holmesgpt()
 
         # Initialize variables
         interactive = not no_interactive
         echo = not no_echo_request
         console = init_log()
 
-        # Set environment variables for Holmes
-        os.environ[CONST_AGENT_CONFIG_PATH_DIR_ENV_KEY] = get_config_dir()
-        os.environ[CONST_AGENT_NAME_ENV_KEY] = CONST_AGENT_NAME
-
         # Detect and read piped input
         piped_data = None
         if not sys.stdin.isatty():
@@ -265,7 +274,7 @@ def aks_agent(
             is_mcp_mode = current_mode == "mcp"
             if interactive:
                 _run_interactive_mode_sync(ai, cmd, resource_group_name, name,
-                                           prompt, console, show_tool_output, is_mcp_mode)
+                                           prompt, console, show_tool_output, is_mcp_mode, telemetry)
             else:
                 _run_noninteractive_mode_sync(ai, config, cmd, resource_group_name, name,
                                               prompt, console, echo, show_tool_output, is_mcp_mode)
@@ -312,13 +321,15 @@ async def _setup_mcp_mode(mcp_manager, config_file: str, model: str, api_key: st
     :return: Enhanced Holmes configuration
     :raises: Exception if MCP setup fails
     """
+    import tempfile
     from pathlib import Path
+
     import yaml
-    import tempfile
     from holmes.config import Config
+
     from .config_generator import ConfigurationGenerator
-    from .user_feedback import ProgressReporter
     from .error_handler import AgentErrorHandler
+    from .user_feedback import ProgressReporter
 
     # Ensure binary is available (download if needed)
     if not mcp_manager.is_binary_available() or not mcp_manager.validate_binary_version():
@@ -602,7 +613,7 @@ def _build_aks_context(cluster_name, resource_group_name, subscription_id, is_mc
 
 
 def _run_interactive_mode_sync(ai, cmd, resource_group_name, name,
-                               prompt, console, show_tool_output, is_mcp_mode):
+                               prompt, console, show_tool_output, is_mcp_mode, telemetry):
     """
     Run interactive mode synchronously - no event loop conflicts.
 
@@ -617,6 +628,7 @@ def _run_interactive_mode_sync(ai, cmd, resource_group_name, name,
     :param console: Console object for output
     :param show_tool_output: Whether to show tool output
     :param is_mcp_mode: Whether running in MCP mode (affects prompt selection)
+    :param telemetry: CLITelemetryClient instance for tracking events
     """
     from holmes.interactive import run_interactive_loop
 
@@ -633,7 +645,8 @@ def _run_interactive_mode_sync(ai, cmd, resource_group_name, name,
         ai, console, prompt, None, None,
         show_tool_output=show_tool_output,
         system_prompt_additions=aks_context,
-        check_version=False
+        check_version=False,
+        feedback_callback=telemetry.track_agent_feedback if telemetry else None
     )
 
 
@@ -653,8 +666,9 @@ def _run_noninteractive_mode_sync(ai, config, cmd, resource_group_name, name,
     :param show_tool_output: Whether to show tool output
     :param is_mcp_mode: Whether running in MCP mode (affects prompt selection)
     """
-    import uuid
     import socket
+    import uuid
+
     from holmes.core.prompt import build_initial_ask_messages
     from holmes.plugins.destinations import DestinationType
     from holmes.plugins.interfaces import Issue
@@ -702,10 +716,12 @@ def _setup_traditional_mode_sync(config_file: str, model: str, api_key: str,
     :param verbose: Enable verbose output
     :return: Traditional Holmes configuration
     """
+    import tempfile
     from pathlib import Path
+
     import yaml
-    import tempfile
     from holmes.config import Config
+
     from .config_generator import ConfigurationGenerator
 
     # Load base config

src/aks-agent/azext_aks_agent/agent/telemetry.py

@@ -4,13 +4,17 @@
 # --------------------------------------------------------------------------------------------
 
 import datetime
+import json
 import logging
 import os
 import platform
 
 from applicationinsights import TelemetryClient
-from azure.cli.core.telemetry import (_get_azure_subscription_id,
-                                      _get_hash_mac_address, _get_user_agent)
+from azure.cli.core.telemetry import (
+    _get_azure_subscription_id,
+    _get_hash_mac_address,
+    _get_user_agent,
+)
 
 DEFAULT_INSTRUMENTATION_KEY = "c301e561-daea-42d9-b9d1-65fca4166704"
 APPLICATIONINSIGHTS_INSTRUMENTATION_KEY_ENV = "APPLICATIONINSIGHTS_INSTRUMENTATION_KEY"
@@ -75,3 +79,21 @@ def _get_application_insights_instrumentation_key(self) -> str:
         return os.getenv(
             APPLICATIONINSIGHTS_INSTRUMENTATION_KEY_ENV, DEFAULT_INSTRUMENTATION_KEY
         )
+
+    def track_agent_feedback(self, feedback):
+        # NOTE: We should try to avoid importing holmesgpt at the top level to prevent dependency issues
+        from holmes.core.feedback import Feedback, FeedbackMetadata
+
+        # Type hint validation for development purposes
+        if not isinstance(feedback, Feedback):
+            raise TypeError(f"Expected Feedback object, got {type(feedback)}")
+
+        # Before privacy team's approval for other user data, we keep only direct user feedback, and model info.
+        feedback_filtered = Feedback()
+        feedback_filtered.user_feedback = feedback.user_feedback
+        feedback_metadata = FeedbackMetadata()
+        feedback_metadata.model = feedback.metadata.model
+        feedback_filtered.metadata = feedback_metadata
+        self.track("AgentCLIFeedback", properties={"feedback": json.dumps(feedback_filtered.to_dict())})
+        # Flush the telemetry data immediately to avoid too much data being sent at once
+        self.flush()