feat(clp-package): Add MCP Server config interface and Docker Compose orchestration. (#1481)

Co-authored-by: Junhao Liao <[email protected]>
Co-authored-by: Junhao Liao <[email protected]>
Co-authored-by: kirkrodrigues <[email protected]>

Author: 3 people

components/clp-mcp-server/clp_mcp_server/clp_mcp_server.py

@@ -2,17 +2,21 @@
 
 import ipaddress
 import logging
+import os
 import socket
 import sys
 from pathlib import Path
 
 import click
-from clp_py_utils.clp_config import CLPConfig
+from clp_py_utils.clp_config import CLPConfig, MCP_SERVER_COMPONENT_NAME
+from clp_py_utils.clp_logging import get_logger, get_logging_formatter, set_logging_level
 from clp_py_utils.core import read_yaml_config_file
 from pydantic import ValidationError
 
 from .server import create_mcp_server
 
+logger = get_logger(MCP_SERVER_COMPONENT_NAME)
+
 
 @click.command()
 @click.option(
@@ -34,10 +38,12 @@ def main(host: str, port: int, config_path: Path) -> int:
     :param config_path: The path to server's configuration file.
     :return: Exit code (0 for success, non-zero for failure).
     """
-    logging.basicConfig(
-        level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-    logger = logging.getLogger(__name__)
+    # Setup logging to file
+    log_file_path = Path(os.getenv("CLP_LOGS_DIR")) / "mcp_server.log"
+    logging_file_handler = logging.FileHandler(filename=log_file_path, encoding="utf-8")
+    logging_file_handler.setFormatter(get_logging_formatter())
+    logger.addHandler(logging_file_handler)
+    set_logging_level(logger, os.getenv("CLP_LOGGING_LEVEL"))
 
     exit_code = 0
 

components/clp-mcp-server/clp_mcp_server/server/server.py

@@ -4,6 +4,8 @@
 
 from clp_py_utils.clp_config import CLPConfig
 from fastmcp import Context, FastMCP
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
 
 from clp_mcp_server.clp_connector import ClpConnector
 
@@ -161,4 +163,14 @@ async def search_by_kql_with_timestamp_range(
 
         return await _execute_kql_query(ctx.session_id, kql_query, begin_ts, end_ts)
 
+    @mcp.custom_route("/health", methods=["GET"])
+    async def health_check(_request: Request) -> PlainTextResponse:
+        """
+        Health check endpoint.
+
+        :param _request: An HTTP request object.
+        :return: A plain text response indicating server is healthy.
+        """
+        return PlainTextResponse("OK")
+
     return mcp

components/clp-package-utils/clp_package_utils/controller.py

@@ -19,6 +19,7 @@
     DB_COMPONENT_NAME,
     DeploymentType,
     GARBAGE_COLLECTOR_COMPONENT_NAME,
+    MCP_SERVER_COMPONENT_NAME,
     QUERY_JOBS_TABLE_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
@@ -47,6 +48,7 @@
     get_clp_home,
     is_retention_period_configured,
     validate_db_config,
+    validate_mcp_server_config,
     validate_queue_config,
     validate_redis_config,
     validate_results_cache_config,
@@ -528,6 +530,37 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict:
 
         return env_vars
 
+    def _set_up_env_for_mcp_server(self) -> EnvVarsDict:
+        """
+        Sets up environment variables and directories for the MCP server component.
+
+        :return: Dictionary of environment variables necessary to launch the component.
+        """
+        component_name = MCP_SERVER_COMPONENT_NAME
+        if self._clp_config.mcp_server is None:
+            logger.info(f"The MCP Server is not configured, skipping {component_name} creation...")
+            return EnvVarsDict()
+        logger.info(f"Setting up environment for {component_name}...")
+
+        logs_dir = self._clp_config.logs_directory / component_name
+        validate_mcp_server_config(self._clp_config, logs_dir)
+        logs_dir.mkdir(parents=True, exist_ok=True)
+
+        env_vars = EnvVarsDict()
+
+        # Connection config
+        env_vars |= {
+            "CLP_MCP_HOST": _get_ip_from_hostname(self._clp_config.mcp_server.host),
+            "CLP_MCP_PORT": str(self._clp_config.mcp_server.port),
+        }
+
+        # Logging config
+        env_vars |= {
+            "CLP_MCP_LOGGING_LEVEL": self._clp_config.mcp_server.logging_level,
+        }
+
+        return env_vars
+
     def _set_up_env_for_garbage_collector(self) -> EnvVarsDict:
         """
         Sets up environment variables for the garbage collector component.
@@ -627,6 +660,8 @@ def start(self) -> None:
         cmd = ["docker", "compose", "--project-name", self._project_name]
         if deployment_type == DeploymentType.BASE:
             cmd += ["--file", "docker-compose.base.yaml"]
+        if self._clp_config.mcp_server is not None:
+            cmd += ["--profile", "mcp"]
         cmd += ["up", "--detach", "--wait"]
         subprocess.run(
             cmd,
@@ -750,6 +785,7 @@ def _set_up_env(self) -> None:
         env_vars |= self._set_up_env_for_query_worker(num_workers)
         env_vars |= self._set_up_env_for_reducer(num_workers)
         env_vars |= self._set_up_env_for_webui(container_clp_config)
+        env_vars |= self._set_up_env_for_mcp_server()
         env_vars |= self._set_up_env_for_garbage_collector()
 
         # Write the environment variables to the `.env` file.

components/clp-package-utils/clp_package_utils/general.py

@@ -20,6 +20,7 @@
     CONTAINER_CLP_HOME,
     CONTAINER_INPUT_LOGS_ROOT_DIR,
     DB_COMPONENT_NAME,
+    MCP_SERVER_COMPONENT_NAME,
     QueryEngine,
     QUEUE_COMPONENT_NAME,
     REDIS_COMPONENT_NAME,
@@ -584,6 +585,14 @@ def validate_webui_config(
     validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port)
 
 
+def validate_mcp_server_config(clp_config: CLPConfig, logs_dir: pathlib.Path):
+    _validate_log_directory(logs_dir, MCP_SERVER_COMPONENT_NAME)
+
+    validate_port(
+        f"{MCP_SERVER_COMPONENT_NAME}.port", clp_config.mcp_server.host, clp_config.mcp_server.port
+    )
+
+
 def validate_path_for_container_mount(path: pathlib.Path) -> None:
     RESTRICTED_PREFIXES: List[pathlib.Path] = [
         CONTAINER_AWS_CONFIG_DIRECTORY,

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -35,6 +35,7 @@
 COMPRESSION_WORKER_COMPONENT_NAME = "compression_worker"
 QUERY_WORKER_COMPONENT_NAME = "query_worker"
 WEBUI_COMPONENT_NAME = "webui"
+MCP_SERVER_COMPONENT_NAME = "mcp_server"
 GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector"
 
 # Action names
@@ -575,6 +576,14 @@ class SweepInterval(BaseModel):
     search_result: PositiveInt = 30
 
 
+class McpServer(BaseModel):
+    DEFAULT_PORT: ClassVar[int] = 8000
+
+    host: DomainStr = "localhost"
+    port: Port = DEFAULT_PORT
+    logging_level: LoggingLevel = "INFO"
+
+
 class GarbageCollector(BaseModel):
     logging_level: LoggingLevel = "INFO"
     sweep_interval: SweepInterval = SweepInterval()
@@ -611,6 +620,7 @@ class CLPConfig(BaseModel):
     garbage_collector: GarbageCollector = GarbageCollector()
     credentials_file_path: SerializablePath = CLP_DEFAULT_CREDENTIALS_FILE_PATH
 
+    mcp_server: Optional[McpServer] = None
     presto: Optional[Presto] = None
 
     archive_output: ArchiveOutput = ArchiveOutput()

components/package-template/src/etc/clp-config.yml

@@ -68,6 +68,11 @@
 #  results_metadata_collection_name: "results-metadata"
 #  rate_limit: 1000
 #
+#mcp_server:
+#  host: "localhost"
+#  port: 8000
+#  logging_level: "INFO"
+#
 ## Where archives should be output to
 #archive_output:
 #  storage:

docs/src/dev-docs/design-deployment-orchestration.md

@@ -45,6 +45,8 @@ graph LR
   query_worker["query-worker"]
   reducer["reducer"]
   webui["webui"]
+  mcp_server["mcp-server"]
+
   garbage_collector["garbage-collector"]
 
   %% One-time jobs
@@ -64,6 +66,8 @@ graph LR
   results_cache_indices_creator -->|completed_successfully| reducer
   db_table_creator -->|completed_successfully| webui
   results_cache_indices_creator -->|completed_successfully| webui
+  db_table_creator -->|completed_successfully| mcp_server
+  results_cache_indices_creator -->|completed_successfully| mcp_server
   db_table_creator -->|completed_successfully| garbage_collector
   results_cache_indices_creator -->|completed_successfully| garbage_collector
 
@@ -94,7 +98,11 @@ graph LR
     webui
     garbage_collector
   end
-:::
+
+  subgraph AI
+    mcp_server
+  end
+
 
 +++
 **Figure 1**: Orchestration architecture of the services in the CLP package.
@@ -118,6 +126,7 @@ graph LR
 | query_worker          | Worker processes for search/aggregation jobs                    |
 | reducer               | Reducers for performing the final stages of aggregation jobs    |
 | webui                 | Web server for the UI                                           |
+| mcp_server            | MCP server for AI agent to access CLP functionalities           |
 | garbage_collector     | Process to manage data retention                                |
 
 :::

tools/deployment/package/docker-compose.yaml

@@ -125,3 +125,41 @@ services:
       "--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}",
       "--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}"
     ]
+
+  mcp-server:
+    <<: *service_defaults
+    hostname: "mcp_server"
+    profiles: ["mcp"]
+    environment:
+      CLP_LOGGING_LEVEL: "${CLP_MCP_LOGGING_LEVEL:-INFO}"
+      CLP_LOGS_DIR: "/var/log/mcp_server"
+      CLP_DB_USER: "${CLP_DB_USER}"
+      CLP_DB_PASS: "${CLP_DB_PASS}"
+      PYTHONPATH: "/opt/clp/lib/python3/site-packages"
+    ports:
+      - host_ip: "${CLP_MCP_HOST:-127.0.0.1}"
+        published: "${CLP_MCP_PORT:-8000}"
+        target: 8000
+    volumes:
+      - *volume_clp_config_readonly
+      - *volume_clp_logs
+    depends_on:
+      db-table-creator:
+        condition: "service_completed_successfully"
+      results-cache-indices-creator:
+        condition: "service_completed_successfully"
+    command: [
+      "python3", "-u",
+      "-m", "clp_mcp_server.clp_mcp_server",
+      "--host", "mcp_server",
+      "--port", "8000",
+      "--config-path", "/etc/clp-config.yml",
+    ]
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "curl",
+        "-f",
+        "http://mcp_server:8000/health"
+      ]

tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh

@@ -6,6 +6,7 @@ set -o pipefail
 apt-get update
 DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   ca-certificates \
+  curl \
   libcurl4 \
   libmariadb3 \
   python3