676 feature request display total index size per customer in indicesmanagement (#687)

* feat: add endpoint to fetch total indices size per customer

* feat: add component to display total indices size per customer

* feat: enhance customer indices display with popover for index details

* feat: enable index selection in CustomerIndicesSize component

* precommit fixes

* lint fixes

* chore: update CURRENT_VERSION to 0.1.43

Author: taylorwalton

backend/app/connectors/wazuh_indexer/routes/monitoring.py

@@ -6,12 +6,14 @@
 
 from app.auth.utils import AuthHandler
 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealthResponse
+from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSizeResponse
 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStatsResponse
 from app.connectors.wazuh_indexer.schema.monitoring import NodeAllocationResponse
 from app.connectors.wazuh_indexer.schema.monitoring import ShardsResponse
 
 # from app.connectors.wazuh_indexer.schema import WazuhIndexerResponse, WazuhIndexerListResponse
 from app.connectors.wazuh_indexer.services.monitoring import cluster_healthcheck
+from app.connectors.wazuh_indexer.services.monitoring import indices_size_per_customer
 from app.connectors.wazuh_indexer.services.monitoring import indices_stats
 from app.connectors.wazuh_indexer.services.monitoring import node_allocation
 from app.connectors.wazuh_indexer.services.monitoring import (
@@ -101,6 +103,35 @@ async def get_indices_stats() -> Union[IndicesStatsResponse, HTTPException]:
         raise HTTPException(status_code=500, detail="Failed to retrieve indices stats.")
 
 
+@wazuh_indexer_router.get(
+    "/indices/size-per-customer",
+    response_model=CustomerIndicesSizeResponse,
+    description="Fetch Wazuh Indexer indices size aggregated per customer",
+    dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
+)
+async def get_indices_size_per_customer() -> Union[CustomerIndicesSizeResponse, HTTPException]:
+    """
+    Fetch Wazuh Indexer indices size per customer.
+
+    This endpoint retrieves the total indices size aggregated per customer,
+    where customer is extracted from index names (e.g., wazuh-copilot_37 -> copilot).
+
+    Returns:
+        CustomerIndicesSizeResponse: A Pydantic model representing the indices size per customer.
+
+    Raises:
+        HTTPException: An exception with a 500 status code is raised if the data cannot be retrieved.
+    """
+    try:
+        response = await indices_size_per_customer()
+        return response
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to retrieve indices size per customer: {str(e)}",
+        )
+
+
 @wazuh_indexer_router.get(
     "/shards",
     response_model=ShardsResponse,

backend/app/connectors/wazuh_indexer/schema/monitoring.py

@@ -78,3 +78,17 @@ class ShardsResponse(BaseModel):
     shards: Optional[List[Shards]]
     message: str
     success: bool
+
+
+class CustomerIndicesSize(BaseModel):
+    customer: str
+    total_size_bytes: int
+    total_size_human: str
+    index_count: int
+    indices: List[str]
+
+
+class CustomerIndicesSizeResponse(BaseModel):
+    customer_sizes: Optional[List[CustomerIndicesSize]]
+    message: str
+    success: bool

backend/app/connectors/wazuh_indexer/services/monitoring.py

@@ -1,10 +1,13 @@
+import re
 from typing import Dict
 from typing import Union
 
 from loguru import logger
 
 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealth
 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealthResponse
+from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSize
+from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSizeResponse
 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStats
 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStatsResponse
 from app.connectors.wazuh_indexer.schema.monitoring import NodeAllocation
@@ -153,3 +156,127 @@ async def output_shard_number_to_be_set_based_on_nodes() -> int:
         logger.error(f"Shards check failed with error: {e}")
         e = f"Shards check failed with error: {e}"
         raise Exception(str(e))
+
+
+def parse_size_to_bytes(size_str: str) -> int:
+    """
+    Convert a human-readable size string to bytes.
+    Handles formats like '1.2gb', '500mb', '100kb', '1024b'.
+    """
+    if not size_str or size_str == "Store size not found":
+        return 0
+
+    size_str = size_str.lower().strip()
+
+    # Define multipliers
+    multipliers = {
+        "b": 1,
+        "kb": 1024,
+        "mb": 1024**2,
+        "gb": 1024**3,
+        "tb": 1024**4,
+    }
+
+    # Match number and unit
+    match = re.match(r"^([\d.]+)\s*([a-z]+)$", size_str)
+    if match:
+        value = float(match.group(1))
+        unit = match.group(2)
+        return int(value * multipliers.get(unit, 1))
+
+    # Try to parse as pure number (bytes)
+    try:
+        return int(float(size_str))
+    except ValueError:
+        return 0
+
+
+def bytes_to_human_readable(size_bytes: int) -> str:
+    """Convert bytes to human-readable format."""
+    for unit in ["b", "kb", "mb", "gb", "tb"]:
+        if abs(size_bytes) < 1024.0:
+            return f"{size_bytes:.2f}{unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.2f}pb"
+
+
+def extract_customer_from_index(index_name: str) -> str:
+    """
+    Extract customer name from index name.
+    Pattern: after dash or underscore, before the next underscore or end.
+    Examples:
+        - wazuh-copilot_37 -> copilot
+        - dev-taylor_37 -> taylor
+        - wazuh-509dine2v_0 -> 509dine2v
+    """
+    # Match pattern: prefix-customer_suffix or prefix_customer_suffix
+    match = re.match(r"^[^-_]+-([^_]+)_", index_name)
+    if match:
+        return match.group(1)
+
+    # Fallback: try underscore as first separator
+    match = re.match(r"^[^_]+_([^_]+)_", index_name)
+    if match:
+        return match.group(1)
+
+    return "unknown"
+
+
+async def indices_size_per_customer() -> Union[CustomerIndicesSizeResponse, Dict[str, str]]:
+    """
+    Returns the total indices size aggregated per customer.
+
+    Returns:
+        CustomerIndicesSizeResponse: A Pydantic model containing the indices size per customer.
+
+    Raises:
+        Exception: An exception is raised if the indices stats cannot be retrieved.
+    """
+    logger.info("Collecting Wazuh Indexer indices size per customer")
+    es_client = await create_wazuh_indexer_client("Wazuh-Indexer")
+    try:
+        raw_indices_stats_data = es_client.cat.indices(format="json")
+
+        formatted_indices_stats_data = await format_indices_stats(raw_indices_stats_data)
+
+        # Aggregate by customer
+        customer_data: Dict[str, Dict] = {}
+
+        for index_data in formatted_indices_stats_data:
+            index_name = index_data.get("index", "")
+            store_size = index_data.get("store_size", "0b")
+
+            customer = extract_customer_from_index(index_name)
+            size_bytes = parse_size_to_bytes(store_size)
+
+            if customer not in customer_data:
+                customer_data[customer] = {
+                    "total_size_bytes": 0,
+                    "index_count": 0,
+                    "indices": [],
+                }
+
+            customer_data[customer]["total_size_bytes"] += size_bytes
+            customer_data[customer]["index_count"] += 1
+            customer_data[customer]["indices"].append(index_name)
+
+        # Convert to response models
+        customer_sizes = [
+            CustomerIndicesSize(
+                customer=customer,
+                total_size_bytes=data["total_size_bytes"],
+                total_size_human=bytes_to_human_readable(data["total_size_bytes"]),
+                index_count=data["index_count"],
+                indices=data["indices"],
+            )
+            for customer, data in sorted(customer_data.items())
+        ]
+
+        return CustomerIndicesSizeResponse(
+            customer_sizes=customer_sizes,
+            success=True,
+            message="Successfully collected Wazuh Indexer indices size per customer",
+        )
+    except Exception as e:
+        logger.error(f"Indices size per customer check failed with error: {e}")
+        raise Exception(f"Indices size per customer check failed with error: {e}")

backend/app/version/services/version.py

@@ -7,7 +7,7 @@
 from packaging.version import Version
 
 # Current version - update this with each release
-CURRENT_VERSION = "0.1.42"
+CURRENT_VERSION = "0.1.43"
 VERSION_CHECK_URL = "https://api.github.com/repos/socfortress/CoPilot/releases/latest"
 
 

frontend/src/api/endpoints/wazuh/indices.ts

@@ -14,5 +14,18 @@ export default {
 	},
 	getClusterHealth() {
 		return HttpClient.get<FlaskBaseResponse & { cluster_health: ClusterHealth }>("/wazuh_indexer/health")
-	}
+	},
+	getIndicesSizePerCustomer() {
+    return HttpClient.get<{
+        customer_sizes: {
+            customer: string
+            total_size_bytes: number
+            total_size_human: string
+            index_count: number
+            indices: string[]
+        }[]
+        message: string
+        success: boolean
+    }>(`/wazuh_indexer/indices/size-per-customer`)
+}
 }