feat: add radix prefix hit rate log

Author: sufubao

lightllm/server/router/dynamic_prompt/hybrid_radix_cache.py

@@ -33,10 +33,6 @@ def __init__(self, unique_name, total_token_num, rank_in_node, mem_manager=None)
         super().__init__(unique_name, total_token_num, rank_in_node, mem_manager)
         # 用于缓存需要被驱逐的buffer节点， 应该包含所有有buffer的节点
         self.evict_buffer_set: Set[TreeNode] = SortedSet(key=lambda x: (x.buffer_time,))
-        self.match_count = 0
-        self.log_interval = 1000
-        self.match_len = 0
-        self.hit_len = 0
 
     def free_radix_cache_to_get_enough_buffer(self, need_buffer_num):
         if need_buffer_num > self.mem_manager.get_buffer_can_use_size():
@@ -112,8 +108,6 @@ def insert_for_hybrid_radix_cache(self, reqs):
 
     def match_prefix(self, key, update_refs=False):
         assert len(key) != 0
-        self.match_count = (self.match_count + 1) % self.log_interval
-        self.match_len += len(key)
         ans_value_list = []
         tree_node = self._match_prefix_helper(self.root_node, key, ans_value_list, update_refs=update_refs)
         origin_ans_len = sum(len(v) for v in ans_value_list)
@@ -145,6 +139,7 @@ def match_prefix(self, key, update_refs=False):
             self.mem_manager.free(evict_token_value)
 
         if tree_node == self.root_node:
+            self._inc_hit_rate(len(key), 0)
             return None, origin_ans_len, None
 
         update_node = tree_node
@@ -156,16 +151,7 @@ def match_prefix(self, key, update_refs=False):
             update_node = update_node.parent
 
         value = torch.concat(ans_value_list)
-        # logger.info("HybridRadixCache match_prefix hit tokens: {}".format(len(value)))
-        self.hit_len += len(value)
-        if self.match_count == 0:
-            logger.info(
-                f"HybridRadixCache match_prefix avg hit rate: {self.hit_len / self.match_len:.4f} "
-                f"({self.hit_len}/{self.match_len}) over last {self.log_interval} matches"
-            )
-            self.match_len = 0
-            self.hit_len = 0
-
+        self._inc_hit_rate(len(key), len(value))
         return tree_node, origin_ans_len, value
 
     def add_buffer_idx_to_node(self, node: TreeNode, buffer_idx: int):

lightllm/server/router/dynamic_prompt/radix_cache.py

@@ -5,6 +5,9 @@
 from typing import Tuple, Dict, Set, List, Optional, Union
 from sortedcontainers import SortedSet
 from .shared_arr import SharedArray
+from lightllm.utils.log_utils import init_logger, log_time_ready
+
+logger = init_logger(__name__)
 
 
 class UniqueTimeIdGenerator:
@@ -135,6 +138,34 @@ def __init__(self, unique_name, total_token_num, rank_in_node, mem_manager=None)
         )
         self.tree_total_tokens_num.arr[0] = 0
 
+        self.total_query_tokens = SharedArray(f"{unique_name}_total_query_tokens_{rank_in_node}", (1,), dtype=np.int64)
+        self.total_query_tokens.arr[0] = 0
+        self.total_hit_tokens = SharedArray(f"{unique_name}_total_hit_tokens_{rank_in_node}", (1,), dtype=np.int64)
+        self.total_hit_tokens.arr[0] = 0
+        self.last_log_query_tokens = 0
+        self.last_log_hit_tokens = 0
+
+    def _inc_hit_rate(self, query_len, hit_len):
+        self.total_query_tokens.arr[0] += query_len
+        self.total_hit_tokens.arr[0] += hit_len
+        if log_time_ready("radix_cache_hit_rate", time_count=30):
+            current_total_query = self.total_query_tokens.arr[0]
+            current_total_hit = self.total_hit_tokens.arr[0]
+            window_query = current_total_query - self.last_log_query_tokens
+            window_hit = current_total_hit - self.last_log_hit_tokens
+            window_hit_rate = window_hit / window_query if window_query > 0 else 0.0
+            cumulative_hit_rate = current_total_hit / current_total_query if current_total_query > 0 else 0.0
+
+            label = self.__class__.__name__
+            logger.info(
+                f"{label} Hit Rate: "
+                f"Window {window_hit_rate:.2%} ({window_hit}/{window_query}), "
+                f"Cumulative {cumulative_hit_rate:.2%} ({current_total_hit}/{current_total_query})"
+            )
+
+            self.last_log_query_tokens = current_total_query
+            self.last_log_hit_tokens = current_total_hit
+
     def insert(self, key, value=None) -> Tuple[int, Optional[TreeNode]]:
         if value is None:
             value = key
@@ -248,9 +279,13 @@ def match_prefix(self, key, update_refs=False):
                 value = torch.concat(ans_value_list)
             else:
                 value = torch.zeros((0,), device="cpu", dtype=self._value_dtype)
-            return tree_node, len(value), value
+
+            matched_len = len(value)
+            self._inc_hit_rate(len(key), matched_len)
+            return tree_node, matched_len, value
         else:
             self.dec_node_ref_counter(self.root_node)
+            self._inc_hit_rate(len(key), 0)
             return None, 0, None
 
     def _match_prefix_helper(