delete useless codes

Author: jayfeather9

lightllm/server/router/dynamic_prompt/hiradix_cache.py

@@ -41,9 +41,6 @@ def __init__(self, unique_name, total_token_num, rank_in_node, mem_manager, max_
             self.hi_cache_kv_buffer = None
             self.is_hi_radix_cache = False
 
-    # write a new function, only insert input(after prefill), call after prefill,
-    # then when the decode finishes, do syncronize to see whether this can be free
-    # no buffer, parallel insert inputs
     def insert_disk(self, req_id, key, value):
         if not self.do_store:
             return
@@ -61,95 +58,17 @@ def abort_req_store_task(self, req_id):
         logger.info(f"Aborting req {req_id} unfinished.")
         self.py_cache_service.az5(self.working_tasks[req_id])
 
-    # TODO: finish this function to only update new ones
-    def _reinsert_helper(self, node: TreeNode, key, value, ans_value_list: list, update_refs=False):
-        if node.is_leaf():
-            self.evict_tree_set.discard(node)
-
-        if update_refs:
-            node.ref_counter += 1
-            # from 0 to 1 need update refs token num
-            if node.ref_counter == 1:
-                self.refed_tokens_num.arr[0] += len(node.token_mem_index_value)
-
-        try:
-            if len(key) == 0:
-                return node
-
-            first_key_id = key[0].item()
-            if first_key_id in node.children.keys():
-                child: TreeNode = node.children[first_key_id]
-                prefix_len = match(key, child.token_id_key)
-                if prefix_len == len(key):
-                    if child.is_leaf():
-                        self.evict_tree_set.discard(child)
-                    child.update_time()
-                    ans_value_list.append(child.token_mem_index_value)
-                    if child.is_leaf():
-                        self.evict_tree_set.add(child)
-                    return prefix_len
-
-                elif prefix_len < len(key) and prefix_len < len(child.token_id_key):
-                    if child.is_leaf():
-                        self.evict_tree_set.discard(child)
-
-                    key = key[prefix_len:]
-                    value = value[prefix_len:]
-                    split_parent_node = child.split_node(prefix_len)
-                    new_node = split_parent_node.add_and_return_new_child(key, value)
-                    # update total token num
-                    self.tree_total_tokens_num.arr[0] += len(new_node.token_mem_index_value)
-
-                    if split_parent_node.is_leaf():
-                        self.evict_tree_set.add(split_parent_node)
-                    if new_node.is_leaf():
-                        self.evict_tree_set.add(new_node)
-
-                    if child.is_leaf():
-                        self.evict_tree_set.add(child)
-                    return prefix_len
-                elif prefix_len < len(key) and prefix_len == len(child.token_id_key):
-                    return prefix_len + self._insert_helper(child, key[prefix_len:], value[prefix_len:])
-                else:
-                    assert False, "can not run to here"
-
-            else:
-                new_node = node.add_and_return_new_child(key, value)
-                # update total token num
-                self.tree_total_tokens_num.arr[0] += len(new_node.token_mem_index_value)
-                ans_value_list.append(new_node.token_mem_index_value)
-                if update_refs:
-                    new_node.ref_counter += 1
-                    if new_node.ref_counter == 1:
-                        self.refed_tokens_num.arr[0] += len(new_node.token_mem_index_value)
-                if new_node.is_leaf():
-                    self.evict_tree_set.add(new_node)
-                return new_node
-        finally:
-            node.update_time()
-            if node.is_leaf():
-                self.evict_tree_set.add(node)
-
     def match_prefix(self, key, update_refs=False):
         assert len(key) != 0
         ans_value_list = []
         pull_hi_cache_tensor = torch.tensor([0], dtype=torch.int64).cuda(self.rank_in_node)
         if self.do_store:
-            # st_time = time.time()
             tree_node = self._match_prefix_helper(self.root_node, key, ans_value_list, update_refs=False)
-            # add a parameter if get long enough (>50%)
-            # first_query_time = time.time()
-            # logger.info(f"HiCache of [{self.rank_in_node}]: No.1 First GPU query took {first_query_time - st_time}s")
             max_len = self._query_hi_cache(key)  # x64
-            # hi_cache_q_time = time.time()
-            # logger.info(f"HiCache of [{self.rank_in_node}]: No.2 Disk query {hi_cache_q_time - first_query_time}s")
             logger.info(f"Matched {sum(len(s) for s in ans_value_list)} from gpu and {max_len} from disk.")
             pull_hi_cache_tensor[0] = max_len if (max_len > sum(len(s) for s in ans_value_list)) else 0
-        # hi_cache_q_time = time.time()
         dist.broadcast(pull_hi_cache_tensor, src=0)
-        # logger.info(f"After broadcast on rank {self.rank_in_node}, tensor={pull_hi_cache_tensor}")
         pull_hi_cache = False
-        # logger.info(f"Rank {self.rank_in_node}, {pull_hi_cache=} {pull_hi_cache_tensor=}")
 
         if pull_hi_cache_tensor[0] == 0 and not self.do_store:
             tree_node = self._match_prefix_helper(self.root_node, key, ans_value_list, update_refs=False)
@@ -166,28 +85,15 @@ def match_prefix(self, key, update_refs=False):
                 tree_node = self._match_prefix_helper(self.root_node, key, ans_value_list, update_refs=update_refs)
         if pull_hi_cache:
             buffers = self.mem_manager.alloc(max_len)
-            # before_pull_time = time.time()
-            # logger.info(
-            #     f"HiCache of [{self.rank_in_node}]: No.2.5 Before pull took {before_pull_time - hi_cache_q_time}"
-            # )
             if self.do_store:
                 read_task = self.py_cache_service.create(tokens=key[:max_len], kv_page_indexer=buffers, mode="r")
                 while not read_task.ready():
                     time.sleep(0.05)
             dist.broadcast(self.mem_manager.get_index_kv_buffer(buffers)["kv_buffer"], src=0)
-            # hicache_pull_time = time.time()
-            # logger.info(f"HiCache of [{self.rank_in_node}]: No.3 Disk pull {hicache_pull_time - before_pull_time}s")
             logger.info(f"HiCache pulled one cache with len = {max_len}")
-            # maybe try: add a function to only insert middle part of kv cache
             self._insert_helper(self.root_node, key, buffers)
-            # insert_time = time.time()
-            # logger.info(f"HiCache of [{self.rank_in_node}]: No.4 Reinsert took {insert_time - hicache_pull_time}")
             ans_value_list = []
             tree_node = self._match_prefix_helper(self.root_node, key, ans_value_list, update_refs=update_refs)
-            # logger.info(
-            #     f"HiCache of [{self.rank_in_node}]: No.5 Re match prefix took {time.time() - insert_time}"
-            #     + f" matched {sum(len(s) for s in ans_value_list)} tokens"
-            # )
         if tree_node != self.root_node:
             if len(ans_value_list) != 0:
                 value = torch.concat(ans_value_list)