File tree Expand file tree Collapse file tree 1 file changed +5
-3
lines changed
Expand file tree Collapse file tree 1 file changed +5
-3
lines changed Original file line number Diff line number Diff line change @@ -127,7 +127,8 @@ def _append_logical_block(self) -> None:
127127 self .logical_token_blocks .append (block )
128128
129129 def _append_tokens_to_blocks (self , token_ids : List [int ]) -> None :
130- while token_ids :
130+ cursor = 0
131+ while cursor < len (token_ids ):
131132 if not self .logical_token_blocks :
132133 self ._append_logical_block ()
133134
@@ -137,8 +138,9 @@ def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
137138 last_block = self .logical_token_blocks [- 1 ]
138139
139140 num_empty_slots = last_block .get_num_empty_slots ()
140- last_block .append_tokens (token_ids [:num_empty_slots ])
141- token_ids = token_ids [num_empty_slots :]
141+ last_block .append_tokens (token_ids [cursor :cursor +
142+ num_empty_slots ])
143+ cursor += num_empty_slots
142144
143145 def append_token_id (
144146 self ,
You can’t perform that action at this time.
0 commit comments