File tree Expand file tree Collapse file tree 1 file changed +6
-4
lines changed Expand file tree Collapse file tree 1 file changed +6
-4
lines changed Original file line number Diff line number Diff line change @@ -194,19 +194,21 @@ def __iter__(
194
194
}
195
195
196
196
def _create_prompt (
197
- self , prompt_tokens : int , start_index : int , unique_prefix : Optional [int ] = None
197
+ self ,
198
+ prompt_tokens : int ,
199
+ start_index : int ,
200
+ unique_prefix : Optional [int ] = None , # noqa: ARG002
198
201
) -> list [int ]:
199
202
if prompt_tokens <= 0 :
200
203
return []
201
204
202
205
left = start_index
203
206
right = start_index + 4 * prompt_tokens
204
- start_tokens = [unique_prefix ] if unique_prefix else []
205
207
206
208
while left < right :
207
209
mid = (left + right ) // 2
208
210
test_prompt = self .text_creator .create_text (start_index , mid - start_index )
209
- test_tokens = start_tokens + self .processor .encode (test_prompt )
211
+ test_tokens = self .processor .encode (test_prompt )
210
212
211
213
if len (test_tokens ) == prompt_tokens :
212
214
return test_tokens
@@ -216,7 +218,7 @@ def _create_prompt(
216
218
right = mid
217
219
218
220
final_text = self .text_creator .create_text (start_index , left - start_index )
219
- return start_tokens + self .processor .encode (final_text )
221
+ return self .processor .encode (final_text )
220
222
221
223
222
224
class SyntheticDatasetCreator (DatasetCreator ):
You can’t perform that action at this time.
0 commit comments