@@ -101,10 +101,10 @@ def get_all_shard_instructions(
101
101
info : dataset_info .DatasetInfo ,
102
102
out_file_format : file_adapters .FileFormat ,
103
103
out_path : epath .Path ,
104
- in_file_adapter : Type [file_adapters .FileAdapter ],
105
- out_file_adapter : Type [file_adapters .FileAdapter ],
106
104
) -> list [ShardInstruction ]:
107
105
"""Returns all shard instructions for the given dataset info."""
106
+ in_file_adapter = file_adapters .ADAPTER_FOR_FORMAT [info .file_format ]
107
+ out_file_adapter = file_adapters .ADAPTER_FOR_FORMAT [out_file_format ]
108
108
shard_instructions = []
109
109
for split_info in info .splits .values ():
110
110
shard_instructions .extend (
@@ -179,14 +179,10 @@ def _convert_dataset(
179
179
out_dir .unlink (missing_ok = True )
180
180
out_dir .mkdir (parents = True , exist_ok = True )
181
181
182
- in_file_adapter = file_adapters .ADAPTER_FOR_FORMAT [info .file_format ]
183
- out_file_adapter = file_adapters .ADAPTER_FOR_FORMAT [out_file_format ]
184
182
shard_instructions = get_all_shard_instructions (
185
183
info = info ,
186
184
out_file_format = out_file_format ,
187
185
out_path = out_dir ,
188
- in_file_adapter = in_file_adapter ,
189
- out_file_adapter = out_file_adapter ,
190
186
)
191
187
192
188
if not shard_instructions :
0 commit comments