File tree Expand file tree Collapse file tree 1 file changed +8
-2
lines changed
src/datasets/packaged_modules/csv Expand file tree Collapse file tree 1 file changed +8
-2
lines changed Original file line number Diff line number Diff line change @@ -148,12 +148,18 @@ class Csv(datasets.ArrowBasedBuilder):
148148 def _info (self ):
149149 return datasets .DatasetInfo (features = self .config .features )
150150
151- def _split_generators (self , dl_manager ):
151+ def _available_splits (self ) -> Optional [List [str ]]:
152+ return [str (split ) for split in self .config .data_files ] if isinstance (self .config .data_files , dict ) else None
153+
154+ def _split_generators (self , dl_manager , splits : Optional [List [str ]] = None ):
152155 """We handle string, list and dicts in datafiles"""
153156 if not self .config .data_files :
154157 raise ValueError (f"At least one data file must be specified, but got data_files={ self .config .data_files } " )
155158 dl_manager .download_config .extract_on_the_fly = True
156- data_files = dl_manager .download_and_extract (self .config .data_files )
159+ data_files = self .config .data_files
160+ if splits and isinstance (data_files , dict ):
161+ data_files = {split : data_files [split ] for split in splits }
162+ data_files = dl_manager .download_and_extract (data_files )
157163 splits = []
158164 for split_name , files in data_files .items ():
159165 if isinstance (files , str ):
You can’t perform that action at this time.
0 commit comments