Skip to content

Commit fc9efbc

Browse files
authored
Update csv.py
1 parent fc7659f commit fc9efbc

File tree

1 file changed

+8
-2
lines changed
  • src/datasets/packaged_modules/csv

1 file changed

+8
-2
lines changed

src/datasets/packaged_modules/csv/csv.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,12 +148,18 @@ class Csv(datasets.ArrowBasedBuilder):
148148
def _info(self):
149149
return datasets.DatasetInfo(features=self.config.features)
150150

151-
def _split_generators(self, dl_manager):
151+
def _available_splits(self) -> Optional[List[str]]:
152+
return [str(split) for split in self.config.data_files] if isinstance(self.config.data_files, dict) else None
153+
154+
def _split_generators(self, dl_manager, splits: Optional[List[str]] = None):
152155
"""We handle string, list and dicts in datafiles"""
153156
if not self.config.data_files:
154157
raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
155158
dl_manager.download_config.extract_on_the_fly = True
156-
data_files = dl_manager.download_and_extract(self.config.data_files)
159+
data_files = self.config.data_files
160+
if splits and isinstance(data_files, dict):
161+
data_files = {split: data_files[split] for split in splits}
162+
data_files = dl_manager.download_and_extract(data_files)
157163
splits = []
158164
for split_name, files in data_files.items():
159165
if isinstance(files, str):

0 commit comments

Comments
 (0)