Skip to content

Commit 778d4e1

Browse files
authored
Fix outdated verification_mode values (#5607)
* fix outdated verification mode values: 'none' -> 'no_checks', 'all' -> 'all_checks' * change string repr of verification_mode to enum to avoid bugs if we change values again * fix: delete dir when there are no files
1 parent f1e7442 commit 778d4e1

File tree

1 file changed

+9
-8
lines changed

1 file changed

+9
-8
lines changed

src/datasets/builder.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -359,11 +359,12 @@ def __init__(
359359
with FileLock(lock_path):
360360
if os.path.exists(self._cache_dir): # check if data exist
361361
if len(os.listdir(self._cache_dir)) > 0:
362-
logger.info("Overwrite dataset info from restored data version.")
363-
self.info = DatasetInfo.from_directory(self._cache_dir)
362+
if os.path.exists(path_join(self._cache_dir, config.DATASET_INFO_FILENAME)):
363+
logger.info("Overwrite dataset info from restored data version if exists.")
364+
self.info = DatasetInfo.from_directory(self._cache_dir)
364365
else: # dir exists but no data, remove the empty dir as data aren't available anymore
365366
logger.warning(
366-
f"Old caching folder {self._cache_dir} for dataset {self.name} exists but not data were found. Removing it. "
367+
f"Old caching folder {self._cache_dir} for dataset {self.name} exists but no data were found. Removing it. "
367368
)
368369
os.rmdir(self._cache_dir)
369370

@@ -374,7 +375,7 @@ def __init__(
374375
# Set download manager
375376
self.dl_manager = None
376377

377-
# Record infos even if verification_mode="none"; used by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json
378+
# Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value.
378379
self._record_infos = False
379380

380381
# Enable streaming (e.g. it patches "open" to work with remote files)
@@ -718,10 +719,10 @@ def download_and_prepare(
718719
```
719720
"""
720721
if ignore_verifications != "deprecated":
721-
verification_mode = "none" if ignore_verifications else "full"
722+
verification_mode = VerificationMode.NO_CHECKS if ignore_verifications else VerificationMode.ALL_CHECKS
722723
warnings.warn(
723724
"'ignore_verifications' was deprecated in favor of 'verification_mode' in version 2.9.1 and will be removed in 3.0.0.\n"
724-
f"You can remove this warning by passing 'verification_mode={verification_mode}' instead.",
725+
f"You can remove this warning by passing 'verification_mode={verification_mode.value}' instead.",
725726
FutureWarning,
726727
)
727728
if use_auth_token != "deprecated":
@@ -1078,10 +1079,10 @@ def as_dataset(
10781079
```
10791080
"""
10801081
if ignore_verifications != "deprecated":
1081-
verification_mode = "none" if ignore_verifications else "full"
1082+
verification_mode = verification_mode.NO_CHECKS if ignore_verifications else VerificationMode.ALL_CHECKS
10821083
warnings.warn(
10831084
"'ignore_verifications' was deprecated in favor of 'verification' in version 2.9.1 and will be removed in 3.0.0.\n"
1084-
f"You can remove this warning by passing 'verification_mode={verification_mode}' instead.",
1085+
f"You can remove this warning by passing 'verification_mode={verification_mode.value}' instead.",
10851086
FutureWarning,
10861087
)
10871088
is_local = not is_remote_filesystem(self._fs)

0 commit comments

Comments
 (0)