Don't save original_shard_lengths by default for backward compat (#7906)

don't save original_shard_lengths by default for backward compat

Author: lhoestq

src/datasets/builder.py

@@ -1514,7 +1514,7 @@ def _rename_shard(shard_and_job: tuple[int]):
                 fpath.replace(SUFFIX, ""),
             )
 
-        if total_original_shards > 1:
+        if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS:
             split_generator.split_info.original_shard_lengths = [
                 original_shard_length
                 for original_shard_lengths in original_shard_lengths_per_job
@@ -1792,7 +1792,7 @@ def _rename_shard(shard_id_and_job: tuple[int]):
                 fpath.replace(SUFFIX, ""),
             )
 
-        if total_original_shards > 1:
+        if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS:
             split_generator.split_info.original_shard_lengths = [
                 original_shard_length
                 for original_shard_lengths in original_shard_lengths_per_job

src/datasets/config.py

@@ -167,6 +167,9 @@
 DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
 EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH))
 
+# Cached dataset info options
+SAVE_ORIGINAL_SHARD_LENGTHS = False
+
 # Download count for the website
 HF_UPDATE_DOWNLOAD_COUNTS = (
     os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES

tests/conftest.py

@@ -27,6 +27,9 @@ def set_test_cache_config(tmp_path_factory, monkeypatch):
     test_extracted_datasets_path = test_hf_datasets_cache / "downloads" / "extracted"
     monkeypatch.setattr("datasets.config.EXTRACTED_DATASETS_PATH", str(test_extracted_datasets_path))
 
+    # used in dataset viewer, we may set it to true by default in the future
+    monkeypatch.setattr("datasets.config.SAVE_ORIGINAL_SHARD_LENGTHS", True)
+
 
 @pytest.fixture(autouse=True)
 def disable_implicit_token(monkeypatch):