Added datasets related to issue #6832

Author: ArjunJagdale

src/datasets/arrow_reader.py

@@ -325,6 +325,7 @@ def read_table(filename, in_memory=False) -> Table:
         Returns:
             pyarrow.Table
         """
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
         table_cls = InMemoryTable if in_memory else MemoryMappedTable
         return table_cls.from_file(filename)
 

src/datasets/builder.py

@@ -19,7 +19,9 @@
 import contextlib
 import copy
 import fnmatch
+import hashlib
 import inspect
+import json
 import os
 import posixpath
 import shutil
@@ -89,14 +91,26 @@
 from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs
 from .utils.track import tracked_list
 
-
 if TYPE_CHECKING:
     from .load import DatasetModule
 
 
 logger = logging.get_logger(__name__)
 
 
+def hash_dict(d):
+    """Hash a dictionary into a short hex string (8 characters)."""
+    def sanitize(obj):
+        if isinstance(obj, dict):
+            return {str(k): sanitize(v) for k, v in obj.items()}
+        elif isinstance(obj, (list, tuple)):
+            return [sanitize(i) for i in obj]
+        else:
+            return str(obj)
+    normalized = json.dumps(sanitize(d), sort_keys=True)
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8]
+
+    
 class InvalidConfigName(ValueError):
     pass
 
@@ -391,7 +405,7 @@ def __init__(
         if not is_remote_url(self._cache_dir_root):
             os.makedirs(self._cache_dir_root, exist_ok=True)
             lock_path = os.path.join(
-                self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock"
+                self._cache_dir_root, Path(self._relative_data_dir()).as_posix().replace("/", "_") + ".lock"
             )
             with FileLock(lock_path):
                 if os.path.exists(self._cache_dir):  # check if data exist
@@ -577,11 +591,27 @@ def _create_builder_config(
             download_config=DownloadConfig(token=self.token, storage_options=self.storage_options),
         )
 
-        # compute the config id that is going to be used for caching
+        runtime_only_config_keys = {"drop_metadata", "drop_labels", "drop_audio", "drop_text", "drop_images"}
+        hashable_config_kwargs = {k: v for k, v in config_kwargs.items() if k not in runtime_only_config_keys}
+
         config_id = builder_config.create_config_id(
-            config_kwargs,
+            hashable_config_kwargs,
             custom_features=custom_features,
         )
+
+        if (
+            builder_config.name in self.builder_configs
+            and builder_config != self.builder_configs[builder_config.name]
+        ):
+            builder_config.name = f"custom-{hash_dict(hashable_config_kwargs)}"
+            while builder_config.name in self.builder_configs:
+                builder_config.name += "-x"
+            config_id = builder_config.create_config_id(
+                hashable_config_kwargs,
+                custom_features=custom_features,
+            )
+            logger.info(f"Renamed conflicting config to: {builder_config.name}")
+
         is_custom = (config_id not in self.builder_configs) and config_id != "default"
         if is_custom:
             logger.info(f"Using custom data configuration {config_id}")
@@ -1659,15 +1689,19 @@ def _prepare_split_single(
         shard_id = 0
         num_examples_progress_update = 0
         try:
+            path = fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}")
+            logger.debug("Creating directory: %s", os.path.dirname(path))
+            os.makedirs(os.path.dirname(path), exist_ok=True)
             writer = writer_class(
                 features=self.info.features,
-                path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+                path=path,
                 writer_batch_size=self._writer_batch_size,
                 hash_salt=split_info.name,
                 check_duplicates=check_duplicate_keys,
                 storage_options=self._fs.storage_options,
                 embed_local_files=embed_local_files,
             )
+            
             try:
                 _time = time.time()
                 for key, record in generator:
@@ -1678,9 +1712,12 @@ def _prepare_split_single(
                         total_num_examples += num_examples
                         total_num_bytes += num_bytes
                         shard_id += 1
+                        path = fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}")
+                        logger.debug("Creating directory: %s", os.path.dirname(path))
+                        os.makedirs(os.path.dirname(path), exist_ok=True)
                         writer = writer_class(
                             features=writer._features,
-                            path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+                            path=path,
                             writer_batch_size=self._writer_batch_size,
                             hash_salt=split_info.name,
                             check_duplicates=check_duplicate_keys,
@@ -1908,9 +1945,12 @@ def _prepare_split_single(
         shard_id = 0
         num_examples_progress_update = 0
         try:
+            path = fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}")
+            logger.debug("Creating directory: %s", os.path.dirname(path))
+            os.makedirs(os.path.dirname(path), exist_ok=True)
             writer = writer_class(
                 features=self.info.features,
-                path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+                path=path,
                 writer_batch_size=self._writer_batch_size,
                 storage_options=self._fs.storage_options,
                 embed_local_files=embed_local_files,
@@ -1925,9 +1965,12 @@ def _prepare_split_single(
                         total_num_examples += num_examples
                         total_num_bytes += num_bytes
                         shard_id += 1
+                        path = fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}")
+                        logger.debug("Creating directory: %s", os.path.dirname(path))
+                        os.makedirs(os.path.dirname(path), exist_ok=True)
                         writer = writer_class(
                             features=writer._features,
-                            path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"),
+                            path=path,
                             writer_batch_size=self._writer_batch_size,
                             storage_options=self._fs.storage_options,
                             embed_local_files=embed_local_files,

src/datasets/load.py

@@ -1161,8 +1161,11 @@ def load_dataset_builder(
             error_msg += f'\nFor example `data_files={{"train": "path/to/data/train/*.{example_extensions[0]}"}}`'
         raise ValueError(error_msg)
 
+    runtime_only_config_keys = {"drop_metadata", "drop_labels", "drop_audio", "drop_text", "drop_images"}
+    hashable_config_kwargs = {k: v for k, v in config_kwargs.items() if k not in runtime_only_config_keys}
+    full_config_kwargs = config_kwargs.copy()
+    config_kwargs_for_config = hashable_config_kwargs.copy()
     builder_cls = get_dataset_builder_class(dataset_module, dataset_name=dataset_name)
-    # Instantiate the dataset builder
     builder_instance: DatasetBuilder = builder_cls(
         cache_dir=cache_dir,
         dataset_name=dataset_name,
@@ -1175,7 +1178,7 @@ def load_dataset_builder(
         token=token,
         storage_options=storage_options,
         **builder_kwargs,
-        **config_kwargs,
+        **full_config_kwargs,
     )
     builder_instance._use_legacy_cache_dir_if_possible(dataset_module)
 

src/datasets/packaged_modules/cache/cache.py

@@ -4,7 +4,7 @@
 import shutil
 import time
 from pathlib import Path
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 import pyarrow as pa
 

src/datasets/packaged_modules/csv/csv.py

@@ -1,6 +1,6 @@
 import itertools
 from dataclasses import dataclass
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, List, Optional, Union
 
 import pandas as pd
 import pyarrow as pa

src/datasets/packaged_modules/folder_based_builder/folder_based_builder.py

@@ -3,7 +3,7 @@
 import itertools
 import os
 from dataclasses import dataclass
-from typing import Any, Callable, Iterator, Optional, Union
+from typing import Any, Callable, Iterator, List, Optional, Union
 
 import pandas as pd
 import pyarrow as pa
@@ -71,6 +71,7 @@ def _available_splits(self) -> Optional[List[str]]:
         return [str(split) for split in self.config.data_files] if isinstance(self.config.data_files, dict) else None
 
     def _split_generators(self, dl_manager, splits: Optional[List[str]] = None):
+        data_files = self.config.data_files
         if not self.config.data_files:
             raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
         dl_manager.download_config.extract_on_the_fly = True
@@ -248,7 +249,14 @@ def _set_feature(feature):
         # before building the features
         if self.config.features is None:
             if add_metadata:
-                self.info.features = metadata_features
+                if self.config.drop_metadata and isinstance(metadata_features, dict):
+                    filtered = {
+                        k: v for k, v in metadata_features.items()
+                        if k == self.BASE_COLUMN_NAME  # e.g. "image"
+                    }
+                    self.info.features = datasets.Features(filtered)
+                else:
+                    self.info.features = metadata_features
             elif add_labels:
                 self.info.features = datasets.Features(
                     {

src/datasets/packaged_modules/parquet/parquet.py

@@ -1,6 +1,6 @@
 import itertools
 from dataclasses import dataclass
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 import pyarrow as pa
 import pyarrow.dataset as ds

tests/test_load.py

@@ -49,6 +49,7 @@
 )
 
 
+SAMPLE_DATASET_IDENTIFIER = "hf-internal-testing/librispeech_asr_dummy"
 SAMPLE_DATASET_IDENTIFIER2 = "hf-internal-testing/dataset_with_data_files"  # only has data files
 SAMPLE_DATASET_IDENTIFIER3 = "hf-internal-testing/multi_dir_dataset"  # has multiple data directories
 SAMPLE_DATASET_IDENTIFIER4 = "hf-internal-testing/imagefolder_with_metadata"  # imagefolder with a metadata file inside the train/test directories
@@ -1093,8 +1094,8 @@ def test_load_dataset_specific_splits_then_full(data_dir):
 @pytest.mark.integration
 def test_loading_from_the_datasets_hub():
     with tempfile.TemporaryDirectory() as tmp_dir:
-@@ -1449,6 +1491,28 @@ def test_loading_from_the_datasets_hub():
-            assert len(dataset["validation"]) == 3
+        dataset = load_dataset(SAMPLE_DATASET_IDENTIFIER, cache_dir=tmp_dir)
+        assert len(dataset["validation"]) >= 3
 
 
 @pytest.mark.integration