Improve FS clients creation

Author: vdusek

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -56,21 +56,17 @@ def __init__(
         self,
         *,
         metadata: DatasetMetadata,
-        storage_dir: Path,
+        path_to_dataset: Path,
         lock: asyncio.Lock,
-        directory_name: str | None = None,
     ) -> None:
         """Initialize a new instance.
 
         Preferably use the `FileSystemDatasetClient.open` class method to create a new instance.
         """
         self._metadata = metadata
 
-        self._storage_dir = storage_dir
-        """The base directory where the storage data are being persisted."""
-
-        self._directory_name = directory_name
-        """The directory name to use for this dataset. If None, uses metadata.name or default."""
+        self._path_to_dataset = path_to_dataset
+        """The full path to the dataset directory."""
 
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
@@ -82,14 +78,7 @@ async def get_metadata(self) -> DatasetMetadata:
     @property
     def path_to_dataset(self) -> Path:
         """The full path to the dataset directory."""
-        # Use the explicit directory name if provided, otherwise fall back to metadata.name or default
-        if self._directory_name is not None:
-            return self._storage_dir / self._STORAGE_SUBDIR / self._directory_name
-
-        if self._metadata.name is None:
-            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
-
-        return self._storage_dir / self._STORAGE_SUBDIR / self._metadata.name
+        return self._path_to_dataset
 
     @property
     def path_to_metadata(self) -> Path:
@@ -124,12 +113,12 @@ async def open(
             ValueError: If a dataset with the specified ID is not found, if metadata is invalid,
                 or if both name and alias are provided.
         """
-        # Validate parameters - exactly one of name or alias should be provided (or neither for default)
-        if name is not None and alias is not None:
-            raise ValueError('Cannot specify both name and alias parameters')
+        # Validate parameters
+        specified_params = sum(1 for param in [id, name, alias] if param is not None)
+        if specified_params > 1:
+            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
 
-        storage_dir = Path(configuration.storage_dir)
-        dataset_base_path = storage_dir / cls._STORAGE_SUBDIR
+        dataset_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
         if not dataset_base_path.exists():
             await asyncio.to_thread(dataset_base_path.mkdir, parents=True, exist_ok=True)
@@ -141,21 +130,20 @@ async def open(
                 if not dataset_dir.is_dir():
                     continue
 
-                metadata_path = dataset_dir / METADATA_FILENAME
-                if not metadata_path.exists():
+                path_to_metadata = dataset_dir / METADATA_FILENAME
+                if not path_to_metadata.exists():
                     continue
 
                 try:
-                    file = await asyncio.to_thread(metadata_path.open)
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = DatasetMetadata(**file_content)
                         if metadata.id == id:
                             client = cls(
                                 metadata=metadata,
-                                storage_dir=storage_dir,
+                                path_to_dataset=dataset_base_path / dataset_dir,
                                 lock=asyncio.Lock(),
-                                directory_name=dataset_dir.name,  # Use the actual directory name
                             )
                             await client._update_metadata(update_accessed_at=True)
                             found = True
@@ -170,48 +158,29 @@ async def open(
 
         # Get a new instance by name or alias.
         else:
-            # Determine the directory name and metadata name based on whether this is a named or alias storage
-            if alias is not None:
-                # For alias storages, use the alias as directory name and set metadata.name to None
-                # Special case: alias='default' should use the same directory as default storage
-                directory_name = None if alias == 'default' else alias
-                actual_name = None
-            elif name is not None:
-                # For named storages, use the name as both directory name and metadata.name
-                directory_name = name
-                actual_name = name
-            else:
-                # For default storage (no name or alias), use None for both - same as alias='default'
-                directory_name = None
-                actual_name = None
-
-            dataset_path = (
-                dataset_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT
-                if directory_name is None
-                else dataset_base_path / directory_name
-            )
-            metadata_path = dataset_path / METADATA_FILENAME
+            dataset_dir = Path(name) if name else Path(alias) if alias else Path('default')
+            path_to_dataset = dataset_base_path / dataset_dir
+            path_to_metadata = path_to_dataset / METADATA_FILENAME
 
             # If the dataset directory exists, reconstruct the client from the metadata file.
-            if dataset_path.exists() and metadata_path.exists():
-                file = await asyncio.to_thread(open, metadata_path)
+            if path_to_dataset.exists() and path_to_metadata.exists():
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:
                     await asyncio.to_thread(file.close)
                 try:
                     metadata = DatasetMetadata(**file_content)
-                    # For aliases, ensure the metadata.name is None
-                    if alias is not None:
-                        metadata = metadata.model_copy(update={'name': None})
                 except ValidationError as exc:
-                    raise ValueError(f'Invalid metadata file for dataset "{name}"') from exc
+                    raise ValueError(f'Invalid metadata file for dataset "{name or alias}"') from exc
+
+                # Update metadata name to match the resolution.
+                metadata.name = name
 
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_dataset=path_to_dataset,
                     lock=asyncio.Lock(),
-                    directory_name=directory_name,
                 )
 
                 await client._update_metadata(update_accessed_at=True)
@@ -221,17 +190,16 @@ async def open(
                 now = datetime.now(timezone.utc)
                 metadata = DatasetMetadata(
                     id=crypto_random_object_id(),
-                    name=actual_name,  # Use actual_name which will be None for aliases
+                    name=name,
                     created_at=now,
                     accessed_at=now,
                     modified_at=now,
                     item_count=0,
                 )
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_dataset=path_to_dataset,
                     lock=asyncio.Lock(),
-                    directory_name=directory_name,
                 )
                 await client._update_metadata()
 

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -55,21 +55,17 @@ def __init__(
         self,
         *,
         metadata: KeyValueStoreMetadata,
-        storage_dir: Path,
+        path_to_kvs: Path,
         lock: asyncio.Lock,
-        directory_name: str | None = None,
     ) -> None:
         """Initialize a new instance.
 
         Preferably use the `FileSystemKeyValueStoreClient.open` class method to create a new instance.
         """
         self._metadata = metadata
 
-        self._storage_dir = storage_dir
-        """The base directory where the storage data are being persisted."""
-
-        self._directory_name = directory_name
-        """The directory name to use for this key-value store. If None, uses metadata.name or default."""
+        self._path_to_kvs = path_to_kvs
+        """The full path to the key-value store directory."""
 
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
@@ -81,14 +77,7 @@ async def get_metadata(self) -> KeyValueStoreMetadata:
     @property
     def path_to_kvs(self) -> Path:
         """The full path to the key-value store directory."""
-        # Use the explicit directory name if provided, otherwise fall back to metadata.name or default
-        if self._directory_name is not None:
-            return self._storage_dir / self._STORAGE_SUBDIR / self._directory_name
-
-        if self._metadata.name is None:
-            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
-
-        return self._storage_dir / self._STORAGE_SUBDIR / self._metadata.name
+        return self._path_to_kvs
 
     @property
     def path_to_metadata(self) -> Path:
@@ -127,8 +116,7 @@ async def open(
         if name is not None and alias is not None:
             raise ValueError('Cannot specify both name and alias parameters')
 
-        storage_dir = Path(configuration.storage_dir)
-        kvs_base_path = storage_dir / cls._STORAGE_SUBDIR
+        kvs_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
         if not kvs_base_path.exists():
             await asyncio.to_thread(kvs_base_path.mkdir, parents=True, exist_ok=True)
@@ -140,21 +128,20 @@ async def open(
                 if not kvs_dir.is_dir():
                     continue
 
-                metadata_path = kvs_dir / METADATA_FILENAME
-                if not metadata_path.exists():
+                path_to_metadata = kvs_dir / METADATA_FILENAME
+                if not path_to_metadata.exists():
                     continue
 
                 try:
-                    file = await asyncio.to_thread(metadata_path.open)
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = KeyValueStoreMetadata(**file_content)
                         if metadata.id == id:
                             client = cls(
                                 metadata=metadata,
-                                storage_dir=storage_dir,
+                                path_to_kvs=kvs_base_path / kvs_dir,
                                 lock=asyncio.Lock(),
-                                directory_name=kvs_dir.name,  # Use the actual directory name
                             )
                             await client._update_metadata(update_accessed_at=True)
                             found = True
@@ -169,48 +156,29 @@ async def open(
 
         # Get a new instance by name or alias.
         else:
-            # Determine the directory name and metadata name based on whether this is a named or alias storage
-            if alias is not None:
-                # For alias storages, use the alias as directory name and set metadata.name to None
-                # Special case: alias='default' should use the same directory as default storage
-                directory_name = None if alias == 'default' else alias
-                actual_name = None
-            elif name is not None:
-                # For named storages, use the name as both directory name and metadata.name
-                directory_name = name
-                actual_name = name
-            else:
-                # For default storage (no name or alias), use None for both - same as alias='default'
-                directory_name = None
-                actual_name = None
-
-            kvs_path = (
-                kvs_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT
-                if directory_name is None
-                else kvs_base_path / directory_name
-            )
-            metadata_path = kvs_path / METADATA_FILENAME
+            kvs_dir = Path(name) if name else Path(alias) if alias else Path('default')
+            path_to_kvs = kvs_base_path / kvs_dir
+            path_to_metadata = path_to_kvs / METADATA_FILENAME
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.
-            if kvs_path.exists() and metadata_path.exists():
-                file = await asyncio.to_thread(open, metadata_path)
+            if path_to_kvs.exists() and path_to_metadata.exists():
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:
                     await asyncio.to_thread(file.close)
                 try:
                     metadata = KeyValueStoreMetadata(**file_content)
-                    # For aliases, ensure the metadata.name is None
-                    if alias is not None:
-                        metadata = metadata.model_copy(update={'name': None})
                 except ValidationError as exc:
-                    raise ValueError(f'Invalid metadata file for key-value store "{name}"') from exc
+                    raise ValueError(f'Invalid metadata file for key-value store "{name or alias}"') from exc
+
+                # Update metadata name to match the resolution.
+                metadata.name = name
 
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_kvs=path_to_kvs,
                     lock=asyncio.Lock(),
-                    directory_name=directory_name,
                 )
 
                 await client._update_metadata(update_accessed_at=True)
@@ -220,16 +188,15 @@ async def open(
                 now = datetime.now(timezone.utc)
                 metadata = KeyValueStoreMetadata(
                     id=crypto_random_object_id(),
-                    name=actual_name,  # Use actual_name which will be None for aliases
+                    name=name,
                     created_at=now,
                     accessed_at=now,
                     modified_at=now,
                 )
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_kvs=path_to_kvs,
                     lock=asyncio.Lock(),
-                    directory_name=directory_name,
                 )
                 await client._update_metadata()