Skip to content

Commit 19b7ea3

Browse files
committed
Improve memory clients creation
1 parent eab68bc commit 19b7ea3

File tree

6 files changed

+24
-61
lines changed

6 files changed

+24
-61
lines changed

src/crawlee/storage_clients/_file_system/_dataset_client.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ async def open(
113113
ValueError: If a dataset with the specified ID is not found, if metadata is invalid,
114114
or if both name and alias are provided.
115115
"""
116-
# Validate parameters
116+
# Validate input parameters.
117117
specified_params = sum(1 for param in [id, name, alias] if param is not None)
118118
if specified_params > 1:
119119
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
@@ -174,9 +174,6 @@ async def open(
174174
except ValidationError as exc:
175175
raise ValueError(f'Invalid metadata file for dataset "{name or alias}"') from exc
176176

177-
# Update metadata name to match the resolution.
178-
metadata.name = name
179-
180177
client = cls(
181178
metadata=metadata,
182179
path_to_dataset=path_to_dataset,

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,10 @@ async def open(
112112
ValueError: If a store with the specified ID is not found, if metadata is invalid,
113113
or if both name and alias are provided.
114114
"""
115-
# Validate parameters - exactly one of name or alias should be provided (or neither for default)
116-
if name is not None and alias is not None:
117-
raise ValueError('Cannot specify both name and alias parameters')
115+
# Validate input parameters.
116+
specified_params = sum(1 for param in [id, name, alias] if param is not None)
117+
if specified_params > 1:
118+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
118119

119120
kvs_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
120121

@@ -172,9 +173,6 @@ async def open(
172173
except ValidationError as exc:
173174
raise ValueError(f'Invalid metadata file for key-value store "{name or alias}"') from exc
174175

175-
# Update metadata name to match the resolution.
176-
metadata.name = name
177-
178176
client = cls(
179177
metadata=metadata,
180178
path_to_kvs=path_to_kvs,

src/crawlee/storage_clients/_file_system/_request_queue_client.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,10 @@ async def open(
164164
ValueError: If a queue with the specified ID is not found, if metadata is invalid,
165165
or if both name and alias are provided.
166166
"""
167-
# Validate parameters - exactly one of name or alias should be provided (or neither for default)
168-
if name is not None and alias is not None:
169-
raise ValueError('Cannot specify both name and alias parameters')
167+
# Validate input parameters.
168+
specified_params = sum(1 for param in [id, name, alias] if param is not None)
169+
if specified_params > 1:
170+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
170171

171172
rq_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
172173

@@ -227,9 +228,6 @@ async def open(
227228
except ValidationError as exc:
228229
raise ValueError(f'Invalid metadata file for request queue "{name or alias}"') from exc
229230

230-
# Update metadata name to match the resolution.
231-
metadata.name = name
232-
233231
client = cls(
234232
metadata=metadata,
235233
path_to_rq=path_to_rq,

src/crawlee/storage_clients/_memory/_dataset_client.py

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -72,28 +72,18 @@ async def open(
7272
Raises:
7373
ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided.
7474
"""
75-
# Validate parameters - exactly one of name or alias should be provided (or neither for default)
76-
if name is not None and alias is not None:
77-
raise ValueError('Cannot specify both name and alias parameters')
78-
79-
# Determine the actual name to use in metadata
80-
if alias is not None:
81-
# For alias storages, metadata.name should be None (unnamed storage)
82-
actual_name = None
83-
elif name is not None:
84-
# For named storages, use the provided name
85-
actual_name = name
86-
else:
87-
# For default storage (no name or alias), use None
88-
actual_name = None
75+
# Validate input parameters.
76+
specified_params = sum(1 for param in [id, name, alias] if param is not None)
77+
if specified_params > 1:
78+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
8979

9080
# Create a new dataset
9181
dataset_id = id or crypto_random_object_id()
9282
now = datetime.now(timezone.utc)
9383

9484
metadata = DatasetMetadata(
9585
id=dataset_id,
96-
name=actual_name,
86+
name=name,
9787
created_at=now,
9888
accessed_at=now,
9989
modified_at=now,

src/crawlee/storage_clients/_memory/_key_value_store_client.py

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -70,28 +70,18 @@ async def open(
7070
Raises:
7171
ValueError: If both name and alias are provided.
7272
"""
73-
# Validate parameters - exactly one of name or alias should be provided (or neither for default)
74-
if name is not None and alias is not None:
75-
raise ValueError('Cannot specify both name and alias parameters')
76-
77-
# Determine the actual name to use in metadata
78-
if alias is not None:
79-
# For alias storages, metadata.name should be None (unnamed storage)
80-
actual_name = None
81-
elif name is not None:
82-
# For named storages, use the provided name
83-
actual_name = name
84-
else:
85-
# For default storage (no name or alias), use None
86-
actual_name = None
73+
# Validate input parameters.
74+
specified_params = sum(1 for param in [id, name, alias] if param is not None)
75+
if specified_params > 1:
76+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
8777

8878
# Create a new key-value store
8979
store_id = id or crypto_random_object_id()
9080
now = datetime.now(timezone.utc)
9181

9282
metadata = KeyValueStoreMetadata(
9383
id=store_id,
94-
name=actual_name,
84+
name=name,
9585
created_at=now,
9686
accessed_at=now,
9787
modified_at=now,

src/crawlee/storage_clients/_memory/_request_queue_client.py

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -82,28 +82,18 @@ async def open(
8282
Raises:
8383
ValueError: If both name and alias are provided.
8484
"""
85-
# Validate parameters - exactly one of name or alias should be provided (or neither for default)
86-
if name is not None and alias is not None:
87-
raise ValueError('Cannot specify both name and alias parameters')
88-
89-
# Determine the actual name to use in metadata
90-
if alias is not None:
91-
# For alias storages, metadata.name should be None (unnamed storage)
92-
actual_name = None
93-
elif name is not None:
94-
# For named storages, use the provided name
95-
actual_name = name
96-
else:
97-
# For default storage (no name or alias), use None
98-
actual_name = None
85+
# Validate input parameters.
86+
specified_params = sum(1 for param in [id, name, alias] if param is not None)
87+
if specified_params > 1:
88+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
9989

10090
# Create a new queue
10191
queue_id = id or crypto_random_object_id()
10292
now = datetime.now(timezone.utc)
10393

10494
metadata = RequestQueueMetadata(
10595
id=queue_id,
106-
name=actual_name,
96+
name=name,
10797
created_at=now,
10898
accessed_at=now,
10999
modified_at=now,

0 commit comments

Comments
 (0)