chore: replace os.path with pathlib.Path in tests (#1101)

### Description

- Replace `os.path` with `pathlib.Path` in tests. To improve
readability, and a unified style of working with file paths

Author: Mantisus

src/crawlee/_utils/file.py

@@ -9,7 +9,10 @@
 import re
 import shutil
 from enum import Enum
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pathlib import Path
 
 
 class ContentType(Enum):
@@ -27,7 +30,7 @@ def is_content_type(content_type_enum: ContentType, content_type: str) -> bool:
     return content_type_enum.matches(content_type)
 
 
-async def force_remove(filename: str) -> None:
+async def force_remove(filename: str | Path) -> None:
     """Removes a file, suppressing the FileNotFoundError if it does not exist.
 
     JS-like rm(filename, { force: true }).
@@ -39,7 +42,7 @@ async def force_remove(filename: str) -> None:
         await asyncio.to_thread(os.remove, filename)
 
 
-async def force_rename(src_dir: str, dst_dir: str) -> None:
+async def force_rename(src_dir: str | Path, dst_dir: str | Path) -> None:
     """Renames a directory, ensuring that the destination directory is removed if it exists.
 
     Args:

tests/unit/_utils/test_file.py

@@ -1,9 +1,8 @@
 from __future__ import annotations
 
 import io
-import os
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING
+from pathlib import Path
 
 import pytest
 
@@ -17,9 +16,6 @@
     json_dumps,
 )
 
-if TYPE_CHECKING:
-    from pathlib import Path
-
 
 async def test_json_dumps() -> None:
     assert await json_dumps({'key': 'value'}) == '{\n  "key": "value"\n}'
@@ -112,42 +108,42 @@ def test_determine_file_extension() -> None:
 
 
 async def test_force_remove(tmp_path: Path) -> None:
-    test_file_path = os.path.join(tmp_path, 'test.txt')
+    test_file_path = Path(tmp_path, 'test.txt')
     # Does not crash/raise when the file does not exist
-    assert os.path.exists(test_file_path) is False
+    assert test_file_path.exists() is False
     await force_remove(test_file_path)
-    assert os.path.exists(test_file_path) is False
+    assert test_file_path.exists() is False
 
     # Removes the file if it exists
     with open(test_file_path, 'a', encoding='utf-8'):  # noqa: ASYNC230
         pass
-    assert os.path.exists(test_file_path) is True
+    assert test_file_path.exists() is True
     await force_remove(test_file_path)
-    assert os.path.exists(test_file_path) is False
+    assert test_file_path.exists() is False
 
 
 async def test_force_rename(tmp_path: Path) -> None:
-    src_dir = os.path.join(tmp_path, 'src')
-    dst_dir = os.path.join(tmp_path, 'dst')
-    src_file = os.path.join(src_dir, 'src_dir.txt')
-    dst_file = os.path.join(dst_dir, 'dst_dir.txt')
+    src_dir = Path(tmp_path, 'src')
+    dst_dir = Path(tmp_path, 'dst')
+    src_file = Path(src_dir, 'src_dir.txt')
+    dst_file = Path(dst_dir, 'dst_dir.txt')
     # Won't crash if source directory does not exist
-    assert os.path.exists(src_dir) is False
+    assert src_dir.exists() is False
     await force_rename(src_dir, dst_dir)
 
     # Will remove dst_dir if it exists (also covers normal case)
     # Create the src_dir with a file in it
-    os.mkdir(src_dir)
+    src_dir.mkdir()
     with open(src_file, 'a', encoding='utf-8'):  # noqa: ASYNC230
         pass
     # Create the dst_dir with a file in it
-    os.mkdir(dst_dir)
+    dst_dir.mkdir()
     with open(dst_file, 'a', encoding='utf-8'):  # noqa: ASYNC230
         pass
-    assert os.path.exists(src_file) is True
-    assert os.path.exists(dst_file) is True
+    assert src_file.exists() is True
+    assert dst_file.exists() is True
     await force_rename(src_dir, dst_dir)
-    assert os.path.exists(src_dir) is False
-    assert os.path.exists(dst_file) is False
+    assert src_dir.exists() is False
+    assert dst_file.exists() is False
     # src_dir.txt should exist in dst_dir
-    assert os.path.exists(os.path.join(dst_dir, 'src_dir.txt')) is True
+    assert (dst_dir / 'src_dir.txt').exists() is True

tests/unit/storage_clients/_memory/test_dataset_client.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-import os
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
@@ -53,17 +53,15 @@ async def test_update(dataset_client: DatasetClient) -> None:
 
     old_dataset_info = await dataset_client.get()
     assert old_dataset_info is not None
-    old_dataset_directory = os.path.join(
-        dataset_client._memory_storage_client.datasets_directory, old_dataset_info.name or ''
-    )
-    new_dataset_directory = os.path.join(dataset_client._memory_storage_client.datasets_directory, new_dataset_name)
-    assert os.path.exists(os.path.join(old_dataset_directory, '000000001.json')) is True
-    assert os.path.exists(os.path.join(new_dataset_directory, '000000001.json')) is False
+    old_dataset_directory = Path(dataset_client._memory_storage_client.datasets_directory, old_dataset_info.name or '')
+    new_dataset_directory = Path(dataset_client._memory_storage_client.datasets_directory, new_dataset_name)
+    assert (old_dataset_directory / '000000001.json').exists() is True
+    assert (new_dataset_directory / '000000001.json').exists() is False
 
     await asyncio.sleep(0.1)
     updated_dataset_info = await dataset_client.update(name=new_dataset_name)
-    assert os.path.exists(os.path.join(old_dataset_directory, '000000001.json')) is False
-    assert os.path.exists(os.path.join(new_dataset_directory, '000000001.json')) is True
+    assert (old_dataset_directory / '000000001.json').exists() is False
+    assert (new_dataset_directory / '000000001.json').exists() is True
     # Only modified_at and accessed_at should be different
     assert old_dataset_info.created_at == updated_dataset_info.created_at
     assert old_dataset_info.modified_at != updated_dataset_info.modified_at
@@ -78,10 +76,10 @@ async def test_delete(dataset_client: DatasetClient) -> None:
     await dataset_client.push_items({'abc': 123})
     dataset_info = await dataset_client.get()
     assert dataset_info is not None
-    dataset_directory = os.path.join(dataset_client._memory_storage_client.datasets_directory, dataset_info.name or '')
-    assert os.path.exists(os.path.join(dataset_directory, '000000001.json')) is True
+    dataset_directory = Path(dataset_client._memory_storage_client.datasets_directory, dataset_info.name or '')
+    assert (dataset_directory / '000000001.json').exists() is True
     await dataset_client.delete()
-    assert os.path.exists(os.path.join(dataset_directory, '000000001.json')) is False
+    assert (dataset_directory / '000000001.json').exists() is False
     # Does not crash when called again
     await dataset_client.delete()
 

tests/unit/storage_clients/_memory/test_key_value_store_client.py

@@ -3,8 +3,8 @@
 import asyncio
 import base64
 import json
-import os
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 import pytest
@@ -16,8 +16,6 @@
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecordMetadata
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
     from crawlee.storage_clients import MemoryStorageClient
     from crawlee.storage_clients._memory import KeyValueStoreClient
 
@@ -79,19 +77,17 @@ async def test_update(key_value_store_client: KeyValueStoreClient) -> None:
     await key_value_store_client.set_record('test', {'abc': 123})
     old_kvs_info = await key_value_store_client.get()
     assert old_kvs_info is not None
-    old_kvs_directory = os.path.join(
+    old_kvs_directory = Path(
         key_value_store_client._memory_storage_client.key_value_stores_directory, old_kvs_info.name or ''
     )
-    new_kvs_directory = os.path.join(
-        key_value_store_client._memory_storage_client.key_value_stores_directory, new_kvs_name
-    )
-    assert os.path.exists(os.path.join(old_kvs_directory, 'test.json')) is True
-    assert os.path.exists(os.path.join(new_kvs_directory, 'test.json')) is False
+    new_kvs_directory = Path(key_value_store_client._memory_storage_client.key_value_stores_directory, new_kvs_name)
+    assert (old_kvs_directory / 'test.json').exists() is True
+    assert (new_kvs_directory / 'test.json').exists() is False
 
     await asyncio.sleep(0.1)
     updated_kvs_info = await key_value_store_client.update(name=new_kvs_name)
-    assert os.path.exists(os.path.join(old_kvs_directory, 'test.json')) is False
-    assert os.path.exists(os.path.join(new_kvs_directory, 'test.json')) is True
+    assert (old_kvs_directory / 'test.json').exists() is False
+    assert (new_kvs_directory / 'test.json').exists() is True
     # Only modified_at and accessed_at should be different
     assert old_kvs_info.created_at == updated_kvs_info.created_at
     assert old_kvs_info.modified_at != updated_kvs_info.modified_at
@@ -106,12 +102,10 @@ async def test_delete(key_value_store_client: KeyValueStoreClient) -> None:
     await key_value_store_client.set_record('test', {'abc': 123})
     kvs_info = await key_value_store_client.get()
     assert kvs_info is not None
-    kvs_directory = os.path.join(
-        key_value_store_client._memory_storage_client.key_value_stores_directory, kvs_info.name or ''
-    )
-    assert os.path.exists(os.path.join(kvs_directory, 'test.json')) is True
+    kvs_directory = Path(key_value_store_client._memory_storage_client.key_value_stores_directory, kvs_info.name or '')
+    assert (kvs_directory / 'test.json').exists() is True
     await key_value_store_client.delete()
-    assert os.path.exists(os.path.join(kvs_directory, 'test.json')) is False
+    assert (kvs_directory / 'test.json').exists() is False
     # Does not crash when called again
     await key_value_store_client.delete()
 
@@ -188,7 +182,7 @@ async def test_get_and_set_record(tmp_path: Path, key_value_store_client: KeyVal
     assert bytes_record_info.value.decode('utf-8') == bytes_value.decode('utf-8')
 
     # Test using file descriptor
-    with open(os.path.join(tmp_path, 'test.json'), 'w+', encoding='utf-8') as f:  # noqa: ASYNC230
+    with open(tmp_path / 'test.json', 'w+', encoding='utf-8') as f:  # noqa: ASYNC230
         f.write('Test')
         with pytest.raises(NotImplementedError, match='File-like values are not supported in local memory storage'):
             await key_value_store_client.set_record('file', f)
@@ -283,12 +277,12 @@ async def test_writes_correct_metadata(
     )
 
     # Check that everything was written correctly, both the data and metadata
-    storage_path = os.path.join(memory_storage_client.key_value_stores_directory, key_value_store_name)
-    item_path = os.path.join(storage_path, expected_output['filename'])
-    item_metadata_path = os.path.join(storage_path, f'{expected_output["filename"]}.__metadata__.json')
+    storage_path = Path(memory_storage_client.key_value_stores_directory, key_value_store_name)
+    item_path = Path(storage_path, expected_output['filename'])
+    item_metadata_path = storage_path / f'{expected_output["filename"]}.__metadata__.json'
 
-    assert os.path.exists(item_path)
-    assert os.path.exists(item_metadata_path)
+    assert item_path.exists()
+    assert item_metadata_path.exists()
 
     # Test the actual value of the item
     with open(item_path, 'rb') as item_file:  # noqa: ASYNC230
@@ -396,8 +390,8 @@ async def test_reads_correct_metadata(
     key_value_store_name = crypto_random_object_id()
 
     # Ensure the directory for the store exists
-    storage_path = os.path.join(memory_storage_client.key_value_stores_directory, key_value_store_name)
-    os.makedirs(storage_path, exist_ok=True)
+    storage_path = Path(memory_storage_client.key_value_stores_directory, key_value_store_name)
+    storage_path.mkdir(exist_ok=True, parents=True)
 
     store_metadata = KeyValueStoreMetadata(
         id=crypto_random_object_id(),
@@ -409,12 +403,12 @@ async def test_reads_correct_metadata(
     )
 
     # Write the store metadata to disk
-    storage_metadata_path = os.path.join(storage_path, METADATA_FILENAME)
+    storage_metadata_path = storage_path / METADATA_FILENAME
     with open(storage_metadata_path, mode='wb') as f:  # noqa: ASYNC230
         f.write(store_metadata.model_dump_json().encode('utf-8'))
 
     # Write the test input item to the disk
-    item_path = os.path.join(storage_path, input_data['filename'])
+    item_path = storage_path / input_data['filename']
     with open(item_path, 'wb') as item_file:  # noqa: ASYNC230
         if isinstance(input_data['value'], bytes):
             item_file.write(input_data['value'])
@@ -426,7 +420,7 @@ async def test_reads_correct_metadata(
 
     # Optionally write the metadata to disk if there is some
     if input_data['metadata'] is not None:
-        storage_metadata_path = os.path.join(storage_path, input_data['filename'] + '.__metadata__.json')
+        storage_metadata_path = storage_path / f'{input_data["filename"]}.__metadata__.json'
         with open(storage_metadata_path, 'w', encoding='utf-8') as metadata_file:  # noqa: ASYNC230
             s = await json_dumps(
                 {