feat: Add SqlStorageClient based on sqlalchemy v2+

pyproject.toml

@@ -50,7 +50,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel]"]
+all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql]"]
 adaptive-crawler = [
     "jaro-winkler>=2.0.3",
     "playwright>=1.27.0",
@@ -73,6 +73,10 @@ otel = [
     "opentelemetry-semantic-conventions>=0.54",
     "wrapt>=1.17.0",
 ]
+sql = [
+    "sqlalchemy[asyncio]>=2.0.42,<3.0.0",
+    "aiosqlite>=0.21.0",
+]
 
 [project.scripts]
 crawlee = "crawlee._cli:cli"

src/crawlee/storage_clients/__init__.py

@@ -1,9 +1,21 @@
+from crawlee._utils.try_import import install_import_hook as _install_import_hook
+from crawlee._utils.try_import import try_import as _try_import
+
+# These imports have only mandatory dependencies, so they are imported directly.
 from ._base import StorageClient
 from ._file_system import FileSystemStorageClient
 from ._memory import MemoryStorageClient
 
+_install_import_hook(__name__)
+
+# The following imports are wrapped in try_import to handle optional dependencies,
+# ensuring the module can still function even if these dependencies are missing.
+with _try_import(__name__, 'SQLStorageClient'):
+    from ._sql import SQLStorageClient
+
 __all__ = [
     'FileSystemStorageClient',
     'MemoryStorageClient',
+    'SQLStorageClient',
     'StorageClient',
 ]

src/crawlee/storage_clients/_file_system/_storage_client.py

@@ -31,11 +31,7 @@ class FileSystemStorageClient(StorageClient):
 
     @override
     async def create_dataset_client(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        configuration: Configuration | None = None,
+        self, *, id: str | None = None, name: str | None = None, configuration: Configuration | None = None
     ) -> FileSystemDatasetClient:
         configuration = configuration or Configuration.get_global_configuration()
         client = await FileSystemDatasetClient.open(id=id, name=name, configuration=configuration)

src/crawlee/storage_clients/_sql/__init__.py

@@ -0,0 +1,6 @@
+from ._dataset_client import SQLDatasetClient
+from ._key_value_store_client import SQLKeyValueStoreClient
+from ._request_queue_client import SQLRequestQueueClient
+from ._storage_client import SQLStorageClient
+
+__all__ = ['SQLDatasetClient', 'SQLKeyValueStoreClient', 'SQLRequestQueueClient', 'SQLStorageClient']