feat(storage): mypy storage (#1221)

Author: o-santi

src/storage/Makefile

@@ -1,24 +1,47 @@
+help::
+	@echo "Available commands"
+	@echo "  help           -- (default) print this message"
+
 start-infra:
 	supabase --workdir infra start -x studio,gotrue,postgrest,mailpit,realtime,edge-runtime,logflare,vector,supavisor
+help::
+	@echo "  start-infra    -- start containers for tests"
 
 stop-infra:
 	supabase --workdir infra stop
+help::
+	@echo "  stop-infra     -- stop containers for tests"
+
+tests: mypy pytest
+help::
+	@echo "  tests          -- run all tests for storage3"
 
-tests: pytest
+mypy:
+	uv run --package storage3 mypy src/storage3 tests
+help::
+	@echo "  mypy           -- run mypy on storage3"
 
 pytest: start-infra
 	uv run --package storage3 pytest --cov=./ --cov-report=xml --cov-report=html -vv
+help::
+	@echo "  pytest         -- run pytest on storage3"
 
 build-sync:
 	uv run --package storage3 run-unasync.py
 	sed -i '0,/SyncMock, /{s/SyncMock, //}' tests/_sync/test_bucket.py tests/_sync/test_client.py
 	sed -i 's/SyncMock/Mock/g' tests/_sync/test_bucket.py tests/_sync/test_client.py
-	sed -i 's/SyncClient/Client/g' storage3/_sync/client.py storage3/_sync/bucket.py storage3/_sync/file_api.py tests/_sync/test_bucket.py tests/_sync/test_client.py
-	sed -i 's/self\.session\.aclose/self\.session\.close/g' storage3/_sync/client.py
+	sed -i 's/SyncClient/Client/g' src/storage3/_sync/client.py src/storage3/_sync/bucket.py src/storage3/_sync/file_api.py tests/_sync/test_bucket.py tests/_sync/test_client.py
+	sed -i 's/self\.session\.aclose/self\.session\.close/g' src/storage3/_sync/client.py
+help::
+	@echo "  build-sync     -- generate _sync from _async implementation"
 
 clean:
 	rm -rf htmlcov .pytest_cache .mypy_cache .ruff_cache
 	rm -f .coverage coverage.xml
+help::
+	@echo "  clean          -- clean intermediary files"
 
 build:
 	uv build --package storage3
+help::
+	@echo "  build          -- invoke uv build on storage3 package"

src/storage/pyproject.toml

@@ -36,7 +36,10 @@ repository = "https://github.com/supabase/supabase-py"
 lints = [
   "pre-commit >=4.2.0",
   "ruff >=0.12.1",
-  "unasync >= 0.6.0"
+  "unasync >= 0.6.0",
+  "python-lsp-server (>=1.12.2,<2.0.0)",
+  "pylsp-mypy (>=0.7.0,<0.8.0)",
+  "python-lsp-ruff (>=2.2.2,<3.0.0)",
 ]
 docs = [
   "Sphinx >=7.1.2",
@@ -52,7 +55,6 @@ tests = [
 dev = [
   { include-group = "lints" },
   { include-group = "tests" },
-  { include-group = "lints" },
   { include-group = "docs" },
 ]
 
@@ -85,6 +87,10 @@ filterwarnings = [
     "ignore::DeprecationWarning", # ignore deprecation warnings globally
 ]
 
+[tool.mypy]
+follow_untyped_imports = true # for deprecation module that does not have stubs
+allow_redefinition = true
+
 [tool.uv]
 default-groups = [ "dev" ]
 

src/storage/run-unasync.py

@@ -2,7 +2,7 @@
 
 import unasync
 
-paths = Path("src/functions").glob("**/*.py")
+paths = Path("src/storage3").glob("**/*.py")
 tests = Path("tests").glob("**/*.py")
 
 rules = (unasync._DEFAULT_RULE,)

src/storage/src/storage3/_async/file_api.py

@@ -22,6 +22,7 @@
     RequestMethod,
     SignedUploadURL,
     SignedUrlResponse,
+    TransformOptions,
     UploadData,
     UploadResponse,
     URLOptions,
@@ -165,7 +166,7 @@ async def create_signed_url(
         options
             options to be passed for downloading or transforming the file.
         """
-        json = {"expiresIn": str(expires_in)}
+        json: dict[str, str | bool | TransformOptions] = {"expiresIn": str(expires_in)}
         download_query = ""
         if options.get("download"):
             json.update({"download": options["download"]})
@@ -209,7 +210,10 @@ async def create_signed_urls(
         options
             options to be passed for downloading the file.
         """
-        json = {"paths": paths, "expiresIn": str(expires_in)}
+        json: dict[str, str | bool | None | list[str]] = {
+            "paths": paths,
+            "expiresIn": str(expires_in),
+        }
         download_query = ""
         if options.get("download"):
             json.update({"download": options.get("download")})
@@ -265,9 +269,7 @@ async def get_public_url(self, path: str, options: URLOptions = {}) -> str:
 
         render_path = "render/image" if options.get("transform") else "object"
         transformation_query = (
-            urllib.parse.urlencode(options.get("transform"))
-            if options.get("transform")
-            else None
+            urllib.parse.urlencode(t) if (t := options.get("transform")) else None
         )
 
         if transformation_query:
@@ -322,7 +324,7 @@ async def copy(self, from_path: str, to_path: str) -> dict[str, str]:
         )
         return res.json()
 
-    async def remove(self, paths: list) -> list[dict[str, Any]]:
+    async def remove(self, paths: list[str]) -> list[dict[str, Any]]:
         """
         Deletes files within the same bucket
 
@@ -341,7 +343,7 @@ async def remove(self, paths: list) -> list[dict[str, Any]]:
     async def info(
         self,
         path: str,
-    ) -> list[dict[str, str]]:
+    ) -> dict[str, Any]:
         """
         Lists info for a particular file.
 
@@ -381,7 +383,7 @@ async def list(
         self,
         path: Optional[str] = None,
         options: Optional[ListBucketFilesOptions] = None,
-    ) -> list[dict[str, str]]:
+    ) -> list[dict[str, Any]]:
         """
         Lists all the files within a bucket.
 

src/storage/src/storage3/_sync/file_api.py

@@ -22,6 +22,7 @@
     RequestMethod,
     SignedUploadURL,
     SignedUrlResponse,
+    TransformOptions,
     UploadData,
     UploadResponse,
     URLOptions,
@@ -165,7 +166,7 @@ def create_signed_url(
         options
             options to be passed for downloading or transforming the file.
         """
-        json = {"expiresIn": str(expires_in)}
+        json: dict[str, str | bool | TransformOptions] = {"expiresIn": str(expires_in)}
         download_query = ""
         if options.get("download"):
             json.update({"download": options["download"]})
@@ -209,7 +210,10 @@ def create_signed_urls(
         options
             options to be passed for downloading the file.
         """
-        json = {"paths": paths, "expiresIn": str(expires_in)}
+        json: dict[str, str | bool | None | list[str]] = {
+            "paths": paths,
+            "expiresIn": str(expires_in),
+        }
         download_query = ""
         if options.get("download"):
             json.update({"download": options.get("download")})
@@ -265,9 +269,7 @@ def get_public_url(self, path: str, options: URLOptions = {}) -> str:
 
         render_path = "render/image" if options.get("transform") else "object"
         transformation_query = (
-            urllib.parse.urlencode(options.get("transform"))
-            if options.get("transform")
-            else None
+            urllib.parse.urlencode(t) if (t := options.get("transform")) else None
         )
 
         if transformation_query:
@@ -322,7 +324,7 @@ def copy(self, from_path: str, to_path: str) -> dict[str, str]:
         )
         return res.json()
 
-    def remove(self, paths: list) -> list[dict[str, Any]]:
+    def remove(self, paths: list[str]) -> list[dict[str, Any]]:
         """
         Deletes files within the same bucket
 
@@ -341,7 +343,7 @@ def remove(self, paths: list) -> list[dict[str, Any]]:
     def info(
         self,
         path: str,
-    ) -> list[dict[str, str]]:
+    ) -> dict[str, Any]:
         """
         Lists info for a particular file.
 
@@ -381,7 +383,7 @@ def list(
         self,
         path: Optional[str] = None,
         options: Optional[ListBucketFilesOptions] = None,
-    ) -> list[dict[str, str]]:
+    ) -> list[dict[str, Any]]:
         """
         Lists all the files within a bucket.
 

src/storage/src/storage3/exceptions.py

@@ -6,6 +6,7 @@
 class StorageApiErrorDict(TypedDict):
     name: str
     message: str
+    code: str
     status: int
 
 

src/storage/src/storage3/py.typed

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import AsyncGenerator, Generator
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 from unittest.mock import AsyncMock, Mock, patch
@@ -22,7 +23,7 @@
 
 
 # Global variable to track the ids from the buckets created in the tests run
-temp_test_buckets_ids = []
+temp_test_buckets_ids: list[str] = []
 
 
 @pytest.fixture
@@ -56,14 +57,10 @@ async def afinalizer():
     request.addfinalizer(AsyncFinalizerFactory(afinalizer).finalizer)
 
 
-async def bucket_factory(
-    storage: AsyncStorageClient, uuid_factory: Callable[[], str], public: bool
-) -> str:
-    """Creates a test bucket which will be used in the whole storage tests run and deleted at the end"""
-
-
 @pytest.fixture
-async def bucket(storage: AsyncStorageClient, uuid_factory: Callable[[], str]) -> str:
+async def bucket(
+    storage: AsyncStorageClient, uuid_factory: Callable[[], str]
+) -> AsyncGenerator[str]:
     """Creates a test bucket which will be used in the whole storage tests run and deleted at the end"""
     bucket_id = uuid_factory()
 
@@ -84,7 +81,7 @@ async def bucket(storage: AsyncStorageClient, uuid_factory: Callable[[], str]) -
 @pytest.fixture
 async def public_bucket(
     storage: AsyncStorageClient, uuid_factory: Callable[[], str]
-) -> str:
+) -> AsyncGenerator[str]:
     """Creates a test public bucket which will be used in the whole storage tests run and deleted at the end"""
     bucket_id = uuid_factory()
 
@@ -103,15 +100,17 @@ async def public_bucket(
 
 
 @pytest.fixture
-def storage_file_client(storage: AsyncStorageClient, bucket: str) -> AsyncBucketProxy:
+def storage_file_client(
+    storage: AsyncStorageClient, bucket: str
+) -> Generator[AsyncBucketProxy]:
     """Creates the storage file client for the whole storage tests run"""
     yield storage.from_(bucket)
 
 
 @pytest.fixture
 def storage_file_client_public(
     storage: AsyncStorageClient, public_bucket: str
-) -> AsyncBucketProxy:
+) -> Generator[AsyncBucketProxy]:
     """Creates the storage file client for the whole storage tests run"""
     yield storage.from_(public_bucket)
 
@@ -281,6 +280,7 @@ async def test_client_upload(
     image_info = next((f for f in files if f.get("name") == file.name), None)
 
     assert image == file.file_content
+    assert image_info is not None
     assert image_info.get("metadata", {}).get("mimetype") == file.mime_type
 
 
@@ -308,6 +308,7 @@ async def test_client_update(
     )
 
     assert image == two_files[1].file_content
+    assert image_info is not None
     assert image_info.get("metadata", {}).get("mimetype") == two_files[1].mime_type
 
 
@@ -339,6 +340,7 @@ async def test_client_upload_to_signed_url(
     image_info = next((f for f in files if f.get("name") == file.name), None)
 
     assert image == file.file_content
+    assert image_info is not None
     assert image_info.get("metadata", {}).get("mimetype") == file.mime_type
 
     # Test with file_options=None
@@ -586,6 +588,7 @@ async def test_client_copy(
     copied_info = next(
         (f for f in files if f.get("name") == f"copied_{file.name}"), None
     )
+    assert copied_info is not None
     assert copied_info.get("metadata", {}).get("mimetype") == file.mime_type
 
 
@@ -612,6 +615,7 @@ async def test_client_move(
     # Verify metadata was preserved
     files = await storage_file_client.list(file.bucket_folder)
     moved_info = next((f for f in files if f.get("name") == f"moved_{file.name}"), None)
+    assert moved_info is not None
     assert moved_info.get("metadata", {}).get("mimetype") == file.mime_type
 
 
@@ -628,7 +632,7 @@ async def test_client_remove(
     assert await storage_file_client.exists(file.bucket_path)
 
     # Remove file
-    await storage_file_client.remove(file.bucket_path)
+    await storage_file_client.remove([file.bucket_path])
 
     # Verify file no longer exists
     assert not await storage_file_client.exists(file.bucket_path)

src/storage/tests/_async/test_client.py

@@ -14,18 +14,7 @@ def pytest_configure(config) -> None:
     load_dotenv(dotenv_path="tests/tests.env")
 
 
-@pytest.fixture(scope="package")
-def event_loop() -> Generator[asyncio.AbstractEventLoop]:
-    """Returns an event loop for the current thread"""
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        loop = asyncio.new_event_loop()
-    yield loop
-    loop.close()
-
-
-@pytest.fixture(scope="package")
+@pytest.fixture
 def storage() -> Generator[SyncStorageClient]:
     url = os.environ.get("SUPABASE_TEST_URL")
     assert url is not None, "Must provide SUPABASE_TEST_URL environment variable"