feat(LAB-4123): use http_client instead of httpx

Author: FannyGaudin

pyproject.toml

@@ -44,8 +44,7 @@ dependencies = [
   "filelock >= 3.0.0, < 4.0.0",
   "pip-system-certs >= 4.0.0, < 5.0.0; platform_system=='Windows'",
   "pyrate-limiter >= 3, < 4",
-  "kili-formats == 1.1.0",
-  "httpx >= 0.27.0, < 1.0.0"
+  "kili-formats == 1.1.0"
 ]
 urls = { homepage = "https://github.com/kili-technology/kili-python-sdk" }
 

src/kili/adapters/kili_api_gateway/asset/formatters.py

@@ -2,9 +2,8 @@
 
 import asyncio
 import json
-import os
 
-import httpx
+import requests
 
 from kili.adapters.http_client import HttpClient
 from kili.core.helpers import is_url
@@ -24,40 +23,41 @@ def load_json_from_link(link: str, http_client: HttpClient) -> dict:
     return response.json()
 
 
-async def _download_json_response(url: str) -> dict:
+async def _download_json_response(url: str, http_client: HttpClient) -> dict:
     """Download and parse JSON response from a URL using asyncio.
 
     Args:
         url: URL to download the JSON response from
+        http_client: HttpClient instance with SSL verification already configured
 
     Returns:
         Parsed JSON response as a dictionary
     """
     try:
-        verify_env = os.getenv("KILI_VERIFY")
-        verify = verify_env.lower() in ("true", "1", "yes") if verify_env is not None else True
-
-        async with httpx.AsyncClient(verify=verify) as client:
-            response = await client.get(url, timeout=30.0)
-            response.raise_for_status()
-            return response.json()
-    except (httpx.HTTPError, json.JSONDecodeError):
+        # Run synchronous requests call in a thread
+        response = await asyncio.to_thread(http_client.get, url, timeout=30)
+        response.raise_for_status()
+        return response.json()
+    except (requests.RequestException, json.JSONDecodeError):
         # Return empty dict on error to ensure consistent response format
         return {}
 
 
-async def _download_json_responses_async(url_to_label_mapping: list[tuple[str, dict]]) -> None:
+async def _download_json_responses_async(
+    url_to_label_mapping: list[tuple[str, dict]], http_client: HttpClient
+) -> None:
     """Download JSON responses in parallel using asyncio.
 
     Args:
         url_to_label_mapping: List of tuples (url, label_dict) to download
+        http_client: HttpClient instance with SSL verification already configured
     """
     # Process in batches to limit concurrent connections
     for i in range(0, len(url_to_label_mapping), JSON_RESPONSE_BATCH_SIZE):
         batch = url_to_label_mapping[i : i + JSON_RESPONSE_BATCH_SIZE]
 
         # Download all URLs in the batch in parallel using asyncio.gather
-        download_tasks = [_download_json_response(url) for url, _ in batch]
+        download_tasks = [_download_json_response(url, http_client) for url, _ in batch]
         json_responses = await asyncio.gather(*download_tasks)
 
         # Assign the downloaded responses back to their labels and remove the URL
@@ -67,17 +67,20 @@ async def _download_json_responses_async(url_to_label_mapping: list[tuple[str, d
                 del label["jsonResponseUrl"]
 
 
-def download_json_responses_parallel(url_to_label_mapping: list[tuple[str, dict]]) -> None:
+def download_json_responses_parallel(
+    url_to_label_mapping: list[tuple[str, dict]], http_client: HttpClient
+) -> None:
     """Download JSON responses in parallel and assign to labels.
 
     Args:
         url_to_label_mapping: List of tuples (url, label_dict) to download
+        http_client: HttpClient instance with SSL verification already configured
     """
     if not url_to_label_mapping:
         return
 
     # Run async downloads in a synchronous context
-    asyncio.run(_download_json_responses_async(url_to_label_mapping))
+    asyncio.run(_download_json_responses_async(url_to_label_mapping, http_client))
 
 
 def _parse_label_json_response(label: dict) -> None:
@@ -129,6 +132,6 @@ def load_asset_json_fields(asset: dict, fields: ListOrTuple[str], http_client: H
         _process_label_json_response(asset["latestLabel"], url_to_label_mapping)
 
     if url_to_label_mapping:
-        download_json_responses_parallel(url_to_label_mapping)
+        download_json_responses_parallel(url_to_label_mapping, http_client)
 
     return asset

tests/unit/adapters/kili_api_gateway/asset/test_formatters.py

@@ -56,6 +56,7 @@ class TestDownloadJsonResponsesParallel:
 
     def test_download_single_json_response(self):
         """Test downloading a single JSON response."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -65,7 +66,7 @@ def test_download_single_json_response(self):
         ) as mock_download:
             mock_download.return_value = {"annotation": "data"}
 
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify the JSON was downloaded and assigned
             assert label["jsonResponse"] == {"annotation": "data"}
@@ -75,6 +76,7 @@ def test_download_single_json_response(self):
 
     def test_download_multiple_json_responses(self):
         """Test downloading multiple JSON responses in parallel."""
+        http_client = Mock()
         label1 = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         label2 = {"id": "label2", "jsonResponseUrl": "https://example.com/label2.json"}
         label3 = {"id": "label3", "jsonResponseUrl": "https://example.com/label3.json"}
@@ -86,7 +88,7 @@ def test_download_multiple_json_responses(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url:
@@ -99,7 +101,7 @@ async def mock_download(url):
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             side_effect=mock_download,
         ):
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify all labels got their JSON
             assert label1["jsonResponse"] == {"data": "label1"}
@@ -113,25 +115,27 @@ async def mock_download(url):
 
     def test_download_handles_request_exception(self):
         """Test that request exceptions are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
-        # Mock download to raise httpx.HTTPError
+        # Mock download to return empty dict on error
         with patch(
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             new_callable=AsyncMock,
         ) as mock_download:
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_handles_json_decode_error(self):
         """Test that JSON decode errors are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -143,14 +147,15 @@ def test_download_handles_json_decode_error(self):
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_handles_timeout_error(self):
         """Test that timeout errors are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -162,14 +167,15 @@ def test_download_handles_timeout_error(self):
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_processes_in_batches(self):
         """Test that downloads are processed in batches of JSON_RESPONSE_BATCH_SIZE."""
+        http_client = Mock()
         # Create more labels than batch size
         num_labels = JSON_RESPONSE_BATCH_SIZE + 5
         labels = [
@@ -188,7 +194,7 @@ def test_download_processes_in_batches(self):
             mock_download.return_value = {"data": "test"}
 
             # Call the function
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify all labels got their JSON response
             for label in labels:
@@ -200,6 +206,7 @@ def test_download_processes_in_batches(self):
 
     def test_download_mixed_success_and_failure(self):
         """Test downloading with some successes and some failures."""
+        http_client = Mock()
         label_success = {"id": "success", "jsonResponseUrl": "https://example.com/success.json"}
         label_fail = {"id": "fail", "jsonResponseUrl": "https://example.com/fail.json"}
 
@@ -209,7 +216,7 @@ def test_download_mixed_success_and_failure(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "success" in url:
                 return {"status": "ok"}
             return {}  # Failure returns empty dict
@@ -218,7 +225,7 @@ async def mock_download(url):
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             side_effect=mock_download,
         ):
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Success should have data
             assert label_success["jsonResponse"] == {"status": "ok"}
@@ -283,7 +290,7 @@ def test_load_asset_with_labels_json_response_url(self):
         fields = ["id", "labels.jsonResponse", "labels.jsonResponseUrl"]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url:
@@ -345,7 +352,7 @@ def test_load_asset_with_both_labels_and_latest_label(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url: