feat(LAB-4123): use http_client instead of httpx

Author: FannyGaudin

pyproject.toml

@@ -44,8 +44,7 @@ dependencies = [
   "filelock >= 3.0.0, < 4.0.0",
   "pip-system-certs >= 4.0.0, < 5.0.0; platform_system=='Windows'",
   "pyrate-limiter >= 3, < 4",
-  "kili-formats == 1.1.0",
-  "httpx >= 0.27.0, < 1.0.0"
+  "kili-formats == 1.1.0"
 ]
 urls = { homepage = "https://github.com/kili-technology/kili-python-sdk" }
 

src/kili/adapters/kili_api_gateway/asset/formatters.py

@@ -2,62 +2,53 @@
 
 import asyncio
 import json
-import os
 
-import httpx
+import requests
 
 from kili.adapters.http_client import HttpClient
+from kili.adapters.kili_api_gateway.helpers.http_json import load_json_from_link
 from kili.core.helpers import is_url
 from kili.domain.types import ListOrTuple
 
 # Batch size for parallel JSON response downloads (same as export service)
 JSON_RESPONSE_BATCH_SIZE = 10
 
 
-def load_json_from_link(link: str, http_client: HttpClient) -> dict:
-    """Load json from link (synchronous fallback for non-batch operations)."""
-    if link == "" or not is_url(link):
-        return {}
-
-    response = http_client.get(link, timeout=30)
-    response.raise_for_status()
-    return response.json()
-
-
-async def _download_json_response(url: str) -> dict:
+async def _download_json_response(url: str, http_client: HttpClient) -> dict:
     """Download and parse JSON response from a URL using asyncio.
 
     Args:
         url: URL to download the JSON response from
+        http_client: HttpClient instance with SSL verification already configured
 
     Returns:
         Parsed JSON response as a dictionary
     """
     try:
-        verify_env = os.getenv("KILI_VERIFY")
-        verify = verify_env.lower() in ("true", "1", "yes") if verify_env is not None else True
-
-        async with httpx.AsyncClient(verify=verify) as client:
-            response = await client.get(url, timeout=30.0)
-            response.raise_for_status()
-            return response.json()
-    except (httpx.HTTPError, json.JSONDecodeError):
+        # Run synchronous requests call in a thread
+        response = await asyncio.to_thread(http_client.get, url, timeout=30)
+        response.raise_for_status()
+        return response.json()
+    except (requests.RequestException, json.JSONDecodeError):
         # Return empty dict on error to ensure consistent response format
         return {}
 
 
-async def _download_json_responses_async(url_to_label_mapping: list[tuple[str, dict]]) -> None:
+async def _download_json_responses_async(
+    url_to_label_mapping: list[tuple[str, dict]], http_client: HttpClient
+) -> None:
     """Download JSON responses in parallel using asyncio.
 
     Args:
         url_to_label_mapping: List of tuples (url, label_dict) to download
+        http_client: HttpClient instance with SSL verification already configured
     """
     # Process in batches to limit concurrent connections
     for i in range(0, len(url_to_label_mapping), JSON_RESPONSE_BATCH_SIZE):
         batch = url_to_label_mapping[i : i + JSON_RESPONSE_BATCH_SIZE]
 
         # Download all URLs in the batch in parallel using asyncio.gather
-        download_tasks = [_download_json_response(url) for url, _ in batch]
+        download_tasks = [_download_json_response(url, http_client) for url, _ in batch]
         json_responses = await asyncio.gather(*download_tasks)
 
         # Assign the downloaded responses back to their labels and remove the URL
@@ -67,17 +58,37 @@ async def _download_json_responses_async(url_to_label_mapping: list[tuple[str, d
                 del label["jsonResponseUrl"]
 
 
-def download_json_responses_parallel(url_to_label_mapping: list[tuple[str, dict]]) -> None:
+def download_json_responses_parallel(
+    url_to_label_mapping: list[tuple[str, dict]], http_client: HttpClient
+) -> None:
     """Download JSON responses in parallel and assign to labels.
 
     Args:
         url_to_label_mapping: List of tuples (url, label_dict) to download
+        http_client: HttpClient instance with SSL verification already configured
     """
     if not url_to_label_mapping:
         return
 
-    # Run async downloads in a synchronous context
-    asyncio.run(_download_json_responses_async(url_to_label_mapping))
+    # Check if we're already in an event loop (e.g., Jupyter notebook, async web framework)
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        # No running loop, safe to use asyncio.run() for parallel downloads
+        asyncio.run(_download_json_responses_async(url_to_label_mapping, http_client))
+    else:
+        # Already in a loop - fall back to sequential downloads to avoid RuntimeError
+        # This happens in Jupyter notebooks, FastAPI, and other async contexts
+        for url, label in url_to_label_mapping:
+            try:
+                response = http_client.get(url, timeout=30)
+                response.raise_for_status()
+                label["jsonResponse"] = response.json()
+            except (requests.RequestException, json.JSONDecodeError):
+                label["jsonResponse"] = {}
+
+            if "jsonResponseUrl" in label:
+                del label["jsonResponseUrl"]
 
 
 def _parse_label_json_response(label: dict) -> None:
@@ -129,6 +140,6 @@ def load_asset_json_fields(asset: dict, fields: ListOrTuple[str], http_client: H
         _process_label_json_response(asset["latestLabel"], url_to_label_mapping)
 
     if url_to_label_mapping:
-        download_json_responses_parallel(url_to_label_mapping)
+        download_json_responses_parallel(url_to_label_mapping, http_client)
 
     return asset

src/kili/adapters/kili_api_gateway/helpers/http_json.py

@@ -0,0 +1,22 @@
+"""HTTP JSON helpers for downloading JSON data from URLs."""
+
+from kili.adapters.http_client import HttpClient
+from kili.core.helpers import is_url
+
+
+def load_json_from_link(link: str, http_client: HttpClient) -> dict:
+    """Load json from link.
+
+    Args:
+        link: URL to download JSON from
+        http_client: HttpClient instance with SSL verification already configured
+
+    Returns:
+        Parsed JSON response as a dictionary, or empty dict if link is invalid
+    """
+    if link == "" or not is_url(link):
+        return {}
+
+    response = http_client.get(link, timeout=30)
+    response.raise_for_status()
+    return response.json()

src/kili/adapters/kili_api_gateway/label/formatters.py

@@ -3,20 +3,11 @@
 import json
 
 from kili.adapters.http_client import HttpClient
+from kili.adapters.kili_api_gateway.helpers.http_json import load_json_from_link
 from kili.core.helpers import is_url
 from kili.domain.types import ListOrTuple
 
 
-def load_json_from_link(link: str, http_client: HttpClient) -> dict:
-    """Load json from link."""
-    if link == "" or not is_url(link):
-        return {}
-
-    response = http_client.get(link, timeout=30)
-    response.raise_for_status()
-    return response.json()
-
-
 def load_label_json_fields(label: dict, fields: ListOrTuple[str], http_client: HttpClient) -> dict:
     """Load json fields of a label."""
     if "jsonResponse" in fields:

tests/unit/adapters/kili_api_gateway/asset/test_formatters.py

@@ -13,8 +13,8 @@
     JSON_RESPONSE_BATCH_SIZE,
     download_json_responses_parallel,
     load_asset_json_fields,
-    load_json_from_link,
 )
+from kili.adapters.kili_api_gateway.helpers.http_json import load_json_from_link
 
 
 class TestLoadJsonFromLink:
@@ -56,6 +56,7 @@ class TestDownloadJsonResponsesParallel:
 
     def test_download_single_json_response(self):
         """Test downloading a single JSON response."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -65,7 +66,7 @@ def test_download_single_json_response(self):
         ) as mock_download:
             mock_download.return_value = {"annotation": "data"}
 
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify the JSON was downloaded and assigned
             assert label["jsonResponse"] == {"annotation": "data"}
@@ -75,6 +76,7 @@ def test_download_single_json_response(self):
 
     def test_download_multiple_json_responses(self):
         """Test downloading multiple JSON responses in parallel."""
+        http_client = Mock()
         label1 = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         label2 = {"id": "label2", "jsonResponseUrl": "https://example.com/label2.json"}
         label3 = {"id": "label3", "jsonResponseUrl": "https://example.com/label3.json"}
@@ -86,7 +88,7 @@ def test_download_multiple_json_responses(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url:
@@ -99,7 +101,7 @@ async def mock_download(url):
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             side_effect=mock_download,
         ):
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify all labels got their JSON
             assert label1["jsonResponse"] == {"data": "label1"}
@@ -113,25 +115,27 @@ async def mock_download(url):
 
     def test_download_handles_request_exception(self):
         """Test that request exceptions are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
-        # Mock download to raise httpx.HTTPError
+        # Mock download to return empty dict on error
         with patch(
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             new_callable=AsyncMock,
         ) as mock_download:
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_handles_json_decode_error(self):
         """Test that JSON decode errors are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -143,14 +147,15 @@ def test_download_handles_json_decode_error(self):
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_handles_timeout_error(self):
         """Test that timeout errors are handled gracefully."""
+        http_client = Mock()
         label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
         url_to_label_mapping = [("https://example.com/label1.json", label)]
 
@@ -162,14 +167,15 @@ def test_download_handles_timeout_error(self):
             mock_download.return_value = {}  # Returns empty dict on error
 
             # Should not raise exception
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Should set empty dict on error
             assert label["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label
 
     def test_download_processes_in_batches(self):
         """Test that downloads are processed in batches of JSON_RESPONSE_BATCH_SIZE."""
+        http_client = Mock()
         # Create more labels than batch size
         num_labels = JSON_RESPONSE_BATCH_SIZE + 5
         labels = [
@@ -188,7 +194,7 @@ def test_download_processes_in_batches(self):
             mock_download.return_value = {"data": "test"}
 
             # Call the function
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Verify all labels got their JSON response
             for label in labels:
@@ -200,6 +206,7 @@ def test_download_processes_in_batches(self):
 
     def test_download_mixed_success_and_failure(self):
         """Test downloading with some successes and some failures."""
+        http_client = Mock()
         label_success = {"id": "success", "jsonResponseUrl": "https://example.com/success.json"}
         label_fail = {"id": "fail", "jsonResponseUrl": "https://example.com/fail.json"}
 
@@ -209,7 +216,7 @@ def test_download_mixed_success_and_failure(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "success" in url:
                 return {"status": "ok"}
             return {}  # Failure returns empty dict
@@ -218,7 +225,7 @@ async def mock_download(url):
             "kili.adapters.kili_api_gateway.asset.formatters._download_json_response",
             side_effect=mock_download,
         ):
-            download_json_responses_parallel(url_to_label_mapping)
+            download_json_responses_parallel(url_to_label_mapping, http_client)
 
             # Success should have data
             assert label_success["jsonResponse"] == {"status": "ok"}
@@ -228,6 +235,30 @@ async def mock_download(url):
             assert label_fail["jsonResponse"] == {}
             assert "jsonResponseUrl" not in label_fail
 
+    def test_download_in_existing_event_loop(self):
+        """Test that downloads work when called from within an existing event loop."""
+        import asyncio
+
+        http_client = Mock()
+        label = {"id": "label1", "jsonResponseUrl": "https://example.com/label1.json"}
+        url_to_label_mapping = [("https://example.com/label1.json", label)]
+
+        # Mock the http_client.get response
+        mock_response = Mock()
+        mock_response.json.return_value = {"data": "test"}
+        http_client.get.return_value = mock_response
+
+        # Run inside an existing event loop (simulating Jupyter/FastAPI)
+        async def run_in_loop():
+            download_json_responses_parallel(url_to_label_mapping, http_client)
+
+        asyncio.run(run_in_loop())
+
+        # Verify the JSON was downloaded (sequentially in this case)
+        assert label["jsonResponse"] == {"data": "test"}
+        assert "jsonResponseUrl" not in label
+        http_client.get.assert_called_once_with("https://example.com/label1.json", timeout=30)
+
 
 class TestLoadAssetJsonFields:
     """Test load_asset_json_fields integration with download_json_responses_parallel."""
@@ -283,7 +314,7 @@ def test_load_asset_with_labels_json_response_url(self):
         fields = ["id", "labels.jsonResponse", "labels.jsonResponseUrl"]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url:
@@ -345,7 +376,7 @@ def test_load_asset_with_both_labels_and_latest_label(self):
         ]
 
         # Create mock side effect for different URLs
-        async def mock_download(url):
+        async def mock_download(url, client):
             if "label1" in url:
                 return {"data": "label1"}
             elif "label2" in url: