Merge pull request #374 from maxnoe/remove_requests

Requests → httpx

Author: chaen

diracx-cli/src/diracx/cli/auth.py

@@ -5,6 +5,7 @@
 import asyncio
 import json
 import os
+from asyncio import sleep
 from datetime import datetime, timedelta, timezone
 from typing import Annotated, Optional
 
@@ -92,7 +93,7 @@ async def login(
                 if response.error == "authorization_pending":
                     # TODO: Setting more than 5 seconds results in an error
                     # Related to keep-alive disconnects from uvicon (--timeout-keep-alive)
-                    await asyncio.sleep(2)
+                    await sleep(2)
                     continue
                 raise RuntimeError(f"Device flow failed with {response}")
             break

diracx-client/pyproject.toml

@@ -12,12 +12,12 @@ classifiers = [
     "Topic :: Scientific/Engineering",
     "Topic :: System :: Distributed Computing",
 ]
-dependencies = ["azure-core", "diracx-core", "isodate", "requests"]
+dependencies = ["azure-core", "diracx-core", "isodate", "httpx"]
 dynamic = ["version"]
 
 [project.optional-dependencies]
 testing = ["diracx-testing"]
-types = ["types-requests"]
+types = []
 
 [tool.setuptools.packages.find]
 where = ["src"]

diracx-client/src/diracx/client/generated/_patch.py

@@ -13,7 +13,7 @@
 import importlib.util
 import json
 import jwt
-import requests
+import httpx
 
 from pathlib import Path
 from typing import Any, Dict, List, Optional, cast

diracx-client/src/diracx/client/patches/utils.py

@@ -7,8 +7,8 @@
 import json
 import os
 from diracx.core.utils import EXPIRES_GRACE_SECONDS, serialize_credentials
+import httpx
 import jwt
-import requests
 
 from datetime import datetime, timezone
 from importlib.metadata import PackageNotFoundError, distribution
@@ -43,11 +43,11 @@ def get_openid_configuration(
     endpoint: str, *, verify: bool | str = True
 ) -> Dict[str, str]:
     """Get the openid configuration from the .well-known endpoint"""
-    response = requests.get(
+    response = httpx.get(
         url=parse.urljoin(endpoint, ".well-known/openid-configuration"),
         verify=verify,
     )
-    if not response.ok:
+    if not response.is_success:
         raise RuntimeError("Cannot fetch any information from the .well-known endpoint")
     return response.json()
 
@@ -123,7 +123,7 @@ def refresh_token(
     verify: bool | str = True,
 ) -> TokenResponse:
     """Refresh the access token using the refresh_token flow."""
-    response = requests.post(
+    response = httpx.post(
         url=token_endpoint,
         data={
             "client_id": client_id,

diracx-core/tests/test_s3.py

@@ -5,8 +5,8 @@
 import random
 import secrets
 
+import httpx
 import pytest
-import requests
 from aiobotocore.session import get_session
 
 from diracx.core.s3 import (
@@ -82,9 +82,13 @@ async def test_presigned_upload_moto(moto_s3):
     )
 
     # Upload the file
-    r = requests.post(
-        upload_info["url"], data=upload_info["fields"], files={"file": file_content}
-    )
+    async with httpx.AsyncClient() as client:
+        r = await client.post(
+            upload_info["url"],
+            data=upload_info["fields"],
+            files={"file": file_content},
+        )
+
     assert r.status_code == 204, r.text
 
     # Make sure the object is actually there
@@ -120,12 +124,18 @@ async def test_bucket(minio_client):
     "content,checksum,size,expected_error",
     [
         # Make sure a valid request works
-        [*_random_file(128), 128, None],
+        pytest.param(*_random_file(128), 128, None, id="valid"),
         # Check with invalid sizes
-        [*_random_file(128), 127, "exceeds the maximum"],
-        [*_random_file(128), 129, "smaller than the minimum"],
+        pytest.param(*_random_file(128), 127, "exceeds the maximum", id="maximum"),
+        pytest.param(*_random_file(128), 129, "smaller than the minimum", id="minimum"),
         # Check with invalid checksum
-        [_random_file(128)[0], _random_file(128)[1], 128, "ContentChecksumMismatch"],
+        pytest.param(
+            _random_file(128)[0],
+            _random_file(128)[1],
+            128,
+            "ContentChecksumMismatch",
+            id="checksum",
+        ),
     ],
 )
 async def test_presigned_upload_minio(
@@ -143,9 +153,11 @@ async def test_presigned_upload_minio(
         minio_client, test_bucket, key, "sha256", checksum, size, 60
     )
     # Ensure the URL doesn't work
-    r = requests.post(
-        upload_info["url"], data=upload_info["fields"], files={"file": content}
-    )
+    async with httpx.AsyncClient() as client:
+        r = await client.post(
+            upload_info["url"], data=upload_info["fields"], files={"file": content}
+        )
+
     if expected_error is None:
         assert r.status_code == 204, r.text
         assert await s3_object_exists(minio_client, test_bucket, key)

diracx-routers/pyproject.toml

@@ -43,7 +43,6 @@ types = [
     "types-cachetools",
     "types-python-dateutil",
     "types-PyYAML",
-    "types-requests",
 ]
 
 [project.entry-points."diracx.services"]

diracx-routers/tests/jobs/test_sandboxes.py

@@ -5,8 +5,8 @@
 from copy import deepcopy
 from io import BytesIO
 
+import httpx
 import pytest
-import requests
 from fastapi.testclient import TestClient
 
 from diracx.routers.auth.token import create_token
@@ -57,15 +57,15 @@ def test_upload_then_download(
 
     # Actually upload the file
     files = {"file": ("file", BytesIO(data))}
-    r = requests.post(upload_info["url"], data=upload_info["fields"], files=files)
+    r = httpx.post(upload_info["url"], data=upload_info["fields"], files=files)
     assert r.status_code == 204, r.text
 
     # Make sure we can download it and get the same data back
     r = normal_user_client.get("/api/jobs/sandbox", params={"pfn": sandbox_pfn})
     assert r.status_code == 200, r.text
     download_info = r.json()
     assert download_info["expires_in"] > 5
-    r = requests.get(download_info["url"])
+    r = httpx.get(download_info["url"])
     assert r.status_code == 200, r.text
     assert r.content == data
 

diracx-testing/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "pytest-asyncio",
     "pytest-cov",
     "pytest-xdist",
+    "httpx",
 ]
 dynamic = ["version"]
 

diracx-testing/src/diracx/testing/utils.py

@@ -18,8 +18,8 @@
 from urllib.parse import parse_qs, urljoin, urlparse
 from uuid import uuid4
 
+import httpx
 import pytest
-import requests
 
 if TYPE_CHECKING:
     from diracx.core.settings import DevelopmentSettings
@@ -616,7 +616,7 @@ async def test_login(monkeypatch, capfd, cli_env):
 
     poll_attempts = 0
 
-    def fake_sleep(*args, **kwargs):
+    async def fake_sleep(*args, **kwargs):
         nonlocal poll_attempts
 
         # Keep track of the number of times this is called
@@ -629,13 +629,13 @@ def fake_sleep(*args, **kwargs):
             match = re.search(rf"{cli_env['DIRACX_URL']}[^\n]+", captured.out)
             assert match, captured
 
-            do_device_flow_with_dex(match.group(), cli_env["DIRACX_CA_PATH"])
+            await do_device_flow_with_dex(match.group(), cli_env["DIRACX_CA_PATH"])
 
         # Ensure we don't poll forever
         assert poll_attempts <= 100
 
         # Reduce the sleep duration to zero to speed up the test
-        return unpatched_sleep(0)
+        await unpatched_sleep(0.0)
 
     # We monkeypatch asyncio.sleep to provide a hook to run the actions that
     # would normally be done by a user. This includes capturing the login URL
@@ -650,7 +650,7 @@ def fake_sleep(*args, **kwargs):
 
     # Run the login command
     with monkeypatch.context() as m:
-        m.setattr("asyncio.sleep", fake_sleep)
+        m.setattr("diracx.cli.auth.sleep", fake_sleep)
         await cli.auth.login(vo="diracAdmin", group=None, property=None)
     captured = capfd.readouterr()
     assert "Login successful!" in captured.out
@@ -664,18 +664,22 @@ def fake_sleep(*args, **kwargs):
     return expected_credentials_path.read_text()
 
 
-def do_device_flow_with_dex(url: str, ca_path: str) -> None:
+async def do_device_flow_with_dex(url: str, ca_path: str) -> None:
     """Do the device flow with dex."""
 
     class DexLoginFormParser(HTMLParser):
         def handle_starttag(self, tag, attrs):
             nonlocal action_url
             if "form" in str(tag):
                 assert action_url is None
-                action_url = urljoin(login_page_url, dict(attrs)["action"])
+                action_url = urljoin(str(login_page_url), dict(attrs)["action"])
 
+    ssl_context = ssl.create_default_context(cafile=ca_path)
+    client_kwargs = dict(verify=ssl_context, follow_redirects=True)
     # Get the login page
-    r = requests.get(url, verify=ca_path)
+    async with httpx.AsyncClient(**client_kwargs) as client:
+        r = await client.get(url)
+
     r.raise_for_status()
     login_page_url = r.url  # This is not the same as URL as we redirect to dex
     login_page_body = r.text
@@ -686,19 +690,24 @@ def handle_starttag(self, tag, attrs):
     assert action_url is not None, login_page_body
 
     # Do the actual login
-    r = requests.post(
-        action_url,
-        data={"login": "admin@example.com", "password": "password"},
-        verify=ca_path,
-    )
+    async with httpx.AsyncClient(**client_kwargs) as client:
+        r = await client.post(
+            action_url,
+            data={"login": "admin@example.com", "password": "password"},
+        )
+
     r.raise_for_status()
     approval_url = r.url  # This is not the same as URL as we redirect to dex
     # Do the actual approval
-    r = requests.post(
-        approval_url,
-        {"approval": "approve", "req": parse_qs(urlparse(r.url).query)["req"][0]},
-        verify=ca_path,
-    )
+
+    async with httpx.AsyncClient(**client_kwargs) as client:
+        r = await client.post(
+            approval_url,
+            data={
+                "approval": "approve",
+                "req": parse_qs(urlparse(str(r.url)).query)["req"][0],
+            },
+        )
 
     # This should have redirected to the DiracX page that shows the login is complete
     assert "Please close the window" in r.text

extensions/gubbins/gubbins-client/pyproject.toml

@@ -17,7 +17,7 @@ dynamic = ["version"]
 
 [project.optional-dependencies]
 testing = ["diracx-client[testing]", "diracx-testing"]
-types = ["types-requests"]
+types = []
 
 [tool.setuptools.packages.find]
 where = ["src"]