Skip to content

Commit 7616f7b

Browse files
committed
WIP: Async support in pulp-glue
Replaces requests with aiohttp and changes the api.
1 parent b31ea2e commit 7616f7b

File tree

21 files changed

+746
-563
lines changed

21 files changed

+746
-563
lines changed

.ci/settings/settings.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
ALLOWED_EXPORT_PATHS = ["/tmp"]
44
ANALYTICS = False
55
ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"]
6+
TASK_DIAGNOSTICS = ["memory"]
67

78
if os.environ.get("PULP_HTTPS", "false").lower() == "true":
89
AUTHENTICATION_BACKENDS = "@merge django.contrib.auth.backends.RemoteUserBackend"

CHANGES/pulp-glue/+aiohttp.feature

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
WIP: Added async api to Pulp glue.

CHANGES/pulp-glue/+aiohttp.removal

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Replaced requests with aiohttp.
2+
Breaking change: Reworked the contract around the `AuthProvider` to allow authentication to be coded independently of the underlying library.

lint_requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ mypy==1.19.1
44
shellcheck-py==0.11.0.1
55

66
# Type annotation stubs
7+
types-aiofiles
78
types-pygments
89
types-PyYAML
9-
types-requests
1010
types-setuptools
1111
types-toml
1212

lower_bounds_constraints.lock

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
aiofiles==25.1.0
2+
aiohttp==3.12.0
13
click==8.0.0
24
packaging==20.0
35
PyYAML==5.3

pulp-glue/docs/dev/learn/architecture.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ To this end, `pulp-glue` is the go-to place for all known version-dependent Pulp
88

99
## OpenAPI
1010

11-
This is the part in `pulp_glue` that uses [`requests`](https://requests.readthedocs.io/) to perform low level communication with an `OpenAPI 3` compatible server.
11+
This is the part in `pulp_glue` that uses http to perform low level communication with an `OpenAPI 3` compatible server.
1212
It is not anticipated that users of Pulp Glue need to interact with this abstraction layer.
1313

1414
## Contexts
Lines changed: 132 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,145 @@
11
import typing as t
2-
from datetime import datetime, timedelta
2+
from datetime import datetime
33

4-
import requests
54

6-
7-
class OAuth2ClientCredentialsAuth(requests.auth.AuthBase):
8-
"""
9-
This implements the OAuth2 ClientCredentials Grant authentication flow.
10-
https://datatracker.ietf.org/doc/html/rfc6749#section-4.4
5+
class AuthProviderBase:
116
"""
7+
Base class for auth providers.
128
13-
def __init__(
14-
self,
15-
client_id: str,
16-
client_secret: str,
17-
token_url: str,
18-
scopes: list[str] | None = None,
19-
verify_ssl: str | bool | None = None,
20-
):
21-
self._token_server_auth = requests.auth.HTTPBasicAuth(client_id, client_secret)
22-
self._token_url = token_url
23-
self._scopes = scopes
24-
self._verify_ssl = verify_ssl
9+
This abstract base class will analyze the authentication proposals of the openapi specs.
10+
Different authentication schemes can be implemented in subclasses.
11+
"""
2512

26-
self._access_token: str | None = None
27-
self._expire_at: datetime | None = None
13+
def __init__(self) -> None:
14+
self._oauth2_token: str | None = None
15+
self._oauth2_expires: datetime = datetime.now()
16+
17+
def can_complete_http_basic(self) -> bool:
18+
return False
19+
20+
def can_complete_mutualTLS(self) -> bool:
21+
return False
22+
23+
def can_complete_oauth2_client_credentials(self, scopes: list[str]) -> bool:
24+
return False
25+
26+
def can_complete_scheme(self, scheme: dict[str, t.Any], scopes: list[str]) -> bool:
27+
if scheme["type"] == "http":
28+
if scheme["scheme"] == "basic":
29+
return self.can_complete_http_basic()
30+
elif scheme["type"] == "mutualTLS":
31+
return self.can_complete_mutualTLS()
32+
elif scheme["type"] == "oauth2":
33+
for flow_name, flow in scheme["flows"].items():
34+
if flow_name == "clientCredentials" and self.can_complete_oauth2_client_credentials(
35+
flow["scopes"]
36+
):
37+
return True
38+
return False
39+
40+
def can_complete(
41+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
42+
) -> bool:
43+
for name, scopes in proposal.items():
44+
scheme = security_schemes.get(name)
45+
if scheme is None or not self.can_complete_scheme(scheme, scopes):
46+
return False
47+
# This covers the case where `[]` allows for no auth at all.
48+
return True
49+
50+
async def auth_success_hook(
51+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
52+
) -> None:
53+
pass
54+
55+
async def auth_failure_hook(
56+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
57+
) -> None:
58+
pass
59+
60+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
61+
raise NotImplementedError()
62+
63+
async def oauth2_client_credentials(self) -> tuple[bytes, bytes]:
64+
raise NotImplementedError()
65+
66+
def tls_credentials(self) -> tuple[str, str | None]:
67+
raise NotImplementedError()
68+
69+
70+
class BasicAuthProvider(AuthProviderBase):
71+
"""
72+
AuthProvider providing basic auth with fixed `username`, `password`.
73+
"""
2874

29-
def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest:
30-
if self._expire_at is None or self._expire_at < datetime.now():
31-
self._retrieve_token()
75+
def __init__(self, username: t.AnyStr, password: t.AnyStr):
76+
super().__init__()
77+
self.username: bytes = username.encode("latin1") if isinstance(username, str) else username
78+
self.password: bytes = password.encode("latin1") if isinstance(password, str) else password
3279

33-
assert self._access_token is not None
80+
def can_complete_http_basic(self) -> bool:
81+
return True
3482

35-
request.headers["Authorization"] = f"Bearer {self._access_token}"
83+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
84+
return self.username, self.password
3685

37-
# Call to untyped function "register_hook" in typed context
38-
request.register_hook("response", self._handle401) # type: ignore[no-untyped-call]
3986

40-
return request
87+
class GlueAuthProvider(AuthProviderBase):
88+
"""
89+
AuthProvider allowing to be used with prepared credentials.
90+
"""
4191

42-
def _handle401(
92+
def __init__(
4393
self,
44-
response: requests.Response,
45-
**kwargs: t.Any,
46-
) -> requests.Response:
47-
if response.status_code != 401:
48-
return response
49-
50-
# If we get this far, probably the token is not valid anymore.
51-
52-
# Try to reach for a new token once.
53-
self._retrieve_token()
54-
55-
assert self._access_token is not None
56-
57-
# Consume content and release the original connection
58-
# to allow our new request to reuse the same one.
59-
response.content
60-
response.close()
61-
prepared_new_request = response.request.copy()
62-
63-
prepared_new_request.headers["Authorization"] = f"Bearer {self._access_token}"
64-
65-
# Avoid to enter into an infinity loop.
66-
# Call to untyped function "deregister_hook" in typed context
67-
prepared_new_request.deregister_hook( # type: ignore[no-untyped-call]
68-
"response", self._handle401
69-
)
70-
71-
# "Response" has no attribute "connection"
72-
new_response: requests.Response = response.connection.send(prepared_new_request, **kwargs)
73-
new_response.history.append(response)
74-
new_response.request = prepared_new_request
75-
76-
return new_response
77-
78-
def _retrieve_token(self) -> None:
79-
data = {
80-
"grant_type": "client_credentials",
81-
}
82-
83-
if self._scopes:
84-
data["scope"] = " ".join(self._scopes)
85-
86-
response: requests.Response = requests.post(
87-
self._token_url,
88-
data=data,
89-
auth=self._token_server_auth,
90-
verify=self._verify_ssl,
91-
)
92-
93-
response.raise_for_status()
94-
95-
token = response.json()
96-
self._expire_at = datetime.now() + timedelta(seconds=token["expires_in"])
97-
self._access_token = token["access_token"]
94+
*,
95+
username: t.AnyStr | None = None,
96+
password: t.AnyStr | None = None,
97+
client_id: t.AnyStr | None = None,
98+
client_secret: t.AnyStr | None = None,
99+
cert: str | None = None,
100+
key: str | None = None,
101+
):
102+
super().__init__()
103+
self.username: bytes | None = None
104+
self.password: bytes | None = None
105+
self.client_id: bytes | None = None
106+
self.client_secret: bytes | None = None
107+
self.cert: str | None = cert
108+
self.key: str | None = key
109+
110+
if username is not None:
111+
assert password is not None
112+
self.username = username.encode("latin1") if isinstance(username, str) else username
113+
self.password = password.encode("latin1") if isinstance(password, str) else password
114+
if client_id is not None:
115+
assert client_secret is not None
116+
self.client_id = client_id.encode("latin1") if isinstance(client_id, str) else client_id
117+
self.client_secret = (
118+
client_secret.encode("latin1") if isinstance(client_secret, str) else client_secret
119+
)
120+
121+
if cert is None and key is not None:
122+
raise RuntimeError("Key can only be used together with a cert.")
123+
124+
def can_complete_http_basic(self) -> bool:
125+
return self.username is not None
126+
127+
def can_complete_oauth2_client_credentials(self, scopes: list[str]) -> bool:
128+
return self.client_id is not None
129+
130+
def can_complete_mutualTLS(self) -> bool:
131+
return self.cert is not None
132+
133+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
134+
assert self.username is not None
135+
assert self.password is not None
136+
return self.username, self.password
137+
138+
async def oauth2_client_credentials(self) -> tuple[bytes, bytes]:
139+
assert self.client_id is not None
140+
assert self.client_secret is not None
141+
return self.client_id, self.client_secret
142+
143+
def tls_credentials(self) -> tuple[str, str | None]:
144+
assert self.cert is not None
145+
return (self.cert, self.key)

pulp-glue/pulp_glue/common/context.py

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from contextlib import ExitStack
99

1010
from packaging.specifiers import SpecifierSet
11+
from pulp_glue.common.authentication import GlueAuthProvider
1112
from pulp_glue.common.exceptions import (
1213
NotImplementedFake,
1314
OpenAPIError,
@@ -18,7 +19,7 @@
1819
UnsafeCallError,
1920
)
2021
from pulp_glue.common.i18n import get_translation
21-
from pulp_glue.common.openapi import BasicAuthProvider, OpenAPI
22+
from pulp_glue.common.openapi import OpenAPI
2223

2324
if sys.version_info >= (3, 11):
2425
import tomllib
@@ -201,6 +202,20 @@ def patch_upstream_pulp_replicate_request_body(api: OpenAPI) -> None:
201202
operation.pop("requestBody", None)
202203

203204

205+
@api_quirk(PluginRequirement("core", specifier="<3.85"))
206+
def patch_security_scheme_mutual_tls(api: OpenAPI) -> None:
207+
# Trick to allow tls cert auth on older Pulp.
208+
if (components := api.api_spec.get("components")) is not None:
209+
if (security_schemes := components.get("securitySchemes")) is not None:
210+
# Only if it is going to be idempotent...
211+
if "gluePatchTLS" not in security_schemes:
212+
security_schemes["gluePatchTLS"] = {"type": "mutualTLS"}
213+
for method, path in api.operations.values():
214+
operation = api.api_spec["paths"][path][method]
215+
if "security" in operation:
216+
operation["security"].append({"gluePatchTLS": []})
217+
218+
204219
class PulpContext:
205220
"""
206221
Abstract class for the global PulpContext object.
@@ -334,8 +349,13 @@ def from_config(cls, config: dict[str, t.Any]) -> "t.Self":
334349
api_kwargs: dict[str, t.Any] = {
335350
"base_url": config["base_url"],
336351
}
337-
if "username" in config:
338-
api_kwargs["auth_provider"] = BasicAuthProvider(config["username"], config["password"])
352+
api_kwargs["auth_provider"] = GlueAuthProvider(
353+
**{
354+
k: v
355+
for k, v in config.items()
356+
if k in {"username", "password", "client_id", "client_secret", "cert", "key"}
357+
}
358+
)
339359
if "headers" in config:
340360
api_kwargs["headers"] = dict(
341361
(header.split(":", maxsplit=1) for header in config["headers"])
@@ -384,7 +404,9 @@ def api(self) -> OpenAPI:
384404
# Deprecated for 'auth'.
385405
if not password:
386406
password = self.prompt("password", hide_input=True)
387-
self._api_kwargs["auth_provider"] = BasicAuthProvider(username, password)
407+
self._api_kwargs["auth_provider"] = GlueAuthProvider(
408+
username=username, password=password
409+
)
388410
warnings.warn(
389411
"Using 'username' and 'password' with 'PulpContext' is deprecated. "
390412
"Use an auth provider with the 'auth_provider' argument instead.",
@@ -398,10 +420,10 @@ def api(self) -> OpenAPI:
398420
)
399421
except OpenAPIError as e:
400422
raise PulpException(str(e))
423+
self._patch_api_spec()
401424
# Rerun scheduled version checks
402425
for plugin_requirement in self._needed_plugins:
403426
self.needs_plugin(plugin_requirement)
404-
self._patch_api_spec()
405427
return self._api
406428

407429
@property

0 commit comments

Comments
 (0)