Version 0.11.0 (#190)

* Fix max_keepalive_connections config

* Use .arequest for the async version of the API

* Linting

* Update tests

* Switch to transport API with 'ext' interface

* Run unasync

* Use plain strings in 'ext'. Bump version to 0.11.0

* Version 0.11

* Update CHANGELOG

* Update CHANGELOG

Author: lovelydinosaur

CHANGELOG.md

@@ -4,6 +4,59 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## 0.11.0 (September 22nd, 2020)
+
+The Transport API with 0.11.0 has a couple of significant changes.
+
+Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features
+such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections.
+
+The interface changes from:
+
+```python
+def request(method, url, headers, stream, timeout):
+    return (http_version, status_code, reason, headers, stream)
+```
+
+To instead including an optional dictionary of extensions on the request and response:
+
+```python
+def request(method, url, headers, stream, ext):
+    return (status_code, headers, stream, ext)
+```
+
+Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes.
+
+In particular:
+
+* Trailing headers support.
+* HTTP/2 Server Push
+* sendfile.
+* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming.
+* Exposing debug information out of the API, including template name, template context.
+
+Currently extensions are limited to:
+
+* request: `timeout` - Optional. Timeout dictionary.
+* response: `http_version` - Optional. Include the HTTP version used on the response.
+* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*.
+
+See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this.
+
+Secondly, the async version of `request` is now namespaced as `arequest`.
+
+This allows concrete transports to support both sync and async implementations on the same class.
+
+### Added
+
+- Add curio support. (Pull #168)
+- Add anyio support, with `backend="anyio"`. (Pull #169)
+
+### Changed
+
+- Update the Transport API to use 'ext' for optional extensions. (Pull #190)
+- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189)
+
 ## 0.10.2 (August 20th, 2020)
 
 ### Added

httpcore/__init__.py

@@ -51,7 +51,7 @@
     "WriteError",
     "WriteTimeout",
 ]
-__version__ = "0.10.2"
+__version__ = "0.11.0"
 
 __locals = locals()
 

httpcore/_async/base.py

@@ -1,8 +1,8 @@
 import enum
 from types import TracebackType
-from typing import AsyncIterator, List, Tuple, Type
+from typing import AsyncIterator, Tuple, Type
 
-from .._types import URL, Headers, T, TimeoutDict
+from .._types import URL, Headers, T
 
 
 class NewConnectionRequired(Exception):
@@ -67,8 +67,8 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]:
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
         """
         The interface for sending a single HTTP request, and returning a response.
 
@@ -80,23 +80,17 @@ async def arequest(
         * **headers** - `Optional[List[Tuple[bytes, bytes]]]` - Any HTTP headers
         to send with the request.
         * **stream** - `Optional[AsyncByteStream]` - The body of the HTTP request.
-        * **timeout** - `Optional[Dict[str, Optional[float]]]` - A dictionary of
-        timeout values for I/O operations. Supported keys are "pool" for acquiring a
-        connection from the connection pool, "read" for reading from the connection,
-        "write" for writing to the connection and "connect" for opening the connection.
-        Values are floating point seconds.
+        * **ext** - `Optional[dict]` - A dictionary of optional extensions.
 
         ** Returns:**
 
-        A five-tuple of:
+        A four-tuple of:
 
-        * **http_version** - `bytes` - The HTTP version used by the server,
-        such as `b'HTTP/1.1'`.
         * **status_code** - `int` - The HTTP status code, such as `200`.
-        * **reason_phrase** - `bytes` - Any HTTP reason phrase, such as `b'OK'`.
         * **headers** - `List[Tuple[bytes, bytes]]` - Any HTTP headers included
         on the response.
         * **stream** - `AsyncByteStream` - The body of the HTTP response.
+        * **ext** - `dict` - A dictionary of optional extensions.
         """
         raise NotImplementedError()  # pragma: nocover
 

httpcore/_async/connection.py

@@ -1,5 +1,5 @@
 from ssl import SSLContext
-from typing import List, Optional, Tuple
+from typing import Optional, Tuple, cast
 
 from .._backends.auto import AsyncBackend, AsyncLock, AsyncSocketStream, AutoBackend
 from .._types import URL, Headers, Origin, TimeoutDict
@@ -72,9 +72,12 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]:
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
         assert url_to_origin(url) == self.origin
+        ext = {} if ext is None else ext
+        timeout = cast(TimeoutDict, ext.get("timeout", {}))
+
         async with self.request_lock:
             if self.state == ConnectionState.PENDING:
                 if not self.socket:
@@ -94,7 +97,7 @@ async def arequest(
         logger.trace(
             "connection.arequest method=%r url=%r headers=%r", method, url, headers
         )
-        return await self.connection.arequest(method, url, headers, stream, timeout)
+        return await self.connection.arequest(method, url, headers, stream, ext)
 
     async def _open_socket(self, timeout: TimeoutDict = None) -> AsyncSocketStream:
         scheme, hostname, port = self.origin

httpcore/_async/connection_pool.py

@@ -1,6 +1,6 @@
 import warnings
 from ssl import SSLContext
-from typing import AsyncIterator, Callable, Dict, List, Optional, Set, Tuple
+from typing import AsyncIterator, Callable, Dict, List, Optional, Set, Tuple, cast
 
 from .._backends.auto import AsyncLock, AsyncSemaphore
 from .._backends.base import lookup_async_backend
@@ -153,15 +153,17 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]:
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
         if url[0] not in (b"http", b"https"):
             scheme = url[0].decode("latin-1")
             raise UnsupportedProtocol(f"Unsupported URL protocol {scheme!r}")
         if not url[1]:
             raise LocalProtocolError("Missing hostname in URL.")
 
         origin = url_to_origin(url)
+        ext = {} if ext is None else ext
+        timeout = cast(TimeoutDict, ext.get("timeout", {}))
 
         await self._keepalive_sweep()
 
@@ -190,7 +192,7 @@ async def arequest(
 
             try:
                 response = await connection.arequest(
-                    method, url, headers=headers, stream=stream, timeout=timeout
+                    method, url, headers=headers, stream=stream, ext=ext
                 )
             except NewConnectionRequired:
                 connection = None
@@ -199,10 +201,11 @@ async def arequest(
                 await self._remove_from_pool(connection)
                 raise
 
+        status_code, headers, stream, ext = response
         wrapped_stream = ResponseByteStream(
-            response[4], connection=connection, callback=self._response_closed
+            stream, connection=connection, callback=self._response_closed
         )
-        return response[0], response[1], response[2], response[3], wrapped_stream
+        return status_code, headers, wrapped_stream, ext
 
     async def _get_connection_from_pool(
         self, origin: Origin
@@ -305,10 +308,8 @@ async def _keepalive_sweep(self) -> None:
             await connection.aclose()
 
     async def _add_to_pool(
-        self, connection: AsyncHTTPConnection, timeout: TimeoutDict = None
+        self, connection: AsyncHTTPConnection, timeout: TimeoutDict
     ) -> None:
-        timeout = {} if timeout is None else timeout
-
         logger.trace("adding connection to pool=%r", connection)
         await self._connection_semaphore.acquire(timeout=timeout.get("pool", None))
         async with self._thread_lock:

httpcore/_async/http11.py

@@ -1,5 +1,5 @@
 from ssl import SSLContext
-from typing import AsyncIterator, List, Tuple, Union
+from typing import AsyncIterator, List, Tuple, Union, cast
 
 import h11
 
@@ -53,11 +53,12 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]:
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
         headers = [] if headers is None else headers
         stream = PlainByteStream(b"") if stream is None else stream
-        timeout = {} if timeout is None else timeout
+        ext = {} if ext is None else ext
+        timeout = cast(TimeoutDict, ext.get("timeout", {}))
 
         self.state = ConnectionState.ACTIVE
 
@@ -73,7 +74,11 @@ async def arequest(
             aiterator=self._receive_response_data(timeout),
             aclose_func=self._response_closed,
         )
-        return (http_version, status_code, reason_phrase, headers, response_stream)
+        ext = {
+            "http_version": http_version.decode("ascii", errors="ignore"),
+            "reason": reason_phrase.decode("ascii", errors="ignore"),
+        }
+        return (status_code, headers, response_stream, ext)
 
     async def start_tls(
         self, hostname: bytes, timeout: TimeoutDict = None

httpcore/_async/http2.py

@@ -1,6 +1,5 @@
-from http import HTTPStatus
 from ssl import SSLContext
-from typing import AsyncIterator, Dict, List, Tuple
+from typing import AsyncIterator, Dict, List, Tuple, cast
 
 import h2.connection
 import h2.events
@@ -19,13 +18,6 @@
 logger = get_logger(__name__)
 
 
-def get_reason_phrase(status_code: int) -> bytes:
-    try:
-        return HTTPStatus(status_code).phrase.encode("ascii")
-    except ValueError:
-        return b""
-
-
 class AsyncHTTP2Connection(AsyncBaseHTTPConnection):
     READ_NUM_BYTES = 64 * 1024
     CONFIG = H2Configuration(validate_inbound_headers=False)
@@ -99,9 +91,10 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]:
-        timeout = {} if timeout is None else timeout
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
+        ext = {} if ext is None else ext
+        timeout = cast(TimeoutDict, ext.get("timeout", {}))
 
         async with self.init_lock:
             if not self.sent_connection_init:
@@ -123,7 +116,7 @@ async def arequest(
             h2_stream = AsyncHTTP2Stream(stream_id=stream_id, connection=self)
             self.streams[stream_id] = h2_stream
             self.events[stream_id] = []
-            return await h2_stream.arequest(method, url, headers, stream, timeout)
+            return await h2_stream.arequest(method, url, headers, stream, ext)
         except Exception:  # noqa: PIE786
             self.max_streams_semaphore.release()
             raise
@@ -283,11 +276,12 @@ async def arequest(
         url: URL,
         headers: Headers = None,
         stream: AsyncByteStream = None,
-        timeout: TimeoutDict = None,
-    ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]:
+        ext: dict = None,
+    ) -> Tuple[int, Headers, AsyncByteStream, dict]:
         headers = [] if headers is None else [(k.lower(), v) for (k, v) in headers]
         stream = PlainByteStream(b"") if stream is None else stream
-        timeout = {} if timeout is None else timeout
+        ext = {} if ext is None else ext
+        timeout = cast(TimeoutDict, ext.get("timeout", {}))
 
         # Send the request.
         seen_headers = set(key for key, value in headers)
@@ -301,12 +295,14 @@ async def arequest(
 
         # Receive the response.
         status_code, headers = await self.receive_response(timeout)
-        reason_phrase = get_reason_phrase(status_code)
         response_stream = AsyncIteratorByteStream(
             aiterator=self.body_iter(timeout), aclose_func=self._response_closed
         )
 
-        return (b"HTTP/2", status_code, reason_phrase, headers, response_stream)
+        ext = {
+            "http_version": "HTTP/2",
+        }
+        return (status_code, headers, response_stream, ext)
 
     async def send_headers(
         self,