Skip to content

Commit a084fde

Browse files
committed
Merge pull request #239 from KeepSafe/chunked-encoding
make chunked transfer encoding explicit
2 parents 3a528c1 + 0907ed9 commit a084fde

File tree

9 files changed

+164
-36
lines changed

9 files changed

+164
-36
lines changed

CHANGES.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ CHANGES
3333
- Convert `ConnectionError` to `aiohttp.DisconnectedError` and don't
3434
eat `ConnectionError` exceptions from web handlers.
3535

36+
- Remove hop headers from Response class, wsgi response still uses hop headers.
37+
38+
- Allow to send raw chunked encoded response.
39+
40+
- Allow to encode output bytes stream into chunked encoding.
41+
42+
- Allow to compress output bytes stream with `deflate` encoding.
43+
3644
- Server has 75 seconds keepalive timeout now, was non-keepalive by default.
3745

3846
- Application doesn't accept `**kwargs` anymore (#243).

aiohttp/client.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,6 +539,7 @@ def send(self, writer, reader):
539539
request.add_compression_filter(self.compress)
540540

541541
if self.chunked is not None:
542+
request.enable_chunked_encoding()
542543
request.add_chunking_filter(self.chunked)
543544

544545
request.add_headers(

aiohttp/protocol.py

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,7 @@ class HttpMessage:
518518
status_line = b''
519519
upgrade = False # Connection: UPGRADE
520520
websocket = False # Upgrade: WEBSOCKET
521+
has_chunked_hdr = False # Transfer-encoding: chunked
521522

522523
# subclass can enable auto sending headers with write() call,
523524
# this is useful for wsgi's start_response implementation.
@@ -545,7 +546,7 @@ def force_close(self):
545546
self.closing = True
546547
self.keepalive = False
547548

548-
def force_chunked(self):
549+
def enable_chunked_encoding(self):
549550
self.chunked = True
550551

551552
def keep_alive(self):
@@ -575,6 +576,9 @@ def add_header(self, name, value):
575576
if name == hdrs.CONTENT_LENGTH:
576577
self.length = int(value)
577578

579+
if name == hdrs.TRANSFER_ENCODING:
580+
self.has_chunked_hdr = value.lower().strip() == 'chunked'
581+
578582
if name == hdrs.CONNECTION:
579583
val = value.lower()
580584
# handle websocket
@@ -591,9 +595,6 @@ def add_header(self, name, value):
591595
self.websocket = True
592596
self.headers[name] = value
593597

594-
elif name == hdrs.TRANSFER_ENCODING and not self.chunked:
595-
self.chunked = value.lower().strip() == 'chunked'
596-
597598
elif name not in self.HOP_HEADERS:
598599
# ignore hop-by-hop headers
599600
self.headers.add(name, value)
@@ -612,12 +613,11 @@ def send_headers(self):
612613
assert not self.headers_sent, 'headers have been sent already'
613614
self.headers_sent = True
614615

615-
if (self.chunked is True) or (
616-
self.length is None and
617-
self.version >= HttpVersion11 and
618-
self.status not in (304, 204)):
619-
self.chunked = True
616+
if self.chunked or (self.length is None and
617+
self.version >= HttpVersion11 and
618+
self.status not in (304, 204)):
620619
self.writer = self._write_chunked_payload()
620+
self.headers[hdrs.TRANSFER_ENCODING] = 'chunked'
621621

622622
elif self.length is not None:
623623
self.writer = self._write_length_payload(self.length)
@@ -630,15 +630,15 @@ def send_headers(self):
630630
self._add_default_headers()
631631

632632
# status + headers
633-
hdrs = ''.join(itertools.chain(
633+
headers = ''.join(itertools.chain(
634634
(self.status_line,),
635635
*((k, ': ', v, '\r\n')
636636
for k, v in ((k, value)
637637
for k, value in self.headers.items()))))
638-
hdrs = hdrs.encode('utf-8') + b'\r\n'
638+
headers = headers.encode('utf-8') + b'\r\n'
639639

640-
self.output_length += len(hdrs)
641-
self.transport.write(hdrs)
640+
self.output_length += len(headers)
641+
self.transport.write(headers)
642642

643643
def _add_default_headers(self):
644644
# set the connection header
@@ -649,9 +649,6 @@ def _add_default_headers(self):
649649
else:
650650
connection = 'close'
651651

652-
if self.chunked:
653-
self.headers[hdrs.TRANSFER_ENCODING] = 'chunked'
654-
655652
self.headers[hdrs.CONNECTION] = connection
656653

657654
def write(self, chunk, *, EOF_MARKER=EOF_MARKER, EOL_MARKER=EOL_MARKER):
@@ -795,16 +792,7 @@ class Response(HttpMessage):
795792
http version, (1, 0) stands for HTTP/1.0 and (1, 1) is for HTTP/1.1
796793
"""
797794

798-
HOP_HEADERS = {
799-
hdrs.CONNECTION,
800-
hdrs.KEEP_ALIVE,
801-
hdrs.PROXY_AUTHENTICATE,
802-
hdrs.PROXY_AUTHORIZATION,
803-
hdrs.TE,
804-
hdrs.TRAILER,
805-
hdrs.TRANSFER_ENCODING,
806-
hdrs.UPGRADE,
807-
}
795+
HOP_HEADERS = ()
808796

809797
@staticmethod
810798
def calc_reason(status):

aiohttp/test_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ def _response(self, response, body=None,
287287
hdrs.extend(headers.items())
288288

289289
if chunked:
290-
response.force_chunked()
290+
response.enable_chunked_encoding()
291291

292292
# headers
293293
response.add_headers(*hdrs)

aiohttp/web.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,10 @@ class StreamResponse(HeadersMixin):
391391
def __init__(self, *, status=200, reason=None):
392392
self._body = None
393393
self._keep_alive = None
394+
self._chunked = False
395+
self._chunk_size = None
396+
self._compression = False
397+
self._compression_force = False
394398
self._headers = CIMultiDict()
395399
self._cookies = http.cookies.SimpleCookie()
396400
self.set_status(status, reason)
@@ -412,6 +416,14 @@ def started(self):
412416
def status(self):
413417
return self._status
414418

419+
@property
420+
def chunked(self):
421+
return self._chunked
422+
423+
@property
424+
def compression(self):
425+
return self._compression
426+
415427
@property
416428
def reason(self):
417429
return self._reason
@@ -429,6 +441,16 @@ def keep_alive(self):
429441
def force_close(self):
430442
self._keep_alive = False
431443

444+
def enable_chunked_encoding(self, chunk_size=None):
445+
"""Enables automatic chunked transfer encoding."""
446+
self._chunked = True
447+
self._chunk_size = chunk_size
448+
449+
def enable_compression(self, force=False):
450+
"""Enables response compression with `deflate` encoding."""
451+
self._compression = True
452+
self._compression_force = force
453+
432454
@property
433455
def headers(self):
434456
return self._headers
@@ -557,6 +579,17 @@ def start(self, request):
557579

558580
self._copy_cookies()
559581

582+
if self._compression:
583+
if (self._compression_force or
584+
'deflate' in request.headers.get(
585+
hdrs.ACCEPT_ENCODING, '')):
586+
resp_impl.add_compression_filter()
587+
588+
if self._chunked:
589+
resp_impl.enable_chunked_encoding()
590+
if self._chunk_size:
591+
resp_impl.add_chunking_filter(self._chunk_size)
592+
560593
headers = self.headers.items()
561594
for key, val in headers:
562595
resp_impl.add_header(key, val)

aiohttp/wsgi.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from urllib.parse import urlsplit
1818

1919
import aiohttp
20-
from aiohttp import server, helpers
20+
from aiohttp import server, helpers, hdrs
2121

2222

2323
class WSGIServerHttpProtocol(server.ServerHttpProtocol):
@@ -174,6 +174,17 @@ class WsgiResponse:
174174

175175
status = None
176176

177+
HOP_HEADERS = {
178+
hdrs.CONNECTION,
179+
hdrs.KEEP_ALIVE,
180+
hdrs.PROXY_AUTHENTICATE,
181+
hdrs.PROXY_AUTHORIZATION,
182+
hdrs.TE,
183+
hdrs.TRAILER,
184+
hdrs.TRANSFER_ENCODING,
185+
hdrs.UPGRADE,
186+
}
187+
177188
def __init__(self, writer, message):
178189
self.writer = writer
179190
self.message = message
@@ -192,8 +203,12 @@ def start_response(self, status, headers, exc_info=None):
192203
resp = self.response = aiohttp.Response(
193204
self.writer, status_code,
194205
self.message.version, self.message.should_close)
206+
resp.HOP_HEADERS = self.HOP_HEADERS
195207
resp.add_headers(*headers)
196208

209+
if resp.has_chunked_hdr:
210+
resp.enable_chunked_encoding()
211+
197212
# send headers immediately for websocket connection
198213
if status_code == 101 and resp.upgrade and resp.websocket:
199214
resp.send_headers()

docs/web_reference.rst

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ first positional parameter.
250250

251251

252252
Response classes
253-
-----------------
253+
----------------
254254

255255
For now, :mod:`aiohttp.web` has two classes for the *HTTP response*:
256256
:class:`StreamResponse` and :class:`Response`.
@@ -337,6 +337,18 @@ StreamResponse
337337
Disable :attr:`keep_alive` for connection. There are no ways to
338338
enable it back.
339339

340+
.. attribute:: chunked
341+
342+
Read-only property, incicates if chunked encoding is on.
343+
344+
Can be enabled by :meth:`enable_chunked_encoding` call.
345+
346+
.. method:: enable_chunked_encoding
347+
348+
Enables :attr:`chunked` enacoding for response. There are no ways to
349+
disable it back. With enabled :attr:`chunked` encoding each `write()`
350+
operation encoded in separate chunk.
351+
340352
.. attribute:: headers
341353

342354
:class:`~aiohttp.multidict.CIMultiDict` instance

tests/test_http_protocol.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import asyncio
66
import zlib
77

8-
from aiohttp import protocol
8+
from aiohttp import hdrs, protocol
99

1010

1111
class HttpMessageTests(unittest.TestCase):
@@ -56,7 +56,7 @@ def test_force_close(self):
5656
def test_force_chunked(self):
5757
msg = protocol.Response(self.transport, 200)
5858
self.assertFalse(msg.chunked)
59-
msg.force_chunked()
59+
msg.enable_chunked_encoding()
6060
self.assertTrue(msg.chunked)
6161

6262
def test_keep_alive(self):
@@ -153,6 +153,7 @@ def test_add_headers_connection_keepalive(self):
153153

154154
def test_add_headers_hop_headers(self):
155155
msg = protocol.Response(self.transport, 200)
156+
msg.HOP_HEADERS = (hdrs.TRANSFER_ENCODING,)
156157

157158
msg.add_headers(('connection', 'test'), ('transfer-encoding', 't'))
158159
self.assertEqual([], list(msg.headers))
@@ -195,8 +196,8 @@ def test_default_headers_chunked(self):
195196
self.assertNotIn('TRANSFER-ENCODING', headers)
196197

197198
msg = protocol.Response(self.transport, 200)
198-
msg.force_chunked()
199-
msg._add_default_headers()
199+
msg.enable_chunked_encoding()
200+
msg.send_headers()
200201

201202
headers = [r for r, _ in msg.headers.items()]
202203
self.assertIn('TRANSFER-ENCODING', headers)
@@ -285,7 +286,7 @@ def test_prepare_length(self):
285286

286287
def test_prepare_chunked_force(self):
287288
msg = protocol.Response(self.transport, 200)
288-
msg.force_chunked()
289+
msg.enable_chunked_encoding()
289290

290291
chunked = msg._write_chunked_payload = unittest.mock.Mock()
291292
chunked.return_value = iter([1, 2, 3])
@@ -340,7 +341,7 @@ def test_write_payload_chunked(self):
340341
write = self.transport.write = unittest.mock.Mock()
341342

342343
msg = protocol.Response(self.transport, 200)
343-
msg.force_chunked()
344+
msg.enable_chunked_encoding()
344345
msg.send_headers()
345346

346347
msg.write(b'data')
@@ -355,7 +356,7 @@ def test_write_payload_chunked_multiple(self):
355356
write = self.transport.write = unittest.mock.Mock()
356357

357358
msg = protocol.Response(self.transport, 200)
358-
msg.force_chunked()
359+
msg.enable_chunked_encoding()
359360
msg.send_headers()
360361

361362
msg.write(b'data1')

0 commit comments

Comments
 (0)