Merge branch 'development' into add_proxy_support

Author: fredthomsen

.travis/install.sh

@@ -22,12 +22,15 @@ if [[ "$NGHTTP2" = true ]]; then
     # Now, download and install nghttp2's latest version.
     git clone https://github.com/tatsuhiro-t/nghttp2.git
     cd nghttp2
+    DIR=`pwd`
+    export PYTHONPATH="$DIR/lib/python${TRAVIS_PYTHON_VERSION}/site-packages"
+    mkdir -p $PYTHONPATH
     autoreconf -i
     automake
     autoconf
-    ./configure --disable-threads
+    ./configure --disable-threads --prefix=`pwd`
     make
-    sudo make install
+    make install
 
     # The makefile doesn't install into the active virtualenv. Install again.
     cd python

CONTRIBUTORS.rst

@@ -29,6 +29,9 @@ In chronological order:
 - Jerome De Cuyper (@jdecuyper)
 
   - Updated documentation and tests.
+  - Added support for user-provided SSLContext objects.
+  - Improved support for HTTP/2 error codes.
+  - Added support for graceful connection closure.
 
 - Fred Thomsen (@fredthomsen)
 

HISTORY.rst

@@ -4,13 +4,53 @@ Release History
 dev
 ---
 
+*Feature Enhancement*
+
+- Pay attention to max frame length changes from remote peers. Thanks to
+  @jdecuyper!
+
+*Bugfixes*
+
+- Prevent hyper from emitting oversized frames. Thanks to @jdecuyper!
+- Prevent hyper from emitting RST_STREAM frames whenever it finishes consuming
+  a stream.
+- Prevent hyper from emitting lots of RST_STREAM frames.
+
+*Software Updates*
+
+- Updated hyperframe to version 1.1.1.
+
+0.4.0 (2015-06-21)
+------------------
+
 *New Features*
 
-- Support for upgrading plaintext HTTP/1.1 to plaintext HTTP/2. (`Issue 28`_)
+- HTTP/1.1 and HTTP/2 abstraction layer. Don't specify what version you want to
+  use, just automatically get the best version the server supports!
+- Support for upgrading plaintext HTTP/1.1 to plaintext HTTP/2, with thanks to
+  @fredthomsen! (`Issue #28`_)
 - ``HTTP11Connection`` and ``HTTPConnection`` objects are now both context
   managers.
+- Added support for ALPN negotiation when using PyOpenSSL. (`Issue #31`_)
+- Added support for user-provided SSLContext objects, with thanks to
+  @jdecuyper! (`Issue #8`_)
+- Better support for HTTP/2 error codes, with thanks to @jdecuyper!
+  (`Issue #119`_)
+- More gracefully close connections, with thanks to @jdecuyper! (`Issue #15`_)
+
+*Structural Changes*
+
+- The framing and HPACK layers were stripped out into their own libraries.
+
+*Bugfixes*
+
+- Properly verify hostnames when using PyOpenSSL.
 
+.. _Issue #8: https://github.com/Lukasa/hyper/issues/8
+.. _Issue #15: https://github.com/Lukasa/hyper/issues/15
 .. _Issue #28: https://github.com/Lukasa/hyper/issues/28
+.. _Issue #31: https://github.com/Lukasa/hyper/issues/31
+.. _Issue #119: https://github.com/Lukasa/hyper/issues/119
 
 0.3.1 (2015-04-03)
 ------------------

Makefile

@@ -1,7 +1,7 @@
 .PHONY: certs publish test
 
 certs:
-	curl http://ci.kennethreitz.org/job/ca-bundle/lastSuccessfulBuild/artifact/cacerts.pem -o hyper/certs.pem
+	curl https://mkcert.org/generate/ -o hyper/certs.pem
 
 publish:
 	python setup.py sdist upload

docs/source/conf.py

@@ -55,9 +55,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.3.1'
+version = '0.4.0'
 # The full version, including alpha/beta/rc tags.
-release = '0.3.1'
+release = '0.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

hyper/__init__.py

@@ -6,7 +6,7 @@
 A module for providing an abstraction layer over the differences between
 HTTP/1.1 and HTTP/2.
 """
-__version__ = '0.3.1'
+__version__ = '0.4.0'
 
 from .common.connection import HTTPConnection
 from .http20.connection import HTTP20Connection

hyper/http20/connection.py

@@ -12,7 +12,7 @@
 from ..packages.hyperframe.frame import (
     FRAMES, DataFrame, HeadersFrame, PushPromiseFrame, RstStreamFrame,
     SettingsFrame, Frame, WindowUpdateFrame, GoAwayFrame, PingFrame,
-    BlockedFrame
+    BlockedFrame, FRAME_MAX_LEN, FRAME_MAX_ALLOWED_LEN
 )
 from ..packages.hpack.hpack_compat import Encoder, Decoder
 from .stream import Stream
@@ -113,9 +113,13 @@ def __init_state(self):
         # Streams are stored in a dictionary keyed off their stream IDs. We
         # also save the most recent one for easy access without having to walk
         # the dictionary.
+        # Finally, we add a set of all streams that we or the remote party
+        # forcefully closed with RST_STREAM, to avoid encountering issues where
+        # frames were already in flight before the RST was processed.
         self.streams = {}
         self.recent_stream = None
         self.next_stream_id = 1
+        self.reset_streams = set()
 
         # Header encoding/decoding is at the connection scope, so we embed a
         # header encoder and a decoder. These get passed to child stream
@@ -126,6 +130,7 @@ def __init_state(self):
         # Values for the settings used on an HTTP/2 connection.
         self._settings = {
             SettingsFrame.INITIAL_WINDOW_SIZE: 65535,
+            SettingsFrame.SETTINGS_MAX_FRAME_SIZE: FRAME_MAX_LEN,
         }
 
         # The socket used to send data.
@@ -250,16 +255,17 @@ def _send_preamble(self):
         # The server will also send an initial settings frame, so get it.
         self._recv_cb()
 
-    def close(self):
+    def close(self, error_code=None):
         """
         Close the connection to the server.
 
+        :param error_code: (optional) The error code to reset all streams with.
         :returns: Nothing.
         """
         # Close all streams
         for stream in list(self.streams.values()):
             log.debug("Close stream %d" % stream.stream_id)
-            stream.close()
+            stream.close(error_code)
 
         # Send GoAway frame to the server
         try:
@@ -391,10 +397,14 @@ def receive_frame(self, frame):
             if 'ACK' not in frame.flags:
                 self._update_settings(frame)
 
-                # Need to return an ack.
-                f = SettingsFrame(0)
-                f.flags.add('ACK')
-                self._send_cb(f)
+                # When the setting containing the max frame size value is out
+                # of range, the spec dictates to tear down the connection.
+                # Therefore we make sure the socket is still alive before
+                # returning the ack.
+                if self._sock is not None:
+                    f = SettingsFrame(0)
+                    f.flags.add('ACK')
+                    self._send_cb(f)
         elif frame.type == GoAwayFrame.type:
             # If we get GoAway with error code zero, we are doing a graceful
             # shutdown and all is well. Otherwise, throw an exception.
@@ -404,13 +414,19 @@ def receive_frame(self, frame):
             # code registry otherwise use the frame's additional data.
             if frame.error_code != 0:
                 try:
-                    name, number, description = errors.get_data(frame.error_code)
+                    name, number, description = errors.get_data(
+                        frame.error_code
+                    )
                 except ValueError:
-                    error_string = ("Encountered error code %d, extra data %s" %
-                                   (frame.error_code, frame.additional_data))
+                    error_string = (
+                        "Encountered error code %d, extra data %s" %
+                        (frame.error_code, frame.additional_data)
+                    )
                 else:
-                    error_string = ("Encountered error %s %s: %s" %
-                                   (name, number, description))
+                    error_string = (
+                        "Encountered error %s %s: %s" %
+                        (name, number, description)
+                    )
 
                 raise ConnectionError(error_string)
 
@@ -452,6 +468,25 @@ def _update_settings(self, frame):
 
             self._settings[SettingsFrame.INITIAL_WINDOW_SIZE] = newsize
 
+        if SettingsFrame.SETTINGS_MAX_FRAME_SIZE in frame.settings:
+            new_size = frame.settings[SettingsFrame.SETTINGS_MAX_FRAME_SIZE]
+            if FRAME_MAX_LEN <= new_size <= FRAME_MAX_ALLOWED_LEN:
+                self._settings[SettingsFrame.SETTINGS_MAX_FRAME_SIZE] = (
+                    new_size
+                )
+            else:
+                log.warning(
+                    "Frame size %d is outside of allowed range",
+                    new_size
+                )
+
+                # Tear the connection down with error code PROTOCOL_ERROR
+                self.close(1)
+                error_string = (
+                    "Advertised frame size %d is outside of range" % (new_size)
+                )
+                raise ConnectionError(error_string)
+
     def _new_stream(self, stream_id=None, local_closed=False):
         """
         Returns a new stream object for this connection.
@@ -472,12 +507,18 @@ def _close_stream(self, stream_id, error_code=None):
         """
         Called by a stream when it would like to be 'closed'.
         """
-        if error_code is not None:
-            f = RstStreamFrame(stream_id)
-            f.error_code = error_code
-            self._send_cb(f)
-
-        del self.streams[stream_id]
+        # Graceful shutdown of streams involves not emitting an error code
+        # at all.
+        if error_code:
+            self._send_rst_frame(stream_id, error_code)
+        else:
+            # Just delete the stream.
+            try:
+                del self.streams[stream_id]
+            except KeyError as e:  # pragma: no cover
+                log.warn(
+                    "Stream with id %d does not exist: %s",
+                    stream_id, e)
 
     def _send_cb(self, frame, tolerate_peer_gone=False):
         """
@@ -498,6 +539,14 @@ def _send_cb(self, frame, tolerate_peer_gone=False):
 
         data = frame.serialize()
 
+        max_frame_size = self._settings[SettingsFrame.SETTINGS_MAX_FRAME_SIZE]
+        if frame.body_len > max_frame_size:
+            raise ValueError(
+                     "Frame size %d exceeds maximum frame size setting %d" %
+                     (frame.body_len,
+                      self._settings[SettingsFrame.SETTINGS_MAX_FRAME_SIZE])
+            )
+
         log.info(
             "Sending frame %s on stream %d",
             frame.__class__.__name__,
@@ -507,7 +556,8 @@ def _send_cb(self, frame, tolerate_peer_gone=False):
         try:
             self._sock.send(data)
         except socket.error as e:
-            if not tolerate_peer_gone or e.errno not in (errno.EPIPE, errno.ECONNRESET):
+            if (not tolerate_peer_gone or
+                e.errno not in (errno.EPIPE, errno.ECONNRESET)):
                 raise
 
     def _adjust_receive_window(self, frame_len):
@@ -538,6 +588,15 @@ def _consume_single_frame(self):
         # Parse the header. We can use the returned memoryview directly here.
         frame, length = Frame.parse_frame_header(header)
 
+        if (length > FRAME_MAX_LEN):
+            log.warning(
+                "Frame size exceeded on stream %d (received: %d, max: %d)",
+                frame.stream_id,
+                length,
+                FRAME_MAX_LEN
+            )
+            self._send_rst_frame(frame.stream_id, 6) # 6 = FRAME_SIZE_ERROR
+
         # Read the remaining data from the socket.
         data = self._recv_payload(length)
         self._consume_frame_payload(frame, data)
@@ -598,9 +657,14 @@ def _consume_frame_payload(self, frame, data):
                 # the ENABLE_PUSH setting is 0, but the spec leaves the client
                 # action undefined when they do it anyway. So we just refuse
                 # the stream and go about our business.
-                f = RstStreamFrame(frame.promised_stream_id)
-                f.error_code = 7 # REFUSED_STREAM
-                self._send_cb(f)
+                self._send_rst_frame(frame.promised_stream_id, 7)
+
+        # If this frame was received on a stream that has been reset, drop it.
+        if frame.stream_id in self.reset_streams:
+            log.info(
+                "Stream %s has been reset, dropping frame.", frame.stream_id
+            )
+            return
 
         # Work out to whom this frame should go.
         if frame.stream_id != 0:
@@ -609,12 +673,15 @@ def _consume_frame_payload(self, frame, data):
             except KeyError:
                 # If we receive an unexpected stream identifier then we
                 # cancel the stream with an error of type PROTOCOL_ERROR
-                f = RstStreamFrame(frame.stream_id)
-                f.error_code = 1 # PROTOCOL_ERROR
-                self._send_cb(f)
+                self._send_rst_frame(frame.stream_id, 1)
                 log.warning(
                     "Unexpected stream identifier %d" % (frame.stream_id)
                 )
+
+            # If this is a RST_STREAM frame, we may get more than one (because
+            # of frames in flight). Keep track.
+            if frame.type == RstStreamFrame.type:
+                self.reset_streams.add(frame.stream_id)
         else:
             self.receive_frame(frame)
 
@@ -640,6 +707,24 @@ def _recv_cb(self):
             except ConnectionResetError:
                 break
 
+    def _send_rst_frame(self, stream_id, error_code):
+        """
+        Send reset stream frame with error code and remove stream from map.
+        """
+        f = RstStreamFrame(stream_id)
+        f.error_code = error_code
+        self._send_cb(f)
+
+        try:
+            del self.streams[stream_id]
+        except KeyError as e:  # pragma: no cover
+            log.warn(
+                "Stream with id %d does not exist: %s",
+                stream_id, e)
+
+        # Keep track of the fact that we reset this stream in case there are
+        # other frames in flight.
+        self.reset_streams.add(stream_id)
 
     # The following two methods are the implementation of the context manager
     # protocol.

hyper/http20/exceptions.py

@@ -40,3 +40,10 @@ class ProtocolError(HTTP20Error):
     The remote party violated the HTTP/2 protocol.
     """
     pass
+
+
+class StreamResetError(HTTP20Error):
+    """
+    A stream was forcefully reset by the remote party.
+    """
+    pass