Support python interception for http.client

Author: pimterry

overrides/pythonpath/http/__init__.py

@@ -0,0 +1,8 @@
+# This module _must_ trigger the import for http.client. We need to ensure we preload
+# the real http module (or import http & import http.server don't work properly), but
+# if we do that before importing the client, the client can never be intercepted.
+
+# There might be a cleaner alternative, but for now we just aggressively pre-intercept
+# the client instead.
+
+from . import client

overrides/pythonpath/http/client.py

@@ -0,0 +1,48 @@
+from httptoolkit_intercept import preload_real_module
+
+preload_real_module('http', 'http.client')
+
+import http.client, os, functools
+
+# Re-export all public fields
+from http.client import *
+# Load a few extra notable private fields, for max compatibility
+from http.client import __file__, __doc__
+
+_httpProxy = os.environ['HTTP_PROXY']
+[_proxyHost, _proxyPort] = _httpProxy.split('://')[1].split(':')
+_certPath = os.environ['SSL_CERT_FILE']
+
+# Redirect and then tunnel all plain HTTP connections:
+_http_connection_init = HTTPConnection.__init__
+@functools.wraps(_http_connection_init)
+def _new_http_connection_init(self, host, port=80, *k, **kw):
+    _http_connection_init(self, _proxyHost, _proxyPort, *k, **kw)
+    self.set_tunnel(host, port)
+HTTPConnection.__init__ = _new_http_connection_init
+
+def _build_default_context():
+    import ssl
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.options |= ssl.OP_NO_SSLv2
+    context.options |= ssl.OP_NO_SSLv3
+    return context
+
+# Redirect & tunnel HTTPS connections, and inject our CA certificate:
+_https_connection_init = HTTPSConnection.__init__
+@functools.wraps(_https_connection_init)
+def _new_https_connection_init(self, host, port=443, *k, **kw):
+    context = None
+    if 'context' in kw:
+        context = kw.get('context')
+    elif len(k) > 7:
+        context = k[7]
+
+    if context == None:
+        context = kw['context'] = _build_default_context()
+
+    context.load_verify_locations(_certPath)
+
+    _https_connection_init(self, _proxyHost, _proxyPort, *k, **kw)
+    self.set_tunnel(host, port)
+HTTPSConnection.__init__ = _new_https_connection_init

overrides/pythonpath/httplib.py

@@ -13,6 +13,7 @@
 [_proxyHost, _proxyPort] = _httpProxy.split('://')[1].split(':')
 _certPath = os.environ['SSL_CERT_FILE']
 
+# Redirect and then tunnel all plain HTTP connections:
 _http_connection_init = HTTPConnection.__init__
 @functools.wraps(_http_connection_init)
 def _new_http_connection_init(self, host, port=80, *k, **kw):
@@ -27,6 +28,7 @@ def _build_default_context():
     context.options |= ssl.OP_NO_SSLv3
     return context
 
+# Redirect & tunnel HTTPS connections, and inject our CA certificate:
 _https_connection_init = HTTPSConnection.__init__
 @functools.wraps(_https_connection_init)
 def _new_https_connection_init(self, host, port=443, *k, **kw):

overrides/pythonpath/httptoolkit_intercept.py

@@ -6,20 +6,19 @@
     import imp
     importlib = imp
 
-def preload_real_module(moduleName):
+def preload_real_module(*module_names):
     # Re-importing the real module at the top level of an override fails after deleting it from
-    # sys.modules['httplib'] in Python 2. Some interesting issues there ofc, but doing this 
+    # sys.modules['httplib'] in Python 2. Some interesting issues there ofc, but doing this
     # instead works nicely in both Python 2 & 3.
     import sys, os
 
     override_path = os.path.dirname(os.path.abspath(__file__))
-    # print('override path %s' % override_path)
     original_sys_path = list(sys.path)
-    # print('sys_path %s' % original_sys_path)
     sys.path = [p for p in sys.path if p != override_path and p != '']
-    # print('new sys_path %s' % sys.path)
 
-    del sys.modules[moduleName]
-    __import__(moduleName)
+    for mod in module_names:
+        if mod in sys.modules:
+            del sys.modules[mod]
+        __import__(mod)
 
     sys.path = original_sys_path