Enable docker containers for TLS Interception (#375)

* Move static values to constants.py

* Install openssl in docker containers for TLS interception

* Complete TLS Interception with Docker doc

Author: abhinavsingh

Dockerfile

@@ -20,6 +20,9 @@ LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
 
 COPY --from=builder /deps /usr/local
 
+# Install openssl to enable TLS interception within container
+RUN apk update && apk add openssl
+
 EXPOSE 8899/tcp
 ENTRYPOINT [ "proxy" ]
 CMD [ "--hostname=0.0.0.0" ]

README.md

@@ -63,6 +63,7 @@ Table of Contents
     * [Plugin Ordering](#plugin-ordering)
 * [End-to-End Encryption](#end-to-end-encryption)
 * [TLS Interception](#tls-interception)
+    * [TLS Interception With Docker](#tls-interception-with-docker)
 * [Proxy Over SSH Tunnel](#proxy-over-ssh-tunnel)
     * [Proxy Remote Requests Locally](#proxy-remote-requests-locally)
     * [Proxy Local Requests Remotely](#proxy-local-requests-remotely)
@@ -862,9 +863,94 @@ cached file instead of plain text.
 Now use CA flags with other
 [plugin examples](#plugin-examples) to see them work with `https` traffic.
 
+## TLS Interception With Docker
+
+Important notes about TLS Interception with Docker container:
+
+- Since `v2.2.0`, `proxy.py` docker container also ships with `openssl`.  This allows `proxy.py`
+to generate certificates on the fly for TLS Interception.
+
+- For security reasons, `proxy.py` docker container doesn't ship with CA certificates.
+
+Here is how to start a `proxy.py` docker container
+with TLS Interception:
+
+1. Generate CA certificates on host computer
+
+    ```bash
+    ❯ make ca-certificates
+    ```
+
+2. Copy all generated certificates into a separate directory.  We'll later mount this directory into our docker container
+
+    ```bash
+    ❯ mkdir /tmp/ca-certificates
+    ❯ cp ca-cert.pem ca-key.pem ca-signing-key.pem /tmp/ca-certificates
+    ```
+
+3. Start docker container
+
+    ```bash
+    ❯ docker run -it --rm \
+        -v /tmp/ca-certificates:/tmp/ca-certificates \
+        -p 8899:8899 \
+        abhinavsingh/proxy.py:latest \
+        --hostname 0.0.0.0 \
+        --plugins proxy.plugin.CacheResponsesPlugin \
+        --ca-key-file /tmp/ca-certificates/ca-key.pem \
+        --ca-cert-file /tmp/ca-certificates/ca-cert.pem \
+        --ca-signing-key /tmp/ca-certificates/ca-signing-key.pem
+    ```
+
+    - `-v /tmp/ca-certificates:/tmp/ca-certificates` flag mounts our CA certificate directory in container environment
+    - `--plugins proxy.plugin.CacheResponsesPlugin` enables `CacheResponsesPlugin` so that we can inspect intercepted traffic
+    - `--ca-*` flags enable TLS Interception.
+
+4. From another terminal, try TLS Interception using `curl`. You can omit `--cacert` flag if CA certificate is already trusted by the system.
+
+    ```bash
+    ❯ curl -v \
+        --cacert ca-cert.pem \
+        -x 127.0.0.1:8899 \
+        https://httpbin.org/get
+    ```
+
+5. Verify `issuer` field from response headers.
+
+    ```bash
+    * Server certificate:
+    *  subject: CN=httpbin.org; C=NA; ST=Unavailable; L=Unavailable; O=Unavailable; OU=Unavailable
+    *  start date: Jun 17 09:26:57 2020 GMT
+    *  expire date: Jun 17 09:26:57 2022 GMT
+    *  subjectAltName: host "httpbin.org" matched cert's "httpbin.org"
+    *  issuer: CN=example.com
+    *  SSL certificate verify ok.
+    ```
+
+6. Back on docker terminal, copy response dump path logs.
+
+    ```bash
+    ...[redacted]... [I] access_log:338 - 172.17.0.1:56498 - CONNECT httpbin.org:443 - 1031 bytes - 1216.70 ms
+    ...[redacted]... [I] close:49 - Cached response at /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt
+    ```
+
+7. In another terminal, `cat` the response dump:
+
+    ```bash
+    ❯ docker exec -it $(docker ps | grep proxy.py | awk '{ print $1 }') cat /tmp/httpbin.org-ae1a927d064e4ab386ea319eb38fe251.txt
+    HTTP/1.1 200 OK
+    ...[redacted]...
+    {
+      ...[redacted]...,
+      "url": "http://httpbin.org/get"
+    }
+    ```
+
 Proxy Over SSH Tunnel
 =====================
 
+**This is a WIP and may not work as documented**
+
 Requires `paramiko` to work. See [requirements-tunnel.txt](https://github.com/abhinavsingh/proxy.py/blob/develop/requirements-tunnel.txt)
 
 ## Proxy Remote Requests Locally

proxy/common/constants.py

@@ -10,6 +10,7 @@
 """
 import os
 import time
+import pathlib
 import ipaddress
 
 from typing import List
@@ -75,3 +76,13 @@
 DEFAULT_VERSION = False
 DEFAULT_HTTP_PORT = 80
 DEFAULT_MAX_SEND_SIZE = 16 * 1024
+
+DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy')
+
+# Cor plugins enabled by default or via flags
+PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin'
+PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin'
+PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin'
+PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.DevtoolsProtocolPlugin'
+PLUGIN_DASHBOARD = 'proxy.dashboard.dashboard.ProxyDashboard'
+PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin'

proxy/common/flags.py

@@ -20,7 +20,6 @@
 import multiprocessing
 import sys
 import inspect
-import pathlib
 
 from typing import Optional, Union, Dict, List, TypeVar, Type, cast, Any, Tuple
 
@@ -33,7 +32,9 @@
 from .constants import DEFAULT_PAC_FILE_URL_PATH, DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT
 from .constants import DEFAULT_NUM_WORKERS, DEFAULT_VERSION, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_IPV6_HOSTNAME
 from .constants import DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CLIENT_RECVBUF_SIZE, DEFAULT_STATIC_SERVER_DIR
-from .constants import DEFAULT_ENABLE_DASHBOARD, COMMA, DOT
+from .constants import DEFAULT_ENABLE_DASHBOARD, DEFAULT_DATA_DIRECTORY_PATH, COMMA, DOT
+from .constants import PLUGIN_HTTP_PROXY, PLUGIN_WEB_SERVER, PLUGIN_PAC_FILE
+from .constants import PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_DASHBOARD, PLUGIN_INSPECT_TRAFFIC
 from .version import __version__
 
 __homepage__ = 'https://github.com/abhinavsingh/proxy.py'
@@ -49,9 +50,6 @@
 class Flags:
     """Contains all input flags and inferred input parameters."""
 
-    ROOT_DATA_DIR_NAME = '.proxy.py'
-    GENERATED_CERTS_DIR_NAME = 'certificates'
-
     def __init__(
             self,
             auth_code: Optional[bytes] = DEFAULT_BASIC_AUTH,
@@ -112,16 +110,25 @@ def __init__(
         self.devtools_ws_path: bytes = devtools_ws_path
         self.enable_events: bool = enable_events
 
-        self.proxy_py_data_dir = os.path.join(
-            str(pathlib.Path.home()), self.ROOT_DATA_DIR_NAME)
+        self.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH
         os.makedirs(self.proxy_py_data_dir, exist_ok=True)
 
         self.ca_cert_dir: Optional[str] = ca_cert_dir
         if self.ca_cert_dir is None:
             self.ca_cert_dir = os.path.join(
-                self.proxy_py_data_dir, self.GENERATED_CERTS_DIR_NAME)
+                self.proxy_py_data_dir, 'certificates')
             os.makedirs(self.ca_cert_dir, exist_ok=True)
 
+    def tls_interception_enabled(self) -> bool:
+        return self.ca_key_file is not None and \
+            self.ca_cert_dir is not None and \
+            self.ca_signing_key_file is not None and \
+            self.ca_cert_file is not None
+
+    def encryption_enabled(self) -> bool:
+        return self.keyfile is not None and \
+            self.certfile is not None
+
     @classmethod
     def initialize(
             cls: Type[T],
@@ -138,51 +145,59 @@ def initialize(
                 'A future version of pip will drop support for Python 2.7.')
             sys.exit(1)
 
-        args = Flags.init_parser().parse_args(input_args)
+        parser = Flags.init_parser()
+        args = parser.parse_args(input_args)
 
+        # Print version and exit
         if args.version:
             print(__version__)
             sys.exit(0)
 
-        if (args.cert_file and args.key_file) and \
-                (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file):
-            print('You can either enable end-to-end encryption OR TLS interception,'
-                  'not both together.')
-            sys.exit(1)
-
-        auth_code = None
-        if args.basic_auth:
-            auth_code = b'Basic %s' % base64.b64encode(bytes_(args.basic_auth))
-
+        # Setup logging module
         Flags.setup_logger(args.log_file, args.log_level, args.log_format)
-        Flags.set_open_file_limit(args.open_file_limit)
 
-        http_proxy_plugin = 'proxy.http.proxy.HttpProxyPlugin'
-        web_server_plugin = 'proxy.http.server.HttpWebServerPlugin'
-        pac_file_plugin = 'proxy.http.server.HttpWebServerPacFilePlugin'
-        devtools_protocol_plugin = 'proxy.http.inspector.DevtoolsProtocolPlugin'
-        dashboard_plugin = 'proxy.dashboard.dashboard.ProxyDashboard'
-        inspect_traffic_plugin = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin'
+        # Setup limits
+        Flags.set_open_file_limit(args.open_file_limit)
 
         default_plugins: List[Tuple[str, bool]] = []
         if args.enable_dashboard:
-            default_plugins.append((web_server_plugin, True))
+            default_plugins.append((PLUGIN_WEB_SERVER, True))
             args.enable_static_server = True
-            default_plugins.append((dashboard_plugin, True))
-            default_plugins.append((inspect_traffic_plugin, True))
+            default_plugins.append((PLUGIN_DASHBOARD, True))
+            default_plugins.append((PLUGIN_INSPECT_TRAFFIC, True))
             args.enable_events = True
             args.enable_devtools = True
         if args.enable_devtools:
-            default_plugins.append((devtools_protocol_plugin, True))
-            default_plugins.append((web_server_plugin, True))
+            default_plugins.append((PLUGIN_DEVTOOLS_PROTOCOL, True))
+            default_plugins.append((PLUGIN_WEB_SERVER, True))
         if not args.disable_http_proxy:
-            default_plugins.append((http_proxy_plugin, True))
+            default_plugins.append((PLUGIN_HTTP_PROXY, True))
         if args.enable_web_server or \
                 args.pac_file is not None or \
                 args.enable_static_server:
-            default_plugins.append((web_server_plugin, True))
+            default_plugins.append((PLUGIN_WEB_SERVER, True))
         if args.pac_file is not None:
-            default_plugins.append((pac_file_plugin, True))
+            default_plugins.append((PLUGIN_PAC_FILE, True))
+
+        plugins = Flags.load_plugins(
+            bytes_(
+                '%s,%s' %
+                (text_(COMMA).join(collections.OrderedDict(default_plugins).keys()),
+                    opts.get('plugins', args.plugins))))
+
+        # proxy.py currently cannot serve over HTTPS and perform TLS interception
+        # at the same time.  Check if user is trying to enable both feature
+        # at the same time.
+        if (args.cert_file and args.key_file) and \
+                (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file):
+            print('You can either enable end-to-end encryption OR TLS interception,'
+                  'not both together.')
+            sys.exit(1)
+
+        # Generate auth_code required for basic authentication if enabled
+        auth_code = None
+        if args.basic_auth:
+            auth_code = b'Basic %s' % base64.b64encode(bytes_(args.basic_auth))
 
         return cls(
             auth_code=cast(Optional[bytes], opts.get('auth_code', auth_code)),
@@ -258,23 +273,9 @@ def initialize(
                 opts.get(
                     'enable_events',
                     args.enable_events)),
-            plugins=Flags.load_plugins(
-                bytes_(
-                    '%s,%s' %
-                    (text_(COMMA).join(collections.OrderedDict(default_plugins).keys()),
-                     opts.get('plugins', args.plugins)))),
+            plugins=plugins,
             pid_file=cast(Optional[str], opts.get('pid_file', args.pid_file)))
 
-    def tls_interception_enabled(self) -> bool:
-        return self.ca_key_file is not None and \
-            self.ca_cert_dir is not None and \
-            self.ca_signing_key_file is not None and \
-            self.ca_cert_file is not None
-
-    def encryption_enabled(self) -> bool:
-        return self.keyfile is not None and \
-            self.certfile is not None
-
     @staticmethod
     def init_parser() -> argparse.ArgumentParser:
         """Initializes and returns argument parser."""