TLS Interception Cert Generation (#362)

* Use common.pki for interception certificate generation

* Fix tests

* Dont use certificate fields that we dont need, it leads to certificate generation error on Ubuntu

* Prepare for v2.2.0

* npm audit fix

Author: abhinavsingh

README.md

@@ -795,8 +795,7 @@ response from the server.  Start `proxy.py` as:
 ```
 
 
-> :note: **MacOS users** also need to pass explicit CA file path
-> needed for validation of peer certificates. See --ca-file flag.
+[![NOTE](https://img.shields.io/static/v1?label=MacOS&message=note&color=yellow)](https://github.com/abhinavsingh/proxy.py#flags) Also provide explicit CA bundle path needed for validation of peer certificates. See `--ca-file` flag.
 
 
 Verify TLS interception using `curl`
@@ -1327,7 +1326,7 @@ usage: pki.py [-h] [--password PASSWORD] [--private-key-path PRIVATE_KEY_PATH]
               [--public-key-path PUBLIC_KEY_PATH] [--subject SUBJECT]
               action
 
-proxy.py v2.1.2 : PKI Utility
+proxy.py v2.2.0 : PKI Utility
 
 positional arguments:
   action                Valid actions: remove_passphrase, gen_private_key,
@@ -1518,7 +1517,7 @@ usage: proxy [-h] [--backlog BACKLOG] [--basic-auth BASIC_AUTH]
              [--static-server-dir STATIC_SERVER_DIR] [--threadless]
              [--timeout TIMEOUT] [--version]
 
-proxy.py v2.1.2
+proxy.py v2.2.0
 
 optional arguments:
   -h, --help            show this help message and exit

dashboard/package-lock.json

@@ -37,7 +37,7 @@
     "eslint-plugin-node": "^10.0.0",
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
-    "http-server": "^0.12.1",
+    "http-server": "^0.12.3",
     "jasmine": "^3.5.0",
     "jasmine-ts": "^0.3.0",
     "jquery": "^3.5.0",

dashboard/package.json

@@ -5,7 +5,7 @@ class Proxy < Formula
     Network monitoring, controls & Application development, testing, debugging."
   homepage "https://github.com/abhinavsingh/proxy.py"
   url "https://github.com/abhinavsingh/proxy.py/archive/master.zip"
-  version "2.0.0"
+  version "2.1.2"
 
   depends_on "python"
 

helper/homebrew/stable/proxy.rb

@@ -8,5 +8,5 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-VERSION = (2, 1, 2)
+VERSION = (2, 2, 0)
 __version__ = '.'.join(map(str, VERSION[0:3]))

proxy/common/version.py

@@ -8,14 +8,13 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import logging
 import threading
-import subprocess
 import os
 import ssl
 import socket
 import time
 import errno
-import logging
 from typing import Optional, List, Union, Dict, cast, Any, Tuple
 
 from .plugin import HttpProxyBasePlugin
@@ -28,6 +27,7 @@
 from ...common.types import HasFileno
 from ...common.constants import PROXY_AGENT_HEADER_VALUE
 from ...common.utils import build_http_response, text_
+from ...common.pki import gen_public_key, gen_csr, sign_csr
 
 from ...core.event import eventNames
 from ...core.connection import TcpServerConnection, TcpConnectionUninitializedException
@@ -279,7 +279,8 @@ def on_request_complete(self) -> Union[socket.socket, bool]:
                         'BrokenPipeError when wrapping client')
                     return True
                 except OSError as e:
-                    logger.exception('OSError when wrapping client', exc_info=e)
+                    logger.exception(
+                        'OSError when wrapping client', exc_info=e)
                     return True
                 # Update all plugin connection reference
                 for plugin in self.plugins.values():
@@ -342,6 +343,57 @@ def access_log(self) -> None:
                  self.response.total_size,
                  connection_time_ms))
 
+    def gen_ca_signed_certificate(self, cert_file_path: str) -> None:
+        '''CA signing key (default) is used for generating a public key
+        for common_name, if one already doesn't exist.  Using generated
+        public key a CSR request is generated, which is then signed by
+        CA key and secret.  Again this process only happen if signed
+        certificate doesn't already exist.
+
+        returns signed certificate path.'''
+        assert(self.request.host and self.flags.ca_cert_dir and self.flags.ca_signing_key_file and
+               self.flags.ca_key_file and self.flags.ca_cert_file)
+        public_key_path = os.path.join(self.flags.ca_cert_dir,
+                                       '{0}.{1}'.format(text_(self.request.host), 'pub'))
+        private_key_path = self.flags.ca_signing_key_file
+        private_key_password = ''
+        subject = '/CN={0}'.format(text_(self.request.host))
+        alt_subj_names = [text_(self.request.host), ]
+        validity_in_days = 365 * 2
+        timeout = 10
+
+        # Generate a public key for the common name
+        if not os.path.isfile(public_key_path):
+            logger.debug('Generating public key %s', public_key_path)
+            resp = gen_public_key(public_key_path=public_key_path, private_key_path=private_key_path,
+                                  private_key_password=private_key_password, subject=subject, alt_subj_names=alt_subj_names,
+                                  validity_in_days=validity_in_days, timeout=timeout)
+            assert(resp is True)
+
+        csr_path = os.path.join(self.flags.ca_cert_dir,
+                                '{0}.{1}'.format(text_(self.request.host), 'csr'))
+
+        # Generate a CSR request for this common name
+        if not os.path.isfile(csr_path):
+            logger.debug('Generating CSR %s', csr_path)
+            resp = gen_csr(csr_path=csr_path, key_path=private_key_path, password=private_key_password,
+                           crt_path=public_key_path, timeout=timeout)
+            assert(resp is True)
+
+        ca_key_path = self.flags.ca_key_file
+        ca_key_password = ''
+        ca_crt_path = self.flags.ca_cert_file
+        serial = self.uid.int
+
+        # Sign generated CSR
+        if not os.path.isfile(cert_file_path):
+            logger.debug('Signing CSR %s', cert_file_path)
+            resp = sign_csr(csr_path=csr_path, crt_path=cert_file_path, ca_key_path=ca_key_path,
+                            ca_key_password=ca_key_password, ca_crt_path=ca_crt_path,
+                            serial=str(serial), alt_subj_names=alt_subj_names,
+                            validity_in_days=validity_in_days, timeout=timeout)
+            assert(resp is True)
+
     @staticmethod
     def generated_cert_file_path(ca_cert_dir: str, host: str) -> str:
         return os.path.join(ca_cert_dir, '%s.pem' % host)
@@ -359,21 +411,7 @@ def generate_upstream_certificate(
             self.flags.ca_cert_dir, text_(self.request.host))
         with self.lock:
             if not os.path.isfile(cert_file_path):
-                logger.debug('Generating certificates %s', cert_file_path)
-                # TODO: Parse subject from certificate
-                # Currently we only set CN= field for generated certificates.
-                gen_cert = subprocess.Popen(
-                    ['openssl', 'req', '-new', '-key', self.flags.ca_signing_key_file, '-subj',
-                     f'/C=/ST=/L=/O=/OU=/CN={ text_(self.request.host) }'],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-                sign_cert = subprocess.Popen(
-                    ['openssl', 'x509', '-req', '-days', '365', '-CA', self.flags.ca_cert_file, '-CAkey',
-                     self.flags.ca_key_file, '-set_serial', str(self.uid.int), '-out', cert_file_path],
-                    stdin=gen_cert.stdout,
-                    stderr=subprocess.PIPE)
-                # TODO: Ensure sign_cert success.
-                sign_cert.communicate(timeout=10)
+                self.gen_ca_signed_certificate(cert_file_path)
         return cert_file_path
 
     def wrap_server(self) -> None:

proxy/http/proxy/server.py

@@ -10,7 +10,7 @@
 """
 from setuptools import setup, find_packages
 
-VERSION = (2, 1, 2)
+VERSION = (2, 2, 0)
 __version__ = '.'.join(map(str, VERSION[0:3]))
 __description__ = '''⚡⚡⚡Fast, Lightweight, Pluggable, TLS interception capable proxy server
     focused on Network monitoring, controls & Application development, testing, debugging.'''

setup.py

@@ -30,14 +30,18 @@ class TestHttpProxyTlsInterception(unittest.TestCase):
     @mock.patch('ssl.wrap_socket')
     @mock.patch('ssl.create_default_context')
     @mock.patch('proxy.http.proxy.server.TcpServerConnection')
-    @mock.patch('subprocess.Popen')
+    @mock.patch('proxy.http.proxy.server.gen_public_key')
+    @mock.patch('proxy.http.proxy.server.gen_csr')
+    @mock.patch('proxy.http.proxy.server.sign_csr')
     @mock.patch('selectors.DefaultSelector')
     @mock.patch('socket.fromfd')
     def test_e2e(
             self,
             mock_fromfd: mock.Mock,
             mock_selector: mock.Mock,
-            mock_popen: mock.Mock,
+            mock_sign_csr: mock.Mock,
+            mock_gen_csr: mock.Mock,
+            mock_gen_public_key: mock.Mock,
             mock_server_conn: mock.Mock,
             mock_ssl_context: mock.Mock,
             mock_ssl_wrap: mock.Mock) -> None:
@@ -46,11 +50,17 @@ def test_e2e(
 
         self.mock_fromfd = mock_fromfd
         self.mock_selector = mock_selector
-        self.mock_popen = mock_popen
+        self.mock_sign_csr = mock_sign_csr
+        self.mock_gen_csr = mock_gen_csr
+        self.mock_gen_public_key = mock_gen_public_key
         self.mock_server_conn = mock_server_conn
         self.mock_ssl_context = mock_ssl_context
         self.mock_ssl_wrap = mock_ssl_wrap
 
+        self.mock_sign_csr.return_value = True
+        self.mock_gen_csr.return_value = True
+        self.mock_gen_public_key.return_value = True
+
         ssl_connection = mock.MagicMock(spec=ssl.SSLSocket)
         self.mock_ssl_context.return_value.wrap_socket.return_value = ssl_connection
         self.mock_ssl_wrap.return_value = mock.MagicMock(spec=ssl.SSLSocket)
@@ -118,6 +128,7 @@ def mock_connection() -> Any:
                 fd=self._conn.fileno,
                 events=selectors.EVENT_READ,
                 data=None), selectors.EVENT_READ)], ]
+
         self.protocol_handler.run_once()
 
         # Assert our mocked plugins invocations
@@ -142,8 +153,9 @@ def mock_connection() -> Any:
         self.assertEqual(plain_connection.setblocking.call_count, 2)
         self.mock_ssl_context.return_value.wrap_socket.assert_called_with(
             plain_connection, server_hostname=host)
-        # TODO: Assert Popen arguments, piping, success condition
-        self.assertEqual(self.mock_popen.call_count, 2)
+        self.assertEqual(self.mock_sign_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_public_key.call_count, 1)
         self.assertEqual(ssl_connection.setblocking.call_count, 1)
         self.assertEqual(
             self.mock_server_conn.return_value._conn,

tests/http/test_http_proxy_tls_interception.py

@@ -33,23 +33,33 @@ class TestHttpProxyPluginExamplesWithTlsInterception(unittest.TestCase):
     @mock.patch('ssl.wrap_socket')
     @mock.patch('ssl.create_default_context')
     @mock.patch('proxy.http.proxy.server.TcpServerConnection')
-    @mock.patch('subprocess.Popen')
+    @mock.patch('proxy.http.proxy.server.gen_public_key')
+    @mock.patch('proxy.http.proxy.server.gen_csr')
+    @mock.patch('proxy.http.proxy.server.sign_csr')
     @mock.patch('selectors.DefaultSelector')
     @mock.patch('socket.fromfd')
     def setUp(self,
               mock_fromfd: mock.Mock,
               mock_selector: mock.Mock,
-              mock_popen: mock.Mock,
+              mock_sign_csr: mock.Mock,
+              mock_gen_csr: mock.Mock,
+              mock_gen_public_key: mock.Mock,
               mock_server_conn: mock.Mock,
               mock_ssl_context: mock.Mock,
               mock_ssl_wrap: mock.Mock) -> None:
         self.mock_fromfd = mock_fromfd
         self.mock_selector = mock_selector
-        self.mock_popen = mock_popen
+        self.mock_sign_csr = mock_sign_csr
+        self.mock_gen_csr = mock_gen_csr
+        self.mock_gen_public_key = mock_gen_public_key
         self.mock_server_conn = mock_server_conn
         self.mock_ssl_context = mock_ssl_context
         self.mock_ssl_wrap = mock_ssl_wrap
 
+        self.mock_sign_csr.return_value = True
+        self.mock_gen_csr.return_value = True
+        self.mock_gen_public_key.return_value = True
+
         self.fileno = 10
         self._addr = ('127.0.0.1', 54382)
         self.flags = Flags(
@@ -126,7 +136,10 @@ def send(raw: bytes) -> int:
         )
         self.protocol_handler.run_once()
 
-        self.mock_popen.assert_called()
+        self.assertEqual(self.mock_sign_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_csr.call_count, 1)
+        self.assertEqual(self.mock_gen_public_key.call_count, 1)
+
         self.mock_server_conn.assert_called_once_with('uni.corn', 443)
         self.server.connect.assert_called()
         self.assertEqual(