Merge pull request #12 from MobSF/bump

Migrate to mitmproxy 11.0.0

Author: ajinabraham

.github/workflows/python-tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ['3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v2

http_tools/modules/upstream.py

@@ -0,0 +1,40 @@
+"""Upstream proxy module."""
+
+import http_tools.settings as settings
+
+from mitmproxy import ctx
+from mitmproxy import http
+from mitmproxy.connection import Server
+from mitmproxy.net.server_spec import ServerSpec
+
+
+def load(loader):
+    loader.add_option(
+        name='proxy_ip',
+        typespec=str,
+        default='127.0.0.1',
+        help='Upstream Proxy IP',
+    )
+    loader.add_option(
+        name='proxy_port',
+        typespec=int,
+        default=8000,
+        help='Upstream Proxy Port',
+    )
+
+
+def request(flow: http.HTTPFlow) -> None:
+    if (flow.request.url.endswith('/kill')
+            and flow.request.method == 'GET'
+            and flow.request.port == settings.PROXY_PORT):
+        # Prevent killing the proxy server
+        flow.kill()
+
+    address = (ctx.options.proxy_ip, ctx.options.proxy_port)
+    # Check if the server connection already exists
+    if flow.server_conn.timestamp_start:
+        # Replace the existing server connection with a new one
+        flow.server_conn = Server(address=flow.server_conn.address)
+
+    # Set the upstream proxy (via) server
+    flow.server_conn.via = ServerSpec(('http', address))

http_tools/web/controllers/dashboard.py

@@ -4,6 +4,7 @@
 import os
 import glob
 from pathlib import PurePath
+from urllib.parse import urlparse
 import subprocess
 import threading
 
@@ -67,11 +68,21 @@ def post(self, project):
             self.write({'error': 'No requests found for the project'})
             return
         proxy = self.get_argument('proxy', default='http://127.0.0.1:8080')
+        parsed_url = urlparse(proxy)
         flow_file = os.path.join(settings.FLOWS_DIR, project + '.flow')
-        trd = threading.Thread(target=subprocess.call, args=(
-            ['mitmdump', '-k', '-n', '-m',
-             'upstream:{}'.format(proxy),
-             '--client-replay', flow_file],))
+        script_dir = os.path.join(settings.BASE_PATH, 'modules')
+        # mitmproxy 11.0.0 has issues with client replay and upstream proxy
+        # See: https://github.com/mitmproxy/mitmproxy/issues/7280
+        args = ['mitmdump',
+                '--scripts', os.path.join(script_dir, 'upstream.py'),
+                '--set', 'connection_strategy=lazy',
+                '--set', 'upstream_cert=false',
+                '--set', f'proxy_ip={parsed_url.hostname}',
+                '--set', f'proxy_port={parsed_url.port}',
+                '--ssl-insecure',
+                '--no-server',
+                '--client-replay', flow_file]
+        trd = threading.Thread(target=subprocess.call, args=(args,))
         trd.setDaemon(True)
         trd.start()
         self.write({'success': 'Repeating request to upstream'})

setup.py

@@ -17,7 +17,7 @@ def read(rel_path):
                '[mitmproxy](https://mitmproxy.org/)')
 setup(
     name='http-tools',
-    version='4.0.0',
+    version='5.0.0',
     description=description,
     author='Ajin Abraham',
     author_email='[email protected]',
@@ -42,7 +42,7 @@ def read(rel_path):
     long_description=read('README.md'),
     long_description_content_type='text/markdown',
     install_requires=[
-        'mitmproxy==10.1.5',
-        'markupsafe>=2.1.3',
+        'mitmproxy==11.0.0',
+        'markupsafe>=3.0.2',
     ],
 )

tox.ini

@@ -33,7 +33,7 @@ skip_install = true
 deps =
     bandit
 commands =
-    bandit libsast -r
+    bandit http_tools -r -ll -iii
 
 [testenv:publish]
 skip_install = true
@@ -91,4 +91,6 @@ ignore =
     # Use python sort imports
     SF01,
     # Allow Private member access
+    W503,
+    # Allow line break before binary operator
 radon_max_cc = 10