Proxy fixes

docker/docker-compose.yml

@@ -73,6 +73,12 @@ services:
       - POSTGRES_HOST=postgres
       - POSTGRES_PORT=5432
       - MOBSF_ASYNC_ANALYSIS=1
+    #Proxy options - use it if you behind proxy
+      #- MOBSF_UPSTREAM_PROXY_ENABLED=True  
+      #- MOBSF_UPSTREAM_PROXY_SSL_VERIFY=True
+      #- MOBSF_UPSTREAM_PROXY_TYPE=http
+      #- MOBSF_UPSTREAM_PROXY_IP=host.docker.internal
+      #- MOBSF_UPSTREAM_PROXY_PORT=8080
     healthcheck:
       test: curl -f http://localhost:8000/login/ || exit 1
       interval: 30s

docker/docker-compose_swarm.yml

@@ -29,6 +29,12 @@ services:
       - POSTGRES_HOST=postgres
       - POSTGRES_PORT=5432
       - MOBSF_API_KEY_FILE=/run/secrets/mobsf_api_key
+      #Proxy options - use it if you behind proxy
+      #- MOBSF_UPSTREAM_PROXY_ENABLED=True  
+      #- MOBSF_UPSTREAM_PROXY_SSL_VERIFY=True
+      #- MOBSF_UPSTREAM_PROXY_TYPE=http
+      #- MOBSF_UPSTREAM_PROXY_IP=host.docker.internal
+      #- MOBSF_UPSTREAM_PROXY_PORT=8080
     healthcheck:
       test: curl -f http://localhost:8000/login/ || exit 1
     depends_on:

mobsf/DynamicAnalyzer/tools/apk_patcher.py

@@ -10,11 +10,11 @@
 
 from django.conf import settings
 
+from mobsf.MobSF.proxy import upstream_proxy
 from mobsf.MobSF.utils import (
     find_java_binary,
     is_file_exists,
     is_internet_available,
-    upstream_proxy,
 )
 
 

mobsf/DynamicAnalyzer/tools/webproxy.py

@@ -8,7 +8,7 @@
 
 from django.conf import settings
 
-from mobsf.MobSF.utils import upstream_proxy
+from mobsf.MobSF.proxy import upstream_proxy
 
 logger = logging.getLogger(__name__)
 

mobsf/DynamicAnalyzer/views/android/frida_server_download.py

@@ -9,10 +9,9 @@
 
 from django.conf import settings
 
-from mobsf.MobSF.utils import (
-    is_internet_available,
-    upstream_proxy,
-)
+from mobsf.MobSF.utils import is_internet_available
+
+from mobsf.MobSF.proxy import upstream_proxy
 
 
 logger = logging.getLogger(__name__)
@@ -30,7 +29,7 @@ def clean_up_old_binaries(dirc, version):
                 pass
 
 
-def download_frida_server(url, version, fname, proxies):
+def download_frida_server(url, version, fname, proxies, verify):
     """Download frida-server-binary."""
     try:
         download_dir = Path(settings.DWD_DIR)
@@ -40,6 +39,7 @@ def download_frida_server(url, version, fname, proxies):
                 url,
                 timeout=5,
                 proxies=proxies,
+                verify=verify,
                 stream=True) as r:
             with LZMAFile(r.raw) as f:
                 with open(dwd_loc, 'wb') as flip:
@@ -72,7 +72,7 @@ def update_frida_server(arch, version):
         for item in response.json()['assets']:
             if item['name'] == f'{fserver}.xz':
                 url = item['browser_download_url']
-                return download_frida_server(url, version, fserver, proxies)
+                return download_frida_server(url, version, fserver, proxies, verify)
         return False
     except Exception:
         logger.exception('[ERROR] Fetching Frida Server Release')

mobsf/DynamicAnalyzer/views/ios/corellium_apis.py

@@ -9,10 +9,9 @@
 
 import requests
 
-from mobsf.MobSF.utils import (
-    is_number,
-    upstream_proxy,
-)
+from mobsf.MobSF.utils import is_number
+
+from mobsf.MobSF.proxy import upstream_proxy
 
 
 SUCCESS_RESP = (200, 204)

mobsf/MalwareAnalyzer/views/VirusTotal.py

@@ -4,11 +4,12 @@
 
 from django.conf import settings
 
+from mobsf.MobSF.proxy import upstream_proxy
+
 from mobsf.MobSF.utils import (
     append_scan_status,
     file_size,
     get_config_loc,
-    upstream_proxy,
 )
 
 logger = logging.getLogger(__name__)

mobsf/MobSF/proxy.py

@@ -0,0 +1,67 @@
+import os
+import re
+import logging
+from . import settings
+
+logger = logging.getLogger(__name__)
+
+def upstream_proxy(flaw_type, for_urllib=False):
+    """Set upstream Proxy if needed."""
+    if settings.UPSTREAM_PROXY_ENABLED:
+        if not settings.UPSTREAM_PROXY_USERNAME:
+            proxy_port = str(settings.UPSTREAM_PROXY_PORT)
+            proxy_host = '{}://{}:{}'.format(
+                settings.UPSTREAM_PROXY_TYPE,
+                docker_translate_proxy_ip(settings.UPSTREAM_PROXY_IP),
+                proxy_port)
+            proxies = {flaw_type: proxy_host}
+        else:
+            proxy_port = str(settings.UPSTREAM_PROXY_PORT)
+            proxy_host = '{}://{}:{}@{}:{}'.format(
+                settings.UPSTREAM_PROXY_TYPE,
+                settings.UPSTREAM_PROXY_USERNAME,
+                settings.UPSTREAM_PROXY_PASSWORD,
+                docker_translate_proxy_ip(settings.UPSTREAM_PROXY_IP),
+                proxy_port)
+            proxies = {flaw_type: proxy_host}
+    else:
+        if for_urllib:
+            proxies = {}
+        else:
+            proxies = {flaw_type: None}
+    verify = settings.UPSTREAM_PROXY_SSL_VERIFY in ('1', '"1"')
+    return proxies, verify
+
+
+def docker_translate_localhost(identifier):
+    """Convert localhost to host.docker.internal."""
+    if not identifier:
+        return identifier
+    if not os.getenv('MOBSF_PLATFORM') == 'docker':
+        return identifier
+    try:
+        identifier = identifier.strip()
+        docker_internal = 'host.docker.internal:'
+        if re.match(r'^emulator-\d{4}$', identifier):
+            adb_port = int(identifier.split('emulator-')[1]) + 1
+            # ADB port is console port + 1
+            return f'{docker_internal}{adb_port}'
+        m = re.match(r'^(localhost|127\.0\.0\.1):\d{1,5}$', identifier)
+        if m:
+            adb_port = int(identifier.split(m.group(1))[1].replace(':', ''))
+            return f'{docker_internal}{adb_port}'
+        return identifier
+    except Exception:
+        logger.exception('Failed to convert device '
+                         'identifier for docker connectivity')
+        return identifier
+
+
+def docker_translate_proxy_ip(ip):
+    """Convert localhost proxy ip to host.docker.internal."""
+    if not os.getenv('MOBSF_PLATFORM') == 'docker':
+        return ip
+    if ip and ip.strip() in ('127.0.0.1', 'localhost'):
+        return 'host.docker.internal'
+    return ip
+

mobsf/MobSF/settings.py

@@ -138,6 +138,7 @@
               'releases/latest')
 FRIDA_SERVER = 'https://api.github.com/repos/frida/frida/releases/tags/'
 GOOGLE = 'https://www.google.com'
+PLAYSTORE = 'https://play.google.com'
 BAIDU = 'https://www.baidu.com/'
 APKPURE = 'https://m.apkpure.com/android/{}/download?from=details'
 APKTADA = 'https://apktada.com/download-apk/'

mobsf/MobSF/tools_download.py

@@ -4,13 +4,16 @@
 import tempfile
 import zipfile
 import platform
+import ssl
 from pathlib import Path
 from urllib.request import (
     ProxyHandler,
+    HTTPSHandler,
     Request,
     build_opener,
     getproxies,
 )
+from mobsf.MobSF.proxy import upstream_proxy
 
 logging.basicConfig(
     level=logging.INFO,
@@ -22,9 +25,17 @@
 def download_file(url, file_path):
     req = Request(url)
     system_proxies = getproxies()
-    proxy_handler = ProxyHandler(system_proxies)
-    opener = build_opener(proxy_handler)
-
+    if system_proxies:
+        proxies=system_proxies 
+    else:
+        proxies, verify = upstream_proxy('https', for_urllib=True)
+    proxy_handler = ProxyHandler(proxies)
+    if verify:
+        ssl_context = ssl.create_default_context()
+    else:
+        ssl_context = ssl._create_unverified_context()
+    https_handler = HTTPSHandler(context=ssl_context)
+    opener = build_opener(proxy_handler, https_handler)
     with opener.open(req) as response:
         if response.status == 200:
             file_size = int(response.headers.get('Content-Length', 0))

mobsf/MobSF/utils.py

@@ -35,10 +35,16 @@
 
 import requests
 
+from mobsf.MobSF.proxy import (
+    upstream_proxy,
+    docker_translate_localhost,
+)
+
 from django.shortcuts import render
 from django.utils import timezone
 
 from mobsf.StaticAnalyzer.models import RecentScansDB
+
 from mobsf.MobSF. init import api_key
 
 from . import settings
@@ -73,31 +79,6 @@ class Color(object):
     END = '\033[0m'
 
 
-def upstream_proxy(flaw_type):
-    """Set upstream Proxy if needed."""
-    if settings.UPSTREAM_PROXY_ENABLED:
-        if not settings.UPSTREAM_PROXY_USERNAME:
-            proxy_port = str(settings.UPSTREAM_PROXY_PORT)
-            proxy_host = '{}://{}:{}'.format(
-                settings.UPSTREAM_PROXY_TYPE,
-                docker_translate_proxy_ip(settings.UPSTREAM_PROXY_IP),
-                proxy_port)
-            proxies = {flaw_type: proxy_host}
-        else:
-            proxy_port = str(settings.UPSTREAM_PROXY_PORT)
-            proxy_host = '{}://{}:{}@{}:{}'.format(
-                settings.UPSTREAM_PROXY_TYPE,
-                settings.UPSTREAM_PROXY_USERNAME,
-                settings.UPSTREAM_PROXY_PASSWORD,
-                docker_translate_proxy_ip(settings.UPSTREAM_PROXY_IP),
-                proxy_port)
-            proxies = {flaw_type: proxy_host}
-    else:
-        proxies = {flaw_type: None}
-    verify = settings.UPSTREAM_PROXY_SSL_VERIFY in ('1', '"1"')
-    return proxies, verify
-
-
 def get_system_resources():
     """Get CPU and Memory Available."""
     # Get number of physical cores
@@ -379,39 +360,6 @@ def find_process_by(name):
     return proc
 
 
-def docker_translate_localhost(identifier):
-    """Convert localhost to host.docker.internal."""
-    if not identifier:
-        return identifier
-    if not os.getenv('MOBSF_PLATFORM') == 'docker':
-        return identifier
-    try:
-        identifier = identifier.strip()
-        docker_internal = 'host.docker.internal:'
-        if re.match(r'^emulator-\d{4}$', identifier):
-            adb_port = int(identifier.split('emulator-')[1]) + 1
-            # ADB port is console port + 1
-            return f'{docker_internal}{adb_port}'
-        m = re.match(r'^(localhost|127\.0\.0\.1):\d{1,5}$', identifier)
-        if m:
-            adb_port = int(identifier.split(m.group(1))[1].replace(':', ''))
-            return f'{docker_internal}{adb_port}'
-        return identifier
-    except Exception:
-        logger.exception('Failed to convert device '
-                         'identifier for docker connectivity')
-        return identifier
-
-
-def docker_translate_proxy_ip(ip):
-    """Convert localhost proxy ip to host.docker.internal."""
-    if not os.getenv('MOBSF_PLATFORM') == 'docker':
-        return ip
-    if ip and ip.strip() in ('127.0.0.1', 'localhost'):
-        return 'host.docker.internal'
-    return ip
-
-
 def get_device():
     """Get Device."""
     if os.getenv('ANALYZER_IDENTIFIER'):

mobsf/MobSF/views/apk_downloader.py

@@ -15,12 +15,14 @@
     add_to_recent_scan,
     handle_uploaded_file,
 )
+
+from mobsf.MobSF.proxy import upstream_proxy
+
 from mobsf.MobSF.utils import (
     is_internet_available,
     is_path_traversal,
     is_zip_magic,
     strict_package_check,
-    upstream_proxy,
 )
 
 

mobsf/StaticAnalyzer/views/android/manifest_analysis.py

@@ -9,9 +9,11 @@
 from mobsf.MobSF.utils import (
     append_scan_status,
     is_number,
-    upstream_proxy,
     valid_host,
 )
+
+from mobsf.MobSF.proxy import upstream_proxy
+
 from mobsf.StaticAnalyzer.views.android import (
     network_security,
 )