Remove bmob proxy

Author: Alexandru Meterez

fred/__init__.py

@@ -0,0 +1,15 @@
+from flask import Flask
+
+
+class MyServer(Flask):
+
+    def __init__(self, *args, **kwargs):
+        super(MyServer, self).__init__(*args, **kwargs)
+
+        # instanciate your variables here
+        self.proxy = None
+        self.server = None
+
+
+app = MyServer(__name__, static_url_path='')
+# app = Flask(__name__, static_url_path='')

fred/config/proxy.py

@@ -0,0 +1,18 @@
+from browsermobproxy import Server
+
+
+def init(app):
+    server = Server("utils/browsermob_proxy/bin/browsermob-proxy", options={'port': 8090})
+    server.start()
+    proxy = server.create_proxy()
+
+    app.proxy = proxy
+    app.server = server
+
+
+def get_proxy(app):
+    return app.proxy
+
+
+def get_server(app):
+    return app.server

fred/data/collect.py

@@ -4,7 +4,7 @@
 import time
 import json
 import base64
-from run import proxy
+import requests
 
 SCRIPT = """var body = document.body,
             html = document.documentElement;
@@ -38,19 +38,18 @@ def evaluate(script):
     return base64.b64decode(screenshot['data'])
 
 
-def collect_data(url, output_folder, output_filename):
+def collect_data(url, output_folder, output_filename, proxy_host, proxy_port):
     if not os.path.exists("./tmp"):
         os.mkdir("./tmp")
     output_folder = os.path.join("./tmp", output_folder)
     if not os.path.exists(output_folder):
         os.mkdir(output_folder)
-
     options = webdriver.ChromeOptions()
     options.add_argument("--start-maximized")
     options.add_argument("--force-device-scale-factor=2")
     options.add_argument("--disable-infobars")
     options.add_argument("--headless")
-    options.add_argument('--proxy-server=%s' % proxy.proxy)
+    options.add_argument('--proxy-server=localhost:{}'.format(proxy_port))
     options.add_argument('--no-sandbox')
     options.add_argument('--disable-dev-shm-usage')
     capabilities = DesiredCapabilities.CHROME
@@ -60,14 +59,16 @@ def collect_data(url, output_folder, output_filename):
                               desired_capabilities=capabilities, service_args=["--verbose"])
     driver.maximize_window()
     driver.fullscreen_window()
-    proxy.new_har("Logs")
+    # proxy.new_har("Logs")
+    requests.put('%s/proxy/%s/har' % (proxy_host, proxy_port), {'initialPageTitle': 'Logs'})
     driver.get(url)
-
+    print(driver.get_log('browser'))
     logs = driver.get_log('browser')
     with open(os.path.join(output_folder, output_filename.split('.')[0] + '_js_log.json'), 'w') as f:
         json.dump(logs, f, indent=2)
     with open(os.path.join(output_folder, output_filename.split('.')[0] + '_network_log.json'), 'w') as f:
-        json.dump(proxy.har, f, indent=2)
+        r = requests.get('%s/proxy/%s/har' % (proxy_host, proxy_port))
+        json.dump(r.json(), f, indent=2)
 
     height = driver.execute_script(SCRIPT)
     for i in range(4):

fred/endpoints/verify.py

@@ -4,15 +4,30 @@
 import uuid
 from utils.utils import get_time, check_unique_prefix
 from config.status_codes import STATUS_CODES
+from config.proxy import get_proxy
+import socket
+from urllib.parse import urlparse
 
 states = {}
 id_to_urls = {}
 
 
 class Verify(Resource):
+    def __init__(self, app):
+        self.app = app
+
     def post(self):
         baseline_url = request.json['baseline_url']
         updated_url = request.json['updated_url']
+        proxy = get_proxy(self.app)
+        baseline_url_host = urlparse(baseline_url)
+        updated_url_host = urlparse(updated_url)
+        print(baseline_url_host.netloc)
+        print(socket.gethostbyname(baseline_url))
+        # proxy.remap_hosts(baseline_url_host.netloc, socket.gethostbyname(baseline_url))
+        # proxy.remap_hosts(updated_url_host.netloc, socket.gethostbyname(updated_url))
+        proxy.remap_hosts(baseline_url_host.netloc, 'www.google.com')
+        proxy.remap_hosts(updated_url_host.netloc, 'www.google.com')
         max_depth = request.json['max_depth']
         max_urls = request.json['max_urls']
         prefix = request.json['prefix']
@@ -29,7 +44,8 @@ def post(self):
              '--max-depth',
              max_depth, '--max-urls', max_urls, '--prefix', prefix, '--auth-baseline-username', auth_baseline_username,
              '--auth-baseline-password', auth_baseline_password, '--auth-updated-username', auth_updated_username,
-             '--auth-updated-password', auth_updated_password])
+             '--auth-updated-password', auth_updated_password, '--proxy-host', proxy.host, '--proxy-port',
+             str(proxy.port)])
         if p.poll() is not None and p.poll() > 0:
             return {'Error': 'Failed to launch crawler'}, 406
         id = str(uuid.uuid4().hex)

fred/run.py

@@ -1,3 +1,5 @@
+from config.proxy import get_server, init
+
 from flask import Flask
 from flask_restful import Api
 from flask_cors import CORS
@@ -8,7 +10,10 @@
 from endpoints.get_result import Result
 from apscheduler.schedulers.background import BackgroundScheduler
 from flask import Flask, request, send_from_directory
-from browsermobproxy import Server
+import sys
+
+sys.path.append('../')
+from fred import app
 
 
 def clear_ended():
@@ -18,31 +23,28 @@ def clear_ended():
             del id_to_urls[k]
 
 
-app = Flask(__name__, static_url_path='')
-
-
 @app.route('/static/<path:path>')
 def send_js(path):
     return send_from_directory('frontend', path)
 
 
-if app.config["DEBUG"]:
-    @app.after_request
-    def after_request(response):
-        response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"
-        response.headers["Expires"] = 0
-        response.headers["Pragma"] = "no-cache"
-        return response
-server = Server("utils/browsermob_proxy/bin/browsermob-proxy", options={'port': 8090})
-server.start()
-proxy = server.create_proxy()
-cors = CORS(app, resources={r"*": {"origins": "*"}})
-api = Api(app)
-api.add_resource(Verify, "/api/verify")
-api.add_resource(IDList, "/api/ids")
-api.add_resource(Shutdown, "/api/shutdown")
-api.add_resource(Result, "/api/result")
-
 if __name__ == '__main__':
+    if app.config["DEBUG"]:
+        @app.after_request
+        def after_request(response):
+            response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"
+            response.headers["Expires"] = 0
+            response.headers["Pragma"] = "no-cache"
+            return response
+    cors = CORS(app, resources={r"*": {"origins": "*"}})
+    # print(hex(id(app)))
+    api = Api(app)
+    api.add_resource(Verify, "/api/verify", resource_class_kwargs={'app': app})
+    api.add_resource(IDList, "/api/ids")
+    api.add_resource(Shutdown, "/api/shutdown")
+    api.add_resource(Result, "/api/result")
+    init(app)
+
     app.run(host='0.0.0.0', debug=True)
+    server = get_server(app)
     server.stop()