Update for moving to puppeteer

Author: Alexandru Meterez

fred/.gitignore.swp

@@ -1,15 +1,3 @@
 from flask import Flask
 
-
-class MyServer(Flask):
-
-    def __init__(self, *args, **kwargs):
-        super(MyServer, self).__init__(*args, **kwargs)
-
-        # instanciate your variables here
-        self.proxy = None
-        self.server = None
-
-
-app = MyServer(__name__, static_url_path='')
-# app = Flask(__name__, static_url_path='')
+app = Flask(__name__, static_url_path='')

fred/__init__.py

@@ -8,16 +8,11 @@
 
 
 async def intercept_network_response(response):
-    print("URL:", response.url)
-    print(response.status)
     network.append(str(response.status) + response.url)
-    print("\n\n")
 
 
 async def intercept_console(response):
-    print(response.text)
     javascript.append(response.text)
-    print("\n\n")
 
 
 async def collect_msgs_and_screenshot(url, ss_path):
@@ -44,7 +39,7 @@ def collect_data(url, output_folder, output_filename):
         os.mkdir(output_folder)
 
     asyncio.get_event_loop().run_until_complete(
-        collect_msgs_and_screenshot(url, os.path.join(output_folder, output_filename) + '.png'))
+        collect_msgs_and_screenshot(url, os.path.join(output_folder, output_filename)))
 
     with open(os.path.join(output_folder, output_filename.split('.')[0] + '_js_log.json'), 'w') as f:
         json.dump(javascript, f, indent=2)

fred/config/proxy.py

@@ -1,5 +1,4 @@
 import json
-from urllib.parse import urlparse
 import re
 
 
@@ -11,23 +10,11 @@ def __init__(self, log_file):
     def _js_messages(self):
         messages = []
         for log in self.logs:
-            processed_message = re.sub(r'http:\S+', '', log['message'])
+            processed_message = re.sub(r'http:\S+', '', log)
             messages.append(processed_message)
         return messages
 
 
 class NetworkLogPreprocessor(object):
     def __init__(self, log_file):
-        self.logs = json.load(open(log_file, 'r'))
-        self.network_messages = self._network_messages()
-
-    def _network_messages(self):
-        messages = []
-        for entry in self.logs['log']['entries']:
-            request_text = entry['request']['url']
-            request_path = urlparse(request_text).path
-            response = entry['response']['status']
-            message = request_path + str(response)
-
-            messages.append(message)
-        return messages
+        self.network_messages = json.load(open(log_file, 'r'))

fred/data/collect.py

@@ -4,30 +4,15 @@
 import uuid
 from utils.utils import get_time, check_unique_prefix
 from config.status_codes import STATUS_CODES
-from config.proxy import get_proxy
-import socket
-from urllib.parse import urlparse
 
 states = {}
 id_to_urls = {}
 
 
 class Verify(Resource):
-    def __init__(self, app):
-        self.app = app
-
     def post(self):
         baseline_url = request.json['baseline_url']
         updated_url = request.json['updated_url']
-        proxy = get_proxy(self.app)
-        baseline_url_host = urlparse(baseline_url)
-        updated_url_host = urlparse(updated_url)
-        print(baseline_url_host.netloc)
-        print(socket.gethostbyname(baseline_url))
-        # proxy.remap_hosts(baseline_url_host.netloc, socket.gethostbyname(baseline_url))
-        # proxy.remap_hosts(updated_url_host.netloc, socket.gethostbyname(updated_url))
-        proxy.remap_hosts(baseline_url_host.netloc, 'www.google.com')
-        proxy.remap_hosts(updated_url_host.netloc, 'www.google.com')
         max_depth = request.json['max_depth']
         max_urls = request.json['max_urls']
         prefix = request.json['prefix']
@@ -44,8 +29,7 @@ def post(self):
              '--max-depth',
              max_depth, '--max-urls', max_urls, '--prefix', prefix, '--auth-baseline-username', auth_baseline_username,
              '--auth-baseline-password', auth_baseline_password, '--auth-updated-username', auth_updated_username,
-             '--auth-updated-password', auth_updated_password, '--proxy-host', proxy.host, '--proxy-port',
-             str(proxy.port)])
+             '--auth-updated-password', auth_updated_password])
         if p.poll() is not None and p.poll() > 0:
             return {'Error': 'Failed to launch crawler'}, 406
         id = str(uuid.uuid4().hex)

fred/data/log_preprocess.py

@@ -1,15 +1,11 @@
-from config.proxy import get_server, init
-
-from flask import Flask
 from flask_restful import Api
 from flask_cors import CORS
 from endpoints.verify import Verify
 from endpoints.get_ids import IDList
 from endpoints.shutdown import Shutdown
 from endpoints.verify import states, id_to_urls
 from endpoints.get_result import Result
-from apscheduler.schedulers.background import BackgroundScheduler
-from flask import Flask, request, send_from_directory
+from flask import send_from_directory
 import sys
 
 sys.path.append('../')
@@ -39,12 +35,9 @@ def after_request(response):
     cors = CORS(app, resources={r"*": {"origins": "*"}})
     # print(hex(id(app)))
     api = Api(app)
-    api.add_resource(Verify, "/api/verify", resource_class_kwargs={'app': app})
+    api.add_resource(Verify, "/api/verify")
     api.add_resource(IDList, "/api/ids")
     api.add_resource(Shutdown, "/api/shutdown")
     api.add_resource(Result, "/api/result")
-    init(app)
 
     app.run(host='0.0.0.0', debug=True)
-    server = get_server(app)
-    server.stop()