Merge pull request #2 from adobe/dev.pyppeteer-usage

Dev.pyppeteer usage

Author: alexandrumeterez

fred/.gitignore.swp

@@ -0,0 +1,3 @@
+from flask import Flask
+
+app = Flask(__name__, static_url_path='')

fred/__init__.py

@@ -3,5 +3,4 @@
     2: 'Prediction script failed',
     3: 'Failed to launch inference process',
     4: 'Model does not exist',
-    5: 'CHROMEDRIVER_PATH not provided in env variables'
 }

fred/config/status_codes.py

@@ -1,16 +1,33 @@
 from urllib.parse import urlparse
-from selenium import webdriver
 from bs4 import BeautifulSoup
-from selenium.common.exceptions import InvalidArgumentException
-import os
 from utils.utils import eprint
+from pyppeteer import launch
+import asyncio
 
 
-def _recursive_get_urls(crawled_urls, driver, max_urls, parent_url, domain, depth=0):
+class MyPage(object):
+    def __init__(self):
+        self.source = None
+
+    def set(self, source):
+        self.source = source
+
+
+async def get_page(test_page, url):
+    browser = await launch()
+    page = await browser.newPage()
+    await page.goto(url)
+    out = await page.content()
+    test_page.set(out)
+    await browser.close()
+
+
+def _recursive_get_urls(crawled_urls, test_page, max_urls, parent_url, domain, depth=0):
     if depth == 0 or len(crawled_urls) == max_urls:
         return crawled_urls
-    driver.get(parent_url)
-    html = driver.page_source.encode('utf-8')
+    asyncio.get_event_loop().run_until_complete(get_page(test_page, parent_url))
+
+    html = test_page.source
     soup = BeautifulSoup(html, features='html.parser')
 
     urls = soup.findAll('a')
@@ -23,31 +40,15 @@ def _recursive_get_urls(crawled_urls, driver, max_urls, parent_url, domain, dept
         if urlparse(url).netloc == domain and url not in crawled_urls:
             if len(crawled_urls) <= max_urls:
                 crawled_urls.append(url)
-                print('[LOG] Added: {}'.format(url))
-                _recursive_get_urls(crawled_urls, driver, max_urls, url, domain, depth - 1)
+                eprint('[LOG] Added: {}'.format(url))
+                _recursive_get_urls(crawled_urls, max_urls, url, domain, depth - 1)
 
 
 def get_recursive_urls(parent_url, max_depth, max_urls):
     scraped_urls = [parent_url]
     domain = urlparse(parent_url).netloc
-    if not 'CHROMEDRIVER_PATH' in os.environ:
-        eprint('[ERR] CHROMEDRIVER_PATH not provided in env variables')
-        exit(5)
-    driver_path = os.environ['CHROMEDRIVER_PATH']
-    assert os.path.exists(driver_path), 'No such file {}'.format(driver_path)
-    options = webdriver.ChromeOptions()
-    options.add_argument("--disable-infobars")
-    options.add_argument("--headless")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--disable-dev-shm-usage")
-    driver = webdriver.Chrome(driver_path, chrome_options=options)
-    try:
-        driver.get(parent_url)
-    except InvalidArgumentException:
-        eprint('[ERR] Invalid website')
-        driver.close()
-        exit(1)
-    _recursive_get_urls(scraped_urls, driver, max_urls, parent_url, domain, depth=max_depth)
+    page = MyPage()
+    asyncio.get_event_loop().run_until_complete(get_page(page, parent_url))
+    _recursive_get_urls(scraped_urls, page, max_urls, parent_url, domain, depth=max_depth)
     eprint('[LOG] Finished crawling URLs for {}'.format(parent_url))
-    driver.close()
     return scraped_urls

fred/crawler/crawler.py

@@ -1,41 +1,34 @@
 import os
-from selenium import webdriver
-from selenium.webdriver import DesiredCapabilities
-import time
+import asyncio
 import json
-import base64
-from run import proxy
+from pyppeteer import launch
 
-SCRIPT = """var body = document.body,
-            html = document.documentElement;
-            return Math.max( body.scrollHeight, body.offsetHeight, 
-            html.clientHeight, html.scrollHeight, html.offsetHeight );"""
+network = []
+javascript = []
 
 
-def _chrome_full_screenshot(driver):
-    def send(cmd, params):
-        resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id
-        url = driver.command_executor._url + resource
-        body = json.dumps({'cmd': cmd, 'params': params})
-        response = driver.command_executor._request('POST', url, body)
-        return response.get('value')
+async def intercept_network_response(response):
+    network.append(str(response.status) + response.url)
 
-    def evaluate(script):
-        response = send('Runtime.evaluate', {'returnByValue': True, 'expression': script})
-        return response['result']['value']
 
-    metrics = evaluate( \
-        "({" + \
-        "width: Math.max(window.innerWidth, document.body.scrollWidth, document.documentElement.scrollWidth)|0," + \
-        "height: Math.max(innerHeight, document.body.scrollHeight, document.documentElement.scrollHeight)|0," + \
-        "deviceScaleFactor: window.devicePixelRatio || 1," + \
-        "mobile: typeof window.orientation !== 'undefined'" + \
-        "})")
-    send('Emulation.setDeviceMetricsOverride', metrics)
-    screenshot = send('Page.captureScreenshot', {'format': 'png', 'fromSurface': True})
-    send('Emulation.clearDeviceMetricsOverride', {})
+async def intercept_console(response):
+    javascript.append(response.text)
 
-    return base64.b64decode(screenshot['data'])
+
+async def collect_msgs_and_screenshot(url, ss_path):
+    browser = await launch()
+    page = await browser.newPage()
+
+    page.on('response', intercept_network_response)
+    page.on('console', intercept_console)
+    page.on('requestfailed', intercept_console)
+    page.on('pageerror', intercept_console)
+    page.on('error', intercept_console)
+
+    await page.goto(url)
+    await page.screenshot({'path': ss_path, 'fullPage': True})
+
+    await browser.close()
 
 
 def collect_data(url, output_folder, output_filename):
@@ -45,36 +38,11 @@ def collect_data(url, output_folder, output_filename):
     if not os.path.exists(output_folder):
         os.mkdir(output_folder)
 
-    options = webdriver.ChromeOptions()
-    options.add_argument("--start-maximized")
-    options.add_argument("--force-device-scale-factor=2")
-    options.add_argument("--disable-infobars")
-    options.add_argument("--headless")
-    options.add_argument('--proxy-server=%s' % proxy.proxy)
-    options.add_argument('--no-sandbox')
-    options.add_argument('--disable-dev-shm-usage')
-    capabilities = DesiredCapabilities.CHROME
-    capabilities['goog:loggingPrefs'] = {'browser': 'ALL'}
+    asyncio.get_event_loop().run_until_complete(
+        collect_msgs_and_screenshot(url, os.path.join(output_folder, output_filename)))
 
-    driver = webdriver.Chrome(os.environ['CHROMEDRIVER_PATH'], chrome_options=options,
-                              desired_capabilities=capabilities, service_args=["--verbose"])
-    driver.maximize_window()
-    driver.fullscreen_window()
-    proxy.new_har("Logs")
-    driver.get(url)
-
-    logs = driver.get_log('browser')
     with open(os.path.join(output_folder, output_filename.split('.')[0] + '_js_log.json'), 'w') as f:
-        json.dump(logs, f, indent=2)
-    with open(os.path.join(output_folder, output_filename.split('.')[0] + '_network_log.json'), 'w') as f:
-        json.dump(proxy.har, f, indent=2)
+        json.dump(javascript, f, indent=2)
 
-    height = driver.execute_script(SCRIPT)
-    for i in range(4):
-        driver.execute_script("window.scrollBy(0, " + str(height / 5) + ")")
-        time.sleep(3)
-    png = _chrome_full_screenshot(driver)
-    with open(os.path.join(output_folder, output_filename), 'wb') as f:
-        f.write(png)
-    driver.execute_script("window.scrollTo(0, 0)")
-    driver.close()
+    with open(os.path.join(output_folder, output_filename.split('.')[0] + '_network_log.json'), 'w') as f:
+        json.dump(network, f, indent=2)

fred/data/collect.py

@@ -1,5 +1,4 @@
 import json
-from urllib.parse import urlparse
 import re
 
 
@@ -11,23 +10,11 @@ def __init__(self, log_file):
     def _js_messages(self):
         messages = []
         for log in self.logs:
-            processed_message = re.sub(r'http:\S+', '', log['message'])
+            processed_message = re.sub(r'http:\S+', '', log)
             messages.append(processed_message)
         return messages
 
 
 class NetworkLogPreprocessor(object):
     def __init__(self, log_file):
-        self.logs = json.load(open(log_file, 'r'))
-        self.network_messages = self._network_messages()
-
-    def _network_messages(self):
-        messages = []
-        for entry in self.logs['log']['entries']:
-            request_text = entry['request']['url']
-            request_path = urlparse(request_text).path
-            response = entry['response']['status']
-            message = request_path + str(response)
-
-            messages.append(message)
-        return messages
+        self.network_messages = json.load(open(log_file, 'r'))

fred/data/log_preprocess.py

@@ -1,14 +1,15 @@
-from flask import Flask
 from flask_restful import Api
 from flask_cors import CORS
 from endpoints.verify import Verify
 from endpoints.get_ids import IDList
 from endpoints.shutdown import Shutdown
 from endpoints.verify import states, id_to_urls
 from endpoints.get_result import Result
-from apscheduler.schedulers.background import BackgroundScheduler
-from flask import Flask, request, send_from_directory
-from browsermobproxy import Server
+from flask import send_from_directory
+import sys
+
+sys.path.append('../')
+from fred import app
 
 
 def clear_ended():
@@ -18,31 +19,25 @@ def clear_ended():
             del id_to_urls[k]
 
 
-app = Flask(__name__, static_url_path='')
-
-
 @app.route('/static/<path:path>')
 def send_js(path):
     return send_from_directory('frontend', path)
 
 
-if app.config["DEBUG"]:
-    @app.after_request
-    def after_request(response):
-        response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"
-        response.headers["Expires"] = 0
-        response.headers["Pragma"] = "no-cache"
-        return response
-server = Server("utils/browsermob_proxy/bin/browsermob-proxy", options={'port': 8090})
-server.start()
-proxy = server.create_proxy()
-cors = CORS(app, resources={r"*": {"origins": "*"}})
-api = Api(app)
-api.add_resource(Verify, "/api/verify")
-api.add_resource(IDList, "/api/ids")
-api.add_resource(Shutdown, "/api/shutdown")
-api.add_resource(Result, "/api/result")
-
 if __name__ == '__main__':
+    if app.config["DEBUG"]:
+        @app.after_request
+        def after_request(response):
+            response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate, public, max-age=0"
+            response.headers["Expires"] = 0
+            response.headers["Pragma"] = "no-cache"
+            return response
+    cors = CORS(app, resources={r"*": {"origins": "*"}})
+    # print(hex(id(app)))
+    api = Api(app)
+    api.add_resource(Verify, "/api/verify")
+    api.add_resource(IDList, "/api/ids")
+    api.add_resource(Shutdown, "/api/shutdown")
+    api.add_resource(Result, "/api/result")
+
     app.run(host='0.0.0.0', debug=True)
-    server.stop()