diff --git a/dockerfiles/splash/provision.sh b/dockerfiles/splash/provision.sh index bdd04c8ae..b29ca089a 100755 --- a/dockerfiles/splash/provision.sh +++ b/dockerfiles/splash/provision.sh @@ -215,7 +215,7 @@ for cmd in "$@"; do fi done -if [ $UNKNOWN -eq 1 ]; then +if [ ${UNKNOWN} -eq 1 ]; then echo "Unknown commands encountered, exiting..." exit 1 fi diff --git a/docs/api.rst b/docs/api.rst index 2724b2f91..fee1eff77 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -35,7 +35,7 @@ url : string : required baseurl : string : optional The base url to render the page with. - Base HTML content will be feched from the URL given in the url + Base HTML content will be fetched from the URL given in the url argument, while relative referenced resources in the HTML-text used to render the page are fetched using the URL given in the baseurl argument as base. See also: :ref:`render-html-doesnt-work`. @@ -249,7 +249,7 @@ width : integer : optional .. _arg-height: height : integer : optional - Crop the renderd image to the given height (in pixels). Often used in + Crop the rendered image to the given height (in pixels). Often used in conjunction with the width argument to generate fixed-size thumbnails. .. _arg-render-all: diff --git a/docs/faq.rst b/docs/faq.rst index 4b52037f9..0f3dd146e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -50,7 +50,7 @@ render use resource timeouts. For render.*** endpoints use for :ref:`execute` use either :ref:`splash-resource-timeout` or ``request:set_timeout`` (see :ref:`splash-on-request`). -It is a good practive to always set resource_timeout; something similar to +It is a good practice to always set resource_timeout; something similar to ``resource_timeout=20`` often works well. .. _504-slow-script: diff --git a/splash/browser_tab.py b/splash/browser_tab.py index d685e755a..4c02d3656 100644 --- a/splash/browser_tab.py +++ b/splash/browser_tab.py @@ -999,7 +999,8 @@ def _return_reply(self, callback, url): reply = self.sender() callback(reply) - def _set_request_headers(self, request, headers): + @staticmethod + def _set_request_headers(request, headers): """ Set HTTP headers for the request. """ if isinstance(headers, dict): headers = headers.items() diff --git a/splash/har/log.py b/splash/har/log.py index e09137689..6b78cc589 100644 --- a/splash/har/log.py +++ b/splash/har/log.py @@ -83,14 +83,16 @@ def todict(self): } } - def _get_browser(self): + @staticmethod + def _get_browser(): return { "name": "QWebKit", "version": six.text_type(qWebKitVersion()), "comment": "PyQt %s, Qt %s" % (PYQT_VERSION_STR, QT_VERSION_STR), } - def _empty_page(self, page_id, started_dt): + @staticmethod + def _empty_page(page_id, started_dt): if not isinstance(started_dt, six.string_types): started_dt = format_datetime(started_dt) diff --git a/splash/kernel/completer.py b/splash/kernel/completer.py index 181ef7885..fb8f9a2af 100644 --- a/splash/kernel/completer.py +++ b/splash/kernel/completer.py @@ -119,7 +119,8 @@ def complete_obj_method(self, value, prefix=""): methods = self.completer.obj_attrs(value, False, True) return sorted_with_prefix(prefix, to_unicode_all(methods.values())) - def complete_keyword(self, prefix): + @staticmethod + def complete_keyword(prefix): return sorted_with_prefix(prefix, LUA_KEYWORDS) def complete_global_variable(self, prefix): diff --git a/splash/kernel/inspections.py b/splash/kernel/inspections.py index e03a6f38a..763195a35 100644 --- a/splash/kernel/inspections.py +++ b/splash/kernel/inspections.py @@ -25,7 +25,8 @@ def __init__(self, lua): def parse(self, code, cursor_pos): return self.parser.parse(code, cursor_pos, allow_inside=True) - def doc_repr(self, doc): + @staticmethod + def doc_repr(doc): if not doc.get("signature"): return doc["content"] diff --git a/splash/lua_runtime.py b/splash/lua_runtime.py index 12b5492bc..18df25187 100644 --- a/splash/lua_runtime.py +++ b/splash/lua_runtime.py @@ -109,7 +109,8 @@ def _create_runtime(self, lua_package_path): self._setup_lua_paths(runtime, lua_package_path) return runtime - def _setup_lua_paths(self, lua, lua_package_path): + @staticmethod + def _setup_lua_paths(lua, lua_package_path): root = os.path.join(os.path.dirname(__file__), 'lua_modules') at_root = lambda *p: os.path.abspath(os.path.join(root, *p)) default_path = "{root}/?.lua;{libs}/?.lua".format( diff --git a/splash/network_manager.py b/splash/network_manager.py index d3c2de811..8513b2456 100644 --- a/splash/network_manager.py +++ b/splash/network_manager.py @@ -99,10 +99,12 @@ def __init__(self, verbosity): self._request_ids = itertools.count() assert self.proxyFactory() is None, "Standard QNetworkProxyFactory is not supported" - def _on_ssl_errors(self, reply, errors): + @staticmethod + def _on_ssl_errors(reply, errors): reply.ignoreSslErrors() - def _on_finished(self, reply): + @staticmethod + def _on_finished(reply): reply.deleteLater() def createRequest(self, operation, request, outgoingData=None): @@ -248,12 +250,14 @@ def _get_har(self, request=None): request = self.sender().request() return self._get_webpage_attribute(request, "har") - def _get_webpage_attribute(self, request, attribute): + @staticmethod + def _get_webpage_attribute(request, attribute): web_frame = get_request_webframe(request) if web_frame: return getattr(web_frame.page(), attribute, None) - def _set_webpage_attribute(self, request, attribute, value): + @staticmethod + def _set_webpage_attribute(request, attribute, value): web_frame = get_request_webframe(request) if web_frame: return setattr(web_frame.page(), attribute, value) diff --git a/splash/proxy.py b/splash/proxy.py index a7ed66f8a..fd2634f87 100644 --- a/splash/proxy.py +++ b/splash/proxy.py @@ -57,7 +57,8 @@ def should_use_proxy_list(self, protocol, url): return not bool(self.whitelist) - def _get_default_proxy_list(self): + @staticmethod + def _get_default_proxy_list(): return [QNetworkProxy(QNetworkProxy.DefaultProxy)] def _get_custom_proxy_list(self): diff --git a/splash/qtrender_image.py b/splash/qtrender_image.py index b7bc7fe88..779b97546 100644 --- a/splash/qtrender_image.py +++ b/splash/qtrender_image.py @@ -1,11 +1,11 @@ """This module handles rendering QWebPage into PNG and JPEG images.""" -import sys import array +import sys from abc import ABCMeta, abstractmethod, abstractproperty from io import BytesIO from math import ceil, floor -import six +import six from PIL import Image from PyQt5.QtCore import QBuffer, QPoint, QRect, QSize, Qt from PyQt5.QtGui import QImage, QPainter, QRegion @@ -14,7 +14,6 @@ class QtImageRenderer(object): - # QPainter cannot render a region with any dimension greater than # this value. QPAINTER_MAXSIZE = 32766 @@ -84,7 +83,8 @@ def qimage_to_pil_image(self, qimage): self._qsize_to_tuple(qimage.size()), buf, 'raw', self.pillow_decoder_format) - def swap_byte_order_i32(self, buf): + @staticmethod + def swap_byte_order_i32(buf): """Swap order of bytes in each 32-bit word of given byte sequence.""" arr = array.array('I') arr.fromstring(buf) @@ -298,7 +298,7 @@ def _render_qwebpage_tiled(self, web_rect, render_rect, canvas_size): top = j * tile_qimage.height() painter.setViewport(render_rect.translated(-left, -top)) self.logger.log("Rendering with viewport=%s" - % painter.viewport(), min_level=2) + % painter.viewport(), min_level=2) clip_rect = QRect( QPoint(floor(left / ratio), @@ -329,7 +329,8 @@ def _render_qwebpage_tiled(self, web_rect, render_rect, canvas_size): painter.end() return WrappedPillowImage(canvas) - def _calculate_image_parameters(self, web_viewport, img_width, img_height): + @staticmethod + def _calculate_image_parameters(web_viewport, img_width, img_height): """ :return: (image_viewport, image_size) """ @@ -349,7 +350,8 @@ def _calculate_image_parameters(self, web_viewport, img_width, img_height): image_size = QSize(img_width, img_height) return image_viewport, image_size - def _calculate_tiling(self, to_paint): + @staticmethod + def _calculate_tiling(to_paint): tile_maxsize = defaults.TILE_MAXSIZE tile_hsize = min(tile_maxsize, to_paint.width()) tile_vsize = min(tile_maxsize, to_paint.height()) @@ -360,7 +362,8 @@ def _calculate_tiling(self, to_paint): 'vertical_count': vtiles, 'tile_size': tile_size} - def _qsize_to_tuple(self, sz): + @staticmethod + def _qsize_to_tuple(sz): return sz.width(), sz.height() def _qpainter_needs_tiling(self, render_rect, canvas_size): @@ -371,6 +374,7 @@ def _qpainter_needs_tiling(self, render_rect, canvas_size): class _DummyLogger(object): """Logger to use when no logger is passed into rendering functions.""" + def log(self, *args, **kwargs): pass @@ -384,6 +388,7 @@ class WrappedImage(six.with_metaclass(ABCMeta, object)): use one or another. """ + @abstractproperty def size(self): """ diff --git a/splash/qtrender_lua.py b/splash/qtrender_lua.py index 8411b9198..29b9f355e 100644 --- a/splash/qtrender_lua.py +++ b/splash/qtrender_lua.py @@ -599,7 +599,8 @@ def jpeg(self, width=None, height=None, render_all=False, return None return BinaryCapsule(result, 'image/jpeg') - def _validate_region(self, region): + @staticmethod + def _validate_region(region): if region is not None: try: if isinstance(region, dict): @@ -758,7 +759,8 @@ def callback(reply): def autoload_reset(self): self.tab.autoload_reset() - def _check_mouse_coordinantes(self, x, y): + @staticmethod + def _check_mouse_coordinantes(x, y): def _assert_number(value, name): if not isinstance(value, (int, float)): raise ScriptError({ @@ -1070,7 +1072,8 @@ def get_version(self): }) return versions - def _error_info_to_lua(self, error_info): + @staticmethod + def _error_info_to_lua(error_info): if error_info is None: return "error" res = "%s%s" % (error_info.type.lower(), error_info.code) @@ -1215,7 +1218,8 @@ def clear(self): super(_ExposedBoundRequest, self).clear() self.request = None - def _on_request_required(self, meth, attr_name): + @staticmethod + def _on_request_required(meth, attr_name): raise ScriptError({ "message": "request is used outside a callback", "type": ScriptError.SPLASH_LUA_ERROR, @@ -1320,7 +1324,8 @@ def clear(self): self._info = None self._info_lua = None - def _on_response_required(self, meth, attr_name): + @staticmethod + def _on_response_required(meth, attr_name): raise ScriptError({ "message": "response is used outside a callback", "type": ScriptError.SPLASH_LUA_ERROR, @@ -1494,7 +1499,8 @@ def on_lua_error(self, lua_exception): raise ScriptError(py_info) - def _make_script_error(self, ex): + @staticmethod + def _make_script_error(ex): if not isinstance(ex, (TypeError, ValueError)): return ex diff --git a/splash/resources.py b/splash/resources.py index 76ceea4b9..a0dcbfde0 100644 --- a/splash/resources.py +++ b/splash/resources.py @@ -53,7 +53,8 @@ def render(self, request): self._log_stats(request, {}, error=self._format_error(400, e)) return b"\n" - def _write_error_content(self, request, code, err, content_type=b'text/plain'): + @staticmethod + def _write_error_content(request, code, err, content_type=b'text/plain'): request.setHeader(b"content-type", content_type) request.setResponseCode(code) content = json.dumps(err).encode('utf8') @@ -66,7 +67,8 @@ def _write_error(self, request, code, exc): return self._write_error_content(request, code, err, content_type=b"application/json") - def _format_error(self, code, exc): + @staticmethod + def _format_error(code, exc): err = { "error": code, "type": exc.__class__.__name__, @@ -144,12 +146,14 @@ def render_POST(self, request): return self.render_GET(request) - def _cancel_timer(self, _, timer): + @staticmethod + def _cancel_timer(_, timer): #log.msg("_cancelTimer") timer.cancel() return _ - def _request_failed(self, _, pool_d, timer): + @staticmethod + def _request_failed(_, pool_d, timer): log.msg("Client disconnected: %s" % _.value) timer.cancel() pool_d.cancel() @@ -364,7 +368,8 @@ def render_GET(self, request): return (json.dumps(info, sort_keys=True)).encode('utf-8') - def get_repr(self, render): + @staticmethod + def get_repr(render): if hasattr(render, 'url'): return render.url return render.tab.url @@ -393,7 +398,8 @@ def render_POST(self, request): class PingResource(Resource): isLeaf = True - def render_GET(self, request): + @staticmethod + def render_GET(request): request.setHeader(b"content-type", b"application/json") return (json.dumps({ "status": "ok", @@ -623,7 +629,8 @@ def getChild(self, name, request): return self return Resource.getChild(self, name, request) - def get_example_script(self): + @staticmethod + def get_example_script(): return """ function main(splash) local url = splash.args.url diff --git a/splash/tests/mockserver.py b/splash/tests/mockserver.py index b6262c5de..5d31e0502 100755 --- a/splash/tests/mockserver.py +++ b/splash/tests/mockserver.py @@ -273,7 +273,7 @@ def _delayedRender(self, request_info): request, n = request_info # write 1px black gif gif_data = b'AQABAIAAAAAAAAAAACH5BAAAAAAALAAAAAABAAEAAAICTAEAOw==' - request.write(base64.decodestring(gif_data)) + request.write(base64.decodebytes(gif_data)) if not request._disconnected: request.finish() @@ -764,7 +764,7 @@ class Image(Resource): @use_chunked_encoding def render_GET(self, request): request.setHeader(b"Content-Type", b"image/gif") - return base64.decodestring(b'R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=') + return base64.decodebytes(b'R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=') class SetHeadersResource(Resource): diff --git a/splash/tests/stress.py b/splash/tests/stress.py index fa713fff7..ee214eb26 100644 --- a/splash/tests/stress.py +++ b/splash/tests/stress.py @@ -1,18 +1,22 @@ from __future__ import print_function -import sys, requests, random, optparse, time, json +import json +import optparse +import random +import sys +import time +from collections import Counter from itertools import islice from threading import Thread -from collections import Counter -import requests +import requests import six from six.moves.queue import Queue from .utils import SplashServer, MockServer -class StressTest(): +class StressTest: def __init__(self, reqs, host="localhost:8050", requests=1000, concurrency=50, shuffle=False, verbose=False): self.reqs = reqs self.host = host @@ -47,7 +51,7 @@ def run(self): print("Total requests: %d" % len(args)) print("Concurrency : %d" % self.concurrency) print("Elapsed time : %.3fs" % elapsed) - print("Avg time p/req: %.3fs" % (elapsed/len(args))) + print("Avg time p/req: %.3fs" % (elapsed / len(args))) print("Received (per status code or error):") for c, n in Counter(outputs).items(): print(" %s: %d" % (c, n)) @@ -79,7 +83,6 @@ def worker(host, q, p, verbose=False): class MockArgs(object): - ok_urls = 0.5 error_urls = 0.3 timeout_urls = 0.2 @@ -110,7 +113,6 @@ def __iter__(self): class ArgsFromUrlFile(object): - def __init__(self, urlfile): self.urlfile = urlfile @@ -123,7 +125,6 @@ def __iter__(self): class ArgsFromLogfile(object): - def __init__(self, logfile): self.logfile = logfile @@ -155,7 +156,7 @@ def lua_runonce(script, timeout=60., splash_args=None, **kwargs): splash_args = ['--disable-lua-sandbox', '--allowed-schemes=file,http,https', ] with SplashServer(extra_args=splash_args) as s, \ - MockServer() as ms: + MockServer() as ms: if kwargs.get('url', '').startswith('mock://'): kwargs['url'] = ms.url(kwargs['url'][7:]) params = {'lua_source': script} @@ -211,19 +212,19 @@ def benchmark_png(url, viewport=None, wait=0.5, render_all=1, def parse_opts(): op = optparse.OptionParser() op.add_option("-H", dest="host", default="localhost:8050", - help="splash hostname & port (default: %default)") + help="splash hostname & port (default: %default)") op.add_option("-u", dest="urlfile", metavar="FILE", - help="read urls from FILE instead of using mock server ones") + help="read urls from FILE instead of using mock server ones") op.add_option("-l", dest="logfile", metavar="FILE", - help="read urls from splash log file (useful for replaying)") + help="read urls from splash log file (useful for replaying)") op.add_option("-s", dest="shuffle", action="store_true", default=False, - help="shuffle (randomize) requests (default: %default)") + help="shuffle (randomize) requests (default: %default)") op.add_option("-v", dest="verbose", action="store_true", default=False, - help="verbose mode (default: %default)") + help="verbose mode (default: %default)") op.add_option("-c", dest="concurrency", type="int", default=50, - help="concurrency (default: %default)") + help="concurrency (default: %default)") op.add_option("-n", dest="requests", type="int", default=1000, - help="number of requests (default: %default)") + help="number of requests (default: %default)") return op.parse_args() @@ -238,5 +239,6 @@ def main(): t = StressTest(urls, opts.host, opts.requests, opts.concurrency, opts.shuffle, opts.verbose) t.run() + if __name__ == "__main__": main() diff --git a/splash/ui/style.css b/splash/ui/style.css index 49b160e2f..734c29ce5 100644 --- a/splash/ui/style.css +++ b/splash/ui/style.css @@ -2,7 +2,7 @@ .label { color: #000; font-weight: normal; - font-size: 100%%; + font-size: 100%; } table { @@ -53,7 +53,7 @@ body.no-lua .if-lua { /* Hide stuff that requires lua support if there is no lua .netPageTimingBar {opacity: 0.3; width: 2px; } .timeInfoTip { width: 250px !important; } .customEventBar { background-color: gray; } -._onStarted { background-color: marine; } +._onStarted { background-color: maroon; } ._onPrepareStart { background-color: green; } ._onCustomJsExecuted { background-color: green; } ._onScreenshotPrepared { background-color: magenta; }