[tests] use Flask as test server not python http.server

this will allow us to test more complex scenarios like HTTP error codes
etc.

Author: pawelmhm

requirements-dev.txt

@@ -5,3 +5,4 @@ mock==1.3.0
 pytest==2.9.1
 pytest-cov==2.2.1
 port-for==0.3.1
+Flask==1.1.1

tests/sample_data/testproject/testproject/spiders/testspider.py

@@ -11,3 +11,7 @@ class TestSpider(scrapy.Spider):
     def parse(self, response):
         name = response.xpath('//h1/text()').extract()
         return TestprojectItem(name=name)
+
+    def some_errback(self, err):
+        self.logger.error("Logging some error {}".format(err))
+        return

tests/servers.py

@@ -1,18 +1,17 @@
 # -*- coding: utf-8 -*-
-import six
-from subprocess import Popen, PIPE
-from six.moves.urllib.parse import urljoin
 import fcntl
 import os
 import shutil
 import socket
 import sys
 import tempfile
 import time
+from subprocess import Popen, PIPE
 
 import port_for
+from six.moves.urllib.parse import urljoin
 
-from . import SAMPLE_DATA
+from . import TESTS_PATH
 from .utils import get_testenv, generate_project
 
 DEVNULL = open(os.devnull, 'wb')
@@ -30,12 +29,9 @@ def __init__(self, host='localhost', port=None, cwd=None, shell=False,
         self.stdin = stdin
         self.stdout = stdout
         self.stderr = stderr
-        if six.PY2:
-            command = 'SimpleHTTPServer'
-        else:
-            command = 'http.server'
+
         self.arguments = [
-            sys.executable, '-u', '-m', command, str(self.port)
+            'flask', 'run', '-p', str(self.port)
         ]
 
     def start(self):
@@ -126,4 +122,4 @@ class MockServer(BaseTestServer):
 
     def __init__(self, *args, **kwargs):
         super(MockServer, self).__init__(*args, **kwargs)
-        self.cwd = os.path.join(SAMPLE_DATA, 'testsite')
+        self.cwd = os.path.join(TESTS_PATH, 'testsite')

tests/test_resource_crawl.py

@@ -378,3 +378,16 @@ def test_invalid_json_in_post(self, server):
             assert res_json[k] == v
         msg = "Invalid JSON in POST body"
         assert msg in res_json['message']
+
+    @pytest.mark.parametrize("method", [
+        perform_get, perform_post
+    ])
+    def test_passing_errback(self, server, method):
+        url = server.url("crawl.json")
+        res = method(url,
+                     {"spider_name": "test"},
+                     {"url": server.target_site.url("page1.html"),
+                      'errback': 'some_errback',
+                      'callback': 'raise_some_error'})
+
+        res_json = res.json()

tests/testsite/app.py

@@ -0,0 +1,34 @@
+from flask import Flask
+
+
+app = Flask(__name__)
+
+
+def read_file(filename):
+    with open(filename) as f:
+        return f.read()
+
+
+@app.route('/')
+def base():
+    return b'hello'
+
+
+@app.route('/index.html')
+def index():
+    return read_file('index.html')
+
+
+@app.route('/page1.html')
+def page1():
+    return read_file('page1.html')
+
+
+@app.route('/page2.html')
+def page2():
+    return read_file('page2.html')
+
+
+@app.route('/page3.html')
+def page3():
+    return read_file('page3.html')