Merge branch 'master' into fix-95

Author: pawelmhm

.github/workflows/main.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements-dev.txt
+#    - name: Lint with flake8
+#      run: |
+#        # stop the build if there are Python syntax errors or undefined names
+#        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+#        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+#        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

.travis.yml

@@ -8,20 +8,20 @@
 # > sudo docker run -p 9080:9080 -tid -v ${PROJECT_DIR}:/scrapyrt/project scrapyrt
 #
 
-FROM ubuntu:14.04
+FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
-    apt-get install -y python python-dev  \
+    apt-get install -y python3 python3-dev  \
     libffi-dev libxml2-dev libxslt1-dev zlib1g-dev libssl-dev wget
 
 RUN mkdir -p /scrapyrt/src /scrapyrt/project
 RUN mkdir -p /var/log/scrapyrt
 
 RUN wget -O /tmp/get-pip.py "https://bootstrap.pypa.io/get-pip.py" && \
-    python /tmp/get-pip.py "pip==9.0.1" && \
-    rm /tmp/get-pip.py 
+    python3 /tmp/get-pip.py "pip==19.3.1" && \
+    rm /tmp/get-pip.py
 
 ADD . /scrapyrt/src
 RUN pip install /scrapyrt/src

Dockerfile

@@ -2,8 +2,8 @@
 Scrapyrt (Scrapy realtime)
 ==========================
 
-.. image:: https://travis-ci.org/scrapinghub/scrapyrt.svg?branch=master
-    :target: https://travis-ci.org/scrapinghub/scrapyrt
+.. image:: https://github.com/scrapinghub/scrapyrt/workflows/CI/badge.svg
+   :target: https://github.com/scrapinghub/scrapyrt/actions
 
 .. image:: https://img.shields.io/pypi/pyversions/scrapyrt.svg
     :target: https://pypi.python.org/pypi/scrapyrt

README.rst

@@ -136,16 +136,16 @@ with hopefully helpful error message.
 Examples
 ~~~~~~~~
 
-To run sample `dmoz spider`_ from `Scrapy educational dirbot project`_
-parsing page about Ada programming language::
+To run sample `toscrape-css spider`_ from `Scrapy educational quotesbot project`_
+parsing page about famous quotes::
 
-    curl "http://localhost:9080/crawl.json?spider_name=dmoz&url=http://www.dmoz.org/Computers/Programming/Languages/Ada/"
+    curl "http://localhost:9080/crawl.json?spider_name=toscrape-css&url=http://quotes.toscrape.com/"
 
 
 To run same spider only allowing one request and parsing url
 with callback ``parse_foo``::
 
-    curl "http://localhost:9080/crawl.json?spider_name=dmoz&url=http://www.dmoz.org/Computers/Programming/Languages/Ada/&callback=parse_foo&max_requests=1"
+    curl "http://localhost:9080/crawl.json?spider_name=toscrape-css&url=http://quotes.toscrape.com/&callback=parse_foo&max_requests=1"
 
 POST
 ----
@@ -222,16 +222,16 @@ hopefully helpful error message.
 Examples
 ~~~~~~~~
 
-To schedule spider dmoz with sample url using POST handler::
+To schedule spider toscrape-css with sample url using POST handler::
 
     curl localhost:9080/crawl.json \
-        -d '{"request":{"url":"http://www.dmoz.org/Computers/Programming/Languages/Awk/"}, "spider_name": "dmoz"}'
+        -d '{"request":{"url":"http://quotes.toscrape.com/"}, "spider_name": "toscrape-css"}'
 
 
 to schedule same spider with some meta that will be passed to spider request::
 
     curl localhost:9080/crawl.json \
-        -d '{"request":{"url":"http://www.dmoz.org/Computers/Programming/Languages/Awk/", "meta": {"alfa":"omega"}}, "spider_name": "dmoz"}'
+        -d '{"request":{"url":"http://quotes.toscrape.com/", "meta": {"alfa":"omega"}}, "spider_name": "toscrape-css"}'
 
 Response
 --------
@@ -265,34 +265,34 @@ errors (optional)
 
 Example::
 
-    $ curl "http://localhost:9080/crawl.json?spider_name=dmoz&url=http://www.dmoz.org/Computers/Programming/Languages/Ada/"
+    $ curl "http://localhost:9080/crawl.json?spider_name=toscrape-css&url=http://quotes.toscrape.com/"
     {
         "status": "ok"
-        "spider_name": "dmoz",
+        "spider_name": "toscrape-css",
         "stats": {
-            "start_time": "2014-12-29 16:04:15",
-            "finish_time": "2014-12-29 16:04:16",
+            "start_time": "2019-12-06 13:01:31",
+            "finish_time": "2019-12-06 13:01:35",
             "finish_reason": "finished",
-            "downloader/response_status_count/200": 1,
-            "downloader/response_count": 1,
-            "downloader/response_bytes": 8494,
-            "downloader/request_method_count/GET": 1,
-            "downloader/request_count": 1,
-            "downloader/request_bytes": 247,
-            "item_scraped_count": 16,
-            "log_count/DEBUG": 17,
-            "log_count/INFO": 4,
-            "response_received_count": 1,
-            "scheduler/dequeued": 1,
-            "scheduler/dequeued/memory": 1,
-            "scheduler/enqueued": 1,
-            "scheduler/enqueued/memory": 1
+            "downloader/response_status_count/200": 10,
+            "downloader/response_count": 11,
+            "downloader/response_bytes": 24812,
+            "downloader/request_method_count/GET": 11,
+            "downloader/request_count": 11,
+            "downloader/request_bytes": 2870,
+            "item_scraped_count": 100,
+            "log_count/DEBUG": 111,
+            "log_count/INFO": 9,
+            "response_received_count": 11,
+            "scheduler/dequeued": 10,
+            "scheduler/dequeued/memory": 10,
+            "scheduler/enqueued": 10,
+            "scheduler/enqueued/memory": 10,
         },
         "items": [
             {
-                "description": ...,
-                "name": ...,
-                "url": ...
+                "text": ...,
+                "author": ...,
+                "tags": ...
             },
             ...
         ],
@@ -315,7 +315,7 @@ message
 
 Example::
 
-    $ curl "http://localhost:9080/crawl.json?spider_name=foo&url=http://www.dmoz.org/Computers/Programming/Languages/Ada/"
+    $ curl "http://localhost:9080/crawl.json?spider_name=foo&url=http://quotes.toscrape.com/"
     {
         "status": "error"
         "code": 404,
@@ -456,22 +456,22 @@ in response, for example::
 
     {
         "status": "ok"
-        "spider_name": "dmoz",
+        "spider_name": "toscrape-css",
         "stats": {
-            "start_time": "2014-12-29 17:26:11",
+            "start_time": "2019-12-06 13:11:30"
             "spider_exceptions/Exception": 1,
-            "finish_time": "2014-12-29 17:26:11",
+            "finish_time": "2019-12-06 13:11:31",
             "finish_reason": "finished",
             "downloader/response_status_count/200": 1,
-            "downloader/response_count": 1,
-            "downloader/response_bytes": 8494,
-            "downloader/request_method_count/GET": 1,
-            "downloader/request_count": 1,
-            "downloader/request_bytes": 247,
-            "log_count/DEBUG": 1,
+            "downloader/response_count": 2,
+            "downloader/response_bytes": 2701,
+            "downloader/request_method_count/GET": 2,
+            "downloader/request_count": 2,
+            "downloader/request_bytes": 446,
+            "log_count/DEBUG": 2,
             "log_count/ERROR": 1,
-            "log_count/INFO": 4,
-            "response_received_count": 1,
+            "log_count/INFO": 9,
+            "response_received_count": 2,
             "scheduler/dequeued": 1,
             "scheduler/dequeued/memory": 1,
             "scheduler/enqueued": 1,
@@ -559,8 +559,8 @@ approach described in `Python Logging HOWTO`_ or redirect stdout to a file using
 `bash redirection syntax`_, `supervisord logging`_ etc.
 
 
-.. _dmoz spider: https://github.com/scrapy/dirbot/blob/master/dirbot/spiders/dmoz.py
-.. _Scrapy educational dirbot project: https://github.com/scrapy/dirbot
+.. _toscrape-css spider: https://github.com/scrapy/quotesbot/blob/master/quotesbot/spiders/toscrape-css.py
+.. _Scrapy educational quotesbot project: https://github.com/scrapy/quotesbot
 .. _Scrapy Request: http://doc.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request
 .. _Scrapy Crawler: http://doc.scrapy.org/en/latest/topics/api.html#scrapy.crawler.Crawler
 .. _parse: http://doc.scrapy.org/en/latest/topics/spiders.html#scrapy.spider.Spider.parse

docs/source/api.rst

@@ -1,8 +1,8 @@
-bumpversion==0.5.3
-fabric
-requests==2.22.0
+-r requirements.txt
 mock==1.3.0
-pytest==2.9.1
-pytest-cov==2.2.1
 port-for==0.3.1
-Flask==1.1.1
+Flask==1.1.2
+pytest==6.2.1
+requests==2.25.1
+bumpversion==0.6.0
+flake8==3.8.4

fabfile.py

@@ -1,6 +1,6 @@
 Scrapy>=1.0.0
 service-identity>=1.0.0
-demjson==2.2.4
-six==1.12.0
+demjson>=2.2.4
+six>=1.12.0
 jmespath==0.10.0
-pyasn1==0.4.8
+pyasn1>=0.4.8

requirements-dev.txt

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-from six.moves.configparser import (
-    SafeConfigParser, NoOptionError, NoSectionError
+from configparser import (
+    ConfigParser, NoOptionError, NoSectionError
 )
 import argparse
 import os
@@ -64,7 +64,7 @@ def find_scrapy_project(project):
     project_config_path = closest_scrapy_cfg()
     if not project_config_path:
         raise RuntimeError('Cannot find scrapy.cfg file')
-    project_config = SafeConfigParser()
+    project_config = ConfigParser()
     project_config.read(project_config_path)
     try:
         project_settings = project_config.get('settings', project)

requirements.txt

@@ -24,10 +24,12 @@
     zip_safe=False,
     classifiers=[
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Operating System :: OS Independent',
         'Environment :: Console',
         'Environment :: No Input/Output (Daemon)',