Modernize (#165)

Author: Gallaecio

.github/workflows/main.yml

@@ -1,39 +1,39 @@
 name: CI
-
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
   push:
-    branches: [ master ]
+    branches:
+    - master
   pull_request:
-    branches: [ master ]
-
 jobs:
-  build:
-
+  tox:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        scrapy-version: ["2.11", "2.10"]
-
+        include:
+        - python-version: "3.13"
+          tox: "pre-commit"
+        - python-version: "3.9"
+          tox: "mypy"
+        - python-version: "3.13"
+          tox: "pylint"
+        - python-version: "3.9"
+          tox: "min"
+        - python-version: "3.10"
+        - python-version: "3.11"
+        - python-version: "3.12"
+        - python-version: "3.13"
+        - python-version: "3.13"  # Keep in sync with .readthedocs.yml
+          tox: "docs"
+        - python-version: "3.13"
+          tox: "twinecheck"
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        scrapy-version: ${{ matrix.scrapy-version }}
-    - name: Install dependencies
+    - name: Run tox
       run: |
-        python -m pip install --upgrade pip
-        pip install scrapy==${{ matrix.scrapy-version}}
-        pip install -r requirements-dev.txt
-#    - name: Lint with flake8
-#      run: |
-#        # stop the build if there are Python syntax errors or undefined names
-#        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-#        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-#        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest
+        pip install -U tox
+        tox -e ${{ matrix.tox || 'py' }}

.github/workflows/publish.yml

@@ -1,26 +1,25 @@
 name: Publish
-on: 
-  release:
-    types: [published]
-
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
 jobs:
   publish:
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/p/${{ github.event.repository.name }}
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python 3
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3
-
-    - name: Build distribution
-      run: |
-        pip install -U setuptools wheel
-        python setup.py sdist bdist_wheel
-    
+        python-version: 3.13
+    - run: |
+        python -m pip install --upgrade build
+        python -m build
     - name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        password: ${{ secrets.PYPI_TOKEN }}
+        password: ${{ secrets.PYPI_TOKEN }}

.gitignore

@@ -3,6 +3,8 @@
 /build/
 /dist/
 /*.egg-info/
+/venv/
+/.venv/
 
 # twisted
 dropin.cache
@@ -22,6 +24,7 @@ _trial_temp
 /test_urls*
 testing_scripts*
 coverage.xml
+/.coverage.*
 .python-version
 
 # IDE

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.12.5
+  hooks:
+    - id: ruff-check
+      args: [ --fix ]
+    - id: ruff-format
+- repo: https://github.com/adamchainz/blacken-docs
+  rev: 1.19.1
+  hooks:
+  - id: blacken-docs
+    additional_dependencies:
+    - black==25.1.0
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v1.0.0
+  hooks:
+    - id: sphinx-lint

.readthedocs.yml

@@ -0,0 +1,15 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+build:
+  os: ubuntu-24.04
+  tools:
+    # For available versions, see:
+    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
+    python: "3.13"  # Keep in sync with .github/workflows/main.yml
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

CHANGES.rst

@@ -0,0 +1,35 @@
+Changes
+=======
+
+ScrapyRT 0.17.0 (unreleased)
+----------------------------
+
+-   Added support for Scrapy 2.13+.
+
+-   Lowered the minimum required Scrapy version from 2.10 to 2.7.
+
+-   The ``start_requests`` API parameter is deprecated in favor of a new
+    ``spider_start`` API parameter.
+
+    Same with the corresponding ``__init__`` parameter and attribute of the
+    ``CrawlManager`` class.
+
+
+ScrapyRT 0.16 (2023-02-14)
+--------------------------
+
+- errback method for spider made configurable, errback for spiders will default
+  to None instead of parse
+
+
+ScrapyRT 0.12 (2021-03-08)
+--------------------------
+
+- added crawl arguments for API
+- removed Python 2 support
+- added Python 3.9 support
+- docs clean up
+- removed superfluous requirements (demjson, six)
+- fixed API crash when spider returns bytes in items output
+- updated unit tests
+- development improvements, moved from Travis to Github Workflows

LICENSE

@@ -5,10 +5,10 @@ All rights reserved.
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-    1. Redistributions of source code must retain the above copyright notice, 
+    1. Redistributions of source code must retain the above copyright notice,
        this list of conditions and the following disclaimer.
-    
-    2. Redistributions in binary form must reproduce the above copyright 
+
+    2. Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
 

README.rst

@@ -28,7 +28,7 @@ ScrapyRT (Scrapy realtime)
 Add HTTP API for your `Scrapy <https://scrapy.org/>`_ project in minutes.
 
 You send a request to ScrapyRT with spider name and URL, and in response, you get items collected by a spider
-visiting this URL. 
+visiting this URL.
 
 * All Scrapy project components (e.g. middleware, pipelines, extensions) are supported
 * You run Scrapyrt in Scrapy project directory. It starts HTTP server allowing you to schedule spiders and get spider output in JSON.