Disallow new requests after ≥10 and ≥1% undocumented error responses

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.0
+current_version = 0.6.0
 commit = True
 tag = True
 tag_name = {new_version}

.coveragerc

@@ -1,2 +1,8 @@
 [run]
 branch = true
+
+[report]
+# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:

.github/workflows/publish.yml

@@ -13,11 +13,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.x'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/test.yml

@@ -16,12 +16,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -33,20 +33,22 @@ jobs:
         tox -e py
     - name: coverage
       if: ${{ success() }}
-      run: bash <(curl -s https://codecov.io/bash)
+      uses: codecov/[email protected]
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   check:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.10']
+        python-version: ['3.12']  # Keep in sync with .readthedocs.yml
         tox-job: ["mypy", "docs"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+    - id: isort
+-   repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+    - id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+    - id: flake8
+- repo: https://github.com/adamchainz/blacken-docs
+  rev: 1.19.0
+  hooks:
+  - id: blacken-docs
+    additional_dependencies:
+    - black==24.10.0

.readthedocs.yml

@@ -0,0 +1,14 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+build:
+  os: ubuntu-22.04
+  tools:
+    # For available versions, see:
+    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
+    python: "3.12"  # Keep in sync with .github/workflows/test.yml
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

CHANGES.rst

@@ -1,6 +1,142 @@
 Changes
 =======
 
+0.6.0 (2024-05-29)
+------------------
+
+* Improved how the :ref:`default retry policy <default-retry-policy>` handles
+  :ref:`temporary download errors <zapi-temporary-download-errors>`.
+  Before, 3 HTTP 429 responses followed by a single HTTP 520 response would
+  have prevented a retry. Now, unrelated responses and errors do not count
+  towards the HTTP 520 retry limit.
+
+* Improved how the :ref:`default retry policy <default-retry-policy>` handles
+  network errors. Before, after 15 minutes of unsuccessful responses (e.g. HTTP
+  429), any network error would prevent a retry. Now, network errors must happen
+  15 minutes in a row, without different errors in between, to stop retries.
+
+* Implemented an optional :ref:`aggressive retry policy
+  <aggressive-retry-policy>`, which retries more errors more often, and could
+  be useful for long crawls or websites with a low success rate.
+
+* Improved the exception that is raised when passing an invalid retrying policy
+  object to a :ref:`Python client <api>`.
+
+0.5.2 (2024-05-10)
+------------------
+
+* :class:`~zyte_api.RequestError` now has a :data:`~zyte_api.RequestError.query`
+  attribute with the Zyte API request parameters that caused the error.
+
+0.5.1 (2024-04-16)
+------------------
+
+* :class:`~zyte_api.ZyteAPI` and :class:`~zyte_api.AsyncZyteAPI` sessions no
+  longer need to be used as context managers, and can instead be closed with a
+  ``close()`` method.
+
+0.5.0 (2024-04-05)
+------------------
+
+* Removed Python 3.7 support.
+
+* Added :class:`~zyte_api.ZyteAPI` and :class:`~zyte_api.AsyncZyteAPI` to
+  provide both sync and async Python interfaces with a cleaner API.
+
+* Deprecated ``zyte_api.aio``:
+
+  * Replace ``zyte_api.aio.client.AsyncClient`` with the new
+    :class:`~zyte_api.AsyncZyteAPI` class.
+
+  * Replace ``zyte_api.aio.client.create_session`` with the new
+    :meth:`AsyncZyteAPI.session <zyte_api.AsyncZyteAPI.session>` method.
+
+  * Import ``zyte_api.aio.errors.RequestError``,
+    ``zyte_api.aio.retry.RetryFactory`` and
+    ``zyte_api.aio.retry.zyte_api_retrying`` directly from ``zyte_api`` now.
+
+* When using the command-line interface, you can now use ``--store-errors`` to
+  have error responses be stored alongside successful responses.
+
+* Improved the documentation.
+
+0.4.8 (2023-11-02)
+------------------
+
+* Include the Zyte API request ID value in a new ``.request_id`` attribute
+  in ``zyte_api.aio.errors.RequestError``.
+
+0.4.7 (2023-09-26)
+------------------
+
+* ``AsyncClient`` now lets you set a custom user agent to send to Zyte API.
+
+0.4.6 (2023-09-26)
+------------------
+
+* Increased the client timeout to match the server’s.
+* Mentioned the ``api_key`` parameter of ``AsyncClient`` in the docs example.
+
+0.4.5 (2023-01-03)
+------------------
+
+* w3lib >= 2.1.1 is required in install_requires, to ensure that URLs
+  are escaped properly.
+* unnecessary ``requests`` library is removed from install_requires
+* fixed tox 4 support
+
+0.4.4 (2022-12-01)
+------------------
+
+* Fixed an issue with submitting URLs which contain unescaped symbols
+* New "retrying" argument for AsyncClient.__init__, which allows to set
+  custom retrying policy for the client
+* ``--dont-retry-errors`` argument in the CLI tool
+
+0.4.3 (2022-11-10)
+------------------
+
+* Connections are no longer reused between requests.
+  This reduces the amount of ``ServerDisconnectedError`` exceptions.
+
+0.4.2 (2022-10-28)
+------------------
+* Bump minimum ``aiohttp`` version to 3.8.0, as earlier versions don't support
+  brotli decompression of responses
+* Declared Python 3.11 support
+
+0.4.1 (2022-10-16)
+------------------
+
+* Network errors, like server timeouts or disconnections, are now retried for
+  up to 15 minutes, instead of 5 minutes.
+
+0.4.0 (2022-09-20)
+------------------
+
+* Require to install ``Brotli`` as a dependency. This changes the requests to
+  have ``Accept-Encoding: br`` and automatically decompress brotli responses.
+
+0.3.0 (2022-07-29)
+------------------
+
+Internal AggStats class is cleaned up:
+
+* ``AggStats.n_extracted_queries`` attribute is removed, as it was a duplicate
+  of ``AggStats.n_results``
+* ``AggStats.n_results`` is renamed to ``AggStats.n_success``
+* ``AggStats.n_input_queries`` is removed as redundant and misleading;
+  AggStats got a new ``AggStats.n_processed`` property instead.
+
+This change is backwards incompatible if you used stats directly.
+
+0.2.1 (2022-07-29)
+------------------
+
+* ``aiohttp.client_exceptions.ClientConnectorError`` is now treated as a
+  network error and retried accordingly.
+* Removed the unused ``zyte_api.sync`` module.
+
 0.2.0 (2022-07-14)
 ------------------
 

README.rst

@@ -18,32 +18,101 @@ python-zyte-api
    :target: https://codecov.io/gh/zytedata/zyte-api
    :alt: Coverage report
 
-Python client libraries for `Zyte Data API`_.
+.. description-start
 
-Command-line utility and asyncio-based library are provided by this package.
+Command-line client and Python client library for `Zyte API`_.
+
+.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html
+
+.. description-end
 
 Installation
 ============
 
-::
+.. install-start
+
+.. code-block:: shell
 
     pip install zyte-api
 
-``zyte-api`` requires Python 3.7+.
+.. note:: Python 3.9+ is required.
+
+.. install-end
+
+Basic usage
+===========
+
+.. basic-start
+
+Set your API key
+----------------
+
+.. key-get-start
+
+After you `sign up for a Zyte API account
+<https://app.zyte.com/account/signup/zyteapi>`_, copy `your API key
+<https://app.zyte.com/o/zyte-api/api-access>`_.
+
+.. key-get-end
+
+
+Use the command-line client
+---------------------------
+
+Then you can use the zyte-api command-line client to send Zyte API requests.
+First create a text file with a list of URLs:
+
+.. code-block:: none
+
+    https://books.toscrape.com
+    https://quotes.toscrape.com
+
+And then call ``zyte-api`` from your shell:
+
+.. code-block:: shell
 
-API key
-=======
+    zyte-api url-list.txt --api-key YOUR_API_KEY --output results.jsonl
 
-Make sure you have an API key for the `Zyte Data API`_ service.
-You can set ``ZYTE_API_KEY`` environment
-variable with the key to avoid passing it around explicitly.
 
-Read the `documentation <https://python-zyte-api.readthedocs.io>`_  for more information.
+Use the Python sync API
+-----------------------
 
-License is BSD 3-clause.
+For very basic Python scripts, use the sync API:
+
+.. code-block:: python
+
+    from zyte_api import ZyteAPI
+
+    client = ZyteAPI(api_key="YOUR_API_KEY")
+    response = client.get({"url": "https://toscrape.com", "httpResponseBody": True})
+
+
+Use the Python async API
+------------------------
+
+For asyncio code, use the async API:
+
+.. code-block:: python
+
+    import asyncio
+
+    from zyte_api import AsyncZyteAPI
+
+
+    async def main():
+        client = AsyncZyteAPI(api_key="YOUR_API_KEY")
+        response = await client.get(
+            {"url": "https://toscrape.com", "httpResponseBody": True}
+        )
+
+
+    asyncio.run(main())
+
+.. basic-end
+
+Read the `documentation <https://python-zyte-api.readthedocs.io>`_  for more
+information.
 
 * Documentation: https://python-zyte-api.readthedocs.io
 * Source code: https://github.com/zytedata/python-zyte-api
 * Issue tracker: https://github.com/zytedata/python-zyte-api/issues
-
-.. _Zyte Data API: https://docs.zyte.com/zyte-api/get-started.html