Merge remote-tracking branch 'scrapinghub/master' into six

Author: Gallaecio

.readthedocs.yml

@@ -0,0 +1,15 @@
+version: 2
+formats: all
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+build:
+  os: ubuntu-22.04
+  tools:
+    # For available versions, see:
+    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
+    python: "3.10"  # Keep in sync with .github/workflows/main.yml
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

docs/client/overview.rst

@@ -145,7 +145,7 @@ For example, to run a new job for a given spider with custom parameters::
 
 
 Getting job information
-^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^
 
 To select a specific job for a project, use ``.jobs.get(<jobKey>)``::
 

docs/conf.py

@@ -19,10 +19,6 @@
 import sys
 from datetime import datetime
 
-from docutils import nodes
-from sphinx.util.docfields import TypedField
-from sphinx import addnodes
-
 
 sys.path.insert(0, os.path.abspath('..'))
 
@@ -75,7 +71,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -93,8 +89,7 @@
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-#
-html_theme = 'alabaster'
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -165,49 +160,4 @@
      'Miscellaneous'),
 ]
 
-# Set Sphinx Read The Docs theme
-import sphinx_rtd_theme
-
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-
-# disable cross-reference for ivar
-# patch taken from http://stackoverflow.com/a/41184353/1932023
-def patched_make_field(self, types, domain, items, env=None):
-    # type: (List, unicode, Tuple) -> nodes.field
-    def handle_item(fieldarg, content):
-        par = nodes.paragraph()
-        par += addnodes.literal_strong('', fieldarg)  # Patch: this line added
-        # par.extend(self.make_xrefs(self.rolename, domain, fieldarg,
-        #                           addnodes.literal_strong))
-        if fieldarg in types:
-            par += nodes.Text(' (')
-            # NOTE: using .pop() here to prevent a single type node to be
-            # inserted twice into the doctree, which leads to
-            # inconsistencies later when references are resolved
-            fieldtype = types.pop(fieldarg)
-            if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text):
-                typename = ''.join(n.astext() for n in fieldtype)
-                par.extend(self.make_xrefs(self.typerolename, domain, typename,
-                                           addnodes.literal_emphasis))
-            else:
-                par += fieldtype
-            par += nodes.Text(')')
-        par += nodes.Text(' -- ')
-        par += content
-        return par
-
-    fieldname = nodes.field_name('', self.label)
-    if len(items) == 1 and self.can_collapse:
-        fieldarg, content = items[0]
-        bodynode = handle_item(fieldarg, content)
-    else:
-        bodynode = self.list_type()
-        for fieldarg, content in items:
-            bodynode += nodes.list_item('', handle_item(fieldarg, content))
-    fieldbody = nodes.field_body('', bodynode)
-    return nodes.field('', fieldname, fieldbody)
-
-
-TypedField.make_field = patched_make_field

docs/index.rst

@@ -2,9 +2,6 @@
 Client interface for Scrapinghub API
 ====================================
 
-.. image:: https://secure.travis-ci.org/scrapinghub/python-scrapinghub.svg?branch=master
-   :target: https://travis-ci.org/scrapinghub/python-scrapinghub
-
 The ``scrapinghub`` is a Python library for communicating with the `Scrapinghub API`_.
 
 .. _Scrapinghub API: https://doc.scrapinghub.com/scrapy-cloud.html#scrapycloud

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx==7.2.6
+sphinx-rtd-theme==2.0.0

requirements-test.txt

@@ -1,5 +1,5 @@
 mock
-vcrpy
+vcrpy<6  # https://github.com/kevin1024/vcrpy/issues/885
 pytest
 pytest-cov
 responses

scrapinghub/client/__init__.py

@@ -27,7 +27,7 @@ def request(self, *args, **kwargs):
 
 
 class ScrapinghubClient:
-    r"""Main class to work with Scrapinghub API.
+    """Main class to work with Scrapinghub API.
 
     :param auth: (optional) Scrapinghub APIKEY or other SH auth credentials.
         If not provided, it will read, respectively, from 
@@ -38,7 +38,7 @@ class ScrapinghubClient:
         If you need full access to *Scrapy Cloud* features, you'll need to
         provide a Scrapinghub APIKEY through this argument or deploying ``SH_APIKEY``.
     :param dash_endpoint: (optional) Scrapinghub Dash panel url.
-    :param \*\*kwargs: (optional) Additional arguments for
+    :param kwargs: (optional) Additional arguments for
         :class:`~scrapinghub.hubstorage.HubstorageClient` constructor.
 
     :ivar projects: projects collection,

scrapinghub/client/collections.py

@@ -154,10 +154,10 @@ def __init__(self, client, collections, type_, name):
         self._origin = _Collection(type_, name, collections._origin)
 
     def get(self, key, **params):
-        r"""Get item from collection by key.
+        """Get item from collection by key.
 
         :param key: string item key.
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: an item dictionary if exists.
         :rtype: :class:`dict`
         """
@@ -206,15 +206,15 @@ def count(self, *args, **kwargs):
 
     def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
              endts=None, requests_params=None, **params):
-        r"""A method to iterate through collection items.
+        """A method to iterate through collection items.
 
         :param key: a string key or a list of keys to filter with.
         :param prefix: a string prefix to filter items.
         :param prefixcount: maximum number of values to return per prefix.
         :param startts: UNIX timestamp at which to begin results.
         :param endts: UNIX timestamp at which to end results.
         :param requests_params: (optional) a dict with optional requests params.
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: an iterator over items list.
         :rtype: :class:`collections.abc.Iterable[dict]`
         """
@@ -227,7 +227,7 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
 
     def list(self, key=None, prefix=None, prefixcount=None, startts=None,
              endts=None, requests_params=None, **params):
-        r"""Convenient shortcut to list iter results.
+        """Convenient shortcut to list iter results.
 
         Please note that :meth:`list` method can use a lot of memory and for a
         large amount of logs it's recommended to iterate through it
@@ -240,7 +240,7 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None,
         :param startts: UNIX timestamp at which to begin results.
         :param endts: UNIX timestamp at which to end results.
         :param requests_params: (optional) a dict with optional requests params.
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: a list of items where each item is represented with a dict.
         :rtype: :class:`list[dict]`
         """

scrapinghub/client/frontiers.py

@@ -314,9 +314,9 @@ def add(self, fps):
             writer.write({'fp': fp})
 
     def iter(self, **params):
-        r"""Iterate through fingerprints in the slot.
+        """Iterate through fingerprints in the slot.
 
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: an iterator over fingerprints.
         :rtype: :class:`collections.abc.Iterable[str]`
         """
@@ -326,9 +326,9 @@ def iter(self, **params):
             yield fp.get('fp')
 
     def list(self, **params):
-        r"""List fingerprints in the slot.
+        """List fingerprints in the slot.
 
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: a list of fingerprints.
         :rtype: :class:`list[str]`
         """
@@ -349,10 +349,10 @@ def add(self, fps):
         return origin.add(self._frontier.key, self.key, fps)
 
     def iter(self, mincount=None, **params):
-        r"""Iterate through batches in the queue.
+        """Iterate through batches in the queue.
 
         :param mincount: (optional) limit results with min amount of requests.
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: an iterator over request batches in the queue where each
             batch is represented with a dict with ('id', 'requests') field.
         :rtype: :class:`collections.abc.Iterable[dict]`
@@ -363,10 +363,10 @@ def iter(self, mincount=None, **params):
         return origin.apiget(path, params=params)
 
     def list(self, mincount=None, **params):
-        r"""List request batches in the queue.
+        """List request batches in the queue.
 
         :param mincount: (optional) limit results with min amount of requests.
-        :param \*\*params: (optional) additional query params for the request.
+        :param params: (optional) additional query params for the request.
         :return: a list of request batches in the queue where each batch
             is represented with a dict with ('id', 'requests') field.
         :rtype: :class:`list[dict]`

scrapinghub/client/jobs.py

@@ -43,7 +43,7 @@ def __init__(self, client, project_id, spider=None):
 
     def count(self, spider=None, state=None, has_tag=None, lacks_tag=None,
               startts=None, endts=None, **params):
-        r"""Count jobs with a given set of filters.
+        """Count jobs with a given set of filters.
 
         :param spider: (optional) filter by spider name.
         :param state: (optional) a job state, a string or a list of strings.
@@ -55,7 +55,7 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None,
             in milliseconds.
         :param endts: (optional) UNIX timestamp at which to end results,
             in milliseconds.
-        :param \*\*params: (optional) other filter params.
+        :param params: (optional) other filter params.
 
         :return: jobs count.
         :rtype: :class:`int`
@@ -138,7 +138,7 @@ def cancel(self, keys=None, count=None, **params):
     def iter(self, count=None, start=None, spider=None, state=None,
              has_tag=None, lacks_tag=None, startts=None, endts=None,
              meta=None, **params):
-        r"""Iterate over jobs collection for a given set of params.
+        """Iterate over jobs collection for a given set of params.
 
         :param count: (optional) limit amount of returned jobs.
         :param start: (optional) number of jobs to skip in the beginning.
@@ -154,7 +154,7 @@ def iter(self, count=None, start=None, spider=None, state=None,
             in millisecons.
         :param meta: (optional) request for additional fields, a single
             field name or a list of field names to return.
-        :param \*\*params: (optional) other filter params.
+        :param params: (optional) other filter params.
 
         :return: a generator object over a list of dictionaries of jobs summary
             for a given filter params.
@@ -209,7 +209,7 @@ def iter(self, count=None, start=None, spider=None, state=None,
     def list(self, count=None, start=None, spider=None, state=None,
              has_tag=None, lacks_tag=None, startts=None, endts=None,
              meta=None, **params):
-        r"""Convenient shortcut to list iter results.
+        """Convenient shortcut to list iter results.
 
         :param count: (optional) limit amount of returned jobs.
         :param start: (optional) number of jobs to skip in the beginning.
@@ -225,7 +225,7 @@ def list(self, count=None, start=None, spider=None, state=None,
             in milliseconds.
         :param meta: (optional) request for additional fields, a single
             field name or a list of field names to return.
-        :param \*\*params: (optional) other filter params.
+        :param params: (optional) other filter params.
 
         :return: list of dictionaries of jobs summary for a given filter params.
         :rtype: :class:`list[dict]`
@@ -248,7 +248,7 @@ def list(self, count=None, start=None, spider=None, state=None,
     def run(self, spider=None, units=None, priority=None, meta=None,
             add_tag=None, job_args=None, job_settings=None, cmd_args=None,
             environment=None, **params):
-        r"""Schedule a new job and returns its job key.
+        """Schedule a new job and returns its job key.
 
         :param spider: a spider name string
             (not needed if job is scheduled via :attr:`Spider.jobs`).
@@ -260,7 +260,7 @@ def run(self, spider=None, units=None, priority=None, meta=None,
         :param job_settings: (optional) a dictionary with job settings.
         :param cmd_args: (optional) a string with script command args.
         :param environment: (option) a dictionary with custom environment
-        :param \*\*params: (optional) additional keyword args.
+        :param params: (optional) additional keyword args.
 
         :return: a job instance, representing the scheduled job.
         :rtype: :class:`Job`
@@ -327,12 +327,12 @@ def get(self, job_key):
         return Job(self._client, str(job_key))
 
     def summary(self, state=None, spider=None, **params):
-        r"""Get jobs summary (optionally by state).
+        """Get jobs summary (optionally by state).
 
         :param state: (optional) a string state to filter jobs.
         :param spider: (optional) a spider name (not needed if instantiated
             with :class:`~scrapinghub.client.spiders.Spider`).
-        :param \*\*params: (optional) additional keyword args.
+        :param params: (optional) additional keyword args.
         :return: a list of dictionaries of jobs summary
             for a given filter params grouped by job state.
         :rtype: :class:`list[dict]`
@@ -353,14 +353,14 @@ def summary(self, state=None, spider=None, **params):
 
     def iter_last(self, start=None, start_after=None, count=None,
                   spider=None, **params):
-        r"""Iterate through last jobs for each spider.
+        """Iterate through last jobs for each spider.
 
         :param start: (optional)
         :param start_after: (optional)
         :param count: (optional)
         :param spider: (optional) a spider name (not needed if instantiated
             with :class:`~scrapinghub.client.spiders.Spider`).
-        :param \*\*params: (optional) additional keyword args.
+        :param params: (optional) additional keyword args.
         :return: a generator object over a list of dictionaries of jobs summary
             for a given filter params.
         :rtype: :class:`types.GeneratorType[dict]`
@@ -508,9 +508,9 @@ def close_writers(self):
         self._job.close_writers()
 
     def start(self, **params):
-        r"""Move job to running state.
+        """Move job to running state.
 
-        :param \*\*params: (optional) keyword meta parameters to update.
+        :param params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
         :rtype: :class:`str`
 
@@ -522,9 +522,9 @@ def start(self, **params):
         return self.update(state='running', **params)
 
     def finish(self, **params):
-        r"""Move running job to finished state.
+        """Move running job to finished state.
 
-        :param \*\*params: (optional) keyword meta parameters to update.
+        :param params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
         :rtype: :class:`str`
 
@@ -536,9 +536,9 @@ def finish(self, **params):
         return self.update(state='finished', **params)
 
     def delete(self, **params):
-        r"""Mark finished job for deletion.
+        """Mark finished job for deletion.
 
-        :param \*\*params: (optional) keyword meta parameters to update.
+        :param params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
         :rtype: :class:`str`
 
@@ -550,10 +550,10 @@ def delete(self, **params):
         return self.update(state='deleted', **params)
 
     def update(self, state, **params):
-        r"""Update job state.
+        """Update job state.
 
         :param state: a new job state.
-        :param \*\*params: (optional) keyword meta parameters to update.
+        :param params: (optional) keyword meta parameters to update.
         :return: a previous string job state.
         :rtype: :class:`str`