Convert proxy logic to subclasses system

Author: vshlapakov

scrapinghub/client/activity.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-from .proxy import _Proxy
+from .proxy import _Proxy, format_iter_filters
 from .utils import parse_job_key
 
 
@@ -46,8 +46,10 @@ class Activity(_Proxy):
     """
     def __init__(self, *args, **kwargs):
         super(Activity, self).__init__(*args, **kwargs)
-        self._proxy_methods([('iter', 'list')])
-        self._wrap_iter_methods(['iter'])
+
+    def iter(self, **params):
+        params = format_iter_filters(params)
+        return self._origin.list(**params)
 
     def add(self, values, **kwargs):
         """Add new event to the project activity.

scrapinghub/client/collections.py

@@ -5,11 +5,11 @@
 
 from ..hubstorage.collectionsrt import Collection as _Collection
 
-from .proxy import _Proxy, proxy_methods, wrap_kwargs, format_iter_filters
+from .proxy import _Proxy, _DownloadableProxyMixin, format_iter_filters
 from .utils import update_kwargs
 
 
-class Collections(_Proxy):
+class Collections(_Proxy, _DownloadableProxyMixin):
     """Access to project collections.
 
     Not a public constructor: use :class:`~scrapinghub.client.projects.Project`
@@ -144,42 +144,6 @@ class Collection(object):
     def __init__(self, client, collections, type_, name):
         self._client = client
         self._origin = _Collection(type_, name, collections._origin)
-        proxy_methods(self._origin, self, [
-            'create_writer', 'count',
-            ('iter', 'iter_values'),
-            ('iter_raw_json', 'iter_json'),
-        ])
-        # simplified version of _Proxy._wrap_iter_methods logic
-        # to provide better support for filter param in iter methods
-        for method in ['iter', 'iter_raw_json']:
-            wrapped = wrap_kwargs(getattr(self, method), format_iter_filters)
-            setattr(self, method, wrapped)
-
-    def list(self, key=None, prefix=None, prefixcount=None, startts=None,
-             endts=None, requests_params=None, **params):
-        """Convenient shortcut to list iter results.
-
-        Please note that :meth:`list` method can use a lot of memory and for a
-        large amount of logs it's recommended to iterate through it
-        via :meth:`iter` method (all params and available filters are same for
-        both methods).
-
-        :param key: a string key or a list of keys to filter with.
-        :param prefix: a string prefix to filter items.
-        :param prefixcount: maximum number of values to return per prefix.
-        :param startts: UNIX timestamp at which to begin results.
-        :param endts: UNIX timestamp at which to end results.
-        :param requests_params: (optional) a dict with optional requests params.
-        :param \*\*params: (optional) additional query params for the request.
-        :return: a list of items where each item is represented with a dict.
-        :rtype: :class:`list[dict]`
-        """
-        # FIXME there should be similar docstrings for iter/iter_raw_json
-        # but as we proxy them as-is, it's not in place, should be improved
-        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
-                      startts=startts, endts=endts,
-                      requests_params=requests_params)
-        return list(self.iter(requests_params=None, **params))
 
     def get(self, key, **params):
         """Get item from collection by key.
@@ -215,6 +179,28 @@ def delete(self, keys):
                              "object providing string keys")
         self._origin.delete(keys)
 
+    def count(self, *args, **kwargs):
+        return self._origin._collections.count(
+            self._origin.coltype, self._origin.colname, *args, **kwargs)
+
+    def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
+             endts=None, requests_params=None, **params):
+        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
+                      startts=startts, endts=endts,
+                      requests_params=requests_params)
+        params = format_iter_filters(params)
+        return self._origin._collections.iter_values(
+            self._origin.coltype, self._origin.colname, **params)
+
+    def iter_raw_json(self, key=None, prefix=None, prefixcount=None,
+                      startts=None, endts=None, requests_params=None, **params):
+        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
+                      startts=startts, endts=endts,
+                      requests_params=requests_params)
+        params = format_iter_filters(params)
+        return self._origin._collections.iter_json(
+            self._origin.coltype, self._origin.colname, **params)
+
     def iter_raw_msgpack(self, key=None, prefix=None, prefixcount=None,
                          startts=None, endts=None, requests_params=None,
                          **params):
@@ -234,5 +220,33 @@ def iter_raw_msgpack(self, key=None, prefix=None, prefixcount=None,
         update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
                       startts=startts, endts=endts,
                       requests_params=requests_params)
+        params = format_iter_filters(params)
         return self._origin._collections.iter_msgpack(
             self._origin.coltype, self._origin.colname, **params)
+
+    def list(self, key=None, prefix=None, prefixcount=None, startts=None,
+             endts=None, requests_params=None, **params):
+        """Convenient shortcut to list iter results.
+
+        Please note that :meth:`list` method can use a lot of memory and for a
+        large amount of logs it's recommended to iterate through it
+        via :meth:`iter` method (all params and available filters are same for
+        both methods).
+
+        :param key: a string key or a list of keys to filter with.
+        :param prefix: a string prefix to filter items.
+        :param prefixcount: maximum number of values to return per prefix.
+        :param startts: UNIX timestamp at which to begin results.
+        :param endts: UNIX timestamp at which to end results.
+        :param requests_params: (optional) a dict with optional requests params.
+        :param \*\*params: (optional) additional query params for the request.
+        :return: a list of items where each item is represented with a dict.
+        :rtype: :class:`list[dict]`
+        """
+        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
+                      startts=startts, endts=endts)
+        return list(self.iter(requests_params=requests_params, **params))
+
+    def create_writer(self, **kwargs):
+        return self._origin._collections.create_writer(
+            self._origin.coltype, self._origin.colname, **kwargs)

scrapinghub/client/frontiers.py

@@ -85,7 +85,6 @@ class Frontiers(_Proxy):
     """
     def __init__(self, *args, **kwargs):
         super(Frontiers, self).__init__(*args, **kwargs)
-        self._proxy_methods(['close', 'flush'])
 
     def get(self, name):
         """Get a frontier by name.
@@ -121,6 +120,12 @@ def newcount(self):
         """
         return sum(self._origin.newcount.values())
 
+    def flush(self):
+        self._origin.flush()
+
+    def close(self):
+        self._origin.close()
+
 
 class Frontier(object):
     """Representation of a frontier object.

scrapinghub/client/items.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import
 
-from .proxy import _Proxy
+from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin
 
 
-class Items(_Proxy):
+class Items(_ItemsResourceProxy, _DownloadableProxyMixin):
     """Representation of collection of job items.
 
     Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instanc

scrapinghub/client/logs.py

@@ -1,11 +1,13 @@
 from __future__ import absolute_import
+
 import json
+import logging
 
-from .proxy import _Proxy
+from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin
 from .utils import LogLevel
 
 
-class Logs(_Proxy):
+class Logs(_ItemsResourceProxy, _DownloadableProxyMixin):
     """Representation of collection of job logs.
 
     Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance
@@ -47,11 +49,24 @@ class Logs(_Proxy):
             'time': 1486375511188,
         }]
     """
+    def log(self, message, level=logging.INFO, ts=None, **other):
+        self._origin.log(message, level=level, ts=ts, **other)
+
+    def debug(self, message, **other):
+        self._origin.debug(message, **other)
+
+    def info(self, message, **other):
+        self._origin.info(message, **other)
+
+    def warn(self, message, **other):
+        self._origin.warn(message, **other)
+    warning = warn
+
+    def error(self, message, **other):
+        self._origin.error(message, **other)
 
-    def __init__(self, *args, **kwargs):
-        super(Logs, self).__init__(*args, **kwargs)
-        self._proxy_methods(['log', 'debug', 'info', 'warning', 'warn',
-                             'error', 'batch_write_start'])
+    def batch_write_start(self):
+        return self._origin.batch_write_start()
 
     def _modify_iter_params(self, params):
         """Modify iter() filters on-the-fly.

scrapinghub/client/proxy.py

@@ -3,10 +3,6 @@
 import six
 import json
 
-from ..hubstorage.resourcetype import DownloadableResource
-from ..hubstorage.resourcetype import ItemsResourceType
-from ..hubstorage.collectionsrt import Collections
-
 from .exceptions import wrap_value_too_large
 
 
@@ -30,35 +26,18 @@ def __init__(self, cls, client, key):
         self._client = client
         self._origin = cls(client._hsclient, key)
 
-        if issubclass(cls, ItemsResourceType):
-            self._proxy_methods(['get', 'write', 'flush', 'close',
-                                 'stats', ('iter', 'list')])
-            # redefine write method to wrap hubstorage.ValueTooLarge error
-            origin_method = getattr(self, 'write')
-            setattr(self, 'write', wrap_value_too_large(origin_method))
-
-        # DType iter_values() has more priority than IType list()
-        # plus Collections interface doesn't need the iter methods
-        if issubclass(cls, DownloadableResource) and cls is not Collections:
-            methods = [('iter', 'iter_values'),
-                       ('iter_raw_msgpack', 'iter_msgpack'),
-                       ('iter_raw_json', 'iter_json')]
-            self._proxy_methods(methods)
-            self._wrap_iter_methods([method[0] for method in methods])
-
-    def _proxy_methods(self, methods):
-        """A little helper for cleaner interface."""
-        proxy_methods(self._origin, self, methods)
-
-    def _wrap_iter_methods(self, methods):
-        """Modify kwargs for all passed self.iter* methods."""
-        for method in methods:
-            wrapped = wrap_kwargs(getattr(self, method),
-                                  self._modify_iter_params)
-            setattr(self, method, wrapped)
+    def list(self, *args, **kwargs):
+        """Convenient shortcut to list iter results.
+
+        Please note that :meth:`list` method can use a lot of memory and for a
+        large amount of elements it's recommended to iterate through it via
+        :meth:`iter` method (all params and available filters are same for both
+        methods).
+        """
+        return list(self.iter(*args, **kwargs))
 
     def _modify_iter_params(self, params):
-        """A helper to modify iter() params on-the-fly.
+        """A helper to modify iter*() params on-the-fly.
 
         The method is internal and should be redefined in subclasses.
 
@@ -68,15 +47,43 @@ def _modify_iter_params(self, params):
         """
         return format_iter_filters(params)
 
-    def list(self, *args, **kwargs):
-        """Convenient shortcut to list iter results.
 
-        Please note that :meth:`list` method can use a lot of memory and for a
-        large amount of elements it's recommended to iterate through it via
-        :meth:`iter` method (all params and available filters are same for both
-        methods).
-        """
-        return list(self.iter(*args, **kwargs))
+class _ItemsResourceProxy(_Proxy):
+
+    def get(self, _key, **params):
+        return self._origin.get(_key, **params)
+
+    @wrap_value_too_large
+    def write(self, item):
+        return self._origin.write(item)
+
+    def iter(self, _key=None, **params):
+        params = self._modify_iter_params(params)
+        return self._origin.list(_key, **params)
+
+    def flush(self):
+        self._origin.flush()
+
+    def stats(self):
+        return self._origin.stats()
+
+    def close(self, block=True):
+        self._origin.close(block)
+
+
+class _DownloadableProxyMixin(object):
+
+    def iter(self, _path=None, requests_params=None, **apiparams):
+        apiparams = self._modify_iter_params(apiparams)
+        return self._origin.iter_values(_path, requests_params, **apiparams)
+
+    def iter_raw_json(self, _path=None, requests_params=None, **apiparams):
+        apiparams = self._modify_iter_params(apiparams)
+        return self._origin.iter_json(_path, requests_params, **apiparams)
+
+    def iter_raw_msgpack(self, _path=None, requests_params=None, **apiparams):
+        apiparams = self._modify_iter_params(apiparams)
+        return self._origin.iter_msgpack(_path, requests_params, **apiparams)
 
 
 class _MappingProxy(_Proxy):
@@ -130,25 +137,6 @@ def iter(self):
         return six.iteritems(next(self._origin.apiget()))
 
 
-def proxy_methods(origin, successor, methods):
-    """A helper to proxy methods from origin to successor.
-
-    Accepts a list with strings and tuples:
-
-    - each string defines:
-        a successor method name to proxy 1:1 with origin method
-    - each tuple should consist of 2 strings:
-        a successor method name and an origin method name
-    """
-    for method in methods:
-        if isinstance(method, tuple):
-            successor_name, origin_name = method
-        else:
-            successor_name, origin_name = method, method
-        if not hasattr(successor, successor_name):
-            setattr(successor, successor_name, getattr(origin, origin_name))
-
-
 def format_iter_filters(params):
     """Format iter() filter param on-the-fly.
 
@@ -168,11 +156,3 @@ def format_iter_filters(params):
         if filter_data:
             params['filter'] = filter_data
     return params
-
-
-def wrap_kwargs(fn, kwargs_fn):
-    """Tiny wrapper to prepare modified version of function kwargs"""
-    def wrapped(*args, **kwargs):
-        kwargs = kwargs_fn(kwargs)
-        return fn(*args, **kwargs)
-    return wrapped

scrapinghub/client/requests.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import
 
-from .proxy import _Proxy
+from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin
 
 
-class Requests(_Proxy):
+class Requests(_ItemsResourceProxy, _DownloadableProxyMixin):
     """Representation of collection of job requests.
 
     Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance
@@ -41,6 +41,6 @@ class Requests(_Proxy):
             'url': 'https://example.com'
         }]
     """
-    def __init__(self, *args, **kwargs):
-        super(Requests, self).__init__(*args, **kwargs)
-        self._proxy_methods(['add'])
+    def add(self, url, status, method, rs, parent, duration, ts, fp=None):
+        return self._origin.add(
+            url, status, method, rs, parent, duration, ts, fp=None)

scrapinghub/client/samples.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import
 
-from .proxy import _Proxy
+from .proxy import _ItemsResourceProxy
 
 
-class Samples(_Proxy):
+class Samples(_ItemsResourceProxy):
     """Representation of collection of job samples.
 
     Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance