Add retries for more calls #126 (#131)

Author: hermit-crab

scrapinghub/hubstorage/collectionsrt.py

@@ -63,7 +63,7 @@ def delete(self, _type, _name, _keys):
         return self.apipost((_type, _name, 'deleted'), is_idempotent=True, jl=_keys)
 
     def truncate(self, _name):
-        return self.apipost('delete', params={'name': _name})
+        return self.apipost('delete', params={'name': _name}, is_idempotent=True)
 
     def iter_json(self, _type, _name, requests_params=None, **apiparams):
         return DownloadableResource.iter_json(self, (_type, _name),
@@ -115,7 +115,10 @@ def _batch(self, method, path, total_param, progress=None, **params):
         getparams = dict(params)
         try:
             while True:
-                r = next(self.apirequest(path, method=method, params=getparams))
+                r = next(self.apirequest(
+                    path, method=method, params=getparams,
+                    is_idempotent=method=='GET',
+                ))
                 total += r[total_param]
                 next_start = r.get('nextstart')
                 if next_start is None:

scrapinghub/hubstorage/frontier.py

@@ -59,7 +59,7 @@ def read(self, frontier, slot, mincount=None):
         return self.apiget((frontier, 's', slot, 'q'), params=params)
 
     def delete(self, frontier, slot, ids):
-        self.apipost((frontier, 's', slot, 'q/deleted'), jl=ids)
+        self.apipost((frontier, 's', slot, 'q/deleted'), jl=ids, is_idempotent=True)
 
     def delete_slot(self, frontier, slot):
         self.apidelete((frontier, 's', slot))

scrapinghub/hubstorage/resourcetype.py

@@ -158,6 +158,7 @@ def iter_msgpack(self, _path=None, requests_params=None, **apiparams):
         requests_params = dict(requests_params or {})
         requests_params.setdefault('method', 'GET')
         requests_params.setdefault('stream', True)
+        requests_params.setdefault('is_idempotent', True)
         requests_params = self._enforce_msgpack(**requests_params)
         for chunk in self._retry(self._iter_content, False, _path,
                                  requests_params, **apiparams):
@@ -168,6 +169,7 @@ def iter_json(self, _path=None, requests_params=None, **apiparams):
         requests_params = dict(requests_params or {})
         requests_params.setdefault('method', 'GET')
         requests_params.setdefault('stream', True)
+        requests_params.setdefault('is_idempotent', True)
         for line in self._retry(self._iter_lines, True, _path, requests_params,
                                 **apiparams):
             yield line