Report exceeded max encoded size in MB if applied

Author: vshlapakov

scrapinghub/hubstorage/batchuploader.py

@@ -12,7 +12,7 @@
 import requests
 from collections import deque
 from threading import Thread, Event
-from .utils import xauth, iterqueue
+from .utils import xauth, iterqueue, sizeof_fmt
 from .serialization import jsonencode
 
 logger = logging.getLogger('hubstorage.batchuploader')
@@ -227,8 +227,8 @@ def write(self, item):
         if len(data) > self.maxitemsize:
             truncated_data = data[:self.ERRMSG_DATA_TRUNCATION_LEN] + "..."
             raise ValueTooLarge(
-                'Value exceeds max encoded size of {} bytes: {!r}'
-                .format(self.maxitemsize, truncated_data))
+                'Value exceeds max encoded size of {}: {!r}'
+                .format(sizeof_fmt(self.maxitemsize), truncated_data))
 
         self.itemsq.put(data)
         if self.itemsq.full():

scrapinghub/hubstorage/utils.py

@@ -131,3 +131,21 @@ def apipoll(endpoint, *args, **kwargs):
         if result is not None or (time.time() - start) > max_poll:
             return result
 
+
+def sizeof_fmt(num):
+    """Little helper to get size in human readable form.
+
+    Size is rounded to a closest integer value (for simplicity).
+
+    >>> sizeof_fmt(100)
+    '100 B'
+    >>> sizeof_fmt(1024)
+    '1 KiB'
+    >>> sizeof_fmt(1024*1024 + 100)
+    '1 MiB'
+    """
+    for unit in ['B', 'KiB', 'MiB']:
+        if abs(num) < 1024.0:
+            return "%.0f %s" % (num, unit)
+        num /= 1024.0
+    return "%.0f %s" % (num, 'GiB')

tests/hubstorage/cassetes/test_batchuploader/test_writer_maxitemsize_custom.gz

@@ -0,0 +1 @@
+eJydVQl320QQTtrSltKTq4W2YEyhJo1ky7dDOdI0BOc+nGQbEGEtbbyyZUmj1Sax+/Qe/Lv+LGbXoYQcrwX5+T3tzOzs7PfNN/rjQgpjOXJ9bGzMCxIWUyfxwkDAuA0XcilczJGr6IqZiNDK4FIK7+TIFTRxRl0WC7icwpXR/pkQEwSJ0RpEDK7a8C65i1YaRb7nUJU1f2h0RRj4XsAEXKPk2mhTwPSZ8J4N17Wtx1hkUN/bZ3CDkvtomXYcJoShTohD35j2/fDAWIm9jhfATRtukXEMmoDblFzGlw0W77MY7tjwvt69zeLekMlOvmA2zGpmdZBwLKZkVkwLPqDkDoYQQ23yHGZs4aVUNR/a8BG5iC6X7cPHlFzC1+c0YXDXhnvkIa5akk1mSlZmngaZYsGqZazqVNmasgqZuaUWfELJzWOgLLKgk3D41Ib75ALaSw14QKUuVyQ0kQIepvDZCNo+3pV2GHyuI1cWIKNPd0KXwRcLr6RetUN3ANkUvjzKEXtBBx6Rx7h4me2xQXYqky2OnryVtxq1RnYyk6Uy4egJpO+n8JWQ+ryYgWQiga9TeJzjSGgup3nYFCw2pjtYPnxjwwS5h7ZIg2cIJ6YRnshlO28hqgV4QsmtI6aixJgNsFpV0aQNBrmBjs7QiyYzLtvzFYjmKfbzNhROsm9RvXUaiw5jb6ibCIo2lEgRzc+o8JzMTv9tfy+qUB41yKhIqNhQ1RRP5CegdiZfdRsamoViHaao5FnyQAMsIg+7X2HMhZEgeMaRJYVvdUoZe/CUlJRQkiSayudFEsZIqrnfN4+BZzphP98N25D/m6tICg7f6TL7DO/twvea79WVjRb8IGUKP+Y4ynBaM/XsDOnN2PCc3D4hPSU8mD2F+k82zJ1E/ec3a65pwzz/t9oWbFh8o9qWzlPbsg0rr9W2ekxtazasv4XaNs5kr2XDps5aqldgC+lDjW3n+GtdEa7kxJG0F/wR+XNcM4tqZIrYWAYBkqRUIxzOXOkzd7etdUXdvhcox7k6i1ig2n838foqmVWuV+r1Qr1WqBXKyh172NCJ2lxU+RMaJ6ezH1VwOkm9YKFbBl4ilP0fXWfZYJ635xxvxZtvbg6b1rLXFM1gveLMNKvNXkS2ZuYbpgpyt9dUELSLzepStznQf69+uDicrSwPe97emukah9u8Tg66Tnl9vT3o+3vhTLXld6o7XRFYpF01llYr5R73GwUN0nmSQN+uwF7fPQjj3iiiJ/uhhikOu9iJCobRk8KOkBwH0S+6vX/NcZw8Nn9COQ6S37hJOc6JXW5RjmPg9/87BijlqP02r1GOAnf016MIrta3ao2XKTD+lNT+u3o1k8qyx4/JtiMl05N2/6jd+cK4NP8Ch5o5Rw==

tests/hubstorage/test_batchuploader.py

@@ -41,22 +41,31 @@ def test_writer_maxitemsize(hsclient, hsproject):
     with pytest.raises(ValueTooLarge) as excinfo1:
         writer.write({'b': 'x' * max_size})
     excinfo1.match(
-        r'Value exceeds max encoded size of 1048576 bytes:'
+        r'Value exceeds max encoded size of 1 MiB:'
         ' \'{"b": "x+\\.\\.\\.\'')
 
     with pytest.raises(ValueTooLarge) as excinfo2:
         writer.write({'b'*max_size: 'x'})
     excinfo2.match(
-        r'Value exceeds max encoded size of 1048576 bytes:'
+        r'Value exceeds max encoded size of 1 MiB:'
         ' \'{"b+\\.\\.\\.\'')
 
     with pytest.raises(ValueTooLarge) as excinfo3:
         writer.write({'b'*(max_size//2): 'x'*(max_size//2)})
     excinfo3.match(
-        r'Value exceeds max encoded size of 1048576 bytes:'
+        r'Value exceeds max encoded size of 1 MiB:'
         ' \'{"b+\\.\\.\\.\'')
 
 
+def test_writer_maxitemsize_custom(hsclient, hsproject):
+    _, writer = _job_and_writer(hsclient, hsproject, maxitemsize=512*1024)
+    with pytest.raises(ValueTooLarge) as excinfo:
+        writer.write({'b': 'x' * writer.maxitemsize})
+    excinfo.match(
+        r'Value exceeds max encoded size of 512 KiB:'
+        ' \'{"b": "x+\\.\\.\\.\'')
+
+
 def test_writer_contentencoding(hsclient, hsproject):
     for ce in ('identity', 'gzip'):
         job, writer = _job_and_writer(hsclient, hsproject,

tests/hubstorage/test_utils.py

@@ -0,0 +1,13 @@
+"""
+Test utils module.
+"""
+
+from scrapinghub.hubstorage.utils import sizeof_fmt
+
+
+def test_sizeof_fmt():
+    assert sizeof_fmt(1000) == '1000 B'
+    assert sizeof_fmt(1024) == '1 KiB'
+    assert sizeof_fmt(1024 * 1024) == '1 MiB'
+    assert sizeof_fmt(1024 * 1024 + 100) == '1 MiB'
+    assert sizeof_fmt(1024 * 1024 * 1024) == '1 GiB'