Added an ability to check table's integrity (#257)

* Added an ability to checks table's integrity

* Fixed error message

* Fixed linting

Author: roll

README.md

@@ -214,13 +214,14 @@ Constructor to instantiate `Table` class. If `references` argument is provided,
 
 - `(str/None)` - returns the table's SHA256 hash if it's already read using e.g. `table.read`, otherwise returns `None`. In the middle of an iteration it returns hash of already read contents
 
-#### `table.iter(keyed=Fase, extended=False, cast=True, relations=False, foreign_keys_values=False)`
+#### `table.iter(keyed=Fase, extended=False, cast=True, integrity=False, relations=False, foreign_keys_values=False)`
 
 Iterates through the table data and emits rows cast based on table schema. Data casting can be disabled.
 
 - `keyed (bool)` - iterate keyed rows
 - `extended (bool)` - iterate extended rows
 - `cast (bool)` - disable data casting if false
+- `integrity` (dict) - dictionary in a form of `{'size': <bytes>, 'hash': '<sha256>'}` to check integrity of the table when it's read completely. Both keys are optional.
 - `relations (dict)` - dictionary of foreign key references in a form of `{resource1: [{field1: value1, field2: value2}, ...], ...}`. If provided, foreign key fields will checked and resolved to one of their references (/!\ one-to-many fk are not completely resolved).
 - `foreign_keys_values (dict)` - three-level dictionary of foreign key references optimized to speed up validation process in a form of `{resource1: { (foreign_key_field1, foreign_key_field2) : { (value1, value2) : {one_keyedrow}, ... }}}`. If not provided but relations is true, it will be created before the validation process by *index_foreign_keys_values* method
 - `(exceptions.TableSchemaException)` - raises any error that occurs during this process
@@ -229,13 +230,14 @@ Iterates through the table data and emits rows cast based on table schema. Data
   - `{header1: value1, header2: value2}` - keyed
   - `[rowNumber, [header1, header2], [value1, value2]]` - extended
 
-#### `table.read(keyed=False, extended=False, cast=True, relations=False, limit=None, foreign_keys_values=False)`
+#### `table.read(keyed=False, extended=False, cast=True, integrity=False, relations=False, limit=None, foreign_keys_values=False)`
 
 Read the whole table and returns as array of rows. Count of rows could be limited.
 
 - `keyed (bool)` - flag to emit keyed rows
 - `extended (bool)` - flag to emit extended rows
 - `cast (bool)` - flag to disable data casting if false
+- `integrity` (dict) - dictionary in a form of `{'size': <bytes>, 'hash': '<sha256>'}` to check integrity of the table when it's read completely. Both keys are optional.
 - `relations (dict)` - dict of foreign key references in a form of `{resource1: [{field1: value1, field2: value2}, ...], ...}`. If provided foreign key fields will checked and resolved to its references
 - `limit (int)` - integer limit of rows to return
 - `foreign_keys_values (dict)` - three-level dictionary of foreign key references optimized to speed up validation process in a form of `{resource1: { (foreign_key_field1, foreign_key_field2) : { (value1, value2) : {one_keyedrow}, ... }}}`
@@ -646,10 +648,14 @@ All validation errors.
 
 All value cast errors.
 
-#### `exceptions.RelationError`
+#### `exceptions.IntegrityError`
 
 All integrity errors.
 
+#### `exceptions.RelationError`
+
+All relations errors.
+
 #### `exceptions.StorageError`
 
 All storage errors.

pylama.ini

@@ -6,7 +6,7 @@ ignore = E128,E301,E305,E731
 max_line_length = 100
 
 [pylama:mccabe]
-complexity = 24
+complexity = 36
 
 [pylama:*/__init__.py]
 ignore = W0611

tableschema/exceptions.py

@@ -40,6 +40,10 @@ class CastError(TableSchemaException):
     pass
 
 
+class IntegrityError(TableSchemaException):
+    pass
+
+
 class RelationError(TableSchemaException):
     pass
 

tableschema/table.py

@@ -80,7 +80,8 @@ def hash(self):
         if self.__stream:
             return self.__stream.hash
 
-    def iter(self, keyed=False, extended=False, cast=True, relations=False,
+    def iter(self, keyed=False, extended=False, cast=True,
+             integrity=False, relations=False,
              foreign_keys_values=False):
         """https://github.com/frictionlessdata/tableschema-py#table
         """
@@ -169,16 +170,31 @@ def iter(self, keyed=False, extended=False, cast=True, relations=False,
             else:
                 yield row
 
+        # Check integrity
+        if integrity:
+            violations = []
+            size = integrity.get('size')
+            hash = integrity.get('hash')
+            if size and size != self.__stream.size:
+                violations.append('size "%s"' % self.__stream.size)
+            if hash and hash != self.__stream.hash:
+                violations.append('hash "%s"' % self.__stream.hash)
+            if violations:
+                message = 'Calculated %s differ(s) from declared value(s)'
+                raise exceptions.IntegrityError(message % ' and '.join(violations))
+
         # Close stream
         self.__stream.close()
 
-    def read(self, keyed=False, extended=False, cast=True, relations=False, limit=None,
+    def read(self, keyed=False, extended=False, cast=True, limit=None,
+             integrity=False, relations=False,
              foreign_keys_values=False):
         """https://github.com/frictionlessdata/tableschema-py#table
         """
         result = []
-        rows = self.iter(keyed=keyed, extended=extended, cast=cast, relations=relations,
-                         foreign_keys_values=foreign_keys_values)
+        rows = self.iter(keyed=keyed, extended=extended, cast=cast,
+            integrity=integrity, relations=relations,
+            foreign_keys_values=foreign_keys_values)
         for count, row in enumerate(rows, start=1):
             result.append(row)
             if count == limit:

tests/test_table.py

@@ -194,25 +194,28 @@ def test_read_with_headers_field_names_mismatch():
     assert 'match schema field names' in str(excinfo.value)
 
 
-# Stats
+# Stats/integrity
+
+SIZE = 63
+HASH = '328adab247692a1a405e83c2625d52e366389eabf8a1824931187877e8644774'
 
 def test_size():
     table = Table('data/data.csv')
     table.read()
-    assert table.size == 63
+    assert table.size == SIZE
 
 
 @pytest.mark.skipif(six.PY2, reason='Support only for Python3')
 def test_size_compressed():
     table = Table('data/data.csv.zip')
     table.read()
-    assert table.size == 63
+    assert table.size == SIZE
 
 
 def test_size_remote():
     table = Table(BASE_URL % 'data/data.csv')
     table.read()
-    assert table.size == 63
+    assert table.size == SIZE
 
 
 def test_size_not_read():
@@ -223,27 +226,66 @@ def test_size_not_read():
 def test_hash():
     table = Table('data/data.csv')
     table.read()
-    assert table.hash == '328adab247692a1a405e83c2625d52e366389eabf8a1824931187877e8644774'
+    assert table.hash == HASH
 
 
 @pytest.mark.skipif(six.PY2, reason='Support only for Python3')
 def test_hash_compressed():
     table = Table('data/data.csv.zip')
     table.read()
-    assert table.hash == '328adab247692a1a405e83c2625d52e366389eabf8a1824931187877e8644774'
+    assert table.hash == HASH
 
 
 def test_hash_remote():
     table = Table(BASE_URL % 'data/data.csv')
     table.read()
-    assert table.hash == '328adab247692a1a405e83c2625d52e366389eabf8a1824931187877e8644774'
+    assert table.hash == HASH
 
 
 def test_hash():
     table = Table(BASE_URL % 'data/data.csv')
     assert table.hash is None
 
 
+def test_read_integrity():
+    table = Table('data/data.csv')
+    table.read(integrity={'size': SIZE, 'hash': HASH})
+    assert True
+
+def test_read_integrity_error():
+    table = Table('data/data.csv')
+    with pytest.raises(exceptions.IntegrityError) as excinfo:
+        table.read(integrity={'size': SIZE + 1, 'hash': HASH + 'a'})
+    assert str(SIZE) in str(excinfo.value)
+    assert HASH in str(excinfo.value)
+
+
+def test_read_integrity_size():
+    table = Table('data/data.csv')
+    table.read(integrity={'size': SIZE})
+    assert True
+
+
+def test_read_integrity_size_error():
+    table = Table('data/data.csv')
+    with pytest.raises(exceptions.IntegrityError) as excinfo:
+        table.read(integrity={'size': SIZE + 1})
+    assert str(SIZE) in str(excinfo.value)
+
+
+def test_read_integrity_hash():
+    table = Table('data/data.csv')
+    table.read(integrity={'hash': HASH})
+    assert True
+
+
+def test_read_integrity_hash_error():
+    table = Table('data/data.csv')
+    with pytest.raises(exceptions.IntegrityError) as excinfo:
+        table.read(integrity={'hash': HASH + 'a'})
+    assert HASH in str(excinfo.value)
+
+
 # Foreign keys
 
 FK_SOURCE = [