tests: test rebuild of EC object with 2 corrupt chunks

Author: fvennetier

tests/functional/blob/test_rebuilder.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2018-2019 OpenIO SAS, as part of OpenIO SDS
+# Copyright (C) 2022 OVH SAS
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -21,8 +22,9 @@
 from oio.common.utils import cid_from_name
 from oio.common.constants import OIO_VERSION
 from oio.common.fullpath import encode_fullpath
+from oio.common.storage_method import ECDriverError
 from oio.blob.rebuilder import BlobRebuilder
-from tests.utils import BaseTestCase, random_str
+from tests.utils import BaseTestCase, random_data, random_str
 from tests.functional.blob import convert_to_old_chunk
 
 
@@ -53,7 +55,8 @@ def setUp(self):
             self.rawx_volumes[service_id] = volume
 
         self.api.object_create(
-            self.account, self.container, obj_name=self.path, data="chunk")
+            self.account, self.container, obj_name=self.path,
+            data=random_data(50 * 1024))
         meta, self.chunks = self.api.object_locate(
             self.account, self.container, self.path)
         self.version = meta['version']
@@ -66,6 +69,65 @@ def _chunk_path(self, chunk):
         volume = self.rawx_volumes[volume_id]
         return volume + '/' + chunk_id[:3] + '/' + chunk_id
 
+    def _corrupt_chunk(self, chunk, offset=7):
+        chunk_path = self._chunk_path(chunk)
+        self.logger.debug("Corrupting chunk %s", chunk_path)
+        with open(chunk_path, 'rb+') as chunk_fd:
+            chunk_fd.seek(offset, os.SEEK_SET)
+            last_byte = chunk_fd.read(1)
+            last_byte = chr((ord(last_byte) - 1) % 256)
+            chunk_fd.seek(offset, os.SEEK_SET)
+            chunk_fd.write(last_byte)
+            chunk_fd.flush()
+
+    def test_rebuild_with_corrupt_input(self):
+        """
+        The the rebuild of a missing chunk while 2 other chunks are corrupt.
+        Notice that we corrupt the chunk's EC preamble, not the chunk's data
+        segment.
+        """
+        if self.conf['storage_policy'] != 'EC' \
+           or len(self.conf['services']['rawx']) < 9:
+            self.skipTest("Will run only with 'EC' storage policy "
+                          + "and at least 9 rawx services")
+
+        # pick one chunk, remove it
+        removed_chunk = random.choice(self.chunks)
+        chunk_headers = self.blob_client.chunk_head(removed_chunk['url'])
+        removed_chunk_size = int(chunk_headers['chunk_size'])
+        os.remove(self._chunk_path(removed_chunk))
+        chunks_kept = list(self.chunks)
+        chunks_kept.remove(removed_chunk)
+
+        # pick two chunks, corrupt them
+        for _ in range(2):
+            corrupt_chunk = random.choice(chunks_kept)
+            chunks_kept.remove(corrupt_chunk)
+            self._corrupt_chunk(corrupt_chunk)
+
+        # run the rebuilder, check failure
+        chunk_id = removed_chunk['url'].split('/')[3]
+        chunk_volume = removed_chunk['url'].split('/')[2]
+        conf = self.conf.copy()
+        conf['allow_same_rawx'] = True
+        rebuilder = BlobRebuilder(conf, service_id=chunk_volume,
+                                  logger=self.logger)
+        rebuilder_worker = rebuilder.create_worker(None, None)
+        self.assertRaises(
+            ECDriverError,
+            rebuilder_worker._process_item,
+            (self.ns, self.cid, self.content_id, chunk_id))
+
+        # run the rebuilder with options, check success
+        conf['allow_same_rawx'] = True
+        conf['read_all_available_sources'] = True
+        rebuilder = BlobRebuilder(conf, service_id=chunk_volume,
+                                  logger=self.logger)
+        rebuilder_worker = rebuilder.create_worker(None, None)
+        rebuilt_bytes = rebuilder_worker._process_item(
+            (self.ns, self.cid, self.content_id, chunk_id))
+        self.assertEqual(removed_chunk_size, rebuilt_bytes)
+
     def test_rebuild_old_chunk(self):
         for c in self.chunks:
             convert_to_old_chunk(