Add random read test

Author: Ravi Nagarjun Akella

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeBlocksConan(ConanFile):
     name = "homeblocks"
-    version = "1.0.17"
+    version = "1.0.18"
     homepage = "https://github.com/eBay/HomeBlocks"
     description = "Block Store built on HomeStore"
     topics = ("ebay")

src/lib/volume/tests/test_common.hpp

@@ -213,12 +213,6 @@ class HBTestHelper {
         }
     }
 
-    static void validate_zeros(uint8_t const* buf, uint64_t size) {
-        sisl::io_blob_safe blob(size, 512);
-        std::memset(blob.bytes(), 0, size);
-        RELEASE_ASSERT_EQ(std::memcmp(buf, blob.bytes(), size), 0, "data_buf mismatch");
-    }
-
 private:
     void init_devices(bool is_file, uint64_t dev_size = 0) {
         if (is_file) {

src/lib/volume/tests/test_volume_io.cpp

@@ -174,19 +174,6 @@ class VolumeIOImpl {
             });
     }
 
-<<<<<<< HEAD
-    void verify_all_data() {
-        for (auto& [lba, data_pattern] : m_lba_data) {
-            auto buffer = iomanager.iobuf_alloc(512, 4096);
-            vol_interface_req_ptr req(new vol_interface_req{buffer, lba, 1});
-
-            auto vol_mgr = g_helper->inst()->volume_manager();
-            vol_mgr->read(m_vol_ptr, req).get();
-            test_common::HBTestHelper::validate_data_buf(buffer, 4096, data_pattern);
-            LOGDEBUG("Verify data vol={} lba={} pattern={} {}", m_vol_name, lba, data_pattern,
-                     *r_cast< uint64_t* >(buffer));
-            iomanager.iobuf_free(buffer);
-=======
     void read_and_verify(lba_t start_lba, uint32_t nlbas) {
         auto sz = nlbas * m_vol_ptr->info()->page_size;
         sisl::io_blob_safe read_blob(sz, 512);
@@ -209,7 +196,7 @@ class VolumeIOImpl {
         } 
     }
 
-    void verify_data(uint64_t nlbas_per_io = 1) {
+    void verify_all_data(uint64_t nlbas_per_io = 1) {
         auto start_lba = m_lba_data.begin()->first;
         auto max_lba = m_lba_data.rbegin()->first;
         verify_data(start_lba, max_lba, nlbas_per_io);
@@ -221,7 +208,6 @@ class VolumeIOImpl {
             auto num_lbas_this_round = std::min(nlbas_per_io, max_lba - lba);
             read_and_verify(lba, num_lbas_this_round);
             num_lbas_verified += num_lbas_this_round;
->>>>>>> read API implementation
         }
         LOGINFO("Verified {} lbas for volume {}", num_lbas_verified, m_vol_ptr->info()->name);
     }
@@ -283,36 +269,34 @@ class VolumeIOTest : public ::testing::Test {
         LOGINFO("IO completed");
     }
 
-<<<<<<< HEAD
-    void verify_all_data(shared< VolumeIOImpl > vol_impl = nullptr) {
-        if (vol_impl) {
-            vol_impl->verify_all_data();
-=======
-    void verify_data(shared< VolumeIOImpl > vol_impl = nullptr, uint64_t nlbas_per_io = 1) {
+    void verify_all_data(shared< VolumeIOImpl > vol_impl = nullptr, uint64_t nlbas_per_io = 1) {
         if (vol_impl) {
-            vol_impl->verify_data(nlbas_per_io);
->>>>>>> read API implementation
+            vol_impl->verify_all_data(nlbas_per_io);
             return;
         }
 
         for (auto& vol_impl : m_vols_impl) {
-<<<<<<< HEAD
-            vol_impl->verify_all_data();
+            vol_impl->verify_all_data(nlbas_per_io);
         }
     }
 
     void restart(int shutdown_delay) {
         g_helper->restart(shutdown_delay);
         for (auto& vol_impl : m_vols_impl) {
             vol_impl->reset();
-=======
-            vol_impl->verify_data(nlbas_per_io);
->>>>>>> read API implementation
         }
     }
 
     std::vector< shared< VolumeIOImpl > >& volume_list() { return m_vols_impl; }
 
+    template < typename T >
+    T get_random_number(T min, T max) {
+        static std::random_device rd;
+        static std::mt19937 gen(rd());
+        std::uniform_int_distribution< T > dis(min, max);
+        return dis(gen);
+    }
+
 private:
     std::vector< shared< VolumeIOImpl > > m_vols_impl;
 };
@@ -333,34 +317,25 @@ TEST_F(VolumeIOTest, SingleVolumeWriteData) {
     restart(5);
 
     LOGINFO("Verify data");
-<<<<<<< HEAD
     verify_all_data(vol);
-=======
-    verify_data(vol, 30 /* nlbas_per_io */);
->>>>>>> read API implementation
+    //verify_data(vol, 30 /* nlbas_per_io */);
 
     // Write and verify again on same LBA range to single volume multiple times.
     LOGINFO("Write and verify data with num_iter={} start={} nblks={}", num_iter, start_lba, nblks);
     for (uint32_t i = 0; i < num_iter; i++) {
         generate_io_single(vol, start_lba, nblks);
     }
-<<<<<<< HEAD
 
-    verify_all_data(vol);
-=======
-    verify_data(vol);
-
-    // verify random lba ranges
+    verify_all_data(vol, 30 /* nlbas_per_io */);
 
->>>>>>> read API implementation
     LOGINFO("SingleVolumeWriteData test done.");
 }
 
 TEST_F(VolumeIOTest, SingleVolumeReadData) {
     // Write and verify fixed LBA range to single volume multiple times.
     auto vol = volume_list().back();
-    uint32_t nblks = 500;
-    lba_t start_lba = 1000;
+    uint32_t nblks = 5000;
+    lba_t start_lba = 500;
     uint32_t num_iter = 1;
     LOGINFO("Write and verify data with num_iter={} start={} nblks={}", num_iter, start_lba, nblks);
     for (uint32_t i = 0; i < num_iter; i++) {
@@ -371,7 +346,17 @@ TEST_F(VolumeIOTest, SingleVolumeReadData) {
     vol->verify_data(2000, 3000, 40);
     vol->verify_data(800, 1800, 40);
 
-    LOGINFO("SingleVolumeReadHoles test done.");
+    // random reads
+    num_iter = 100;
+    for(uint32_t i = 0; i < num_iter; i++) {
+        auto start_lba = get_random_number< lba_t >(0, 10000);
+        auto nblks = get_random_number< uint32_t >(1, 64);
+        auto no_lbas_per_io = get_random_number< uint64_t >(1, 50);
+        LOGINFO("iter {}: Read data start={} nblks={} no_lbas_per_io {}", i, start_lba, nblks, no_lbas_per_io);
+        vol->verify_data(start_lba, start_lba + nblks, no_lbas_per_io);
+    }
+
+    LOGINFO("SingleVolumeRead test done.");
 }
 
 TEST_F(VolumeIOTest, MultipleVolumeWriteData) {

src/lib/volume_mgr.cpp

@@ -259,26 +259,29 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::read(const VolumePtr& vol, const
     // TODO: check if the system is accepting ios (shutdown in progress etc)
     RELEASE_ASSERT(vol != nullptr, "VolumePtr is null");
     // Step 1: get the blk ids from index table
-    std::vector< std::pair< VolumeIndexKey, VolumeIndexValue > > out_vector;
-    if(auto index_resp = read_from_index(vol, req, out_vector); index_resp.hasError()) {
+    std::vector< std::pair< VolumeIndexKey, VolumeIndexValue > > index_kvs;
+    if(auto index_resp = read_from_index(vol, req, index_kvs); index_resp.hasError()) {
         LOGE("Failed to read from index table for range=[{}, {}], volume id: {}, error: {}",
              req->lba, req->end_lba(), boost::uuids::to_string(vol->id()), index_resp.error());
         return index_resp;
     }
+    if (index_kvs.empty()) {
+        return folly::Unit();
+    }
 
     // Step 2: Consolidate the blk ids and issue read requests
     std::vector< folly::Future< std::error_code > > futs;
     auto* read_buf = req->buffer;
     DEBUG_ASSERT(read_buf != nullptr, "Read buffer is null");
     auto cur_lba = req->lba;
-    for (uint32_t i = 0, blk_count = 0, start_idx = 0; i < out_vector.size(); ++i, ++cur_lba) {
-        auto const& [key, value] = out_vector[i];
+    for (uint32_t i = 0, blk_count = 0, start_idx = 0; i < index_kvs.size(); ++i, ++cur_lba) {
+        auto const& [key, value] = index_kvs[i];
         // cur_lba is used to keep track of the holes
         // move the read buffer by the size of the holes
         if(cur_lba != key.key()) {
             if(blk_count > 0) {
                 // submit the read for the previous blkids
-                submit_read_to_backend(read_buf, futs, out_vector[start_idx], blk_count, vol, req->part_of_batch);
+                submit_read_to_backend(read_buf, futs, index_kvs[start_idx], blk_count, vol, req->part_of_batch);
                 start_idx = i;
                 blk_count = 0;
             }
@@ -288,30 +291,30 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::read(const VolumePtr& vol, const
         }
 
         // Contiguous blkids are merged into a single read request
-        bool is_contiguous = (i == 0 || value.blkid().blk_num() == out_vector[i-1].second.blkid().blk_num() + 1);
+        bool is_contiguous = (i == 0 || (value.blkid().blk_num() == index_kvs[i-1].second.blkid().blk_num() + 1
+                                && value.blkid().chunk_num() == index_kvs[i-1].second.blkid().chunk_num())); 
         if(is_contiguous) {
             blk_count++;
-            if(i < out_vector.size() - 1) {
+            if(i < index_kvs.size() - 1) {
                 continue;
             }
         }
         // submit the read for the previous blkids
-        submit_read_to_backend(read_buf, futs, out_vector[start_idx], blk_count, vol, req->part_of_batch);
-        if(out_vector[start_idx].second.blkid().blk_num() + blk_count - 1 == value.blkid().blk_num()) {
-            // this is the last entry in the out vector
+        submit_read_to_backend(read_buf, futs, index_kvs[start_idx], blk_count, vol, req->part_of_batch);
+        if(index_kvs[start_idx].second.blkid().blk_num() + blk_count - 1 == value.blkid().blk_num()) {
+            // this is the last entry in the index_kvs
             continue;
         }
 
         // reset the blkids and size for the next read
         blk_count = 0;
         start_idx = i;
-        if(i == out_vector.size() - 1) {
-            submit_read_to_backend(read_buf, futs, out_vector[start_idx], 1, vol, req->part_of_batch);
+        if(i == index_kvs.size() - 1) {
+            submit_read_to_backend(read_buf, futs, index_kvs[start_idx], 1, vol, req->part_of_batch);
         }
     }
 
-    return out_vector.empty() ? folly::Unit() :
-      folly::collectAllUnsafe(futs).thenValue([out_vector = std::move(out_vector), buf = req->buffer
+    return  folly::collectAllUnsafe(futs).thenValue([index_kvs = std::move(index_kvs), buf = req->buffer
         , start_lba = req->lba, blk_size = vol->rd()->get_blk_size()](auto&& vf) -> VolumeManager::Result< folly::Unit > {
         for (auto const& err_c : vf) {
             if (sisl_unlikely(err_c.value())) {
@@ -320,8 +323,8 @@ VolumeManager::NullAsyncResult HomeBlocksImpl::read(const VolumePtr& vol, const
             }
             // verify the checksum
             auto read_buf = buf;
-            for(uint64_t cur_lba = start_lba, i = 0; i < out_vector.size(); ++i, ++cur_lba) {
-                auto const& [key, value] = out_vector[i];
+            for(uint64_t cur_lba = start_lba, i = 0; i < index_kvs.size(); ++i, ++cur_lba) {
+                auto const& [key, value] = index_kvs[i];
                 // ignore the holes
                 if(cur_lba != key.key()) {
                     read_buf += (key.key() - cur_lba) * blk_size;