Merge pull request #359 from cruzdb/init-stripes

Init stripes

Author: dotnwat

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Pending
 
+* separate view and epoch-versioned-view abstractions
+* use a scalable view / objectmap implementation
+* moved away from protocol buffers to use only flatbuffers
+* implemented a large set of unit test coverage for views
+* initialize first stripe for new logs
+
 # v0.5.0
 
 * be/ceph: added support for controlling omap vs bytestream storage

src/include/zlog/options.h

@@ -26,6 +26,9 @@ struct Options {
   bool create_if_missing = false;
   bool error_if_exists = false;
 
+  // add stripes to the initial view for new logs.
+  bool create_initial_view_stripes = true;
+
   // schedule a background task to initialize stripe objects whenever a new
   // stripe is created. this should always be set to true, and is only false for
   // in testing scenarios (e.g. synchronous object init in i/o path).
@@ -37,8 +40,6 @@ struct Options {
 
   uint32_t max_inflight_ops = 1024;
 
-  bool create_initial_view_stripes = true;
-
   ///////////////////////////////////////////////////////////////////
 
   // number of I/O threads

src/libzlog/log.cc

@@ -15,7 +15,8 @@ Log::~Log() {}
 
 int create_or_open(const Options& options,
     Backend *backend, const std::string& name,
-    std::string *hoid_out, std::string *prefix_out)
+    std::string *hoid_out, std::string *prefix_out,
+    bool *created)
 {
   std::string hoid;
   std::string prefix;
@@ -54,7 +55,9 @@ int create_or_open(const Options& options,
         return ret;
       }
     }
-
+    if (created) {
+      *created = true;
+    }
     break;
   }
 
@@ -81,10 +84,11 @@ int Log::Open(const Options& options,
     }
   }
 
+  bool created = false;
   std::string hoid;
   std::string prefix;
   int ret = create_or_open(options, backend.get(),
-      name, &hoid, &prefix);
+      name, &hoid, &prefix, &created);
   if (ret) {
     return ret;
   }
@@ -110,8 +114,14 @@ int Log::Open(const Options& options,
   // gh#343
   impl->striper.update_current_view(0);
 
-  // TODO: initialize the first stripe so that cost isn't incurred by clients
-  // when they start performing I/O.
+  // kick start initialization of the first stripe
+  if (options.init_stripe_on_create && created) {
+    // if there actually is a stripe. this is controlled by the
+    // create_init_view_stripes option
+    if (!impl->striper.view()->object_map().empty()) {
+      impl->striper.async_init_stripe(0);
+    }
+  }
 
   *logpp = impl.release();
 

src/libzlog/log_impl.h

@@ -253,6 +253,7 @@ class LogImpl : public Log {
 
 int create_or_open(const Options& options,
     Backend *backend, const std::string& name,
-    std::string *hoid_out, std::string *prefix_out);
+    std::string *hoid_out, std::string *prefix_out,
+    bool *created);
 
 }

src/libzlog/view.cc

@@ -27,10 +27,27 @@ std::string View::create_initial(const Options& options)
 {
   flatbuffers::FlatBufferBuilder fbb;
 
-  // - next_stripe_id = 0
-  // - no stripes
-  // TODO: if (options.create_initial_view_stripes) ...
-  const auto object_map = zlog::fbs::CreateObjectMapDirect(fbb, 0, nullptr, 0);
+  // below the prefix is discarded when the object map is encoded
+  // TODO: it would be nice to reformulate the abstractions to avoid needing to
+  // use a placeholder prefix here. the multistripe defines a layout, but
+  // shouldn't define a static object naming implied by requiring this prefix
+  // parameter.
+  std::map<uint64_t, MultiStripe> stripes;
+  if (options.create_initial_view_stripes) {
+    stripes.emplace(0,
+        MultiStripe(
+          "<<UNUSED PREFIX>>",
+          0,
+          options.stripe_width,
+          options.stripe_slots,
+          0,
+          1,
+          options.stripe_width * options.stripe_slots - 1));
+  }
+
+  const auto object_map = stripes.empty() ?
+    zlog::fbs::CreateObjectMapDirect(fbb, 0, nullptr, 0) :
+    ObjectMap(1, stripes, 0).encode(fbb);
 
   auto builder = zlog::fbs::ViewBuilder(fbb);
   builder.add_object_map(object_map);

src/zlog.cc

@@ -159,7 +159,7 @@ int handle_log(std::vector<std::string> command, std::shared_ptr<zlog::Backend>
     options.create_if_missing = true;
     std::string hoid, prefix;
     int ret = zlog::create_or_open(options, backend.get(), command[1],
-        &hoid, &prefix);
+        &hoid, &prefix, nullptr);
     switch (ret) {
       case 0:
         break;