MB-39292: 3/n Change from JSON to flatbuffers+crc32c

Change from a JSON format to a flatbuffers defined structure.
Reload of the manifest can use the flatbuffers Verify which
checks that the loaded data appears to be a valid 'Manifest'.

A CRC (using crc32c) is further added to the stored data so
that we can detect changes to the data which may not be
detected by Verify (e.g. a string value "dog" changing to "cat").

Detected issues are logged with CRITICAL - the bucket is allowed to
continue warm-up and the in-memory Bucket::Manifest is initialised as
the 'epoch' Manifest, the bucket will then pick-up the current
manifest from ns_server, any Manifest is a successor of epoch.

Change-Id: I319c3e76ef5c07da0680e602f3e342d3d8affa58
Reviewed-on: http://review.couchbase.org/c/kv_engine/+/137162
Reviewed-by: Dave Rigby <[email protected]>
Tested-by: Build Bot <[email protected]>

Author: jimwwalker

engines/ep/CMakeLists.txt

@@ -190,6 +190,14 @@ ADD_CUSTOM_COMMAND(OUTPUT
                     src/collections/events.fbs
                   COMMENT "Generating flatbuffers - collection events")
 
+ADD_CUSTOM_COMMAND(OUTPUT
+                     ${CMAKE_CURRENT_BINARY_DIR}/src/collections/manifest_generated.h
+                  COMMAND
+                     ${FLATC} -o ${CMAKE_CURRENT_BINARY_DIR}/src/collections/ --cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/collections/manifest.fbs
+                  DEPENDS
+                    src/collections/manifest.fbs
+                  COMMENT "Generating flatbuffers - collection manifest")
+
 # Custom build target to generate source files that we normally generate as part
 # of a build.
 add_custom_target(generated_source_files
@@ -202,6 +210,7 @@ add_custom_target(generated_source_files
             ${CMAKE_CURRENT_BINARY_DIR}/src/collections/kvstore_generated.h
             ${CMAKE_CURRENT_BINARY_DIR}/src/collections/kvstore_flatbuffers_schema.cc
             ${CMAKE_CURRENT_BINARY_DIR}/src/collections/events_generated.h
+            ${CMAKE_CURRENT_BINARY_DIR}/src/collections/manifest_generated.h
             generate_audit_descriptors)
 
 # Collections library for things required outside of ep-engine

engines/ep/src/collections/manager.cc

@@ -315,14 +315,18 @@ void Collections::Manager::addScopeStats(KVBucket& bucket,
 
 void Collections::Manager::warmupLoadManifest(const std::string& dbpath) {
     auto rv = Collections::PersistManifestTask::tryAndLoad(dbpath);
-    if (rv) {
+    if (rv.has_value()) {
         EP_LOG_INFO(
-                "Collections::Manager::warmupLoadManifest: loaded uid:{:#x}",
-                rv->getUid());
-        *currentManifest.wlock() = std::move(*rv);
-    } else {
-        EP_LOG_INFO("Collections::Manager::warmupLoadManifest: nothing loaded");
-    }
+                "Collections::Manager::warmupLoadManifest: starting at "
+                "uid:{:#x}",
+                rv.value().getUid());
+        *currentManifest.wlock() = rv.value();
+    }
+    // else tryAndLoad detected (and logged) some kind of corruption issue.
+    // If this corruption occurred at the same time as some issue in the
+    // forward flow of the Manifest, KV can't validate that any change to the
+    // manifest is a legal successor (Manifest::isSuccessor) - we will just have
+    // to accept what we're given and carry on.
 }
 
 /**

engines/ep/src/collections/manager.h

@@ -324,7 +324,8 @@ class Manager {
 
     friend std::ostream& operator<<(std::ostream& os, const Manager& manager);
 
-    /// Store the most recent (current) manifest received
+    /// Store the most recent (current) manifest received - this default
+    /// constructs as the 'epoch' Manifest
     folly::Synchronized<Manifest> currentManifest;
 
     /// Serialise updates to the manifest (set_collections core)

engines/ep/src/collections/manifest.cc

@@ -19,6 +19,7 @@
 #include "bucket_logger.h"
 #include "collections/collections_constants.h"
 #include "collections/collections_types.h"
+#include "collections/manifest_generated.h"
 #include "ep_engine.h"
 #include "kv_bucket.h"
 #include "statistics/collector.h"
@@ -301,6 +302,85 @@ nlohmann::json Manifest::toJson(
     return manifest;
 }
 
+flatbuffers::DetachedBuffer Manifest::toFlatbuffer() const {
+    flatbuffers::FlatBufferBuilder builder;
+    std::vector<flatbuffers::Offset<Collections::Persist::Scope>> fbScopes;
+
+    for (const auto& scope : scopes) {
+        std::vector<flatbuffers::Offset<Collections::Persist::Collection>>
+                fbCollections;
+
+        for (const auto& c : scope.second.collections) {
+            auto newEntry = Collections::Persist::CreateCollection(
+                    builder,
+                    uint32_t(c.id),
+                    c.maxTtl.has_value(),
+                    c.maxTtl.value_or(std::chrono::seconds(0)).count(),
+                    builder.CreateString(collections.at(c.id).name.data(),
+                                         collections.at(c.id).name.size()));
+            fbCollections.push_back(newEntry);
+        }
+        auto collectionVector = builder.CreateVector(fbCollections);
+
+        auto newEntry = Collections::Persist::CreateScope(
+                builder,
+                scope.first,
+                builder.CreateString(scope.second.name.data(),
+                                     scope.second.name.size()),
+                collectionVector);
+        fbScopes.push_back(newEntry);
+    }
+
+    auto scopeVector = builder.CreateVector(fbScopes);
+    auto toWrite =
+            Collections::Persist::CreateManifest(builder, uid, scopeVector);
+    builder.Finish(toWrite);
+    return builder.Release();
+}
+
+// sibling of toFlatbuffer, construct a Manifest from a flatbuffer format
+Manifest::Manifest(std::string_view flatbufferData, Manifest::FlatBuffers tag)
+    : defaultCollectionExists(false), scopes(), collections(), uid(0) {
+    flatbuffers::Verifier v(
+            reinterpret_cast<const uint8_t*>(flatbufferData.data()),
+            flatbufferData.size());
+    if (!v.VerifyBuffer<Collections::Persist::Manifest>(nullptr)) {
+        std::stringstream ss;
+        ss << "Collections::Manifest::Manifest(FlatBuffers): flatbufferData "
+              "invalid, ptr:"
+           << reinterpret_cast<const void*>(flatbufferData.data())
+           << ", size:" << flatbufferData.size();
+
+        throw std::invalid_argument(ss.str());
+    }
+
+    auto manifest = flatbuffers::GetRoot<Collections::Persist::Manifest>(
+            reinterpret_cast<const uint8_t*>(flatbufferData.data()));
+
+    uid = manifest->uid();
+    for (const Collections::Persist::Scope* scope : *manifest->scopes()) {
+        std::vector<CollectionEntry> scopeCollections;
+
+        for (const Collections::Persist::Collection* collection :
+             *scope->collections()) {
+            cb::ExpiryLimit maxTtl;
+            CollectionID cid(collection->collectionId());
+            if (collection->ttlValid()) {
+                maxTtl = std::chrono::seconds(collection->maxTtl());
+            }
+            this->collections.emplace(std::make_pair(
+                    cid,
+                    Collection{scope->scopeId(), collection->name()->str()}));
+            enableDefaultCollection(cid);
+            scopeCollections.push_back({cid, maxTtl});
+        }
+
+        this->scopes.emplace(
+                scope->scopeId(),
+                Scope{scope->name()->str(), std::move(scopeCollections)});
+    }
+}
+
 void Manifest::addCollectionStats(KVBucket& bucket,
                                   const void* cookie,
                                   const AddStatFn& add_stat) const {

engines/ep/src/collections/manifest.fbs

@@ -0,0 +1,51 @@
+//
+//     Copyright 2020 Couchbase, Inc
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.
+//
+
+//
+// Flatbuffer types used in the serialisation of the Collections::Manifest
+// object. These are used to store as a file a copy of the current manifest
+// and use it in warm-up.
+//
+
+namespace Collections.Persist;
+
+table Collection {
+    collectionId:uint;
+    ttlValid:bool;
+    maxTtl:uint;
+    name:string;
+}
+
+table Scope {
+    scopeId:uint;
+    name:string;
+    collections:[Collection];
+}
+
+table Manifest {
+   uid:ulong;
+   scopes:[Scope];
+}
+
+table ManifestWithCrc {
+   crc:uint;
+   manifest:[ubyte];
+}
+
+// This is the main type representing a Collections::Manifest
+root_type Manifest;
+// This wraps the flatbuffer Manifest with a CRC for integrity checking
+root_type ManifestWithCrc;

engines/ep/src/collections/manifest.h

@@ -29,6 +29,10 @@
 
 class KVBucket;
 
+namespace flatbuffers {
+class DetachedBuffer;
+}
+
 namespace Collections {
 
 static const size_t MaxScopeOrCollectionNameSize = 251;
@@ -69,6 +73,9 @@ class Manifest {
      */
     explicit Manifest(std::string_view json);
 
+    struct FlatBuffers {};
+    explicit Manifest(std::string_view flatbufferData, FlatBuffers tag);
+
     bool doesDefaultCollectionExist() const {
         return defaultCollectionExists;
     }
@@ -229,6 +236,11 @@ class Manifest {
     nlohmann::json toJson(
             const Collections::IsVisibleFunction& isVisible) const;
 
+    /**
+     * @return flatbuffer representation of this object
+     */
+    flatbuffers::DetachedBuffer toFlatbuffer() const;
+
     /**
      * Add stats for collection. Each collection is tested for
      * Privilege::SimpleStats and 'added' if the user has the privilege.