C++ migration: use C++ serializer for communicating with the local store (#4116)

After this PR is merged, the Objective-C serializers can be completely removed from the project. Also, this PR changes the `Message` class quite a bit; it's probably in its final form now.

I did some performance testing locally and came to the conclusion that replacing Objective-C serializers with their C++ versions brings no performance downgrade; actually, there seems to be a very slight improvement. I tested writing large batches and reading large collections:
* both are network-dominated; wall time is about twice longer than CPU time;
* serialization represents a pretty small percentage of the total CPU time;
* for writing, C++ serialization is a little more efficient, taking ~6% of total CPU time, compared to ~8% in the Objective-C version. I've seen similar results for reading.

All integration tests pass, under Address Sanitizer as well.

Author: var-const

Firestore/Example/Benchmarks/FSTLevelDBBenchmarkTests.mm

@@ -19,13 +19,12 @@
 
 #include <cstdint>
 
-#import "Firestore/Source/Local/FSTLocalSerializer.h"
-#import "Firestore/Source/Remote/FSTSerializerBeta.h"
-
 #include "Firestore/core/src/firebase/firestore/local/leveldb_key.h"
 #include "Firestore/core/src/firebase/firestore/local/leveldb_transaction.h"
+#include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
 #include "Firestore/core/src/firebase/firestore/model/document_key.h"
 #include "Firestore/core/src/firebase/firestore/model/types.h"
+#include "Firestore/core/src/firebase/firestore/remote/serializer.h"
 #include "Firestore/core/src/firebase/firestore/util/filesystem.h"
 #include "Firestore/core/src/firebase/firestore/util/path.h"
 #include "Firestore/core/src/firebase/firestore/util/string_format.h"
@@ -38,9 +37,11 @@
 using firebase::firestore::local::LevelDbRemoteDocumentKey;
 using firebase::firestore::local::LevelDbTargetDocumentKey;
 using firebase::firestore::local::LevelDbTransaction;
+using firebase::firestore::local::LocalSerializer;
 using firebase::firestore::model::DatabaseId;
 using firebase::firestore::model::DocumentKey;
 using firebase::firestore::model::TargetId;
+using firebase::firestore::remote::Serializer;
 using firebase::firestore::util::StringFormat;
 using firebase::firestore::util::Path;
 
@@ -59,13 +60,13 @@
 
 FSTLevelDB *LevelDBPersistence() {
   DatabaseId db_id("p", "d");
-  auto remoteSerializer = [[FSTSerializerBeta alloc] initWithDatabaseID:db_id];
-  auto serializer = [[FSTLocalSerializer alloc] initWithRemoteSerializer:remoteSerializer];
+  Serializer remoteSerializer{db_id};
+  LocalSerializer serializer{std::move(remoteSerializer)};
 
   FSTLevelDB *db;
   Path path = util::TempDir().AppendUtf8("FSTLevelDBBenchmarkTests");
   util::Status status = [FSTLevelDB dbWithDirectory:std::move(path)
-                                         serializer:serializer
+                                         serializer:std::move(serializer)
                                           lruParams:local::LruParams::Disabled()
                                                 ptr:&db];
   if (!status.ok()) {

Firestore/Example/FuzzTests/FuzzingTargets/FSTFuzzTestSerializer.mm

@@ -21,6 +21,7 @@
 #import "Firestore/Example/FuzzTests/FuzzingTargets/FSTFuzzTestSerializer.h"
 
 #include "Firestore/core/src/firebase/firestore/model/database_id.h"
+#include "Firestore/core/src/firebase/firestore/nanopb/message.h"
 #include "Firestore/core/src/firebase/firestore/nanopb/reader.h"
 #include "Firestore/core/src/firebase/firestore/remote/serializer.h"
 
@@ -29,18 +30,17 @@
 namespace fuzzing {
 
 using firebase::firestore::model::DatabaseId;
-using firebase::firestore::nanopb::Reader;
+using firebase::firestore::nanopb::StringReader;
 using firebase::firestore::remote::Serializer;
 
 int FuzzTestDeserialization(const uint8_t *data, size_t size) {
   Serializer serializer{DatabaseId{"project"}};
 
   @autoreleasepool {
     @try {
-      Reader reader(data, size);
-      google_firestore_v1_Value nanopb_proto{};
-      reader.ReadNanopbMessage(google_firestore_v1_Value_fields, &nanopb_proto);
-      serializer.DecodeFieldValue(&reader, nanopb_proto);
+      StringReader reader{data, size};
+      auto message = Message<google_firestore_v1_Value>::TryParse(&reader);
+      serializer.DecodeFieldValue(&reader, *message);
     } @catch (...) {
       // Caught exceptions are ignored because the input might be malformed and
       // the deserialization might throw an error as intended. Fuzzing focuses on

Firestore/Example/Tests/Local/FSTLevelDBMigrationsTests.mm

@@ -37,6 +37,7 @@
 NS_ASSUME_NONNULL_BEGIN
 
 using firebase::firestore::Error;
+using firebase::firestore::firestore_client_TargetGlobal;
 using firebase::firestore::local::LevelDbCollectionParentKey;
 using firebase::firestore::local::LevelDbDir;
 using firebase::firestore::local::LevelDbDocumentMutationKey;
@@ -55,6 +56,7 @@
 using firebase::firestore::model::DocumentKey;
 using firebase::firestore::model::ListenSequenceNumber;
 using firebase::firestore::model::TargetId;
+using firebase::firestore::nanopb::Message;
 using firebase::firestore::testutil::Key;
 using firebase::firestore::util::OrderedCode;
 using firebase::firestore::util::Path;
@@ -88,12 +90,12 @@ - (void)tearDown {
 }
 
 - (void)testAddsTargetGlobal {
-  FSTPBTargetGlobal *metadata = LevelDbQueryCache::ReadMetadata(_db.get());
-  XCTAssertNil(metadata, @"Not expecting metadata yet, we should have an empty db");
+  auto metadata = LevelDbQueryCache::TryReadMetadata(_db.get());
+  XCTAssert(!metadata, @"Not expecting metadata yet, we should have an empty db");
   LevelDbMigrations::RunMigrations(_db.get());
 
-  metadata = LevelDbQueryCache::ReadMetadata(_db.get());
-  XCTAssertNotNil(metadata, @"Migrations should have added the metadata");
+  metadata = LevelDbQueryCache::TryReadMetadata(_db.get());
+  XCTAssert(metadata, @"Migrations should have added the metadata");
 }
 
 - (void)testSetsVersionNumber {
@@ -168,9 +170,9 @@ - (void)testDropsTheQueryCache {
       ASSERT_FOUND(transaction, key);
     }
 
-    FSTPBTargetGlobal *metadata = LevelDbQueryCache::ReadMetadata(_db.get());
-    XCTAssertNotNil(metadata, @"Metadata should have been added");
-    XCTAssertEqual(metadata.targetCount, 0);
+    auto metadata = LevelDbQueryCache::TryReadMetadata(_db.get());
+    XCTAssert(metadata, @"Metadata should have been added");
+    XCTAssertEqual(metadata.value()->target_count, 0);
   }
 }
 
@@ -211,9 +213,9 @@ - (void)testAddsSentinelRows {
     LevelDbTransaction transaction(_db.get(), "Setup");
 
     // Set up target global
-    FSTPBTargetGlobal *metadata = LevelDbQueryCache::ReadMetadata(_db.get());
+    auto metadata = LevelDbQueryCache::ReadMetadata(_db.get());
     // Expect that documents missing a row will get the new number
-    metadata.highestListenSequenceNumber = new_sequence_number;
+    metadata->highest_listen_sequence_number = new_sequence_number;
     transaction.Put(LevelDbTargetGlobalKey::Key(), metadata);
 
     // Set up some documents (we only need the keys)

Firestore/core/src/firebase/firestore/core/firestore_client.mm

@@ -20,21 +20,20 @@
 #include <memory>
 #include <utility>
 
-#import "Firestore/Source/Local/FSTLocalSerializer.h"
-#import "Firestore/Source/Remote/FSTSerializerBeta.h"
-
 #include "Firestore/core/src/firebase/firestore/api/settings.h"
 #include "Firestore/core/src/firebase/firestore/auth/credentials_provider.h"
 #include "Firestore/core/src/firebase/firestore/core/database_info.h"
 #include "Firestore/core/src/firebase/firestore/core/event_manager.h"
 #include "Firestore/core/src/firebase/firestore/core/view.h"
 #include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
+#include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
 #include "Firestore/core/src/firebase/firestore/local/memory_persistence.h"
 #include "Firestore/core/src/firebase/firestore/model/database_id.h"
 #include "Firestore/core/src/firebase/firestore/model/document_set.h"
 #include "Firestore/core/src/firebase/firestore/model/mutation.h"
 #include "Firestore/core/src/firebase/firestore/remote/datastore.h"
 #include "Firestore/core/src/firebase/firestore/remote/remote_store.h"
+#include "Firestore/core/src/firebase/firestore/remote/serializer.h"
 #include "Firestore/core/src/firebase/firestore/util/async_queue.h"
 #include "Firestore/core/src/firebase/firestore/util/delayed_constructor.h"
 #include "Firestore/core/src/firebase/firestore/util/exception.h"
@@ -59,6 +58,7 @@
 using auth::CredentialsProvider;
 using auth::User;
 using local::LevelDbPersistence;
+using local::LocalSerializer;
 using local::LocalStore;
 using local::LruParams;
 using local::MemoryPersistence;
@@ -71,6 +71,7 @@
 using model::OnlineState;
 using remote::Datastore;
 using remote::RemoteStore;
+using remote::Serializer;
 using util::AsyncQueue;
 using util::DelayedConstructor;
 using util::DelayedOperation;
@@ -153,13 +154,10 @@ new FirestoreClient(database_info, std::move(credentials_provider),
     Path dir = LevelDbPersistence::StorageDirectory(
         database_info_, LevelDbPersistence::AppDataDirectory());
 
-    FSTSerializerBeta* remote_serializer = [[FSTSerializerBeta alloc]
-        initWithDatabaseID:database_info_.database_id()];
-    FSTLocalSerializer* serializer =
-        [[FSTLocalSerializer alloc] initWithRemoteSerializer:remote_serializer];
+    Serializer remote_serializer{database_info_.database_id()};
 
     auto created = LevelDbPersistence::Create(
-        std::move(dir), serializer,
+        std::move(dir), LocalSerializer{std::move(remote_serializer)},
         LruParams::WithCacheSize(settings.cache_size_bytes()));
     // If leveldb fails to start then just throw up our hands: the error is
     // unrecoverable. There's nothing an end-user can do and nearly all

Firestore/core/src/firebase/firestore/local/CMakeLists.txt

@@ -88,8 +88,8 @@ cc_library(
     memory_remote_document_cache.mm
     mutation_queue.h
     persistence.h
+    proto_sizer.cc
     proto_sizer.h
-    proto_sizer.mm
     query_cache.h
     query_data.cc
     query_data.h

Firestore/core/src/firebase/firestore/local/leveldb_migrations.cc

@@ -25,6 +25,7 @@
 #include "Firestore/core/src/firebase/firestore/local/memory_index_manager.h"
 #include "Firestore/core/src/firebase/firestore/model/document_key.h"
 #include "Firestore/core/src/firebase/firestore/model/types.h"
+#include "Firestore/core/src/firebase/firestore/nanopb/message.h"
 #include "Firestore/core/src/firebase/firestore/nanopb/reader.h"
 #include "Firestore/core/src/firebase/firestore/nanopb/writer.h"
 #include "absl/strings/match.h"
@@ -39,7 +40,8 @@ using leveldb::Status;
 using leveldb::WriteOptions;
 using model::DocumentKey;
 using model::ResourcePath;
-using nanopb::Reader;
+using nanopb::Message;
+using nanopb::StringReader;
 using nanopb::Writer;
 
 namespace {
@@ -113,8 +115,7 @@ void ClearQueryCache(leveldb::DB* db) {
   firestore_client_TargetGlobal target_global{};
 
   nanopb::StringWriter writer;
-  writer.WriteNanopbMessage(firestore_client_TargetGlobal_fields,
-                            &target_global);
+  writer.Write(firestore_client_TargetGlobal_fields, &target_global);
   transaction.Put(LevelDbTargetGlobalKey::Key(), writer.Release());
 
   SaveVersion(3, &transaction);
@@ -172,15 +173,14 @@ void RemoveAcknowledgedMutations(leveldb::DB* db) {
   for (; it->Valid() && absl::StartsWith(it->key(), mutation_queue_start);
        it->Next()) {
     HARD_ASSERT(key.Decode(it->key()), "Failed to decode mutation queue key");
-    firestore_client_MutationQueue mutation_queue{};
-    Reader reader(it->value());
-    reader.ReadNanopbMessage(firestore_client_MutationQueue_fields,
-                             &mutation_queue);
+    StringReader reader(it->value());
+    auto mutation_queue =
+        Message<firestore_client_MutationQueue>::TryParse(&reader);
     HARD_ASSERT(reader.status().ok(), "Failed to deserialize MutationQueue");
     RemoveMutationBatches(&transaction, key.user_id(),
-                          mutation_queue.last_acknowledged_batch_id);
+                          mutation_queue->last_acknowledged_batch_id);
     RemoveMutationDocuments(&transaction, key.user_id(),
-                            mutation_queue.last_acknowledged_batch_id);
+                            mutation_queue->last_acknowledged_batch_id);
   }
 
   SaveVersion(5, &transaction);
@@ -195,11 +195,10 @@ model::ListenSequenceNumber GetHighestSequenceNumber(
   std::string bytes;
   transaction->Get(LevelDbTargetGlobalKey::Key(), &bytes);
 
-  firestore_client_TargetGlobal target_global{};
-  Reader reader(bytes);
-  reader.ReadNanopbMessage(firestore_client_TargetGlobal_fields,
-                           &target_global);
-  return target_global.highest_listen_sequence_number;
+  StringReader reader(bytes);
+  auto target_global =
+      Message<firestore_client_TargetGlobal>::TryParse(&reader);
+  return target_global->highest_listen_sequence_number;
 }
 
 /**

Firestore/core/src/firebase/firestore/local/leveldb_mutation_queue.h

@@ -17,36 +17,29 @@
 #ifndef FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_LOCAL_LEVELDB_MUTATION_QUEUE_H_
 #define FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_LOCAL_LEVELDB_MUTATION_QUEUE_H_
 
-#if !defined(__OBJC__)
-#error "For now, this file must only be included by ObjC source files."
-#endif  // !defined(__OBJC__)
-
-#import <Foundation/Foundation.h>
-
 #include <set>
 #include <string>
 #include <vector>
 
+#include "Firestore/Protos/nanopb/firestore/local/mutation.nanopb.h"
 #include "Firestore/core/include/firebase/firestore/timestamp.h"
 #include "Firestore/core/src/firebase/firestore/auth/user.h"
 #include "Firestore/core/src/firebase/firestore/local/leveldb_key.h"
 #include "Firestore/core/src/firebase/firestore/local/mutation_queue.h"
 #include "Firestore/core/src/firebase/firestore/model/document_key.h"
 #include "Firestore/core/src/firebase/firestore/model/document_key_set.h"
 #include "Firestore/core/src/firebase/firestore/model/types.h"
+#include "Firestore/core/src/firebase/firestore/nanopb/message.h"
 #include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
 #include "leveldb/db.h"
 
-@class FSTLocalSerializer;
-@class FSTPBMutationQueue;
-
-NS_ASSUME_NONNULL_BEGIN
-
 namespace firebase {
 namespace firestore {
 namespace local {
 
 class LevelDbPersistence;
+class LocalSerializer;
 
 /**
  * Returns one larger than the largest batch ID that has been stored. If there
@@ -58,7 +51,7 @@ class LevelDbMutationQueue : public MutationQueue {
  public:
   LevelDbMutationQueue(const auth::User& user,
                        LevelDbPersistence* db,
-                       FSTLocalSerializer* serializer);
+                       LocalSerializer* serializer);
 
   void Start() override;
 
@@ -97,7 +90,7 @@ class LevelDbMutationQueue : public MutationQueue {
 
   nanopb::ByteString GetLastStreamToken() override;
 
-  void SetLastStreamToken(const nanopb::ByteString& stream_token) override;
+  void SetLastStreamToken(nanopb::ByteString stream_token) override;
 
  private:
   /**
@@ -107,7 +100,7 @@ class LevelDbMutationQueue : public MutationQueue {
   std::vector<model::MutationBatch> AllMutationBatchesWithIds(
       const std::set<model::BatchId>& batch_ids);
 
-  std::string mutation_queue_key() {
+  std::string mutation_queue_key() const {
     return LevelDbMutationQueueKey::Key(user_id_);
   }
 
@@ -116,14 +109,16 @@ class LevelDbMutationQueue : public MutationQueue {
   }
 
   /** Parses the MutationQueue metadata from the given LevelDB row contents. */
-  FSTPBMutationQueue* _Nullable MetadataForKey(const std::string& key);
+  nanopb::Message<firestore_client_MutationQueue> MetadataForKey(
+      const std::string& key);
 
   model::MutationBatch ParseMutationBatch(absl::string_view encoded);
 
   // The LevelDbMutationQueue instance is owned by LevelDbPersistence.
   LevelDbPersistence* db_;
 
-  FSTLocalSerializer* serializer_;
+  // Owned by LevelDbPersistence.
+  LocalSerializer* serializer_ = nullptr;
 
   /**
    * The normalized user_id (i.e. after converting null to empty) as used in our
@@ -143,13 +138,11 @@ class LevelDbMutationQueue : public MutationQueue {
   /**
    * A write-through cache copy of the metadata describing the current queue.
    */
-  FSTPBMutationQueue* _Nullable metadata_;
+  nanopb::Message<firestore_client_MutationQueue> metadata_;
 };
 
 }  // namespace local
 }  // namespace firestore
 }  // namespace firebase
 
-NS_ASSUME_NONNULL_END
-
 #endif  // FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_LOCAL_LEVELDB_MUTATION_QUEUE_H_