Skip to content

Commit 4124b48

Browse files
amabluea-maurice
authored andcommitted
Added schemas for the flatbuffer types that will be written to disk for the
storage engine, as well as conversion functions to the C++ types that they back. PiperOrigin-RevId: 311436824
1 parent 019aa23 commit 4124b48

9 files changed

+534
-1
lines changed

cmake/external/flatbuffers.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ if(TARGET flatbuffers OR NOT DOWNLOAD_FLATBUFFERS)
1818
return()
1919
endif()
2020

21-
set(version 97ffc590e336b800d346bdf2f4876394bb353f97)
21+
set(version 99aa1ef21dd9dc3f9d4fb0eb82f4b59d0bb5e4c5)
2222

2323
ExternalProject_Add(
2424
flatbuffers

database/CMakeLists.txt

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,22 @@ if (NOT OPENSSL_FOUND)
6565
endif()
6666
endif()
6767

68+
# Flatbuffer schemas used by the desktop implementation.
69+
set(desktop_flatbuffers_schema_dir
70+
"${CMAKE_CURRENT_LIST_DIR}/src/desktop/persistence/schemas")
71+
set(desktop_flatbuffers_schemas
72+
"${desktop_flatbuffers_schema_dir}/persisted_compound_write.fbs"
73+
"${desktop_flatbuffers_schema_dir}/persisted_query_params.fbs"
74+
"${desktop_flatbuffers_schema_dir}/persisted_query_spec.fbs"
75+
"${desktop_flatbuffers_schema_dir}/persisted_tracked_query.fbs"
76+
"${desktop_flatbuffers_schema_dir}/persisted_user_write_record.fbs")
77+
78+
flatbuffers_generate_headers(
79+
TARGET desktop_flatbuffers
80+
INCLUDE "${CMAKE_CURRENT_LIST_DIR}"
81+
INCLUDE_PREFIX "database/src/desktop/persistence"
82+
SCHEMAS ${desktop_flatbuffers_schemas})
83+
6884
# Source files used by the desktop implementation.
6985
set(desktop_SRCS
7086
src/desktop/connection/connection.cc
@@ -96,6 +112,7 @@ set(desktop_SRCS
96112
src/desktop/database_reference_desktop.cc
97113
src/desktop/disconnection_desktop.cc
98114
src/desktop/mutable_data_desktop.cc
115+
src/desktop/persistence/flatbuffer_conversions.cc
99116
src/desktop/persistence/in_memory_persistence_storage_engine.cc
100117
src/desktop/persistence/noop_persistence_manager.cc
101118
src/desktop/persistence/persistence_manager.cc
@@ -137,6 +154,7 @@ else()
137154

138155
set(additional_link_LIB
139156
flatbuffers
157+
desktop_flatbuffers
140158
firebase_rest_lib
141159
${OPENSSL_CRYPTO_LIBRARY}
142160
libuWS)
Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "database/src/desktop/persistence/flatbuffer_conversions.h"
16+
17+
#include "app/src/include/firebase/variant.h"
18+
#include "app/src/variant_util.h"
19+
#include "database/src/common/query_spec.h"
20+
#include "database/src/desktop/core/compound_write.h"
21+
#include "database/src/desktop/core/tracked_query_manager.h"
22+
#include "database/src/desktop/persistence/persisted_compound_write_generated.h"
23+
#include "database/src/desktop/persistence/persisted_tracked_query_generated.h"
24+
#include "database/src/desktop/persistence/persisted_user_write_record_generated.h"
25+
#include "database/src/desktop/persistence/persistence_storage_engine.h"
26+
#include "flatbuffers/flatbuffers.h"
27+
#include "flatbuffers/util.h"
28+
29+
using firebase::database::internal::persistence::CreatePersistedCompoundWrite;
30+
using firebase::database::internal::persistence::CreatePersistedQueryParams;
31+
using firebase::database::internal::persistence::CreatePersistedQuerySpec;
32+
using firebase::database::internal::persistence::CreatePersistedTrackedQuery;
33+
using firebase::database::internal::persistence::CreateTreeKeyValuePair;
34+
using firebase::util::FlexbufferToVariant;
35+
using firebase::util::VariantToFlexbuffer;
36+
using flatbuffers::FlatBufferBuilder;
37+
using flatbuffers::Offset;
38+
using flatbuffers::Vector;
39+
40+
namespace firebase {
41+
namespace database {
42+
namespace internal {
43+
44+
static void VariantTreeFromFlatbuffer(const persistence::VariantTreeNode* node,
45+
Tree<Variant>* out_tree) {
46+
if (node->value()) {
47+
out_tree->set_value(FlexbufferToVariant(node->value_flexbuffer_root()));
48+
}
49+
if (node->children()) {
50+
for (const persistence::TreeKeyValuePair* kvp : *node->children()) {
51+
const char* key = kvp->key()->c_str();
52+
const persistence::VariantTreeNode* value = kvp->subtree();
53+
Tree<Variant>* subtree = out_tree->GetOrMakeSubtree(Path(key));
54+
VariantTreeFromFlatbuffer(value, subtree);
55+
}
56+
}
57+
}
58+
59+
CompoundWrite CompoundWriteFromFlatbuffer(
60+
const persistence::PersistedCompoundWrite* persisted_compound_write) {
61+
if (!persisted_compound_write->write_tree()) {
62+
return CompoundWrite();
63+
}
64+
Tree<Variant> write_tree;
65+
const persistence::VariantTreeNode* node =
66+
persisted_compound_write->write_tree();
67+
VariantTreeFromFlatbuffer(node, &write_tree);
68+
return CompoundWrite(write_tree);
69+
}
70+
71+
QueryParams QueryParamsFromFlatbuffer(
72+
const persistence::PersistedQueryParams* persisted_query_params) {
73+
QueryParams params;
74+
// Set by Query::OrderByPriority(), Query::OrderByChild(),
75+
// Query::OrderByKey(), and Query::OrderByValue().
76+
// Default is kOrderByPriority.
77+
params.order_by =
78+
static_cast<QueryParams::OrderBy>(persisted_query_params->order_by());
79+
if (persisted_query_params->order_by_child()) {
80+
params.order_by_child = persisted_query_params->order_by_child()->str();
81+
}
82+
if (persisted_query_params->start_at_value()) {
83+
params.start_at_value = FlexbufferToVariant(
84+
persisted_query_params->start_at_value_flexbuffer_root());
85+
}
86+
if (persisted_query_params->start_at_child_key()) {
87+
params.start_at_child_key =
88+
persisted_query_params->start_at_child_key()->str();
89+
}
90+
if (persisted_query_params->end_at_value()) {
91+
params.end_at_value = FlexbufferToVariant(
92+
persisted_query_params->end_at_value_flexbuffer_root());
93+
}
94+
if (persisted_query_params->end_at_child_key()) {
95+
params.end_at_child_key = persisted_query_params->end_at_child_key()->str();
96+
}
97+
if (persisted_query_params->equal_to_value()) {
98+
params.equal_to_value = FlexbufferToVariant(
99+
persisted_query_params->equal_to_value_flexbuffer_root());
100+
}
101+
if (persisted_query_params->equal_to_child_key()) {
102+
params.equal_to_child_key =
103+
persisted_query_params->equal_to_child_key()->str();
104+
}
105+
params.limit_first = persisted_query_params->limit_first();
106+
params.limit_last = persisted_query_params->limit_last();
107+
return params;
108+
}
109+
110+
QuerySpec QuerySpecFromFlatbuffer(
111+
const persistence::PersistedQuerySpec* persisted_query_spec) {
112+
QuerySpec query_spec;
113+
if (persisted_query_spec->path()) {
114+
query_spec.path = Path(persisted_query_spec->path()->c_str());
115+
}
116+
if (persisted_query_spec->params()) {
117+
query_spec.params =
118+
QueryParamsFromFlatbuffer(persisted_query_spec->params());
119+
}
120+
return query_spec;
121+
}
122+
123+
TrackedQuery TrackedQueryFromFlatbuffer(
124+
const persistence::PersistedTrackedQuery* persisted_tracked_query) {
125+
TrackedQuery tracked_query;
126+
tracked_query.query_id = persisted_tracked_query->query_id();
127+
if (persisted_tracked_query->query_spec()) {
128+
tracked_query.query_spec =
129+
QuerySpecFromFlatbuffer(persisted_tracked_query->query_spec());
130+
}
131+
tracked_query.last_use = persisted_tracked_query->last_use();
132+
tracked_query.complete = persisted_tracked_query->complete();
133+
tracked_query.active = persisted_tracked_query->active();
134+
return tracked_query;
135+
}
136+
137+
UserWriteRecord UserWriteRecordFromFlatbuffer(
138+
const persistence::PersistedUserWriteRecord* persisted_user_write_record) {
139+
UserWriteRecord user_write_record;
140+
user_write_record.write_id = persisted_user_write_record->write_id();
141+
if (persisted_user_write_record->path()) {
142+
user_write_record.path = Path(persisted_user_write_record->path()->c_str());
143+
}
144+
user_write_record.is_overwrite = persisted_user_write_record->is_overwrite();
145+
if (user_write_record.is_overwrite) {
146+
if (persisted_user_write_record->overwrite()) {
147+
user_write_record.overwrite = FlexbufferToVariant(
148+
persisted_user_write_record->overwrite_flexbuffer_root());
149+
}
150+
} else {
151+
if (persisted_user_write_record->merge()) {
152+
user_write_record.merge =
153+
CompoundWriteFromFlatbuffer(persisted_user_write_record->merge());
154+
}
155+
}
156+
user_write_record.visible = persisted_user_write_record->visible();
157+
return user_write_record;
158+
}
159+
160+
static Offset<persistence::VariantTreeNode> FlatbufferFromVariantTreeNode(
161+
FlatBufferBuilder* builder, const Tree<Variant>& node) {
162+
Offset<Vector<uint8_t>> value_offset = 0;
163+
if (node.value().has_value()) {
164+
value_offset =
165+
builder->CreateVector(VariantToFlexbuffer(node.value().value()));
166+
}
167+
Offset<Vector<Offset<persistence::TreeKeyValuePair>>> children_vector_offset =
168+
0;
169+
if (!node.children().empty()) {
170+
std::vector<Offset<persistence::TreeKeyValuePair>> children_offsets;
171+
for (auto& kvp : node.children()) {
172+
const std::string& key = kvp.first;
173+
const Tree<Variant>& value = kvp.second;
174+
children_offsets.push_back(CreateTreeKeyValuePair(
175+
*builder, builder->CreateString(key),
176+
FlatbufferFromVariantTreeNode(builder, value)));
177+
}
178+
children_vector_offset =
179+
builder->CreateVector<Offset<persistence::TreeKeyValuePair>>(
180+
children_offsets);
181+
}
182+
return CreateVariantTreeNode(*builder, value_offset, children_vector_offset);
183+
}
184+
185+
Offset<persistence::PersistedCompoundWrite> FlatbufferFromCompoundWrite(
186+
FlatBufferBuilder* builder, const CompoundWrite& compound_write) {
187+
return persistence::CreatePersistedCompoundWrite(
188+
*builder,
189+
FlatbufferFromVariantTreeNode(builder, compound_write.write_tree()));
190+
}
191+
192+
Offset<persistence::PersistedQueryParams> FlatbufferFromQueryParams(
193+
FlatBufferBuilder* builder, const QueryParams& params) {
194+
return CreatePersistedQueryParams(
195+
*builder, static_cast<persistence::OrderBy>(params.order_by),
196+
builder->CreateString(params.order_by_child),
197+
builder->CreateVector(VariantToFlexbuffer(params.start_at_value)),
198+
builder->CreateString(params.start_at_child_key),
199+
builder->CreateVector(VariantToFlexbuffer(params.end_at_value)),
200+
builder->CreateString(params.end_at_child_key),
201+
builder->CreateVector(VariantToFlexbuffer(params.equal_to_value)),
202+
builder->CreateString(params.equal_to_child_key), params.limit_first,
203+
params.limit_last);
204+
}
205+
206+
Offset<persistence::PersistedQuerySpec> FlatbufferFromQuerySpec(
207+
FlatBufferBuilder* builder, const QuerySpec& query_spec) {
208+
return CreatePersistedQuerySpec(
209+
*builder, builder->CreateString(query_spec.path.str()),
210+
FlatbufferFromQueryParams(builder, query_spec.params));
211+
}
212+
213+
flatbuffers::Offset<persistence::PersistedTrackedQuery>
214+
FlatbufferFromTrackedQuery(flatbuffers::FlatBufferBuilder* builder,
215+
const TrackedQuery& tracked_query) {
216+
return CreatePersistedTrackedQuery(
217+
*builder, tracked_query.query_id,
218+
FlatbufferFromQuerySpec(builder, tracked_query.query_spec),
219+
tracked_query.last_use, tracked_query.complete, tracked_query.active);
220+
}
221+
222+
flatbuffers::Offset<persistence::PersistedUserWriteRecord>
223+
FlatbufferFromUserWriteRecord(flatbuffers::FlatBufferBuilder* builder,
224+
const UserWriteRecord& user_write_record) {
225+
return CreatePersistedUserWriteRecord(
226+
*builder, user_write_record.write_id,
227+
builder->CreateString(user_write_record.path.str()),
228+
user_write_record.is_overwrite
229+
? builder->CreateVector(
230+
VariantToFlexbuffer(user_write_record.overwrite))
231+
: 0,
232+
user_write_record.is_overwrite
233+
? 0
234+
: FlatbufferFromCompoundWrite(builder, user_write_record.merge),
235+
user_write_record.visible, user_write_record.is_overwrite);
236+
}
237+
238+
} // namespace internal
239+
} // namespace database
240+
} // namespace firebase
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef FIREBASE_DATABASE_CLIENT_CPP_SRC_DESKTOP_PERSISTENCE_FLATBUFFER_CONVERSIONS_H_
16+
#define FIREBASE_DATABASE_CLIENT_CPP_SRC_DESKTOP_PERSISTENCE_FLATBUFFER_CONVERSIONS_H_
17+
18+
#include "app/src/include/firebase/variant.h"
19+
#include "database/src/common/query_spec.h"
20+
#include "database/src/desktop/core/compound_write.h"
21+
#include "database/src/desktop/core/tracked_query_manager.h"
22+
#include "database/src/desktop/persistence/persisted_compound_write_generated.h"
23+
#include "database/src/desktop/persistence/persisted_query_params_generated.h"
24+
#include "database/src/desktop/persistence/persisted_query_spec_generated.h"
25+
#include "database/src/desktop/persistence/persisted_tracked_query_generated.h"
26+
#include "database/src/desktop/persistence/persisted_user_write_record_generated.h"
27+
#include "database/src/desktop/persistence/persistence_storage_engine.h"
28+
#include "flatbuffers/flatbuffers.h"
29+
30+
namespace firebase {
31+
namespace database {
32+
namespace internal {
33+
34+
// These functions convert serialized flatbuffers into their in-memory
35+
// counterparts.
36+
//
37+
// TODO(amablue): Consider using the Flatbuffers object API here, and removing
38+
// the non-generated versions of these structs.
39+
CompoundWrite CompoundWriteFromFlatbuffer(
40+
const persistence::PersistedCompoundWrite* persisted_compound_write);
41+
QueryParams QueryParamsFromFlatbuffer(
42+
const persistence::PersistedQueryParams* persisted_query_params);
43+
QuerySpec QuerySpecFromFlatbuffer(
44+
const persistence::PersistedQuerySpec* persisted_query_spec);
45+
TrackedQuery TrackedQueryFromFlatbuffer(
46+
const persistence::PersistedTrackedQuery* persisted_tracked_query);
47+
UserWriteRecord UserWriteRecordFromFlatbuffer(
48+
const persistence::PersistedUserWriteRecord* persisted_user_write_record);
49+
50+
// These functions convert in-memory data structures into their flatbuffer
51+
// counterparts so they can be serialzied to disk.
52+
flatbuffers::Offset<persistence::PersistedCompoundWrite>
53+
FlatbufferFromCompoundWrite(flatbuffers::FlatBufferBuilder* builder,
54+
const CompoundWrite& compound_write);
55+
flatbuffers::Offset<persistence::PersistedQueryParams>
56+
FlatbufferFromQueryParams(flatbuffers::FlatBufferBuilder* builder,
57+
const QueryParams& params);
58+
flatbuffers::Offset<persistence::PersistedQuerySpec> FlatbufferFromQuerySpec(
59+
flatbuffers::FlatBufferBuilder* builder, const QuerySpec& query_spec);
60+
flatbuffers::Offset<persistence::PersistedTrackedQuery>
61+
FlatbufferFromTrackedQuery(flatbuffers::FlatBufferBuilder* builder,
62+
const TrackedQuery& tracked_query);
63+
flatbuffers::Offset<persistence::PersistedUserWriteRecord>
64+
FlatbufferFromUserWriteRecord(flatbuffers::FlatBufferBuilder* builder,
65+
const UserWriteRecord& user_write_record);
66+
67+
} // namespace internal
68+
} // namespace database
69+
} // namespace firebase
70+
71+
#endif // FIREBASE_DATABASE_CLIENT_CPP_SRC_DESKTOP_PERSISTENCE_FLATBUFFER_CONVERSIONS_H_
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright 2020 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
namespace firebase.database.internal.persistence;
16+
17+
table TreeKeyValuePair {
18+
key:string (id: 0);
19+
subtree:VariantTreeNode (id: 1);
20+
}
21+
22+
table VariantTreeNode {
23+
value:[ubyte] (id: 0, flexbuffer);
24+
children:[TreeKeyValuePair] (id: 1);
25+
}
26+
27+
table PersistedCompoundWrite {
28+
write_tree:VariantTreeNode (id: 0);
29+
}
30+
31+
// This only has a root type for testing purposes, so they are easier to
32+
// construct.
33+
root_type PersistedCompoundWrite;

0 commit comments

Comments
 (0)