Add setIndexConfiguration for client side indexing (#10090)

* Add setIndexConfiguration for client side indexing

* Address feedback

* Templates in headers

Co-authored-by: Wu-Hui <[email protected]>

Author: cherylEnkidu

Firestore/CHANGELOG.md

@@ -1,3 +1,7 @@
+# Unreleased
+- [added] Expose client side indexing feature with `FIRFirestore.setIndexConfigurationFromJSON` and
+  `FIRFirestore.setIndexConfigurationFromStream` (#10090).
+
 # 9.5.0
 - [fixed] Fixed an intermittent crash if `ListenerRegistration::Remove()` was
   invoked concurrently (#10065).

Firestore/Example/Firestore.xcodeproj/project.pbxproj

@@ -707,6 +707,7 @@
 		71719F9F1E33DC2100824A3D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 71719F9D1E33DC2100824A3D /* LaunchScreen.storyboard */; };
 		71E2B154C4FB63F7B7CC4B50 /* target_id_generator_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380CF82019382300D97691 /* target_id_generator_test.cc */; };
 		722F9A798F39F7D1FE7CF270 /* CodableGeoPointTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5495EB022040E90200EBA509 /* CodableGeoPointTests.swift */; };
+		7264B73291F7F1EB454C45B1 /* FIRIndexingTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 795AA8FC31D2AF6864B07D39 /* FIRIndexingTests.mm */; };
 		7281C2F04838AFFDF6A762DF /* memory_remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1CA9800A53669EFBFFB824E3 /* memory_remote_document_cache_test.cc */; };
 		72AD91671629697074F2545B /* ordered_code_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB380D03201BC6E400D97691 /* ordered_code_test.cc */; };
 		72B25B2D698E4746143D5B74 /* memory_lru_garbage_collector_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9765D47FA12FA283F4EFAD02 /* memory_lru_garbage_collector_test.cc */; };
@@ -1077,6 +1078,7 @@
 		C09BDBA73261578F9DA74CEE /* firebase_auth_credentials_provider_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = F869D85E900E5AF6CD02E2FC /* firebase_auth_credentials_provider_test.mm */; };
 		C0AD8DB5A84CAAEE36230899 /* status_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54A0352C20A3B3D7003E0143 /* status_test.cc */; };
 		C0EFC5FB79517679C377C252 /* schedule_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 9B0B005A79E765AF02793DCE /* schedule_test.cc */; };
+		C10417B067155BE78E19807D /* FIRIndexingTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 795AA8FC31D2AF6864B07D39 /* FIRIndexingTests.mm */; };
 		C1237EE2A74F174A3DF5978B /* memory_target_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 2286F308EFB0534B1BDE05B9 /* memory_target_cache_test.cc */; };
 		C15F5F1E7427738F20C2D789 /* offline_spec_test.json in Resources */ = {isa = PBXBuildFile; fileRef = 54DA12A11F315EE100DD57A1 /* offline_spec_test.json */; };
 		C19214F5B43AA745A7FC2FC1 /* maybe_document.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE7E20B89AAC00B5BCE7 /* maybe_document.pb.cc */; };
@@ -1292,6 +1294,7 @@
 		F0EA84FB66813F2BC164EF7C /* token_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = A082AFDD981B07B5AD78FDE8 /* token_test.cc */; };
 		F10A3E4E164A5458DFF7EDE6 /* leveldb_remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 0840319686A223CC4AD3FAB1 /* leveldb_remote_document_cache_test.cc */; };
 		F19B749671F2552E964422F7 /* FIRListenerRegistrationTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E06B202154D500B64F25 /* FIRListenerRegistrationTests.mm */; };
+		F1EAEE9DF819C017A9506AEB /* FIRIndexingTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 795AA8FC31D2AF6864B07D39 /* FIRIndexingTests.mm */; };
 		F272A8C41D2353700A11D1FB /* field_mask_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 549CCA5320A36E1F00BCEB75 /* field_mask_test.cc */; };
 		F27347560A963E8162C56FF3 /* target_index_matcher_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 63136A2371C0C013EC7A540C /* target_index_matcher_test.cc */; };
 		F2AB7EACA1B9B1A7046D3995 /* FSTSyncEngineTestDriver.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E02E20213FFC00B64F25 /* FSTSyncEngineTestDriver.mm */; };
@@ -1642,6 +1645,7 @@
 		776530F066E788C355B78457 /* FIRBundlesTests.mm */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.objcpp; path = FIRBundlesTests.mm; sourceTree = "<group>"; };
 		78EE0BFC7E60C4929458A0EA /* resource.pb.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = resource.pb.h; sourceTree = "<group>"; };
 		79507DF8378D3C42F5B36268 /* string_win_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = string_win_test.cc; sourceTree = "<group>"; };
+		795AA8FC31D2AF6864B07D39 /* FIRIndexingTests.mm */ = {isa = PBXFileReference; includeInIndex = 1; path = FIRIndexingTests.mm; sourceTree = "<group>"; };
 		79D4CD6A707ED3F7A6D2ECF5 /* view_testing.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = view_testing.h; sourceTree = "<group>"; };
 		79EAA9F7B1B9592B5F053923 /* bundle_spec_test.json */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.json; path = bundle_spec_test.json; sourceTree = "<group>"; };
 		7B65C996438B84DBC7616640 /* CodableTimestampTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = CodableTimestampTests.swift; sourceTree = "<group>"; };
@@ -2636,6 +2640,7 @@
 				5492E06C202154D500B64F25 /* FIRDatabaseTests.mm */,
 				5492E06A202154D500B64F25 /* FIRFieldsTests.mm */,
 				6161B5012047140400A99DBB /* FIRFirestoreSourceTests.mm */,
+				795AA8FC31D2AF6864B07D39 /* FIRIndexingTests.mm */,
 				5492E06B202154D500B64F25 /* FIRListenerRegistrationTests.mm */,
 				D5B25E7E7D6873CBA4571841 /* FIRNumericTransformTests.mm */,
 				5492E069202154D500B64F25 /* FIRQueryTests.mm */,
@@ -3982,6 +3987,7 @@
 				BC2D0A8EA272A0058F6C2B9E /* FIRFirestoreSourceTests.mm in Sources */,
 				CBC1C0459C73BB4B06998401 /* FIRFirestoreTests.mm in Sources */,
 				61D35E0DE04E70D3BC243A65 /* FIRGeoPointTests.mm in Sources */,
+				7264B73291F7F1EB454C45B1 /* FIRIndexingTests.mm in Sources */,
 				F19B749671F2552E964422F7 /* FIRListenerRegistrationTests.mm in Sources */,
 				08F44F7DF9A3EF0D35C8FB57 /* FIRNumericTransformTests.mm in Sources */,
 				6DBB3DB3FD6B4981B7F26A55 /* FIRQuerySnapshotTests.mm in Sources */,
@@ -4201,6 +4207,7 @@
 				8A6C809B9F81C30B7333FCAA /* FIRFirestoreSourceTests.mm in Sources */,
 				457171CE2510EEA46F7D8A30 /* FIRFirestoreTests.mm in Sources */,
 				4809D7ACAA9414E3192F04FF /* FIRGeoPointTests.mm in Sources */,
+				C10417B067155BE78E19807D /* FIRIndexingTests.mm in Sources */,
 				E2B15548A3B6796CE5A01975 /* FIRListenerRegistrationTests.mm in Sources */,
 				B03F286F3AEC3781C386C646 /* FIRNumericTransformTests.mm in Sources */,
 				BB894A81FDF56EEC19CC29F8 /* FIRQuerySnapshotTests.mm in Sources */,
@@ -4654,6 +4661,7 @@
 				6161B5032047140C00A99DBB /* FIRFirestoreSourceTests.mm in Sources */,
 				25C167BAA4284FC951206E1F /* FIRFirestoreTests.mm in Sources */,
 				1B6E74BA33B010D76DB1E2F9 /* FIRGeoPointTests.mm in Sources */,
+				F1EAEE9DF819C017A9506AEB /* FIRIndexingTests.mm in Sources */,
 				5492E074202154D600B64F25 /* FIRListenerRegistrationTests.mm in Sources */,
 				D5B252EE3F4037405DB1ECE3 /* FIRNumericTransformTests.mm in Sources */,
 				6C143182916AC638707DB854 /* FIRQuerySnapshotTests.mm in Sources */,

Firestore/Example/Tests/Integration/API/FIRIndexingTests.mm

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <FirebaseFirestore/FirebaseFirestore.h>
+
+#import <XCTest/XCTest.h>
+
+#import "Firestore/Example/Tests/Util/FSTIntegrationTestCase.h"
+
+@interface FIRIndexingTests : FSTIntegrationTestCase
+@end
+
+@implementation FIRIndexingTests
+
+// Clears persistence for each test method to have a clean start.
+- (void)setUp {
+  [super setUp];
+  self.db = [self firestore];
+  XCTestExpectation* exp = [self expectationWithDescription:@"clear persistence"];
+  [self.db clearPersistenceWithCompletion:^(NSError*) {
+    [exp fulfill];
+  }];
+  [self awaitExpectation:exp];
+}
+
+- (void)testCanConfigureIndexes {
+  NSString* json = @"{\n"
+                    "\t\"indexes\": [{\n"
+                    "\t\t\t\"collectionGroup\": \"restaurants\",\n"
+                    "\t\t\t\"queryScope\": \"COLLECTION\",\n"
+                    "\t\t\t\"fields\": [{\n"
+                    "\t\t\t\t\t\"fieldPath\": \"price\",\n"
+                    "\t\t\t\t\t\"order\": \"ASCENDING\"\n"
+                    "\t\t\t\t},\n"
+                    "\t\t\t\t{\n"
+                    "\t\t\t\t\t\"fieldPath\": \"avgRating\",\n"
+                    "\t\t\t\t\t\"order\": \"DESCENDING\"\n"
+                    "\t\t\t\t}\n"
+                    "\t\t\t]\n"
+                    "\t\t},\n"
+                    "\t\t{\n"
+                    "\t\t\t\"collectionGroup\": \"restaurants\",\n"
+                    "\t\t\t\"queryScope\": \"COLLECTION\",\n"
+                    "\t\t\t\"fields\": [{\n"
+                    "\t\t\t\t\"fieldPath\": \"price\",\n"
+                    "\t\t\t\t\"order\": \"ASCENDING\"\n"
+                    "\t\t\t}]\n"
+                    "\t\t}\n"
+                    "\t],\n"
+                    "\t\"fieldOverrides\": []\n"
+                    "}";
+
+  [self.db setIndexConfigurationFromJSON:json
+                              completion:^(NSError* error) {
+                                XCTAssertNil(error);
+                              }];
+}
+
+- (void)testBadJsonDoesNotCrashClient {
+  [self.db setIndexConfigurationFromJSON:@"{,"
+                              completion:^(NSError* error) {
+                                XCTAssertNotNil(error);
+                                XCTAssertEqualObjects(error.domain, FIRFirestoreErrorDomain);
+                                XCTAssertEqual(error.code, FIRFirestoreErrorCodeInvalidArgument);
+                              }];
+}
+
+- (void)testBadIndexDoesNotCrashClient {
+  NSString* json = @"{\n"
+                    "\t\"indexes\": [{\n"
+                    "\t\t\"collectionGroup\": \"restaurants\",\n"
+                    "\t\t\"queryScope\": \"COLLECTION\",\n"
+                    "\t\t\"fields\": [{\n"
+                    "\t\t\t\"fieldPath\": \"price\",\n"
+                    "\t\t\t\"order\": \"ASCENDING\",\n"
+                    "\t\t]}\n"
+                    "\t}],\n"
+                    "\t\"fieldOverrides\": []\n"
+                    "}";
+
+  [self.db setIndexConfigurationFromJSON:json
+                              completion:^(NSError* error) {
+                                XCTAssertNotNil(error);
+                                XCTAssertEqualObjects(error.domain, FIRFirestoreErrorDomain);
+                                XCTAssertEqual(error.code, FIRFirestoreErrorCodeInvalidArgument);
+                              }];
+}
+
+@end

Firestore/Source/API/FIRFirestore.mm

@@ -87,6 +87,7 @@
 using firebase::firestore::util::ThrowInvalidArgument;
 using firebase::firestore::util::kLogLevelDebug;
 using firebase::firestore::util::kLogLevelNotice;
+using firebase::firestore::util::StreamReadResult;
 
 using UserUpdateBlock = id _Nullable (^)(FIRTransaction *, NSError **);
 using UserTransactionCompletion = void (^)(id _Nullable, NSError *_Nullable);
@@ -206,6 +207,31 @@ - (void)setSettings:(FIRFirestoreSettings *)settings {
   }
 }
 
+- (void)setIndexConfigurationFromJSON:(NSString *)json
+                           completion:(nullable void (^)(NSError *_Nullable error))completion {
+  _firestore->SetIndexConfiguration(MakeString(json), MakeCallback(completion));
+}
+
+- (void)setIndexConfigurationFromStream:(NSInputStream *)stream
+                             completion:(nullable void (^)(NSError *_Nullable error))completion {
+  auto input = absl::make_unique<ByteStreamApple>(stream);
+  auto callback = MakeCallback(completion);
+
+  std::string json;
+  bool eof = false;
+  while (!eof) {
+    StreamReadResult result = input->Read(1024ul);
+    if (!result.ok()) {
+      callback(result.status());
+      return;
+    }
+    eof = result.eof();
+    json.append(std::move(result).ValueOrDie());
+  }
+
+  _firestore->SetIndexConfiguration(json, callback);
+}
+
 - (FIRCollectionReference *)collectionWithPath:(NSString *)collectionPath {
   if (!collectionPath) {
     ThrowInvalidArgument("Collection path cannot be nil.");

Firestore/Source/Public/FirebaseFirestore/FIRFirestore.h

@@ -72,6 +72,53 @@ NS_SWIFT_NAME(Firestore)
  */
 @property(strong, nonatomic, readonly) FIRApp *app;
 
+#pragma mark - Configure FieldIndexes
+
+/**
+ * This method is in preview. API signature and functionality are subject to change.
+ *
+ * Configures indexing for local query execution. Any previous index configuration is overridden.
+ *
+ * The index entries themselves are created asynchronously. You can continue to use queries
+ * that require indexing even if the indices are not yet available. Query execution will
+ * automatically start using the index once the index entries have been written.
+ *
+ * The method accepts the JSON format exported by the Firebase CLI (`firebase
+ * firestore:indexes`). If the JSON format is invalid, the completion block will be
+ * invoked with a NSError.
+ *
+ * @param json The JSON format exported by the Firebase CLI.
+ * @param completion A block to execute when setting is in a final state. The `error` parameter
+ * will be set if the block is invoked due to an error.
+ */
+- (void)setIndexConfigurationFromJSON:(NSString *)json
+                           completion:(nullable void (^)(NSError *_Nullable error))completion
+    NS_SWIFT_NAME(setIndexConfiguration(_:completion:));
+
+/**
+ * This method is in preview. API signature and functionality are subject to change.
+ *
+ * Configures indexing for local query execution. Any previous index configuration is overridden.
+ *
+ * The index entries themselves are created asynchronously. You can continue to use queries
+ * that require indexing even if the indices are not yet available. Query execution will
+ * automatically start using the index once the index entries have been written.
+ *
+ * Indexes are only supported with LevelDB persistence. Invoke `set_persistence_enabled(true)`
+ * before setting an index configuration. If LevelDB is not enabled, any index configuration
+ * will be rejected.
+ *
+ * The method accepts the JSON format exported by the Firebase CLI (`firebase
+ * firestore:indexes`). If the JSON format is invalid, this method ignores the changes.
+ *
+ * @param stream An input stream from which the configuration can be read.
+ * @param completion A block to execute when setting is in a final state. The `error` parameter
+ * will be set if the block is invoked due to an error.
+ */
+- (void)setIndexConfigurationFromStream:(NSInputStream *)stream
+                             completion:(nullable void (^)(NSError *_Nullable error))completion
+    NS_SWIFT_NAME(setIndexConfiguration(_:completion:));
+
 #pragma mark - Collections and Documents
 
 /**

Firestore/core/src/api/firestore.cc

@@ -17,6 +17,7 @@
 #include "Firestore/core/src/api/firestore.h"
 
 #include <utility>
+#include <vector>
 
 #include "Firestore/core/src/api/collection_reference.h"
 #include "Firestore/core/src/api/document_reference.h"
@@ -31,13 +32,18 @@
 #include "Firestore/core/src/credentials/empty_credentials_provider.h"
 #include "Firestore/core/src/local/leveldb_persistence.h"
 #include "Firestore/core/src/model/document_key.h"
+#include "Firestore/core/src/model/field_path.h"
 #include "Firestore/core/src/model/resource_path.h"
 #include "Firestore/core/src/remote/firebase_metadata_provider.h"
 #include "Firestore/core/src/remote/grpc_connection.h"
 #include "Firestore/core/src/util/async_queue.h"
+#include "Firestore/core/src/util/exception.h"
 #include "Firestore/core/src/util/executor.h"
 #include "Firestore/core/src/util/hard_assert.h"
+#include "Firestore/core/src/util/json_reader.h"
+#include "Firestore/core/src/util/log.h"
 #include "Firestore/core/src/util/status.h"
+#include "Firestore/third_party/nlohmann_json/json.hpp"
 #include "absl/memory/memory.h"
 
 namespace firebase {
@@ -49,13 +55,18 @@ using core::DatabaseInfo;
 using core::FirestoreClient;
 using credentials::AuthCredentialsProvider;
 using local::LevelDbPersistence;
+using model::FieldIndex;
+using model::FieldPath;
 using model::ResourcePath;
+using model::Segment;
+using nlohmann::json;
 using remote::FirebaseMetadataProvider;
 using remote::GrpcConnection;
 using util::AsyncQueue;
 using util::Empty;
 using util::Executor;
 using util::Status;
+using util::ThrowInvalidArgument;
 
 const int kDefaultTransactionMaxAttempts = 5;
 
@@ -242,6 +253,70 @@ DatabaseInfo Firestore::MakeDatabaseInfo() const {
                       settings_.ssl_enabled());
 }
 
+void Firestore::SetIndexConfiguration(const std::string& config,
+                                      const util::StatusCallback& callback) {
+  EnsureClientConfigured();
+
+  util::JsonReader reader;
+  if (!settings_.persistence_enabled()) {
+    LOG_DEBUG("Cannot enable indexes when persistence is disabled.");
+    callback(util::Status::OK());
+    return;
+  }
+
+  auto json_object =
+      nlohmann::json::parse(config.begin(), config.end(),
+                            /*callback=*/nullptr, /*allow_exceptions=*/false);
+  if (json_object.is_discarded()) {
+    callback(Status(Error::kErrorInvalidArgument, "Invalid Json format."));
+    return;
+  }
+
+  std::vector<FieldIndex> parsed_indexes;
+  const std::vector<json> default_vector;
+  const auto& json_indexes =
+      reader.OptionalArray("indexes", json_object, default_vector);
+  for (const auto& json_index : json_indexes) {
+    const std::string& collection_group =
+        reader.RequiredString("collectionGroup", json_index);
+    std::vector<Segment> segments;
+    const auto& json_fields =
+        reader.OptionalArray("fields", json_index, default_vector);
+    for (const auto& json_field : json_fields) {
+      FieldPath field_path = FieldPath::FromServerFormat(
+                                 reader.RequiredString("fieldPath", json_field))
+                                 .ValueOrDie();
+      std::string default_string;
+      if ("CONTAINS" ==
+          reader.OptionalString("arrayConfig", json_field, default_string)) {
+        segments.emplace_back(
+            Segment(std::move(field_path), Segment::Kind::kContains));
+      } else if ("ASCENDING" ==
+                 reader.OptionalString("order", json_field, default_string)) {
+        segments.emplace_back(
+            Segment(std::move(field_path), Segment::Kind::kAscending));
+      } else {
+        segments.emplace_back(
+            Segment(std::move(field_path), Segment::Kind::kDescending));
+      }
+    }
+
+    if (reader.status() != util::Status::OK()) {
+      callback(reader.status());
+      return;
+    }
+
+    parsed_indexes.emplace_back(
+        FieldIndex(FieldIndex::UnknownId(), collection_group,
+                   std::move(segments), FieldIndex::InitialState()));
+  }
+
+  client_->ConfigureFieldIndexes(std::move(parsed_indexes));
+
+  callback(util::Status::OK());
+  return;
+}
+
 std::shared_ptr<LoadBundleTask> Firestore::LoadBundle(
     std::unique_ptr<util::ByteStream> bundle_data) {
   EnsureClientConfigured();

Firestore/core/src/api/firestore.h

@@ -107,6 +107,9 @@ class Firestore : public std::enable_shared_from_this<Firestore> {
   void EnableNetwork(util::StatusCallback callback);
   void DisableNetwork(util::StatusCallback callback);
 
+  void SetIndexConfiguration(const std::string& config,
+                             const util::StatusCallback& callback);
+
   std::shared_ptr<api::LoadBundleTask> LoadBundle(
       std::unique_ptr<util::ByteStream> bundle_data);
   void GetNamedQuery(const std::string& name, api::QueryCallback callback);