Skip to content

Commit b0f2783

Browse files
schmidt-sebastianschmidt-sebastian
authored
Add IndexConfiguration (#2856)
1 parent d318e06 commit b0f2783

File tree

13 files changed

+593
-9
lines changed

13 files changed

+593
-9
lines changed

‎firebase-firestore/src/androidTest/java/com/google/firebase/firestore/IndexingTest.java

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Copyright 2021 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.firebase.firestore;
16+
17+
import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testFirestore;
18+
19+
import androidx.test.ext.junit.runners.AndroidJUnit4;
20+
import com.google.android.gms.tasks.Task;
21+
import com.google.android.gms.tasks.Tasks;
22+
import com.google.firebase.firestore.local.Persistence;
23+
import com.google.firebase.firestore.testutil.Assert;
24+
import com.google.firebase.firestore.testutil.IntegrationTestUtil;
25+
import java.util.concurrent.ExecutionException;
26+
import org.junit.After;
27+
import org.junit.BeforeClass;
28+
import org.junit.Test;
29+
import org.junit.runner.RunWith;
30+
31+
@RunWith(AndroidJUnit4.class)
32+
public class IndexingTest {
33+
/** Current state of indexing support. Used for restoring after test run. */
34+
private static final boolean supportsIndexing = Persistence.INDEXING_SUPPORT_ENABLED;
35+
36+
@BeforeClass
37+
public static void beforeClass() {
38+
Persistence.INDEXING_SUPPORT_ENABLED = true;
39+
}
40+
41+
@BeforeClass
42+
public static void afterClass() {
43+
Persistence.INDEXING_SUPPORT_ENABLED = supportsIndexing;
44+
}
45+
46+
@After
47+
public void tearDown() {
48+
IntegrationTestUtil.tearDown();
49+
}
50+
51+
@Test
52+
public void testCanConfigureIndices() throws ExecutionException, InterruptedException {
53+
FirebaseFirestore db = testFirestore();
54+
Task<Void> indexTask =
55+
db.configureIndices(
56+
"{\n"
57+
+ " \"indexes\": [\n"
58+
+ " {\n"
59+
+ " \"collectionGroup\": \"restaurants\",\n"
60+
+ " \"queryScope\": \"COLLECTION\",\n"
61+
+ " \"fields\": [\n"
62+
+ " {\n"
63+
+ " \"fieldPath\": \"price\",\n"
64+
+ " \"order\": \"ASCENDING\"\n"
65+
+ " },\n"
66+
+ " {\n"
67+
+ " \"fieldPath\": \"avgRating\",\n"
68+
+ " \"order\": \"DESCENDING\"\n"
69+
+ " }\n"
70+
+ " ]\n"
71+
+ " },\n"
72+
+ " {\n"
73+
+ " \"collectionGroup\": \"restaurants\",\n"
74+
+ " \"queryScope\": \"COLLECTION\",\n"
75+
+ " \"fields\": [\n"
76+
+ " {\n"
77+
+ " \"fieldPath\": \"price\",\n"
78+
+ " \"order\": \"ASCENDING\"\n"
79+
+ " }"
80+
+ " ]\n"
81+
+ " }\n"
82+
+ " ],\n"
83+
+ " \"fieldOverrides\": []\n"
84+
+ "}");
85+
Tasks.await(indexTask);
86+
}
87+
88+
@Test
89+
public void testBadJsonDoesNotCrashClient() {
90+
FirebaseFirestore db = testFirestore();
91+
92+
Assert.assertThrows(IllegalArgumentException.class, () -> db.configureIndices("{,"));
93+
}
94+
95+
@Test
96+
public void testBadIndexDoesNotCrashClient() {
97+
FirebaseFirestore db = testFirestore();
98+
Assert.assertThrows(
99+
IllegalArgumentException.class,
100+
() ->
101+
db.configureIndices(
102+
"{\n"
103+
+ " \"indexes\": [\n"
104+
+ " {\n"
105+
+ " \"collectionGroup\": \"restaurants\",\n"
106+
+ " \"queryScope\": \"COLLECTION\",\n"
107+
+ " \"fields\": [\n"
108+
+ " {\n"
109+
+ " \"fieldPath\": \"price\",\n"
110+
+ " \"order\": \"INVALID\"\n"
111+
+ " ,\n"
112+
+ " ]\n"
113+
+ " }\n"
114+
+ " ],\n"
115+
+ " \"fieldOverrides\": []\n"
116+
+ "}"));
117+
}
118+
119+
// TODO(indexing): Add tests that validate that indices are active
120+
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
import com.google.firebase.firestore.core.FirestoreClient;
3939
import com.google.firebase.firestore.local.SQLitePersistence;
4040
import com.google.firebase.firestore.model.DatabaseId;
41+
import com.google.firebase.firestore.model.FieldIndex;
42+
import com.google.firebase.firestore.model.FieldPath;
4143
import com.google.firebase.firestore.model.ResourcePath;
4244
import com.google.firebase.firestore.remote.FirestoreChannel;
4345
import com.google.firebase.firestore.remote.GrpcMetadataProvider;
@@ -51,7 +53,12 @@
5153
import java.io.ByteArrayInputStream;
5254
import java.io.InputStream;
5355
import java.nio.ByteBuffer;
56+
import java.util.ArrayList;
57+
import java.util.List;
5458
import java.util.concurrent.Executor;
59+
import org.json.JSONArray;
60+
import org.json.JSONException;
61+
import org.json.JSONObject;
5562

5663
/**
5764
* Represents a Cloud Firestore database and is the entry point for all Cloud Firestore operations.
@@ -264,6 +271,69 @@ public FirebaseApp getApp() {
264271
return firebaseApp;
265272
}
266273

274+
/**
275+
* Configures Indexing for local query execution. Any previous index configuration is overridden.
276+
* The Task resolves once the index configuration has been persisted.
277+
*
278+
* <p>The index entries themselves are created asynchronously. You can continue to use queries
279+
* that require indexing even if the indices are not yet available. Query execution will
280+
* automatically start using the index once the index entries have been written.
281+
*
282+
* <p>The method accepts the JSON format exported by the Firebase CLI (`firebase
283+
* firestore:indexes`). If the JSON format is invalid, this method rejects the returned task.
284+
*
285+
* @param json The JSON format exported by the Firebase CLI.
286+
* @return A task that resolves once all indices are successfully configured.
287+
*/
288+
@VisibleForTesting
289+
Task<Void> configureIndices(String json) {
290+
ensureClientConfigured();
291+
292+
// Preconditions.checkState(BuildConfig.ENABLE_INDEXING, "Indexing support is not yet
293+
// available.");
294+
295+
List<FieldIndex> parsedIndices = new ArrayList<>();
296+
297+
// See https://firebase.google.com/docs/reference/firestore/indexes/#json_format for the
298+
// format of the index definition. Unlike the backend, the SDK does not distinguish between
299+
// collection ID and collection group indices and hence the queryScope field is ignored.
300+
301+
try {
302+
JSONObject jsonObject = new JSONObject(json);
303+
304+
if (jsonObject.has("indexes")) {
305+
JSONArray indices = jsonObject.getJSONArray("indexes");
306+
for (int i = 0; i < indices.length(); ++i) {
307+
JSONObject definition = indices.getJSONObject(i);
308+
FieldIndex fieldIndex = new FieldIndex(definition.getString("collectionGroup"));
309+
310+
JSONArray fields = definition.optJSONArray("fields");
311+
for (int f = 0; fields != null && f < fields.length(); ++f) {
312+
JSONObject field = fields.getJSONObject(f);
313+
FieldPath fieldPath = FieldPath.fromServerFormat(field.getString("fieldPath"));
314+
if ("CONTAINS".equals(field.optString("arrayConfig"))) {
315+
fieldIndex = fieldIndex.withAddedField(fieldPath, FieldIndex.Segment.Kind.CONTAINS);
316+
} else if ("ASCENDING".equals(field.optString("order"))) {
317+
fieldIndex = fieldIndex.withAddedField(fieldPath, FieldIndex.Segment.Kind.ASCENDING);
318+
} else if ("DESCENDING".equals(field.optString("order"))) {
319+
fieldIndex = fieldIndex.withAddedField(fieldPath, FieldIndex.Segment.Kind.DESCENDING);
320+
} else {
321+
throw new IllegalArgumentException(
322+
String.format(
323+
"Index definition not valid (for field \"%s\" of collection \"%s\")",
324+
fieldPath, fieldIndex.getCollectionId()));
325+
}
326+
parsedIndices.add(fieldIndex);
327+
}
328+
}
329+
}
330+
} catch (JSONException e) {
331+
throw new IllegalArgumentException("Failed to parse index configuration", e);
332+
}
333+
334+
return client.configureIndices(parsedIndices);
335+
}
336+
267337
/**
268338
* Gets a {@code CollectionReference} instance that refers to the collection at the specified path
269339
* within the database.

‎firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import com.google.firebase.firestore.local.QueryResult;
3939
import com.google.firebase.firestore.model.Document;
4040
import com.google.firebase.firestore.model.DocumentKey;
41+
import com.google.firebase.firestore.model.FieldIndex;
4142
import com.google.firebase.firestore.model.mutation.Mutation;
4243
import com.google.firebase.firestore.remote.Datastore;
4344
import com.google.firebase.firestore.remote.GrpcMetadataProvider;
@@ -301,6 +302,11 @@ public Task<Query> getNamedQuery(String queryName) {
301302
return completionSource.getTask();
302303
}
303304

305+
public Task<Void> configureIndices(List<FieldIndex> fieldIndices) {
306+
verifyNotTerminated();
307+
return asyncQueue.enqueue(() -> localStore.configureIndices(fieldIndices));
308+
}
309+
304310
public void removeSnapshotsInSyncListener(EventListener<Void> listener) {
305311
// Checks for shutdown but does not raise error, allowing remove after shutdown to be a no-op.
306312
if (isTerminated()) {

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexManager.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.google.firebase.firestore.local;
1616

17+
import com.google.firebase.firestore.model.FieldIndex;
1718
import com.google.firebase.firestore.model.ResourcePath;
1819
import java.util.List;
1920

@@ -39,4 +40,12 @@ public interface IndexManager {
3940
* being either a document location or the empty path for a root-level collection).
4041
*/
4142
List<ResourcePath> getCollectionParents(String collectionId);
43+
44+
/**
45+
* Adds a field path index.
46+
*
47+
* <p>Values for this index are persisted asynchronously. The index will only be used for query
48+
* execution once values are persisted.
49+
*/
50+
void addFieldIndex(FieldIndex index);
4251
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalSerializer.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import com.google.firebase.firestore.core.Query.LimitType;
2323
import com.google.firebase.firestore.core.Target;
2424
import com.google.firebase.firestore.model.DocumentKey;
25+
import com.google.firebase.firestore.model.FieldIndex;
2526
import com.google.firebase.firestore.model.MutableDocument;
2627
import com.google.firebase.firestore.model.ObjectValue;
2728
import com.google.firebase.firestore.model.SnapshotVersion;
2829
import com.google.firebase.firestore.model.mutation.Mutation;
2930
import com.google.firebase.firestore.model.mutation.MutationBatch;
3031
import com.google.firebase.firestore.remote.RemoteSerializer;
32+
import com.google.firestore.admin.v1.Index;
3133
import com.google.firestore.v1.DocumentTransform.FieldTransform;
3234
import com.google.firestore.v1.Write;
3335
import com.google.firestore.v1.Write.Builder;
@@ -287,4 +289,32 @@ public BundledQuery decodeBundledQuery(com.google.firestore.bundle.BundledQuery
287289

288290
return new BundledQuery(target, limitType);
289291
}
292+
293+
public Index encodeFieldIndex(FieldIndex fieldIndex) {
294+
Index.Builder index = Index.newBuilder();
295+
// The Mobile SDKs treat all indices as collection group indices, as we run all collection group
296+
// queries against each collection separately.
297+
index.setQueryScope(Index.QueryScope.COLLECTION_GROUP);
298+
299+
for (FieldIndex.Segment segment : fieldIndex) {
300+
Index.IndexField.Builder indexField = Index.IndexField.newBuilder();
301+
indexField.setFieldPath(segment.getFieldPath().canonicalString());
302+
switch (segment.getKind()) {
303+
case ASCENDING:
304+
indexField.setOrder(Index.IndexField.Order.ASCENDING);
305+
break;
306+
case DESCENDING:
307+
indexField.setOrder(Index.IndexField.Order.DESCENDING);
308+
break;
309+
case CONTAINS:
310+
indexField.setArrayConfig(Index.IndexField.ArrayConfig.CONTAINS);
311+
break;
312+
default:
313+
throw fail("Unknown index kind %s", segment.getKind());
314+
}
315+
index.addFields(indexField);
316+
}
317+
318+
return index.build();
319+
}
290320
}

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.firebase.firestore.core.TargetIdGenerator;
3333
import com.google.firebase.firestore.model.Document;
3434
import com.google.firebase.firestore.model.DocumentKey;
35+
import com.google.firebase.firestore.model.FieldIndex;
3536
import com.google.firebase.firestore.model.MutableDocument;
3637
import com.google.firebase.firestore.model.ObjectValue;
3738
import com.google.firebase.firestore.model.ResourcePath;
@@ -106,6 +107,9 @@ public final class LocalStore implements BundleCallback {
106107
/** Manages our in-memory or durable persistence. */
107108
private final Persistence persistence;
108109

110+
/** Manages the list of active field and collection indices. */
111+
private final IndexManager indexManager;
112+
109113
/** The set of all mutations that have been sent but not yet been applied to the backend. */
110114
private MutationQueue mutationQueue;
111115

@@ -145,8 +149,8 @@ public LocalStore(Persistence persistence, QueryEngine queryEngine, User initial
145149
targetIdGenerator = TargetIdGenerator.forTargetCache(targetCache.getHighestTargetId());
146150
mutationQueue = persistence.getMutationQueue(initialUser);
147151
remoteDocuments = persistence.getRemoteDocumentCache();
148-
localDocuments =
149-
new LocalDocumentsView(remoteDocuments, mutationQueue, persistence.getIndexManager());
152+
indexManager = persistence.getIndexManager();
153+
localDocuments = new LocalDocumentsView(remoteDocuments, mutationQueue, indexManager);
150154

151155
this.queryEngine = queryEngine;
152156
queryEngine.setLocalDocumentsView(localDocuments);
@@ -182,8 +186,7 @@ public ImmutableSortedMap<DocumentKey, Document> handleUserChange(User user) {
182186
List<MutationBatch> newBatches = mutationQueue.getAllMutationBatches();
183187

184188
// Recreate our LocalDocumentsView using the new MutationQueue.
185-
localDocuments =
186-
new LocalDocumentsView(remoteDocuments, mutationQueue, persistence.getIndexManager());
189+
localDocuments = new LocalDocumentsView(remoteDocuments, mutationQueue, indexManager);
187190
queryEngine.setLocalDocumentsView(localDocuments);
188191

189192
// Union the old/new changed keys.
@@ -697,6 +700,17 @@ public void saveNamedQuery(NamedQuery namedQuery, ImmutableSortedSet<DocumentKey
697700
"Get named query", () -> bundleCache.getNamedQuery(queryName));
698701
}
699702

703+
public void configureIndices(List<FieldIndex> fieldIndices) {
704+
persistence.runTransaction(
705+
"Configure indices",
706+
() -> {
707+
// TODO(indexing): Disable no longer active indices
708+
for (FieldIndex fieldIndex : fieldIndices) {
709+
indexManager.addFieldIndex(fieldIndex);
710+
}
711+
});
712+
}
713+
700714
/** Mutable state for the transaction in allocateQuery. */
701715
private static class AllocateQueryHolder {
702716
TargetData cached;

‎firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryIndexManager.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import static com.google.firebase.firestore.util.Assert.hardAssert;
1717

18+
import com.google.firebase.firestore.model.FieldIndex;
1819
import com.google.firebase.firestore.model.ResourcePath;
1920
import java.util.ArrayList;
2021
import java.util.Collections;
@@ -37,6 +38,11 @@ public List<ResourcePath> getCollectionParents(String collectionId) {
3738
return collectionParentsIndex.getEntries(collectionId);
3839
}
3940

41+
@Override
42+
public void addFieldIndex(FieldIndex index) {
43+
// Field indices are not supported with memory persistence.
44+
}
45+
4046
/**
4147
* Internal implementation of the collection-parent index. Also used for in-memory caching by
4248
* SQLiteIndexManager and initial index population in SQLiteSchema.

0 commit comments

Comments
 (0)