Skip to content

Commit 986eb5a

Browse files
author
Burak Serdar
committed
Add in-memory indexing during association if result set is large
1 parent 081811b commit 986eb5a

File tree

9 files changed

+183
-85
lines changed

9 files changed

+183
-85
lines changed

‎crud/src/main/java/com/redhat/lightblue/assoc/ep/Assemble.java‎

Lines changed: 61 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.HashMap;
2323
import java.util.List;
2424
import java.util.ArrayList;
25+
import java.util.Set;
2526

2627
import java.util.stream.Collectors;
2728

@@ -41,14 +42,19 @@
4142
import com.redhat.lightblue.metadata.CompositeMetadata;
4243

4344
import com.redhat.lightblue.assoc.BindQuery;
45+
import com.redhat.lightblue.assoc.QueryFieldInfo;
46+
import com.redhat.lightblue.assoc.AnalyzeQuery;
4447

4548
import com.redhat.lightblue.mindex.MemDocIndex;
4649
import com.redhat.lightblue.mindex.GetIndexKeySpec;
50+
import com.redhat.lightblue.mindex.GetIndexLookupSpec;
4751
import com.redhat.lightblue.mindex.KeySpec;
52+
import com.redhat.lightblue.mindex.LookupSpec;
4853

4954
import com.redhat.lightblue.eval.QueryEvaluator;
5055

5156
import com.redhat.lightblue.util.Path;
57+
import com.redhat.lightblue.util.JsonDoc;
5258

5359
/**
5460
* There are two sides to an Assemble step: Assemble gets results from the
@@ -60,6 +66,13 @@ public class Assemble extends Step<ResultDocument> {
6066

6167
private static final Logger LOGGER = LoggerFactory.getLogger(Assemble.class);
6268

69+
/**
70+
* This is used for testing. This is the threshold for the number
71+
* of slots above which we'll use the memory indexing. If we have
72+
* slots fewer than this, then we don't use in-memory index.
73+
*/
74+
public static int MEM_INDEX_THRESHOLD=16;
75+
6376
private final ExecutionBlock[] destinationBlocks;
6477
private final Source<ResultDocument> source;
6578
private Map<ExecutionBlock, Assemble> destinations;
@@ -156,7 +169,7 @@ public DocAndQ(ResultDocument doc) {
156169
}
157170
}
158171

159-
private static class BatchAssembler {
172+
private class BatchAssembler {
160173
private List<DocAndQ> docs = new ArrayList<>();
161174
private List<QueryExpression> queries = new ArrayList<>();
162175
private final int batchSize;
@@ -196,9 +209,15 @@ public void commit() {
196209
combinedQuery = null;
197210
}
198211
List<ResultDocument> destResults = dest.getResultList(combinedQuery, ctx);
212+
int numSlots=0;
213+
for(ResultDocument doc:destResults) {
214+
List<ChildSlot> slots=doc.getSlots().get(aq.getReference());
215+
if(slots!=null)
216+
numSlots+=slots.size();
217+
}
199218
// Try to build an index from results
200219
MemDocIndex docIndex=null;
201-
if(aq.getQuery()!=null&&destResults.size()>2) {
220+
if(aq.getQuery()!=null&&numSlots>MEM_INDEX_THRESHOLD) {
202221
// Lets see if we can build a key spec from this query
203222
GetIndexKeySpec giks=new GetIndexKeySpec(aq.getQueryFieldInfo());
204223
KeySpec keySpec=giks.iterate(aq.getQuery());
@@ -242,10 +261,10 @@ private JsonNode toJson(Step.ToJsonCb<Step> scb,Step.ToJsonCb<ExecutionBlock> bc
242261
* Associates child documents obtained from 'aq' to all the slots in the
243262
* parent document
244263
*/
245-
public static void associateDocs(ResultDocument parentDoc,
246-
List<ResultDocument> childDocs,
247-
AssociationQuery aq,
248-
MemDocIndex childIndex) {
264+
public void associateDocs(ResultDocument parentDoc,
265+
List<ResultDocument> childDocs,
266+
AssociationQuery aq,
267+
MemDocIndex childIndex) {
249268
if(!childDocs.isEmpty()) {
250269
CompositeMetadata childMetadata = childDocs.get(0).getBlock().getMetadata();
251270
List<ChildSlot> slots = parentDoc.getSlots().get(aq.getReference());
@@ -314,16 +333,45 @@ public static void associateDocs(CompositeMetadata childMetadata,
314333
}
315334
}
316335

317-
private static void associateDocsWithIndex(CompositeMetadata childMetadata,
318-
ResultDocument parentDoc,
319-
Path destFieldName,
320-
BindQuery binders,
321-
List<ResultDocument> childDocs,
322-
AssociationQuery aq,
323-
MemDocIndex childIndex) {
336+
private void associateDocsWithIndex(CompositeMetadata childMetadata,
337+
ResultDocument parentDoc,
338+
Path destFieldName,
339+
BindQuery binders,
340+
List<ResultDocument> childDocs,
341+
AssociationQuery aq,
342+
MemDocIndex childIndex) {
324343
LOGGER.debug("Associating docs using index");
344+
QueryExpression boundQuery = binders.iterate(aq.getQuery());
345+
LOGGER.debug("Association query:{}", boundQuery);
346+
QueryEvaluator qeval = QueryEvaluator.getInstance(boundQuery, childMetadata);
347+
AnalyzeQuery analyzer=new AnalyzeQuery(block.rootMd,aq.getReference());
348+
analyzer.iterate(boundQuery);
349+
List<QueryFieldInfo> qfi=analyzer.getFieldInfo();
350+
GetIndexLookupSpec gils=new GetIndexLookupSpec(qfi);
351+
LookupSpec ls=gils.iterate(boundQuery);
352+
LOGGER.debug("Lookup spec:"+ls);
353+
List<ResultDocument> docs=reorder(childDocs,childIndex.find(ls));
354+
ArrayNode destNode=null;
355+
for (ResultDocument childDoc : childDocs) {
356+
if (qeval.evaluate(childDoc.getDoc()).getResult()) {
357+
destNode=ensureDestNodeExists(parentDoc,destNode,destFieldName);
358+
destNode.add(childDoc.getDoc().getRoot());
359+
}
360+
}
325361
}
326362

363+
/**
364+
* Returns the documents in foundList in the order of originalList
365+
*/
366+
private static List<ResultDocument> reorder(List<ResultDocument> originalList,Set<JsonDoc> foundList) {
367+
List<ResultDocument> ret=new ArrayList<>(foundList.size());
368+
for(ResultDocument d:originalList) {
369+
if(foundList.contains(d.getDoc()))
370+
ret.add(d);
371+
}
372+
return ret;
373+
}
374+
327375
@Override
328376
public JsonNode toJson() {
329377
return toJson(Step::toJson,ExecutionBlock::toJson);

‎crud/src/main/java/com/redhat/lightblue/assoc/ep/AssociationQuery.java‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import com.redhat.lightblue.assoc.BoundObject;
1010
import com.redhat.lightblue.assoc.Conjunct;
1111
import com.redhat.lightblue.assoc.RewriteQuery;
12+
import com.redhat.lightblue.assoc.QueryFieldInfo;
13+
import com.redhat.lightblue.assoc.AnalyzeQuery;
1214

1315
import com.redhat.lightblue.query.QueryExpression;
1416

@@ -22,7 +24,24 @@ public class AssociationQuery {
2224
private final ResolvedReferenceField reference;
2325
// If non-null, query is either always true or always false
2426
private final Boolean always;
27+
private final List<QueryFieldInfo> qfi;
28+
private final List<AssociationQueryConjunct> aqConjuncts=new ArrayList<>();
2529

30+
public static class AssociationQueryConjunct {
31+
public final QueryExpression query;
32+
public final List<QueryFieldInfo> qfi;
33+
34+
AssociationQueryConjunct(QueryExpression q,List<QueryFieldInfo> qfi) {
35+
this.query=q;
36+
this.qfi=qfi;
37+
}
38+
39+
@Override
40+
public String toString() {
41+
return query.toString();
42+
}
43+
}
44+
2645
public AssociationQuery(CompositeMetadata root,
2746
CompositeMetadata currentEntity,
2847
ResolvedReferenceField reference,
@@ -32,6 +51,7 @@ public AssociationQuery(CompositeMetadata root,
3251
List<QueryExpression> queries = new ArrayList<>(conjuncts.size());
3352
int numTrue=0;
3453
int numFalse=0;
54+
qfi=new ArrayList<>();
3555
for (Conjunct c : conjuncts) {
3656
RewriteQuery.RewriteQueryResult result = rewriter.rewriteQuery(c.getClause(), c.getFieldInfo());
3757
if(result.query instanceof RewriteQuery.TruePH) {
@@ -41,7 +61,13 @@ public AssociationQuery(CompositeMetadata root,
4161
numFalse++;
4262
} else {
4363
queries.add(result.query);
44-
}
64+
// Analyze the query as if it is a root entity query
65+
AnalyzeQuery aq=new AnalyzeQuery(currentEntity,null);
66+
aq.iterate(result.query);
67+
AssociationQueryConjunct q=new AssociationQueryConjunct(result.query,aq.getFieldInfo());
68+
aqConjuncts.add(q);
69+
qfi.addAll(q.qfi);
70+
}
4571
fieldBindings.addAll(result.bindings);
4672
}
4773
if(queries.isEmpty()) {
@@ -59,6 +85,14 @@ public AssociationQuery(CompositeMetadata root,
5985
}
6086
}
6187

88+
public List<AssociationQueryConjunct> getConjuncts() {
89+
return aqConjuncts;
90+
}
91+
92+
public List<QueryFieldInfo> getQueryFieldInfo() {
93+
return qfi;
94+
}
95+
6296
public Boolean getAlways() {
6397
return always;
6498
}

‎crud/src/main/java/com/redhat/lightblue/assoc/ep/ExecutionBlock.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
*/
2424
public class ExecutionBlock {
2525

26-
private final CompositeMetadata rootMd;
26+
public final CompositeMetadata rootMd;
2727

2828
/**
2929
* The query plan node corresponding to this execution block

‎crud/src/main/java/com/redhat/lightblue/mindex/ArrayKeySpec.java‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@
2727
import com.fasterxml.jackson.databind.JsonNode;
2828
import com.fasterxml.jackson.databind.node.ArrayNode;
2929

30-
import com.redhat.lightblue.metadata.ArrayField;
31-
import com.redhat.lightblue.metadata.ArrayElement;
30+
import com.redhat.lightblue.assoc.QueryFieldInfo;
3231

3332
import com.redhat.lightblue.util.JsonDoc;
3433
import com.redhat.lightblue.util.Path;
@@ -41,32 +40,13 @@
4140
*/
4241
public class ArrayKeySpec implements KeySpec,Comparator<ArrayKey> {
4342
final KeySpec[] keyFields;
44-
final ArrayField array;
45-
46-
final ArrayElement element;
4743
final Path arrayName;
4844

49-
public ArrayKeySpec(ArrayField array,KeySpec[] fields) {
45+
public ArrayKeySpec(QueryFieldInfo array,KeySpec[] fields) {
5046
this.keyFields=fields;
51-
this.array=array;
52-
this.arrayName=array.getFullPath();
53-
this.element=array.getElement();
47+
this.arrayName=array.getEntityRelativeFieldNameWithContext();
5448
}
5549

56-
public ArrayKeySpec(ArrayField containingArray,ArrayField array,KeySpec[] fields) {
57-
this.keyFields=fields;
58-
this.array=array;
59-
Path arrName=containingArray.getElement().getFullPath();
60-
Path full=array.getFullPath();
61-
if(full.prefix(arrName.numSegments()).equals(arrName)) {
62-
this.arrayName=array.getFullPath().suffix(-arrName.numSegments());
63-
} else {
64-
this.arrayName=array.getFullPath();
65-
}
66-
67-
this.element=array.getElement();
68-
}
69-
7050
@Override
7151
public int compareKeys(Key k1,Key k2) {
7252
return compare( (ArrayKey)k1,(ArrayKey)k2);

‎crud/src/main/java/com/redhat/lightblue/mindex/GetIndexKeySpec.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@ protected KeySpec processArrayMatchExpression(QueryExpression nestedExpression,P
9494
ret=null;
9595
} else {
9696
if(nestedSpec instanceof CompositeKeySpec) {
97-
ret=new ArrayKeySpec(enclosingArrayMd, ((CompositeKeySpec)nestedSpec).keyFields);
97+
ret=new ArrayKeySpec(enclosingArray,((CompositeKeySpec)nestedSpec).keyFields);
9898
} else {
99-
ret=new ArrayKeySpec(enclosingArrayMd ,new KeySpec[]{nestedSpec});
99+
ret=new ArrayKeySpec(enclosingArray,new KeySpec[]{nestedSpec});
100100
}
101101
}
102102
return ret;

‎crud/src/main/java/com/redhat/lightblue/mindex/IndexQueryProcessorBase.java‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ abstract class IndexQueryProcessorBase<T> extends QueryIteratorSkeleton<T> {
3535
public static final String ERR_INDEX_ANALYZE="crud:index:QueryAnalysisError";
3636

3737
protected final List<QueryFieldInfo> fieldInfo;
38-
protected ArrayField enclosingArrayMd;
38+
protected QueryFieldInfo enclosingArray;
3939

4040
public IndexQueryProcessorBase(List<QueryFieldInfo> fields) {
4141
this.fieldInfo=fields;
@@ -53,11 +53,7 @@ protected QueryFieldInfo findFieldInfo(Path field, QueryExpression clause) {
5353
}
5454

5555
protected SimpleKeySpec simpleKeySpec(QueryFieldInfo finfo) {
56-
if(enclosingArrayMd==null) {
57-
return new SimpleKeySpec(finfo.getFieldMd());
58-
} else {
59-
return new SimpleKeySpec(enclosingArrayMd,finfo.getFieldMd());
60-
}
56+
return new SimpleKeySpec(finfo);
6157
}
6258

6359
@Override
@@ -176,11 +172,10 @@ protected T itrNaryLogicalExpression(NaryLogicalExpression q, Path context) {
176172

177173
@Override
178174
protected T itrArrayMatchExpression(ArrayMatchExpression q, Path context) {
179-
ArrayField oldArray=enclosingArrayMd;
180-
QueryFieldInfo finfo=findFieldInfo(q.getArray(),q);
181-
enclosingArrayMd=(ArrayField)finfo.getFieldMd();
175+
QueryFieldInfo oldArray=enclosingArray;
176+
enclosingArray=findFieldInfo(q.getArray(),q);
182177
T ret=processArrayMatchExpression(q.getElemMatch(), new Path(new Path(context, q.getArray()), Path.ANYPATH));
183-
enclosingArrayMd=oldArray;
178+
enclosingArray=oldArray;
184179
return ret;
185180
}
186181
}

‎crud/src/main/java/com/redhat/lightblue/mindex/SimpleKeySpec.java‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import com.redhat.lightblue.metadata.FieldTreeNode;
2929
import com.redhat.lightblue.metadata.ArrayField;
3030

31+
import com.redhat.lightblue.assoc.QueryFieldInfo;
32+
3133
import com.redhat.lightblue.util.JsonDoc;
3234
import com.redhat.lightblue.util.Path;
3335
import com.redhat.lightblue.util.KeyValueCursor;
@@ -39,28 +41,14 @@ public class SimpleKeySpec implements KeySpec,Comparator<SimpleKey> {
3941
final FieldTreeNode fieldMd;
4042
final Path fullName;
4143
final Type type;
42-
43-
public SimpleKeySpec(FieldTreeNode fieldMd) {
44-
this.fieldMd=fieldMd;
45-
this.fullName=fieldMd.getFullPath();
46-
this.type=fieldMd.getType();
47-
}
4844

49-
/**
50-
* Create a simple key spec under an array. This initializes the key relative to the array
51-
*/
52-
public SimpleKeySpec(ArrayField containingArray,FieldTreeNode fieldMd) {
53-
this.fieldMd=fieldMd;
54-
Path arrayName=containingArray.getElement().getFullPath();
55-
Path full=fieldMd.getFullPath();
56-
if(full.prefix(arrayName.numSegments()).equals(arrayName)) {
57-
this.fullName=fieldMd.getFullPath().suffix(-arrayName.numSegments());
58-
} else {
59-
this.fullName=fieldMd.getFullPath();
60-
}
45+
public SimpleKeySpec(QueryFieldInfo finfo) {
46+
this.fieldMd=finfo.getFieldMd();
47+
this.fullName=finfo.getEntityRelativeFieldName();
6148
this.type=fieldMd.getType();
6249
}
6350

51+
6452
@Override
6553
public int compareKeys(Key k1,Key k2) {
6654
return compare( (SimpleKey)k1,(SimpleKey)k2);

‎crud/src/test/java/com/redhat/lightblue/mediator/CompositeFinderWithIndexingTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
Copyright 2013 Red Hat, Inc. and/or its affiliates.
3+
4+
This file is part of lightblue.
5+
6+
This program is free software: you can redistribute it and/or modify
7+
it under the terms of the GNU General Public License as published by
8+
the Free Software Foundation, either version 3 of the License, or
9+
(at your option) any later version.
10+
11+
This program is distributed in the hope that it will be useful,
12+
but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
GNU General Public License for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with this program. If not, see <http://www.gnu.org/licenses/>.
18+
*/
19+
package com.redhat.lightblue.mediator;
20+
21+
import org.junit.After;
22+
import org.junit.Before;
23+
24+
import com.redhat.lightblue.assoc.ep.Assemble;
25+
26+
public class CompositeFinderWithIndexingTest extends CompositeFinderTest {
27+
28+
@Before
29+
public void setup() {
30+
// Force use of mem index for all cases
31+
Assemble.MEM_INDEX_THRESHOLD=0;
32+
}
33+
@After
34+
public void restore() {
35+
// Force use of mem index for all cases
36+
Assemble.MEM_INDEX_THRESHOLD=16;
37+
}
38+
}

0 commit comments

Comments
 (0)