Skip to content

Commit eda8541

Browse files
pm-oscpm-osc
committed
#4860 allow ordering list properties via mixed index backend
Signed-off-by: pm-osc <[email protected]>
1 parent 1b92960 commit eda8541

File tree

13 files changed

+264
-12
lines changed

13 files changed

+264
-12
lines changed

‎docs/schema/index-management/index-performance.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,13 +285,17 @@ When using `order().by()` it is important to note that:
285285
results. All results will be retrieved and then sorted in-memory.
286286
For large result sets, this can be very expensive.
287287

288-
- Mixed indexes support ordering natively and efficiently. However,
288+
- Mixed indexes support ordering property keys with **SINGLE** cardinality
289+
natively and efficiently. However,
289290
the property key used in the order().by() method must have been
290291
previously added to the mixed indexed for native result ordering
291292
support. This is important in cases where the the order().by() key
292293
is different from the query keys. If the property key is not part of
293294
the index, then sorting requires loading all results into memory.
294295

296+
- Only Elasticsearch and Solr mixed index backends support ordering
297+
property keys with **LIST** cardinality natively and efficiently.
298+
295299
### Label Constraint
296300

297301
In many cases it is desirable to only index vertices or edges with a

‎janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java‎

Lines changed: 189 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@
154154
import static org.janusgraph.testutil.JanusGraphAssert.assertNoBackendHit;
155155
import static org.janusgraph.testutil.JanusGraphAssert.assertNotEmpty;
156156
import static org.janusgraph.testutil.JanusGraphAssert.assertTraversal;
157+
import static org.janusgraph.testutil.JanusGraphAssert.assertTraversalAndIndexUsage;
157158
import static org.junit.jupiter.api.Assertions.assertEquals;
158159
import static org.junit.jupiter.api.Assertions.assertFalse;
159160
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -186,16 +187,18 @@ public abstract class JanusGraphIndexTest extends JanusGraphBaseTest {
186187
public final boolean supportsGeoPoint;
187188
public final boolean supportsNumeric;
188189
public final boolean supportsText;
190+
public final boolean supportsOrderingListProperty;
189191

190192
public IndexFeatures indexFeatures;
191193

192194
private static final Logger log =
193195
LoggerFactory.getLogger(JanusGraphIndexTest.class);
194196

195-
protected JanusGraphIndexTest(boolean supportsGeoPoint, boolean supportsNumeric, boolean supportsText) {
197+
protected JanusGraphIndexTest(boolean supportsGeoPoint, boolean supportsNumeric, boolean supportsText, boolean supportsOrderingListProperty) {
196198
this.supportsGeoPoint = supportsGeoPoint;
197199
this.supportsNumeric = supportsNumeric;
198200
this.supportsText = supportsText;
201+
this.supportsOrderingListProperty = supportsOrderingListProperty;
199202
}
200203

201204
protected String[] getIndexBackends() {
@@ -4434,4 +4437,189 @@ public void testGetIndexInfo() throws DecoderException {
44344437
assertEquals(1, indexInfo.getCompositeIndexType().getInlineFieldKeys().length);
44354438
assertEquals("id", indexInfo.getCompositeIndexType().getInlineFieldKeys()[0]);
44364439
}
4440+
4441+
@Test
4442+
public void testOrderingListProperty() {
4443+
PropertyKey name1 = mgmt.makePropertyKey("name1").dataType(String.class).cardinality(Cardinality.SINGLE)
4444+
.make();
4445+
PropertyKey name2 = mgmt.makePropertyKey("name2").dataType(String.class).cardinality(Cardinality.LIST)
4446+
.make();
4447+
PropertyKey gender = mgmt.makePropertyKey("gender").dataType(String.class).cardinality(Cardinality.SINGLE).make();
4448+
PropertyKey age1 = mgmt.makePropertyKey("age1").dataType(Double.class).cardinality(Cardinality.SINGLE).make();
4449+
PropertyKey age2 = mgmt.makePropertyKey("age2").dataType(Double.class).cardinality(Cardinality.LIST).make();
4450+
PropertyKey birth1 = mgmt.makePropertyKey("birth1").dataType(Date.class).cardinality(Cardinality.SINGLE).make();
4451+
PropertyKey birth2 = mgmt.makePropertyKey("birth2").dataType(Date.class).cardinality(Cardinality.LIST).make();
4452+
4453+
mgmt.buildIndex("listPropertyOrdering", Vertex.class)
4454+
.addKey(name1, Mapping.STRING.asParameter())
4455+
.addKey(name2, Mapping.STRING.asParameter())
4456+
.addKey(gender, Mapping.STRING.asParameter())
4457+
.addKey(age1)
4458+
.addKey(age2)
4459+
.addKey(birth1)
4460+
.addKey(birth2)
4461+
.buildMixedIndex(INDEX);
4462+
finishSchema();
4463+
4464+
Vertex v1 = tx.addVertex("gender", "male", "name1", "value1", "name2", "value1", "age1", 1, "age2", 1, "birth1", "2010-01-01T00:00:00", "birth2", "2010-01-01T00:00:00");
4465+
Vertex v2 = tx.addVertex("gender", "female", "name1", "value2", "name2", "value2", "age1", 2, "age2", 2, "birth1", "2011-01-01T00:00:00", "birth2", "2011-01-01T00:00:00");
4466+
Vertex v3 = tx.addVertex("gender", "male", "name1", "value3", "name2", "value3", "name2", "value8", "age1", 3, "age2", 3, "birth1", "2012-01-01T00:00:00", "birth2", "2012-01-01T00:00:00");
4467+
Vertex v4 = tx.addVertex("gender", "female", "name1", "value4", "name2", "value4", "name2", "value7", "age1", 4, "age2", 4, "birth1", "2013-01-01T00:00:00", "birth2", "2013-01-01T00:00:00");
4468+
Vertex v5 = tx.addVertex("gender", "male", "name1", "value5", "name2", "value5", "age1", 5, "age2", 5, "birth1", "2014-01-01T00:00:00", "birth2", "2014-01-01T00:00:00");
4469+
Vertex v6 = tx.addVertex("gender", "female", "name1", "value6", "name2", "value6", "age1", 6, "age2", 6, "birth1", "2015-01-01T00:00:00", "birth2", "2015-01-01T00:00:00");
4470+
tx.commit();
4471+
4472+
clopen(option(FORCE_INDEX_USAGE), false);
4473+
4474+
final GraphTraversalSource g = tx.traversal();
4475+
org.apache.tinkerpop.gremlin.process.traversal.Order ORDER_DESC = org.apache.tinkerpop.gremlin.process.traversal.Order.desc;
4476+
4477+
// ordering without using index on SINGLE cardinality property
4478+
Supplier<GraphTraversal<?, Vertex>> tFullscanSingle = () -> g.V().order().by("name1");
4479+
assertTraversalAndIndexUsage(
4480+
Arrays.asList(
4481+
"query=[]",
4482+
"_fullscan=true"
4483+
),
4484+
tFullscanSingle, v1, v2, v3, v4, v5, v6
4485+
);
4486+
4487+
// ordering without using index on LIST cardinality property with multiple values
4488+
// throws IllegalStateException with message "Multiple properties exist for the provided key, use Vertex.properties(name2)"
4489+
Exception exception = assertThrows(IllegalStateException.class, () -> {
4490+
g.V().order().by("name2").toList();
4491+
});
4492+
assertTrue(exception.getMessage().contains("Multiple properties exist for the provided key, use Vertex.properties(name2)"));
4493+
4494+
///////////////////////////////////////////////////
4495+
// ordering SINGLE cardinality properties
4496+
// ordering SINGLE cardinality String property
4497+
Supplier<GraphTraversal<?, Vertex>> tSingleString = () -> g.V().has("name1").order().by("name1", ORDER_DESC);
4498+
assertTraversalAndIndexUsage(
4499+
Arrays.asList(
4500+
"_condition=(name1 <> null)",
4501+
"_query=[(name1 <> null)][DESC(name1)]:listPropertyOrdering"
4502+
),
4503+
tSingleString, v6, v5, v4, v3, v2, v1
4504+
);
4505+
4506+
// ordering SINGLE cardinality Double property
4507+
Supplier<GraphTraversal<?, Vertex>> tSingleDouble = () -> g.V().has("age1").order().by("age1", ORDER_DESC);
4508+
assertTraversalAndIndexUsage(
4509+
Arrays.asList(
4510+
"_condition=(age1 <> null)",
4511+
"_query=[(age1 <> null)][DESC(age1)]:listPropertyOrdering"
4512+
),
4513+
tSingleDouble, v6, v5, v4, v3, v2, v1
4514+
);
4515+
4516+
// ordering SINGLE cardinality Date property
4517+
Supplier<GraphTraversal<?, Vertex>> tSingleDate = () -> g.V().has("birth1").order().by("birth1", ORDER_DESC);
4518+
assertTraversalAndIndexUsage(
4519+
Arrays.asList(
4520+
"_condition=(birth1 <> null)",
4521+
"_query=[(birth1 <> null)][DESC(birth1)]:listPropertyOrdering"
4522+
),
4523+
tSingleDate, v6, v5, v4, v3, v2, v1
4524+
);
4525+
4526+
/////////////////////////////////////////////////
4527+
// ordering SINGLE cardinality properties with filtering
4528+
// ordering SINGLE cardinality String property
4529+
Supplier<GraphTraversal<?, Vertex>> tSingleStringFilter = () -> g.V().has("gender", "female").has("name1").order().by("name1", ORDER_DESC);
4530+
assertTraversalAndIndexUsage(
4531+
Arrays.asList(
4532+
"_condition=(gender = female AND name1 <> null)",
4533+
"_query=[(gender = female AND name1 <> null)][DESC(name1)]:listPropertyOrdering"
4534+
),
4535+
tSingleStringFilter, v6, v4, v2
4536+
);
4537+
4538+
// ordering SINGLE cardinality Double property
4539+
Supplier<GraphTraversal<?, Vertex>> tSingleDoubleFilter = () -> g.V().has("gender", "female").has("age1").order().by("age1", ORDER_DESC);
4540+
assertTraversalAndIndexUsage(
4541+
Arrays.asList(
4542+
"_condition=(gender = female AND age1 <> null)",
4543+
"_query=[(gender = female AND age1 <> null)][DESC(age1)]:listPropertyOrdering"
4544+
),
4545+
tSingleDoubleFilter, v6, v4, v2
4546+
);
4547+
4548+
// ordering SINGLE cardinality Date property
4549+
Supplier<GraphTraversal<?, Vertex>> tSingleDateFilter = () -> g.V().has("gender", "female").has("birth1").order().by("birth1", ORDER_DESC);
4550+
assertTraversalAndIndexUsage(
4551+
Arrays.asList(
4552+
"_condition=(gender = female AND birth1 <> null)",
4553+
"_query=[(gender = female AND birth1 <> null)][DESC(birth1)]:listPropertyOrdering"
4554+
),
4555+
tSingleDateFilter, v6, v4, v2
4556+
);
4557+
4558+
// certain mixed index backend specific part since Lucene does not support ordering for list properties
4559+
if (supportsOrderingListProperty) {
4560+
///////////////////////////////////////////////////
4561+
// ordering LIST cardinality properties
4562+
// ordering LIST cardinality String property
4563+
Supplier<GraphTraversal<?, Vertex>> tListString = () -> g.V().has("name2").order().by("name2", ORDER_DESC);
4564+
assertTraversalAndIndexUsage(
4565+
Arrays.asList(
4566+
"_condition=(name2 <> null)",
4567+
"_query=[(name2 <> null)][DESC(name2)]:listPropertyOrdering"
4568+
),
4569+
tListString, v3, v4, v6, v5, v2, v1
4570+
);
4571+
4572+
// ordering LIST cardinality Double property
4573+
Supplier<GraphTraversal<?, Vertex>> tListDouble = () -> g.V().has("age2").order().by("age2", ORDER_DESC);
4574+
assertTraversalAndIndexUsage(
4575+
Arrays.asList(
4576+
"_condition=(age2 <> null)",
4577+
"_query=[(age2 <> null)][DESC(age2)]:listPropertyOrdering"
4578+
),
4579+
tListDouble, v6, v5, v4, v3, v2, v1
4580+
);
4581+
4582+
// ordering LIST cardinality Date property
4583+
Supplier<GraphTraversal<?, Vertex>> tListDate = () -> g.V().has("birth2").order().by("birth2", ORDER_DESC);
4584+
assertTraversalAndIndexUsage(
4585+
Arrays.asList(
4586+
"_condition=(birth2 <> null)",
4587+
"_query=[(birth2 <> null)][DESC(birth2)]:listPropertyOrdering"
4588+
),
4589+
tListDate, v6, v5, v4, v3, v2, v1
4590+
);
4591+
4592+
/////////////////////////////////////////////////
4593+
// ordering LIST cardinality properties with filtering
4594+
// ordering LIST cardinality String property
4595+
Supplier<GraphTraversal<?, Vertex>> tListStringFilter = () -> g.V().has("gender", "female").has("name2").order().by("name2", ORDER_DESC);
4596+
assertTraversalAndIndexUsage(
4597+
Arrays.asList(
4598+
"_condition=(gender = female AND name2 <> null)",
4599+
"_query=[(gender = female AND name2 <> null)][DESC(name2)]:listPropertyOrdering"
4600+
),
4601+
tListStringFilter, v4, v6, v2
4602+
);
4603+
4604+
// ordering LIST cardinality Double property
4605+
Supplier<GraphTraversal<?, Vertex>> tListDoubleFilter = () -> g.V().has("gender", "female").has("age2").order().by("age2", ORDER_DESC);
4606+
assertTraversalAndIndexUsage(
4607+
Arrays.asList(
4608+
"_condition=(gender = female AND age2 <> null)",
4609+
"_query=[(gender = female AND age2 <> null)][DESC(age2)]:listPropertyOrdering"
4610+
),
4611+
tListDoubleFilter, v6, v4, v2
4612+
);
4613+
4614+
// ordering LIST cardinality Date property
4615+
Supplier<GraphTraversal<?, Vertex>> tListDateFilter = () -> g.V().has("gender", "female").has("birth2").order().by("birth2", ORDER_DESC);
4616+
assertTraversalAndIndexUsage(
4617+
Arrays.asList(
4618+
"_condition=(gender = female AND birth2 <> null)",
4619+
"_query=[(gender = female AND birth2 <> null)][DESC(birth2)]:listPropertyOrdering"
4620+
),
4621+
tListDateFilter, v6, v4, v2
4622+
);
4623+
}
4624+
}
44374625
}

‎janusgraph-backend-testutils/src/main/java/org/janusgraph/testutil/JanusGraphAssert.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.List;
3838
import java.util.Map;
3939
import java.util.Set;
40+
import java.util.function.Supplier;
4041
import java.util.stream.IntStream;
4142
import java.util.stream.Stream;
4243

@@ -79,6 +80,30 @@ public static<V extends Element> void assertNotEmpty(Object object) {
7980
assertFalse(isEmpty(object));
8081
}
8182

83+
public static <E extends Element> void assertTraversalAndIndexUsage(List<String> containsText,
84+
Supplier<GraphTraversal<?, E>> traversal,
85+
E... expectedElements) {
86+
87+
// assert expected results
88+
assertTraversal(traversal.get(), expectedElements);
89+
90+
// assert index usage
91+
String profileStr = traversal.get().profile().toList().toString();
92+
93+
// let's drop the postfix coming from dynamic field definition so comparison will not fail for Solr
94+
// all dynamic field definitions are in janusgraph-solr/src/test/resources/solr/core-template/schema.xml
95+
// multi-valued date and date postfix
96+
profileStr = profileStr.replace("_dts", "").replace("_dt", "");
97+
// multi-valued string and string postfix
98+
profileStr = profileStr.replace("_ss", "").replace("_s", "");
99+
// multi-valued double and double postfix
100+
profileStr = profileStr.replace("_ds", "").replace("_d", "");
101+
102+
for (String text : containsText) {
103+
assertTrue(profileStr.contains(text));
104+
}
105+
}
106+
82107
public static<E extends Element> void assertTraversal(GraphTraversal<?, E> req, E... expectedElements) {
83108
for (final E expectedElement : expectedElements) {
84109
assertEquals(expectedElement, req.next());

‎janusgraph-core/src/main/java/org/janusgraph/diskstorage/indexing/IndexFeatures.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,13 @@ public class IndexFeatures {
3939
private final boolean supportsGeoContains;
4040
private final boolean supportsGeoExists;
4141
private final boolean supportsNotQueryNormalForm;
42+
private final boolean supportsOrderingListProperty;
4243
private final Set<Cardinality> supportedCardinalities;
4344

4445
public IndexFeatures(boolean supportsDocumentTTL, Mapping defaultMap, Set<Mapping> supportedMap,
4546
String wildcardField, Set<Cardinality> supportedCardinalities, boolean supportsNanoseconds,
4647
boolean supportCustomAnalyzer, boolean supportsGeoContains, boolean supportGeoExists,
47-
boolean supportsNotQueryNormalForm) {
48+
boolean supportsNotQueryNormalForm, boolean supportsOrderingListProperty) {
4849

4950
Preconditions.checkArgument(defaultMap!=null && defaultMap!=Mapping.DEFAULT);
5051
Preconditions.checkArgument(supportedMap!=null && !supportedMap.isEmpty()
@@ -59,6 +60,7 @@ public IndexFeatures(boolean supportsDocumentTTL, Mapping defaultMap, Set<Mappin
5960
this.supportsGeoContains = supportsGeoContains;
6061
this.supportsGeoExists = supportGeoExists;
6162
this.supportsNotQueryNormalForm = supportsNotQueryNormalForm;
63+
this.supportsOrderingListProperty = supportsOrderingListProperty;
6264
}
6365

6466
public boolean supportsDocumentTTL() {
@@ -101,6 +103,10 @@ public boolean supportNotQueryNormalForm() {
101103
return supportsNotQueryNormalForm;
102104
}
103105

106+
public boolean supportsOrderingListProperty() {
107+
return supportsOrderingListProperty;
108+
}
109+
104110
public static class Builder {
105111

106112
private boolean supportsDocumentTTL = false;
@@ -113,6 +119,7 @@ public static class Builder {
113119
private boolean supportsGeoContains;
114120
private boolean supportsGeoExists;
115121
private boolean supportNotQueryNormalForm;
122+
private boolean supportsOrderingListProperty;
116123

117124
public Builder supportsDocumentTTL() {
118125
supportsDocumentTTL=true;
@@ -164,10 +171,15 @@ public Builder supportNotQueryNormalForm() {
164171
return this;
165172
}
166173

174+
public Builder supportsOrderingListProperty() {
175+
this.supportsOrderingListProperty = true;
176+
return this;
177+
}
178+
167179
public IndexFeatures build() {
168180
return new IndexFeatures(supportsDocumentTTL, defaultStringMapping, Collections.unmodifiableSet(new HashSet<>(supportedMappings)),
169181
wildcardField, Collections.unmodifiableSet(new HashSet<>(supportedCardinalities)), supportsNanoseconds, supportsCustomAnalyzer,
170-
supportsGeoContains, supportsGeoExists, supportNotQueryNormalForm);
182+
supportsGeoContains, supportsGeoExists, supportNotQueryNormalForm, supportsOrderingListProperty);
171183
}
172184
}
173185
}

‎janusgraph-core/src/main/java/org/janusgraph/graphdb/database/IndexSerializer.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,19 @@ public boolean supportsExistsQuery(final MixedIndexType index, final ParameterIn
181181
return true;
182182
}
183183

184+
public boolean allMixedIndexBackendSupportsOrderingListProperty() {
185+
boolean allMixedIndexBackendSupportsOrderingListProperty = false;
186+
for (Map.Entry<String, ? extends IndexInformation> entry : mixedIndexes.entrySet()) {
187+
// if any of the mixed index backends does not support ordering list property, let's return false
188+
if (!entry.getValue().getFeatures().supportsOrderingListProperty()) {
189+
return false;
190+
}
191+
allMixedIndexBackendSupportsOrderingListProperty = true;
192+
}
193+
194+
return allMixedIndexBackendSupportsOrderingListProperty;
195+
}
196+
184197
public IndexFeatures features(final MixedIndexType index) {
185198
return getMixedIndex(index).getFeatures();
186199
}

‎janusgraph-core/src/main/java/org/janusgraph/graphdb/query/graph/GraphCentricQueryBuilder.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
package org.janusgraph.graphdb.query.graph;
1616

1717
import com.google.common.base.Preconditions;
18-
import org.janusgraph.core.Cardinality;
18+
//import org.janusgraph.core.Cardinality;
1919
import org.janusgraph.core.JanusGraphEdge;
2020
import org.janusgraph.core.JanusGraphElement;
2121
import org.janusgraph.core.JanusGraphQuery;
@@ -201,8 +201,8 @@ public GraphCentricQueryBuilder orderBy(String keyName, org.apache.tinkerpop.gr
201201
Preconditions.checkArgument(key!=null && order!=null,"Need to specify and key and an order");
202202
Preconditions.checkArgument(Comparable.class.isAssignableFrom(key.dataType()),
203203
"Can only order on keys with comparable data type. [%s] has datatype [%s]", key.name(), key.dataType());
204-
Preconditions.checkArgument(key.cardinality()== Cardinality.SINGLE,
205-
"Ordering is undefined on multi-valued key [%s]", key.name());
204+
//Preconditions.checkArgument(key.cardinality()== Cardinality.SINGLE,
205+
// "Ordering is undefined on multi-valued key [%s]", key.name());
206206
Preconditions.checkArgument(!orders.containsKey(key), "orders [%s] already contains key [%s]", orders, key);
207207
orders.add(key, Order.convert(order));
208208
return this;

0 commit comments

Comments
 (0)