Skip to content

Commit 4d78aa9

Browse files
aseovicaseovic
committed
Enh 36422086 - [35884452->24.09] Improve filter re-ordering algorithm for composite filters (re-enable)
[git-p4: depot-paths = "//dev/coherence-ce/main/": change = 107707]
1 parent 2136f45 commit 4d78aa9

File tree

7 files changed

+40
-53
lines changed

7 files changed

+40
-53
lines changed

‎prj/coherence-core/src/main/java/com/tangosol/util/filter/AllFilter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
33
*
44
* Licensed under the Universal Permissive License v 1.0 as shown at
55
* https://oss.oracle.com/licenses/upl.
@@ -137,7 +137,7 @@ protected Set<Filter<?>> simplifyFilters()
137137
protected Filter applyIndex(Map mapIndexes, Set setKeys,
138138
QueryContext ctx, QueryRecord.PartialResult.TraceStep step)
139139
{
140-
//optimizeFilterOrder(mapIndexes, setKeys);
140+
optimizeFilterOrder(mapIndexes, setKeys);
141141

142142
Filter[] aFilter = m_aFilter;
143143
int cFilters = aFilter.length;

‎prj/coherence-core/src/main/java/com/tangosol/util/filter/AnyFilter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
33
*
44
* Licensed under the Universal Permissive License v 1.0 as shown at
55
* https://oss.oracle.com/licenses/upl.
@@ -149,7 +149,7 @@ protected Set<Filter<?>> simplifyFilters()
149149
protected Filter applyIndex(Map mapIndexes, Set setKeys, QueryContext ctx,
150150
QueryRecord.PartialResult.TraceStep step)
151151
{
152-
//optimizeFilterOrder(mapIndexes, setKeys);
152+
optimizeFilterOrder(mapIndexes, setKeys);
153153

154154
Filter[] aFilter = m_aFilter;
155155
int cFilters = aFilter.length;

‎prj/coherence-core/src/main/java/com/tangosol/util/filter/ArrayFilter.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
33
*
44
* Licensed under the Universal Permissive License v 1.0 as shown at
55
* https://oss.oracle.com/licenses/upl.
@@ -21,7 +21,6 @@
2121
import com.tangosol.util.QueryContext;
2222

2323
import java.util.Arrays;
24-
import java.util.LinkedHashSet;
2524
import java.util.Map;
2625
import java.util.Set;
2726

@@ -254,7 +253,7 @@ protected void optimizeFilterOrder(Map mapIndexes, Set setKeys)
254253
return;
255254
}
256255

257-
Set<Filter> setFilter = new LinkedHashSet<>(Arrays.asList(m_aFilter));
256+
Set<Filter<?>> setFilter = simplifyFilters();
258257
int cFilters = setFilter.size();
259258
WeightedFilter[] aWeighted = new WeightedFilter[cFilters];
260259
Filter<?>[] aFilter = new Filter[cFilters];

‎prj/test/functional/filter/src/main/java/filter/AbstractFilterTests.java

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
33
*
44
* Licensed under the Universal Permissive License v 1.0 as shown at
55
* https://oss.oracle.com/licenses/upl.
@@ -76,8 +76,6 @@
7676
import java.io.FileInputStream;
7777
import java.io.FileOutputStream;
7878
import java.io.IOException;
79-
import java.nio.file.Files;
80-
import java.nio.file.Path;
8179
import java.util.Collections;
8280
import org.junit.Test;
8381

@@ -297,7 +295,7 @@ protected void doTest(NamedCache cacheTest, int cItems,
297295
}
298296

299297
// 2) fill the cache and just execute all the queries
300-
if (!loadData(cacheTest, "people-" + cItems + ".bin"))
298+
if (!loadData(cacheTest, "people-" + cItems + ".bin", fDebug))
301299
{
302300
Person.fillRandom(cacheTest, cItems);
303301
saveData(cacheTest, "people-" + cItems + ".bin");
@@ -388,9 +386,9 @@ protected void doTest(NamedCache cacheTest, int cItems,
388386
}
389387
}
390388

391-
// the default is 4, which will test 64 random combinations of filters
389+
// the default is 10, which will test 1,000 random combinations of filters
392390
// specify 0 to run all possible combinations (all 64 thousand of them...)
393-
int cIterations = Config.getInteger("test.filters.iterations", 4);
391+
int cIterations = Config.getInteger("test.filters.iterations", 10);
394392
boolean fRandom = true;
395393

396394
if (cIterations == 0)
@@ -583,15 +581,18 @@ private void saveData(NamedCache cache, String sFileName)
583581
}
584582
}
585583

586-
private boolean loadData(NamedCache cache, String sFileName)
584+
private boolean loadData(NamedCache cache, String sFileName, boolean fDebug)
587585
{
588586
try (FileInputStream inFile = new FileInputStream(sFileName))
589587
{
590588
Map data = new HashMap(cache);
591589
DataInput in = new DataInputStream(inFile);
592590
ExternalizableHelper.readMap(in, data, getClass().getClassLoader());
593591
cache.putAll(data);
594-
System.out.printf("\nLoaded %d cache entries from %s", cache.size(), sFileName);
592+
if (fDebug)
593+
{
594+
System.out.printf("Loaded %d cache entries from %s\n", cache.size(), sFileName);
595+
}
595596
return true;
596597
}
597598
catch (IOException e)

‎prj/test/functional/filter/src/main/java/filter/QueryRecordReporterTests.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates.
33
*
44
* Licensed under the Universal Permissive License v 1.0 as shown at
55
* https://oss.oracle.com/licenses/upl.
@@ -10,15 +10,18 @@
1010

1111
import com.oracle.bedrock.testsupport.deferred.Eventually;
1212
import com.oracle.bedrock.runtime.coherence.CoherenceCacheServer;
13+
1314
import com.tangosol.net.CacheFactory;
1415
import com.tangosol.net.DistributedCacheService;
1516
import com.tangosol.net.NamedCache;
1617
import com.tangosol.net.partition.PartitionSet;
18+
1719
import com.tangosol.util.Filter;
1820
import com.tangosol.util.Resources;
1921
import com.tangosol.util.SimpleQueryRecord;
2022
import com.tangosol.util.SimpleQueryRecordReporter;
2123
import com.tangosol.util.aggregator.QueryRecorder;
24+
2225
import com.tangosol.util.filter.AllFilter;
2326
import com.tangosol.util.filter.AlwaysFilter;
2427
import com.tangosol.util.filter.AndFilter;
@@ -27,12 +30,14 @@
2730
import com.tangosol.util.filter.LessEqualsFilter;
2831
import com.tangosol.util.filter.LikeFilter;
2932
import com.tangosol.util.filter.PartitionedFilter;
33+
3034
import com.oracle.coherence.testing.AbstractFunctionalTest;
3135

3236
import data.pof.Address;
3337

3438
import filter.nestinglevel1.nestinglevel2.nestinglevel3.IntegerToStringPersonKeyExtractor;
3539
import filter.nestinglevel1.nestinglevel2.nestinglevel3.StringToIntegerPersonAddressZipExtractor;
40+
3641
import org.junit.AfterClass;
3742
import org.junit.Before;
3843
import org.junit.BeforeClass;
@@ -43,7 +48,9 @@
4348
import java.io.ObjectInputStream;
4449
import java.io.ObjectOutputStream;
4550
import java.io.Serializable;
51+
4652
import java.net.URL;
53+
4754
import java.util.Calendar;
4855
import java.util.NoSuchElementException;
4956
import java.util.Scanner;
@@ -145,7 +152,7 @@ public void testReporterDoubleIndex() throws Exception
145152

146153
out(" * * * * * * * Double-Index Query * * * * * * * * * * * * * * (testReporterDoubleIndex)");
147154
testReporter(cache, filterAll, QueryRecorder.RecordType.EXPLAIN, "explain-DoubleIndex.ptn");
148-
//testReporter(cache, filterAll, QueryRecorder.RecordType.TRACE, "trace-DoubleIndex.ptn");
155+
testReporter(cache, filterAll, QueryRecorder.RecordType.TRACE, "trace-DoubleIndex.ptn");
149156

150157
cache.removeIndex(IntegerToStringPersonKeyExtractor.INSTANCE);
151158
cache.removeIndex(StringToIntegerPersonAddressZipExtractor.INSTANCE);

‎prj/test/functional/filter/src/main/resources/explain-DoubleIndex.ptn

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,20 @@
22
Explain Plan
33
Filter Name Index Cost
44
======================================================================================
5-
AllFilter\s+\| ---- \| 0
5+
AllFilter \| ---- \| 0
66
LikeFilter\(filter\.nestinglevel1\.nestinglevel2\.nestinglev\.\.\. \(0\) \| 0 \| 11
7-
AndFilter\s+\| ---- \| 0
8-
BetweenFilter\(filter\.nestinglevel1\.nestinglevel2\.nesti\.\.\. \(1\) \| 1 \| (66|67)
9-
EqualsFilter\(\.getFirstName\(\), Hector\)\s+\| 2 \| 100000
10-
LessEqualsFilter\(\.getAge\(\), 98\)\s+\| 3 \| 100000
7+
BetweenFilter\(filter\.nestinglevel1\.nestinglevel2\.nesting\.\.\. \(1\) \| 1 \| (66|67)
8+
LessEqualsFilter\(\.getAge\(\), 98\) \| 2 \| 100000
9+
EqualsFilter\(\.getFirstName\(\), Hector\) \| 3 \| 100000
1110

1211

1312
Index Lookups
1413
Index Description Extractor Ordered
1514
======================================================================================
1615
0 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B,\.\.\. \(2\)\s+\| filter\.nestinglevel1\.nes\.\.\. \(3\) \| true
1716
1 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B, Size=3\s+\| filter\.nestinglevel1\.nes\.\.\. \(4\) \| true
18-
2 \| No index found \| \.getFirstName\(\) \| false
19-
3 \| No index found \| \.getAge\(\) \| false
17+
2 \| No index found \| \.getAge\(\) \| false
18+
3 \| No index found \| \.getFirstName\(\) \| false
2019

2120

2221
Complete filter and index descriptions

‎prj/test/functional/filter/src/main/resources/trace-DoubleIndex.ptn

Lines changed: 10 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,23 @@
22
Trace
33
Filter Name Index Effectiveness Duration
44
======================================================================================
5-
AllFilter[\s]*\| ---- \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
6-
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
7-
AndFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
8-
BetweenFilter\(filter\.nestingle\.\.\. \(1\) \| 1 \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
9-
EqualsFilter\(\.getFirstName\(\), Hector\) \| 2 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
10-
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| 3 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
11-
AllFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
12-
EqualsFilter\(\.getFirstName\(\), Hector\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
13-
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
14-
15-
PartitionSet\{[0-9]+\.\.[0-9]+\}
16-
17-
18-
Trace
19-
Filter Name Index Effectiveness Duration
20-
======================================================================================
21-
AllFilter[\s]*\| ---- \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
22-
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 50\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
23-
AndFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
24-
BetweenFilter\(filter\.nestingle\.\.\. \(1\) \| 1 \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
25-
EqualsFilter\(\.getFirstName\(\), Hector\) \| 2 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
26-
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| 3 \| [0-9]+\|[0-9]+\(0%\)[\s]*\| [0-9]+[\s]*
27-
AllFilter[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
28-
EqualsFilter\(\.getFirstName\(\), Hector\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
29-
LessEqualsFilter\(\.getAge\(\), 98\)[\s]*\| ---- \| [0-9]+\|[0-9]+\([0-9]+%\)[\s]*\| [0-9]+[\s]*
30-
31-
PartitionSet\{[0-9]+\.\.[0-9]+\}
5+
AllFilter \| ---- \| 100\|7\(93%\) \| [0-9]+[\s]*
6+
LikeFilter\(filter\.nestinglevel1\.\.\.\. \(0\) \| 0 \| 100\|11\(89%\) \| [0-9]+[\s]*
7+
BetweenFilter\(filter\.nestingleve\.\.\. \(1\) \| 1 \| 11\|7\(36%\) \| [0-9]+[\s]*
8+
LessEqualsFilter\(\.getAge\(\), 98\) \| 2 \| 7\|7\(0%\) \| [0-9]+[\s]*
9+
EqualsFilter\(\.getFirstName\(\), Hector\) \| 3 \| 7\|7\(0%\) \| [0-9]+[\s]*
10+
AllFilter \| ---- \| 7\|1\(85%\) \| [0-9]+[\s]*
11+
LessEqualsFilter\(\.getAge\(\), 98\) \| ---- \| 7\|7\(0%\) \| [0-9]+[\s]*
12+
EqualsFilter\(\.getFirstName\(\), Hector\) \| ---- \| 7\|1\(85%\) \| [0-9]+[\s]*
3213

3314

3415
Index Lookups
3516
Index Description Extractor Ordered
3617
======================================================================================
3718
0 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B,\.\.\. \(2\)\s+\| filter\.nestinglevel1\.nes\.\.\. \(3\) \| true
3819
1 \| Partitioned: Footprint=[0-9]+[\.[0-9]+]*[K\|M]*B, Size=3 \| filter\.nestinglevel1\.nes\.\.\. \(4\) \| true
39-
2 \| No index found \| \.getFirstName\(\) \| false
40-
3 \| No index found \| \.getAge\(\) \| false
20+
2 \| No index found \| \.getAge\(\) \| false
21+
3 \| No index found \| \.getFirstName\(\) \| false
4122

4223

4324
Complete filter and index descriptions

0 commit comments

Comments
 (0)