Skip to content

Commit 86f5d98

Browse files
donghun-chodonghun-cho
committed
[#12936] Reduce TraceIndex filters using min/max values
1 parent 3b6112e commit 86f5d98

File tree

4 files changed

+102
-53
lines changed

4 files changed

+102
-53
lines changed

‎commons-server/src/main/java/com/navercorp/pinpoint/common/server/scatter/TraceIndexFilterBuilder.java‎

Lines changed: 71 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.navercorp.pinpoint.common.hbase.HbaseColumnFamily;
44
import com.navercorp.pinpoint.common.hbase.HbaseTables;
5-
import com.navercorp.pinpoint.common.server.util.pair.LongPair;
65
import org.apache.hadoop.hbase.CompareOperator;
76
import org.apache.hadoop.hbase.filter.BinaryComponentComparator;
87
import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
@@ -17,88 +16,117 @@
1716
import java.util.List;
1817

1918
public class TraceIndexFilterBuilder {
19+
private static final long DEFAULT_ELAPSED_MIN = 0L;
20+
private static final long DEFAULT_ELAPSED_MAX = Integer.MAX_VALUE;
21+
private static final byte MIN_ELAPSED_BYTE = TraceIndexRowKeyUtils.toElapsedByte(DEFAULT_ELAPSED_MIN);
22+
2023
private static final HbaseColumnFamily INDEX = HbaseTables.TRACE_INDEX;
2124
private static final HbaseColumnFamily META = HbaseTables.TRACE_INDEX_META;
2225
private static final byte[] META_QUALIFIER_RPC = HbaseTables.TRACE_INDEX_META_QUALIFIER_RPC;
2326
private static final int ROW_FILTER_OFFSET = TraceIndexRowKeyUtils.SALTED_ROW_TIMESTAMP_OFFSET + 8 + 8;
2427

25-
private LongPair elapsedMinMax;
28+
private long elapsedMin = DEFAULT_ELAPSED_MIN;
29+
private long elapsedMax = DEFAULT_ELAPSED_MAX;
2630
private Boolean success;
2731
private String agentId;
2832
private String rpcRegex;
2933

3034
public TraceIndexFilterBuilder() {
3135
}
3236

33-
public Filter build(boolean enableAdditionalRowFilters, boolean enableValueFilter) {
37+
public FilterList build(boolean enableAdditionalRowFilters, boolean enableValueFilter) {
3438
List<Filter> filters = new ArrayList<>();
3539
// row filters
3640
if (enableAdditionalRowFilters) {
37-
if (success != null) {
38-
filters.add(createSuccessRowFilter(success));
39-
}
40-
if (agentId != null) {
41-
filters.add(createAgentIdHashRowFilter(agentId));
42-
}
43-
if (elapsedMinMax != null) {
44-
filters.add(createElapsedByteRowFilter(elapsedMinMax));
45-
}
41+
filters.addAll(createSuccessRowFilter(success));
42+
filters.addAll(createAgentIdHashRowFilter(agentId));
43+
filters.addAll(createElapsedByteRowFilter(elapsedMin, elapsedMax));
4644
}
4745

4846
// value filters
4947
if (enableValueFilter) {
50-
if (elapsedMinMax != null) {
51-
filters.add(createElapsedValueFilter(elapsedMinMax));
52-
}
53-
if (rpcRegex != null) {
54-
filters.add(createRpcRegexValueFilter(rpcRegex));
55-
}
48+
filters.addAll(createElapsedValueFilter(elapsedMin, elapsedMax));
49+
filters.addAll(createRpcRegexValueFilter(rpcRegex));
5650
}
5751
return new FilterList(filters);
5852
}
5953

60-
private Filter createElapsedByteRowFilter(LongPair elapsedMinMax) {
61-
List<Byte> allowedBytes = TraceIndexRowKeyUtils.toElapsedByteList(elapsedMinMax);
62-
if (allowedBytes.size() == 1) {
63-
return new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(new byte[]{allowedBytes.get(0)}, ROW_FILTER_OFFSET));
64-
} else {
65-
return new FilterList(
66-
new RowFilter(CompareOperator.GREATER_OR_EQUAL, new BinaryComponentComparator(new byte[]{allowedBytes.get(0)}, ROW_FILTER_OFFSET)),
67-
new RowFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComponentComparator(new byte[]{allowedBytes.get(allowedBytes.size() - 1)}, ROW_FILTER_OFFSET))
68-
);
54+
private List<Filter> createElapsedByteRowFilter(long elapsedMin, long elapsedMax) {
55+
if (elapsedMin <= DEFAULT_ELAPSED_MIN && elapsedMax >= DEFAULT_ELAPSED_MAX) {
56+
return List.of();
57+
}
58+
Byte minByte = null;
59+
Byte maxByte = null;
60+
if (elapsedMin > DEFAULT_ELAPSED_MIN) {
61+
minByte = TraceIndexRowKeyUtils.toElapsedByte(elapsedMin);
62+
}
63+
if (elapsedMax < DEFAULT_ELAPSED_MAX) {
64+
maxByte = TraceIndexRowKeyUtils.toElapsedByte(elapsedMax);
65+
}
66+
67+
// if min and max are the same, use EQUAL filter
68+
if (minByte != null && minByte.equals(maxByte)) {
69+
return List.of(new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(new byte[]{minByte}, ROW_FILTER_OFFSET)));
70+
}
71+
72+
// otherwise, use range filters
73+
List<Filter> filters = new ArrayList<>(2);
74+
if (minByte != null && minByte > MIN_ELAPSED_BYTE) {
75+
filters.add(new RowFilter(CompareOperator.GREATER_OR_EQUAL, new BinaryComponentComparator(new byte[]{minByte}, ROW_FILTER_OFFSET)));
76+
}
77+
if (maxByte != null) {
78+
filters.add(new RowFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComponentComparator(new byte[]{maxByte}, ROW_FILTER_OFFSET)));
6979
}
80+
return filters;
7081
}
7182

72-
private Filter createSuccessRowFilter(boolean success) {
83+
private List<Filter> createSuccessRowFilter(Boolean success) {
84+
if (success == null) {
85+
return List.of();
86+
}
7387
// 0 for success
7488
BinaryComponentComparator comparator = new BinaryComponentComparator(new byte[]{0}, ROW_FILTER_OFFSET + 1); // elapsed(1)
7589
if (success) {
76-
return new RowFilter(CompareOperator.EQUAL, comparator);
90+
return List.of(new RowFilter(CompareOperator.EQUAL, comparator));
7791
} else {
78-
return new RowFilter(CompareOperator.NOT_EQUAL, comparator);
92+
return List.of(new RowFilter(CompareOperator.NOT_EQUAL, comparator));
7993
}
8094
}
8195

82-
private Filter createAgentIdHashRowFilter(String agentId) {
96+
private List<Filter> createAgentIdHashRowFilter(String agentId) {
97+
if (agentId == null) {
98+
return List.of();
99+
}
83100
short agentIdHash = TraceIndexRowKeyUtils.toAgentIdHash(agentId);
84-
return new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(Bytes.toBytes(agentIdHash), ROW_FILTER_OFFSET + 2)); // elapsed(1) + error(1)
101+
return List.of(new RowFilter(CompareOperator.EQUAL, new BinaryComponentComparator(Bytes.toBytes(agentIdHash), ROW_FILTER_OFFSET + 2))); // elapsed(1) + error(1)
102+
}
103+
104+
private List<Filter> createElapsedValueFilter(long elapsedMin, long elapsedMax) {
105+
List<Filter> filters = new ArrayList<>();
106+
if (elapsedMin > DEFAULT_ELAPSED_MIN) {
107+
int yLow = (int) elapsedMin;
108+
filters.add(new SingleColumnValueFilter(INDEX.getName(), INDEX.getName(), CompareOperator.GREATER_OR_EQUAL, new BinaryPrefixComparator(Bytes.toBytes(yLow))));
109+
}
110+
if (elapsedMax < DEFAULT_ELAPSED_MAX) {
111+
int yHigh = (int) elapsedMax;
112+
filters.add(new SingleColumnValueFilter(INDEX.getName(), INDEX.getName(), CompareOperator.LESS_OR_EQUAL, new BinaryPrefixComparator(Bytes.toBytes(yHigh))));
113+
}
114+
return filters;
85115
}
86116

87-
private Filter createElapsedValueFilter(LongPair elapsedMinMax) {
88-
int yLow = (int) elapsedMinMax.first();
89-
int yHigh = (int) elapsedMinMax.second();
90-
return new FilterList(
91-
new SingleColumnValueFilter(INDEX.getName(), INDEX.getName(), CompareOperator.GREATER_OR_EQUAL, new BinaryPrefixComparator(Bytes.toBytes(yLow))),
92-
new SingleColumnValueFilter(INDEX.getName(), INDEX.getName(), CompareOperator.LESS_OR_EQUAL, new BinaryPrefixComparator(Bytes.toBytes(yHigh)))
93-
);
117+
private List<Filter> createRpcRegexValueFilter(String rpcRegex) {
118+
if (rpcRegex == null) {
119+
return List.of();
120+
}
121+
return List.of(new SingleColumnValueFilter(META.getName(), META_QUALIFIER_RPC, CompareOperator.EQUAL, new RegexStringComparator(rpcRegex)));
94122
}
95123

96-
private Filter createRpcRegexValueFilter(String rpcRegex) {
97-
return new SingleColumnValueFilter(META.getName(), META_QUALIFIER_RPC, CompareOperator.EQUAL, new RegexStringComparator(rpcRegex));
124+
public void setElapsedMin(long elapsedMin) {
125+
this.elapsedMin = elapsedMin;
98126
}
99127

100-
public void setElapsedMinMax(LongPair elapsedMinMax) {
101-
this.elapsedMinMax = elapsedMinMax;
128+
public void setElapsedMax(long elapsedMax) {
129+
this.elapsedMax = elapsedMax;
102130
}
103131

104132
public void setSuccess(Boolean success) {

‎commons-server/src/main/java/com/navercorp/pinpoint/common/server/scatter/TraceIndexRowKeyUtils.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,6 @@ public static byte toElapsedByte(long elapsed) {
7070
return fuzzyRowKeyFactory.getKey(elapsed);
7171
}
7272

73-
public static List<Byte> toElapsedByteList(LongPair elapsedMinMax) {
74-
return fuzzyRowKeyFactory.getRangeKey(elapsedMinMax.second(), elapsedMinMax.first());
75-
}
76-
7773
public static byte toErrorByte(int errorCode) {
7874
return errorCode == 0 ? (byte) 0 : (byte) 1;
7975
} // zero or non-zero

‎commons-server/src/test/java/com/navercorp/pinpoint/common/server/scatter/TraceIndexFilterBuilderTest.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package com.navercorp.pinpoint.common.server.scatter;
22

33
import com.navercorp.pinpoint.common.hbase.HbaseTables;
4-
import com.navercorp.pinpoint.common.server.util.pair.LongPair;
54
import com.navercorp.pinpoint.common.trace.ServiceType;
65
import org.apache.hadoop.hbase.KeyValue;
76
import org.apache.hadoop.hbase.filter.Filter;
7+
import org.apache.hadoop.hbase.filter.FilterList;
88
import org.apache.hadoop.hbase.util.Bytes;
99
import org.assertj.core.api.Assertions;
1010
import org.junit.jupiter.api.Test;
@@ -47,7 +47,8 @@ public void agentIdHashRowFilterTest() throws IOException {
4747
@Test
4848
public void elapsedByteRowFilterTest() throws IOException {
4949
TraceIndexFilterBuilder builder = new TraceIndexFilterBuilder();
50-
builder.setElapsedMinMax(new LongPair(250, 350));
50+
builder.setElapsedMin(250L);
51+
builder.setElapsedMax(350L);
5152
Filter filter = builder.build(true, false);
5253

5354
// elapsed time with different byte slots
@@ -64,10 +65,20 @@ public void elapsedByteRowFilterTest() throws IOException {
6465
Assertions.assertThat(isFilterOutRow(filter, createIndexKeyValue(rowKey3, value))).isEqualTo(true);
6566
}
6667

68+
@Test
69+
public void elapsedByteRowFilterMinTest() throws IOException {
70+
TraceIndexFilterBuilder builder = new TraceIndexFilterBuilder();
71+
builder.setElapsedMin(0L);
72+
FilterList filter = builder.build(true, false);
73+
74+
Assertions.assertThat(filter.getFilters()).hasSize(0);
75+
}
76+
6777
@Test
6878
public void elapsedTimeValueFilterTest() throws IOException {
6979
TraceIndexFilterBuilder builder = new TraceIndexFilterBuilder();
70-
builder.setElapsedMinMax(new LongPair(50, 200));
80+
builder.setElapsedMin(50L);
81+
builder.setElapsedMax(200L);
7182
Filter filter = builder.build(false, true);
7283

7384
byte[] rowKey = createTestRowKey();
@@ -80,6 +91,16 @@ public void elapsedTimeValueFilterTest() throws IOException {
8091
Assertions.assertThat(getFilterReturnCode(filter, createIndexKeyValue(rowKey, value3))).isEqualTo(Filter.ReturnCode.NEXT_ROW);
8192
}
8293

94+
@Test
95+
public void elapsedTimeValueFilterMinMaxTest() throws IOException {
96+
TraceIndexFilterBuilder builder = new TraceIndexFilterBuilder();
97+
builder.setElapsedMin(0L);
98+
builder.setElapsedMax(Long.MAX_VALUE); // integer max is enough
99+
FilterList filter = builder.build(false, true);
100+
101+
Assertions.assertThat(filter.getFilters()).hasSize(0);
102+
}
103+
83104
@Test
84105
public void rpcRegexValueFilterTest() throws IOException {
85106
TraceIndexFilterBuilder builder = new TraceIndexFilterBuilder();

‎web/src/main/java/com/navercorp/pinpoint/web/scatter/dao/hbase/HbaseTraceIndexDao.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import com.navercorp.pinpoint.common.hbase.wd.RowKeyDistributor;
2828
import com.navercorp.pinpoint.common.server.scatter.TraceIndexFilterBuilder;
2929
import com.navercorp.pinpoint.common.server.scatter.TraceIndexRowKeyUtils;
30-
import com.navercorp.pinpoint.common.server.util.pair.LongPair;
3130
import com.navercorp.pinpoint.common.timeseries.time.Range;
3231
import com.navercorp.pinpoint.web.config.ScatterChartProperties;
3332
import com.navercorp.pinpoint.web.scatter.DragArea;
@@ -42,6 +41,7 @@
4241
import com.navercorp.pinpoint.web.vo.LimitedScanResult;
4342
import org.apache.hadoop.hbase.TableName;
4443
import org.apache.hadoop.hbase.client.Scan;
44+
import org.apache.hadoop.hbase.filter.FilterList;
4545
import org.apache.logging.log4j.LogManager;
4646
import org.apache.logging.log4j.Logger;
4747
import org.springframework.beans.factory.annotation.Qualifier;
@@ -173,13 +173,17 @@ private Scan createScan(int serviceUid, String applicationName, int serviceTypeC
173173

174174
private void setHbaseFilter(Scan scan, DragAreaQuery dragAreaQuery, String rpcRegex) {
175175
TraceIndexFilterBuilder filterBuilder = new TraceIndexFilterBuilder();
176-
filterBuilder.setElapsedMinMax(new LongPair(dragAreaQuery.getDragArea().getYLow(), dragAreaQuery.getDragArea().getYHigh()));
176+
filterBuilder.setElapsedMin(dragAreaQuery.getDragArea().getYLow());
177+
filterBuilder.setElapsedMax(dragAreaQuery.getDragArea().getYHigh());
177178
filterBuilder.setAgentId(dragAreaQuery.getAgentId());
178179
filterBuilder.setRpcRegex(rpcRegex);
179180
if (dragAreaQuery.getDotStatus() != null) {
180181
filterBuilder.setSuccess(dragAreaQuery.getDotStatus() == Dot.Status.SUCCESS);
181182
}
182-
scan.setFilter(filterBuilder.build(scatterChartProperties.isEnableHbaseRowFilter(), scatterChartProperties.isEnableHbaseValueFilter()));
183+
FilterList filter = filterBuilder.build(scatterChartProperties.isEnableHbaseRowFilter(), scatterChartProperties.isEnableHbaseValueFilter());
184+
if (!filter.getFilters().isEmpty()) {
185+
scan.setFilter(filter);
186+
}
183187
}
184188

185189
private TraceIndexMetaMapper createDotMetaMapper(String applicationName) {

0 commit comments

Comments
 (0)