Skip to content

Commit 068cb8b

Browse files
songkant-awssongkant-aws
committed
Improved patterns command with new algorithm (#3263)
* Improved patterns command with new algorithm Signed-off-by: Songkan Tang <songkant@amazon.com> * Minor change log parser default configurations Signed-off-by: Songkan Tang <songkant@amazon.com> * Refactor a bit and add partial unit tests Signed-off-by: Songkan Tang <songkant@amazon.com> * Add more unit tests Signed-off-by: Songkan Tang <songkant@amazon.com> * Amend patterns user facing doc Signed-off-by: Songkan Tang <songkant@amazon.com> * Add average time benchmark for patterns window functions Signed-off-by: Songkan Tang <songkant@amazon.com> * Add new default pattern method setting to allow change Signed-off-by: Songkan Tang <songkant@amazon.com> * Update unit tests per injected setting Signed-off-by: Songkan Tang <songkant@amazon.com> * Update patterns command user facing doc after introducing setting Signed-off-by: Songkan Tang <songkant@amazon.com> * Complement more unit test cases Signed-off-by: Songkan Tang <songkant@amazon.com> * Adjust patterns.rst file format Signed-off-by: Songkan Tang <songkant@amazon.com> * Handle null like ExprValue cases and fix additional doctest Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix spotless style check Signed-off-by: Songkan Tang <songkant@amazon.com> * Minor doctest fix style Signed-off-by: Songkan Tang <songkant@amazon.com> --------- Signed-off-by: Songkan Tang <songkant@amazon.com>
1 parent 03083d7 commit 068cb8b

File tree

41 files changed

+2227
-129
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+2227
-129
lines changed

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/PatternsWindowFunctionBenchmark.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.operator.predicate;
7+
8+
import static org.opensearch.sql.data.model.ExprTupleValue.fromExprValueMap;
9+
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
10+
11+
import com.google.common.collect.ImmutableList;
12+
import com.google.common.collect.ImmutableMap;
13+
import com.google.common.collect.Iterators;
14+
import com.google.common.collect.PeekingIterator;
15+
import java.util.concurrent.TimeUnit;
16+
import org.openjdk.jmh.annotations.Benchmark;
17+
import org.openjdk.jmh.annotations.BenchmarkMode;
18+
import org.openjdk.jmh.annotations.Fork;
19+
import org.openjdk.jmh.annotations.Measurement;
20+
import org.openjdk.jmh.annotations.Mode;
21+
import org.openjdk.jmh.annotations.OutputTimeUnit;
22+
import org.openjdk.jmh.annotations.Scope;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
import org.opensearch.sql.common.patterns.BrainLogParser;
26+
import org.opensearch.sql.data.model.ExprStringValue;
27+
import org.opensearch.sql.data.model.ExprValue;
28+
import org.opensearch.sql.expression.DSL;
29+
import org.opensearch.sql.expression.Expression;
30+
import org.opensearch.sql.expression.NamedArgumentExpression;
31+
import org.opensearch.sql.expression.ReferenceExpression;
32+
import org.opensearch.sql.expression.window.WindowDefinition;
33+
import org.opensearch.sql.expression.window.frame.BufferPatternRowsWindowFrame;
34+
import org.opensearch.sql.expression.window.frame.CurrentRowWindowFrame;
35+
import org.opensearch.sql.expression.window.frame.WindowFrame;
36+
37+
@Warmup(iterations = 1)
38+
@Measurement(iterations = 3)
39+
@BenchmarkMode(Mode.AverageTime)
40+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
41+
@State(Scope.Thread)
42+
@Fork(value = 1)
43+
public class PatternsWindowFunctionBenchmark {
44+
45+
private static final String TEST_MESSAGE_1 =
46+
"12.132.31.17 - - [2018-07-22T05:36:25.812Z] \\\"GET /opensearch HTTP/1.1\\\" 200 9797"
47+
+ " \\\"-\\\" \\\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421"
48+
+ " Firefox/6.0a1\\\"";
49+
private static final String TEST_MESSAGE_2 =
50+
"129.138.185.193 - - [2018-07-22T05:39:39.668Z] \\\"GET /opensearch HTTP/1.1\\\" 404 9920"
51+
+ " \\\"-\\\" \\\"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421"
52+
+ " Firefox/6.0a1\\\"";
53+
private static final String TEST_MESSAGE_3 =
54+
"240.58.187.246 - - [2018-07-22T06:02:46.006Z] \\\"GET /opensearch HTTP/1.1\\\" 500 6936"
55+
+ " \\\"-\\\" \\\"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko)"
56+
+ " Chrome/11.0.696.50 Safari/534.24\\\"";
57+
58+
private PeekingIterator<ExprValue> tuples;
59+
private final BufferPatternRowsWindowFrame bufferWindowFrame =
60+
new BufferPatternRowsWindowFrame(
61+
new WindowDefinition(ImmutableList.of(), ImmutableList.of()),
62+
new BrainLogParser(),
63+
new NamedArgumentExpression("message", new ReferenceExpression("message", STRING)));
64+
65+
@Benchmark
66+
public void testSimplePattern() {
67+
CurrentRowWindowFrame windowFrame =
68+
new CurrentRowWindowFrame(new WindowDefinition(ImmutableList.of(), ImmutableList.of()));
69+
70+
run(windowFrame, DSL.simple_pattern(DSL.ref("message", STRING)));
71+
}
72+
73+
@Benchmark
74+
public void testBrain() {
75+
BufferPatternRowsWindowFrame windowFrame =
76+
new BufferPatternRowsWindowFrame(
77+
new WindowDefinition(ImmutableList.of(), ImmutableList.of()),
78+
new BrainLogParser(),
79+
new NamedArgumentExpression("message", new ReferenceExpression("message", STRING)));
80+
81+
run(windowFrame, DSL.brain(DSL.ref("message", STRING)));
82+
}
83+
84+
private void run(WindowFrame windowFrame, Expression windowFunction) {
85+
tuples =
86+
Iterators.peekingIterator(
87+
Iterators.forArray(
88+
tuple(TEST_MESSAGE_1), tuple(TEST_MESSAGE_2), tuple(TEST_MESSAGE_3)));
89+
while (tuples.hasNext() || windowFrame.hasNext()) {
90+
windowFrame.load(tuples);
91+
windowFunction.valueOf(windowFrame);
92+
}
93+
}
94+
95+
private ExprValue tuple(String message) {
96+
return fromExprValueMap(ImmutableMap.of("message", new ExprStringValue(message)));
97+
}
98+
}

0 commit comments

Comments
 (0)