Skip to content

Commit 826bbad

Browse files
parkertimminsparkertimmins
committed
A full test that should in theory work
1 parent 6e36e5e commit 826bbad

File tree

1 file changed

+147
-35
lines changed

1 file changed

+147
-35
lines changed

‎x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextRandomTests.java‎

Lines changed: 147 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -16,101 +16,213 @@
1616
import org.elasticsearch.action.bulk.BulkRequest;
1717
import org.elasticsearch.action.bulk.BulkResponse;
1818
import org.elasticsearch.action.index.IndexRequest;
19+
import org.elasticsearch.action.search.SearchRequest;
20+
import org.elasticsearch.action.search.SearchRequestBuilder;
21+
import org.elasticsearch.action.search.SearchResponse;
1922
import org.elasticsearch.action.support.IndicesOptions;
2023
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2124
import org.elasticsearch.common.settings.Settings;
2225
import org.elasticsearch.common.time.DateFormatter;
26+
import org.elasticsearch.common.util.CollectionUtils;
2327
import org.elasticsearch.core.Tuple;
28+
import org.elasticsearch.datageneration.queries.LeafQueryGenerator;
29+
import org.elasticsearch.datastreams.DataStreamsPlugin;
2430
import org.elasticsearch.index.IndexMode;
2531
import org.elasticsearch.index.IndexSettings;
2632
import org.elasticsearch.index.mapper.DateFieldMapper;
33+
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
34+
import org.elasticsearch.index.query.MatchQueryBuilder;
35+
import org.elasticsearch.index.query.QueryBuilder;
36+
import org.elasticsearch.index.query.QueryBuilders;
37+
import org.elasticsearch.license.LicenseSettings;
38+
import org.elasticsearch.plugins.Plugin;
39+
import org.elasticsearch.search.SearchHit;
2740
import org.elasticsearch.test.ESIntegTestCase;
2841
import org.elasticsearch.test.ESTestCase;
42+
import org.elasticsearch.test.InternalSettingsPlugin;
43+
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
44+
import org.elasticsearch.test.transport.MockTransportService;
2945
import org.elasticsearch.xcontent.XContentBuilder;
3046
import org.elasticsearch.xcontent.XContentFactory;
3147
import org.elasticsearch.xcontent.XContentType;
3248
import org.elasticsearch.xcontent.json.JsonXContent;
49+
import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
50+
import org.elasticsearch.xpack.core.XPackPlugin;
3351
import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction;
52+
import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
53+
import org.junit.Before;
3454

3555
import java.io.IOException;
3656
import java.time.Instant;
3757
import java.time.ZoneOffset;
3858
import java.time.ZonedDateTime;
39-
import java.util.Arrays;
40-
import java.util.UUID;
59+
import java.util.*;
4160
import java.util.concurrent.TimeUnit;
61+
import java.util.stream.Collectors;
62+
63+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
4264

4365
public class PatternedTextRandomTests extends ESIntegTestCase {
4466

67+
@Override
68+
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
69+
return Settings.builder()
70+
.put(super.nodeSettings(nodeOrdinal, otherSettings))
71+
// .put("xpack.license.self_generated.type", "trial")
72+
// .put("cluster.logsdb.enabled", "true")
73+
.put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial")
74+
.build();
75+
}
76+
4577

4678

79+
@Override
80+
protected Collection<Class<? extends Plugin>> nodePlugins() {
81+
// return Arrays.asList(LocalStateCompositeXPackPlugin.class);
82+
return Arrays.asList(MapperExtrasPlugin.class, LogsDBPlugin.class, LocalStateCompositeXPackPlugin.class);
83+
// return CollectionUtils.appendToCopy(super.nodePlugins(), LogsDBPlugin.class, LocalStateCompositeXPackPlugin.class);
84+
// return List.of(InternalSettingsPlugin.class, XPackPlugin.class, LogsDBPlugin.class, DataStreamsPlugin.class);
85+
}
86+
87+
private static final String INDEX = "test_index";
88+
private static final String MATCH_ONLY_TEXT_FIELD = "field_match_only_text";
89+
private static final String PATTERNED_TEXT_FIELD = "field_patterned_text";
90+
91+
@Before
92+
public void setup() {
93+
assumeTrue("Only when patterned_text feature flag is enabled", PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled());
94+
}
95+
4796
public void test() throws IOException {
48-
var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
49-
String index = "test_index";
97+
var settings = Settings.builder();
98+
// var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
5099
var mappings = XContentFactory.jsonBuilder()
51100
.startObject()
52101
.startObject("properties")
53102
.startObject("@timestamp")
54103
.field("type", "date")
55104
.endObject()
56-
.startObject("field_patterned_text")
105+
.startObject(PATTERNED_TEXT_FIELD)
57106
.field("type", "patterned_text")
58107
.endObject()
59-
.startObject("field_match_only_text")
108+
.startObject(MATCH_ONLY_TEXT_FIELD)
60109
.field("type", "match_only_text")
61110
.endObject()
62111
.endObject()
63112
.endObject();
64113

65-
var createRequest = new CreateIndexRequest(index)
114+
var createRequest = new CreateIndexRequest(INDEX)
66115
.settings(settings)
67116
.mapping(mappings);
68117

69118
var createResponse = safeGet(admin().indices().create(createRequest));
70119
assertTrue(createResponse.isAcknowledged());
71120

72-
BulkRequest bulkRequest = new BulkRequest();
73121
int numDocs = randomIntBetween(1, 300);
74-
long timestamp = System.currentTimeMillis();
75-
for (int i = 0; i < numDocs; i++) {
76-
timestamp += TimeUnit.SECONDS.toMillis(1);
77-
String logMessage = randomMessage();
78-
var indexRequest = new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE)
79-
.source(
80-
JsonXContent.contentBuilder()
81-
.startObject()
82-
.field("@timestamp", timestamp)
83-
.field("field_patterned_text", logMessage)
84-
.field("field_match_only_text", logMessage);
85-
.endObject()
122+
List<String> logMessages = generateMessages(numDocs);
123+
indexDocs(logMessages);
124+
125+
for (var message : logMessages) {
126+
List<String> queryTerms = randomQueryParts(message);
127+
128+
var patternedTextQueries = generateQueries(PATTERNED_TEXT_FIELD, queryTerms);
129+
var matchOnlyQueries = generateQueries(MATCH_ONLY_TEXT_FIELD, queryTerms);
130+
131+
for (int i = 0; i < patternedTextQueries.size(); ++i) {
132+
var ptQuery = patternedTextQueries.get(i);
133+
var motQuery = matchOnlyQueries.get(i);
134+
135+
var ptResponse = client().prepareSearch(INDEX).setQuery(ptQuery).setSize(numDocs).get();
136+
var motResponse = client().prepareSearch(INDEX).setQuery(motQuery).setSize(numDocs).get();
137+
138+
assertNoFailures(ptResponse);
139+
assertNoFailures(motResponse);
140+
141+
// assertTrue(motResponse.getHits().getTotalHits().value() > 0);
142+
assertEquals(
143+
motResponse.getHits().getTotalHits().value(),
144+
ptResponse.getHits().getTotalHits().value()
86145
);
87-
bulkRequest.add(indexRequest);
88-
}
89-
BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
90146

91-
client().index(
147+
var motDocIds = Arrays.stream(motResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet());
148+
var ptDocIds = Arrays.stream(ptResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toSet());
92149

93-
)
94-
);
95-
safeGet(
96-
indicesAdmin().refresh(
97-
new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden())
98-
)
99-
);
150+
assertEquals(motDocIds, ptDocIds);
151+
}
152+
}
153+
}
100154

101-
int numDocs = randomIntBetween(100, 1000);
155+
public List<QueryBuilder> generateQueries(String field, List<String> queryTerms) {
156+
var results = new ArrayList<QueryBuilder>();
102157

158+
for (var queryTerm : queryTerms) {
159+
results.add(QueryBuilders.termQuery(field, queryTerm));
160+
results.add(QueryBuilders.matchQuery(field, queryTerm));
161+
results.add(QueryBuilders.matchPhraseQuery(field, queryTerm));
162+
}
103163

104-
new BulkRequest()
105-
.
164+
return results;
165+
}
106166

107-
for (int i = 0; i < numDocs; i++) {
167+
private static List<String> randomQueryParts(String value) {
168+
var values = new ArrayList<String>();
169+
var tokenizerRegex = "[\\s\\p{Punct}]+";
170+
List<String> tokens = Arrays.stream(value.split(tokenizerRegex)).filter(t -> t.isEmpty() == false).toList();
171+
172+
// full value
173+
values.add(value);
174+
// random sub-phrase
175+
values.add(randomSubstring(value));
108176

177+
if (tokens.isEmpty() == false) {
178+
// random term
179+
values.add(randomFrom(tokens));
180+
// random sub-phrase
181+
values.add(getSubPhrase(tokens));
109182
}
183+
return values;
184+
}
185+
186+
private static String randomSubstring(String value) {
187+
int low = ESTestCase.randomIntBetween(0, value.length() - 1);
188+
int hi = ESTestCase.randomIntBetween(low + 1, value.length());
189+
return value.substring(low, hi);
190+
}
110191

192+
private static String getSubPhrase(List<String> tokens) {
193+
int low = ESTestCase.randomIntBetween(0, tokens.size() - 1);
194+
int hi = ESTestCase.randomIntBetween(low + 1, tokens.size());
195+
return String.join(" ", tokens.subList(low, hi));
111196
}
112197

198+
private List<String> generateMessages(int numDocs) {
199+
List<String> logMessages = new ArrayList<>();
200+
for (int i = 0; i < numDocs; i++) {
201+
logMessages.add(randomMessage());
202+
}
203+
return logMessages;
204+
}
113205

206+
private void indexDocs(List<String> logMessages) throws IOException {
207+
BulkRequest bulkRequest = new BulkRequest();
208+
long timestamp = System.currentTimeMillis();
209+
for (var msg : logMessages) {
210+
timestamp += TimeUnit.SECONDS.toMillis(1);
211+
var indexRequest = new IndexRequest(INDEX).opType(DocWriteRequest.OpType.CREATE)
212+
.source(
213+
JsonXContent.contentBuilder()
214+
.startObject()
215+
.field("@timestamp", timestamp)
216+
.field("field_patterned_text", msg)
217+
.field("field_match_only_text", msg)
218+
.endObject()
219+
);
220+
bulkRequest.add(indexRequest);
221+
}
222+
BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
223+
assertFalse(bulkResponse.hasFailures());
224+
safeGet(indicesAdmin().refresh(new RefreshRequest(INDEX).indicesOptions(IndicesOptions.lenientExpandOpenHidden())));
225+
}
114226

115227
public static String randomMessage() {
116228
if (rarely()) {

0 commit comments

Comments
 (0)