Skip to content

Commit f3ed9b3

Browse files
luigidellaquilaluigidellaquila
authored
ES|QL query log (elastic#124094)
1 parent c3222ab commit f3ed9b3

File tree

19 files changed

+906
-36
lines changed

19 files changed

+906
-36
lines changed

‎distribution/src/config/log4j2.properties‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,26 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index
129129
logger.index_indexing_slowlog.level = trace
130130
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
131131
logger.index_indexing_slowlog.additivity = false
132+
133+
134+
######## ES|QL query log JSON ####################
135+
appender.esql_querylog_rolling.type = RollingFile
136+
appender.esql_querylog_rolling.name = esql_querylog_rolling
137+
appender.esql_querylog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
138+
.cluster_name}_esql_querylog.json
139+
appender.esql_querylog_rolling.layout.type = ECSJsonLayout
140+
appender.esql_querylog_rolling.layout.dataset = elasticsearch.esql_querylog
141+
142+
appender.esql_querylog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
143+
.cluster_name}_esql_querylog-%i.json.gz
144+
appender.esql_querylog_rolling.policies.type = Policies
145+
appender.esql_querylog_rolling.policies.size.type = SizeBasedTriggeringPolicy
146+
appender.esql_querylog_rolling.policies.size.size = 1GB
147+
appender.esql_querylog_rolling.strategy.type = DefaultRolloverStrategy
148+
appender.esql_querylog_rolling.strategy.max = 4
149+
#################################################
150+
151+
logger.esql_querylog_rolling.name = esql.querylog
152+
logger.esql_querylog_rolling.level = trace
153+
logger.esql_querylog_rolling.appenderRef.esql_querylog_rolling.ref = esql_querylog_rolling
154+
logger.esql_querylog_rolling.additivity = false

‎docs/changelog/124094.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124094
2+
summary: ES|QL slow log
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,9 @@ public interface SlowLogFieldProvider {
1919
* @param indexSettings settings for the index
2020
*/
2121
SlowLogFields create(IndexSettings indexSettings);
22+
23+
/**
24+
* Create a field provider without index level settings
25+
*/
26+
SlowLogFields create();
2227
}

‎server/src/main/java/org/elasticsearch/index/SlowLogFields.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,12 @@ public interface SlowLogFields {
2727
* @return map of field name to value
2828
*/
2929
Map<String, String> searchFields();
30+
31+
/**
32+
* Slow log fields for query
33+
* @return map of field name to value
34+
*/
35+
default Map<String, String> queryFields() {
36+
return Map.of();
37+
}
3038
}

‎server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public class IndicesServiceBuilder {
8383
QueryRewriteInterceptor queryRewriteInterceptor = null;
8484
SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
8585
@Override
86-
public SlowLogFields create(IndexSettings indexSettings) {
86+
public SlowLogFields create() {
8787
return new SlowLogFields() {
8888
@Override
8989
public Map<String, String> indexFields() {
@@ -96,6 +96,12 @@ public Map<String, String> searchFields() {
9696
}
9797
};
9898
}
99+
100+
@Override
101+
public SlowLogFields create(IndexSettings indexSettings) {
102+
return create();
103+
}
104+
99105
};
100106

101107
public IndicesServiceBuilder settings(Settings settings) {

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 59 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@
112112
import org.elasticsearch.index.IndexMode;
113113
import org.elasticsearch.index.IndexSettingProvider;
114114
import org.elasticsearch.index.IndexSettingProviders;
115+
import org.elasticsearch.index.IndexSettings;
115116
import org.elasticsearch.index.IndexingPressure;
116117
import org.elasticsearch.index.SlowLogFieldProvider;
117118
import org.elasticsearch.index.SlowLogFields;
@@ -806,26 +807,65 @@ private void construct(
806807
List<? extends SlowLogFieldProvider> slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class);
807808
// NOTE: the response of index/search slow log fields below must be calculated dynamically on every call
808809
// because the responses may change dynamically at runtime
809-
SlowLogFieldProvider slowLogFieldProvider = indexSettings -> {
810-
final List<SlowLogFields> fields = new ArrayList<>();
811-
for (var provider : slowLogFieldProviders) {
812-
fields.add(provider.create(indexSettings));
813-
}
814-
return new SlowLogFields() {
815-
@Override
816-
public Map<String, String> indexFields() {
817-
return fields.stream()
818-
.flatMap(f -> f.indexFields().entrySet().stream())
819-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
810+
SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
811+
public SlowLogFields create() {
812+
final List<SlowLogFields> fields = new ArrayList<>();
813+
for (var provider : slowLogFieldProviders) {
814+
fields.add(provider.create());
820815
}
816+
return new SlowLogFields() {
817+
@Override
818+
public Map<String, String> indexFields() {
819+
return fields.stream()
820+
.flatMap(f -> f.indexFields().entrySet().stream())
821+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
822+
}
823+
824+
@Override
825+
public Map<String, String> searchFields() {
826+
return fields.stream()
827+
.flatMap(f -> f.searchFields().entrySet().stream())
828+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
829+
}
830+
831+
@Override
832+
public Map<String, String> queryFields() {
833+
return fields.stream()
834+
.flatMap(f -> f.queryFields().entrySet().stream())
835+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
836+
}
837+
};
838+
}
821839

822-
@Override
823-
public Map<String, String> searchFields() {
824-
return fields.stream()
825-
.flatMap(f -> f.searchFields().entrySet().stream())
826-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
840+
public SlowLogFields create(IndexSettings indexSettings) {
841+
final List<SlowLogFields> fields = new ArrayList<>();
842+
for (var provider : slowLogFieldProviders) {
843+
fields.add(provider.create(indexSettings));
827844
}
828-
};
845+
return new SlowLogFields() {
846+
@Override
847+
public Map<String, String> indexFields() {
848+
return fields.stream()
849+
.flatMap(f -> f.indexFields().entrySet().stream())
850+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
851+
}
852+
853+
@Override
854+
public Map<String, String> searchFields() {
855+
return fields.stream()
856+
.flatMap(f -> f.searchFields().entrySet().stream())
857+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
858+
}
859+
860+
@Override
861+
public Map<String, String> queryFields() {
862+
return fields.stream()
863+
.flatMap(f -> f.queryFields().entrySet().stream())
864+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
865+
}
866+
};
867+
}
868+
829869
};
830870

831871
IndicesService indicesService = new IndicesServiceBuilder().settings(settings)
@@ -914,7 +954,8 @@ public Map<String, String> searchFields() {
914954
dataStreamGlobalRetentionSettings,
915955
documentParsingProvider,
916956
taskManager,
917-
projectResolver
957+
projectResolver,
958+
slowLogFieldProvider
918959
);
919960

920961
Collection<?> pluginComponents = pluginsService.flatMap(plugin -> {

‎server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.env.Environment;
2121
import org.elasticsearch.env.NodeEnvironment;
2222
import org.elasticsearch.features.FeatureService;
23+
import org.elasticsearch.index.SlowLogFieldProvider;
2324
import org.elasticsearch.indices.IndicesService;
2425
import org.elasticsearch.indices.SystemIndices;
2526
import org.elasticsearch.plugins.Plugin;
@@ -53,5 +54,6 @@ public record PluginServiceInstances(
5354
DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings,
5455
DocumentParsingProvider documentParsingProvider,
5556
TaskManager taskManager,
56-
ProjectResolver projectResolver
57+
ProjectResolver projectResolver,
58+
SlowLogFieldProvider slowLogFieldProvider
5759
) implements Plugin.PluginServices {}

‎server/src/main/java/org/elasticsearch/plugins/Plugin.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.features.FeatureService;
3030
import org.elasticsearch.index.IndexModule;
3131
import org.elasticsearch.index.IndexSettingProvider;
32+
import org.elasticsearch.index.SlowLogFieldProvider;
3233
import org.elasticsearch.indices.IndicesService;
3334
import org.elasticsearch.indices.SystemIndices;
3435
import org.elasticsearch.plugins.internal.DocumentParsingProvider;
@@ -180,6 +181,11 @@ public interface PluginServices {
180181
* The project resolver for the cluster. This should be used to determine the active project against which a request should execute
181182
*/
182183
ProjectResolver projectResolver();
184+
185+
/**
186+
* Provider for additional SlowLog fields
187+
*/
188+
SlowLogFieldProvider slowLogFieldProvider();
183189
}
184190

185191
/**

‎server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ static void setFields(Map<String, String> fields) {
213213
}
214214

215215
@Override
216-
public SlowLogFields create(IndexSettings indexSettings) {
216+
public SlowLogFields create() {
217217
return new SlowLogFields() {
218218
@Override
219219
public Map<String, String> indexFields() {
@@ -226,6 +226,12 @@ public Map<String, String> searchFields() {
226226
}
227227
};
228228
}
229+
230+
@Override
231+
public SlowLogFields create(IndexSettings indexSettings) {
232+
return create();
233+
}
234+
229235
}
230236

231237
public static class TestAnotherSlowLogFieldProvider implements SlowLogFieldProvider {
@@ -237,7 +243,7 @@ static void setFields(Map<String, String> fields) {
237243
}
238244

239245
@Override
240-
public SlowLogFields create(IndexSettings indexSettings) {
246+
public SlowLogFields create() {
241247
return new SlowLogFields() {
242248
@Override
243249
public Map<String, String> indexFields() {
@@ -250,6 +256,11 @@ public Map<String, String> searchFields() {
250256
}
251257
};
252258
}
259+
260+
@Override
261+
public SlowLogFields create(IndexSettings indexSettings) {
262+
return create();
263+
}
253264
}
254265

255266
@Override

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/LogType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public enum LogType {
1515
AUDIT("%s_audit.json"),
1616
SEARCH_SLOW("%s_index_search_slowlog.json"),
1717
INDEXING_SLOW("%s_index_indexing_slowlog.json"),
18+
ESQL_SLOW("%s_esql_slowlog.json"),
1819
DEPRECATION("%s_deprecation.json");
1920

2021
private final String filenameFormat;

0 commit comments

Comments
 (0)