Skip to content

Commit fdf062b

Browse files
luigidellaquilaluigidellaquila
authored
ES|QL query log (elastic#124094) (elastic#125120)
1 parent 5e39d22 commit fdf062b

File tree

19 files changed

+907
-37
lines changed

19 files changed

+907
-37
lines changed

‎distribution/src/config/log4j2.properties‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,26 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index
129129
logger.index_indexing_slowlog.level = trace
130130
logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
131131
logger.index_indexing_slowlog.additivity = false
132+
133+
134+
######## ES|QL query log JSON ####################
135+
appender.esql_querylog_rolling.type = RollingFile
136+
appender.esql_querylog_rolling.name = esql_querylog_rolling
137+
appender.esql_querylog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
138+
.cluster_name}_esql_querylog.json
139+
appender.esql_querylog_rolling.layout.type = ECSJsonLayout
140+
appender.esql_querylog_rolling.layout.dataset = elasticsearch.esql_querylog
141+
142+
appender.esql_querylog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
143+
.cluster_name}_esql_querylog-%i.json.gz
144+
appender.esql_querylog_rolling.policies.type = Policies
145+
appender.esql_querylog_rolling.policies.size.type = SizeBasedTriggeringPolicy
146+
appender.esql_querylog_rolling.policies.size.size = 1GB
147+
appender.esql_querylog_rolling.strategy.type = DefaultRolloverStrategy
148+
appender.esql_querylog_rolling.strategy.max = 4
149+
#################################################
150+
151+
logger.esql_querylog_rolling.name = esql.querylog
152+
logger.esql_querylog_rolling.level = trace
153+
logger.esql_querylog_rolling.appenderRef.esql_querylog_rolling.ref = esql_querylog_rolling
154+
logger.esql_querylog_rolling.additivity = false

‎docs/changelog/124094.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124094
2+
summary: ES|QL slow log
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,9 @@ public interface SlowLogFieldProvider {
1919
* @param indexSettings settings for the index
2020
*/
2121
SlowLogFields create(IndexSettings indexSettings);
22+
23+
/**
24+
* Create a field provider without index level settings
25+
*/
26+
SlowLogFields create();
2227
}

‎server/src/main/java/org/elasticsearch/index/SlowLogFields.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,12 @@ public interface SlowLogFields {
2727
* @return map of field name to value
2828
*/
2929
Map<String, String> searchFields();
30+
31+
/**
32+
* Slow log fields for query
33+
* @return map of field name to value
34+
*/
35+
default Map<String, String> queryFields() {
36+
return Map.of();
37+
}
3038
}

‎server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public class IndicesServiceBuilder {
8383
QueryRewriteInterceptor queryRewriteInterceptor = null;
8484
SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
8585
@Override
86-
public SlowLogFields create(IndexSettings indexSettings) {
86+
public SlowLogFields create() {
8787
return new SlowLogFields() {
8888
@Override
8989
public Map<String, String> indexFields() {
@@ -96,6 +96,12 @@ public Map<String, String> searchFields() {
9696
}
9797
};
9898
}
99+
100+
@Override
101+
public SlowLogFields create(IndexSettings indexSettings) {
102+
return create();
103+
}
104+
99105
};
100106

101107
public IndicesServiceBuilder settings(Settings settings) {

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 59 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@
112112
import org.elasticsearch.index.IndexMode;
113113
import org.elasticsearch.index.IndexSettingProvider;
114114
import org.elasticsearch.index.IndexSettingProviders;
115+
import org.elasticsearch.index.IndexSettings;
115116
import org.elasticsearch.index.IndexingPressure;
116117
import org.elasticsearch.index.SlowLogFieldProvider;
117118
import org.elasticsearch.index.SlowLogFields;
@@ -810,26 +811,65 @@ private void construct(
810811
List<? extends SlowLogFieldProvider> slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class);
811812
// NOTE: the response of index/search slow log fields below must be calculated dynamically on every call
812813
// because the responses may change dynamically at runtime
813-
SlowLogFieldProvider slowLogFieldProvider = indexSettings -> {
814-
final List<SlowLogFields> fields = new ArrayList<>();
815-
for (var provider : slowLogFieldProviders) {
816-
fields.add(provider.create(indexSettings));
817-
}
818-
return new SlowLogFields() {
819-
@Override
820-
public Map<String, String> indexFields() {
821-
return fields.stream()
822-
.flatMap(f -> f.indexFields().entrySet().stream())
823-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
814+
SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
815+
public SlowLogFields create() {
816+
final List<SlowLogFields> fields = new ArrayList<>();
817+
for (var provider : slowLogFieldProviders) {
818+
fields.add(provider.create());
824819
}
820+
return new SlowLogFields() {
821+
@Override
822+
public Map<String, String> indexFields() {
823+
return fields.stream()
824+
.flatMap(f -> f.indexFields().entrySet().stream())
825+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
826+
}
827+
828+
@Override
829+
public Map<String, String> searchFields() {
830+
return fields.stream()
831+
.flatMap(f -> f.searchFields().entrySet().stream())
832+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
833+
}
834+
835+
@Override
836+
public Map<String, String> queryFields() {
837+
return fields.stream()
838+
.flatMap(f -> f.queryFields().entrySet().stream())
839+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
840+
}
841+
};
842+
}
825843

826-
@Override
827-
public Map<String, String> searchFields() {
828-
return fields.stream()
829-
.flatMap(f -> f.searchFields().entrySet().stream())
830-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
844+
public SlowLogFields create(IndexSettings indexSettings) {
845+
final List<SlowLogFields> fields = new ArrayList<>();
846+
for (var provider : slowLogFieldProviders) {
847+
fields.add(provider.create(indexSettings));
831848
}
832-
};
849+
return new SlowLogFields() {
850+
@Override
851+
public Map<String, String> indexFields() {
852+
return fields.stream()
853+
.flatMap(f -> f.indexFields().entrySet().stream())
854+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
855+
}
856+
857+
@Override
858+
public Map<String, String> searchFields() {
859+
return fields.stream()
860+
.flatMap(f -> f.searchFields().entrySet().stream())
861+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
862+
}
863+
864+
@Override
865+
public Map<String, String> queryFields() {
866+
return fields.stream()
867+
.flatMap(f -> f.queryFields().entrySet().stream())
868+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
869+
}
870+
};
871+
}
872+
833873
};
834874

835875
IndicesService indicesService = new IndicesServiceBuilder().settings(settings)
@@ -917,7 +957,8 @@ public Map<String, String> searchFields() {
917957
systemIndices,
918958
dataStreamGlobalRetentionSettings,
919959
documentParsingProvider,
920-
taskManager
960+
taskManager,
961+
slowLogFieldProvider
921962
);
922963

923964
Collection<?> pluginComponents = pluginsService.flatMap(plugin -> {

‎server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.env.Environment;
2020
import org.elasticsearch.env.NodeEnvironment;
2121
import org.elasticsearch.features.FeatureService;
22+
import org.elasticsearch.index.SlowLogFieldProvider;
2223
import org.elasticsearch.indices.IndicesService;
2324
import org.elasticsearch.indices.SystemIndices;
2425
import org.elasticsearch.plugins.Plugin;
@@ -51,5 +52,6 @@ public record PluginServiceInstances(
5152
SystemIndices systemIndices,
5253
DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings,
5354
DocumentParsingProvider documentParsingProvider,
54-
TaskManager taskManager
55+
TaskManager taskManager,
56+
SlowLogFieldProvider slowLogFieldProvider
5557
) implements Plugin.PluginServices {}

‎server/src/main/java/org/elasticsearch/plugins/Plugin.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.features.FeatureService;
2828
import org.elasticsearch.index.IndexModule;
2929
import org.elasticsearch.index.IndexSettingProvider;
30+
import org.elasticsearch.index.SlowLogFieldProvider;
3031
import org.elasticsearch.indices.IndicesService;
3132
import org.elasticsearch.indices.SystemIndices;
3233
import org.elasticsearch.plugins.internal.DocumentParsingProvider;
@@ -173,6 +174,11 @@ public interface PluginServices {
173174
* to track task removal by registering a RemovedTaskListener.
174175
*/
175176
TaskManager taskManager();
177+
178+
/**
179+
* Provider for additional SlowLog fields
180+
*/
181+
SlowLogFieldProvider slowLogFieldProvider();
176182
}
177183

178184
/**

‎server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ static void setFields(Map<String, String> fields) {
210210
}
211211

212212
@Override
213-
public SlowLogFields create(IndexSettings indexSettings) {
213+
public SlowLogFields create() {
214214
return new SlowLogFields() {
215215
@Override
216216
public Map<String, String> indexFields() {
@@ -223,6 +223,12 @@ public Map<String, String> searchFields() {
223223
}
224224
};
225225
}
226+
227+
@Override
228+
public SlowLogFields create(IndexSettings indexSettings) {
229+
return create();
230+
}
231+
226232
}
227233

228234
public static class TestAnotherSlowLogFieldProvider implements SlowLogFieldProvider {
@@ -234,7 +240,7 @@ static void setFields(Map<String, String> fields) {
234240
}
235241

236242
@Override
237-
public SlowLogFields create(IndexSettings indexSettings) {
243+
public SlowLogFields create() {
238244
return new SlowLogFields() {
239245
@Override
240246
public Map<String, String> indexFields() {
@@ -247,6 +253,11 @@ public Map<String, String> searchFields() {
247253
}
248254
};
249255
}
256+
257+
@Override
258+
public SlowLogFields create(IndexSettings indexSettings) {
259+
return create();
260+
}
250261
}
251262

252263
@Override

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/LogType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ public enum LogType {
1414
SERVER_JSON("%s_server.json"),
1515
AUDIT("%s_audit.json"),
1616
SEARCH_SLOW("%s_index_search_slowlog.json"),
17-
INDEXING_SLOW("%s_index_indexing_slowlog.json");
17+
INDEXING_SLOW("%s_index_indexing_slowlog.json"),
18+
ESQL_QUERY("%s_esql_querylog.json");
1819

1920
private final String filenameFormat;
2021

0 commit comments

Comments
 (0)