Skip to content

Commit bb77d49

Browse files
astefanastefan
authored
ESQL: use field_caps native nested fields filtering (#121918)
* [8.x] ESQL: use field_caps native nested fields filtering (#117201) (#117375) (#121645) * Just filter the nested fields natively with field_caps support (cherry picked from commit 73381db) * Add import
1 parent 2f9f364 commit bb77d49

File tree

4 files changed

+335
-22
lines changed

4 files changed

+335
-22
lines changed

‎docs/changelog/117201.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 117201
2+
summary: "Use `field_caps` native nested fields filtering"
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 117054

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java‎

Lines changed: 319 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.client.ResponseException;
1515
import org.elasticsearch.common.Strings;
1616
import org.elasticsearch.common.network.NetworkAddress;
17+
import org.elasticsearch.common.settings.Settings;
1718
import org.elasticsearch.core.CheckedConsumer;
1819
import org.elasticsearch.geo.GeometryTestUtils;
1920
import org.elasticsearch.index.mapper.BlockLoader;
@@ -27,6 +28,7 @@
2728
import org.elasticsearch.xcontent.XContentBuilder;
2829
import org.elasticsearch.xcontent.XContentType;
2930
import org.elasticsearch.xcontent.json.JsonXContent;
31+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
3032
import org.hamcrest.Matcher;
3133
import org.junit.Before;
3234

@@ -1106,6 +1108,323 @@ public void testTypeConflictInObject() throws IOException {
11061108
);
11071109
}
11081110

1111+
/**
1112+
* Test for https://github.com/elastic/elasticsearch/issues/117054 fix
1113+
*/
1114+
public void testOneNestedSubField_AndSameNameSupportedField() throws IOException {
1115+
assumeIndexResolverNestedFieldsNameClashFixed();
1116+
ESRestTestCase.createIndex("test", Settings.EMPTY, """
1117+
"properties": {
1118+
"Responses": {
1119+
"properties": {
1120+
"process": {
1121+
"type": "nested",
1122+
"properties": {
1123+
"pid": {
1124+
"type": "long"
1125+
}
1126+
}
1127+
}
1128+
}
1129+
},
1130+
"process": {
1131+
"properties": {
1132+
"parent": {
1133+
"properties": {
1134+
"command_line": {
1135+
"type": "wildcard",
1136+
"fields": {
1137+
"text": {
1138+
"type": "text"
1139+
}
1140+
}
1141+
}
1142+
}
1143+
}
1144+
}
1145+
}
1146+
}
1147+
""");
1148+
1149+
Map<String, Object> result = runEsql("FROM test");
1150+
assertMap(
1151+
result,
1152+
matchesMapWithOptionalTook(result.get("took")).entry(
1153+
"columns",
1154+
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
1155+
).entry("values", Collections.EMPTY_LIST)
1156+
);
1157+
1158+
index("test", """
1159+
{"Responses.process.pid": 123,"process.parent.command_line":"run.bat"}""");
1160+
1161+
result = runEsql("FROM test");
1162+
assertMap(
1163+
result,
1164+
matchesMapWithOptionalTook(result.get("took")).entry(
1165+
"columns",
1166+
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
1167+
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
1168+
);
1169+
1170+
result = runEsql("""
1171+
FROM test | where process.parent.command_line == "run.bat"
1172+
""");
1173+
assertMap(
1174+
result,
1175+
matchesMapWithOptionalTook(result.get("took")).entry(
1176+
"columns",
1177+
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
1178+
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
1179+
);
1180+
1181+
ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test | SORT Responses.process.pid"));
1182+
String err = EntityUtils.toString(e.getResponse().getEntity());
1183+
assertThat(err, containsString("line 1:18: Unknown column [Responses.process.pid]"));
1184+
1185+
e = expectThrows(ResponseException.class, () -> runEsql("""
1186+
FROM test
1187+
| SORT Responses.process.pid
1188+
| WHERE Responses.process IS NULL
1189+
"""));
1190+
err = EntityUtils.toString(e.getResponse().getEntity());
1191+
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
1192+
}
1193+
1194+
public void testOneNestedSubField_AndSameNameSupportedField_TwoIndices() throws IOException {
1195+
assumeIndexResolverNestedFieldsNameClashFixed();
1196+
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
1197+
"properties": {
1198+
"Responses": {
1199+
"properties": {
1200+
"process": {
1201+
"type": "nested",
1202+
"properties": {
1203+
"pid": {
1204+
"type": "long"
1205+
}
1206+
}
1207+
}
1208+
}
1209+
}
1210+
}
1211+
""");
1212+
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
1213+
"properties": {
1214+
"process": {
1215+
"properties": {
1216+
"parent": {
1217+
"properties": {
1218+
"command_line": {
1219+
"type": "wildcard",
1220+
"fields": {
1221+
"text": {
1222+
"type": "text"
1223+
}
1224+
}
1225+
}
1226+
}
1227+
}
1228+
}
1229+
}
1230+
}
1231+
""");
1232+
index("test1", """
1233+
{"Responses.process.pid": 123}""");
1234+
index("test2", """
1235+
{"process.parent.command_line":"run.bat"}""");
1236+
1237+
Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line ASC NULLS FIRST");
1238+
assertMap(
1239+
result,
1240+
matchesMapWithOptionalTook(result.get("took")).entry(
1241+
"columns",
1242+
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
1243+
).entry("values", List.of(matchesList().item(null).item(null), matchesList().item("run.bat").item("run.bat")))
1244+
);
1245+
1246+
result = runEsql("""
1247+
FROM test* | where process.parent.command_line == "run.bat"
1248+
""");
1249+
assertMap(
1250+
result,
1251+
matchesMapWithOptionalTook(result.get("took")).entry(
1252+
"columns",
1253+
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
1254+
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
1255+
);
1256+
1257+
ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT Responses.process.pid"));
1258+
String err = EntityUtils.toString(e.getResponse().getEntity());
1259+
assertThat(err, containsString("line 1:19: Unknown column [Responses.process.pid]"));
1260+
1261+
e = expectThrows(ResponseException.class, () -> runEsql("""
1262+
FROM test*
1263+
| SORT Responses.process.pid
1264+
| WHERE Responses.process IS NULL
1265+
"""));
1266+
err = EntityUtils.toString(e.getResponse().getEntity());
1267+
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
1268+
}
1269+
1270+
public void testOneNestedField_AndSameNameSupportedField_TwoIndices() throws IOException {
1271+
assumeIndexResolverNestedFieldsNameClashFixed();
1272+
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
1273+
"properties": {
1274+
"Responses": {
1275+
"properties": {
1276+
"process": {
1277+
"type": "nested",
1278+
"properties": {
1279+
"pid": {
1280+
"type": "long"
1281+
}
1282+
}
1283+
}
1284+
}
1285+
},
1286+
"process": {
1287+
"properties": {
1288+
"parent": {
1289+
"properties": {
1290+
"command_line": {
1291+
"type": "wildcard",
1292+
"fields": {
1293+
"text": {
1294+
"type": "text"
1295+
}
1296+
}
1297+
}
1298+
}
1299+
}
1300+
}
1301+
}
1302+
}
1303+
""");
1304+
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
1305+
"properties": {
1306+
"Responses": {
1307+
"properties": {
1308+
"process": {
1309+
"type": "integer",
1310+
"fields": {
1311+
"pid": {
1312+
"type": "long"
1313+
}
1314+
}
1315+
}
1316+
}
1317+
},
1318+
"process": {
1319+
"properties": {
1320+
"parent": {
1321+
"properties": {
1322+
"command_line": {
1323+
"type": "wildcard",
1324+
"fields": {
1325+
"text": {
1326+
"type": "text"
1327+
}
1328+
}
1329+
}
1330+
}
1331+
}
1332+
}
1333+
}
1334+
}
1335+
""");
1336+
index("test1", """
1337+
{"Responses.process.pid": 111,"process.parent.command_line":"run1.bat"}""");
1338+
index("test2", """
1339+
{"Responses.process": 222,"process.parent.command_line":"run2.bat"}""");
1340+
1341+
Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line");
1342+
assertMap(
1343+
result,
1344+
matchesMapWithOptionalTook(result.get("took")).entry(
1345+
"columns",
1346+
List.of(
1347+
columnInfo("Responses.process", "integer"),
1348+
columnInfo("Responses.process.pid", "long"),
1349+
columnInfo("process.parent.command_line", "keyword"),
1350+
columnInfo("process.parent.command_line.text", "text")
1351+
)
1352+
)
1353+
.entry(
1354+
"values",
1355+
List.of(
1356+
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
1357+
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
1358+
)
1359+
)
1360+
);
1361+
1362+
result = runEsql("""
1363+
FROM test* | where Responses.process.pid == 111
1364+
""");
1365+
assertMap(
1366+
result,
1367+
matchesMapWithOptionalTook(result.get("took")).entry(
1368+
"columns",
1369+
List.of(
1370+
columnInfo("Responses.process", "integer"),
1371+
columnInfo("Responses.process.pid", "long"),
1372+
columnInfo("process.parent.command_line", "keyword"),
1373+
columnInfo("process.parent.command_line.text", "text")
1374+
)
1375+
).entry("values", List.of())
1376+
);
1377+
1378+
result = runEsql("FROM test* | SORT process.parent.command_line");
1379+
assertMap(
1380+
result,
1381+
matchesMapWithOptionalTook(result.get("took")).entry(
1382+
"columns",
1383+
List.of(
1384+
columnInfo("Responses.process", "integer"),
1385+
columnInfo("Responses.process.pid", "long"),
1386+
columnInfo("process.parent.command_line", "keyword"),
1387+
columnInfo("process.parent.command_line.text", "text")
1388+
)
1389+
)
1390+
.entry(
1391+
"values",
1392+
List.of(
1393+
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
1394+
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
1395+
)
1396+
)
1397+
);
1398+
1399+
result = runEsql("""
1400+
FROM test*
1401+
| SORT process.parent.command_line
1402+
| WHERE Responses.process IS NULL
1403+
""");
1404+
assertMap(
1405+
result,
1406+
matchesMapWithOptionalTook(result.get("took")).entry(
1407+
"columns",
1408+
List.of(
1409+
columnInfo("Responses.process", "integer"),
1410+
columnInfo("Responses.process.pid", "long"),
1411+
columnInfo("process.parent.command_line", "keyword"),
1412+
columnInfo("process.parent.command_line.text", "text")
1413+
)
1414+
).entry("values", List.of(matchesList().item(null).item(null).item("run1.bat").item("run1.bat")))
1415+
);
1416+
}
1417+
1418+
private void assumeIndexResolverNestedFieldsNameClashFixed() throws IOException {
1419+
// especially for BWC tests but also for regular tests
1420+
var capsName = EsqlCapabilities.Cap.FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER.name().toLowerCase(Locale.ROOT);
1421+
boolean requiredClusterCapability = clusterHasCapability("POST", "/_query", List.of(), List.of(capsName)).orElse(false);
1422+
assumeTrue(
1423+
"This test makes sense for versions that have the fix for https://github.com/elastic/elasticsearch/issues/117054",
1424+
requiredClusterCapability
1425+
);
1426+
}
1427+
11091428
private CheckedConsumer<XContentBuilder, IOException> empNoInObject(String empNoType) {
11101429
return index -> {
11111430
index.startObject("properties");

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,12 @@ public enum Cap {
412412
/**
413413
* Fix pushdown of LIMIT past MV_EXPAND
414414
*/
415-
ADD_LIMIT_INSIDE_MV_EXPAND;
415+
ADD_LIMIT_INSIDE_MV_EXPAND,
416+
417+
/**
418+
* Fix for https://github.com/elastic/elasticsearch/issues/117054
419+
*/
420+
FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER;
416421

417422
private final boolean enabled;
418423

0 commit comments

Comments
 (0)