Skip to content

Commit d3b9fa3

Browse files
GalLaloucheGalLalouche
committed
ESQL: Support for _index metadata field in CsvTests (elastic#121261)
* ESQL: Support for _index metadata field in CsvTests * Extract INDEX constant to MetadataAttribute * Add comment on capability
1 parent 70b4cee commit d3b9fa3

File tree

7 files changed

+49
-36
lines changed

7 files changed

+49
-36
lines changed

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public class MetadataAttribute extends TypedAttribute {
3232
public static final String TIMESTAMP_FIELD = "@timestamp";
3333
public static final String TSID_FIELD = "_tsid";
3434
public static final String SCORE = "_score";
35+
public static final String INDEX = "_index";
3536

3637
static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
3738
Attribute.class,
@@ -42,7 +43,7 @@ public class MetadataAttribute extends TypedAttribute {
4243
private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
4344
"_version",
4445
tuple(DataType.LONG, false), // _version field is not searchable
45-
"_index",
46+
INDEX,
4647
tuple(DataType.KEYWORD, true),
4748
IdFieldMapper.NAME,
4849
tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-remote.csv-spec‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ max:integer |_index:keyword
3939
;
4040

4141
metaIndexAliasedInAggs
42-
required_capability: metadata_fields
42+
required_capability: index_metadata_field
4343
required_capability: metadata_fields_remote_test
4444
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i | SORT _i;
4545

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata.csv-spec‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ max:integer |_index:keyword
4040
;
4141

4242
metaIndexSorted
43-
required_capability: metadata_fields
43+
required_capability: index_metadata_field
4444
from employees metadata _index | sort _index, emp_no desc | keep emp_no, _index | limit 2;
4545

4646

@@ -50,7 +50,7 @@ emp_no:integer |_index:keyword
5050
;
5151

5252
metaIndexWithInPredicate
53-
required_capability: metadata_fields
53+
required_capability: index_metadata_field
5454
from employees metadata _index | where _index in ("employees", "foobar") | sort emp_no desc | keep emp_no, _index | limit 2;
5555

5656

@@ -60,7 +60,7 @@ emp_no:integer |_index:keyword
6060
;
6161

6262
metaIndexAliasedInAggs
63-
required_capability: metadata_fields
63+
required_capability: index_metadata_field
6464
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
6565

6666

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ mc:l | count:l
133133

134134
multiIndexIpString
135135
required_capability: union_types
136-
required_capability: metadata_fields
136+
required_capability: index_metadata_field
137137
required_capability: casting_operator
138138
required_capability: union_types_remove_fields
139139

@@ -162,7 +162,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
162162

163163
multiIndexIpStringRename
164164
required_capability: union_types
165-
required_capability: metadata_fields
165+
required_capability: index_metadata_field
166166
required_capability: casting_operator
167167
required_capability: union_types_remove_fields
168168

@@ -191,7 +191,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
191191

192192
multiIndexIpStringRenameToString
193193
required_capability: union_types
194-
required_capability: metadata_fields
194+
required_capability: index_metadata_field
195195
required_capability: union_types_remove_fields
196196

197197
FROM sample_data, sample_data_str METADATA _index
@@ -219,7 +219,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
219219

220220
multiIndexWhereIpString
221221
required_capability: union_types
222-
required_capability: metadata_fields
222+
required_capability: index_metadata_field
223223
required_capability: union_types_remove_fields
224224

225225
FROM sample_data, sample_data_str METADATA _index
@@ -237,7 +237,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected
237237

238238
multiIndexWhereIpStringLike
239239
required_capability: union_types
240-
required_capability: metadata_fields
240+
required_capability: index_metadata_field
241241
required_capability: union_types_remove_fields
242242

243243
FROM sample_data, sample_data_str METADATA _index
@@ -445,7 +445,7 @@ count:long | message:keyword
445445

446446
multiIndexMissingIpToString
447447
required_capability: union_types
448-
required_capability: metadata_fields
448+
required_capability: index_metadata_field
449449
required_capability: union_types_missing_field
450450

451451
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -480,7 +480,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450
480480

481481
multiIndexMissingIpToIp
482482
required_capability: union_types
483-
required_capability: metadata_fields
483+
required_capability: index_metadata_field
484484
required_capability: union_types_missing_field
485485

486486
FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
@@ -515,7 +515,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
515515

516516
multiIndexTsLong
517517
required_capability: union_types
518-
required_capability: metadata_fields
518+
required_capability: index_metadata_field
519519
required_capability: union_types_remove_fields
520520

521521
FROM sample_data, sample_data_ts_long METADATA _index
@@ -543,7 +543,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
543543

544544
multiIndexTsLongRename
545545
required_capability: union_types
546-
required_capability: metadata_fields
546+
required_capability: index_metadata_field
547547
required_capability: union_types_remove_fields
548548

549549
FROM sample_data, sample_data_ts_long METADATA _index
@@ -573,7 +573,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
573573
multiIndexTsNanosRename
574574
required_capability: to_date_nanos
575575
required_capability: union_types
576-
required_capability: metadata_fields
576+
required_capability: index_metadata_field
577577
required_capability: union_types_remove_fields
578578

579579
FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -602,7 +602,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
602602
multiIndexTsNanosRenameToNanos
603603
required_capability: to_date_nanos
604604
required_capability: union_types
605-
required_capability: metadata_fields
605+
required_capability: index_metadata_field
606606
required_capability: union_types_remove_fields
607607

608608
FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -631,7 +631,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 34502
631631
multiIndex sort millis and nanos as nanos
632632
required_capability: to_date_nanos
633633
required_capability: union_types
634-
required_capability: metadata_fields
634+
required_capability: index_metadata_field
635635
required_capability: union_types_remove_fields
636636

637637
FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -660,7 +660,7 @@ sample_data | 2023-10-23T12:15:03.360000000Z | 172.21.2.162 | 34502
660660
multiIndex sort millis and nanos as millis
661661
required_capability: to_date_nanos
662662
required_capability: union_types
663-
required_capability: metadata_fields
663+
required_capability: index_metadata_field
664664
required_capability: union_types_remove_fields
665665

666666
FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -691,7 +691,7 @@ multiIndexTsNanosRenameToNanosWithFiltering
691691
required_capability: to_date_nanos
692692
required_capability: date_nanos_binary_comparison
693693
required_capability: union_types
694-
required_capability: metadata_fields
694+
required_capability: index_metadata_field
695695
required_capability: union_types_remove_fields
696696

697697
FROM sample_data, sample_data_ts_nanos METADATA _index
@@ -716,7 +716,7 @@ sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 12323
716716

717717
multiIndexTsLongRenameToString
718718
required_capability: union_types
719-
required_capability: metadata_fields
719+
required_capability: index_metadata_field
720720
required_capability: union_types_remove_fields
721721

722722
FROM sample_data, sample_data_ts_long METADATA _index
@@ -744,7 +744,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
744744

745745
multiIndexWhereTsLong
746746
required_capability: union_types
747-
required_capability: metadata_fields
747+
required_capability: index_metadata_field
748748
required_capability: union_types_remove_fields
749749

750750
FROM sample_data, sample_data_ts_long METADATA _index
@@ -979,7 +979,7 @@ count:long | message:keyword
979979

980980
multiIndexIpStringTsLong
981981
required_capability: union_types
982-
required_capability: metadata_fields
982+
required_capability: index_metadata_field
983983
required_capability: union_types_remove_fields
984984
required_capability: to_date_nanos
985985

@@ -1022,7 +1022,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
10221022

10231023
multiIndexIpStringTsLongDropped
10241024
required_capability: union_types
1025-
required_capability: metadata_fields
1025+
required_capability: index_metadata_field
10261026
required_capability: to_date_nanos
10271027

10281028
FROM sample_data* METADATA _index
@@ -1064,7 +1064,7 @@ sample_data_ts_nanos | 8268153 | Connection error
10641064

10651065
multiIndexIpStringTsLongRename
10661066
required_capability: union_types
1067-
required_capability: metadata_fields
1067+
required_capability: index_metadata_field
10681068
required_capability: union_types_remove_fields
10691069
required_capability: to_date_nanos
10701070

@@ -1107,7 +1107,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
11071107

11081108
multiIndexIpStringTsLongRenameDropped
11091109
required_capability: union_types
1110-
required_capability: metadata_fields
1110+
required_capability: index_metadata_field
11111111
required_capability: to_date_nanos
11121112

11131113
FROM sample_data* METADATA _index
@@ -1149,7 +1149,7 @@ sample_data_ts_nanos | 8268153 | Connection error
11491149

11501150
multiIndexIpStringTsLongRenameToString
11511151
required_capability: union_types
1152-
required_capability: metadata_fields
1152+
required_capability: index_metadata_field
11531153
required_capability: union_types_remove_fields
11541154
required_capability: to_date_nanos
11551155

@@ -1192,7 +1192,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
11921192

11931193
multiIndexWhereIpStringTsLong
11941194
required_capability: union_types
1195-
required_capability: metadata_fields
1195+
required_capability: index_metadata_field
11961196
required_capability: union_types_remove_fields
11971197
required_capability: to_date_nanos
11981198

@@ -1226,7 +1226,7 @@ count:long | message:keyword
12261226

12271227
multiIndexWhereIpStringLikeTsLong
12281228
required_capability: union_types
1229-
required_capability: metadata_fields
1229+
required_capability: index_metadata_field
12301230
required_capability: union_types_remove_fields
12311231
required_capability: to_date_nanos
12321232

@@ -1260,7 +1260,7 @@ count:long | message:keyword
12601260

12611261
multiIndexMultiColumnTypesRename
12621262
required_capability: union_types
1263-
required_capability: metadata_fields
1263+
required_capability: index_metadata_field
12641264
required_capability: union_types_remove_fields
12651265
required_capability: to_date_nanos
12661266

@@ -1279,7 +1279,7 @@ null | null | 8268153 | Connectio
12791279

12801280
multiIndexMultiColumnTypesRenameAndKeep
12811281
required_capability: union_types
1282-
required_capability: metadata_fields
1282+
required_capability: index_metadata_field
12831283
required_capability: union_types_remove_fields
12841284
required_capability: to_date_nanos
12851285

@@ -1299,7 +1299,7 @@ sample_data_ts_nanos | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015123456
12991299

13001300
multiIndexMultiColumnTypesRenameAndDrop
13011301
required_capability: union_types
1302-
required_capability: metadata_fields
1302+
required_capability: index_metadata_field
13031303
required_capability: union_types_remove_fields
13041304
required_capability: to_date_nanos
13051305

@@ -1591,7 +1591,7 @@ FROM sample_data, sample_data_ts_long
15911591

15921592
shortIntegerWidening
15931593
required_capability: union_types
1594-
required_capability: metadata_fields
1594+
required_capability: index_metadata_field
15951595
required_capability: casting_operator
15961596
required_capability: union_types_numeric_widening
15971597

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,17 @@ public enum Cap {
121121
* Cast string literals to a desired data type for IN predicate and more types for BinaryComparison.
122122
*/
123123
STRING_LITERAL_AUTO_CASTING_EXTENDED,
124-
125124
/**
126125
* Support for metadata fields.
127126
*/
128127
METADATA_FIELDS,
129128

129+
/**
130+
* Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently
131+
* supported.
132+
*/
133+
INDEX_METADATA_FIELD,
134+
130135
/**
131136
* Support for timespan units abbreviations
132137
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.index.query.QueryBuilder;
2222
import org.elasticsearch.threadpool.ThreadPool;
2323
import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
24+
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
2425
import org.elasticsearch.xpack.esql.core.type.DataType;
2526
import org.elasticsearch.xpack.esql.core.type.DateEsField;
2627
import org.elasticsearch.xpack.esql.core.type.EsField;
@@ -50,7 +51,7 @@
5051

5152
public class IndexResolver {
5253
public static final Set<String> ALL_FIELDS = Set.of("*");
53-
public static final Set<String> INDEX_METADATA_FIELD = Set.of("_index");
54+
public static final Set<String> INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX);
5455
public static final String UNMAPPED = "unmapped";
5556

5657
public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.elasticsearch.xpack.esql.core.expression.Attribute;
4949
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
5050
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
51+
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
5152
import org.elasticsearch.xpack.esql.core.type.DataType;
5253
import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
5354
import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
@@ -292,6 +293,10 @@ private Block getBlockForMultiType(DocBlock indexDoc, MultiTypeEsField multiType
292293
private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) {
293294
var indexId = docBlock.asVector().shards().getInt(0);
294295
var indexPage = indexPages.get(indexId);
296+
if (MetadataAttribute.INDEX.equals(columnName)) {
297+
return docBlock.blockFactory()
298+
.newConstantBytesRefBlockWith(new BytesRef(indexPage.index), blockCopier.docIndices.getPositionCount());
299+
}
295300
int columnIndex = indexPage.columnIndex(columnName)
296301
.orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames));
297302
var originalData = indexPage.page.getBlock(columnIndex);
@@ -410,8 +415,9 @@ private Block extractBlockForColumn(
410415
) {
411416
foreachIndexDoc(docBlock, indexDoc -> {
412417
TestBlockCopier blockCopier = blockCopier(dataType, extractPreference, indexDoc.asVector().docs());
413-
Block blockForIndex = extractBlock.apply(indexDoc, blockCopier);
414-
blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
418+
try (Block blockForIndex = extractBlock.apply(indexDoc, blockCopier)) {
419+
blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
420+
}
415421
});
416422
var result = blockBuilder.build();
417423
assert result.getPositionCount() == docBlock.getPositionCount()

0 commit comments

Comments
 (0)