Skip to content

Tests/8.x csv tests metadata index #121499

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 3, 2025
Merged

Tests/8.x csv tests metadata index #121499

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...ql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class MetadataAttribute extends TypedAttribute {
public static final String TIMESTAMP_FIELD = "@timestamp";
public static final String TSID_FIELD = "_tsid";
public static final String SCORE = "_score";
public static final String INDEX = "_index";

static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
Attribute.class,
Expand All @@ -42,7 +43,7 @@ public class MetadataAttribute extends TypedAttribute {
private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
"_version",
tuple(DataType.LONG, false), // _version field is not searchable
"_index",
INDEX,
tuple(DataType.KEYWORD, true),
IdFieldMapper.NAME,
tuple(DataType.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-remote.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ max:integer |_index:keyword
;

metaIndexAliasedInAggs
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: metadata_fields_remote_test
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i | SORT _i;

Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ max:integer |_index:keyword
;

metaIndexSorted
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | sort _index, emp_no desc | keep emp_no, _index | limit 2;


Expand All @@ -50,7 +50,7 @@ emp_no:integer |_index:keyword
;

metaIndexWithInPredicate
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | where _index in ("employees", "foobar") | sort emp_no desc | keep emp_no, _index | limit 2;


Expand All @@ -60,7 +60,7 @@ emp_no:integer |_index:keyword
;

metaIndexAliasedInAggs
required_capability: metadata_fields
required_capability: index_metadata_field
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;


Expand Down
54 changes: 27 additions & 27 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ mc:l | count:l

multiIndexIpString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_remove_fields

Expand Down Expand Up @@ -162,7 +162,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexIpStringRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_remove_fields

Expand Down Expand Up @@ -191,7 +191,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexIpStringRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_str METADATA _index
Expand Down Expand Up @@ -219,7 +219,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexWhereIpString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_str METADATA _index
Expand All @@ -237,7 +237,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected

multiIndexWhereIpStringLike
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_str METADATA _index
Expand Down Expand Up @@ -445,7 +445,7 @@ count:long | message:keyword

multiIndexMissingIpToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_missing_field

FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
Expand Down Expand Up @@ -480,7 +480,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450

multiIndexMissingIpToIp
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_missing_field

FROM sample_data, sample_data_str, missing_ip_sample_data METADATA _index
Expand Down Expand Up @@ -515,7 +515,7 @@ sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_long METADATA _index
Expand Down Expand Up @@ -543,7 +543,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexTsLongRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_long METADATA _index
Expand Down Expand Up @@ -573,7 +573,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsNanosRename
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_nanos METADATA _index
Expand Down Expand Up @@ -602,7 +602,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233
multiIndexTsNanosRenameToNanos
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_nanos METADATA _index
Expand Down Expand Up @@ -631,7 +631,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 34502
multiIndex sort millis and nanos as nanos
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_nanos METADATA _index
Expand Down Expand Up @@ -660,7 +660,7 @@ sample_data | 2023-10-23T12:15:03.360000000Z | 172.21.2.162 | 34502
multiIndex sort millis and nanos as millis
required_capability: to_date_nanos
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_nanos METADATA _index
Expand Down Expand Up @@ -691,7 +691,7 @@ multiIndexTsNanosRenameToNanosWithFiltering
required_capability: to_date_nanos
required_capability: date_nanos_binary_comparison
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_nanos METADATA _index
Expand All @@ -716,7 +716,7 @@ sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 12323

multiIndexTsLongRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_long METADATA _index
Expand Down Expand Up @@ -744,7 +744,7 @@ sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexWhereTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields

FROM sample_data, sample_data_ts_long METADATA _index
Expand Down Expand Up @@ -979,7 +979,7 @@ count:long | message:keyword

multiIndexIpStringTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1022,7 +1022,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexIpStringTsLongDropped
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: to_date_nanos

FROM sample_data* METADATA _index
Expand Down Expand Up @@ -1064,7 +1064,7 @@ sample_data_ts_nanos | 8268153 | Connection error

multiIndexIpStringTsLongRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1107,7 +1107,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexIpStringTsLongRenameDropped
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: to_date_nanos

FROM sample_data* METADATA _index
Expand Down Expand Up @@ -1149,7 +1149,7 @@ sample_data_ts_nanos | 8268153 | Connection error

multiIndexIpStringTsLongRenameToString
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1192,7 +1192,7 @@ sample_data_ts_nanos | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233

multiIndexWhereIpStringTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1226,7 +1226,7 @@ count:long | message:keyword

multiIndexWhereIpStringLikeTsLong
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1260,7 +1260,7 @@ count:long | message:keyword

multiIndexMultiColumnTypesRename
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand All @@ -1279,7 +1279,7 @@ null | null | 8268153 | Connectio

multiIndexMultiColumnTypesRenameAndKeep
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand All @@ -1299,7 +1299,7 @@ sample_data_ts_nanos | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015123456

multiIndexMultiColumnTypesRenameAndDrop
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: union_types_remove_fields
required_capability: to_date_nanos

Expand Down Expand Up @@ -1591,7 +1591,7 @@ FROM sample_data, sample_data_ts_long

shortIntegerWidening
required_capability: union_types
required_capability: metadata_fields
required_capability: index_metadata_field
required_capability: casting_operator
required_capability: union_types_numeric_widening

Expand Down
7 changes: 7 additions & 0 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,13 @@ public enum Cap {

/** Capability for remote metadata test */
METADATA_FIELDS_REMOTE_TEST(false),

/**
* Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently
* supported.
*/
INDEX_METADATA_FIELD,

/**
* WIP on Join planning
* - Introduce BinaryPlan and co
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.DateEsField;
import org.elasticsearch.xpack.esql.core.type.EsField;
Expand Down Expand Up @@ -50,7 +51,7 @@

public class IndexResolver {
public static final Set<String> ALL_FIELDS = Set.of("*");
public static final Set<String> INDEX_METADATA_FIELD = Set.of("_index");
public static final Set<String> INDEX_METADATA_FIELD = Set.of(MetadataAttribute.INDEX);
public static final String UNMAPPED = "unmapped";

public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()
Expand Down
10 changes: 8 additions & 2 deletions ...ql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.elasticsearch.xpack.esql.core.expression.Attribute;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
Expand Down Expand Up @@ -289,6 +290,10 @@ private Block getBlockForMultiType(DocBlock indexDoc, MultiTypeEsField multiType
private Block extractBlockForSingleDoc(DocBlock docBlock, String columnName, TestBlockCopier blockCopier) {
var indexId = docBlock.asVector().shards().getInt(0);
var indexPage = indexPages.get(indexId);
if (MetadataAttribute.INDEX.equals(columnName)) {
return docBlock.blockFactory()
.newConstantBytesRefBlockWith(new BytesRef(indexPage.index), blockCopier.docIndices.getPositionCount());
}
int columnIndex = indexPage.columnIndex(columnName)
.orElseThrow(() -> new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", columnName, indexPage.columnNames));
var originalData = indexPage.page.getBlock(columnIndex);
Expand Down Expand Up @@ -408,8 +413,9 @@ private Block extractBlockForColumn(
TestBlockCopier blockCopier = mapToDocValues
? TestSpatialPointStatsBlockCopier.create(indexDoc.asVector().docs(), dataType)
: new TestBlockCopier(indexDoc.asVector().docs());
Block blockForIndex = extractBlock.apply(indexDoc, blockCopier);
blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
try (Block blockForIndex = extractBlock.apply(indexDoc, blockCopier)) {
blockBuilder.copyFrom(blockForIndex, 0, blockForIndex.getPositionCount());
}
});
var result = blockBuilder.build();
assert result.getPositionCount() == docBlock.getPositionCount()
Expand Down