Skip to content

Commit 4afee66

Browse files
bpinteabpintea
authored
Merge branch 'main' into esql/ds/up_fault_limit
2 parents 5dae9d8 + 5d29b03 commit 4afee66

File tree

27 files changed

+624
-153
lines changed

27 files changed

+624
-153
lines changed

‎docs/reference/elasticsearch/rest-apis/api-conventions.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,10 +308,10 @@ You can also exclude clusters from a list of clusters to search using the `-` ch
308308
Multi-target APIs that can target indices support the following query string parameters:
309309

310310
`ignore_unavailable`
311-
: (Optional, Boolean) If `false`, the request returns an error if it targets a missing or closed index. Defaults to `false`.
311+
: (Optional, Boolean) If `false`, the request returns an error if it targets a concrete (non-wildcarded) index, alias, or data stream that is missing, closed, or otherwise unavailable. If `true`, unavailable concrete targets are silently ignored. Defaults to `false`.
312312

313313
`allow_no_indices`
314-
: (Optional, Boolean) If `false`, the request returns an error if any wildcard expression, [index alias](docs-content://manage-data/data-store/aliases.md), or `_all` value targets only missing or closed indices. This behavior applies even if the request targets other open indices. For example, a request targeting `foo*,bar*` returns an error if an index starts with `foo` but no index starts with `bar`.
314+
: (Optional, Boolean) If `false`, the request returns an error (1) if any wildcard expression (including `_all` and `*`) resolves to zero matching indices or (2) if the complete set of resolved indices, [aliases](docs-content://manage-data/data-store/aliases.md) or data streams is empty after all expressions are evaluated. If `true`, index expressions that resolve to no indices are allowed and the request returns an empty result.
315315

316316
`expand_wildcards`
317317
: (Optional, string) Type of index that wildcard patterns can match. If the request can target data streams, this argument determines whether wildcard expressions match hidden data streams. Supports comma-separated values, such as `open,hidden`. Valid values are:

‎server/src/main/java/org/elasticsearch/index/codec/CodecService.java‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.apache.lucene.codecs.FilterCodec;
1515
import org.apache.lucene.codecs.lucene104.Lucene104Codec;
1616
import org.elasticsearch.common.util.BigArrays;
17-
import org.elasticsearch.common.util.FeatureFlag;
1817
import org.elasticsearch.core.Nullable;
1918
import org.elasticsearch.index.codec.tsdb.ES93TSDBDefaultCompressionLucene103Codec;
2019
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
@@ -33,8 +32,6 @@
3332
*/
3433
public class CodecService implements CodecProvider {
3534

36-
public static final boolean ZSTD_STORED_FIELDS_FEATURE_FLAG = new FeatureFlag("zstd_stored_fields").isEnabled();
37-
3835
private final Map<String, Codec> codecs;
3936

4037
public static final String DEFAULT_CODEC = "default";
@@ -50,19 +47,15 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays,
5047

5148
boolean useSyntheticId = mapperService != null && mapperService.getIndexSettings().useTimeSeriesSyntheticId();
5249

53-
var legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, bigArrays, threadPool);
50+
var bestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene104Codec.Mode.BEST_SPEED, mapperService, bigArrays, threadPool);
5451
if (useSyntheticId) {
5552
// Use the default Lucene compression when the synthetic id is used even if the ZSTD feature flag is enabled
56-
codecs.put(DEFAULT_CODEC, new ES93TSDBDefaultCompressionLucene103Codec(legacyBestSpeedCodec));
57-
} else if (ZSTD_STORED_FIELDS_FEATURE_FLAG) {
58-
codecs.put(
59-
DEFAULT_CODEC,
60-
new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays, threadPool)
61-
);
53+
codecs.put(DEFAULT_CODEC, new ES93TSDBDefaultCompressionLucene103Codec(bestSpeedCodec));
6254
} else {
63-
codecs.put(DEFAULT_CODEC, legacyBestSpeedCodec);
55+
codecs.put(DEFAULT_CODEC, bestSpeedCodec);
6456
}
65-
codecs.put(LEGACY_DEFAULT_CODEC, legacyBestSpeedCodec);
57+
// We can't remove this now
58+
codecs.put(LEGACY_DEFAULT_CODEC, bestSpeedCodec);
6659

6760
codecs.put(
6861
BEST_COMPRESSION_CODEC,

‎server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99

1010
package org.elasticsearch.index.codec;
1111

12+
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
1213
import org.elasticsearch.common.settings.Settings;
1314
import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
1415
import org.elasticsearch.test.ESSingleNodeTestCase;
1516

17+
import static org.elasticsearch.index.codec.CodecTests.getLucene90StoredFieldsFormatMode;
1618
import static org.hamcrest.Matchers.equalTo;
1719

1820
public class CodecIntegrationTests extends ESSingleNodeTestCase {
@@ -47,15 +49,14 @@ public void testDefaultCodecLogsdb() {
4749
assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION));
4850
}
4951

50-
public void testDefaultCodec() {
51-
assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG);
52-
52+
public void testDefaultCodec() throws Exception {
5353
var indexService = createIndex("index1");
54-
var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0)
54+
var storedFieldsFormat = (Lucene90StoredFieldsFormat) indexService.getShard(0)
5555
.getEngineOrNull()
5656
.config()
5757
.getCodec()
5858
.storedFieldsFormat();
59-
assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_SPEED));
59+
var mode = getLucene90StoredFieldsFormatMode(storedFieldsFormat);
60+
assertThat(mode, equalTo(Lucene90StoredFieldsFormat.Mode.BEST_SPEED));
6061
}
6162
}

‎server/src/test/java/org/elasticsearch/index/codec/CodecTests.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.TransportVersion;
2323
import org.elasticsearch.common.settings.Settings;
2424
import org.elasticsearch.common.util.BigArrays;
25+
import org.elasticsearch.core.SuppressForbidden;
2526
import org.elasticsearch.env.Environment;
2627
import org.elasticsearch.index.IndexMode;
2728
import org.elasticsearch.index.IndexSettings;
@@ -50,19 +51,18 @@
5051
public class CodecTests extends ESTestCase {
5152

5253
public void testResolveDefaultCodecs() throws Exception {
53-
assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG);
5454
CodecService codecService = createCodecService();
55-
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
56-
assertThat(codecService.codec("default"), instanceOf(Elasticsearch93Lucene104Codec.class));
55+
var codec = codecService.codec("default");
56+
assertThat(codec, instanceOf(CodecService.DeduplicateFieldInfosCodec.class));
57+
codec = ((CodecService.DeduplicateFieldInfosCodec) codec).delegate();
58+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
5759
}
5860

5961
public void testDefault() throws Exception {
60-
assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG);
6162
Codec codec = createCodecService().codec("default");
62-
assertEquals(
63-
"Zstd814StoredFieldsFormat(compressionMode=ZSTD(level=1), chunkSize=14336, maxDocsPerChunk=128, blockShift=10)",
64-
codec.storedFieldsFormat().toString()
65-
);
63+
Lucene90StoredFieldsFormat storedFieldsFormat = (Lucene90StoredFieldsFormat) codec.storedFieldsFormat();
64+
var mode = getLucene90StoredFieldsFormatMode(storedFieldsFormat);
65+
assertEquals(Lucene90StoredFieldsFormat.Mode.BEST_SPEED, mode);
6666
}
6767

6868
public void testTSDBDefault() throws Exception {
@@ -178,4 +178,12 @@ private CodecService createCodecService(boolean syntheticIdEnabled) throws IOExc
178178
return new CodecService(service, BigArrays.NON_RECYCLING_INSTANCE, null);
179179
}
180180

181+
@SuppressForbidden(reason = "access violation required in order to read private field for this test")
182+
static Lucene90StoredFieldsFormat.Mode getLucene90StoredFieldsFormatMode(Lucene90StoredFieldsFormat storedFieldsFormat)
183+
throws NoSuchFieldException, IllegalAccessException {
184+
var modeField = Lucene90StoredFieldsFormat.class.getDeclaredField("mode");
185+
modeField.setAccessible(true);
186+
return (Lucene90StoredFieldsFormat.Mode) modeField.get(storedFieldsFormat);
187+
}
188+
181189
}

‎server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
import org.elasticsearch.index.analysis.NamedAnalyzer;
4242
import org.elasticsearch.index.codec.CodecService;
4343
import org.elasticsearch.index.codec.LegacyPerFieldMapperCodec;
44-
import org.elasticsearch.index.codec.PerFieldMapperCodec;
4544
import org.elasticsearch.xcontent.ToXContent;
4645
import org.elasticsearch.xcontent.XContentBuilder;
4746
import org.elasticsearch.xcontent.XContentFactory;
@@ -154,16 +153,11 @@ public void testPostingsFormat() throws IOException {
154153
MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
155154
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
156155
Codec codec = codecService.codec("default");
157-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
158-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
159-
assertThat(((PerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(latestLuceneCPClass));
160-
} else {
161-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
162-
codec = deduplicateFieldInfosCodec.delegate();
163-
}
164-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
165-
assertThat(((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(latestLuceneCPClass));
156+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
157+
codec = deduplicateFieldInfosCodec.delegate();
166158
}
159+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
160+
assertThat(((LegacyPerFieldMapperCodec) codec).getPostingsFormatForField("field"), instanceOf(latestLuceneCPClass));
167161
}
168162

169163
public void testDefaultConfiguration() throws IOException {

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java‎

Lines changed: 20 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.elasticsearch.index.IndexVersions;
3434
import org.elasticsearch.index.codec.CodecService;
3535
import org.elasticsearch.index.codec.LegacyPerFieldMapperCodec;
36-
import org.elasticsearch.index.codec.PerFieldMapperCodec;
3736
import org.elasticsearch.index.codec.vectors.BFloat16;
3837
import org.elasticsearch.index.codec.vectors.diskbbq.es94.ES940DiskBBQVectorsFormat;
3938
import org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat;
@@ -1974,17 +1973,11 @@ public void testKnnVectorsFormat() throws IOException {
19741973
);
19751974
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
19761975
Codec codec = codecService.codec("default");
1977-
KnnVectorsFormat knnVectorsFormat;
1978-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
1979-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
1980-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
1981-
} else {
1982-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
1983-
codec = deduplicateFieldInfosCodec.delegate();
1984-
}
1985-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
1986-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
1976+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
1977+
codec = deduplicateFieldInfosCodec.delegate();
19871978
}
1979+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
1980+
KnnVectorsFormat knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
19881981
String expectedString = "ES93HnswVectorsFormat(name=ES93HnswVectorsFormat, maxConn="
19891982
+ (setM ? m : DEFAULT_MAX_CONN)
19901983
+ ", beamWidth="
@@ -2049,17 +2042,11 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException {
20492042
);
20502043
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
20512044
Codec codec = codecService.codec("default");
2052-
KnnVectorsFormat knnVectorsFormat;
2053-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
2054-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
2055-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2056-
} else {
2057-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2058-
codec = deduplicateFieldInfosCodec.delegate();
2059-
}
2060-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2061-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2045+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2046+
codec = deduplicateFieldInfosCodec.delegate();
20622047
}
2048+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2049+
KnnVectorsFormat knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
20632050
VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
20642051
String encoding = quantizedFlatFormat.equals("int4_flat") ? "PACKED_NIBBLE" : "SEVEN_BIT";
20652052
assertThat(
@@ -2100,17 +2087,11 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException {
21002087
);
21012088
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
21022089
Codec codec = codecService.codec("default");
2103-
KnnVectorsFormat knnVectorsFormat;
2104-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
2105-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
2106-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2107-
} else {
2108-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2109-
codec = deduplicateFieldInfosCodec.delegate();
2110-
}
2111-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2112-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2090+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2091+
codec = deduplicateFieldInfosCodec.delegate();
21132092
}
2093+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2094+
KnnVectorsFormat knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
21142095
VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
21152096
assertThat(
21162097
knnVectorsFormat,
@@ -2157,17 +2138,11 @@ public void testKnnBBQHNSWVectorsFormat() throws IOException {
21572138
);
21582139
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
21592140
Codec codec = codecService.codec("default");
2160-
KnnVectorsFormat knnVectorsFormat;
2161-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
2162-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
2163-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2164-
} else {
2165-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2166-
codec = deduplicateFieldInfosCodec.delegate();
2167-
}
2168-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2169-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2141+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2142+
codec = deduplicateFieldInfosCodec.delegate();
21702143
}
2144+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2145+
KnnVectorsFormat knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
21712146
String expectedPrefix = "ES93HnswBinaryQuantizedVectorsFormat(name=ES93HnswBinaryQuantizedVectorsFormat, maxConn="
21722147
+ m
21732148
+ ", beamWidth="
@@ -2213,17 +2188,11 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException {
22132188
}));
22142189
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE, null);
22152190
Codec codec = codecService.codec("default");
2216-
KnnVectorsFormat knnVectorsFormat;
2217-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
2218-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
2219-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2220-
} else {
2221-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2222-
codec = deduplicateFieldInfosCodec.delegate();
2223-
}
2224-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2225-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2191+
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2192+
codec = deduplicateFieldInfosCodec.delegate();
22262193
}
2194+
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2195+
KnnVectorsFormat knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
22272196
VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null);
22282197
assertThat(
22292198
knnVectorsFormat,

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public enum FeatureFlag {
2828
RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null),
2929
TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
3030
ESQL_VIEWS("es.esql_views_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
31+
ESQL_EXTERNAL_DATASOURCES("es.esql_external_datasources_feature_flag_enabled=true", Version.fromString("9.4.0"), null),
3132
EXTENDED_DOC_VALUES_PARAMS("es.extended_doc_values_options_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
3233
TSDB_NO_SEQNO("es.tsdb_no_tsbd_feature_flag_enabled=true", Version.fromString("9.4.0"), null),
3334
IGNORED_SOURCE_AS_DOC_VALUES_FF("es.ignored_source_as_doc_values_feature_flag_enabled=true", Version.fromString("9.4.0"), null);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/esql/EsqlFeatureFlags.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@ public class EsqlFeatureFlags {
1717
* A feature flag to enable ESQL views REST API functionality.
1818
*/
1919
public static final FeatureFlag ESQL_VIEWS_FEATURE_FLAG = new FeatureFlag("esql_views");
20+
21+
/**
22+
* A feature flag to enable the EXTERNAL command and external data source access in ES|QL.
23+
*/
24+
public static final FeatureFlag ESQL_EXTERNAL_DATASOURCES_FEATURE_FLAG = new FeatureFlag("esql_external_datasources");
2025
}

0 commit comments

Comments
 (0)