Skip to content

Commit 8126546

Browse files
masseykemasseyke
authored
Merge branch 'main' into fail-data-stream-reindex-if-any-document-fails
2 parents 7d43186 + 01efd5e commit 8126546

File tree

21 files changed

+747
-788
lines changed

21 files changed

+747
-788
lines changed

‎docs/changelog/121556.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 121556
2+
summary: Enable New Semantic Text Format Only On Newly Created Indices
3+
area: Mapping
4+
type: bug
5+
issues: []

‎libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,10 @@
5252
* <p>
5353
* A bit like Mockito but way more painful.
5454
*/
55-
public class DummyImplementations {
55+
class DummyImplementations {
56+
57+
static class DummyLocaleServiceProvider extends LocaleServiceProvider {
5658

57-
public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
5859
@Override
5960
public Locale[] getAvailableLocales() {
6061
throw unexpected();

‎libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,6 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
9696

9797
private static final Map<String, CheckAction> checkActions = Stream.concat(
9898
Stream.<Entry<String, CheckAction>>of(
99-
entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
100-
entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
101-
entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
10299
entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
103100
entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
104101
entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -341,11 +338,6 @@ private static void systemExit() {
341338
System.exit(123);
342339
}
343340

344-
private static void staticMethodNeverEntitledViaReflection() throws Exception {
345-
Method systemExit = System.class.getMethod("exit", int.class);
346-
systemExit.invoke(null, 123);
347-
}
348-
349341
private static void createClassLoader() throws IOException {
350342
try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
351343
logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -356,11 +348,6 @@ private static void processBuilder_start() throws IOException {
356348
new ProcessBuilder("").start();
357349
}
358350

359-
private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
360-
Method processBuilderStart = ProcessBuilder.class.getMethod("start");
361-
processBuilderStart.invoke(new ProcessBuilder(""));
362-
}
363-
364351
private static void processBuilder_startPipeline() throws IOException {
365352
ProcessBuilder.startPipeline(List.of());
366353
}
@@ -399,10 +386,6 @@ private static void setHttpsConnectionProperties() {
399386
new DummyLocaleServiceProvider();
400387
}
401388

402-
private static void constructorNeverEntitledViaReflection() throws Exception {
403-
DummyLocaleServiceProvider.class.getConstructor().newInstance();
404-
}
405-
406389
private static void breakIteratorProvider$() {
407390
new DummyBreakIteratorProvider();
408391
}

‎muted-tests.yml‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -384,9 +384,6 @@ tests:
384384
- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
385385
method: testDependentVariableIsAliasToNested
386386
issue: https://github.com/elastic/elasticsearch/issues/121415
387-
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
388-
method: testLookupExplosionBigStringManyMatches
389-
issue: https://github.com/elastic/elasticsearch/issues/121465
390387
- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
391388
method: testClientSecretRotation
392389
issue: https://github.com/elastic/elasticsearch/issues/120985
@@ -396,9 +393,6 @@ tests:
396393
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
397394
method: test {yaml=cluster.health/10_basic/cluster health basic test}
398395
issue: https://github.com/elastic/elasticsearch/issues/121478
399-
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
400-
method: testLookupExplosionManyMatches
401-
issue: https://github.com/elastic/elasticsearch/issues/121481
402396
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
403397
method: testGetUsersWithProfileUid
404398
issue: https://github.com/elastic/elasticsearch/issues/121483
@@ -408,15 +402,6 @@ tests:
408402
- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
409403
method: testTransformLifecycle_RangeQueryThatMatchesNoShards
410404
issue: https://github.com/elastic/elasticsearch/issues/121480
411-
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
412-
method: testStopQueryLocal
413-
issue: https://github.com/elastic/elasticsearch/issues/121487
414-
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
415-
method: testSuccessfulPathways
416-
issue: https://github.com/elastic/elasticsearch/issues/121488
417-
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
418-
method: testAsyncQueriesWithLimit0
419-
issue: https://github.com/elastic/elasticsearch/issues/121489
420405
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
421406
method: testSuggestProfilesWithHint
422407
issue: https://github.com/elastic/elasticsearch/issues/121116
@@ -449,6 +434,9 @@ tests:
449434
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
450435
method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
451436
issue: https://github.com/elastic/elasticsearch/issues/121536
437+
- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryUnavailableRemotesIT
438+
method: testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue
439+
issue: https://github.com/elastic/elasticsearch/issues/121578
452440

453441
# Examples:
454442
#

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ static TransportVersion def(int id) {
172172
public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_0_00);
173173
public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00);
174174
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
175+
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01);
175176
public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
176177
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
177178
/*

‎server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.cluster.metadata.IndexMetadata;
1616
import org.elasticsearch.common.settings.Setting;
1717
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.index.IndexVersion;
1819
import org.elasticsearch.index.IndexVersions;
1920
import org.elasticsearch.index.query.SearchExecutionContext;
2021

@@ -41,6 +42,10 @@ public abstract class InferenceMetadataFieldsMapper extends MetadataFieldMapper
4142
Setting.Property.InternalIndex
4243
);
4344

45+
// Check index version SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP because that index version was added in the same serverless promotion
46+
// where the new format was enabled by default
47+
public static final IndexVersion USE_NEW_SEMANTIC_TEXT_FORMAT_BY_DEFAULT = IndexVersions.SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP;
48+
4449
public static final String NAME = "_inference_fields";
4550
public static final String CONTENT_TYPE = "_inference_fields";
4651

@@ -86,10 +91,12 @@ public abstract ValueFetcher valueFetcher(
8691
*/
8792
public static boolean isEnabled(Settings settings) {
8893
var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings);
89-
if (version.before(IndexVersions.INFERENCE_METADATA_FIELDS)
90-
&& version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) {
94+
if ((version.before(IndexVersions.INFERENCE_METADATA_FIELDS)
95+
&& version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false)
96+
|| (version.before(USE_NEW_SEMANTIC_TEXT_FORMAT_BY_DEFAULT) && USE_LEGACY_SEMANTIC_TEXT_FORMAT.exists(settings) == false)) {
9197
return false;
9298
}
99+
93100
return USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false;
94101
}
95102

‎test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,6 @@ public void testFetchMvLongs() throws IOException {
615615
assertMap(map, matchesMap().entry("columns", columns));
616616
}
617617

618-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106683")
619618
public void testFetchTooManyMvLongs() throws IOException {
620619
initMvLongsIndex(500, 100, 1000);
621620
assertCircuitBreaks(() -> fetchMvLongs());
@@ -636,7 +635,8 @@ public void testLookupExplosion() throws IOException {
636635

637636
public void testLookupExplosionManyMatches() throws IOException {
638637
assertCircuitBreaks(() -> {
639-
Map<?, ?> result = lookupExplosion(1500, 10000);
638+
// 1500, 10000 is enough locally, but some CI machines need more.
639+
Map<?, ?> result = lookupExplosion(2000, 10000);
640640
logger.error("should have failed but got {}", result);
641641
});
642642
}
@@ -664,7 +664,8 @@ public void testLookupExplosionBigString() throws IOException {
664664

665665
public void testLookupExplosionBigStringManyMatches() throws IOException {
666666
assertCircuitBreaks(() -> {
667-
Map<?, ?> result = lookupExplosionBigString(500, 1);
667+
// 500, 1 is enough to make it fail locally but some CI needs more
668+
Map<?, ?> result = lookupExplosionBigString(800, 1);
668669
logger.error("should have failed but got {}", result);
669670
});
670671
}

‎x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -786,6 +786,45 @@
786786
},
787787
"sampling": {
788788
"properties": {
789+
"tail": {
790+
"properties": {
791+
"dynamic_service_groups": {
792+
"type": "long"
793+
},
794+
"events": {
795+
"properties": {
796+
"dropped": {
797+
"type": "long"
798+
},
799+
"failed_writes": {
800+
"type": "long"
801+
},
802+
"head_unsampled": {
803+
"type": "long"
804+
},
805+
"processed": {
806+
"type": "long"
807+
},
808+
"sampled": {
809+
"type": "long"
810+
},
811+
"stored": {
812+
"type": "long"
813+
}
814+
}
815+
},
816+
"storage": {
817+
"properties": {
818+
"lsm_size": {
819+
"type": "long"
820+
},
821+
"value_log_size": {
822+
"type": "long"
823+
}
824+
}
825+
}
826+
}
827+
},
789828
"transactions_dropped": {
790829
"type": "long"
791830
}
@@ -2219,6 +2258,54 @@
22192258
},
22202259
"sampling": {
22212260
"properties": {
2261+
"tail": {
2262+
"properties": {
2263+
"dynamic_service_groups": {
2264+
"type": "alias",
2265+
"path": "beat.stats.apm_server.sampling.tail.dynamic_service_groups"
2266+
},
2267+
"events": {
2268+
"properties": {
2269+
"dropped": {
2270+
"type": "alias",
2271+
"path": "beat.stats.apm_server.sampling.tail.events.dropped"
2272+
},
2273+
"failed_writes": {
2274+
"type": "alias",
2275+
"path": "beat.stats.apm_server.sampling.tail.events.failed_writes"
2276+
},
2277+
"head_unsampled": {
2278+
"type": "alias",
2279+
"path": "beat.stats.apm_server.sampling.tail.events.head_unsampled"
2280+
},
2281+
"processed": {
2282+
"type": "alias",
2283+
"path": "beat.stats.apm_server.sampling.tail.events.processed"
2284+
},
2285+
"sampled": {
2286+
"type": "alias",
2287+
"path": "beat.stats.apm_server.sampling.tail.events.sampled"
2288+
},
2289+
"stored": {
2290+
"type": "alias",
2291+
"path": "beat.stats.apm_server.sampling.tail.events.stored"
2292+
}
2293+
}
2294+
},
2295+
"storage": {
2296+
"properties": {
2297+
"lsm_size": {
2298+
"type": "alias",
2299+
"path": "beat.stats.apm_server.sampling.tail.storage.lsm_size"
2300+
},
2301+
"value_log_size": {
2302+
"type": "alias",
2303+
"path": "beat.stats.apm_server.sampling.tail.storage.value_log_size"
2304+
}
2305+
}
2306+
}
2307+
}
2308+
},
22222309
"transactions_dropped": {
22232310
"type": "alias",
22242311
"path": "beat.stats.apm_server.sampling.transactions_dropped"

‎x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,45 @@
966966
},
967967
"sampling": {
968968
"properties": {
969+
"tail": {
970+
"properties": {
971+
"dynamic_service_groups": {
972+
"type": "long"
973+
},
974+
"events": {
975+
"properties": {
976+
"dropped": {
977+
"type": "long"
978+
},
979+
"failed_writes": {
980+
"type": "long"
981+
},
982+
"head_unsampled": {
983+
"type": "long"
984+
},
985+
"processed": {
986+
"type": "long"
987+
},
988+
"sampled": {
989+
"type": "long"
990+
},
991+
"stored": {
992+
"type": "long"
993+
}
994+
}
995+
},
996+
"storage": {
997+
"properties": {
998+
"lsm_size": {
999+
"type": "long"
1000+
},
1001+
"value_log_size": {
1002+
"type": "long"
1003+
}
1004+
}
1005+
}
1006+
}
1007+
},
9691008
"transactions_dropped": {
9701009
"type": "long"
9711010
}

0 commit comments

Comments
 (0)