Skip to content

Commit 1e53258

Browse files
DiannaHohenseeDiannaHohensee
committed
Merge branch 'main' into 2025/07/28/write-load-decider
2 parents 85f4448 + d557309 commit 1e53258

File tree

94 files changed

+1550
-538
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+1550
-538
lines changed

‎docs/changelog/132833.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132833
2+
summary: Adding simulate ingest effective mapping
3+
area: Ingest Node
4+
type: enhancement
5+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
4040
import org.elasticsearch.index.mapper.FieldMapper;
4141
import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues;
42+
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
4243
import org.elasticsearch.index.mapper.MapperBuilderContext;
4344
import org.elasticsearch.index.mapper.NumberFieldMapper;
4445
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
@@ -380,7 +381,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
380381
}
381382
// Multi fields don't have fallback synthetic source.
382383
if (isSyntheticSource && blContext.parentField(name()) == null) {
383-
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
384+
return new FallbackSyntheticSourceBlockLoader(
385+
fallbackSyntheticSourceBlockLoaderReader(),
386+
name(),
387+
IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated())
388+
) {
384389
@Override
385390
public Builder builder(BlockFactory factory, int expectedCount) {
386391
return factory.doubles(expectedCount);

‎muted-tests.yml‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -468,9 +468,6 @@ tests:
468468
- class: org.elasticsearch.xpack.ml.integration.TextEmbeddingQueryIT
469469
method: testHybridSearch
470470
issue: https://github.com/elastic/elasticsearch/issues/132703
471-
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
472-
method: test {yaml=indices.simulate_index_template/10_basic/Simulate index template with data stream with mapping and setting overrides}
473-
issue: https://github.com/elastic/elasticsearch/issues/132702
474471
- class: org.elasticsearch.xpack.ml.integration.RevertModelSnapshotIT
475472
method: testRevertModelSnapshot
476473
issue: https://github.com/elastic/elasticsearch/issues/132733
@@ -507,9 +504,39 @@ tests:
507504
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
508505
method: test {p0=search/410_named_queries/named_queries_with_score}
509506
issue: https://github.com/elastic/elasticsearch/issues/132906
510-
- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT
511-
method: test {p0=esql/230_folding/CIDR_MATCH with non-foldable CIDRs}
512-
issue: https://github.com/elastic/elasticsearch/issues/132931
507+
- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests
508+
method: test40VerifyAutogeneratedCredentials
509+
issue: https://github.com/elastic/elasticsearch/issues/132877
510+
- class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests
511+
method: test50CredentialAutogenerationOnlyOnce
512+
issue: https://github.com/elastic/elasticsearch/issues/132878
513+
- class: org.elasticsearch.upgrades.TransformSurvivesUpgradeIT
514+
method: testTransformRollingUpgrade
515+
issue: https://github.com/elastic/elasticsearch/issues/132892
516+
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
517+
method: testFetchCoerced
518+
issue: https://github.com/elastic/elasticsearch/issues/132893
519+
- class: org.elasticsearch.xpack.eql.planner.QueryTranslatorTests
520+
method: testMatchOptimization
521+
issue: https://github.com/elastic/elasticsearch/issues/132894
522+
- class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
523+
method: testUniqueDeprecationResponsesMergedTogether
524+
issue: https://github.com/elastic/elasticsearch/issues/132895
525+
- class: org.elasticsearch.search.CCSDuelIT
526+
method: testTermsAggs
527+
issue: https://github.com/elastic/elasticsearch/issues/132879
528+
- class: org.elasticsearch.search.CCSDuelIT
529+
method: testTermsAggsWithProfile
530+
issue: https://github.com/elastic/elasticsearch/issues/132880
531+
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
532+
method: testFetchMany
533+
issue: https://github.com/elastic/elasticsearch/issues/132948
534+
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
535+
method: testFetch
536+
issue: https://github.com/elastic/elasticsearch/issues/132956
537+
- class: org.elasticsearch.cluster.ClusterInfoServiceIT
538+
method: testMaxQueueLatenciesInClusterInfo
539+
issue: https://github.com/elastic/elasticsearch/issues/132957
513540

514541
# Examples:
515542
#

‎qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,6 @@ public void testFieldCollapsingSortByField() throws Exception {
670670
}
671671
}
672672

673-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40005")
674673
public void testTermsAggs() throws Exception {
675674
assumeMultiClusterSetup();
676675
{
@@ -685,7 +684,6 @@ public void testTermsAggs() throws Exception {
685684
}
686685
}
687686

688-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40005")
689687
public void testTermsAggsWithProfile() throws Exception {
690688
assumeMultiClusterSetup();
691689
{

‎qa/packaging/src/test/java/org/elasticsearch/packaging/test/ArchiveGenerateInitialCredentialsTests.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,6 @@ public void test30NoAutogenerationWhenDaemonized() throws Exception {
7878
stopElasticsearch();
7979
}
8080

81-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/84407")
8281
public void test40VerifyAutogeneratedCredentials() throws Exception {
8382
/* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
8483
assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);
@@ -97,7 +96,6 @@ public void test40VerifyAutogeneratedCredentials() throws Exception {
9796
stopElasticsearch();
9897
}
9998

100-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/84407")
10199
public void test50CredentialAutogenerationOnlyOnce() throws Exception {
102100
/* Windows issue awaits fix: https://github.com/elastic/elasticsearch/issues/49340 */
103101
assumeTrue("expect command isn't on Windows", distribution.platform != Distribution.Platform.WINDOWS);

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2182,3 +2182,131 @@ setup:
21822182
- match: { docs.0.doc._index: "test" }
21832183
- match: { docs.0.doc._source.foo: "bar" }
21842184
- match: { docs.0.doc.error.type: "document_parsing_exception" }
2185+
2186+
---
2187+
"Test effective mapping":
2188+
2189+
# This creates two templates, where the first reroutes to the second. Then we simulate ingesting and make sure that
2190+
# the effective_mapping is for the index where the document eventually would land. Also, the second index is really
2191+
# a data stream, so we expect to see a @timestamp field.
2192+
2193+
- skip:
2194+
features:
2195+
- headers
2196+
- allowed_warnings
2197+
2198+
- do:
2199+
headers:
2200+
Content-Type: application/json
2201+
ingest.put_pipeline:
2202+
id: "reroute-pipeline"
2203+
body: >
2204+
{
2205+
"processors": [
2206+
{
2207+
"reroute": {
2208+
"destination": "second-index"
2209+
}
2210+
}
2211+
]
2212+
}
2213+
- match: { acknowledged: true }
2214+
2215+
- do:
2216+
allowed_warnings:
2217+
- "index template [first-index-template] has index patterns [first-index*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [first-index-template] will take precedence during new index creation"
2218+
indices.put_index_template:
2219+
name: first-index-template
2220+
body:
2221+
index_patterns: first-index*
2222+
template:
2223+
settings:
2224+
default_pipeline: "reroute-pipeline"
2225+
mappings:
2226+
dynamic: strict
2227+
properties:
2228+
foo:
2229+
type: text
2230+
2231+
- do:
2232+
allowed_warnings:
2233+
- "index template [second-index-template] has index patterns [second-index*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [second-index-template] will take precedence during new index creation"
2234+
indices.put_index_template:
2235+
name: second-index-template
2236+
body:
2237+
index_patterns: second-index*
2238+
template:
2239+
mappings:
2240+
dynamic: strict
2241+
properties:
2242+
bar:
2243+
type: text
2244+
2245+
- do:
2246+
indices.put_index_template:
2247+
name: second-index-template
2248+
body:
2249+
index_patterns: second-index*
2250+
template:
2251+
lifecycle:
2252+
data_retention: "7d"
2253+
mappings:
2254+
dynamic: strict
2255+
properties:
2256+
bar:
2257+
type: text
2258+
data_stream: {}
2259+
2260+
- do:
2261+
indices.create_data_stream:
2262+
name: second-index
2263+
- is_true: acknowledged
2264+
2265+
- do:
2266+
cluster.health:
2267+
wait_for_status: yellow
2268+
2269+
- do:
2270+
indices.put_data_stream_mappings:
2271+
name: second-index
2272+
body:
2273+
properties:
2274+
foo:
2275+
type: boolean
2276+
2277+
- match: { data_streams.0.applied_to_data_stream: true }
2278+
2279+
# Here is the meat of the test. We simulate ingesting into first-index, knowing it will be rerouted to second-index,
2280+
# which is actually a data stream. So we expect the effective_mapping to contain the fields from second-index
2281+
# (including the implicit @timestamp field) and not second-index. Plus, it ought to include fields from the
2282+
# mapping_addition that we pass in.
2283+
- do:
2284+
headers:
2285+
Content-Type: application/json
2286+
simulate.ingest:
2287+
body: >
2288+
{
2289+
"docs": [
2290+
{
2291+
"_index": "first-index",
2292+
"_id": "id",
2293+
"_source": {
2294+
"foo": "bar"
2295+
}
2296+
}
2297+
],
2298+
"mapping_addition": {
2299+
"dynamic": "strict",
2300+
"properties": {
2301+
"baz": {
2302+
"type": "keyword"
2303+
}
2304+
}
2305+
}
2306+
}
2307+
- length: { docs: 1 }
2308+
- match: { docs.0.doc._index: "second-index" }
2309+
- not_exists: docs.0.doc.effective_mapping._doc.properties.foo
2310+
- match: { [email protected]: "date" }
2311+
- match: { docs.0.doc.effective_mapping._doc.properties.bar.type: "text" }
2312+
- match: { docs.0.doc.effective_mapping._doc.properties.baz.type: "keyword" }

‎server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ public void testMustAcceptDataLossToImportDanglingIndex() throws Exception {
176176
* other will be considered dangling, and can therefore be listed and
177177
* deleted through the API
178178
*/
179-
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108288")
180179
public void testDanglingIndexCanBeDeleted() throws Exception {
181180
final Settings settings = buildSettings(1, true);
182181
internalCluster().startNodes(3, settings);

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,7 @@ static TransportVersion def(int id) {
364364
public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_137_0_00);
365365
public static final TransportVersion SIMULATE_INGEST_MAPPING_MERGE_TYPE = def(9_138_0_00);
366366
public static final TransportVersion ESQL_LOOKUP_JOIN_ON_MANY_FIELDS = def(9_139_0_00);
367+
public static final TransportVersion SIMULATE_INGEST_EFFECTIVE_MAPPING = def(9_140_0_00);
367368

368369
/*
369370
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.elasticsearch.common.util.concurrent.AtomicArray;
3636
import org.elasticsearch.common.xcontent.XContentHelper;
3737
import org.elasticsearch.core.Nullable;
38-
import org.elasticsearch.core.Tuple;
3938
import org.elasticsearch.features.FeatureService;
4039
import org.elasticsearch.features.NodeFeature;
4140
import org.elasticsearch.index.IndexSettingProvider;
@@ -144,14 +143,13 @@ protected void doInternalExecute(
144143
DocWriteRequest<?> docRequest = bulkRequest.requests.get(i);
145144
assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests";
146145
IndexRequest request = (IndexRequest) docRequest;
147-
Tuple<Collection<String>, Exception> validationResult = validateMappings(
146+
ValidationResult validationResult = validateMappings(
148147
componentTemplateSubstitutions,
149148
indexTemplateSubstitutions,
150149
mappingAddition,
151150
request,
152151
mappingMergeReason
153152
);
154-
Exception mappingValidationException = validationResult.v2();
155153
responses.set(
156154
i,
157155
BulkItemResponse.success(
@@ -164,8 +162,9 @@ protected void doInternalExecute(
164162
request.source(),
165163
request.getContentType(),
166164
request.getExecutedPipelines(),
167-
validationResult.v1(),
168-
mappingValidationException
165+
validationResult.ignoredFields,
166+
validationResult.validationException,
167+
validationResult.effectiveMapping
169168
)
170169
)
171170
);
@@ -193,7 +192,7 @@ private MapperService.MergeReason getMergeReason(String mergeType) {
193192
* @return a Tuple containing: (1) in v1 the names of any fields that would be ignored upon indexing and (2) in v2 the mapping
194193
* exception if the source does not match the mappings, otherwise null
195194
*/
196-
private Tuple<Collection<String>, Exception> validateMappings(
195+
private ValidationResult validateMappings(
197196
Map<String, ComponentTemplate> componentTemplateSubstitutions,
198197
Map<String, ComposableIndexTemplate> indexTemplateSubstitutions,
199198
Map<String, Object> mappingAddition,
@@ -211,6 +210,7 @@ private Tuple<Collection<String>, Exception> validateMappings(
211210
);
212211

213212
ProjectMetadata project = projectResolver.getProjectMetadata(clusterService.state());
213+
CompressedXContent effectiveMapping = null;
214214
Exception mappingValidationException = null;
215215
Collection<String> ignoredFields = List.of();
216216
IndexAbstraction indexAbstraction = project.getIndicesLookup().get(request.index());
@@ -222,8 +222,8 @@ private Tuple<Collection<String>, Exception> validateMappings(
222222
*/
223223
IndexMetadata imd = project.getIndexSafe(indexAbstraction.getWriteIndex(request, project));
224224
CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
225-
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
226-
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse, mappingMergeReason);
225+
effectiveMapping = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
226+
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, effectiveMapping, request, sourceToParse, mappingMergeReason);
227227
} else {
228228
/*
229229
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -281,8 +281,8 @@ private Tuple<Collection<String>, Exception> validateMappings(
281281
indexSettingProviders
282282
);
283283
CompressedXContent mappings = template.mappings();
284-
CompressedXContent mergedMappings = mergeMappings(mappings, mappingAddition);
285-
ignoredFields = validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse, mappingMergeReason);
284+
effectiveMapping = mergeMappings(mappings, mappingAddition);
285+
ignoredFields = validateUpdatedMappings(mappings, effectiveMapping, request, sourceToParse, mappingMergeReason);
286286
} else {
287287
List<IndexTemplateMetadata> matchingTemplates = findV1Templates(simulatedProjectMetadata, request.index(), false);
288288
if (matchingTemplates.isEmpty() == false) {
@@ -295,23 +295,27 @@ private Tuple<Collection<String>, Exception> validateMappings(
295295
matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
296296
xContentRegistry
297297
);
298-
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
299-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse, mappingMergeReason);
298+
effectiveMapping = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
299+
ignoredFields = validateUpdatedMappings(null, effectiveMapping, request, sourceToParse, mappingMergeReason);
300300
} else {
301301
/*
302302
* The index matched no templates and had no mapping of its own. If there were component template substitutions
303303
* or index template substitutions, they didn't match anything. So just apply the mapping addition if it exists,
304304
* and validate.
305305
*/
306-
final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition);
307-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse, mappingMergeReason);
306+
effectiveMapping = mergeMappings(null, mappingAddition);
307+
ignoredFields = validateUpdatedMappings(null, effectiveMapping, request, sourceToParse, mappingMergeReason);
308308
}
309309
}
310310
}
311311
} catch (Exception e) {
312312
mappingValidationException = e;
313313
}
314-
return Tuple.tuple(ignoredFields, mappingValidationException);
314+
return new ValidationResult(effectiveMapping, mappingValidationException, ignoredFields);
315+
}
316+
317+
private record ValidationResult(CompressedXContent effectiveMapping, Exception validationException, Collection<String> ignoredFields) {
318+
315319
}
316320

317321
/*

0 commit comments

Comments
 (0)