From 9705810e08ce4320cc06d36a8d024de658026509 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 20 Aug 2025 22:04:10 +0200 Subject: [PATCH 1/2] Allow dynamic defaults in managed system indices System indices with non-dynamic mappings normally reject `put mapping` requests that attempt to modify the mapping of a managed index. However, for fields like `dense_vector` and `semantic_text`, some mapping options depend on the shape of the ingested data. The first ingestion often requires a dynamic update to set these defaults. To support this for system indices, this change marks the relevant `put mapping` requests as originating from bulk ingestion, which bypasses the system index mapping check. This is safe because: * such updates are already validated against invalid dynamic updates, and * an error would be raised earlier if a new field is added when `dynamic` is disabled. Fixes #133171 naming --- .../indices/create/CreateSystemIndicesIT.java | 20 +++++++----- .../indices/TestSystemIndexDescriptor.java | 32 +++++++++++++++++++ .../action/index/MappingUpdatedAction.java | 1 + 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 77c6d7d5dff12..645fbeb916ad7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -84,7 +84,7 @@ protected Collection> nodePlugins() { * settings when it is first used, when it is referenced via its alias. */ public void testSystemIndexIsAutoCreatedViaAlias() { - doCreateTest(() -> indexDoc(INDEX_NAME, "1", "foo", "bar"), PRIMARY_INDEX_NAME); + doCreateTest(() -> indexDoc(INDEX_NAME, "1", "foo", "bar"), PRIMARY_INDEX_NAME, false); } /** @@ -93,7 +93,7 @@ public void testSystemIndexIsAutoCreatedViaAlias() { * index name. */ public void testSystemIndexIsAutoCreatedViaConcreteName() { - doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "foo", "bar"), PRIMARY_INDEX_NAME); + doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "foo", "bar"), PRIMARY_INDEX_NAME, false); } /** @@ -144,7 +144,7 @@ public void testNonPrimarySystemIndexCreationThrowsError() { * settings when it is explicitly created, when it is referenced via its alias. */ public void testCreateSystemIndexViaAlias() { - doCreateTest(() -> assertAcked(prepareCreate(INDEX_NAME)), PRIMARY_INDEX_NAME); + doCreateTest(() -> assertAcked(prepareCreate(INDEX_NAME)), PRIMARY_INDEX_NAME, false); } /** @@ -153,7 +153,11 @@ public void testCreateSystemIndexViaAlias() { * concrete index name. */ public void testCreateSystemIndexViaConcreteName() { - doCreateTest(() -> assertAcked(prepareCreate(PRIMARY_INDEX_NAME)), PRIMARY_INDEX_NAME); + doCreateTest(() -> assertAcked(prepareCreate(PRIMARY_INDEX_NAME)), PRIMARY_INDEX_NAME, false); + } + + public void testSystemIndexIsAutoCreatedWithDynamicDefault() { + doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "vector", new int[] {1, 2,3}), PRIMARY_INDEX_NAME, true); } private void createSystemAliasViaV1Template(String indexName, String primaryIndexName) throws Exception { @@ -265,12 +269,12 @@ public void testCreateSystemAliasViaComposableTemplateWithAllowsTemplates() thro ); } - private void doCreateTest(Runnable runnable, String concreteIndex) { + private void doCreateTest(Runnable runnable, String concreteIndex, boolean expandVectorDefault) { // Trigger the creation of the system index runnable.run(); ensureGreen(INDEX_NAME); - assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings(), concreteIndex); + assertMappingsAndSettings(TestSystemIndexDescriptor.getOldMappings(expandVectorDefault), concreteIndex); // Remove the index and alias... assertAcked(indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).removeAlias(concreteIndex, INDEX_NAME).get()); @@ -283,7 +287,7 @@ private void doCreateTest(Runnable runnable, String concreteIndex) { runnable.run(); ensureGreen(INDEX_NAME); - assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings(), concreteIndex); + assertMappingsAndSettings(TestSystemIndexDescriptor.getNewMappings(expandVectorDefault), concreteIndex); assertAliases(concreteIndex); } @@ -362,7 +366,7 @@ private void assertMappingsAndSettings(String expectedMappings, String concreteI ); final Map sourceAsMap = mappings.get(concreteIndex).getSourceAsMap(); - assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false))); + assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, true))); final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings( new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index 03e7660d6d46e..3b70ec87cfda2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -97,6 +97,10 @@ public MappingsVersion getMappingsVersion() { } public static String getOldMappings() { + return getOldMappings(false); + } + + public static String getOldMappings(boolean includeVectorDims) { try { final XContentBuilder builder = jsonBuilder(); @@ -112,6 +116,18 @@ public static String getOldMappings() { builder.startObject("foo"); builder.field("type", "text"); builder.endObject(); + + builder.startObject("vector"); + builder.field("type", "dense_vector"); + if (includeVectorDims) { + builder.field("dims", 3); + } + builder.field("index", true); + builder.field("similarity", "cosine"); + builder.startObject("index_options"); + builder.field("type", "flat"); + builder.endObject(); + builder.endObject(); } builder.endObject(); } @@ -124,6 +140,10 @@ public static String getOldMappings() { } public static String getNewMappings() { + return getNewMappings(false); + } + + public static String getNewMappings(boolean includeVectorDims) { try { final XContentBuilder builder = jsonBuilder(); @@ -142,6 +162,18 @@ public static String getNewMappings() { builder.startObject("foo"); builder.field("type", "text"); builder.endObject(); + + builder.startObject("vector"); + builder.field("type", "dense_vector"); + if (includeVectorDims) { + builder.field("dims", 3); + } + builder.field("index", true); + builder.field("similarity", "cosine"); + builder.startObject("index_options"); + builder.field("type", "flat"); + builder.endObject(); + builder.endObject(); } builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java index 4152bd422b159..25a1a0cd561eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/index/MappingUpdatedAction.java @@ -111,6 +111,7 @@ protected void sendUpdateMapping(Index index, Mapping mappingUpdate, ActionListe putMappingRequest.source(mappingUpdate.toString(), XContentType.JSON); putMappingRequest.masterNodeTimeout(dynamicMappingUpdateTimeout); putMappingRequest.ackTimeout(TimeValue.ZERO); + putMappingRequest.origin("bulk"); client.execute( TransportAutoPutMappingAction.TYPE, putMappingRequest, From f92d55c89ae4fa9212cfff9fff3fc658ebb91fa2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 20 Aug 2025 21:09:34 +0000 Subject: [PATCH 2/2] [CI] Auto commit changes from spotless --- .../action/admin/indices/create/CreateSystemIndicesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 645fbeb916ad7..765df53c9031a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -157,7 +157,7 @@ public void testCreateSystemIndexViaConcreteName() { } public void testSystemIndexIsAutoCreatedWithDynamicDefault() { - doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "vector", new int[] {1, 2,3}), PRIMARY_INDEX_NAME, true); + doCreateTest(() -> indexDoc(PRIMARY_INDEX_NAME, "1", "vector", new int[] { 1, 2, 3 }), PRIMARY_INDEX_NAME, true); } private void createSystemAliasViaV1Template(String indexName, String primaryIndexName) throws Exception {