Merge remote-tracking branch 'upstream/main' into 09162025/ReshardSplitRequestOnSourceTwoPass

Refresh

Author: ankikuma

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/ResolveTransportVersionConflictFuncTest.groovy

@@ -87,4 +87,13 @@ class ResolveTransportVersionConflictFuncTest extends AbstractTransportVersionFu
         assertUpperBound("9.1", "new_tv,8012002")
         assertUpperBound("8.19", "new_tv,7123002")
     }
+
+    def "no new transport version is idempotent"() {
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertUpperBound("9.2", "existing_92,8123000")
+    }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -428,7 +428,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
 
     def "upper bounds files must exist for backport branches"() {
         when:
-        def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
+        def result = runGenerateTask("--name", "new_tv", "--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
 
         then:
         assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -294,4 +294,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "only current upper bound validated on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.0'
+            }
+        """
+        referableAndReferencedTransportVersion("some_tv", "8124000,8012004")
+        transportVersionUpperBound("9.1", "some_tv", "8012004")
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources").build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -98,14 +98,14 @@ public void run() throws IOException {
         List<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
         String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
 
-        List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
-        Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
-
         getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
         if (targetDefinitionName.isEmpty()) {
             // TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
             // resetAllUpperBounds(resources);
         } else {
+            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
+            Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
+
             List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
             // (Re)write the definition file.
             resources.writeDefinition(new TransportVersionDefinition(targetDefinitionName, ids, true));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -83,6 +83,7 @@ public void validateTransportVersions() throws IOException {
         Map<String, TransportVersionDefinition> allDefinitions = collectAllDefinitions(referableDefinitions, unreferableDefinitions);
         Map<Integer, List<IdAndDefinition>> idsByBase = collectIdsByBase(allDefinitions.values());
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
+        TransportVersionUpperBound currentUpperBound = upperBounds.get(getCurrentUpperBoundName().get());
         boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
 
         for (var definition : referableDefinitions.values()) {
@@ -94,14 +95,21 @@ public void validateTransportVersions() throws IOException {
         }
 
         for (var entry : idsByBase.entrySet()) {
-            validateBase(entry.getKey(), entry.getValue());
+            int baseId = entry.getKey();
+            // on main we validate all bases, but on release branches we only validate up to the current upper bound
+            if (onReleaseBranch == false || baseId <= currentUpperBound.definitionId().base()) {
+                validateBase(baseId, entry.getValue());
+            }
         }
 
-        for (var upperBound : upperBounds.values()) {
-            validateUpperBound(upperBound, allDefinitions, idsByBase);
-        }
+        if (onReleaseBranch) {
+            // on release branches we only check the current upper bound, others may be inaccurate
+            validateUpperBound(currentUpperBound, allDefinitions, idsByBase);
+        } else {
+            for (var upperBound : upperBounds.values()) {
+                validateUpperBound(upperBound, allDefinitions, idsByBase);
+            }
 
-        if (onReleaseBranch == false) {
             validatePrimaryIds(resources, upperBounds, allDefinitions);
         }
     }

docs/changelog/134585.yaml

@@ -0,0 +1,5 @@
+pr: 134585
+summary: Add classes to represent raw docs sampling configs
+area: Ingest Node
+type: feature
+issues: []

docs/changelog/135309.yaml

@@ -0,0 +1,5 @@
+pr: 135309
+summary: Enable semantic search CCS when ccs_minimize_roundtrips=true
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/135479.yaml

@@ -0,0 +1,6 @@
+pr: 135479
+summary: Correctly apply field path to JSON processor when adding contents to document
+  root
+area: Ingest Node
+type: bug
+issues: []

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java

@@ -149,6 +149,10 @@ public static void apply(
         boolean strictJsonParsing
     ) {
         Object value = apply(ctx.get(fieldName), allowDuplicateKeys, strictJsonParsing);
+        mergeParsedJson(ctx, value, conflictStrategy);
+    }
+
+    private static void mergeParsedJson(Map<String, Object> ctx, Object value, ConflictStrategy conflictStrategy) {
         if (value instanceof Map) {
             @SuppressWarnings("unchecked")
             Map<String, Object> map = (Map<String, Object>) value;
@@ -184,10 +188,11 @@ public static void recursiveMerge(Map<String, Object> target, Map<String, Object
 
     @Override
     public IngestDocument execute(IngestDocument document) throws Exception {
+        Object value = apply(document.getFieldValue(field, Object.class), allowDuplicateKeys, strictJsonParsing);
         if (addToRoot) {
-            apply(document.getSourceAndMetadata(), field, allowDuplicateKeys, addToRootConflictStrategy, strictJsonParsing);
+            mergeParsedJson(document.getSourceAndMetadata(), value, addToRootConflictStrategy);
         } else {
-            document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class), allowDuplicateKeys, strictJsonParsing));
+            document.setFieldValue(targetField, value);
         }
         return document;
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.IngestPipelineTestUtils;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -166,6 +167,28 @@ public void testAddToRoot() throws Exception {
         assertEquals("see", sourceAndMetadata.get("c"));
     }
 
+    public void testAddToRootNestedField() throws Exception {
+        String processorTag = randomAlphaOfLength(3);
+        String randomTargetField = randomAlphaOfLength(2);
+        JsonProcessor jsonProcessor = new JsonProcessor(processorTag, null, "a.b", randomTargetField, true, REPLACE, false);
+
+        String json = "{\"a\": 1, \"b\": 2}";
+        Map<String, Object> subfield = new HashMap<>();
+        subfield.put("b", json);
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("a", subfield);
+        document.put("c", "see");
+
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        ingestDocument = IngestPipelineTestUtils.runWithRandomAccessPattern(ingestDocument, jsonProcessor);
+
+        Map<String, Object> sourceAndMetadata = ingestDocument.getSourceAndMetadata();
+        assertEquals(1, sourceAndMetadata.get("a"));
+        assertEquals(2, sourceAndMetadata.get("b"));
+        assertEquals("see", sourceAndMetadata.get("c"));
+    }
+
     public void testDuplicateKeys() throws Exception {
         String processorTag = randomAlphaOfLength(3);
         JsonProcessor lenientJsonProcessor = new JsonProcessor(processorTag, null, "a", null, true, REPLACE, true);