Merge branch 'main' into skippers/always-on-for-index-false

Author: romseygeek

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
+distributionSha256Sum=f86344275d1b194688dd330abf9f6f2344cd02872ffee035f2d1ea2fd60cf7f3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-9.2.0
+9.2.1

docs/changelog/138624.yaml

@@ -0,0 +1,6 @@
+pr: 138624
+summary: Handle individual doc parsing failure in bulk request with pipeline
+area: Ingest Node
+type: bug
+issues:
+ - 138445

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
+distributionSha256Sum=f86344275d1b194688dd330abf9f6f2344cd02872ffee035f2d1ea2fd60cf7f3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DerParser.java

@@ -138,7 +138,12 @@ private int getLength() throws IOException {
         int n = derInputStream.read(bytes);
         if (n < num) throw new IOException("Invalid DER: length too short");
 
-        return new BigInteger(1, bytes).intValue();
+        int len = new BigInteger(1, bytes).intValue();
+        if (len < 0) {
+            throw new IOException("Invalid DER: length larger than max-int");
+        }
+
+        return len;
     }
 
     /**

modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
@@ -44,6 +45,11 @@ public class ReindexPlugin extends Plugin implements ActionPlugin {
 
     public static final ActionType<ListTasksResponse> RETHROTTLE_ACTION = new ActionType<>("cluster:admin/reindex/rethrottle");
 
+    /**
+     * Whether the feature flag to guard the work to make reindex more resilient while it is under development.
+     */
+    static boolean REINDEX_RESILIENCE_ENABLED = new FeatureFlag("reindex_resilience").isEnabled();
+
     @Override
     public List<ActionHandler> getActions() {
         return Arrays.asList(

modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java

@@ -50,6 +50,7 @@
 import java.util.stream.IntStream;
 
 import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.bytes.BytesReferenceTestUtils.equalBytes;
 import static org.elasticsearch.core.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -174,7 +175,7 @@ private SearchResponse createSearchResponse() {
     private void assertSameHits(List<? extends ScrollableHitSource.Hit> actual, SearchHit[] expected) {
         assertEquals(actual.size(), expected.length);
         for (int i = 0; i < actual.size(); ++i) {
-            assertEquals(actual.get(i).getSource(), expected[i].getSourceRef());
+            assertThat(expected[i].getSourceRef(), equalBytes(actual.get(i).getSource()));
             assertEquals(actual.get(i).getIndex(), expected[i].getIndex());
             assertEquals(actual.get(i).getVersion(), expected[i].getVersion());
             assertEquals(actual.get(i).getPrimaryTerm(), expected[i].getPrimaryTerm());

modules/reindex/src/test/java/org/elasticsearch/reindex/RoundTripTests.java

@@ -38,6 +38,7 @@
 import java.util.Map;
 
 import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.bytes.BytesReferenceTestUtils.equalBytes;
 
 /**
  * Round trip tests for all {@link Writeable} things declared in this plugin.
@@ -144,7 +145,7 @@ private void assertRequestEquals(ReindexRequest request, ReindexRequest tripped)
             assertNotNull(tripped.getRemoteInfo());
             assertEquals(request.getRemoteInfo().getScheme(), tripped.getRemoteInfo().getScheme());
             assertEquals(request.getRemoteInfo().getHost(), tripped.getRemoteInfo().getHost());
-            assertEquals(request.getRemoteInfo().getQuery(), tripped.getRemoteInfo().getQuery());
+            assertThat(tripped.getRemoteInfo().getQuery(), equalBytes(request.getRemoteInfo().getQuery()));
             assertEquals(request.getRemoteInfo().getUsername(), tripped.getRemoteInfo().getUsername());
             assertEquals(request.getRemoteInfo().getPassword(), tripped.getRemoteInfo().getPassword());
             assertEquals(request.getRemoteInfo().getHeaders(), tripped.getRemoteInfo().getHeaders());

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -59,6 +59,7 @@
 import java.util.Collections;
 import java.util.Map;
 
+import static org.elasticsearch.common.bytes.BytesReferenceTestUtils.equalBytes;
 import static org.elasticsearch.common.io.Streams.readFully;
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING;
@@ -219,7 +220,7 @@ public void testWriteFileMultipleOfChunkSize() throws IOException {
             container.writeBlob(randomPurpose(), key, new BytesArray(initialValue), true);
 
             BytesReference reference = readFully(container.readBlob(randomPurpose(), key));
-            assertEquals(new BytesArray(initialValue), reference);
+            assertThat(reference, equalBytes(new BytesArray(initialValue)));
 
             container.deleteBlobsIgnoringIfNotExists(randomPurpose(), Iterators.single(key));
         }

modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java

@@ -79,6 +79,7 @@
 
 import static fixture.gcs.TestUtils.createServiceAccount;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.elasticsearch.common.bytes.BytesReferenceTestUtils.equalBytes;
 import static org.elasticsearch.common.io.Streams.readFully;
 import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
 import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.randomBytes;
@@ -582,7 +583,7 @@ public void testCompareAndExchangeWhenThrottled() throws IOException {
         final OptionalBytesReference updateResult = safeAwait(
             l -> container.compareAndExchangeRegister(randomPurpose(), key, new BytesArray(data), new BytesArray(updatedData), l)
         );
-        assertEquals(new BytesArray(data), updateResult.bytesReference());
+        assertThat(updateResult.bytesReference(), equalBytes(new BytesArray(data)));
 
         assertEquals(0, requestHandlers.size());
         container.delete(randomPurpose());
@@ -601,7 +602,7 @@ public void testContentsChangeWhileStreaming() throws IOException {
         container.writeBlob(randomPurpose(), key, new BytesArray(initialValue), true);
 
         BytesReference reference = readFully(container.readBlob(randomPurpose(), key));
-        assertEquals(new BytesArray(initialValue), reference);
+        assertThat(reference, equalBytes(new BytesArray(initialValue)));
 
         try (InputStream inputStream = container.readBlob(randomPurpose(), key)) {
             // Trigger the first chunk to load