Merge branch 'main' into 2025/04/25/S3RepositoryAnalysisRestIT-failure

Author: DaveCTurner

docs/changelog/127259.yaml

@@ -21,7 +21,6 @@
 import org.apache.lucene.tests.analysis.CannedTokenStream;
 import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -44,7 +43,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
@@ -257,9 +255,4 @@ public void testDocValuesLoadedFromSynthetic() throws IOException {
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
-
-    @Override
-    protected Function<Object, Object> loadBlockExpected() {
-        return v -> ((BytesRef) v).utf8ToString();
-    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -377,7 +377,6 @@ public SyntheticSourceExample example(int maxVals) {
                 return new SyntheticSourceExample(
                     example.expectedForSyntheticSource(),
                     example.expectedForSyntheticSource(),
-                    example.expectedForBlockLoader(),
                     example.mapping()
                 );
             }
@@ -400,7 +399,7 @@ public SyntheticSourceExample example(int maxValues) {
                 if (v.malformedOutput == null) {
                     return new SyntheticSourceExample(v.input, v.output, this::mapping);
                 }
-                return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping);
+                return new SyntheticSourceExample(v.input, v.malformedOutput, this::mapping);
             }
             List<Value> values = randomList(1, maxValues, this::generateValue);
             List<Object> in = values.stream().map(Value::input).toList();
@@ -479,11 +478,6 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
         }
     }
 
-    protected BlockReaderSupport getSupportedReaders(MapperService mapper, String loaderFieldName) {
-        assumeTrue("Disabled, tested by ScaledFloatFieldBlockLoaderTests instead", false);
-        return null;
-    }
-
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

@@ -36,8 +36,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -213,17 +211,13 @@ public boolean preservesExactSource() {
             public SyntheticSourceExample example(int maxValues) {
                 if (randomBoolean()) {
                     var value = generateValue();
-                    return new SyntheticSourceExample(value.text, value.text, value.tokenCount, this::mapping);
+                    return new SyntheticSourceExample(value.text, value.text, this::mapping);
                 }
 
                 var values = randomList(1, 5, this::generateValue);
-
                 var textArray = values.stream().map(Value::text).toList();
 
-                var blockExpectedList = values.stream().map(Value::tokenCount).filter(Objects::nonNull).sorted().toList();
-                var blockExpected = blockExpectedList.size() == 1 ? blockExpectedList.get(0) : blockExpectedList;
-
-                return new SyntheticSourceExample(textArray, textArray, blockExpected, this::mapping);
+                return new SyntheticSourceExample(textArray, textArray, this::mapping);
             }
 
             private record Value(String text, Integer tokenCount) {}
@@ -258,11 +252,6 @@ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
         };
     }
 
-    protected Function<Object, Object> loadBlockExpected() {
-        // we can get either a number from doc values or null
-        return v -> v != null ? (Number) v : null;
-    }
-
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java

@@ -205,6 +205,7 @@ public void testClientConnectionCloseMidStream() throws Exception {
 
             // await stream handler is ready and request full content
             var handler = clientContext.awaitRestChannelAccepted(opaqueId);
+            assertBusy(() -> assertNotEquals(0, handler.stream.bufSize()));
 
             assertFalse(handler.isClosed());
 
@@ -214,6 +215,7 @@ public void testClientConnectionCloseMidStream() throws Exception {
             assertEquals(requestTransmittedLength, handler.readUntilClose());
 
             assertTrue(handler.isClosed());
+            assertEquals(0, handler.stream.bufSize());
         }
     }
 
@@ -230,6 +232,7 @@ public void testServerCloseConnectionMidStream() throws Exception {
 
             // await stream handler is ready and request full content
             var handler = clientContext.awaitRestChannelAccepted(opaqueId);
+            assertBusy(() -> assertNotEquals(0, handler.stream.bufSize()));
             assertFalse(handler.isClosed());
 
             // terminate connection on server and wait resources are released
@@ -238,6 +241,7 @@ public void testServerCloseConnectionMidStream() throws Exception {
             handler.channel.request().getHttpChannel().close();
             assertThat(safeGet(exceptionFuture), instanceOf(ClosedChannelException.class));
             assertTrue(handler.isClosed());
+            assertBusy(() -> assertEquals(0, handler.stream.bufSize()));
         }
     }
 
@@ -253,6 +257,7 @@ public void testServerExceptionMidStream() throws Exception {
 
             // await stream handler is ready and request full content
             var handler = clientContext.awaitRestChannelAccepted(opaqueId);
+            assertBusy(() -> assertNotEquals(0, handler.stream.bufSize()));
             assertFalse(handler.isClosed());
 
             // terminate connection on server and wait resources are released
@@ -264,6 +269,7 @@ public void testServerExceptionMidStream() throws Exception {
             final var exception = asInstanceOf(RuntimeException.class, safeGet(exceptionFuture));
             assertEquals(ServerRequestHandler.SIMULATED_EXCEPTION_MESSAGE, exception.getMessage());
             safeAwait(handler.closedLatch);
+            assertBusy(() -> assertEquals(0, handler.stream.bufSize()));
         }
     }
 
@@ -304,7 +310,7 @@ public void testClientBackpressure() throws Exception {
                 });
                 handler.readBytes(partSize);
             }
-            assertTrue(handler.receivedLastChunk);
+            assertTrue(handler.stream.hasLast());
         }
     }
 

modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4IncrementalRequestHandlingIT.java

@@ -15,7 +15,6 @@
 import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpRequestDecoder;
-import io.netty.handler.codec.http.LastHttpContent;
 
 import org.elasticsearch.http.HttpPreRequest;
 import org.elasticsearch.http.netty4.internal.HttpHeadersAuthenticatorUtils;
@@ -49,9 +48,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
         }
         if (aggregating || msg instanceof FullHttpRequest) {
             super.channelRead(ctx, msg);
-            if (msg instanceof LastHttpContent == false) {
-                ctx.read(); // HttpObjectAggregator is tricky with auto-read off, it might not call read again, calling on its behalf
-            }
         } else {
             streamContentSizeHandler.channelRead(ctx, msg);
         }

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/MissingReadDetector.java

@@ -123,7 +123,6 @@ private void handleRequest(ChannelHandlerContext ctx, HttpRequest request) {
                 isContinueExpected = true;
             } else {
                 ctx.writeAndFlush(EXPECTATION_FAILED_CLOSE.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE);
-                ctx.read();
                 return;
             }
         }
@@ -137,7 +136,6 @@ private void handleRequest(ChannelHandlerContext ctx, HttpRequest request) {
                 decoder.reset();
             }
             ctx.writeAndFlush(TOO_LARGE.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
-            ctx.read();
         } else {
             ignoreContent = false;
             currentContentLength = 0;
@@ -152,13 +150,11 @@ private void handleRequest(ChannelHandlerContext ctx, HttpRequest request) {
     private void handleContent(ChannelHandlerContext ctx, HttpContent msg) {
         if (ignoreContent) {
             msg.release();
-            ctx.read();
         } else {
             currentContentLength += msg.content().readableBytes();
             if (currentContentLength > maxContentLength) {
                 msg.release();
                 ctx.writeAndFlush(TOO_LARGE_CLOSE.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE);
-                ctx.read();
             } else {
                 ctx.fireChannelRead(msg);
             }