Merge branch 'main' into add_min_score_linear_retriever

Author: mridula-s109

docs/changelog/126786.yaml

@@ -0,0 +1,5 @@
+pr: 126786
+summary: Account for time taken to write index buffers in `IndexingMemoryController`
+area: Distributed
+type: enhancement
+issues: []

docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/qstr.md

@@ -527,7 +527,9 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex,
         CommonStats stats = new CommonStats();
         stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes());
         stats.store = new StoreStats();
-        stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1, 0.123, 0.234));
+        stats.indexing = new IndexingStats(
+            new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, targetWriteLoad, 1, 0.123, 0.234)
+        );
         return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0);
     }
 

docs/reference/query-languages/esql/kibana/definition/functions/qstr.json

@@ -29,6 +29,7 @@
 import java.util.Optional;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class IndexingMemoryControllerIT extends ESSingleNodeTestCase {
@@ -37,7 +38,9 @@ public class IndexingMemoryControllerIT extends ESSingleNodeTestCase {
     protected Settings nodeSettings() {
         return Settings.builder()
             .put(super.nodeSettings())
-            // small indexing buffer so that we can trigger refresh after buffering 100 deletes
+            // small indexing buffer so that
+            // 1. We can trigger refresh after buffering 100 deletes
+            // 2. Indexing memory Controller writes indexing buffers in sync with indexing on the indexing thread
             .put("indices.memory.index_buffer_size", "1kb")
             .build();
     }
@@ -111,4 +114,22 @@ public void testDeletesAloneCanTriggerRefresh() throws Exception {
         }
         assertThat(shard.getEngineOrNull().getIndexBufferRAMBytesUsed(), lessThanOrEqualTo(ByteSizeUnit.KB.toBytes(1)));
     }
+
+    /* When there is memory pressure, we write indexing buffers to disk on the same thread as the indexing thread,
+     * @see org.elasticsearch.indices.IndexingMemoryController.
+     * This test verifies that we update the stats that capture the combined time for indexing + writing the
+     * indexing buffers.
+     * Note that the small indices.memory.index_buffer_size setting is required for this test to work.
+     */
+    public void testIndexingUpdatesRelevantStats() throws Exception {
+        IndexService indexService = createIndex("index", indexSettings(1, 0).put("index.refresh_interval", -1).build());
+        IndexShard shard = indexService.getShard(0);
+        prepareIndex("index").setSource("field", randomUnicodeOfCodepointLengthBetween(10, 25)).get();
+        // Check that
+        assertThat(shard.indexingStats().getTotal().getTotalIndexingExecutionTimeInMillis(), greaterThan(0L));
+        assertThat(
+            shard.indexingStats().getTotal().getTotalIndexingExecutionTimeInMillis(),
+            greaterThan(shard.indexingStats().getTotal().getIndexTime().getMillis())
+        );
+    }
 }

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java

@@ -163,6 +163,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion BATCHED_QUERY_PHASE_VERSION_BACKPORT_8_X = def(8_841_0_19);
     public static final TransportVersion SEARCH_INCREMENTAL_TOP_DOCS_NULL_BACKPORT_8_19 = def(8_841_0_20);
     public static final TransportVersion ML_INFERENCE_SAGEMAKER_8_19 = def(8_841_0_21);
+    public static final TransportVersion ESQL_REPORT_ORIGINAL_TYPES_BACKPORT_8_19 = def(8_841_0_22);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
@@ -234,7 +235,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_00_0);
     public static final TransportVersion PINNED_RETRIEVER = def(9_068_0_00);
     public static final TransportVersion ML_INFERENCE_SAGEMAKER = def(9_069_0_00);
-    public static final TransportVersion RANK_DOCS_QUERY_MIN_SCORE = def(9_070_0_00);
+    public static final TransportVersion WRITE_LOAD_INCLUDES_BUFFER_WRITES = def(9_070_00_0);
+    public static final TransportVersion RANK_DOCS_QUERY_MIN_SCORE = def(9_071_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,