Merge branch 'main' into pkar/msearch-flatworld-search-project-routing

Author: pawankartik-elastic

benchmarks/build.gradle

@@ -48,6 +48,7 @@ dependencies {
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   api(project(':x-pack:plugin:mapper-exponential-histogram'))
+  api(project(':x-pack:plugin:logsdb'))
   implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
   implementation project(path: ':libs:exponential-histogram')

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/common/MappingParsingBenchmark.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.indices.common;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.mapper.MapperMetrics;
+import org.elasticsearch.index.mapper.MapperRegistry;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptCompiler;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xpack.logsdb.LogsDBPlugin;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+@Fork(value = 1)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class MappingParsingBenchmark {
+    static {
+        // For Elasticsearch900Lucene101Codec:
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging();
+        LogConfigurator.setNodeName("test");
+    }
+
+    private static final String MAPPING = """
+        {
+          "_doc": {
+            "dynamic": false,
+            "properties": {
+              "@timestamp": {
+                "type": "date"
+              },
+              "host": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              "message": {
+                "type": "pattern_text"
+              }
+            }
+          }
+        }
+        \s""";
+
+    @Param("1024")
+    private int numIndices;
+
+    private List<MapperService> mapperServices;
+    private CompressedXContent compressedMapping;
+
+    private Random random = new Random();
+    private static final String CHARS = "abcdefghijklmnopqrstuvwxyz1234567890";
+
+    private String randomIndexName() {
+        StringBuilder b = new StringBuilder();
+        for (int i = 0; i < 10; i++) {
+            b.append(CHARS.charAt(random.nextInt(CHARS.length())));
+        }
+        return b.toString();
+    }
+
+    @Setup
+    public void setUp() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", 1)
+            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put("index.mode", "logsdb")
+            .put("index.logsdb.sort_on_host_name", true)
+            .put("index.logsdb.sort_on_message_template", true)
+            .build();
+
+        LogsDBPlugin logsDBPlugin = new LogsDBPlugin(settings);
+
+        Set<Setting<?>> definedSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        definedSettings.addAll(logsDBPlugin.getSettings().stream().filter(Setting::hasIndexScope).toList());
+        IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, definedSettings);
+
+        mapperServices = new ArrayList<>(numIndices);
+        for (int i = 0; i < numIndices; i++) {
+            IndexMetadata meta = IndexMetadata.builder(randomIndexName()).settings(settings).build();
+            IndexSettings indexSettings = new IndexSettings(meta, settings, indexScopedSettings);
+            MapperRegistry mapperRegistry = new IndicesModule(List.of(logsDBPlugin)).getMapperRegistry();
+            SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
+            BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
+            MapperService mapperService = new MapperService(
+                () -> TransportVersion.current(),
+                indexSettings,
+                IndexAnalyzers.of(Map.of()),
+                XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(ClusterModule.getNamedXWriteables()))
+                    .withDeprecationHandler(LoggingDeprecationHandler.INSTANCE),
+                similarityService,
+                mapperRegistry,
+                () -> {
+                    throw new UnsupportedOperationException();
+                },
+                new ProvidedIdFieldMapper(() -> true),
+                new ScriptCompiler() {
+                    @Override
+                    public <T> T compile(Script script, ScriptContext<T> scriptContext) {
+                        throw new UnsupportedOperationException();
+                    }
+                },
+                bitsetFilterCache::getBitSetProducer,
+                MapperMetrics.NOOP
+            );
+
+            mapperServices.add(mapperService);
+        }
+
+        compressedMapping = new CompressedXContent(MAPPING);
+    }
+
+    @Benchmark
+    public void mappingParsingBenchmark() {
+        for (MapperService service : mapperServices) {
+            service.merge("_doc", compressedMapping, MapperService.MergeReason.MAPPING_UPDATE);
+        }
+    }
+}

docs/changelog/132388.yaml

@@ -0,0 +1,5 @@
+pr: 132388
+summary: Added NVIDIA support to Inference Plugin
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/138803.yaml

@@ -0,0 +1,5 @@
+pr: 138803
+summary: Enable CPS
+area: SQL
+type: enhancement
+issues: []

docs/changelog/139061.yaml

@@ -0,0 +1,5 @@
+pr: 139061
+summary: "Disallow index types updates to bbq_disk, revert"
+area: Vector Search
+type: bug
+issues: []

docs/reference/elasticsearch/index-settings/bbq.md

@@ -136,6 +136,9 @@ PUT bbq_flat-index
 ```{applies_to}
 stack: ga 9.2
 ```
+:::{note}
+This feature requires an [Enterprise subscription](https://www.elastic.co/subscriptions).
+:::
 
 When you set a dense vector field’s `index_options` parameter to `type: bbq_disk`, {{es}} uses the DiskBBQ algorithm, a disk-based alternative to HNSW for [kNN search](https://www.elastic.co/docs//solutions/search/vector/knn) on compressed vectors. DiskBBQ stores the vector data on disk instead of in memory, lowering RAM requirements and reducing the overall cost of vector storage and search.
 
@@ -265,4 +268,4 @@ You can change oversampling from the default 3× to another value. Refer to [Ove
 - [Better Binary Quantization (BBQ) in Lucene and {{es}}](https://www.elastic.co/search-labs/blog/better-binary-quantization-lucene-elasticsearch) - Learn how BBQ works, its benefits, and how it reduces memory usage while preserving search accuracy.
 - [Introducing a new vector storage format: DiskBBQ](https://www.elastic.co/search-labs/blog/diskbbq-elasticsearch-introduction) - Learn how DiskBBQ improves vector search in low-memory environments and compares to HNSW in speed and cost-effectiveness. 
 - [Dense vector field type](https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/dense-vector) - Find code examples for using `bbq_hnsw` `index_type`.
-- [kNN search](https://www.elastic.co/docs/solutions/search/vector/knn) - Learn about the search algorithm that BBQ works with.
+- [kNN search](https://www.elastic.co/docs/solutions/search/vector/knn) - Learn about the search algorithm that BBQ works with.

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -362,7 +362,7 @@ $$$dense-vector-index-options$$$
     * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatic scalar quantization. Only supports `element_type` of `float`.
     * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatic half-byte scalar quantization. Only supports `element_type` of `float`.
     * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`.
-    * {applies_to}`stack: ga 9.2` `bbq_disk` - This utilizes a variant of [k-means clustering algorithm](https://en.wikipedia.org/wiki/K-means_clustering) in addition to automatic binary quantization to partition vectors and search subspaces rather than an entire graph structure as in with HNSW. Only supports `element_type` of `float`.  This combines the benefits of BBQ quantization with partitioning to further reduces the required memory overhead when compared with HNSW and can effectively be run at the smallest possible RAM and heap sizes when HNSW would otherwise cause swapping and grind to a halt.  DiskBBQ largely scales linearly with the total RAM.  And search performance is enhanced at scale as a subset of the total vector space is loaded.
+    * {applies_to}`stack: ga 9.2` `bbq_disk` - This utilizes a variant of [k-means clustering algorithm](https://en.wikipedia.org/wiki/K-means_clustering) in addition to automatic binary quantization to partition vectors and search subspaces rather than an entire graph structure as in with HNSW. Only supports `element_type` of `float`.  This combines the benefits of BBQ quantization with partitioning to further reduces the required memory overhead when compared with HNSW and can effectively be run at the smallest possible RAM and heap sizes when HNSW would otherwise cause swapping and grind to a halt.  DiskBBQ largely scales linearly with the total RAM.  And search performance is enhanced at scale as a subset of the total vector space is loaded. This requires an [Enterprise subscription](https://www.elastic.co/subscriptions).
 
 `m`
 :   (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types.