Merge remote-tracking branch 'origin/tsdb/use-large-numeric-blocks' into tsdb/use-large-numeric-blocks

Author: kkrik-es

docs/changelog/138559.yaml

@@ -0,0 +1,5 @@
+pr: 138559
+summary: Accept `project_routing` as query parameter
+area: EQL
+type: enhancement
+issues: []

docs/changelog/138568.yaml

@@ -0,0 +1,5 @@
+pr: 138568
+summary: Use doc values skipper for `_tsid` in synthetic `_id` postings
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/138718.yaml

@@ -0,0 +1,5 @@
+pr: 138718
+summary: Add `project_routing` option
+area: SQL
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBSyntheticIdFieldsProducer.java

@@ -280,22 +280,44 @@ private BytesRef lookupTsIdOrd(int tsIdOrdinal) throws IOException {
         }
 
         /**
-         * Scan all documents to find the first document that has a _tsid equal or greater than the provided _tsid ordinal, returning its
-         * document ID. If no document is found, the method returns {@link DocIdSetIterator#NO_MORE_DOCS}.
+         * Use a doc values skipper to find a starting document ID for the provided _tsid ordinal. The returned document ID might have the
+         * exact _tsid ordinal provided, or a lower one.
          *
-         * Warning: This method is very slow because it potentially scans all documents in the segment.
+         * @param tsIdOrd the _tsid ordinal
+         * @return a docID to start scanning documents from in order to find the first document ID matching the provided _tsid
+         * @throws IOException if any I/O exception occurs
          */
-        private int slowScanToFirstDocWithTsIdOrdinalEqualOrGreaterThan(int tsIdOrd) throws IOException {
+        private int findStartDocIDForTsIdOrd(int tsIdOrd) throws IOException {
+            var skipper = docValuesProducer.getSkipper(tsIdFieldInfo);
+            assert skipper != null;
+            if (skipper.minValue() > tsIdOrd || tsIdOrd > skipper.maxValue()) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+            skipper.advance(tsIdOrd, Long.MAX_VALUE);
+            return skipper.minDocID(0);
+        }
+
+        /**
+         * Find the first document that has a _tsid equal or greater than the provided _tsid ordinal, returning its document ID. If no
+         * document is found, the method returns {@link DocIdSetIterator#NO_MORE_DOCS}.
+         *
+         * Warning: This method can be slow because it potentially scans many documents in the segment.
+         */
+        private int findFirstDocWithTsIdOrdinalEqualOrGreaterThan(int tsIdOrd) throws IOException {
+            final int startDocId = findStartDocIDForTsIdOrd(tsIdOrd);
+            if (startDocId == DocIdSetIterator.NO_MORE_DOCS) {
+                return startDocId;
+            }
             // recreate even if doc values are already on the same ordinal, to ensure the method returns the first doc
-            if (tsIdDocValues == null || (cachedTsIdOrd != -1 && cachedTsIdOrd >= tsIdOrd)) {
+            if (tsIdDocValues == null || (cachedTsIdOrd != -1 && cachedTsIdOrd >= tsIdOrd) || tsIdDocValues.docID() > startDocId) {
                 tsIdDocValues = docValuesProducer.getSorted(tsIdFieldInfo);
                 cachedTsIdOrd = -1;
                 cachedTsId = null;
             }
             assert 0 <= tsIdOrd : tsIdOrd;
             assert tsIdOrd < tsIdDocValues.getValueCount() : tsIdOrd;
 
-            for (int docID = 0; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) {
+            for (int docID = startDocId; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) {
                 boolean found = tsIdDocValues.advanceExact(docID);
                 assert found : "No value found for field [" + tsIdFieldInfo.getName() + " and docID " + docID;
                 var ord = tsIdDocValues.ordValue();
@@ -313,22 +335,25 @@ private int slowScanToFirstDocWithTsIdOrdinalEqualOrGreaterThan(int tsIdOrd) thr
         }
 
         /**
-         * Scan all documents to find the first document that has a _tsid equal to the provided _tsid ordinal, returning its
-         * document ID. If no document is found, the method returns {@link DocIdSetIterator#NO_MORE_DOCS}.
+         * Find the first document that has a _tsid equal to the provided _tsid ordinal, returning its document ID. If no document is found,
+         * the method returns {@link DocIdSetIterator#NO_MORE_DOCS}.
          *
-         * Warning: This method is very slow because it potentially scans all documents in the segment.
+         * Warning: This method can be slow because it potentially scans many documents in the segment.
          */
-        private int slowScanToFirstDocWithTsIdOrdinalEqualTo(int tsIdOrd) throws IOException {
+        private int findFirstDocWithTsIdOrdinalEqualTo(int tsIdOrd) throws IOException {
+            final int startDocId = findStartDocIDForTsIdOrd(tsIdOrd);
+            assert startDocId != DocIdSetIterator.NO_MORE_DOCS : startDocId;
+
             // recreate even if doc values are already on the same ordinal, to ensure the method returns the first doc
-            if (tsIdDocValues == null || (cachedTsIdOrd != -1 && cachedTsIdOrd >= tsIdOrd)) {
+            if (tsIdDocValues == null || (cachedTsIdOrd != -1 && cachedTsIdOrd >= tsIdOrd) || tsIdDocValues.docID() > startDocId) {
                 tsIdDocValues = docValuesProducer.getSorted(tsIdFieldInfo);
                 cachedTsIdOrd = -1;
                 cachedTsId = null;
             }
             assert 0 <= tsIdOrd : tsIdOrd;
             assert tsIdOrd < tsIdDocValues.getValueCount() : tsIdOrd;
 
-            for (int docID = 0; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) {
+            for (int docID = startDocId; docID != DocIdSetIterator.NO_MORE_DOCS; docID = tsIdDocValues.nextDoc()) {
                 boolean found = tsIdDocValues.advanceExact(docID);
                 assert found : "No value found for field [" + tsIdFieldInfo.getName() + " and docID " + docID;
                 var ord = tsIdDocValues.ordValue();
@@ -441,7 +466,7 @@ public SeekStatus seekCeil(BytesRef id) throws IOException {
                 tsIdOrd = -tsIdOrd - 1;
                 // set the terms enum on the first non-matching document
                 if (tsIdOrd < docValues.getTsIdValueCount()) {
-                    int docID = docValues.slowScanToFirstDocWithTsIdOrdinalEqualOrGreaterThan(tsIdOrd);
+                    int docID = docValues.findFirstDocWithTsIdOrdinalEqualOrGreaterThan(tsIdOrd);
                     if (docID != DocIdSetIterator.NO_MORE_DOCS) {
                         current = new SyntheticTerm(
                             docID,
@@ -461,8 +486,8 @@ public SeekStatus seekCeil(BytesRef id) throws IOException {
             // _tsid found, extract the timestamp
             final long timestamp = TsidExtractingIdFieldMapper.extractTimestampFromSyntheticId(id);
 
-            // Slow scan to the first document matching the _tsid
-            final int startDocID = docValues.slowScanToFirstDocWithTsIdOrdinalEqualTo(tsIdOrd);
+            // Find the first document matching the _tsid
+            final int startDocID = docValues.findFirstDocWithTsIdOrdinalEqualTo(tsIdOrd);
             assert startDocID >= 0 : startDocID;
 
             int docID = startDocID;

server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java

@@ -75,6 +75,7 @@ public static ParsedDocument noopTombstone(SeqNoFieldMapper.SeqNoIndexOptions se
      * The returned document consists only _uid, _seqno, _term and _version fields; other metadata fields are excluded.
      * @param id                the id of the deleted document
      */
+    // used by tests
     public static ParsedDocument deleteTombstone(SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions, String id) {
         return deleteTombstone(seqNoIndexOptions, false /* ignored */, false, id, null /* ignored */);
     }
@@ -101,6 +102,9 @@ public static ParsedDocument deleteTombstone(
             // Use a synthetic _id field which is not indexed nor stored
             document.add(IdFieldMapper.syntheticIdField(id));
 
+            // Add doc values fields that are used to synthesize the synthetic _id.
+            // Note: It is not strictly required for tombstones documents but we decided to add them so that iterating and seeking synthetic
+            // _id terms over tombstones also work as if a regular _id field was present.
             var timeSeriesId = TsidExtractingIdFieldMapper.extractTimeSeriesIdFromSyntheticId(uid);
             var timestamp = TsidExtractingIdFieldMapper.extractTimestampFromSyntheticId(uid);
             var routingHash = TsidExtractingIdFieldMapper.extractRoutingHashBytesFromSyntheticId(uid);

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestTestCase.java

@@ -61,7 +61,10 @@ public void checkSearchContent() throws Exception {
             """, validQuery), "query malformed, empty clause found" },
         { String.format(Locale.ROOT, """
             {"query": "%s", "max_samples_per_key": 0}
-            """, validQuery), "max_samples_per_key must be greater than 0" } };
+            """, validQuery), "max_samples_per_key must be greater than 0" },
+        { String.format(Locale.ROOT, """
+            {"query": "%s", "project_routing": "foo"}
+            """, validQuery), "[project_routing] is only allowed when cross-project search is enabled" } };
 
     public void testBadRequests() throws Exception {
         createIndex(defaultValidationIndexName, (String) null);

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java

@@ -90,6 +90,7 @@ public class EqlSearchRequest extends LegacyActionRequest implements IndicesRequ
     static final String KEY_MAX_SAMPLES_PER_KEY = "max_samples_per_key";
     static final String KEY_ALLOW_PARTIAL_SEARCH_RESULTS = "allow_partial_search_results";
     static final String KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS = "allow_partial_sequence_results";
+    static final String KEY_PROJECT_ROUTING = "project_routing";
 
     static final ParseField FILTER = new ParseField(KEY_FILTER);
     static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD);
@@ -106,6 +107,7 @@ public class EqlSearchRequest extends LegacyActionRequest implements IndicesRequ
     static final ParseField MAX_SAMPLES_PER_KEY = new ParseField(KEY_MAX_SAMPLES_PER_KEY);
     static final ParseField ALLOW_PARTIAL_SEARCH_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEARCH_RESULTS);
     static final ParseField ALLOW_PARTIAL_SEQUENCE_RESULTS = new ParseField(KEY_ALLOW_PARTIAL_SEQUENCE_RESULTS);
+    static final ParseField PROJECT_ROUTING = new ParseField(KEY_PROJECT_ROUTING);
 
     private static final ObjectParser<EqlSearchRequest, Void> PARSER = objectParser(EqlSearchRequest::new);
 
@@ -301,6 +303,7 @@ protected static <R extends EqlSearchRequest> ObjectParser<R, Void> objectParser
         parser.declareInt(EqlSearchRequest::maxSamplesPerKey, MAX_SAMPLES_PER_KEY);
         parser.declareBoolean(EqlSearchRequest::allowPartialSearchResults, ALLOW_PARTIAL_SEARCH_RESULTS);
         parser.declareBoolean(EqlSearchRequest::allowPartialSequenceResults, ALLOW_PARTIAL_SEQUENCE_RESULTS);
+        parser.declareString(EqlSearchRequest::projectRouting, PROJECT_ROUTING);
         return parser;
     }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlSearchAction.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.eql.action.EqlSearchAction;
 import org.elasticsearch.xpack.eql.action.EqlSearchRequest;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse;
+import org.elasticsearch.xpack.ql.InvalidArgumentException;
 
 import java.io.IOException;
 import java.util.List;
@@ -87,6 +88,10 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             eqlRequest.allowPartialSequenceResults(
                 request.paramAsBoolean("allow_partial_sequence_results", eqlRequest.allowPartialSequenceResults())
             );
+            eqlRequest.projectRouting(request.param("project_routing", eqlRequest.getProjectRouting()));
+            if (crossProjectEnabled == false && eqlRequest.getProjectRouting() != null) {
+                throw new InvalidArgumentException("[project_routing] is only allowed when cross-project search is enabled");
+            }
         }
 
         return channel -> {

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java

@@ -68,6 +68,7 @@ public void testSearchRequestParser() throws IOException {
               "timestamp_field": "tsf",
               "event_category_field": "etf",
               "size": "101",
+              "project_routing": "_alias:_origin",
               "query": "file where user != 'SYSTEM' by file_path"
             }""", EqlSearchRequest::fromXContent);
         assertArrayEquals(new String[] { "endgame-*" }, request.indices());
@@ -81,6 +82,7 @@ public void testSearchRequestParser() throws IOException {
         assertEquals(101, request.size());
         assertEquals(1000, request.fetchSize());
         assertEquals("file where user != 'SYSTEM' by file_path", request.query());
+        assertEquals("_alias:_origin", request.getProjectRouting());
     }
 
     private EqlSearchRequest generateRequest(String index, String json, Function<XContentParser, EqlSearchRequest> fromXContent)

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java

@@ -81,6 +81,7 @@ protected EqlSearchRequest createTestInstance() {
                 .keepOnCompletion(randomBoolean())
                 .allowPartialSearchResults(randomBoolean())
                 .allowPartialSequenceResults(randomBoolean())
+                .projectRouting(randomAlphaOfLength(10))
                 .fetchFields(randomFetchFields)
                 .runtimeMappings(randomRuntimeMappings())
                 .resultPosition(randomFrom("tail", "head"))
@@ -143,6 +144,7 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, T
         mutatedInstance.allowPartialSequenceResults(
             version.supports(TransportVersions.V_8_18_0) ? instance.allowPartialSequenceResults() : false
         );
+        mutatedInstance.projectRouting(instance.getProjectRouting());
 
         return mutatedInstance;
     }