Merge branch 'main' into eis-headers

Author: davidkyle

docs/changelog/135204.yaml

@@ -0,0 +1,6 @@
+pr: 135204
+summary: Make `_tsid` available in metadata
+area: ES|QL
+type: enhancement
+issues:
+ - 133205

docs/changelog/135337.yaml

@@ -0,0 +1,5 @@
+pr: 135337
+summary: Do not pass `ProjectMetadata` to lazy index permissions builder
+area: Security
+type: enhancement
+issues: []

docs/reference/enrich-processor/geoip-processor.md

@@ -206,14 +206,18 @@ PUT my_ip_locations
 
 If you can’t [automatically update](#geoip-automatic-updates) your IP geolocation databases from the Elastic endpoint, you have a few other options:
 
-* [Use a proxy endpoint](#use-proxy-geoip-endpoint)
+* [Use a reverse proxy endpoint](#use-proxy-geoip-endpoint)
 * [Use a custom endpoint](#use-custom-geoip-endpoint)
 * [Manually update your IP geolocation databases](#manually-update-geoip-databases)
 
 $$$use-proxy-geoip-endpoint$$$
-**Use a proxy endpoint**
+**Use a reverse proxy endpoint**
 
-If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure proxy. You can then specify the proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
+If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure reverse proxy. You can then specify the reverse proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
+
+:::{note}
+True HTTP proxy support for GeoIP database downloads is not currently available in {{es}}.
+:::
 
 In a strict setup the following domains may need to be added to the allowed domains list:
 

docs/reference/query-languages/esql/_snippets/functions/description/count_distinct.md

@@ -25,6 +25,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * Provides the method getValueAsText that is a best-effort optimization for UTF8 fields. If the
+ * {@link UTF8StreamJsonParser} has already parsed the text, then the caller should fall back to getText.
+ * This is sufficient because when we call getText, jackson stores the parsed UTF-16 value for future calls.
+ * Once we've already got the parsed UTF-16 value, the optimization isn't necessary.
+ */
 public class ESUTF8StreamJsonParser extends UTF8StreamJsonParser implements OptimizedTextCapable {
     protected int stringEnd = -1;
     protected int stringLength;
@@ -53,6 +59,7 @@ public ESUTF8StreamJsonParser(
      */
     @Override
     public Text getValueAsText() throws IOException {
+        // _tokenIncomplete is true when UTF8StreamJsonParser has already processed this value.
         if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete) {
             if (lastOptimisedValue != null) {
                 return new Text(new XContentString.UTF8Bytes(lastOptimisedValue), stringLength);
@@ -148,33 +155,33 @@ protected Text _finishAndReturnText() throws IOException {
 
     @Override
     public JsonToken nextToken() throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextToken();
     }
 
     @Override
     public boolean nextFieldName(SerializableString str) throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextFieldName(str);
     }
 
     @Override
     public String nextFieldName() throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextFieldName();
     }
 
     /**
-     * Resets the current token state before moving to the next.
+     * Resets the current token state before moving to the next. It resets the _inputPtr and the
+     * _tokenIncomplete only if {@link UTF8StreamJsonParser#getText()} or {@link UTF8StreamJsonParser#getValueAsString()}
+     * hasn't run yet.
      */
-    private void maybeResetCurrentTokenState() {
+    private void resetCurrentTokenState() {
         if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
             _inputPtr = stringEnd;
             _tokenIncomplete = false;
-            lastOptimisedValue = null;
         }
+        lastOptimisedValue = null;
+        stringEnd = -1;
     }
 }

docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json

@@ -16,6 +16,7 @@
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.core.exc.InputCoercionException;
 import com.fasterxml.jackson.core.exc.StreamConstraintsException;
+import com.fasterxml.jackson.core.filter.FilteringParserDelegate;
 import com.fasterxml.jackson.core.io.JsonEOFException;
 
 import org.elasticsearch.core.IOUtils;
@@ -26,6 +27,7 @@
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentString;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.provider.OptimizedTextCapable;
 import org.elasticsearch.xcontent.provider.XContentParserConfigurationImpl;
 import org.elasticsearch.xcontent.support.AbstractXContentParser;
 
@@ -143,7 +145,19 @@ public String text() throws IOException {
 
     @Override
     public XContentString optimizedText() throws IOException {
-        // TODO: enable utf-8 parsing optimization once verified it is completely safe
+        if (currentToken().isValue() == false) {
+            throwOnNoText();
+        }
+        var parser = this.parser;
+        if (parser instanceof FilteringParserDelegate delegate) {
+            parser = delegate.delegate();
+        }
+        if (parser instanceof OptimizedTextCapable optimizedTextCapableParser) {
+            var bytesRef = optimizedTextCapableParser.getValueAsText();
+            if (bytesRef != null) {
+                return bytesRef;
+            }
+        }
         return new Text(text());
     }
 

docs/reference/query-languages/esql/kibana/docs/functions/count_distinct.md

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ESCborParserTests extends ESTestCase {
 
@@ -54,8 +55,15 @@ private void testStringValue(String expected) throws IOException {
         assertThat(text.hasBytes(), equalTo(true));
         assertThat(text.stringLength(), equalTo(expected.length()));
         assertThat(text.string(), equalTo(expected));
+        // Retrieve twice
         assertThat(parser.getValueAsText().string(), equalTo(expected));
         assertThat(parser.getValueAsString(), equalTo(expected));
+        // Use the getText() to ensure _tokenIncomplete works
+        assertThat(parser.getText(), equalTo(expected));
+        // The optimisation is not used after the getText()
+        assertThat(parser.getValueAsText(), nullValue());
+        // Original CBOR getValueAsString works.
+        assertThat(parser.getValueAsString(), equalTo(expected));
         assertThat(parser.nextToken(), equalTo(JsonToken.END_OBJECT));
     }
 }

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParser.java

@@ -13,14 +13,23 @@
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
 
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.FilterXContentParserWrapper;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.XContentString;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.provider.XContentParserConfigurationImpl;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Locale;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class ESUTF8StreamJsonParserTests extends ESTestCase {
 
@@ -272,12 +281,17 @@ public void testGetValueRandomized() throws IOException {
                     var text = parser.getValueAsText();
                     assertTextRef(text.bytes(), currVal);
                     assertThat(text.stringLength(), Matchers.equalTo(currVal.length()));
-
-                    // Retrieve it twice to ensure it works as expected
+                    // Retrieve it a second time
                     text = parser.getValueAsText();
+                    assertThat(text, Matchers.notNullValue());
                     assertTextRef(text.bytes(), currVal);
                     assertThat(text.stringLength(), Matchers.equalTo(currVal.length()));
+                    // Use getText()
+                    assertThat(parser.getText(), Matchers.equalTo(text.string()));
+                    // After retrieving it with getText() we do not use the optimised value anymore.
+                    assertThat(parser.getValueAsText(), Matchers.nullValue());
                 } else {
+                    assertThat(parser.getText(), Matchers.notNullValue());
                     assertThat(parser.getValueAsText(), Matchers.nullValue());
                     assertThat(parser.getValueAsString(), Matchers.equalTo(currVal));
                     // Retrieve it twice to ensure it works as expected
@@ -288,4 +302,97 @@ public void testGetValueRandomized() throws IOException {
         });
     }
 
+    /**
+     * This test compares the retrieval of an optimised text against the baseline.
+     */
+    public void testOptimisedParser() throws Exception {
+        for (int i = 0; i < 200; i++) {
+            String json = randomJsonInput(randomIntBetween(1, 6));
+            try (
+                var baselineParser = XContentHelper.createParser(
+                    XContentParserConfiguration.EMPTY,
+                    new BytesArray(json),
+                    XContentType.JSON
+                );
+                var optimisedParser = TestXContentParser.create(json)
+            ) {
+                var expected = baselineParser.mapOrdered();
+                var actual = optimisedParser.mapOrdered();
+                assertThat(expected, Matchers.equalTo(actual));
+            }
+        }
+    }
+
+    private String randomJsonInput(int depth) {
+        StringBuilder sb = new StringBuilder();
+        sb.append('{');
+        int numberOfFields = randomIntBetween(1, 10);
+        for (int i = 0; i < numberOfFields; i++) {
+            sb.append("\"k-").append(randomAlphanumericOfLength(10)).append("\":");
+            if (depth == 0 || randomBoolean()) {
+                if (randomIntBetween(0, 9) == 0) {
+                    sb.append(
+                        IntStream.range(0, randomIntBetween(1, 10))
+                            .mapToObj(ignored -> randomUTF8Value())
+                            .collect(Collectors.joining(",", "[", "]"))
+                    );
+                } else {
+                    sb.append(randomUTF8Value());
+                }
+            } else {
+                sb.append(randomJsonInput(depth - 1));
+            }
+            if (i < numberOfFields - 1) {
+                sb.append(',');
+            }
+        }
+        sb.append("}");
+        return sb.toString();
+    }
+
+    private String randomUTF8Value() {
+        return "\"" + buildRandomInput(randomIntBetween(10, 50)).input + "\"";
+    }
+
+    /**
+     * This XContentParser introduces a random mix of getText() and getOptimisedText()
+     * to simulate different access patterns for optimised fields.
+     */
+    private static class TestXContentParser extends FilterXContentParserWrapper {
+
+        TestXContentParser(XContentParser delegate) {
+            super(delegate);
+        }
+
+        static TestXContentParser create(String input) throws IOException {
+            JsonFactory factory = new ESJsonFactoryBuilder().build();
+            assertThat(factory, Matchers.instanceOf(ESJsonFactory.class));
+
+            JsonParser parser = factory.createParser(StandardCharsets.UTF_8.encode(input).array());
+            assertThat(parser, Matchers.instanceOf(ESUTF8StreamJsonParser.class));
+            return new TestXContentParser(new JsonXContentParser(XContentParserConfigurationImpl.EMPTY, parser));
+        }
+
+        @Override
+        public String text() throws IOException {
+            if (randomIntBetween(0, 9) < 8) {
+                return super.text();
+            } else {
+                return super.optimizedText().string();
+            }
+        }
+
+        @Override
+        public XContentString optimizedText() throws IOException {
+            int extraCalls = randomIntBetween(0, 5);
+            for (int i = 0; i < extraCalls; i++) {
+                if (randomBoolean()) {
+                    super.optimizedText();
+                } else {
+                    super.text();
+                }
+            }
+            return super.optimizedText();
+        }
+    }
 }