Merge branch '9.0' into backport/9.0/pr-121887

Author: drempapis

docs/changelog/132018.yaml

@@ -0,0 +1,5 @@
+pr: 132018
+summary: Fix decoding of non-ascii field names in ignored source
+area: Mapping
+type: bug
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/HardcodedEntitlements.java

@@ -92,8 +92,9 @@ private static List<Scope> createServerEntitlements(Path pidFile) {
                     new CreateClassLoaderEntitlement(),
                     new FilesEntitlement(
                         List.of(
-                            // TODO: what in es.base is accessing shared repo?
+                            // necessary due to lack of delegation ES-12382
                             FilesEntitlement.FileData.ofBaseDirPath(SHARED_REPO, READ_WRITE),
+                            FilesEntitlement.FileData.ofBaseDirPath(SHARED_DATA, READ_WRITE),
                             FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE)
                         )
                     )
@@ -122,6 +123,7 @@ private static List<Scope> createServerEntitlements(Path pidFile) {
                     new FilesEntitlement(
                         List.of(
                             FilesEntitlement.FileData.ofBaseDirPath(CONFIG, READ),
+                            FilesEntitlement.FileData.ofBaseDirPath(SHARED_DATA, READ_WRITE),
                             FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE)
                         )
                     )
@@ -130,7 +132,12 @@ private static List<Scope> createServerEntitlements(Path pidFile) {
             new Scope(
                 "org.apache.lucene.misc",
                 List.of(
-                    new FilesEntitlement(List.of(FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE))),
+                    new FilesEntitlement(
+                        List.of(
+                            FilesEntitlement.FileData.ofBaseDirPath(SHARED_DATA, READ_WRITE),
+                            FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE)
+                        )
+                    ),
                     new ReadStoreAttributesEntitlement()
                 )
             ),
@@ -145,7 +152,12 @@ private static List<Scope> createServerEntitlements(Path pidFile) {
                 "org.elasticsearch.nativeaccess",
                 List.of(
                     new LoadNativeLibrariesEntitlement(),
-                    new FilesEntitlement(List.of(FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE)))
+                    new FilesEntitlement(
+                        List.of(
+                            FilesEntitlement.FileData.ofBaseDirPath(SHARED_DATA, READ_WRITE),
+                            FilesEntitlement.FileData.ofBaseDirPath(DATA, READ_WRITE)
+                        )
+                    )
                 )
             )
         );

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -182,8 +182,9 @@ private static BaseDir parseBaseDir(String baseDir) {
             case "config" -> BaseDir.CONFIG;
             case "data" -> BaseDir.DATA;
             case "home" -> BaseDir.USER_HOME;
+            case "shared_data" -> BaseDir.SHARED_DATA;
             // it would be nice to limit this to just ES modules, but we don't have a way to plumb that through to here
-            // however, we still don't document in the error case below that shared_repo is valid
+            // however, we still don't document in the error case below that shared_repo and shared_data is valid
             case "shared_repo" -> BaseDir.SHARED_REPO;
             default -> throw new PolicyValidationException(
                 "invalid relative directory: " + baseDir + ", valid values: [config, data, home]"

plugins/store-smb/src/main/plugin-metadata/entitlement-policy.yaml

@@ -3,3 +3,6 @@ ALL-UNNAMED:
       - relative_path: "indices/"
         relative_to: data
         mode: read_write
+      - relative_path: ""
+        relative_to: shared_data
+        mode: read_write

server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

@@ -181,8 +181,12 @@ static NameValue decode(Object field) {
         int encodedSize = ByteUtils.readIntLE(bytes, 0);
         int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET;
         int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET;
-        String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8);
-        BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4);
+
+        String decoded = new String(bytes, 4, bytes.length - 4, StandardCharsets.UTF_8);
+        String name = decoded.substring(0, nameSize);
+        int nameByteCount = name.getBytes(StandardCharsets.UTF_8).length;
+
+        BytesRef value = new BytesRef(bytes, 4 + nameByteCount, bytes.length - nameByteCount - 4);
         return new NameValue(name, parentOffset, value, null);
     }
 

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java

@@ -10,12 +10,14 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -122,6 +124,15 @@ public void testIgnoredString() throws IOException {
         );
     }
 
+    public void testIgnoredStringFullUnicode() throws IOException {
+        String value = randomUnicodeOfCodepointLengthBetween(5, 20);
+        String fieldName = randomUnicodeOfCodepointLength(5);
+
+        String expected = Strings.toString(JsonXContent.contentBuilder().startObject().field(fieldName, value).endObject());
+
+        assertEquals(expected, getSyntheticSourceWithFieldLimit(b -> b.field(fieldName, value)));
+    }
+
     public void testIgnoredInt() throws IOException {
         int value = randomInt();
         assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)));

x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/entitlement-policy.yaml

@@ -6,3 +6,6 @@ org.elasticsearch.searchablesnapshots:
       - relative_path: indices
         relative_to: data
         mode: read_write
+      - relative_path: ""
+        relative_to: shared_data
+        mode: read_write