Skip to content

Commit 0a1ab00

Browse files
authored
Merge branch 'main' into exp-histo-topn
2 parents 9bcaee1 + fafb162 commit 0a1ab00

File tree

49 files changed

+1748
-902
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+1748
-902
lines changed

docs/changelog/134708.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
pr: 134708
2-
summary: Default `semantic_text` fields to ELSER on EIS when available
2+
summary: Default [semantic_text](/reference/elasticsearch/mapping-reference/semantic-text.md) fields to use [ELSER on EIS](docs-content://explore-analyze/elastic-inference/eis.md#elser-on-eis) when available
33
area: Mapping
44
type: enhancement
55
issues: []

docs/changelog/137297.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 137297
2+
summary: Fixed inconsistency in the `isSyntheticSourceEnabled` flag
3+
area: Mapping
4+
type: bug
5+
issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -437,8 +437,16 @@ When updating documents that contain `semantic_text` fields, it’s important to
437437

438438
If you want to avoid unnecessary inference and keep existing embeddings:
439439

440-
* Use **partial updates through the Bulk API**.
441-
* Omit any `semantic_text` fields that did not change from the `doc` object in your request.
440+
* Use **partial updates through the Bulk API**.
441+
* Omit any `semantic_text` fields that did not change from the `doc` object in your request.
442+
443+
### Scripted updates
444+
445+
For indices containing `semantic_text` fields, updates that use scripts have the
446+
following behavior:
447+
448+
-**Supported:** [Update API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update)
449+
-**Not supported:** [Bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk-1). Scripted updates will fail even if the script targets non-`semantic_text` fields.
442450

443451
## Returning semantic field embeddings in `_source`
444452

@@ -578,18 +586,6 @@ PUT my-index-000004
578586
```
579587
% TEST[skip:Requires inference endpoint]
580588

581-
## Updates to `semantic_text` fields [update-script]
582-
583-
For indices containing `semantic_text` fields, updates that use scripts have the
584-
following behavior:
585-
586-
* Are supported through
587-
the [Update API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update).
588-
* Are not supported through
589-
the [Bulk API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk-1)
590-
and will fail. Even if the script targets non-`semantic_text` fields, the
591-
update will fail when the index contains a `semantic_text` field.
592-
593589
## `copy_to` and multi-fields support [copy-to-support]
594590

595591
The semantic_text field type can serve as the target

libs/entitlement/tools/common/build.gradle

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,17 @@ apply plugin: 'elasticsearch.build'
1212
tasks.named('forbiddenApisMain').configure {
1313
replaceSignatureFiles 'jdk-signatures'
1414
}
15+
16+
dependencies {
17+
implementation(project(':libs:core'))
18+
implementation 'org.ow2.asm:asm:9.9'
19+
implementation 'org.ow2.asm:asm-util:9.9'
20+
}
21+
22+
tasks.named("dependencyLicenses").configure {
23+
mapping from: /asm-.*/, to: 'asm'
24+
}
25+
26+
tasks.named("thirdPartyAudit").configure {
27+
ignoreMissingClasses()
28+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
Copyright (c) 2012 France Télécom
2+
All rights reserved.
3+
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions
6+
are met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in the
11+
documentation and/or other materials provided with the distribution.
12+
3. Neither the name of the copyright holders nor the names of its
13+
contributors may be used to endorse or promote products derived from
14+
this software without specific prior written permission.
15+
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26+
THE POSSIBILITY OF SUCH DAMAGE.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.entitlement.tools;
11+
12+
import org.elasticsearch.core.Tuple;
13+
import org.objectweb.asm.ClassReader;
14+
import org.objectweb.asm.ClassVisitor;
15+
import org.objectweb.asm.MethodVisitor;
16+
import org.objectweb.asm.Opcodes;
17+
18+
import java.io.IOException;
19+
import java.lang.constant.ClassDesc;
20+
import java.util.Collections;
21+
import java.util.Comparator;
22+
import java.util.Map;
23+
import java.util.Set;
24+
import java.util.TreeMap;
25+
import java.util.TreeSet;
26+
import java.util.function.Predicate;
27+
import java.util.stream.Stream;
28+
29+
import static java.util.Collections.emptySet;
30+
31+
public class AccessibleJdkMethods {
32+
33+
private static final Set<AccessibleMethod.Descriptor> EXCLUDES = Set.of(
34+
new AccessibleMethod.Descriptor("toString", "()Ljava/lang/String;", true, false),
35+
new AccessibleMethod.Descriptor("hashCode", "()I", true, false),
36+
new AccessibleMethod.Descriptor("equals", "(Ljava/lang/Object;)Z", true, false),
37+
new AccessibleMethod.Descriptor("close", "()V", true, false)
38+
);
39+
40+
public record AccessibleMethod(Descriptor descriptor, boolean isFinal, boolean isDeprecated) {
41+
public record Descriptor(String method, String descriptor, boolean isPublic, boolean isStatic) {
42+
public static final Comparator<Descriptor> COMPARATOR = Comparator.comparing(Descriptor::method)
43+
.thenComparing(Descriptor::descriptor)
44+
.thenComparing(Descriptor::isStatic);
45+
}
46+
47+
public static final Comparator<AccessibleMethod> COMPARATOR = Comparator.comparing(
48+
AccessibleMethod::descriptor,
49+
Descriptor.COMPARATOR
50+
);
51+
}
52+
53+
public record ModuleClass(String module, String clazz) {
54+
public static final Comparator<ModuleClass> COMPARATOR = Comparator.comparing(ModuleClass::module)
55+
.thenComparing(ModuleClass::clazz);
56+
}
57+
58+
public static Stream<Tuple<ModuleClass, AccessibleMethod>> loadAccessibleMethods(Predicate<String> modulePredicate) throws IOException {
59+
// 1st: map class names to module names (including later excluded modules) for lookup in 2nd step
60+
final Map<String, String> moduleNameByClass = Utils.loadClassToModuleMapping();
61+
final Map<String, Set<String>> exportsByModule = Utils.loadExportsByModule();
62+
final AccessibleMethodsVisitor visitor = new AccessibleMethodsVisitor(modulePredicate, moduleNameByClass, exportsByModule);
63+
// 2nd: calculate accessible implementations of classes in included modules
64+
Utils.walkJdkModules(modulePredicate, exportsByModule, (moduleName, moduleClasses, moduleExports) -> {
65+
for (var classFile : moduleClasses) {
66+
// visit class once (skips if class was already visited earlier due to a dependency on it)
67+
visitor.visitOnce(new ModuleClass(moduleName, Utils.internalClassName(classFile, moduleName)));
68+
}
69+
});
70+
71+
return visitor.getAccessibleMethods().entrySet().stream().flatMap(e -> e.getValue().stream().map(m -> Tuple.tuple(e.getKey(), m)));
72+
}
73+
74+
private static class AccessibleMethodsVisitor extends ClassVisitor {
75+
private final Map<ModuleClass, Set<AccessibleMethod>> inheritableAccessByClass = new TreeMap<>(ModuleClass.COMPARATOR);
76+
private final Map<ModuleClass, Set<AccessibleMethod>> accessibleImplementationsByClass = new TreeMap<>(ModuleClass.COMPARATOR);
77+
78+
private final Predicate<String> modulePredicate;
79+
private final Map<String, String> moduleNameByClass;
80+
private final Map<String, Set<String>> exportsByModule;
81+
82+
private Set<AccessibleMethod> accessibleImplementations;
83+
private Set<AccessibleMethod> inheritableAccess;
84+
85+
private ModuleClass moduleClass;
86+
private boolean isPublicClass;
87+
private boolean isFinalClass;
88+
private boolean isDeprecatedClass;
89+
private boolean isExported;
90+
91+
AccessibleMethodsVisitor(
92+
Predicate<String> modulePredicate,
93+
Map<String, String> moduleNameByClass,
94+
Map<String, Set<String>> exportsByModule
95+
) {
96+
super(Opcodes.ASM9);
97+
this.modulePredicate = modulePredicate;
98+
this.moduleNameByClass = moduleNameByClass;
99+
this.exportsByModule = exportsByModule;
100+
}
101+
102+
private static Set<AccessibleMethod> newSortedSet() {
103+
return new TreeSet<>(AccessibleMethod.COMPARATOR);
104+
}
105+
106+
Map<ModuleClass, Set<AccessibleMethod>> getAccessibleMethods() {
107+
return Collections.unmodifiableMap(accessibleImplementationsByClass);
108+
}
109+
110+
void visitOnce(ModuleClass moduleClass) {
111+
if (accessibleImplementationsByClass.containsKey(moduleClass)) {
112+
return;
113+
}
114+
if (moduleClass.clazz.startsWith("com/sun/") && moduleClass.clazz.contains("/internal/")) {
115+
// skip com.sun.*.internal classes as they are not part of the supported JDK API
116+
// even if methods override some publicly visible API
117+
return;
118+
}
119+
try {
120+
ClassReader cr = new ClassReader(moduleClass.clazz);
121+
cr.accept(this, 0);
122+
} catch (IOException e) {
123+
throw new RuntimeException(e);
124+
}
125+
}
126+
127+
@Override
128+
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
129+
final Set<AccessibleMethod> currentInheritedAccess = newSortedSet();
130+
if (superName != null) {
131+
var superModuleClass = getModuleClassFromName(superName);
132+
visitOnce(superModuleClass);
133+
currentInheritedAccess.addAll(inheritableAccessByClass.getOrDefault(superModuleClass, emptySet()));
134+
}
135+
if (interfaces != null && interfaces.length > 0) {
136+
for (var interfaceName : interfaces) {
137+
var interfaceModuleClass = getModuleClassFromName(interfaceName);
138+
visitOnce(interfaceModuleClass);
139+
currentInheritedAccess.addAll(inheritableAccessByClass.getOrDefault(interfaceModuleClass, emptySet()));
140+
}
141+
}
142+
// only initialize local state AFTER visiting all dependencies above!
143+
super.visit(version, access, name, signature, superName, interfaces);
144+
this.moduleClass = getModuleClassFromName(name);
145+
this.isExported = getModuleExports(moduleClass.module()).contains(getPackageName(name));
146+
this.isPublicClass = (access & Opcodes.ACC_PUBLIC) != 0;
147+
this.isFinalClass = (access & Opcodes.ACC_FINAL) != 0;
148+
this.isDeprecatedClass = (access & Opcodes.ACC_DEPRECATED) != 0;
149+
this.inheritableAccess = currentInheritedAccess;
150+
this.accessibleImplementations = newSortedSet();
151+
}
152+
153+
private ModuleClass getModuleClassFromName(String name) {
154+
String module = moduleNameByClass.get(name);
155+
if (module == null) {
156+
throw new IllegalStateException("Unknown module for class: " + name);
157+
}
158+
return new ModuleClass(module, name);
159+
}
160+
161+
private Set<String> getModuleExports(String module) {
162+
Set<String> exports = exportsByModule.get(module);
163+
if (exports == null) {
164+
throw new IllegalStateException("Unknown exports for module: " + module);
165+
}
166+
return exports;
167+
}
168+
169+
@Override
170+
public void visitEnd() {
171+
super.visitEnd();
172+
if (accessibleImplementationsByClass.put(moduleClass, unmodifiableSet(accessibleImplementations)) != null
173+
|| inheritableAccessByClass.put(moduleClass, unmodifiableSet(inheritableAccess)) != null) {
174+
throw new IllegalStateException("Class " + moduleClass.clazz() + " was already visited!");
175+
}
176+
}
177+
178+
private static Set<AccessibleMethod> unmodifiableSet(Set<AccessibleMethod> set) {
179+
return set.isEmpty() ? emptySet() : Collections.unmodifiableSet(set);
180+
}
181+
182+
private static String getPackageName(String className) {
183+
return ClassDesc.ofInternalName(className).packageName();
184+
}
185+
186+
@Override
187+
public final MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
188+
MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
189+
boolean isPublic = (access & Opcodes.ACC_PUBLIC) != 0;
190+
boolean isProtected = (access & Opcodes.ACC_PROTECTED) != 0;
191+
boolean isFinal = (access & Opcodes.ACC_FINAL) != 0;
192+
boolean isStatic = (access & Opcodes.ACC_STATIC) != 0;
193+
boolean isDeprecated = (access & Opcodes.ACC_DEPRECATED) != 0;
194+
if ((isPublic || isProtected) == false) {
195+
return mv;
196+
}
197+
198+
var methodDescriptor = new AccessibleMethod.Descriptor(name, descriptor, isPublic, isStatic);
199+
var method = new AccessibleMethod(methodDescriptor, isFinal, isDeprecatedClass || isDeprecated);
200+
if (isPublicClass && isExported && EXCLUDES.contains(methodDescriptor) == false) {
201+
// class is public and exported, to be accessible outside the JDK the method must be either:
202+
// - public or
203+
// - protected if not a final class
204+
if (isPublic || isFinalClass == false) {
205+
if (modulePredicate.test(moduleClass.module)) {
206+
accessibleImplementations.add(method);
207+
}
208+
// if public and not static, the method can be accessible on non-public and non-exported subclasses,
209+
// but skip constructors
210+
if (isPublic && isStatic == false && name.equals("<init>") == false) {
211+
inheritableAccess.add(method);
212+
}
213+
}
214+
} else if (inheritableAccess.contains(method)) {
215+
if (modulePredicate.test(moduleClass.module)) {
216+
accessibleImplementations.add(method);
217+
}
218+
}
219+
return mv;
220+
}
221+
}
222+
}

libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,7 @@ public static String toString(EnumSet<ExternalAccess> externalAccesses) {
2424
return externalAccesses.stream().map(Enum::toString).collect(Collectors.joining(DELIMITER));
2525
}
2626

27-
public static EnumSet<ExternalAccess> fromPermissions(
28-
boolean packageExported,
29-
boolean publicClass,
30-
boolean publicMethod,
31-
boolean protectedMethod
32-
) {
27+
public static EnumSet<ExternalAccess> fromPermissions(boolean publicAccessible, boolean publicMethod, boolean protectedMethod) {
3328
if (publicMethod && protectedMethod) {
3429
throw new IllegalArgumentException();
3530
}
@@ -41,7 +36,7 @@ public static EnumSet<ExternalAccess> fromPermissions(
4136
externalAccesses.add(ExternalAccess.PROTECTED_METHOD);
4237
}
4338

44-
if (packageExported && publicClass) {
39+
if (publicAccessible) {
4540
externalAccesses.add(ExternalAccess.PUBLIC_CLASS);
4641
}
4742
return externalAccesses;

0 commit comments

Comments
 (0)