Skip to content

Commit fcf682f

Browse files
markjhoymarkjhoy
authored
Merge branch 'main' into markjhoy/default_token_pruning_sparse_vector
2 parents 6e50539 + 62b2e86 commit fcf682f

File tree

102 files changed

+904
-1812
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+904
-1812
lines changed

‎build-tools-internal/src/main/resources/checkstyle.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@
6868

6969
<!-- Unused imports are forbidden -->
7070
<module name="UnusedImports" />
71+
<module name="SuppressionCommentFilter">
72+
<property name="offCommentFormat" value="begin generated imports"/>
73+
<property name="onCommentFormat" value="end generated imports"/>
74+
<property name="checkFormat" value="UnusedImports"/>
75+
</module>
7176

7277
<!-- Non-inner classes must be in files that match their names. -->
7378
<module name="OuterTypeFilename" />

‎docs/changelog/120488.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 120488
2+
summary: Publish queue latency metrics from tracked thread pools
3+
area: "Infra/Metrics"
4+
type: enhancement
5+
issues: []

‎docs/changelog/127658.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127658
2+
summary: Append all data to Chat Completion buffer
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java‎

Lines changed: 3 additions & 243 deletions
Original file line numberDiff line numberDiff line change
@@ -10,51 +10,19 @@
1010
package org.elasticsearch.entitlement.initialization;
1111

1212
import org.elasticsearch.core.Booleans;
13-
import org.elasticsearch.core.Strings;
1413
import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
1514
import org.elasticsearch.entitlement.bridge.EntitlementChecker;
1615
import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker;
17-
import org.elasticsearch.entitlement.runtime.policy.FileAccessTree;
1816
import org.elasticsearch.entitlement.runtime.policy.PathLookup;
1917
import org.elasticsearch.entitlement.runtime.policy.Policy;
2018
import org.elasticsearch.entitlement.runtime.policy.PolicyManager;
21-
import org.elasticsearch.entitlement.runtime.policy.PolicyUtils;
22-
import org.elasticsearch.entitlement.runtime.policy.Scope;
23-
import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
24-
import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
25-
import org.elasticsearch.entitlement.runtime.policy.entitlements.ExitVMEntitlement;
26-
import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement;
27-
import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.FileData;
28-
import org.elasticsearch.entitlement.runtime.policy.entitlements.InboundNetworkEntitlement;
29-
import org.elasticsearch.entitlement.runtime.policy.entitlements.LoadNativeLibrariesEntitlement;
30-
import org.elasticsearch.entitlement.runtime.policy.entitlements.ManageThreadsEntitlement;
31-
import org.elasticsearch.entitlement.runtime.policy.entitlements.OutboundNetworkEntitlement;
32-
import org.elasticsearch.entitlement.runtime.policy.entitlements.ReadStoreAttributesEntitlement;
33-
import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
34-
import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement;
3519

3620
import java.lang.instrument.Instrumentation;
3721
import java.lang.reflect.Constructor;
3822
import java.lang.reflect.InvocationTargetException;
39-
import java.nio.file.Path;
40-
import java.util.ArrayList;
41-
import java.util.Collections;
42-
import java.util.HashSet;
43-
import java.util.List;
4423
import java.util.Map;
4524
import java.util.Set;
4625

47-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.CONFIG;
48-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.DATA;
49-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.LIB;
50-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.LOGS;
51-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.MODULES;
52-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.PLUGINS;
53-
import static org.elasticsearch.entitlement.runtime.policy.PathLookup.BaseDir.SHARED_REPO;
54-
import static org.elasticsearch.entitlement.runtime.policy.Platform.LINUX;
55-
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
56-
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
57-
5826
/**
5927
* Called by the agent during {@code agentmain} to configure the entitlement system,
6028
* instantiate and configure an {@link EntitlementChecker},
@@ -108,151 +76,11 @@ private static PolicyManager createPolicyManager() {
10876
Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
10977
PathLookup pathLookup = bootstrapArgs.pathLookup();
11078

111-
List<Scope> serverScopes = new ArrayList<>();
112-
List<FileData> serverModuleFileDatas = new ArrayList<>();
113-
Collections.addAll(
114-
serverModuleFileDatas,
115-
// Base ES directories
116-
FileData.ofBaseDirPath(PLUGINS, READ),
117-
FileData.ofBaseDirPath(MODULES, READ),
118-
FileData.ofBaseDirPath(CONFIG, READ),
119-
FileData.ofBaseDirPath(LOGS, READ_WRITE),
120-
FileData.ofBaseDirPath(LIB, READ),
121-
FileData.ofBaseDirPath(DATA, READ_WRITE),
122-
FileData.ofBaseDirPath(SHARED_REPO, READ_WRITE),
123-
// exclusive settings file
124-
FileData.ofRelativePath(Path.of("operator/settings.json"), CONFIG, READ_WRITE).withExclusive(true),
125-
// OS release on Linux
126-
FileData.ofPath(Path.of("/etc/os-release"), READ).withPlatform(LINUX),
127-
FileData.ofPath(Path.of("/etc/system-release"), READ).withPlatform(LINUX),
128-
FileData.ofPath(Path.of("/usr/lib/os-release"), READ).withPlatform(LINUX),
129-
// read max virtual memory areas
130-
FileData.ofPath(Path.of("/proc/sys/vm/max_map_count"), READ).withPlatform(LINUX),
131-
FileData.ofPath(Path.of("/proc/meminfo"), READ).withPlatform(LINUX),
132-
// load averages on Linux
133-
FileData.ofPath(Path.of("/proc/loadavg"), READ).withPlatform(LINUX),
134-
// control group stats on Linux. cgroup v2 stats are in an unpredicable
135-
// location under `/sys/fs/cgroup`, so unfortunately we have to allow
136-
// read access to the entire directory hierarchy.
137-
FileData.ofPath(Path.of("/proc/self/cgroup"), READ).withPlatform(LINUX),
138-
FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ).withPlatform(LINUX),
139-
// // io stats on Linux
140-
FileData.ofPath(Path.of("/proc/self/mountinfo"), READ).withPlatform(LINUX),
141-
FileData.ofPath(Path.of("/proc/diskstats"), READ).withPlatform(LINUX)
142-
);
143-
if (pathLookup.pidFile() != null) {
144-
serverModuleFileDatas.add(FileData.ofPath(pathLookup.pidFile(), READ_WRITE));
145-
}
146-
147-
Collections.addAll(
148-
serverScopes,
149-
new Scope(
150-
"org.elasticsearch.base",
151-
List.of(
152-
new CreateClassLoaderEntitlement(),
153-
new FilesEntitlement(
154-
List.of(
155-
// TODO: what in es.base is accessing shared repo?
156-
FileData.ofBaseDirPath(SHARED_REPO, READ_WRITE),
157-
FileData.ofBaseDirPath(DATA, READ_WRITE)
158-
)
159-
)
160-
)
161-
),
162-
new Scope("org.elasticsearch.xcontent", List.of(new CreateClassLoaderEntitlement())),
163-
new Scope(
164-
"org.elasticsearch.server",
165-
List.of(
166-
new ExitVMEntitlement(),
167-
new ReadStoreAttributesEntitlement(),
168-
new CreateClassLoaderEntitlement(),
169-
new InboundNetworkEntitlement(),
170-
new LoadNativeLibrariesEntitlement(),
171-
new ManageThreadsEntitlement(),
172-
new FilesEntitlement(serverModuleFileDatas)
173-
)
174-
),
175-
new Scope("java.desktop", List.of(new LoadNativeLibrariesEntitlement())),
176-
new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())),
177-
new Scope(
178-
"org.apache.lucene.core",
179-
List.of(
180-
new LoadNativeLibrariesEntitlement(),
181-
new ManageThreadsEntitlement(),
182-
new FilesEntitlement(List.of(FileData.ofBaseDirPath(CONFIG, READ), FileData.ofBaseDirPath(DATA, READ_WRITE)))
183-
)
184-
),
185-
new Scope(
186-
"org.apache.lucene.misc",
187-
List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))), new ReadStoreAttributesEntitlement())
188-
),
189-
new Scope(
190-
"org.apache.logging.log4j.core",
191-
List.of(new ManageThreadsEntitlement(), new FilesEntitlement(List.of(FileData.ofBaseDirPath(LOGS, READ_WRITE))))
192-
),
193-
new Scope(
194-
"org.elasticsearch.nativeaccess",
195-
List.of(new LoadNativeLibrariesEntitlement(), new FilesEntitlement(List.of(FileData.ofBaseDirPath(DATA, READ_WRITE))))
196-
)
197-
);
198-
199-
// conditionally add FIPS entitlements if FIPS only functionality is enforced
200-
if (Booleans.parseBoolean(System.getProperty("org.bouncycastle.fips.approved_only"), false)) {
201-
// if custom trust store is set, grant read access to its location, otherwise use the default JDK trust store
202-
String trustStore = System.getProperty("javax.net.ssl.trustStore");
203-
Path trustStorePath = trustStore != null
204-
? Path.of(trustStore)
205-
: Path.of(System.getProperty("java.home")).resolve("lib/security/jssecacerts");
206-
207-
Collections.addAll(
208-
serverScopes,
209-
new Scope(
210-
"org.bouncycastle.fips.tls",
211-
List.of(
212-
new FilesEntitlement(List.of(FileData.ofPath(trustStorePath, READ))),
213-
new ManageThreadsEntitlement(),
214-
new OutboundNetworkEntitlement()
215-
)
216-
),
217-
new Scope(
218-
"org.bouncycastle.fips.core",
219-
// read to lib dir is required for checksum validation
220-
List.of(new FilesEntitlement(List.of(FileData.ofBaseDirPath(LIB, READ))), new ManageThreadsEntitlement())
221-
)
222-
);
223-
}
224-
225-
var serverPolicy = new Policy(
226-
"server",
227-
bootstrapArgs.serverPolicyPatch() == null
228-
? serverScopes
229-
: PolicyUtils.mergeScopes(serverScopes, bootstrapArgs.serverPolicyPatch().scopes())
230-
);
231-
232-
// agents run without a module, so this is a special hack for the apm agent
233-
// this should be removed once https://github.com/elastic/elasticsearch/issues/109335 is completed
234-
// See also modules/apm/src/main/plugin-metadata/entitlement-policy.yaml
235-
List<Entitlement> agentEntitlements = List.of(
236-
new CreateClassLoaderEntitlement(),
237-
new ManageThreadsEntitlement(),
238-
new SetHttpsConnectionPropertiesEntitlement(),
239-
new OutboundNetworkEntitlement(),
240-
new WriteSystemPropertiesEntitlement(Set.of("AsyncProfiler.safemode")),
241-
new LoadNativeLibrariesEntitlement(),
242-
new FilesEntitlement(
243-
List.of(
244-
FileData.ofBaseDirPath(LOGS, READ_WRITE),
245-
FileData.ofPath(Path.of("/proc/meminfo"), READ),
246-
FileData.ofPath(Path.of("/sys/fs/cgroup/"), READ)
247-
)
248-
)
249-
);
250-
251-
validateFilesEntitlements(pluginPolicies, pathLookup);
79+
FilesEntitlementsValidation.validate(pluginPolicies, pathLookup);
25280

25381
return new PolicyManager(
254-
serverPolicy,
255-
agentEntitlements,
82+
HardcodedEntitlements.serverPolicy(pathLookup.pidFile(), bootstrapArgs.serverPolicyPatch()),
83+
HardcodedEntitlements.agentEntitlements(),
25684
pluginPolicies,
25785
EntitlementBootstrap.bootstrapArgs().scopeResolver(),
25886
EntitlementBootstrap.bootstrapArgs().sourcePaths(),
@@ -262,74 +90,6 @@ private static PolicyManager createPolicyManager() {
26290
);
26391
}
26492

265-
// package visible for tests
266-
static void validateFilesEntitlements(Map<String, Policy> pluginPolicies, PathLookup pathLookup) {
267-
Set<Path> readAccessForbidden = new HashSet<>();
268-
pathLookup.getBaseDirPaths(PLUGINS).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
269-
pathLookup.getBaseDirPaths(MODULES).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
270-
pathLookup.getBaseDirPaths(LIB).forEach(p -> readAccessForbidden.add(p.toAbsolutePath().normalize()));
271-
Set<Path> writeAccessForbidden = new HashSet<>();
272-
pathLookup.getBaseDirPaths(CONFIG).forEach(p -> writeAccessForbidden.add(p.toAbsolutePath().normalize()));
273-
for (var pluginPolicy : pluginPolicies.entrySet()) {
274-
for (var scope : pluginPolicy.getValue().scopes()) {
275-
var filesEntitlement = scope.entitlements()
276-
.stream()
277-
.filter(x -> x instanceof FilesEntitlement)
278-
.map(x -> ((FilesEntitlement) x))
279-
.findFirst();
280-
if (filesEntitlement.isPresent()) {
281-
var fileAccessTree = FileAccessTree.withoutExclusivePaths(filesEntitlement.get(), pathLookup, null);
282-
validateReadFilesEntitlements(pluginPolicy.getKey(), scope.moduleName(), fileAccessTree, readAccessForbidden);
283-
validateWriteFilesEntitlements(pluginPolicy.getKey(), scope.moduleName(), fileAccessTree, writeAccessForbidden);
284-
}
285-
}
286-
}
287-
}
288-
289-
private static IllegalArgumentException buildValidationException(
290-
String componentName,
291-
String moduleName,
292-
Path forbiddenPath,
293-
FilesEntitlement.Mode mode
294-
) {
295-
return new IllegalArgumentException(
296-
Strings.format(
297-
"policy for module [%s] in [%s] has an invalid file entitlement. Any path under [%s] is forbidden for mode [%s].",
298-
moduleName,
299-
componentName,
300-
forbiddenPath,
301-
mode
302-
)
303-
);
304-
}
305-
306-
private static void validateReadFilesEntitlements(
307-
String componentName,
308-
String moduleName,
309-
FileAccessTree fileAccessTree,
310-
Set<Path> readForbiddenPaths
311-
) {
312-
313-
for (Path forbiddenPath : readForbiddenPaths) {
314-
if (fileAccessTree.canRead(forbiddenPath)) {
315-
throw buildValidationException(componentName, moduleName, forbiddenPath, READ);
316-
}
317-
}
318-
}
319-
320-
private static void validateWriteFilesEntitlements(
321-
String componentName,
322-
String moduleName,
323-
FileAccessTree fileAccessTree,
324-
Set<Path> writeForbiddenPaths
325-
) {
326-
for (Path forbiddenPath : writeForbiddenPaths) {
327-
if (fileAccessTree.canWrite(forbiddenPath)) {
328-
throw buildValidationException(componentName, moduleName, forbiddenPath, READ_WRITE);
329-
}
330-
}
331-
}
332-
33393
/**
33494
* If bytecode verification is enabled, ensure these classes get loaded before transforming/retransforming them.
33595
* For these classes, the order in which we transform and verify them matters. Verification during class transformation is at least an

0 commit comments

Comments
 (0)