Skip to content

Commit 5f4a433

Browse files
fang-xing-esqlfang-xing-esql
authored
Merge branch 'main' into double-question-marks
2 parents 0c65ac4 + 359743b commit 5f4a433

File tree

22 files changed

+301
-90
lines changed

22 files changed

+301
-90
lines changed

‎Could‎

Whitespace-only changes.

‎benchmarks/build.gradle‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ base {
2525
archivesName = 'elasticsearch-benchmarks'
2626
}
2727

28-
tasks.named("test").configure { enabled = false }
2928
tasks.named("javadoc").configure { enabled = false }
3029

3130
configurations {
@@ -52,8 +51,10 @@ dependencies {
5251
api "org.openjdk.jmh:jmh-core:$versions.jmh"
5352
annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
5453
// Dependencies of JMH
55-
runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.4'
54+
runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.2'
5655
runtimeOnly 'org.apache.commons:commons-math3:3.6.1'
56+
57+
testImplementation(project(':test:framework'))
5758
}
5859

5960
// enable the JMH's BenchmarkProcessor to generate the final benchmark classes

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.apache.lucene.util.BytesRef;
1313
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
14+
import org.elasticsearch.common.logging.LogConfigurator;
1415
import org.elasticsearch.common.settings.Settings;
1516
import org.elasticsearch.common.util.BigArrays;
1617
import org.elasticsearch.compute.data.Block;
@@ -28,6 +29,8 @@
2829
import org.elasticsearch.compute.operator.EvalOperator;
2930
import org.elasticsearch.compute.operator.Operator;
3031
import org.elasticsearch.core.TimeValue;
32+
import org.elasticsearch.logging.LogManager;
33+
import org.elasticsearch.logging.Logger;
3134
import org.elasticsearch.xpack.esql.core.expression.Expression;
3235
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
3336
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@@ -89,9 +92,16 @@ public class EvalBenchmark {
8992
static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
9093

9194
static {
95+
LogConfigurator.configureESLogging();
9296
// Smoke test all the expected values and force loading subclasses more like prod
97+
selfTest();
98+
}
99+
100+
static void selfTest() {
101+
Logger log = LogManager.getLogger(EvalBenchmark.class);
93102
try {
94103
for (String operation : EvalBenchmark.class.getField("operation").getAnnotationsByType(Param.class)[0].value()) {
104+
log.info("self testing {}", operation);
95105
run(operation);
96106
}
97107
} catch (NoSuchFieldException e) {

‎benchmarks/src/test/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmarkTests.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.compute.operator;
11+
12+
import org.elasticsearch.test.ESTestCase;
13+
14+
public class EvalBenchmarkTests extends ESTestCase {
15+
public void testSelfTest() {
16+
EvalBenchmark.selfTest();
17+
}
18+
}

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
6666
"-Dlog4j2.disable.jmx=true",
6767
"-Dlog4j2.formatMsgNoLookups=true",
6868
"-Djava.locale.providers=CLDR",
69+
// Enable vectorization for whatever version we are running. This ensures we use vectorization even when running EA builds.
70+
"-Dorg.apache.lucene.vectorization.upperJavaFeatureVersion=" + Runtime.version().feature(),
6971
// Pass through distribution type
7072
"-Des.distribution.type=" + distroType
7173
),

‎docs/changelog/124823.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124823
2+
summary: Report failures on partial results
3+
area: "ES|QL"
4+
type: enhancement
5+
issues: []

‎docs/changelog/124931.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124931
2+
summary: This PR fixes a bug whereby partial snapshots of system datastreams could be used to restore system features.
3+
area: "Snapshot/Restore"
4+
type: bug
5+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ private static PolicyManager createPolicyManager() {
213213
new FilesEntitlement(serverModuleFileDatas)
214214
)
215215
),
216+
new Scope("java.desktop", List.of(new LoadNativeLibrariesEntitlement())),
216217
new Scope("org.apache.httpcomponents.httpclient", List.of(new OutboundNetworkEntitlement())),
217218
new Scope("io.netty.transport", List.of(new InboundNetworkEntitlement(), new OutboundNetworkEntitlement())),
218219
new Scope(

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java‎

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@
5959
import static java.util.zip.ZipFile.OPEN_READ;
6060

6161
public class PolicyManager {
62-
private static final Logger logger = LogManager.getLogger(PolicyManager.class);
62+
/**
63+
* Use this if you don't have a {@link ModuleEntitlements} in hand.
64+
*/
65+
private static final Logger generalLogger = LogManager.getLogger(PolicyManager.class);
6366

6467
static final String UNKNOWN_COMPONENT_NAME = "(unknown)";
6568
static final String SERVER_COMPONENT_NAME = "(server)";
@@ -76,7 +79,8 @@ public class PolicyManager {
7679
record ModuleEntitlements(
7780
String componentName,
7881
Map<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType,
79-
FileAccessTree fileAccess
82+
FileAccessTree fileAccess,
83+
Logger logger
8084
) {
8185

8286
ModuleEntitlements {
@@ -101,8 +105,13 @@ private FileAccessTree getDefaultFileAccess(String componentName, Path component
101105
}
102106

103107
// pkg private for testing
104-
ModuleEntitlements defaultEntitlements(String componentName, Path componentPath) {
105-
return new ModuleEntitlements(componentName, Map.of(), getDefaultFileAccess(componentName, componentPath));
108+
ModuleEntitlements defaultEntitlements(String componentName, Path componentPath, String moduleName) {
109+
return new ModuleEntitlements(
110+
componentName,
111+
Map.of(),
112+
getDefaultFileAccess(componentName, componentPath),
113+
getLogger(componentName, moduleName)
114+
);
106115
}
107116

108117
// pkg private for testing
@@ -116,7 +125,8 @@ ModuleEntitlements policyEntitlements(String componentName, Path componentPath,
116125
return new ModuleEntitlements(
117126
componentName,
118127
entitlements.stream().collect(groupingBy(Entitlement::getClass)),
119-
FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPath, exclusivePaths)
128+
FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPath, exclusivePaths),
129+
getLogger(componentName, moduleName)
120130
);
121131
}
122132

@@ -255,17 +265,17 @@ private void neverEntitled(Class<?> callerClass, Supplier<String> operationDescr
255265
return;
256266
}
257267

258-
String componentName = getEntitlements(requestingClass).componentName();
268+
ModuleEntitlements entitlements = getEntitlements(requestingClass);
259269
notEntitled(
260270
Strings.format(
261271
"component [%s], module [%s], class [%s], operation [%s]",
262-
componentName,
272+
entitlements.componentName(),
263273
requestingClass.getModule().getName(),
264274
requestingClass,
265275
operationDescription.get()
266276
),
267277
callerClass,
268-
componentName
278+
entitlements
269279
);
270280
}
271281

@@ -323,7 +333,7 @@ public void checkFileRead(Class<?> callerClass, File file) {
323333
private static boolean isPathOnDefaultFilesystem(Path path) {
324334
var pathFileSystemClass = path.getFileSystem().getClass();
325335
if (path.getFileSystem().getClass() != DEFAULT_FILESYSTEM_CLASS) {
326-
logger.trace(
336+
generalLogger.trace(
327337
() -> Strings.format(
328338
"File entitlement trivially allowed: path [%s] is for a different FileSystem class [%s], default is [%s]",
329339
path.toString(),
@@ -383,7 +393,7 @@ public void checkFileRead(Class<?> callerClass, Path path, boolean followLinks)
383393
realPath == null ? path : Strings.format("%s -> %s", path, realPath)
384394
),
385395
callerClass,
386-
entitlements.componentName()
396+
entitlements
387397
);
388398
}
389399
}
@@ -413,7 +423,7 @@ public void checkFileWrite(Class<?> callerClass, Path path) {
413423
path
414424
),
415425
callerClass,
416-
entitlements.componentName()
426+
entitlements
417427
);
418428
}
419429
}
@@ -502,18 +512,19 @@ private void checkFlagEntitlement(
502512
PolicyParser.getEntitlementTypeName(entitlementClass)
503513
),
504514
callerClass,
505-
classEntitlements.componentName()
515+
classEntitlements
506516
);
507517
}
508-
logger.debug(
509-
() -> Strings.format(
510-
"Entitled: component [%s], module [%s], class [%s], entitlement [%s]",
511-
classEntitlements.componentName(),
512-
requestingClass.getModule().getName(),
513-
requestingClass,
514-
PolicyParser.getEntitlementTypeName(entitlementClass)
515-
)
516-
);
518+
classEntitlements.logger()
519+
.debug(
520+
() -> Strings.format(
521+
"Entitled: component [%s], module [%s], class [%s], entitlement [%s]",
522+
classEntitlements.componentName(),
523+
requestingClass.getModule().getName(),
524+
requestingClass,
525+
PolicyParser.getEntitlementTypeName(entitlementClass)
526+
)
527+
);
517528
}
518529

519530
public void checkWriteProperty(Class<?> callerClass, String property) {
@@ -524,15 +535,16 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
524535

525536
ModuleEntitlements entitlements = getEntitlements(requestingClass);
526537
if (entitlements.getEntitlements(WriteSystemPropertiesEntitlement.class).anyMatch(e -> e.properties().contains(property))) {
527-
logger.debug(
528-
() -> Strings.format(
529-
"Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
530-
entitlements.componentName(),
531-
requestingClass.getModule().getName(),
532-
requestingClass,
533-
property
534-
)
535-
);
538+
entitlements.logger()
539+
.debug(
540+
() -> Strings.format(
541+
"Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
542+
entitlements.componentName(),
543+
requestingClass.getModule().getName(),
544+
requestingClass,
545+
property
546+
)
547+
);
536548
return;
537549
}
538550
notEntitled(
@@ -544,26 +556,34 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
544556
property
545557
),
546558
callerClass,
547-
entitlements.componentName()
559+
entitlements
548560
);
549561
}
550562

551-
private void notEntitled(String message, Class<?> callerClass, String componentName) {
563+
private void notEntitled(String message, Class<?> callerClass, ModuleEntitlements entitlements) {
552564
var exception = new NotEntitledException(message);
553565
// Don't emit a log for muted classes, e.g. classes containing self tests
554566
if (mutedClasses.contains(callerClass) == false) {
555-
var moduleName = callerClass.getModule().getName();
556-
var loggerSuffix = "." + componentName + "." + ((moduleName == null) ? ALL_UNNAMED : moduleName);
557-
var notEntitledLogger = LogManager.getLogger(PolicyManager.class.getName() + loggerSuffix);
558-
String frameInfoSuffix = StackWalker.getInstance(RETAIN_CLASS_REFERENCE)
559-
.walk(this::findRequestingFrame)
560-
.map(frame -> "\n\tat " + frame)
561-
.orElse("");
562-
notEntitledLogger.warn("Not entitled: " + message + frameInfoSuffix);
567+
entitlements.logger().warn("Not entitled:", exception);
563568
}
564569
throw exception;
565570
}
566571

572+
private static Logger getLogger(String componentName, String moduleName) {
573+
var loggerSuffix = "." + componentName + "." + ((moduleName == null) ? ALL_UNNAMED : moduleName);
574+
return MODULE_LOGGERS.computeIfAbsent(PolicyManager.class.getName() + loggerSuffix, LogManager::getLogger);
575+
}
576+
577+
/**
578+
* We want to use the same {@link Logger} object for a given name, because we want {@link ModuleEntitlements}
579+
* {@code equals} and {@code hashCode} to work.
580+
* <p>
581+
* This would not be required if LogManager
582+
* <a href="https://github.com/elastic/elasticsearch/issues/87511">memoized the loggers</a>,
583+
* but here we are.
584+
*/
585+
private static final ConcurrentHashMap<String, Logger> MODULE_LOGGERS = new ConcurrentHashMap<>();
586+
567587
public void checkManageThreadsEntitlement(Class<?> callerClass) {
568588
checkEntitlementPresent(callerClass, ManageThreadsEntitlement.class);
569589
}
@@ -596,7 +616,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
596616
if (pluginName != null) {
597617
var pluginEntitlements = pluginsEntitlements.get(pluginName);
598618
if (pluginEntitlements == null) {
599-
return defaultEntitlements(pluginName, sourcePaths.get(pluginName));
619+
return defaultEntitlements(pluginName, sourcePaths.get(pluginName), requestingModule.getName());
600620
} else {
601621
return getModuleScopeEntitlements(
602622
pluginEntitlements,
@@ -617,7 +637,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
617637
);
618638
}
619639

620-
return defaultEntitlements(UNKNOWN_COMPONENT_NAME, null);
640+
return defaultEntitlements(UNKNOWN_COMPONENT_NAME, null, requestingModule.getName());
621641
}
622642

623643
private static String getScopeName(Module requestingModule) {
@@ -638,7 +658,7 @@ static Path getComponentPathFromClass(Class<?> requestingClass) {
638658
return Paths.get(codeSource.getLocation().toURI());
639659
} catch (Exception e) {
640660
// If we get a URISyntaxException, or any other Exception due to an invalid URI, we return null to safely skip this location
641-
logger.info(
661+
generalLogger.info(
642662
"Cannot get component path for [{}]: [{}] cannot be converted to a valid Path",
643663
requestingClass.getName(),
644664
codeSource.getLocation().toString()
@@ -655,7 +675,7 @@ private ModuleEntitlements getModuleScopeEntitlements(
655675
) {
656676
var entitlements = scopeEntitlements.get(scopeName);
657677
if (entitlements == null) {
658-
return defaultEntitlements(componentName, componentPath);
678+
return defaultEntitlements(componentName, componentPath, scopeName);
659679
}
660680
return policyEntitlements(componentName, componentPath, scopeName, entitlements);
661681
}
@@ -698,18 +718,18 @@ Optional<StackFrame> findRequestingFrame(Stream<StackFrame> frames) {
698718
* @return true if permission is granted regardless of the entitlement
699719
*/
700720
private static boolean isTriviallyAllowed(Class<?> requestingClass) {
701-
if (logger.isTraceEnabled()) {
702-
logger.trace("Stack trace for upcoming trivially-allowed check", new Exception());
721+
if (generalLogger.isTraceEnabled()) {
722+
generalLogger.trace("Stack trace for upcoming trivially-allowed check", new Exception());
703723
}
704724
if (requestingClass == null) {
705-
logger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library");
725+
generalLogger.debug("Entitlement trivially allowed: no caller frames outside the entitlement library");
706726
return true;
707727
}
708728
if (systemModules.contains(requestingClass.getModule())) {
709-
logger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName());
729+
generalLogger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName());
710730
return true;
711731
}
712-
logger.trace("Entitlement not trivially allowed");
732+
generalLogger.trace("Entitlement not trivially allowed");
713733
return false;
714734
}
715735

0 commit comments

Comments
 (0)