[WIP] Non-modular plugins with synthetic module

server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java

@@ -38,6 +38,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
@@ -100,8 +101,8 @@ public record LayerAndLoader(ModuleLayer layer, ClassLoader loader) {
             Objects.requireNonNull(loader);
         }
 
-        public static LayerAndLoader ofLoader(ClassLoader loader) {
-            return new LayerAndLoader(ModuleLayer.boot(), loader);
+        public static LayerAndLoader ofLoader(UberModuleClassLoader loader) {
+            return new LayerAndLoader(loader.getLayer(), loader);
         }
     }
 
@@ -254,7 +255,7 @@ static LayerAndLoader createSPI(
             );
         } else {
             logger.debug(() -> "Loading bundle: " + plugin.getName() + ", creating spi, non-modular");
-            return LayerAndLoader.ofLoader(URLClassLoader.newInstance(bundle.spiUrls.toArray(new URL[0]), parentLoader));
+            return new LayerAndLoader(ModuleLayer.boot(), URLClassLoader.newInstance(bundle.spiUrls.toArray(new URL[0]), parentLoader));
         }
     }
 
@@ -267,26 +268,39 @@ static LayerAndLoader createPlugin(
     ) {
         final PluginDescriptor plugin = bundle.plugin;
         if (plugin.getModuleName().isPresent()) {
-            logger.debug(() -> "Loading bundle: " + plugin.getName() + ", modular");
+            logger.info(() -> "Loading bundle: " + plugin.getName() + ", modular");
             var parentLayers = Stream.concat(
                 Stream.ofNullable(spiLayerAndLoader != null ? spiLayerAndLoader.layer() : null),
                 extendedPlugins.stream().map(LoadedPluginLayer::spiModuleLayer)
             ).toList();
             return createPluginModuleLayer(bundle, pluginParentLoader, parentLayers, qualifiedExports);
         } else if (plugin.isStable()) {
-            logger.debug(() -> "Loading bundle: " + plugin.getName() + ", non-modular as synthetic module");
+            logger.info(() -> "Loading bundle: " + plugin.getName() + ", non-modular as synthetic module");
             return LayerAndLoader.ofLoader(
                 UberModuleClassLoader.getInstance(
                     pluginParentLoader,
-                    ModuleLayer.boot(),
+                    List.of(ModuleLayer.boot()),
                     "synthetic." + toModuleName(plugin.getName()),
                     bundle.allUrls,
                     Set.of("org.elasticsearch.server") // TODO: instead of denying server, allow only jvm + stable API modules
                 )
             );
         } else {
-            logger.debug(() -> "Loading bundle: " + plugin.getName() + ", non-modular");
-            return LayerAndLoader.ofLoader(URLClassLoader.newInstance(bundle.urls.toArray(URL[]::new), pluginParentLoader));
+            var syntheticName = "synthetic." + toModuleName2(plugin.getName());
+            logger.info(() -> "Loading bundle: " + plugin.getName() + ", non-modular, as synthetic module " + syntheticName);
+            var parentLayers = Stream.concat(
+                Stream.ofNullable(spiLayerAndLoader != null ? spiLayerAndLoader.layer() : null),
+                Stream.concat(extendedPlugins.stream().map(LoadedPluginLayer::spiModuleLayer), Stream.of(ModuleLayer.boot()))
+            ).toList();
+            var parentModuleNames = parentLayers.stream()
+                .flatMap(l -> l.modules().stream())
+                .map(Module::getName)
+                .distinct()
+                .collect(Collectors.joining(";"));
+            logger.info("Parent modules for {}: {}", syntheticName, parentModuleNames);
+            return LayerAndLoader.ofLoader(
+                UberModuleClassLoader.getInstance(pluginParentLoader, parentLayers, syntheticName, bundle.allUrls, Set.of())
+            );
         }
     }
 
@@ -324,7 +338,7 @@ static LayerAndLoader createPluginModuleLayer(
         );
     }
 
-    static LayerAndLoader createModuleLayer(
+    private static LayerAndLoader createModuleLayer(
         String className,
         String moduleName,
         Path[] paths,
@@ -370,6 +384,16 @@ static String toModuleName(String name) {
         return result;
     }
 
+    static String toModuleName2(String name) {
+        String result = name.replace('-', '_')
+            .replaceAll("\\W+", ".") // replace non-alphanumeric character strings with dots
+            .replaceAll("(^[^A-Za-z_]*)", "") // trim non-alpha or underscore characters from start
+            .replaceAll("\\.$", "") // trim trailing dot
+            .toLowerCase(Locale.getDefault());
+        assert ModuleSupport.isPackageName(result);
+        return result;
+    }
+
     static final String toPackageName(String className) {
         assert className.endsWith(".") == false;
         int index = className.lastIndexOf('.');

server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java

@@ -28,6 +28,7 @@
 import java.security.PrivilegedAction;
 import java.security.SecureClassLoader;
 import java.util.Enumeration;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -58,43 +59,66 @@ public class UberModuleClassLoader extends SecureClassLoader implements AutoClos
     private final ModuleLayer.Controller moduleController;
     private final Set<String> packageNames;
 
-    private static Map<String, Set<String>> getModuleToServiceMap(ModuleLayer moduleLayer) {
-        Set<String> unqualifiedExports = moduleLayer.modules()
+    private static Set<String> unqualifiedExportsForLayer(ModuleLayer layer) {
+        return layer.modules()
             .stream()
             .flatMap(module -> module.getDescriptor().exports().stream())
             .filter(Predicate.not(ModuleDescriptor.Exports::isQualified))
             .map(ModuleDescriptor.Exports::source)
             .collect(Collectors.toSet());
-        return moduleLayer.modules()
+    }
+
+    private static Set<ModuleDescriptor> readableModulesForLayer(ModuleLayer layer) {
+        return layer.modules()
             .stream()
             .map(Module::getDescriptor)
-            .filter(ModuleSupport::hasAtLeastOneUnqualifiedExport)
+            .filter(md -> ModuleSupport.hasAtLeastOneUnqualifiedExport(md) || md.isOpen() || md.isAutomatic())
+            .collect(Collectors.toSet());
+    }
+
+    private static Map<String, Set<String>> getModuleToServiceMap(List<ModuleLayer> moduleLayers) {
+        Set<String> allUnqualifiedExports = new HashSet<>();
+        Set<ModuleDescriptor> allReadableModules = new HashSet<>();
+        for (var layer : moduleLayers) {
+            allUnqualifiedExports.addAll(unqualifiedExportsForLayer(layer));
+            allReadableModules.addAll(readableModulesForLayer(layer));
+            for (var parentLayer : layer.parents()) {
+                // ModuleLayer.boot() contains all ES libs too, so we want it to be an explicit parent
+                if (parentLayer != ModuleLayer.boot()) {
+                    allUnqualifiedExports.addAll(unqualifiedExportsForLayer(parentLayer));
+                    allReadableModules.addAll(readableModulesForLayer(parentLayer));
+                    // TODO: recurse?
+                }
+            }
+        }
+
+        return allReadableModules.stream()
             .collect(
                 Collectors.toMap(
                     ModuleDescriptor::name,
                     md -> md.provides()
                         .stream()
                         .map(ModuleDescriptor.Provides::service)
-                        .filter(name -> unqualifiedExports.contains(packageName(name)))
+                        .filter(name -> allUnqualifiedExports.contains(packageName(name)))
                         .collect(Collectors.toSet())
                 )
             );
     }
 
     static UberModuleClassLoader getInstance(ClassLoader parent, String moduleName, Set<URL> jarUrls) {
-        return getInstance(parent, ModuleLayer.boot(), moduleName, jarUrls, Set.of());
+        return getInstance(parent, List.of(ModuleLayer.boot()), moduleName, jarUrls, Set.of());
     }
 
     @SuppressWarnings("removal")
     static UberModuleClassLoader getInstance(
         ClassLoader parent,
-        ModuleLayer parentLayer,
+        List<ModuleLayer> parentLayers,
         String moduleName,
         Set<URL> jarUrls,
         Set<String> moduleDenyList
     ) {
         Path[] jarPaths = jarUrls.stream().map(UberModuleClassLoader::urlToPathUnchecked).toArray(Path[]::new);
-        var parentLayerModuleToServiceMap = getModuleToServiceMap(parentLayer);
+        var parentLayerModuleToServiceMap = getModuleToServiceMap(parentLayers);
         Set<String> requires = parentLayerModuleToServiceMap.keySet()
             .stream()
             .filter(Predicate.not(moduleDenyList::contains))
@@ -110,10 +134,15 @@ static UberModuleClassLoader getInstance(
             jarPaths,
             requires,
             uses,
-            s -> isPackageInLayers(s, parentLayer)
+            s -> isPackageInLayers(s, parentLayers)
         );
         // TODO: check that denied modules are not brought as transitive dependencies (or switch to allow-list?)
-        Configuration cf = parentLayer.configuration().resolve(finder, ModuleFinder.of(), Set.of(moduleName));
+        Configuration cf = Configuration.resolve(
+            finder,
+            parentLayers.stream().map(ModuleLayer::configuration).toList(),
+            ModuleFinder.of(),
+            Set.of(moduleName)
+        );
 
         Set<String> packageNames = finder.find(moduleName).map(ModuleReference::descriptor).map(ModuleDescriptor::packages).orElseThrow();
 
@@ -122,20 +151,25 @@ static UberModuleClassLoader getInstance(
             moduleName,
             jarUrls.toArray(new URL[0]),
             cf,
-            parentLayer,
+            parentLayers,
             packageNames
         );
         return AccessController.doPrivileged(pa);
     }
 
-    private static boolean isPackageInLayers(String packageName, ModuleLayer moduleLayer) {
-        if (moduleLayer.modules().stream().map(Module::getPackages).anyMatch(p -> p.contains(packageName))) {
+    private static boolean isPackageInLayers(String packageName, List<ModuleLayer> moduleLayers) {
+        if (moduleLayers.stream().flatMap(x -> x.modules().stream()).map(Module::getPackages).anyMatch(p -> p.contains(packageName))) {
             return true;
         }
-        if (moduleLayer.parents().equals(List.of(ModuleLayer.empty()))) {
-            return false;
+        for (var moduleLayer : moduleLayers) {
+            if (moduleLayer.parents().equals(List.of(ModuleLayer.empty()))) {
+                continue;
+            }
+            if (moduleLayer.parents().stream().anyMatch(ml -> isPackageInLayers(packageName, List.of(ml)))) {
+                return true;
+            }
         }
-        return moduleLayer.parents().stream().anyMatch(ml -> isPackageInLayers(packageName, ml));
+        return false;
     }
 
     /**
@@ -146,7 +180,7 @@ private UberModuleClassLoader(
         String moduleName,
         URL[] jarURLs,
         Configuration cf,
-        ModuleLayer mparent,
+        List<ModuleLayer> mparent,
         Set<String> packageNames
     ) {
         super(parent);
@@ -157,7 +191,7 @@ private UberModuleClassLoader(
         // Defining a module layer tells the Java virtual machine about the
         // classes that may be loaded from the module, and is what makes the
         // Class::getModule call return the name of our ubermodule.
-        this.moduleController = ModuleLayer.defineModules(cf, List.of(mparent), s -> this);
+        this.moduleController = ModuleLayer.defineModules(cf, mparent, s -> this);
         this.module = this.moduleController.layer().findModule(moduleName).orElseThrow();
 
         this.packageNames = packageNames;

test/external-modules/jvm-crash/src/main/java/org/elasticsearch/test/jvm_crash/RestJvmCrashAction.java

@@ -30,7 +30,8 @@ public class RestJvmCrashAction implements RestHandler {
     static {
         try {
             AccessController.doPrivileged((PrivilegedExceptionAction<?>) () -> {
-                Class<?> unsafe = Class.forName("sun.misc.Unsafe");
+                // Bypass UberModuleClassLoader, as server does not have our very dangerous permissions
+                Class<?> unsafe = ClassLoader.getSystemClassLoader().loadClass("sun.misc.Unsafe");
 
                 FREE_MEMORY = unsafe.getMethod("freeMemory", long.class);
                 Field f = unsafe.getDeclaredField("theUnsafe");

test/external-modules/jvm-crash/src/main/plugin-metadata/plugin-security.policy

@@ -3,4 +3,5 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.lang.RuntimePermission "getClassLoader";
 };

x-pack/plugin/spatial/src/main/java/module-info.java

@@ -0,0 +1,34 @@
+module org.elasticsearch.spatial {
+    requires org.apache.lucene.spatial3d;
+    requires org.elasticsearch.h3;
+    requires org.elasticsearch.legacy.geo;
+    requires org.elasticsearch.painless.spi;
+    requires org.elasticsearch.xcore;
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.xcontent;
+    requires org.elasticsearch.base;
+    requires org.elasticsearch.geo;
+    requires org.apache.lucene.core;
+
+    exports org.elasticsearch.xpack.spatial.action to org.elasticsearch.server;
+    exports org.elasticsearch.xpack.spatial.common; // to vector-tile
+    exports org.elasticsearch.xpack.spatial; // to vector-tile
+    exports org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; // to vector-tile
+
+    exports org.elasticsearch.xpack.spatial.index.query;
+    exports org.elasticsearch.xpack.spatial.index.fielddata;
+    exports org.elasticsearch.xpack.spatial.index.fielddata.plain;
+    exports org.elasticsearch.xpack.spatial.index.mapper;
+
+    exports org.elasticsearch.xpack.spatial.ingest;
+    exports org.elasticsearch.xpack.spatial.script.field;
+
+    exports org.elasticsearch.xpack.spatial.search.aggregations;
+    exports org.elasticsearch.xpack.spatial.search.aggregations.metrics;
+    exports org.elasticsearch.xpack.spatial.search.aggregations.support;
+    exports org.elasticsearch.xpack.spatial.search.runtime;
+
+    opens org.elasticsearch.xpack.spatial to org.elasticsearch.painless.spi; // whitelist resource access
+
+    provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.spatial.SpatialPainlessExtension;
+}

x-pack/plugin/sql/build.gradle

@@ -12,7 +12,7 @@ esplugin {
   name = 'x-pack-sql'
   description 'The Elasticsearch plugin that powers SQL for Elasticsearch'
   classname = 'org.elasticsearch.xpack.sql.plugin.SqlPlugin'
-  extendedPlugins = ['x-pack-ql', 'lang-painless']
+  extendedPlugins = ['x-pack-ql', 'lang-painless', 'x-pack-core']
 }
 
 ext {