Support adding to injector instances from fields

build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateBundleManifestTask.java

@@ -51,11 +51,12 @@ public GenerateBundleManifestTask(WorkerExecutor workerExecutor, ExecOperations
     @TaskAction
     public void scanPluginClasses() {
         File outputFile = projectLayout.getBuildDirectory().file(NAMED_COMPONENTS_PATH).get().getAsFile();
+        File moduleDirectory = projectLayout.getProjectDirectory().getAsFile();
 
         ExecResult execResult = LoggedExec.javaexec(execOperations, spec -> {
             spec.classpath(pluginScannerClasspath.plus(getClasspath()).getAsPath());
             spec.getMainClass().set("org.elasticsearch.plugin.scanner.ManifestBuilder");
-            spec.args(outputFile);
+            spec.args(outputFile, moduleDirectory);
             spec.setErrorOutput(System.err);
             spec.setStandardOutput(System.out);
         });

libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extension.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.TYPE;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(value = { TYPE, FIELD })
+public @interface Extension {
+}

libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ManifestBuilder.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.plugin.Component;
 import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.Extension;
 import org.elasticsearch.plugin.MultipleRegistryEntries;
 import org.elasticsearch.plugin.NamedComponent;
 import org.elasticsearch.plugin.RegistryCtor;
@@ -21,34 +22,43 @@
 import org.objectweb.asm.AnnotationVisitor;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.FieldVisitor;
 import org.objectweb.asm.MethodVisitor;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.lang.reflect.Modifier;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiFunction;
+import java.util.stream.Stream;
 
 public class ManifestBuilder {
 
     // main method to be used by gradle build plugin
     public static void main(String[] args) throws IOException {
         List<ClassReader> classReaders = ClassReaders.ofClassPath();
 
-        List<String> components = findComponents(classReaders);
+        List<String> componentsClasses = findComponents(classReaders);
         Map<String, List<EntryInfo>> registries = findRegistries(classReaders);
         Map<String, List<NamedComponentInfo>> namedComponents = findNamedComponents(classReaders);
+        Map<String, Set<String>> extensionsFields = findExtensionsFields(ClassReaders.ofPaths(Stream.of(Path.of(args[1]))));
+
         Path outputFile = Path.of(args[0]);
-        ManifestBuilder.writeToFile(components, registries, namedComponents, outputFile);
+        ManifestBuilder.writeToFile(componentsClasses, extensionsFields, registries, namedComponents, outputFile);
     }
 
-    public static void writeToFile(List<String> components, Map<String, List<EntryInfo>> registries, Map<String, List<NamedComponentInfo>> namedComponents, Path outputFile) throws IOException {
+    public static void writeToFile(List<String> componentsClasses, Map<String, Set<String>> extensionsFields,
+                                   Map<String, List<EntryInfo>> registries,
+                                   Map<String, List<NamedComponentInfo>> namedComponents, Path outputFile) throws IOException {
         Files.createDirectories(outputFile.getParent());
 
         try (OutputStream outputStream = Files.newOutputStream(outputFile)) {
@@ -57,7 +67,17 @@ public static void writeToFile(List<String> components, Map<String, List<EntryIn
 
                 builder.startObject();
 
-                builder.array("components", components.toArray(new String[0]));
+                builder.array("components", componentsClasses.toArray(new String[0]));
+
+                builder.startObject("extensions_fields");
+                for (Map.Entry<String, Set<String>> entry : extensionsFields.entrySet()) {
+                    builder.startArray(entry.getKey());
+                    for (var value : entry.getValue()) {
+                        builder.value(value);
+                    }
+                    builder.endArray();
+                }
+                builder.endObject();
 
                 builder.startObject("registries");
                 for (var entry : registries.entrySet()) {
@@ -242,6 +262,63 @@ public AnnotationVisitor visitAnnotation(String name, String descriptor) {
         return registries;
     }
 
+    private static Map<String, Set<String>> findExtensionsFields(List<ClassReader> classReaders) {
+        Set<String> extensibleClasses = new HashSet<>();
+        Map<String, Set<String>> extensionFields = new HashMap<>();
+
+        // TODO: merge with component scanner?
+        for (ClassReader classReader : classReaders) {
+            classReader.accept(new ClassVisitor(Opcodes.ASM9) {
+                private String currentClassName;
+
+                @Override
+                public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+                    currentClassName = pathToClassName(name);
+                }
+
+                @Override
+                public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) {
+                    if (descriptor.equals(Type.getDescriptor(Extensible.class))) {
+                        extensibleClasses.add(currentClassName);
+                    }
+                    return super.visitAnnotation(descriptor, visible);
+                }
+
+                @Override
+                public FieldVisitor visitField(int access, String fieldName, String descriptor, String signature, Object value) {
+                    if (Modifier.isStatic(access) == false || Modifier.isFinal(access) == false) {
+                        return super.visitField(access, fieldName, descriptor, signature, value);
+                    }
+                    return new FieldVisitor(Opcodes.ASM9) {
+                        @Override
+                        public AnnotationVisitor visitAnnotation(String annotationDescriptor, boolean visible) {
+                            if (annotationDescriptor.equals(Type.getDescriptor(Extension.class))) {
+                                Type type = Type.getType(descriptor);
+                                extensionFields.compute(type.getClassName(), (k, set) -> {
+                                    if (set == null) {
+                                        set = new HashSet<>();
+                                    }
+                                    set.add(currentClassName + "#" + fieldName);
+                                    return set;
+                                });
+                            }
+                            return super.visitAnnotation(annotationDescriptor, visible);
+                        }
+                    };
+                }
+            }, ClassReader.SKIP_CODE);
+        }
+
+        if (extensibleClasses.containsAll(extensionFields.keySet()) == false) {
+            System.out.println(extensibleClasses);
+            System.out.println(extensionFields);
+            extensionFields.keySet().removeAll(extensibleClasses);
+            // TODO: we don't scan dependencies now, so we can't check that class actually extends the right thing
+//            throw new RuntimeException("Some extension fields are not defined as extensible classes: " + extensionFields.values());
+        }
+        return extensionFields;
+    }
+
     private static String pathToClassName(String classWithSlashes) {
         return classWithSlashes.replace('/', '.');
     }

modules/repository-azure/src/main/java/module-info.java

@@ -29,4 +29,5 @@
     requires reactor.netty.core;
     requires reactor.netty.http;
     requires com.azure.storage.blob.batch;
+    requires org.elasticsearch.plugin;
 }

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -19,13 +19,13 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.plugin.Extension;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.threadpool.ExecutorBuilder;
-import org.elasticsearch.threadpool.ScalingExecutorBuilder;
+import org.elasticsearch.threadpool.ScalingExecutorBuilderSpec;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.security.AccessController;
@@ -115,19 +115,13 @@ public List<Setting<?>> getSettings() {
         );
     }
 
-    @Override
-    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settingsToUse) {
-        return List.of(executorBuilder(), nettyEventLoopExecutorBuilder(settingsToUse));
-    }
+    @Extension
+    public static final ScalingExecutorBuilderSpec REPOSITORY_THREAD_POOL = new ScalingExecutorBuilderSpec(REPOSITORY_THREAD_POOL_NAME,
+        0, 5, TimeValue.timeValueSeconds(30L), false);
 
-    public static ExecutorBuilder<?> executorBuilder() {
-        return new ScalingExecutorBuilder(REPOSITORY_THREAD_POOL_NAME, 0, 5, TimeValue.timeValueSeconds(30L), false);
-    }
-
-    public static ExecutorBuilder<?> nettyEventLoopExecutorBuilder(Settings settings) {
-        int eventLoopThreads = AzureClientProvider.eventLoopThreadsFromSettings(settings);
-        return new ScalingExecutorBuilder(NETTY_EVENT_LOOP_THREAD_POOL_NAME, 0, eventLoopThreads, TimeValue.timeValueSeconds(30L), false);
-    }
+    @Extension
+    public static final ScalingExecutorBuilderSpec NETTY_EVENT_LOOP_THREAD_POOL = new ScalingExecutorBuilderSpec(NETTY_EVENT_LOOP_THREAD_POOL_NAME,
+        0, AzureClientProvider::eventLoopThreadsFromSettings, TimeValue.timeValueSeconds(30L), false);
 
     @Override
     public void reload(Settings settingsToLoad) {

server/src/main/java/org/elasticsearch/injection/Injector.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.injection.api.Inject;
 import org.elasticsearch.injection.spec.AmbiguousSpec;
 import org.elasticsearch.injection.spec.ExistingInstanceSpec;
+import org.elasticsearch.injection.spec.ExistingMultipleInstancesSpec;
 import org.elasticsearch.injection.spec.InjectionSpec;
 import org.elasticsearch.injection.spec.MethodHandleSpec;
 import org.elasticsearch.injection.spec.ParameterModifier;
@@ -26,6 +27,7 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.LinkedHashMap;
@@ -120,6 +122,22 @@ public Injector addInstances(Collection<?> objects) {
         return this;
     }
 
+    public <T> Injector addExtensionInstances(Class<T> type, Collection<?> objects) {
+        UnambiguousSpec spec = seedSpecs.computeIfAbsent(type, i -> new ExistingMultipleInstancesSpec(type, new ArrayList<>()));
+        if (spec instanceof ExistingMultipleInstancesSpec == false) {
+            throw new IllegalStateException("There's already an object for " + type);
+        }
+        ((ExistingMultipleInstancesSpec) spec).instances().addAll(objects);
+        return this;
+    }
+
+    public Injector addExtensionsInstances(Map<Class<?>, ? extends Collection<?>> objects) {
+        for (Map.Entry<Class<?>, ? extends Collection<?>> entry : objects.entrySet()) {
+            addExtensionInstances(entry.getKey(), entry.getValue());
+        }
+        return this;
+    }
+
     /**
      * Indicates that <code>object</code> is to be injected for parameters of type <code>type</code>.
      * The given object is treated as though it had been instantiated by the injector.
@@ -156,6 +174,8 @@ private PlanInterpreter doInjection() {
         specMap.values().forEach((spec) -> {
             if (spec instanceof ExistingInstanceSpec e) {
                 existingInstances.put(e.requestedType(), e.instance());
+            } else if (spec instanceof ExistingMultipleInstancesSpec s) {
+                existingInstances.put(s.requestedType(), s.instances());
             }
         });
         PlanInterpreter interpreter = new PlanInterpreter(existingInstances, new ProxyPool());
@@ -200,7 +220,7 @@ private static Map<Class<?>, InjectionSpec> specClosure(Map<Class<?>, Unambiguou
             }
 
             InjectionSpec spec = seedMap.get(c);
-            if (spec instanceof ExistingInstanceSpec) {
+            if (spec instanceof ExistingInstanceSpec || spec instanceof ExistingMultipleInstancesSpec) {
                 // simple!
                 result.put(c, spec);
                 continue;

server/src/main/java/org/elasticsearch/injection/PlanInterpreter.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.injection.spec.MethodHandleSpec;
+import org.elasticsearch.injection.spec.ParameterModifier;
 import org.elasticsearch.injection.spec.ParameterSpec;
 import org.elasticsearch.injection.step.CreateCollectionProxyStep;
 import org.elasticsearch.injection.step.CreateInstanceProxyStep;
@@ -70,7 +71,15 @@ final class PlanInterpreter {
 
     PlanInterpreter(Map<Class<?>, Object> existingInstances, ProxyPool proxyPool) {
         this.proxyPool = proxyPool;
-        existingInstances.forEach(this::addInstance);
+        for (Map.Entry<Class<?>, Object> entry : existingInstances.entrySet()) {
+            Class<?> type = entry.getKey();
+            Object value = entry.getValue();
+            if (value instanceof Collection<?> values) {
+                addInstances(type, values);
+            } else {
+                addInstance(type, value);
+            }
+        }
     }
 
     /**
@@ -181,7 +190,11 @@ private Object instantiate(MethodHandleSpec spec) {
     }
 
     private Object parameterValue(ParameterSpec parameterSpec) {
-        return theInstanceOf(parameterSpec.formalType());
+        if (parameterSpec.modifiers().contains(ParameterModifier.COLLECTION)) {
+            return instancesOf(parameterSpec.injectableType());
+        } else {
+            return theInstanceOf(parameterSpec.formalType());
+        }
     }
 
 }

server/src/main/java/org/elasticsearch/injection/Planner.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.injection.spec.AmbiguousSpec;
 import org.elasticsearch.injection.spec.ExistingInstanceSpec;
+import org.elasticsearch.injection.spec.ExistingMultipleInstancesSpec;
 import org.elasticsearch.injection.spec.InjectionSpec;
 import org.elasticsearch.injection.spec.MethodHandleSpec;
 import org.elasticsearch.injection.spec.ParameterSpec;
@@ -117,6 +118,10 @@ private void planForSpec(InjectionSpec spec, int depth) {
                 // Nothing to do. The injector will already have the required object.
                 logger.trace("{}- Plan {}", indent(depth), e);
             }
+            case ExistingMultipleInstancesSpec e -> {
+                // Nothing to do. The injector will already have the required objects.
+                logger.trace("{}- Plan {}", indent(depth), e);
+            }
             case MethodHandleSpec m -> {
                 for (var p : m.parameters()) {
                     planParameter(p, depth);

server/src/main/java/org/elasticsearch/injection/spec/ExistingMultipleInstancesSpec.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.injection.spec;
+
+import java.util.Collection;
+
+/**
+ * Indicates that a type should be injected by passing a particular collection {@link #instances}.
+ */
+public record ExistingMultipleInstancesSpec(Class<?> requestedType, Collection<Object> instances) implements UnambiguousSpec {
+    @Override
+    public String toString() {
+        // Don't call instance.toString; who knows what that will return
+        return "ExistingMultipleInstanceSpec[" + "requestedType=" + requestedType + ']';
+    }
+
+    public void addInstances(Collection<?> instances) {
+        this.instances.addAll(instances);
+    }
+}

server/src/main/java/org/elasticsearch/injection/spec/UnambiguousSpec.java

@@ -12,4 +12,5 @@
 /**
  * Indicates that there is just one way to inject {@link #requestedType}.
  */
-public sealed interface UnambiguousSpec extends InjectionSpec permits ExistingInstanceSpec, MethodHandleSpec, SubtypeSpec {}
+public sealed interface UnambiguousSpec extends InjectionSpec
+    permits ExistingInstanceSpec, ExistingMultipleInstancesSpec, MethodHandleSpec, SubtypeSpec {}