Make internal classloader instrumentation indy compatible

instrumentation/internal/internal-class-loader/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/internal/classloader/BootDelegationInstrumentation.java

@@ -47,10 +47,7 @@ public class BootDelegationInstrumentation implements TypeInstrumentation {
   public ElementMatcher<TypeDescription> typeMatcher() {
     // just an optimization to exclude common class loaders that are known to delegate to the
     // bootstrap loader (or happen to _be_ the bootstrap loader)
-    return not(namedOneOf(
-            "java.lang.ClassLoader",
-            "com.ibm.oti.vm.BootstrapClassLoader",
-            "io.opentelemetry.javaagent.bootstrap.AgentClassLoader"))
+    return not(namedOneOf("java.lang.ClassLoader", "com.ibm.oti.vm.BootstrapClassLoader"))
         .and(extendsClass(named("java.lang.ClassLoader")));
   }
 
@@ -136,13 +133,14 @@ public static Class<?> onEnter(@Advice.Argument(0) String name) {
       return null;
     }
 
-    @Advice.OnMethodExit(onThrowable = Throwable.class)
-    public static void onExit(
-        @Advice.Return(readOnly = false) Class<?> result,
-        @Advice.Enter Class<?> resultFromBootstrapLoader) {
+    @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
+    @Advice.AssignReturned.ToReturned
+    public static Class<?> onExit(
+        @Advice.Return Class<?> result, @Advice.Enter Class<?> resultFromBootstrapLoader) {
       if (resultFromBootstrapLoader != null) {
-        result = resultFromBootstrapLoader;
+        return resultFromBootstrapLoader;
       }
+      return result;
     }
   }
 }

instrumentation/internal/internal-class-loader/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/internal/classloader/ClassLoaderInstrumentationModule.java

@@ -25,13 +25,10 @@ public boolean defaultEnabled(ConfigProperties config) {
     return true;
   }
 
-  @Override
-  public boolean isIndyModule() {
-    return false;
-  }
-
   @Override
   public boolean isHelperClass(String className) {
+    // TODO: this can be removed when we drop inlined-advice support
+    // The advices can directly access this class in the AgentClassLoader with invokedynamic Advice
     return className.equals("io.opentelemetry.javaagent.tooling.Constants");
   }
 

instrumentation/internal/internal-class-loader/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/internal/classloader/DefineClassInstrumentation.java

@@ -52,9 +52,15 @@ public static DefineClassContext onEnter(
           classLoader, className, classBytes, offset, length);
     }
 
+    // TODO: the ToReturned does nothing except for signaling the AdviceTransformer that it must
+    // not touch this advice
+    // this is done because we do not want the return values to be wrapped in array types
     @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
-    public static void onExit(@Advice.Enter DefineClassContext context) {
+    @Advice.AssignReturned.ToReturned
+    public static Class<?> onExit(
+        @Advice.Enter DefineClassContext context, @Advice.Return Class<?> returned) {
       DefineClassHelper.afterDefineClass(context);
+      return returned;
     }
   }
 
@@ -68,9 +74,15 @@ public static DefineClassContext onEnter(
       return DefineClassHelper.beforeDefineClass(classLoader, className, classBytes);
     }
 
+    // TODO: the ToReturned does nothing except for signaling the AdviceTransformer that it must
+    // not touch this advice
+    // this is done because we do not want the return values to be wrapped in array types
     @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
-    public static void onExit(@Advice.Enter DefineClassContext context) {
+    @Advice.AssignReturned.ToReturned
+    public static Class<?> onExit(
+        @Advice.Enter DefineClassContext context, @Advice.Return Class<?> returned) {
       DefineClassHelper.afterDefineClass(context);
+      return returned;
     }
   }
 }

instrumentation/internal/internal-class-loader/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/internal/classloader/LoadInjectedClassInstrumentation.java

@@ -65,11 +65,13 @@ public static Class<?> onEnter(
     }
 
     @Advice.OnMethodExit(onThrowable = Throwable.class)
-    public static void onExit(
-        @Advice.Return(readOnly = false) Class<?> result, @Advice.Enter Class<?> loadedClass) {
+    @Advice.AssignReturned.ToReturned
+    public static Class<?> onExit(
+        @Advice.Return Class<?> result, @Advice.Enter Class<?> loadedClass) {
       if (loadedClass != null) {
-        result = loadedClass;
+        return loadedClass;
       }
+      return result;
     }
   }
 }

instrumentation/internal/internal-class-loader/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/internal/classloader/ResourceInjectionInstrumentation.java

@@ -61,37 +61,37 @@ public void transform(TypeTransformer transformer) {
   public static class GetResourceAdvice {
 
     @Advice.OnMethodExit(suppress = Throwable.class)
-    public static void onExit(
+    @Advice.AssignReturned.ToReturned
+    public static URL onExit(
         @Advice.This ClassLoader classLoader,
         @Advice.Argument(0) String name,
-        @Advice.Return(readOnly = false) URL resource) {
-      if (resource != null) {
-        return;
-      }
-
-      URL helper = HelperResources.loadOne(classLoader, name);
-      if (helper != null) {
-        resource = helper;
+        @Advice.Return URL resource) {
+      if (resource == null) {
+        URL helper = HelperResources.loadOne(classLoader, name);
+        if (helper != null) {
+          return helper;
+        }
       }
+      return resource;
     }
   }
 
   @SuppressWarnings("unused")
   public static class GetResourcesAdvice {
 
     @Advice.OnMethodExit(suppress = Throwable.class)
-    public static void onExit(
+    @Advice.AssignReturned.ToReturned
+    public static Enumeration<URL> onExit(
         @Advice.This ClassLoader classLoader,
         @Advice.Argument(0) String name,
-        @Advice.Return(readOnly = false) Enumeration<URL> resources) {
+        @Advice.Return Enumeration<URL> resources) {
       List<URL> helpers = HelperResources.loadAll(classLoader, name);
       if (helpers.isEmpty()) {
-        return;
+        return resources;
       }
 
       if (!resources.hasMoreElements()) {
-        resources = Collections.enumeration(helpers);
-        return;
+        return Collections.enumeration(helpers);
       }
 
       List<URL> result = Collections.list(resources);
@@ -108,31 +108,30 @@ public static void onExit(
           result.add(helperUrl);
         }
       }
-
-      resources = Collections.enumeration(result);
+      return Collections.enumeration(result);
     }
   }
 
   @SuppressWarnings("unused")
   public static class GetResourceAsStreamAdvice {
 
     @Advice.OnMethodExit(suppress = Throwable.class)
-    public static void onExit(
+    @Advice.AssignReturned.ToReturned
+    public static InputStream onExit(
         @Advice.This ClassLoader classLoader,
         @Advice.Argument(0) String name,
-        @Advice.Return(readOnly = false) InputStream inputStream) {
-      if (inputStream != null) {
-        return;
-      }
-
-      URL helper = HelperResources.loadOne(classLoader, name);
-      if (helper != null) {
-        try {
-          inputStream = helper.openStream();
-        } catch (IOException ignored) {
-          // ClassLoader.getResourceAsStream also ignores io exceptions from opening the stream
+        @Advice.Return InputStream inputStream) {
+      if (inputStream == null) {
+        URL helper = HelperResources.loadOne(classLoader, name);
+        if (helper != null) {
+          try {
+            return helper.openStream();
+          } catch (IOException ignored) {
+            // ClassLoader.getResourceAsStream also ignores io exceptions from opening the stream
+          }
         }
       }
+      return inputStream;
     }
   }
 }

javaagent-bootstrap/src/main/java/io/opentelemetry/javaagent/bootstrap/AgentClassLoader.java

@@ -142,6 +142,15 @@ private static int getJavaVersion() {
     return Integer.parseInt(javaSpecVersion);
   }
 
+  @Override
+  public Class<?> loadClass(String name) throws ClassNotFoundException {
+    // We explicitly override loadClass from ClassLoader to ensure
+    // that loadClass is properly excluded from our internal ClassLoader Instrumentations
+    // (e.g. LoadInjectedClassInstrumentation, BooDelegationInstrumentation)
+    // Otherwise this will cause recursion in invokedynamic linkage
+    return loadClass(name, false);
+  }
+
   @Override
   public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
     // ContextStorageOverride is meant for library instrumentation we don't want it to apply to our
@@ -158,7 +167,14 @@ public Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundExce
       }
       // search from parent and urls added to this loader
       if (clazz == null) {
-        clazz = super.loadClass(name, false);
+        try {
+          clazz = Class.forName(name, false, this.getParent());
+        } catch (ClassNotFoundException e) {
+          // ignore
+        }
+      }
+      if (clazz == null) {
+        clazz = super.findClass(name);
       }
       if (resolve) {
         resolveClass(clazz);

javaagent-bootstrap/src/main/java/io/opentelemetry/javaagent/bootstrap/IndyBootstrapDispatcher.java

@@ -54,8 +54,7 @@ public static CallSite bootstrap(
     return callSite;
   }
 
-  // package visibility for testing
-  static MethodHandle generateNoopMethodHandle(MethodType methodType) {
+  public static MethodHandle generateNoopMethodHandle(MethodType methodType) {
     Class<?> returnType = methodType.returnType();
     MethodHandle noopNoArg;
     if (returnType == void.class) {

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/instrumentation/indy/AdviceBootstrapState.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.instrumentation.indy;
+
+import java.lang.invoke.MutableCallSite;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import javax.annotation.Nullable;
+
+class AdviceBootstrapState implements AutoCloseable {
+
+  private static final ThreadLocal<Map<Key, AdviceBootstrapState>> stateForCurrentThread =
+      ThreadLocal.withInitial(HashMap::new);
+
+  private final Key key;
+  private int recursionDepth;
+  @Nullable private MutableCallSite nestedCallSite;
+
+  /**
+   * We have to eagerly initialize to not cause a lambda construction during {@link #enter(Class,
+   * String, String, String, String)}.
+   */
+  private static final Function<Key, AdviceBootstrapState> CONSTRUCTOR = AdviceBootstrapState::new;
+
+  private AdviceBootstrapState(Key key) {
+    this.key = key;
+    // enter will increment it by one, so 0 is the value for non-recursive calls
+    recursionDepth = -1;
+  }
+
+  static void initialize() {
+    // Eager initialize everything because we could run into recursions doing this during advice
+    // bootstrapping
+    stateForCurrentThread.get();
+    stateForCurrentThread.remove();
+    try {
+      Class.forName(Key.class.getName());
+    } catch (ClassNotFoundException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  static AdviceBootstrapState enter(
+      Class<?> instrumentedClass,
+      String moduleClassName,
+      String adviceClassName,
+      String adviceMethodName,
+      String adviceMethodDescriptor) {
+    Key key =
+        new Key(
+            instrumentedClass,
+            moduleClassName,
+            adviceClassName,
+            adviceMethodName,
+            adviceMethodDescriptor);
+    AdviceBootstrapState state = stateForCurrentThread.get().computeIfAbsent(key, CONSTRUCTOR);
+    state.recursionDepth++;
+    return state;
+  }
+
+  public boolean isNestedInvocation() {
+    return recursionDepth > 0;
+  }
+
+  public MutableCallSite getOrInitMutableCallSite(Supplier<MutableCallSite> initializer) {
+    if (nestedCallSite == null) {
+      nestedCallSite = initializer.get();
+    }
+    return nestedCallSite;
+  }
+
+  public void initMutableCallSite(MutableCallSite mutableCallSite) {
+    if (nestedCallSite != null) {
+      throw new IllegalStateException("callsite has already been initialized");
+    }
+    nestedCallSite = mutableCallSite;
+  }
+
+  @Nullable
+  public MutableCallSite getMutableCallSite() {
+    return nestedCallSite;
+  }
+
+  @Override
+  public void close() {
+    if (recursionDepth == 0) {
+      Map<Key, AdviceBootstrapState> stateMap = stateForCurrentThread.get();
+      stateMap.remove(key);
+      if (stateMap.isEmpty()) {
+        // Do not leave an empty map dangling as thread local
+        stateForCurrentThread.remove();
+      }
+    } else {
+      recursionDepth--;
+    }
+  }
+
+  /** Key uniquely identifying a single invokedynamic instruction inserted for an advice */
+  private static class Key {
+
+    private final Class<?> instrumentedClass;
+    private final String moduleClassName;
+    private final String adviceClassName;
+    private final String adviceMethodName;
+    private final String adviceMethodDescriptor;
+
+    private Key(
+        Class<?> instrumentedClass,
+        String moduleClassName,
+        String adviceClassName,
+        String adviceMethodName,
+        String adviceMethodDescriptor) {
+      this.instrumentedClass = instrumentedClass;
+      this.moduleClassName = moduleClassName;
+      this.adviceClassName = adviceClassName;
+      this.adviceMethodName = adviceMethodName;
+      this.adviceMethodDescriptor = adviceMethodDescriptor;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || !(o instanceof Key)) {
+        return false;
+      }
+
+      Key that = (Key) o;
+      return instrumentedClass.equals(that.instrumentedClass)
+          && moduleClassName.equals(that.moduleClassName)
+          && adviceClassName.equals(that.adviceClassName)
+          && adviceMethodName.equals(that.adviceMethodName)
+          && adviceMethodDescriptor.equals(that.adviceMethodDescriptor);
+    }
+
+    @Override
+    public int hashCode() {
+      int result = instrumentedClass.hashCode();
+      result = 31 * result + moduleClassName.hashCode();
+      result = 31 * result + adviceClassName.hashCode();
+      result = 31 * result + adviceMethodName.hashCode();
+      result = 31 * result + adviceMethodDescriptor.hashCode();
+      return result;
+    }
+  }
+}