Allow pre-Java 6 classes to be transformed in the debugger (#9591)

Allow pre-Java 6 classes to be transformed in the debugger

Pre-Java 6 classes can contain JSR/RET instructions which are not
supported by ASM when the COMPUTE_FRAMES is used. This leads to
exceptions like this:

13:34:29.407 [Test worker] ERROR com.datadog.debugger.agent.DebuggerTransformer - Cannot write classfile for class: org.apache.felix.gogo.runtime.Pipe Exception:
java.lang.IllegalArgumentException: JSR/RET are not supported with computeFrames option
        at org.objectweb.asm.Frame.execute(Frame.java:1028)
        at org.objectweb.asm.MethodWriter.visitJumpInsn(MethodWriter.java:1147)
        at org.objectweb.asm.tree.JumpInsnNode.accept(JumpInsnNode.java:79)
        at org.objectweb.asm.tree.InsnList.accept(InsnList.java:144)
        at org.objectweb.asm.tree.MethodNode.accept(MethodNode.java:749)
        at org.objectweb.asm.tree.MethodNode.accept(MethodNode.java:647)
        at org.objectweb.asm.tree.ClassNode.accept(ClassNode.java:468)
        at com.datadog.debugger.agent.DebuggerTransformer.writeClassFile(DebuggerTransformer.java:492)
        ...

This uses ASM's JSRInlinerAdapter to rewrite the problematic JSR/RET
instructions as the class file is written.
---------

Co-authored-by: DJ Gregor <[email protected]>

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerTransformer.java

@@ -59,8 +59,11 @@
 import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.pool.TypePool;
 import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
 import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.JSRInlinerAdapter;
 import org.objectweb.asm.tree.ClassNode;
 import org.objectweb.asm.tree.MethodNode;
 import org.objectweb.asm.tree.analysis.Analyzer;
@@ -487,9 +490,10 @@ private byte[] writeClassFile(
       classNode.version = Opcodes.V1_8;
     }
     ClassWriter writer = new SafeClassWriter(loader);
+    ClassVisitor visitor = new JsrInliningClassVisitor(writer);
     LOGGER.debug("Generating bytecode for class: {}", Strings.getClassName(classFilePath));
     try {
-      classNode.accept(writer);
+      classNode.accept(visitor);
     } catch (Throwable t) {
       LOGGER.error("Cannot write classfile for class: {} Exception: ", classFilePath, t);
       reportInstrumentationFails(definitions, Strings.getClassName(classFilePath));
@@ -929,6 +933,26 @@ private static Path dumpClassFile(String className, byte[] classfileBuffer) {
     }
   }
 
+  /**
+   * A {@link org.objectweb.asm.ClassVisitor} that uses {@link
+   * org.objectweb.asm.commons.JSRInlinerAdapter} to remove JSR instructions and inlines the
+   * referenced subroutines. This allows pre-Java 6 classes with finally blocks to be successfully
+   * transformed. Without this an IllegalArgumentException for "JSR/RET are not supported with
+   * computeFrames option" would be thrown when writing the transformed class.
+   */
+  static class JsrInliningClassVisitor extends ClassVisitor {
+    protected JsrInliningClassVisitor(ClassVisitor parent) {
+      super(Opcodes.ASM9, parent);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(
+        int access, String name, String descriptor, String signature, String[] exceptions) {
+      MethodVisitor mv = super.visitMethod(access, name, descriptor, signature, exceptions);
+      return new JSRInlinerAdapter(mv, access, name, descriptor, signature, exceptions);
+    }
+  }
+
   static class SafeClassWriter extends ClassWriter {
     private final ClassLoader classLoader;
 

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/CapturedSnapshotTest.java

@@ -275,6 +275,24 @@ public void veryOldClassFile() throws Exception {
     assertOneSnapshot(listener);
   }
 
+  /**
+   * Ensure older pre-Java 6 class files with JSR/RET can be rewritten without "JSR/RET are not
+   * supported with computeFrames option" exceptions being thrown.
+   */
+  @Test
+  public void veryOldClassFileWithJsrRet() throws Exception {
+    final String CLASS_NAME = "antlr.Tool"; // compiled with jdk 1.2
+    TestSnapshotListener listener = installMethodProbe(CLASS_NAME, "copyFile", null);
+    Class<?> testClass = Class.forName(CLASS_NAME);
+    assertNotNull(testClass);
+    try {
+      Reflect.onClass(testClass).create().call("copyFile", null, null);
+    } catch (Throwable t) {
+      // ignore
+    }
+    assertOneSnapshot(listener);
+  }
+
   @Test
   public void oldClass1_1() throws Exception {
     final String CLASS_NAME = "org.apache.commons.lang.BooleanUtils"; // compiled with jdk 1.1