Add support for SymDB to scan directories (#8306)

When the location of loaded class is a directory (directory provided
into the classpath or war extracted into a temp dir) we need to walk
the directory for scanning class files.
We avoid following the file link to prevent cycles.

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBEnablement.java

@@ -19,6 +19,7 @@
 import java.lang.instrument.Instrumentation;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.time.LocalDateTime;
@@ -163,8 +164,8 @@ private void extractSymbolForLoadedClasses(SymDBReport symDBReport) {
       }
       File jarPathFile = jarPath.toFile();
       if (jarPathFile.isDirectory()) {
-        // we are not supporting class directories (classpath) but only jar files
-        symDBReport.addDirectoryJar(jarPath.toString());
+        scanDirectory(jarPath, alreadyScannedJars, baos, buffer, symDBReport);
+        alreadyScannedJars.add(jarPath.toString());
         continue;
       }
       if (alreadyScannedJars.contains(jarPath.toString())) {
@@ -188,6 +189,46 @@ private void extractSymbolForLoadedClasses(SymDBReport symDBReport) {
     }
   }
 
+  private void scanDirectory(
+      Path jarPath,
+      Set<String> alreadyScannedJars,
+      ByteArrayOutputStream baos,
+      byte[] buffer,
+      SymDBReport symDBReport) {
+    try {
+      Files.walk(jarPath)
+          // explicitly no follow links walking the directory to avoid cycles
+          .filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS))
+          .filter(path -> path.toString().endsWith(".class"))
+          .filter(
+              path ->
+                  !classNameFilter.isExcluded(
+                      Strings.getClassName(trimPrefixes(jarPath.relativize(path).toString()))))
+          .forEach(path -> parseFileEntry(path, jarPath, baos, buffer));
+      alreadyScannedJars.add(jarPath.toString());
+    } catch (IOException e) {
+      symDBReport.addIOException(jarPath.toString(), e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void parseFileEntry(Path path, Path jarPath, ByteArrayOutputStream baos, byte[] buffer) {
+    LOGGER.debug("parsing file class: {}", path.toString());
+    try {
+      try (InputStream inputStream = Files.newInputStream(path)) {
+        int readBytes;
+        baos.reset();
+        while ((readBytes = inputStream.read(buffer)) != -1) {
+          baos.write(buffer, 0, readBytes);
+        }
+        symbolAggregator.parseClass(
+            path.getFileName().toString(), baos.toByteArray(), jarPath.toString());
+      }
+    } catch (IOException ex) {
+      LOGGER.debug("Exception during parsing file class: {}", path, ex);
+    }
+  }
+
   private void parseJarEntry(
       JarEntry jarEntry, JarFile jarFile, Path jarPath, ByteArrayOutputStream baos, byte[] buffer) {
     LOGGER.debug("parsing jarEntry class: {}", jarEntry.getName());

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBReport.java

@@ -14,18 +14,13 @@ public class SymDBReport {
   private static final Logger LOGGER = LoggerFactory.getLogger(SymDBReport.class);
 
   private final Set<String> missingJars = new HashSet<>();
-  private final Set<String> directoryJars = new HashSet<>();
   private final Map<String, String> ioExceptions = new HashMap<>();
   private final List<String> locationErrors = new ArrayList<>();
 
   public void addMissingJar(String jarPath) {
     missingJars.add(jarPath);
   }
 
-  public void addDirectoryJar(String jarPath) {
-    directoryJars.add(jarPath);
-  }
-
   public void addIOException(String jarPath, IOException e) {
     ioExceptions.put(jarPath, e.toString());
   }
@@ -40,8 +35,6 @@ public void report() {
             + locationErrors
             + " Missing jars: "
             + missingJars
-            + " Directory jars: "
-            + directoryJars
             + " IOExceptions: "
             + ioExceptions;
     LOGGER.info(content);

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymDBEnablementTest.java

@@ -4,7 +4,9 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -119,6 +121,28 @@ public void parseLoadedClass() throws ClassNotFoundException, IOException, URISy
         captor.getAllValues().get(1));
   }
 
+  @Test
+  public void parseLoadedClassFromDirectory()
+      throws ClassNotFoundException, IOException, URISyntaxException {
+    URL classFilesUrl = getClass().getResource("/");
+    URLClassLoader urlClassLoader = new URLClassLoader(new URL[] {classFilesUrl}, null);
+    Class<?> testClass = urlClassLoader.loadClass(getClass().getTypeName());
+    when(instr.getAllLoadedClasses()).thenReturn(new Class[] {testClass});
+    when(config.getThirdPartyIncludes())
+        .thenReturn(
+            Stream.of("com.datadog.debugger.", "org.springframework.samples.")
+                .collect(Collectors.toSet()));
+    SymbolAggregator symbolAggregator = mock(SymbolAggregator.class);
+    SymDBEnablement symDBEnablement =
+        new SymDBEnablement(instr, config, symbolAggregator, ClassNameFiltering.allowAll());
+    symDBEnablement.startSymbolExtraction();
+    verify(instr).addTransformer(any(SymbolExtractionTransformer.class));
+    ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
+    verify(symbolAggregator, atLeastOnce()).parseClass(captor.capture(), any(), anyString());
+    // verify that we called parseClass on this test class
+    assertTrue(captor.getAllValues().contains(getClass().getSimpleName() + ".class"));
+  }
+
   @Test
   public void noDuplicateSymbolExtraction() {
     final String CLASS_NAME_PATH = "com/datadog/debugger/symbol/SymbolExtraction01";