GH-799 Improve dependency loader performance with parallel downloading and relocation. (#802)

* Improve dependency loader performance.

* Fix

* Add Spotify Completable Futures version to Versions.kt

---------

Co-authored-by: Martin <[email protected]>

Author: Rollczi

buildSrc/src/main/kotlin/Versions.kt

@@ -40,6 +40,8 @@ object Versions {
 
     const val CAFFEINE = "3.1.8"
 
+    const val SPOTIFY_COMPLETABLE_FUTURES = "0.3.6"
+
     // tests
     const val EXPRESSIBLE_JUNIT = "1.3.6"
     const val GROOVY_ALL = "3.0.21"

buildSrc/src/main/kotlin/eternalcore-repositories.gradle.kts

@@ -5,11 +5,8 @@ plugins {
 repositories {
     mavenCentral()
     maven { url = uri("https://hub.spigotmc.org/nexus/content/repositories/snapshots/") }
-    maven { url = uri("https://repo.panda-lang.org/releases/") }
     maven { url = uri("https://repo.papermc.io/repository/maven-public/") }
-    maven { url = uri("https://repository.minecodes.pl/releases/") }
-    maven { url = uri("https://repository.minecodes.pl/snapshots/") }
-    maven { url = uri("https://repo.eternalcode.pl/snapshots/") }
+    maven { url = uri("https://repo.panda-lang.org/releases/") }
     maven { url = uri("https://repo.eternalcode.pl/releases/") }
     maven { url = uri("https://repo.extendedclip.com/content/repositories/placeholderapi/") }
 }

eternalcore-plugin/build.gradle.kts

@@ -32,6 +32,10 @@ eternalShadowCompiler {
     }
 }
 
+dependencies {
+    implementation("com.spotify:completable-futures:${Versions.SPOTIFY_COMPLETABLE_FUTURES}")
+}
+
 tasks {
     runServer {
         minecraftVersion("1.20.4")

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/dependency/Dependency.java

@@ -29,7 +29,7 @@ public ResourceLocator toMavenJar(Repository repository, String classifier) {
         String url = String.format(
             JAR_MAVEN_FORMAT_WITH_CLASSIFIER,
             repository.url(),
-            this.groupId,
+            this.groupId.replace(".", "/"),
             this.artifactId,
             this.version,
             this.artifactId,

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/dependency/DependencyCollector.java

@@ -12,7 +12,7 @@ public boolean hasScannedDependency(Dependency dependency) {
         return this.fullScannedDependencies.containsKey(dependency.getGroupArtifactId());
     }
 
-    public void scannedDependency(Dependency dependency) {
+    public void addScannedDependency(Dependency dependency) {
         Dependency current = this.fullScannedDependencies.get(dependency.getGroupArtifactId());
 
         if (current == null) {
@@ -25,13 +25,13 @@ public void scannedDependency(Dependency dependency) {
         }
     }
 
-    public void scannedDependencies(Collection<Dependency> dependencies) {
+    public void addScannedDependencies(Collection<Dependency> dependencies) {
         for (Dependency dependency : dependencies) {
-            this.scannedDependency(dependency);
+            this.addScannedDependency(dependency);
         }
     }
 
-    public Collection<Dependency> scannedDependencies() {
+    public Collection<Dependency> getScannedDependencies() {
         return Collections.unmodifiableCollection(this.fullScannedDependencies.values());
     }
 

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/dependency/DependencyLoadEntry.java

@@ -0,0 +1,6 @@
+package com.eternalcode.core.loader.dependency;
+
+import java.nio.file.Path;
+
+public record DependencyLoadEntry(Dependency dependency, Path path) {
+}

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/dependency/DependencyLoadResult.java

@@ -2,9 +2,7 @@
 
 import com.eternalcode.core.loader.classloader.IsolatedClassLoader;
 
-import java.nio.file.Path;
-import java.util.Collection;
 import java.util.List;
 
-public record DependencyLoadResult(IsolatedClassLoader loader, Collection<Dependency> dependencies, List<Path> paths) {
+public record DependencyLoadResult(IsolatedClassLoader loader, List<DependencyLoadEntry> paths) {
 }

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/dependency/DependencyLoaderImpl.java

@@ -8,6 +8,7 @@
 import com.eternalcode.core.loader.relocation.RelocationHandler;
 import com.eternalcode.core.loader.repository.Repository;
 
+import com.spotify.futures.CompletableFutures;
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
@@ -18,27 +19,34 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
 public class DependencyLoaderImpl implements DependencyLoader {
 
     private static final String LOCAL_REPOSITORY_PATH = "localRepository";
 
+    private final Executor executor;
     private final Logger logger;
     private final DependencyDownloader downloadDependency;
     private final RelocationHandler relocationHandler;
     private final DependencyScanner pomXmlScanner;
+    private final Repository localRepository;
 
     private final Map<Dependency, Path> loaded = new HashMap<>();
 
     public DependencyLoaderImpl(Logger logger, File dataFolder, List<Repository> repositories) {
         Path localRepositoryPath = this.setupCacheDirectory(dataFolder);
-        Repository localRepository = Repository.localRepository(localRepositoryPath);
+        this.localRepository = Repository.localRepository(localRepositoryPath);
 
         List<Repository> allRepositories = new ArrayList<>();
         allRepositories.add(localRepository);
         allRepositories.addAll(repositories);
 
+        this.executor = Executors.newCachedThreadPool();
         this.logger = logger;
         this.pomXmlScanner = new PomXmlScanner(allRepositories, localRepository);
         this.downloadDependency = new DependencyDownloader(logger, localRepository, allRepositories);
@@ -64,39 +72,43 @@ public DependencyLoadResult load(IsolatedClassLoader loader, List<Dependency> de
             collector = this.pomXmlScanner.findAllChildren(collector, dependency);
         }
 
-        collector.scannedDependencies(dependencies);
-        this.logger.info("Found " + collector.scannedDependencies().size() + " dependencies");
+        collector.addScannedDependencies(dependencies);
+        this.logger.info("Found " + collector.getScannedDependencies().size() + " dependencies");
 
-        List<Path> paths = new ArrayList<>();
+        List<CompletableFuture<DependencyLoadEntry>> futures = new ArrayList<>();
 
-        for (Dependency dependency : collector.scannedDependencies()) {
-            Path loaded = this.loaded.get(dependency);
+        for (Dependency dependency : collector.getScannedDependencies()) {
+            CompletableFuture<DependencyLoadEntry> pathToDependency = CompletableFuture.supplyAsync(() -> {
+                Path loaded = this.loaded.get(dependency);
 
-            if (loaded != null) {
-                // TODO: jeśli już wcześniej pobrano zależność to można zweryfikować czy relokacja jest poprawna (może jakiś plik z informacjami o relokacjach?)
-                loader.addPath(loaded);
-                paths.add(loaded);
-                continue;
-            }
+                if (loaded != null) {
+                    return new DependencyLoadEntry(dependency, loaded);
+                }
 
-            Path downloadedDependencyPath = this.downloadDependency.downloadDependency(dependency);
+                Path downloadedDependencyPath = this.downloadDependency.downloadDependency(dependency);
 
-            if (this.relocationHandler == null) {
-                this.loaded.put(dependency, downloadedDependencyPath);
-                loader.addPath(downloadedDependencyPath);
-                paths.add(downloadedDependencyPath);
-                continue;
-            }
+                if (this.relocationHandler == null) {
+                    return new DependencyLoadEntry(dependency, downloadedDependencyPath);
+                }
 
-            // TODO: to jest trochę blocking można to zrobić w parallel streamie
-            Path relocatedDependency = this.relocationHandler.relocateDependency(downloadedDependencyPath, dependency, relocations);
+                Path relocatedDependency = this.relocationHandler.relocateDependency(localRepository, downloadedDependencyPath, dependency, relocations);
 
-            this.loaded.put(dependency, relocatedDependency);
-            loader.addPath(relocatedDependency);
-            paths.add(relocatedDependency);
+                return new DependencyLoadEntry(dependency, relocatedDependency);
+            }, this.executor);
+
+            futures.add(pathToDependency);
+        }
+
+        List<DependencyLoadEntry> dependencyLoadEntries = CompletableFutures.allAsList(futures)
+            .orTimeout(60, TimeUnit.MINUTES)
+            .join();
+
+        for (DependencyLoadEntry dependencyLoadEntry : dependencyLoadEntries) {
+            loader.addPath(dependencyLoadEntry.path());
+            this.loaded.put(dependencyLoadEntry.dependency(), dependencyLoadEntry.path());
         }
 
-        return new DependencyLoadResult(loader, collector.scannedDependencies(), paths);
+        return new DependencyLoadResult(loader, dependencyLoadEntries);
     }
 
     @Override

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/pom/PomXmlScanner.java

@@ -68,7 +68,7 @@ public DependencyCollector findAllChildren(DependencyCollector collector, Depend
                 }
 
                 collector = this.findAllChildren(collector, firstChild);
-                collector.scannedDependency(firstChild);
+                collector.addScannedDependency(firstChild);
             }
 
             break;

eternalcore-plugin/src/main/java/com/eternalcode/core/loader/relocation/RelocationHandler.java

@@ -79,12 +79,11 @@ private RelocationHandler(IsolatedClassLoader classLoader, Constructor<?> jarRel
         this.jarRelocatorRunMethod = jarRelocatorRunMethod;
     }
 
-    public Path relocateDependency(Path dependencyPath, Dependency dependency, List<Relocation> relocations) {
+    public Path relocateDependency(Repository localRepository, Path dependencyPath, Dependency dependency, List<Relocation> relocations) {
         if (relocations.isEmpty()) {
             return dependencyPath;
         }
 
-        Repository localRepository = Repository.localRepository(dependencyPath.getParent());
         Path relocatedJar = dependency.toMavenJar(localRepository, "relocated").toPath();
 
         if (Files.exists(relocatedJar)) {