Skip to content

Commit 89c8b91

Browse files
FalsePatternFalsePattern
committed
added caching to dependency loader - should fix the long pause times
1 parent a0da689 commit 89c8b91

File tree

1 file changed

+120
-84
lines changed

1 file changed

+120
-84
lines changed

‎src/main/java/com/falsepattern/lib/internal/impl/dependencies/DependencyLoaderImpl.java

Lines changed: 120 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545

4646
import cpw.mods.fml.relauncher.FMLLaunchHandler;
4747

48+
import java.io.BufferedInputStream;
4849
import java.io.BufferedOutputStream;
4950
import java.io.File;
5051
import java.io.IOException;
@@ -59,6 +60,8 @@
5960
import java.util.ArrayList;
6061
import java.util.Arrays;
6162
import java.util.HashMap;
63+
import java.util.HashSet;
64+
import java.util.List;
6265
import java.util.Map;
6366
import java.util.Objects;
6467
import java.util.Set;
@@ -93,6 +96,52 @@ public class DependencyLoaderImpl {
9396
thread.setName("Dependency Download Thread " + counter.incrementAndGet());
9497
return thread;
9598
});
99+
private static final File libDir;
100+
101+
static {
102+
103+
var homeDir = System.getProperty("minecraft.sharedDataDir");
104+
if (homeDir == null) {
105+
homeDir = System.getenv("MINECRAFT_SHARED_DATA_DIR");
106+
if (homeDir == null) {
107+
homeDir = FileUtil.getMinecraftHome().getAbsolutePath();
108+
}
109+
}
110+
val modsDir = Paths.get(homeDir, "mods").toFile();
111+
val oldLibDir = new File(modsDir, "falsepattern");
112+
libDir = new File(homeDir, "falsepattern");
113+
if (!libDir.exists()) {
114+
if (!libDir.mkdirs()) {
115+
LOG.fatal("Failed to create directory {}", libDir);
116+
throw new RuntimeException("Failed to create directory " + libDir);
117+
}
118+
}
119+
if (oldLibDir.exists()) {
120+
LOG.info("Migrating old library folder. From: " + oldLibDir.getAbsolutePath() + ", To: " + libDir.getAbsolutePath());
121+
val oldFiles = oldLibDir.listFiles();
122+
if (oldFiles != null) {
123+
for (val file: oldFiles) {
124+
try {
125+
Files.move(file.toPath(), libDir.toPath().resolve(oldLibDir.toPath().relativize(file.toPath())), StandardCopyOption.REPLACE_EXISTING);
126+
} catch (IOException e) {
127+
LOG.warn("Failed to move file " + file.getName() + " to new dir! Deleting instead.");
128+
try {
129+
Files.deleteIfExists(file.toPath());
130+
} catch (IOException ex) {
131+
LOG.warn("Failed to delete file " + file.getPath() + "!");
132+
file.deleteOnExit();
133+
}
134+
}
135+
}
136+
}
137+
try {
138+
Files.deleteIfExists(oldLibDir.toPath());
139+
} catch (IOException e) {
140+
LOG.warn("Failed to delete old library directory!");
141+
oldLibDir.deleteOnExit();
142+
}
143+
}
144+
}
96145

97146
public static void addMavenRepo(String url) {
98147
mavenRepositories.add(url);
@@ -185,24 +234,25 @@ public static CompletableFuture<Void> loadLibrariesAsync(Library... libraries) {
185234
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
186235
}
187236

188-
private static Stream<URL> scanSourceMetaInf(URL source) {
237+
private static boolean scanForDepSpecs(URL source, List<URL> output) {
189238
if (!source.getProtocol().equals("file")) {
190239
LOG.warn("Skipping non-file source: {}", source);
191-
return Stream.empty();
240+
return false;
192241
}
193242
LOG.debug("Scanning {} for dependencies", source);
194243
val fileName = source.getFile();
195-
val output = new ArrayList<URL>();
244+
boolean found = false;
196245
if (fileName.endsWith(".jar")) {
197246
//Scan jar file for json in META-INF, add them to the list
198-
try (val inputStream = source.openStream(); val jarFile = new JarInputStream(inputStream)) {
247+
try (val inputStream = new BufferedInputStream(source.openStream(), 65536); val jarFile = new JarInputStream(inputStream)) {
199248
ZipEntry entry;
200249
while ((entry = jarFile.getNextEntry()) != null) {
201250
if (!entry.getName().startsWith("META-INF") || !entry.getName().endsWith(".json")) {
202251
continue;
203252
}
204253
try {
205254
output.add(new URL("jar:" + source + "!/" + entry.getName()));
255+
found = true;
206256
} catch (MalformedURLException e) {
207257
LOG.error("Failed to add json source {} to dependency source list: {}", entry.getName(), e);
208258
}
@@ -214,29 +264,30 @@ private static Stream<URL> scanSourceMetaInf(URL source) {
214264
val dir = new File(fileName);
215265
if (!dir.exists() || !dir.isDirectory()) {
216266
LOG.warn("Skipping non-directory, nor jar source: {}", source);
217-
return Stream.empty();
267+
return false;
218268
}
219269
//Scan directory for json in META-INF, add them to the list
220270
val metaInf = new File(dir, "META-INF");
221271
if (!metaInf.exists() || !metaInf.isDirectory()) {
222-
return Stream.empty();
272+
return false;
223273
}
224274
val files = metaInf.listFiles();
225275
if (files == null) {
226-
return Stream.empty();
276+
return false;
227277
}
228278
for (val file : files) {
229279
if (!file.getName().endsWith(".json")) {
230280
continue;
231281
}
232282
try {
233283
output.add(file.toURI().toURL());
284+
found = true;
234285
} catch (MalformedURLException e) {
235286
LOG.error("Failed to add json source {} to dependency source list: {}", file.getName(), e);
236287
}
237288
}
238289
}
239-
return output.stream();
290+
return found;
240291
}
241292

242293
private static Stream<URL> grabSourceCandidatesFromFolder(File folder) {
@@ -275,40 +326,67 @@ public static void scanDeps() {
275326
LOG.debug("Discovering dependency source candidates...");
276327
val modsDir = new File(FileUtil.getMinecraftHome(), "mods");
277328
val mods1710Dir = new File(modsDir, "1.7.10");
278-
val dependencySpecs = Stream.of(Launch.classLoader.getSources().stream(),
279-
grabSourceCandidatesFromFolder(modsDir),
280-
grabSourceCandidatesFromFolder(mods1710Dir))
281-
.flatMap((i) -> i)
282-
.flatMap(DependencyLoaderImpl::scanSourceMetaInf)
283-
.map((source) -> {
284-
//Convert source to GSON json
285-
try (val is = source.openStream()) {
286-
val jsonRaw = new JsonParser().parse(new InputStreamReader(is));
287-
if (!jsonRaw.isJsonObject()) {
288-
return null;
289-
}
290-
val json = jsonRaw.getAsJsonObject();
291-
if (!(json.has("identifier") &&
292-
json.get("identifier")
293-
.getAsString()
294-
.equals("falsepatternlib_dependencies")
295-
)) {
296-
return null;
297-
}
298-
val builder = new GsonBuilder();
299-
builder.excludeFieldsWithoutExposeAnnotation();
300-
val gson = builder.create();
301-
json.remove("identifier");
302-
val root = gson.fromJson(json, DepRoot.class);
303-
root.source(source.toString());
304-
return root;
305-
} catch (Exception e) {
306-
LOG.error("Failed to read json from source {}: {}", source, e);
307-
return null;
308-
}
309-
})
310-
.filter(Objects::nonNull)
311-
.collect(Collectors.toSet());
329+
long start = System.currentTimeMillis();
330+
val urlsWithoutDeps = new HashSet<String>();
331+
val depCache = new File(libDir, ".depscan_cache");
332+
if (depCache.exists()) {
333+
try {
334+
urlsWithoutDeps.addAll(Files.readAllLines(depCache.toPath()));
335+
} catch (IOException e) {
336+
LOG.error("Could not read dependency scanner cache", e);
337+
}
338+
}
339+
val candidates = Stream.of(Launch.classLoader.getSources().stream(),
340+
grabSourceCandidatesFromFolder(modsDir),
341+
grabSourceCandidatesFromFolder(mods1710Dir))
342+
.flatMap((i) -> i)
343+
.filter((url) -> !urlsWithoutDeps.contains(url.toString()))
344+
.collect(Collectors.toList());
345+
val urls = new ArrayList<URL>();
346+
for (val candidate: candidates) {
347+
if (!scanForDepSpecs(candidate, urls)) {
348+
urlsWithoutDeps.add(candidate.toString());
349+
}
350+
}
351+
try (val out = Files.newBufferedWriter(depCache.toPath())) {
352+
for (val noDep: urlsWithoutDeps) {
353+
out.append(noDep).append(System.lineSeparator());
354+
}
355+
} catch (IOException e) {
356+
LOG.error("Could not write dependency scanner cache", e);
357+
}
358+
val dependencySpecs = urls.stream()
359+
.map((source) -> {
360+
//Convert source to GSON json
361+
try (val is = new BufferedInputStream(source.openStream())) {
362+
val jsonRaw = new JsonParser().parse(new InputStreamReader(is));
363+
if (!jsonRaw.isJsonObject()) {
364+
return null;
365+
}
366+
val json = jsonRaw.getAsJsonObject();
367+
if (!(json.has("identifier") &&
368+
json.get("identifier")
369+
.getAsString()
370+
.equals("falsepatternlib_dependencies")
371+
)) {
372+
return null;
373+
}
374+
val builder = new GsonBuilder();
375+
builder.excludeFieldsWithoutExposeAnnotation();
376+
val gson = builder.create();
377+
json.remove("identifier");
378+
val root = gson.fromJson(json, DepRoot.class);
379+
root.source(source.toString());
380+
return root;
381+
} catch (Exception e) {
382+
LOG.error("Failed to read json from source {}: {}", source, e);
383+
return null;
384+
}
385+
})
386+
.filter(Objects::nonNull)
387+
.collect(Collectors.toSet());
388+
long end = System.currentTimeMillis();
389+
LOG.debug("Discovered {} dependency source candidates in {}ms", dependencySpecs.size(), end - start);
312390
mavenRepositories.addAll(dependencySpecs.stream()
313391
.flatMap((dep) -> dep.repositories().stream())
314392
.collect(Collectors.toSet()));
@@ -407,7 +485,6 @@ private static class DependencyLoadTask {
407485
private String artifact;
408486
private String mavenJarName;
409487
private String jarName;
410-
private File libDir;
411488
private File file;
412489

413490
private void load() {
@@ -477,50 +554,9 @@ private void alreadyLoaded() {
477554
}
478555

479556
private void setupPaths() {
480-
var homeDir = System.getProperty("minecraft.sharedDataDir");
481-
if (homeDir == null) {
482-
homeDir = System.getenv("MINECRAFT_SHARED_DATA_DIR");
483-
if (homeDir == null) {
484-
homeDir = FileUtil.getMinecraftHome().getAbsolutePath();
485-
}
486-
}
487-
val modsDir = Paths.get(homeDir, "mods").toFile();
488-
val oldLibDir = new File(modsDir, "falsepattern");
489-
libDir = new File(homeDir, "falsepattern");
490557
mavenJarName =
491558
String.format("%s-%s%s.jar", artifactId, preferredVersion, (suffix != null) ? ("-" + suffix) : "");
492559
jarName = groupId + "-" + mavenJarName;
493-
if (!libDir.exists()) {
494-
if (!libDir.mkdirs()) {
495-
LOG.fatal("Failed to create directory {}", libDir);
496-
throw new RuntimeException("Failed to create directory " + libDir);
497-
}
498-
}
499-
if (oldLibDir.exists()) {
500-
LOG.info("Migrating old library folder. From: " + oldLibDir.getAbsolutePath() + ", To: " + libDir.getAbsolutePath());
501-
val oldFiles = oldLibDir.listFiles();
502-
if (oldFiles != null) {
503-
for (val file: oldFiles) {
504-
try {
505-
Files.move(file.toPath(), libDir.toPath().resolve(oldLibDir.toPath().relativize(file.toPath())), StandardCopyOption.REPLACE_EXISTING);
506-
} catch (IOException e) {
507-
LOG.warn("Failed to move file " + file.getName() + " to new dir! Deleting instead.");
508-
try {
509-
Files.deleteIfExists(file.toPath());
510-
} catch (IOException ex) {
511-
LOG.warn("Failed to delete file " + file.getPath() + "!");
512-
file.deleteOnExit();
513-
}
514-
}
515-
}
516-
}
517-
try {
518-
Files.deleteIfExists(oldLibDir.toPath());
519-
} catch (IOException e) {
520-
LOG.warn("Failed to delete old library directory!");
521-
oldLibDir.deleteOnExit();
522-
}
523-
}
524560
file = new File(libDir, jarName);
525561
}
526562

0 commit comments

Comments
 (0)