Skip to content

Commit 6ca97de

Browse files
qqndrewqqndrew
committed
Use MD5 Hashes to skip previously packaged components
1 parent ba567b6 commit 6ca97de

File tree

8 files changed

+107
-29
lines changed

8 files changed

+107
-29
lines changed

‎API/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
<parent>
3232
<groupId>org.ohnlp.backbone</groupId>
3333
<artifactId>backbone-parent</artifactId>
34-
<version>3.0.6</version>
34+
<version>3.0.7</version>
3535
</parent>
3636

3737
<artifactId>api</artifactId>

‎Core/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>org.ohnlp.backbone</groupId>
99
<artifactId>backbone-parent</artifactId>
10-
<version>3.0.6</version>
10+
<version>3.0.7</version>
1111
</parent>
1212

1313
<artifactId>core</artifactId>

‎Example-Backbone-Configs/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>org.ohnlp.backbone</groupId>
99
<artifactId>backbone-parent</artifactId>
10-
<version>3.0.6</version>
10+
<version>3.0.7</version>
1111
</parent>
1212

1313
<artifactId>example-backbone-configs</artifactId>

‎IO/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>org.ohnlp.backbone</groupId>
99
<artifactId>backbone-parent</artifactId>
10-
<version>3.0.6</version>
10+
<version>3.0.7</version>
1111
</parent>
1212

1313
<groupId>org.ohnlp.backbone.io</groupId>

‎Plugin-Manager/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>org.ohnlp.backbone</groupId>
99
<artifactId>backbone-parent</artifactId>
10-
<version>3.0.6</version>
10+
<version>3.0.7</version>
1111
</parent>
1212

1313
<artifactId>plugin-manager</artifactId>

‎Plugin-Manager/src/main/java/org/ohnlp/backbone/pluginmanager/PluginManager.java‎

Lines changed: 100 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,26 @@
11
package org.ohnlp.backbone.pluginmanager;
22

3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
5+
import com.fasterxml.jackson.databind.node.ObjectNode;
6+
37
import java.io.File;
48
import java.io.IOException;
9+
import java.math.BigInteger;
510
import java.nio.file.*;
611
import java.nio.file.attribute.BasicFileAttributes;
12+
import java.security.MessageDigest;
13+
import java.security.NoSuchAlgorithmException;
714
import java.util.*;
15+
import java.util.concurrent.atomic.AtomicBoolean;
816
import java.util.stream.Collectors;
917

1018
public class PluginManager {
1119

1220
public static void main(String... args) throws IOException {
21+
File hashFile = new File("bin", "last_modified.json");
22+
23+
ObjectNode hashes = hashFile.exists() ? (ObjectNode) new ObjectMapper().readTree(hashFile) : JsonNodeFactory.instance.objectNode();
1324
Set<String> builds = new HashSet<>();
1425
if (args.length > 0) {
1526
builds = Arrays.stream(args).map(String::toLowerCase).collect(Collectors.toSet());
@@ -22,72 +33,139 @@ public static void main(String... args) throws IOException {
2233
if (f.getName().startsWith("Backbone-Core") && !f.getName().endsWith("Packaged.jar")) {
2334
File source = f;
2435
String type = f.getName().substring(14, f.getName().length() - 4);
36+
if (!hashes.has(type)) {
37+
hashes.set(type, JsonNodeFactory.instance.objectNode());
38+
}
39+
ObjectNode typeHash = (ObjectNode) hashes.get(type);
2540
if (builds.size() == 0 || builds.contains(type.toLowerCase(Locale.ROOT))) {
41+
System.out.println("Repackaging files changed since last packaging of " + type + " platform-specific JAR:");
2642
File target = new File("bin/Backbone-Core-" + type + "-Packaged.jar");
27-
Files.copy(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
28-
install(target, modules, configs, resources);
29-
System.out.println("Successfully Packaged Platform-Specific JAR: " + target.getAbsolutePath());
43+
// Do a complete rebuild in update in case of class conflicts between submodules
44+
// TODO actually check this in the future (i.e., by preventing overwrites in org.ohnlp.backbone
45+
// classes during install()
46+
boolean invalidateAllHashes = false;
47+
boolean updated = false;
48+
if (!checkAndUpdateMD5ChecksumRegistry(false, typeHash, f)) {
49+
System.out.println("- " + getRelativePath(f));
50+
Files.copy(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
51+
invalidateAllHashes = true;
52+
updated = true;
53+
}
54+
updated = install(invalidateAllHashes, target, typeHash, modules, configs, resources) || updated;
55+
if (!updated) {
56+
System.out.println("No changed files found");
57+
} else {
58+
System.out.println("Successfully Packaged Platform-Specific JAR: " + target.getAbsolutePath());
59+
}
3060
}
3161
}
3262
}
3363
}
64+
// Write hashes out
65+
new ObjectMapper().writerWithDefaultPrettyPrinter().writeValue(hashFile, hashes);
3466
System.out.println("Packaging complete!");
3567
}
3668

69+
private static boolean checkAndUpdateMD5ChecksumRegistry(boolean invalidateAllHashes, ObjectNode hashRegistry, File f) {
70+
try {
71+
byte[] data = Files.readAllBytes(Paths.get(f.getPath()));
72+
byte[] hash = MessageDigest.getInstance("MD5").digest(data);
73+
String checksum = new BigInteger(1, hash).toString(16);
74+
String relativePath = getRelativePath(f);
75+
if (invalidateAllHashes
76+
|| !hashRegistry.has(relativePath)
77+
|| !hashRegistry.get(relativePath).asText().equals(checksum)) {
78+
hashRegistry.put(relativePath, checksum);
79+
return false;
80+
} else {
81+
return true;
82+
}
83+
} catch (IOException | NoSuchAlgorithmException e) {
84+
e.printStackTrace();
85+
}
86+
return false;
87+
}
88+
3789
/**
3890
* Packs an OHNLP backbone executable with the specified set of modules, configurations, and resources
3991
*
92+
* @param invalidateAllHashes Whether to ignore/replace all hashes
4093
* @param target The target file for packaging/into which items should be installed
94+
* @param hashRegistry A registry of currently
4195
* @param modules A list of module jar files to install
4296
* @param configurations A set of configuration files to install
4397
* @param resources A set of resource directories to install
4498
*/
45-
public static void install(File target, List<File> modules, List<File> configurations, List<File> resources) {
99+
public static boolean install(boolean invalidateAllHashes, File target, ObjectNode hashRegistry, List<File> modules, List<File> configurations, List<File> resources) {
100+
AtomicBoolean updated = new AtomicBoolean(false);
46101
Map<String, String> env = new HashMap<>();
47102
env.put("create", "false");
48103
try (FileSystem fs = FileSystems.newFileSystem(target.toPath(), PluginManager.class.getClassLoader())) {
49104
for (File module : modules) {
50-
try (FileSystem srcFs = FileSystems.newFileSystem(module.toPath(), PluginManager.class.getClassLoader())) {
51-
Path srcRoot = srcFs.getPath("/");
52-
Files.walkFileTree(srcRoot, new SimpleFileVisitor<Path>() {
53-
@Override
54-
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
55-
String fName = path.getFileName().toString().toLowerCase(Locale.ROOT);
56-
if (fName.endsWith(".sf") || fName.endsWith(".dsa") || fName.endsWith(".rsa")) {
105+
if (!checkAndUpdateMD5ChecksumRegistry(invalidateAllHashes, hashRegistry, module)) {
106+
updated.set(true);
107+
System.out.println("- " + getRelativePath(module));
108+
try (FileSystem srcFs = FileSystems.newFileSystem(module.toPath(), PluginManager.class.getClassLoader())) {
109+
Path srcRoot = srcFs.getPath("/");
110+
Files.walkFileTree(srcRoot, new SimpleFileVisitor<Path>() {
111+
@Override
112+
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
113+
String fName = path.getFileName().toString().toLowerCase(Locale.ROOT);
114+
if (fName.endsWith(".sf") || fName.endsWith(".dsa") || fName.endsWith(".rsa")) {
115+
return FileVisitResult.CONTINUE;
116+
}
117+
Path tgtPath = fs.getPath(path.toString());
118+
Files.createDirectories(tgtPath.getParent());
119+
Files.copy(path, tgtPath, StandardCopyOption.REPLACE_EXISTING);
57120
return FileVisitResult.CONTINUE;
58121
}
59-
Path tgtPath = fs.getPath(path.toString());
60-
Files.createDirectories(tgtPath.getParent());
61-
Files.copy(path, tgtPath, StandardCopyOption.REPLACE_EXISTING);
62-
return FileVisitResult.CONTINUE;
63-
}
64-
});
122+
});
123+
}
65124
}
66125
}
67126
Files.createDirectories(fs.getPath("/configs"));
68127
for (File config : configurations) {
69-
Files.copy(config.toPath(), fs.getPath("/configs/" + config.getName()), StandardCopyOption.REPLACE_EXISTING);
128+
if (!checkAndUpdateMD5ChecksumRegistry(invalidateAllHashes, hashRegistry, config)) {
129+
updated.set(true);
130+
System.out.println("- " + getRelativePath(config));
131+
Files.copy(config.toPath(), fs.getPath("/configs/" + config.getName()), StandardCopyOption.REPLACE_EXISTING);
132+
}
70133
}
71134
Files.createDirectories(fs.getPath("/resources"));
135+
// Always update all resources
72136
for (File resource : resources) {
73137
String resourceDir = resource.getParentFile().toPath().toAbsolutePath().toString();
74138
if (resource.isDirectory()) {
75139
// Recursively find all files in directory (up to arbitrary max depth of 999
76140
Files.find(resource.toPath(), 999, (p, bfa) -> bfa.isRegularFile()).forEach(p -> {
77141
try {
78-
Path filePath = fs.getPath(p.toAbsolutePath().toString().replace(resourceDir, "/resources"));
79-
Files.createDirectories(filePath.getParent());
80-
Files.copy(p, filePath, StandardCopyOption.REPLACE_EXISTING);
142+
if (!checkAndUpdateMD5ChecksumRegistry(invalidateAllHashes, hashRegistry, p.toFile())) {
143+
System.out.println("- " + getRelativePath(p.toFile()));
144+
updated.set(true);
145+
Path filePath = fs.getPath(p.toAbsolutePath().toString().replace(resourceDir, "/resources"));
146+
Files.createDirectories(filePath.getParent());
147+
Files.copy(p, filePath, StandardCopyOption.REPLACE_EXISTING);
148+
}
81149
} catch (IOException e) {
82150
throw new IllegalStateException(e);
83151
}
84152
});
85153
} else {
86-
Files.copy(resource.toPath(), fs.getPath("/resources/" + resource.getName()), StandardCopyOption.REPLACE_EXISTING);
154+
if (!checkAndUpdateMD5ChecksumRegistry(invalidateAllHashes, hashRegistry, resource)) {
155+
System.out.println("- " + getRelativePath(resource));
156+
updated.set(true);
157+
Files.copy(resource.toPath(), fs.getPath("/resources/" + resource.getName()), StandardCopyOption.REPLACE_EXISTING);
158+
}
87159
}
88160
}
89161
} catch (IOException e) {
90162
throw new IllegalStateException(e);
91163
}
164+
return updated.get();
165+
}
166+
167+
private static String getRelativePath(File f) {
168+
File root = new File(".");
169+
return root.toURI().relativize(f.toURI()).getPath();
92170
}
93171
}

‎Transforms/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>org.ohnlp.backbone</groupId>
99
<artifactId>backbone-parent</artifactId>
10-
<version>3.0.6</version>
10+
<version>3.0.7</version>
1111
</parent>
1212

1313
<groupId>org.ohnlp.backbone.transforms</groupId>

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>org.ohnlp.backbone</groupId>
88
<artifactId>backbone-parent</artifactId>
9-
<version>3.0.6</version>
9+
<version>3.0.7</version>
1010

1111

1212
<properties>

0 commit comments

Comments
 (0)