Skip to content

Commit b508db2

Browse files
paxtonharepaxtonhare
committed
fixed #522
1 parent ca7eae0 commit b508db2

File tree

3 files changed

+75
-16
lines changed

3 files changed

+75
-16
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,11 @@
3636

3737
import java.io.File;
3838
import java.io.FileInputStream;
39-
import java.io.IOException;
4039
import java.nio.file.Path;
41-
import java.nio.file.Paths;
4240
import java.util.ArrayList;
4341
import java.util.List;
4442
import java.util.Properties;
43+
import java.util.regex.Pattern;
4544

4645
public class FlowManager extends ResourceManager {
4746
private static final String HUB_NS = "http://marklogic.com/data-hub";
@@ -135,6 +134,16 @@ else if (flowType.equals(FlowType.HARMONIZE)) {
135134
return flows;
136135
}
137136

137+
public Flow getFlowFromProperties(Path propertiesFile) {
138+
String quotedSeparator = Pattern.quote(File.separator);
139+
String flowTypeRegex = ".+" + quotedSeparator + "(input|harmonize)" + quotedSeparator + ".+";
140+
FlowType flowType = propertiesFile.toString().replaceAll(flowTypeRegex, "$1").equals("input")
141+
? FlowType.INPUT : FlowType.HARMONIZE;
142+
143+
String entityName = propertiesFile.toString().replaceAll(".+" + quotedSeparator + "([^/\\\\]+)" + quotedSeparator + "(input|harmonize)" + quotedSeparator + ".+", "$1");
144+
return getLocalFlow(entityName, propertiesFile.getParent(), flowType);
145+
}
146+
138147
private Flow getLocalFlow(String entityName, Path flowDir, FlowType flowType) {
139148
try {
140149
String flowName = flowDir.getFileName().toString();

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.marklogic.client.document.JSONDocumentManager;
1111
import com.marklogic.client.document.XMLDocumentManager;
1212
import com.marklogic.client.ext.modulesloader.Modules;
13+
import com.marklogic.client.ext.modulesloader.ModulesManager;
1314
import com.marklogic.client.ext.modulesloader.impl.AssetFileLoader;
1415
import com.marklogic.client.ext.modulesloader.impl.DefaultModulesLoader;
1516
import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager;
@@ -19,25 +20,25 @@
1920
import com.marklogic.client.io.Format;
2021
import com.marklogic.client.io.StringHandle;
2122
import com.marklogic.com.marklogic.client.ext.file.CacheBusterDocumentFileProcessor;
23+
import com.marklogic.com.marklogic.client.ext.modulesloader.impl.EntityDefModulesFinder;
2224
import com.marklogic.com.marklogic.client.ext.modulesloader.impl.UserModulesFinder;
2325
import com.marklogic.hub.FlowManager;
2426
import com.marklogic.hub.HubConfig;
25-
import com.marklogic.com.marklogic.client.ext.modulesloader.impl.EntityDefModulesFinder;
2627
import com.marklogic.hub.deploy.util.HubFileFilter;
2728
import com.marklogic.hub.error.LegacyFlowsException;
2829
import com.marklogic.hub.flow.Flow;
2930
import org.apache.commons.io.IOUtils;
3031
import org.springframework.core.io.Resource;
3132
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
3233

33-
import java.io.File;
3434
import java.io.IOException;
3535
import java.io.InputStream;
3636
import java.nio.file.FileVisitResult;
3737
import java.nio.file.Files;
3838
import java.nio.file.Path;
3939
import java.nio.file.SimpleFileVisitor;
4040
import java.nio.file.attribute.BasicFileAttributes;
41+
import java.util.Date;
4142
import java.util.List;
4243
import java.util.regex.Pattern;
4344

@@ -109,6 +110,14 @@ boolean isEntityDir(Path dir, Path startPath) {
109110
return dirStr.matches(regex);
110111
}
111112

113+
boolean isFlowPropertiesFile(Path dir) {
114+
Path parent = dir.getParent();
115+
return dir.toFile().isFile() &&
116+
dir.getFileName().toString().endsWith(".properties") &&
117+
parent.toString().matches(".*[/\\\\](input|harmonize)[/\\\\][^/\\\\]+$") &&
118+
dir.getFileName().toString().equals(parent.getFileName().toString() + ".properties");
119+
}
120+
112121
@Override
113122
public void execute(CommandContext context) {
114123
FlowManager flowManager = new FlowManager(hubConfig);
@@ -143,14 +152,7 @@ public void execute(CommandContext context) {
143152
XMLDocumentManager documentManager = hubConfig.newModulesDbClient().newXMLDocumentManager();
144153
DocumentWriteSet documentWriteSet = documentManager.newWriteSet();
145154

146-
for (Flow flow : flows) {
147-
StringHandle handle = new StringHandle(flow.serialize());
148-
handle.setFormat(Format.XML);
149-
documentWriteSet.add(flow.getFlowDbPath(), handle);
150-
}
151-
if (documentWriteSet.size() > 0) {
152-
documentManager.write(documentWriteSet);
153-
}
155+
ModulesManager modulesManager = modulesLoader.getModulesManager();
154156

155157
Files.walkFileTree(startPath, new SimpleFileVisitor<Path>() {
156158
@Override
@@ -173,18 +175,39 @@ else if (isEntityDir(dir, startPath.toAbsolutePath())) {
173175
meta.getCollections().add("http://marklogic.com/entity-services/models");
174176
documentPermissionsParser.parsePermissions(hubConfig.modulePermissions, meta.getPermissions());
175177
for (Resource r : modules.getAssets()) {
176-
InputStream inputStream = r.getInputStream();
177-
StringHandle handle = new StringHandle(IOUtils.toString(inputStream));
178-
inputStream.close();
179-
entityDocMgr.write("/entities/" + r.getFilename(), meta, handle);
178+
if (modulesManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) {
179+
InputStream inputStream = r.getInputStream();
180+
StringHandle handle = new StringHandle(IOUtils.toString(inputStream));
181+
inputStream.close();
182+
entityDocMgr.write("/entities/" + r.getFilename(), meta, handle);
183+
modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date());
184+
}
180185
}
181186
return FileVisitResult.CONTINUE;
182187
}
183188
else {
184189
return FileVisitResult.CONTINUE;
185190
}
186191
}
192+
193+
@Override
194+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
195+
throws IOException
196+
{
197+
if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) {
198+
Flow flow = flowManager.getFlowFromProperties(file);
199+
StringHandle handle = new StringHandle(flow.serialize());
200+
handle.setFormat(Format.XML);
201+
documentWriteSet.add(flow.getFlowDbPath(), handle);
202+
modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date());
203+
}
204+
return FileVisitResult.CONTINUE;
205+
}
187206
});
207+
208+
if (documentWriteSet.size() > 0) {
209+
documentManager.write(documentWriteSet);
210+
}
188211
}
189212
threadPoolTaskExecutor.shutdown();
190213
} catch (IOException e) {

‎marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,33 @@ public void testGetLocalFlows() throws IOException {
187187
FileUtils.deleteDirectory(Paths.get("./del-me-dir").toFile());
188188
}
189189

190+
@Test
191+
public void testGetFlowFromProperties() throws IOException {
192+
Scaffolding scaffolding = new Scaffolding("del-me-dir", stagingClient);
193+
scaffolding.createEntity("my-entity");
194+
195+
FlowManager fm = new FlowManager(getHubConfig("del-me-dir"));
196+
197+
allCombos((codeFormat, dataFormat, flowType) -> {
198+
String flowName = flowType.toString() + "-" + codeFormat.toString() + "-" + dataFormat.toString();
199+
scaffolding.createFlow("my-entity", flowName, flowType, codeFormat, dataFormat);
200+
});
201+
202+
203+
allCombos((codeFormat, dataFormat, flowType) -> {
204+
String flowName = flowType.toString() + "-" + codeFormat.toString() + "-" + dataFormat.toString();
205+
Path propertiesFile = Paths.get("del-me-dir", "plugins", "entities", "my-entity", flowType.toString(), flowName, flowName + ".properties");
206+
Flow flow = fm.getFlowFromProperties(propertiesFile);
207+
assertEquals(flowName, flow.getName());
208+
assertEquals("my-entity", flow.getEntityName());
209+
assertEquals(codeFormat, flow.getCodeFormat());
210+
assertEquals(dataFormat, flow.getDataFormat());
211+
assertEquals(flowType, flow.getType());
212+
});
213+
214+
FileUtils.deleteDirectory(Paths.get("./del-me-dir").toFile());
215+
}
216+
190217
@Test
191218
public void testGetFlows() {
192219
clearDatabases(HubConfig.DEFAULT_MODULES_DB_NAME);

0 commit comments

Comments
 (0)