#53 Reworked FileLoader, added processors to DocumentFileReader, now supporting collections.properties and permissions.properties

Author: Rob Rudin

src/main/java/com/marklogic/client/file/CollectionsDocumentFileProcessor.java

@@ -1,83 +1,28 @@
 package com.marklogic.client.file;
 
-import com.marklogic.client.helper.LoggingObject;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Properties;
 
 /**
  * Looks for a special file in each directory - defaults to collections.properties - that contains properties where the
  * key is the name of a file in the directory, and the value is a comma-delimited list of collections to load the file
- * into (which means you can't use a comma in any collection name). This will by default NOT load the configured
- * properties file, as that's expected to just be configuration data and not something that's intended to be loaded
- * into MarkLogic.
+ * into (which means you can't use a comma in any collection name).
  */
-public class CollectionsDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
-
-	private String collectionsFilename = "collections.properties";
-
-	// Used to avoid checking for and loading the properties for every file in a directory
-	private Map<File, Properties> propertiesCache = new HashMap<>();
-
-	/**
-	 * @param documentFile
-	 * @return
-	 */
-	@Override
-	public DocumentFile processDocumentFile(DocumentFile documentFile) {
-		File file = documentFile.getFile();
-		String name = file.getName();
-		if (collectionsFilename.equals(name)) {
-			return null;
-		}
-
-		File collectionsFile = new File(file.getParentFile(), collectionsFilename);
-		if (collectionsFile.exists()) {
-			try {
-				Properties props = loadProperties(collectionsFile);
-				if (props.containsKey(name)) {
-					String value = props.getProperty(name);
-					documentFile.getDocumentMetadata().withCollections(value.split(","));
-				}
-			} catch (IOException e) {
-				logger.warn("Unable to load properties from collections file: " + collectionsFile.getAbsolutePath(), e);
-			}
-		}
+public class CollectionsDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor {
 
-		return documentFile;
+	public CollectionsDocumentFileProcessor() {
+		this("collections.properties");
 	}
 
-	protected Properties loadProperties(File collectionsFile) throws IOException {
-		Properties props = null;
-		if (propertiesCache.containsKey(collectionsFile)) {
-			props = propertiesCache.get(collectionsFile);
-		}
-		if (props != null) {
-			return props;
-		}
-		props = new Properties();
-		FileReader reader = null;
-		try {
-			reader = new FileReader(collectionsFile);
-			props.load(reader);
-			propertiesCache.put(collectionsFile, props);
-			return props;
-		} finally {
-			if (reader != null) {
-				reader.close();
-			}
-		}
+	public CollectionsDocumentFileProcessor(String propertiesFilename) {
+		super(propertiesFilename);
 	}
 
-	public Map<File, Properties> getPropertiesCache() {
-		return propertiesCache;
-	}
-
-	public void setCollectionsFilename(String collectionsFilename) {
-		this.collectionsFilename = collectionsFilename;
+	@Override
+	protected void processProperties(DocumentFile documentFile, Properties properties) {
+		String name = documentFile.getFile().getName();
+		if (properties.containsKey(name)) {
+			String value = properties.getProperty(name);
+			documentFile.getDocumentMetadata().withCollections(value.split(","));
+		}
 	}
 }

src/main/java/com/marklogic/client/file/DefaultDocumentFileFinder.java

@@ -0,0 +1,175 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.helper.LoggingObject;
+import org.springframework.util.ClassUtils;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Non-threadsafe implementation that implements FileVisitor as a way of descending one or more file paths.
+ */
+public class DefaultDocumentFileReader extends LoggingObject implements FileVisitor<Path>, DocumentFileReader {
+
+	private Path currentRootPath;
+	private List<FileFilter> fileFilters;
+	private List<DocumentFile> documentFiles;
+	private List<DocumentFileProcessor> documentFileProcessors;
+
+	public DefaultDocumentFileReader() {
+		initialize();
+	}
+
+	/**
+	 * Walk the file tree at each of the given paths, applying any configured DocumentFileProcessor instances on each
+	 * DocumentFile that is constructed by a File.
+	 *
+	 * @param paths
+	 * @return
+	 */
+	public List<DocumentFile> readDocumentFiles(String... paths) {
+		documentFiles = new ArrayList<>();
+		for (String path : paths) {
+			if (logger.isDebugEnabled()) {
+				logger.debug(format("Finding documents at path: %s", path));
+			}
+			this.currentRootPath = Paths.get(path);
+			try {
+				Files.walkFileTree(this.currentRootPath, this);
+			} catch (IOException ie) {
+				throw new RuntimeException(format("IO error while walking file tree at path: %s", path), ie);
+			}
+		}
+		return documentFiles;
+	}
+
+	@Override
+	public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+		boolean accept = acceptPath(dir, attrs);
+		if (accept) {
+			if (logger.isTraceEnabled()) {
+				logger.trace("Visiting directory: " + dir);
+			}
+			return FileVisitResult.CONTINUE;
+		} else {
+			if (logger.isTraceEnabled()) {
+				logger.trace("Skipping directory: " + dir);
+			}
+			return FileVisitResult.SKIP_SUBTREE;
+		}
+	}
+
+	@Override
+	public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+		if (acceptPath(path, attrs)) {
+			DocumentFile documentFile = buildDocumentFile(path, currentRootPath);
+			documentFile = processDocumentFile(documentFile);
+			if (documentFile != null) {
+				this.documentFiles.add(documentFile);
+			}
+		}
+		return FileVisitResult.CONTINUE;
+	}
+
+	/**
+	 * If any of the configured FileFilter objects do not accept the Path, then it is not accepted.
+	 *
+	 * @param path
+	 * @param attrs
+	 * @return
+	 */
+	protected boolean acceptPath(Path path, BasicFileAttributes attrs) {
+		if (fileFilters != null) {
+			File file = path.toFile();
+			for (FileFilter filter : fileFilters) {
+				if (!filter.accept(file)) {
+					return false;
+				}
+			}
+		}
+		return true;
+	}
+
+	protected DocumentFile buildDocumentFile(Path path, Path currentRootPath) {
+		Path relPath = currentRootPath.relativize(path);
+		String uri = "/" + relPath.toString().replace("\\", "/");
+		File f = path.toFile();
+		return new DocumentFile(uri, f);
+	}
+
+	protected DocumentFile processDocumentFile(DocumentFile documentFile) {
+		for (DocumentFileProcessor processor : documentFileProcessors) {
+			documentFile = processor.processDocumentFile(documentFile);
+			if (documentFile == null) {
+				break;
+			}
+		}
+		return documentFile;
+	}
+
+	protected void initialize() {
+		CollectionsDocumentFileProcessor cdfp = new CollectionsDocumentFileProcessor();
+		PermissionsDocumentFileProcessor pdfp = new PermissionsDocumentFileProcessor();
+
+		addFileFilter(cdfp);
+		addFileFilter(pdfp);
+
+		addDocumentFileProcessor(cdfp);
+		addDocumentFileProcessor(pdfp);
+		addDocumentFileProcessor(new FormatDocumentFileProcessor());
+	}
+
+	public DocumentFileProcessor getDocumentFileProcessor(String classShortName) {
+		for (DocumentFileProcessor processor : documentFileProcessors) {
+			if (ClassUtils.getShortName(processor.getClass()).equals(classShortName)) {
+				return processor;
+			}
+		}
+		return null;
+	}
+
+	public void addDocumentFileProcessor(DocumentFileProcessor processor) {
+		if (documentFileProcessors == null) {
+			documentFileProcessors = new ArrayList<>();
+		}
+		documentFileProcessors.add(processor);
+	}
+
+	public void addFileFilter(FileFilter fileFilter) {
+		if (fileFilters == null) {
+			fileFilters = new ArrayList<>();
+		}
+		fileFilters.add(fileFilter);
+	}
+
+	public List<DocumentFileProcessor> getDocumentFileProcessors() {
+		return documentFileProcessors;
+	}
+
+	public void setDocumentFileProcessors(List<DocumentFileProcessor> documentFileProcessors) {
+		this.documentFileProcessors = documentFileProcessors;
+	}
+
+	public List<FileFilter> getFileFilters() {
+		return fileFilters;
+	}
+
+	public void setFileFilters(List<FileFilter> fileFilters) {
+		this.fileFilters = fileFilters;
+	}
+}