#49 Can now load schemas from subdirectories

Author: Rob Rudin

CHANGELOG.mdown

@@ -8,7 +8,7 @@ Releases past 2.7.0 now tracked in Github
 
 ## 2.6.1
 
-* [#17](https://github.com/rjrudin/ml-javaclient-util/issues/17) Extension metadata can now be loaded from the classpath
+* [#17](https://github.com/rjrudin/ml-javaclient-util/issues/17) Extension documentMetadata can now be loaded from the classpath
 
 ## 2.6
 

src/main/java/com/marklogic/client/file/DefaultDocumentFileFinder.java

@@ -0,0 +1,75 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.helper.LoggingObject;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+
+public class DefaultDocumentFileFinder extends LoggingObject implements FileVisitor<Path>, DocumentFileFinder {
+
+	private Path currentAssetPath;
+	private FileFilter fileFilter;
+	private List<DocumentFile> documentFiles;
+
+	public List<DocumentFile> findDocumentFiles(String... paths) {
+		documentFiles = new ArrayList<>();
+		for (String path : paths) {
+			if (logger.isDebugEnabled()) {
+				logger.debug(format("Finding documents at path: %s", path));
+			}
+			this.currentAssetPath = Paths.get(path);
+			try {
+				Files.walkFileTree(this.currentAssetPath, this);
+			} catch (IOException ie) {
+				throw new RuntimeException(format("IO error while walking file tree at path: %s", path), ie);
+			}
+		}
+		return documentFiles;
+	}
+
+	@Override
+	public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+		boolean accept = fileFilter == null || fileFilter.accept(dir.toFile());
+		if (accept) {
+			if (logger.isTraceEnabled()) {
+				logger.trace("Visiting directory: " + dir);
+			}
+			return FileVisitResult.CONTINUE;
+		} else {
+			if (logger.isTraceEnabled()) {
+				logger.trace("Skipping directory: " + dir);
+			}
+			return FileVisitResult.SKIP_SUBTREE;
+		}
+	}
+
+	@Override
+	public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+		if (fileFilter == null || fileFilter.accept(path.toFile())) {
+			Path relPath = currentAssetPath.relativize(path);
+			String uri = "/" + relPath.toString().replace("\\", "/");
+			File f = path.toFile();
+			this.documentFiles.add(new DocumentFile(uri, f));
+		}
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+		return FileVisitResult.CONTINUE;
+	}
+
+	public void setFileFilter(FileFilter fileFilter) {
+		this.fileFilter = fileFilter;
+	}
+}

src/main/java/com/marklogic/client/file/DefaultFileLoader.java

@@ -0,0 +1,77 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.batch.BatchWriter;
+import com.marklogic.client.helper.LoggingObject;
+import org.springframework.util.ClassUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DefaultFileLoader extends LoggingObject implements FileLoader {
+
+	private DocumentFileFinder documentFileFinder;
+	private BatchWriter batchWriter;
+	private List<DocumentFileProcessor> documentFileProcessors;
+
+	public DefaultFileLoader(BatchWriter batchWriter) {
+		this(batchWriter, new DefaultDocumentFileFinder());
+	}
+
+	/**
+	 * @param batchWriter This is assumed to have already been initialized, as this class is not interested in managing
+	 *                    the lifecycle of a BatchWriter
+	 * @param documentFileFinder
+	 */
+	public DefaultFileLoader(BatchWriter batchWriter, DocumentFileFinder documentFileFinder) {
+		this.batchWriter = batchWriter;
+		this.documentFileFinder = documentFileFinder;
+		initializeDocumentFileProcessors();
+	}
+
+	protected void initializeDocumentFileProcessors() {
+		documentFileProcessors = new ArrayList<>();
+		documentFileProcessors.add(new FormatDocumentFileProcessor());
+	}
+
+	@Override
+	public List<DocumentFile> loadFiles(String... paths) {
+		List<DocumentFile> documentFiles = documentFileFinder.findDocumentFiles(paths);
+		processDocumentFiles(documentFiles);
+		batchWriter.write(documentFiles);
+		return documentFiles;
+	}
+
+	protected List<DocumentFile> processDocumentFiles(List<DocumentFile> documentFiles) {
+		for (DocumentFile file : documentFiles) {
+			for (DocumentFileProcessor processor : documentFileProcessors) {
+				if (processor.supportsDocumentFile(file)) {
+					processor.processDocumentFile(file);
+				}
+			}
+		}
+		return documentFiles;
+	}
+	public DocumentFileProcessor getDocumentFileProcessor(String classShortName) {
+		for (DocumentFileProcessor processor : documentFileProcessors) {
+			if (ClassUtils.getShortName(processor.getClass()).equals(classShortName)) {
+				return processor;
+			}
+		}
+		return null;
+	}
+
+	public void addDocumentFileProcessor(DocumentFileProcessor processor) {
+		if (documentFileProcessors == null) {
+			documentFileProcessors = new ArrayList<>();
+		}
+		documentFileProcessors.add(processor);
+	}
+
+	public List<DocumentFileProcessor> getDocumentFileProcessors() {
+		return documentFileProcessors;
+	}
+
+	public void setDocumentFileProcessors(List<DocumentFileProcessor> documentFileProcessors) {
+		this.documentFileProcessors = documentFileProcessors;
+	}
+}

src/main/java/com/marklogic/client/file/DocumentFile.java

@@ -0,0 +1,82 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.document.DocumentWriteOperation;
+import com.marklogic.client.io.DocumentMetadataHandle;
+import com.marklogic.client.io.FileHandle;
+import com.marklogic.client.io.Format;
+import com.marklogic.client.io.marker.AbstractWriteHandle;
+import com.marklogic.client.io.marker.DocumentMetadataWriteHandle;
+
+import java.io.File;
+
+/**
+ * Encapsulates a file that should be written to MarkLogic as a single document. Implements DocumentWriteOperation so
+ * that it can be easily written via the Java Client API.
+ */
+public class DocumentFile implements DocumentWriteOperation {
+
+	private String uri;
+	private File file;
+	private Format format;
+	private DocumentMetadataHandle documentMetadata;
+
+	public DocumentFile(String uri, File file) {
+		this.uri = uri;
+		this.file = file;
+		this.documentMetadata = new DocumentMetadataHandle();
+	}
+
+	public String getUri() {
+		return uri;
+	}
+
+	public File getFile() {
+		return file;
+	}
+
+	public String getFileExtension() {
+		String name = file.getName();
+		int pos = name.lastIndexOf('.');
+		return pos < 0 ? name : name.substring(pos + 1);
+	}
+
+	@Override
+	public OperationType getOperationType() {
+		return OperationType.DOCUMENT_WRITE;
+	}
+
+	@Override
+	public DocumentMetadataWriteHandle getMetadata() {
+		return documentMetadata;
+	}
+
+	@Override
+	public AbstractWriteHandle getContent() {
+		FileHandle h = new FileHandle(file);
+		return format != null ? h.withFormat(format) : h;
+	}
+
+	public void setFormat(Format format) {
+		this.format = format;
+	}
+
+	public DocumentMetadataHandle getDocumentMetadata() {
+		return documentMetadata;
+	}
+
+	public void setDocumentMetadata(DocumentMetadataHandle documentMetadata) {
+		this.documentMetadata = documentMetadata;
+	}
+
+	public void setUri(String uri) {
+		this.uri = uri;
+	}
+
+	public void setFile(File file) {
+		this.file = file;
+	}
+
+	public Format getFormat() {
+		return format;
+	}
+}

src/main/java/com/marklogic/client/file/DocumentFileFinder.java

@@ -0,0 +1,8 @@
+package com.marklogic.client.file;
+
+import java.util.List;
+
+public interface DocumentFileFinder {
+
+	List<DocumentFile> findDocumentFiles(String... paths);
+}

src/main/java/com/marklogic/client/file/DocumentFileProcessor.java

@@ -0,0 +1,12 @@
+package com.marklogic.client.file;
+
+/**
+ * Callback-style interface for processing - i.e. do whatever you want with - a DocumentFile instance before it's
+ * written to MarkLogic.
+ */
+public interface DocumentFileProcessor {
+
+	boolean supportsDocumentFile(DocumentFile documentFile);
+
+	void processDocumentFile(DocumentFile documentFile);
+}

src/main/java/com/marklogic/client/file/DocumentMetadataSetter.java

@@ -0,0 +1,6 @@
+package com.marklogic.client.file;
+
+public interface DocumentMetadataSetter {
+
+	void setMetadata(DocumentFile documentFile);
+}

src/main/java/com/marklogic/client/file/FileLoader.java

@@ -0,0 +1,12 @@
+package com.marklogic.client.file;
+
+import java.util.List;
+
+/**
+ * Interface for loading files from a set of paths, returning a list of the files that were loaded as DocumentFile
+ * objects.
+ */
+public interface FileLoader {
+
+	List<DocumentFile> loadFiles(String... paths);
+}

src/main/java/com/marklogic/client/file/FormatDocumentFileProcessor.java

@@ -0,0 +1,43 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.io.Format;
+import com.marklogic.client.modulesloader.impl.FormatGetter;
+import com.marklogic.client.modulesloader.xcc.DefaultDocumentFormatGetter;
+
+/**
+ * Delegates to DefaultDocumentFormatGetter by default for determining what Format to use for the File in a given
+ * DocumentFile.
+ */
+public class FormatDocumentFileProcessor implements DocumentFileProcessor {
+
+	private FormatGetter formatGetter;
+
+	public FormatDocumentFileProcessor() {
+		this(new DefaultDocumentFormatGetter());
+	}
+
+	public FormatDocumentFileProcessor(FormatGetter formatGetter) {
+		this.formatGetter = formatGetter;
+	}
+
+	@Override
+	public boolean supportsDocumentFile(DocumentFile documentFile) {
+		return true;
+	}
+
+	@Override
+	public void processDocumentFile(DocumentFile documentFile) {
+		Format format = formatGetter.getFormat(documentFile.getFile());
+		if (format != null) {
+			documentFile.setFormat(format);
+		}
+	}
+
+	public FormatGetter getFormatGetter() {
+		return formatGetter;
+	}
+
+	public void setFormatGetter(FormatGetter formatGetter) {
+		this.formatGetter = formatGetter;
+	}
+}