Merge pull request #186 from BillFarber/feature/cascadeCollectionsAndPermissions

DEVEXP-527: Starting on tests for cascading collections and permissions.

Author: BillFarber

src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2023 MarkLogic Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.client.ext.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Properties;
+import java.util.Stack;
+
+/**
+ * Adds a stack to store Properties objects while traversing a directory tree.
+ */
+abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor {
+	final private Stack<Properties> propertiesStack = new Stack<>();
+
+	protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilename) {
+		super(propertiesFilename);
+	}
+
+	protected void preVisitDirectory(Path dir) throws IOException {
+		File collectionsPropertiesFile = new File(dir.toFile(), this.getPropertiesFilename());
+		if (collectionsPropertiesFile.exists()) {
+			this.loadProperties(collectionsPropertiesFile);
+		} else {
+			if (!propertiesStack.isEmpty()) {
+				this.setProperties(propertiesStack.peek());
+			} else {
+				this.setProperties(new Properties());
+			}
+		}
+		propertiesStack.push(this.getProperties());
+	}
+
+	protected void postVisitDirectory() {
+		propertiesStack.pop();
+		if (!propertiesStack.isEmpty()) {
+			this.setProperties(propertiesStack.peek());
+		}
+	}
+}

src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java

@@ -22,7 +22,7 @@
  * key is the name of a file in the directory, and the value is a comma-delimited list of collections to load the file
  * into (which means you can't use a comma in any collection name).
  */
-public class CollectionsFileDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor {
+public class CollectionsFileDocumentFileProcessor extends CascadingPropertiesDrivenDocumentFileProcessor {
 
 	private String delimiter = ",";
 

src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java

@@ -22,8 +22,7 @@
 import java.net.URISyntaxException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 /**
  * Non-threadsafe implementation that implements FileVisitor as a way of descending one or more file paths.
@@ -105,6 +104,10 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 			if (logger.isDebugEnabled()) {
 				logger.debug("Visiting directory: " + dir);
 			}
+
+			collectionsFileDocumentFileProcessor.preVisitDirectory(dir);
+			permissionsFileDocumentFileProcessor.preVisitDirectory(dir);
+
 			return FileVisitResult.CONTINUE;
 		} else {
 			if (logger.isDebugEnabled()) {
@@ -127,6 +130,10 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
 		if (exc != null) {
 			logger.warn("Error in postVisitDirectory: " + exc.getMessage(), exc);
 		}
+
+		collectionsFileDocumentFileProcessor.postVisitDirectory();
+		permissionsFileDocumentFileProcessor.postVisitDirectory();
+
 		return FileVisitResult.CONTINUE;
 	}
 

src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java

@@ -24,7 +24,7 @@
  * Looks for a special file in each directory - defaults to permissions.properties - that contains properties where the
  * key is the name of a file in the directory, and the value is a comma-delimited list of role,capability,role,capability,etc.
  */
-public class PermissionsFileDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor {
+public class PermissionsFileDocumentFileProcessor extends CascadingPropertiesDrivenDocumentFileProcessor {
 
 	private DocumentPermissionsParser documentPermissionsParser;
 

src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java

@@ -22,8 +22,6 @@
 import java.io.FileFilter;
 import java.io.FileReader;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Properties;
 
 /**
@@ -34,10 +32,9 @@ public abstract class PropertiesDrivenDocumentFileProcessor extends LoggingObjec
 
 	protected final static String WILDCARD_KEY = "*";
 
-	private String propertiesFilename;
+	private final String propertiesFilename;
 
-	// Used to avoid checking for and loading the properties for every file in a directory
-	private Map<File, Properties> propertiesCache = new HashMap<>();
+	private Properties properties;
 
 	private TokenReplacer tokenReplacer;
 
@@ -60,36 +57,17 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
 		if (!accept(file)) {
 			return null;
 		}
-
-		File propertiesFile = new File(file.getParentFile(), propertiesFilename);
-		if (propertiesFile.exists()) {
-			try {
-				Properties props = loadProperties(propertiesFile);
-				processProperties(documentFile, props);
-			} catch (IOException e) {
-				logger.warn("Unable to load properties from file: " + propertiesFile.getAbsolutePath(), e);
-			}
-		}
-
+		processProperties(documentFile, properties);
 		return documentFile;
 	}
 
 	protected abstract void processProperties(DocumentFile documentFile, Properties properties);
 
 	protected Properties loadProperties(File propertiesFile) throws IOException {
-		Properties props = null;
-		if (propertiesCache.containsKey(propertiesFile)) {
-			props = propertiesCache.get(propertiesFile);
-		}
-		if (props != null) {
-			return props;
-		}
-
-		props = new Properties();
+		properties = new Properties();
 		try (FileReader reader = new FileReader(propertiesFile)) {
-			props.load(reader);
-			propertiesCache.put(propertiesFile, props);
-			return props;
+			properties.load(reader);
+			return properties;
 		}
 	}
 
@@ -101,10 +79,6 @@ protected String getPropertyValue(Properties properties, String propertyName) {
 		return tokenReplacer != null && value != null ? tokenReplacer.replaceTokens(value) : value;
 	}
 
-	public Map<File, Properties> getPropertiesCache() {
-		return propertiesCache;
-	}
-
 	public String getPropertiesFilename() {
 		return propertiesFilename;
 	}
@@ -116,4 +90,12 @@ public void setTokenReplacer(TokenReplacer tokenReplacer) {
 	protected TokenReplacer getTokenReplacer() {
 		return tokenReplacer;
 	}
+
+	protected void setProperties(Properties properties) {
+		this.properties = properties;
+	}
+
+	protected Properties getProperties() {
+		return this.properties;
+	}
 }

src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java

@@ -16,6 +16,7 @@
 package com.marklogic.client.ext;
 
 import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.io.DocumentMetadataHandle;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -27,6 +28,12 @@
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import java.util.Set;
+import java.util.function.Consumer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 @ExtendWith(SpringExtension.class)
 @ContextConfiguration(classes = {TestConfig.class})
 public abstract class AbstractIntegrationTest {
@@ -58,6 +65,36 @@ protected DatabaseClient newClient(String database) {
 		clientConfig.setDatabase(currentDatabase);
 		return client;
 	}
+
+	protected final void verifyMetadata(String uri, Consumer<DocumentMetadataHandle> verifier) {
+		verifier.accept(client.newJSONDocumentManager().readMetadata(uri, new DocumentMetadataHandle()));
+	}
+
+	protected final void verifyCollections(String uri, String... collections) {
+		verifyMetadata(uri, metadata -> {
+			assertEquals(collections.length, metadata.getCollections().size());
+			for (String collection : collections) {
+				assertTrue(metadata.getCollections().contains(collection), "Did not find expected collection: " +
+					collection + "; actual collections: " + metadata.getCollections());
+			}
+		});
+	}
+
+	protected final void verifyPermissions(String uri, String... permissionsRolesAndCapabilities) {
+		verifyMetadata(uri, metadata -> {
+			for (int i = 0; i < permissionsRolesAndCapabilities.length; i += 2) {
+				String role = permissionsRolesAndCapabilities[i];
+				assertTrue(metadata.getPermissions().containsKey(role), "Did not find permissions with role: " +
+					role + "; actual permissions: " + metadata.getPermissions());
+
+				DocumentMetadataHandle.Capability capability =
+					DocumentMetadataHandle.Capability.valueOf(permissionsRolesAndCapabilities[i + 1].toUpperCase());
+				Set<DocumentMetadataHandle.Capability> capabilities = metadata.getPermissions().get(role);
+				assertTrue(capabilities.contains(capability), "Did not find permission for role: " + role +
+					" with capability: " + capability + "; actual capabilities: " + capabilities);
+			}
+		});
+	}
 }
 
 @Configuration

src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2023 MarkLogic Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.client.ext.file;
+
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.ext.AbstractIntegrationTest;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class CascadeCollectionsAndPermissionsTest extends AbstractIntegrationTest {
+
+	final private String PARENT_COLLECTION = "ParentCollection";
+	final private String CHILD_COLLECTION = "ChildCollection";
+
+	@BeforeEach
+	public void setup() {
+		client = newClient(MODULES_DATABASE);
+		DatabaseClient modulesClient = client;
+		modulesClient.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval();
+	}
+
+	@Test
+	public void parentWithBothProperties() {
+		String directory = "src/test/resources/process-files/cascading-metadata-test/parent1-withCP";
+		GenericFileLoader loader = new GenericFileLoader(client);
+		loader.loadFiles(directory);
+
+		verifyCollections( "/child1_1-noCP/test.json", PARENT_COLLECTION);
+		verifyPermissions( "/child1_1-noCP/test.json", "rest-writer", "update");
+
+		verifyCollections( "/child1_2-withCP/test.json", CHILD_COLLECTION);
+		verifyPermissions( "/child1_2-withCP/test.json", "rest-reader", "read");
+
+		verifyCollections( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", CHILD_COLLECTION);
+		verifyPermissions( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", "rest-reader", "read");
+
+		verifyCollections("/child1/child1.json", "ParentCollection");
+		verifyPermissions("/child1/child1.json", "rest-writer", "update");
+
+		verifyCollections("/child2/child2.json", "child2");
+		verifyPermissions("/child2/child2.json", "app-user", "read");
+
+		verifyCollections("/parent.json", "ParentCollection");
+		verifyPermissions("/parent.json", "rest-writer", "update");
+	}
+
+	@Test
+	public void parentWithNoProperties() {
+		String directory = "src/test/resources/process-files/cascading-metadata-test/parent2-noCP";
+		GenericFileLoader loader = new GenericFileLoader(client);
+		loader.loadFiles(directory);
+
+		verifyCollections( "/child2_1-withCP/test.json", CHILD_COLLECTION);
+		verifyPermissions( "/child2_1-withCP/test.json", "rest-reader", "read");
+
+		verifyCollections( "/child2_2-noCP/test.json");
+		verifyPermissions( "/child2_2-noCP/test.json");
+
+		verifyCollections( "/child2_3-withCnoP/test.json", PARENT_COLLECTION);
+		verifyPermissions( "/child2_3-withCnoP/test.json");
+
+		verifyCollections( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", PARENT_COLLECTION);
+		verifyPermissions( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", "rest-reader", "read");
+	}
+}

src/test/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessorTest.java

@@ -19,6 +19,7 @@
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Properties;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -28,9 +29,12 @@ public class CollectionsFileDocumentFileProcessorTest {
 	private CollectionsFileDocumentFileProcessor processor = new CollectionsFileDocumentFileProcessor();
 
 	@Test
-	public void wildcard() {
+	public void wildcard() throws IOException {
 		File testDir = new File("src/test/resources/process-files/wildcard-test");
 
+		File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename());
+		processor.loadProperties(collectionsPropertiesFile);
+
 		DocumentFile file = new DocumentFile("/test.json", new File(testDir, "test.json"));
 		processor.processDocumentFile(file);
 		assertTrue(file.getDocumentMetadata().getCollections().contains("json-data"));
@@ -45,9 +49,12 @@ public void wildcard() {
 	}
 
 	@Test
-	public void replaceTokens() {
+	public void replaceTokens() throws IOException {
 		File testDir = new File("src/test/resources/process-files/token-test");
 
+		File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename());
+		processor.loadProperties(collectionsPropertiesFile);
+
 		DefaultTokenReplacer tokenReplacer = new DefaultTokenReplacer();
 		Properties props = new Properties();
 		props.setProperty("%%someCollection%%", "this-was-replaced");