fixed keeping of empty directories with .keep files in them

Author: jscancella

build.gradle

@@ -73,3 +73,11 @@ pmdTest.enabled = false
 pmd {
   ruleSets = ["java-basic", "java-braces", "java-empty"]
 }
+
+//Keep this for easy viewing of html findbugs report
+//tasks.withType(FindBugs) {
+//  reports {
+//      xml.enabled = false
+//      html.enabled = true
+//  }
+//}

src/main/java/gov/loc/repository/bagit/creator/AddPayloadToBagManifestVistor.java

@@ -50,7 +50,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 
   @Override
   public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)throws IOException{
-    if(!includeHiddenFiles && Files.isHidden(path)){
+    if(!includeHiddenFiles && Files.isHidden(path) && !path.endsWith(".keep")){
       logger.debug("Skipping [{}] since we are ignoring hidden files", path);
     }
     else{

src/main/java/gov/loc/repository/bagit/reader/BagReader.java

@@ -21,6 +21,7 @@
 import gov.loc.repository.bagit.hash.BagitAlgorithmNameToSupportedAlgorithmMapping;
 import gov.loc.repository.bagit.hash.StandardBagitAlgorithmNameToSupportedAlgorithmMapping;
 import gov.loc.repository.bagit.hash.SupportedAlgorithm;
+import gov.loc.repository.bagit.util.PathUtils;
 import gov.loc.repository.bagit.verify.PayloadFileExistsInManifestVistor;
 import javafx.util.Pair;
 
@@ -129,11 +130,13 @@ public Bag readAllManifests(Path rootDir, Bag bag) throws IOException{
     DirectoryStream<Path> manifests = getAllManifestFiles(rootDir);
 
     for (Path path : manifests){
-      if(path.getFileName().toString().startsWith("tagmanifest-")){
+      String filename = PathUtils.getFilename(path);
+      
+      if(filename.startsWith("tagmanifest-")){
         logger.debug("Found tag manifest [{}]", path);
         newBag.getTagManifests().add(readManifest(path, bag.getRootDir()));
       }
-      else if(path.getFileName().toString().startsWith("manifest-")){
+      else if(filename.startsWith("manifest-")){
         logger.debug("Found payload manifest [{}]", path);
         newBag.getPayLoadManifests().add(readManifest(path, bag.getRootDir()));
       }
@@ -145,7 +148,9 @@ else if(path.getFileName().toString().startsWith("manifest-")){
   protected DirectoryStream<Path> getAllManifestFiles(Path rootDir) throws IOException{
     DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
       public boolean accept(Path file) throws IOException {
-        return file.getFileName().toString().startsWith("tagmanifest-") || file.getFileName().toString().startsWith("manifest-");
+        if(file == null || file.getFileName() == null){ return false;}
+        String filename = PathUtils.getFilename(file);
+        return filename.startsWith("tagmanifest-") || filename.startsWith("manifest-");
       }
     };
 
@@ -161,7 +166,7 @@ public boolean accept(Path file) throws IOException {
    */
   public Manifest readManifest(Path manifestFile, Path bagRootDir) throws IOException{
     logger.debug("Reading manifest [{}]", manifestFile);
-    String alg = manifestFile.getFileName().toString().split("[-\\.]")[1];
+    String alg = PathUtils.getFilename(manifestFile).split("[-\\.]")[1];
     SupportedAlgorithm algorithm = nameMapping.getMessageDigestName(alg);
 
     Manifest manifest = new Manifest(algorithm);

src/main/java/gov/loc/repository/bagit/util/PathUtils.java

@@ -0,0 +1,24 @@
+package gov.loc.repository.bagit.util;
+
+import java.nio.file.Path;
+
+public class PathUtils {
+
+  /**
+   * Needed to get rid of findbugs "dodgy code warnings" in regards to getting the filename of a path as a string
+   * 
+   * @param path
+   * @return the filename or an empty string
+   */
+  public static String getFilename(Path path){
+    String filename = "";
+    if(path != null){
+      Path filenamePath = path.getFileName();
+      if(filenamePath != null){
+        filename = filenamePath.toString();
+      }
+    }
+    
+    return filename;
+  }
+}

src/main/java/gov/loc/repository/bagit/verify/BagVerifier.java

@@ -35,6 +35,7 @@
 import gov.loc.repository.bagit.reader.BagReader;
 import gov.loc.repository.bagit.tasks.CheckIfFileExistsTask;
 import gov.loc.repository.bagit.tasks.CheckManifestHashsTask;
+import gov.loc.repository.bagit.util.PathUtils;
 import javafx.util.Pair;
 
 /**
@@ -257,7 +258,7 @@ protected void checkIfAtLeastOnePayloadManifestsExist(Path rootDir, Version vers
     }
 
     for(Path path : directoryStream){
-      if(path.getFileName().toString().startsWith("manifest-")){
+      if(PathUtils.getFilename(path).startsWith("manifest-")){
         logger.debug("Found payload manifest file [{}]", path.getFileName());
         hasAtLeastOneManifest = true;
       }
@@ -282,7 +283,8 @@ protected Set<Path> getAllFilesListedInManifests(Bag bag) throws IOException{
     BagReader reader = new BagReader(nameMapping);
 
     for(Path path : directoryStream){
-      if(path.getFileName().toString().startsWith("tagmanifest-") || path.getFileName().toString().startsWith("manifest-")){
+      String filename = PathUtils.getFilename(path);
+      if(filename.startsWith("tagmanifest-") || filename.startsWith("manifest-")){
         logger.debug("Getting files and checksums listed in [{}]", path);
         Manifest manifest = reader.readManifest(path, bag.getRootDir());
         filesListedInManifests.addAll(manifest.getFileToChecksumMap().keySet());

src/main/java/gov/loc/repository/bagit/verify/FileCountAndTotalSizeVistor.java

@@ -36,10 +36,15 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 
   @Override
   public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException{
-    count++;
-    long size = Files.size(path);
-    logger.debug("File [{}] hash a size of [{}] bytes", path, size);
-    totalSize += size;
+    if(!ignoreHiddenFiles && Files.isHidden(path) && !path.endsWith(".keep")){
+      logger.debug("Skipping [{}] since we are ignoring hidden files", path);
+    }
+    else{
+      count++;
+      long size = Files.size(path);
+      logger.debug("File [{}] hash a size of [{}] bytes", path, size);
+      totalSize += size;
+    }
 
     return FileVisitResult.CONTINUE;
   }

src/main/java/gov/loc/repository/bagit/writer/BagWriter.java

@@ -191,8 +191,9 @@ protected static void writeAdditionalTagPayloadFiles(Set<Manifest> manifests, Pa
       for(Entry<Path, String> entry : manifest.getFileToChecksumMap().entrySet()){
         Path relativeLocation = bagRootDir.relativize(entry.getKey());
         Path writeTo = outputDir.resolve(relativeLocation);
-        if(!Files.exists(writeTo)){
-          Files.createDirectories(writeTo.getParent());
+        Path writeToParent = writeTo.getParent();
+        if(!Files.exists(writeTo) && writeToParent != null){
+          Files.createDirectories(writeToParent);
           Files.copy(entry.getKey(), writeTo);
         }
       }

src/test/java/gov/loc/repository/bagit/creator/AddPayloadToBagManifestVistorTest.java

@@ -0,0 +1,29 @@
+package gov.loc.repository.bagit.creator;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import gov.loc.repository.bagit.domain.Manifest;
+import gov.loc.repository.bagit.hash.StandardSupportedAlgorithms;
+
+public class AddPayloadToBagManifestVistorTest extends Assert {
+
+  @Test
+  public void includeDotKeepFilesInManifest() throws Exception{
+    Manifest manifest = new Manifest(StandardSupportedAlgorithms.MD5);
+    MessageDigest messageDigest = MessageDigest.getInstance("MD5");
+    boolean includeHiddenFiles = false;
+    Path start = Paths.get(getClass().getClassLoader().getResource("dotKeepExampleBag").getFile()).resolve("data");
+    
+    AddPayloadToBagManifestVistor sut = new AddPayloadToBagManifestVistor(manifest, messageDigest, includeHiddenFiles);
+    Files.walkFileTree(start, sut);
+    
+    assertEquals(1, manifest.getFileToChecksumMap().size());
+    assertTrue(manifest.getFileToChecksumMap().containsKey(start.resolve("fooDir/.keep")));
+  }
+}

src/test/resources/dotKeepExampleBag/bagit.txt

@@ -0,0 +1,2 @@
+BagIt-Version: 0.97
+Tag-File-Character-Encoding: UTF-8