Skip to content

Commit 98080c2

Browse files
johnscancellajohnscancella
committed
Merge pull request #34 from danshim/master
Bagit-java issue 31 : Hidden files should be ignored.
2 parents d78bbf2 + ce42831 commit 98080c2

18 files changed

Lines changed: 178 additions & 73 deletions

‎src/main/java/gov/loc/repository/bagit/Bag.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,9 @@ enum Format {
238238
BagConstants getBagConstants();
239239

240240
BagPartFactory getBagPartFactory();
241-
241+
242+
BagFactory getBagFactory();
243+
242244
/**
243245
* <p>Contains names for constants associated with a bag.
244246
* BagIt defines and reserves several names, and some of those names

‎src/main/java/gov/loc/repository/bagit/BagFactory.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import gov.loc.repository.bagit.Bag.BagConstants;
44
import gov.loc.repository.bagit.Bag.BagPartFactory;
5+
import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter;
6+
import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter;
57
import gov.loc.repository.bagit.impl.PreBagImpl;
68

79
import java.io.File;
@@ -23,7 +25,7 @@
2325
*
2426
* @see Bag
2527
*/
26-
public class BagFactory {
28+
public class BagFactory {
2729

2830
/**
2931
* <p>Specifies the mechanism used to load a bag from disk.
@@ -73,7 +75,9 @@ public static Version valueOfString(String versionString) {
7375
}
7476

7577
}
76-
78+
79+
private FileSystemNodeFilter defaultNodeFilter;
80+
7781
/**
7882
* The latest version of the BagIt spec. Currently, this
7983
* is {@link Version#V0_97 0.97}.
@@ -84,7 +88,11 @@ public static Version valueOfString(String versionString) {
8488
* Creates an instance of a bag factory.
8589
*/
8690
public BagFactory() {
87-
91+
this.defaultNodeFilter = new NotHiddenFileSystemNodeFilter();
92+
}
93+
94+
public BagFactory(FileSystemNodeFilter defaultNodeFilter) {
95+
this.defaultNodeFilter = defaultNodeFilter;
8896
}
8997

9098
/**
@@ -300,5 +308,8 @@ public PreBag createPreBag(File dir) {
300308
preBag.setFile(dir);
301309
return preBag;
302310
}
303-
311+
312+
public FileSystemNodeFilter getDefaultNodeFilter() {
313+
return this.defaultNodeFilter;
314+
}
304315
}

‎src/main/java/gov/loc/repository/bagit/BagHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class BagHelper {
3030
public static String getVersion(File bagFile) {
3131
DirNode bagFileDirNode = null;
3232
try {
33-
bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile);
33+
bagFileDirNode = FileSystemFactory.getDirNodeForBag(bagFile, new BagFactory());
3434
log.trace(MessageFormat.format("BagFileDirNode has filepath {0} and is a {1}", bagFileDirNode.getFilepath(), bagFileDirNode.getClass().getSimpleName()));
3535

3636
FileNode bagItNode = bagFileDirNode.childFile(BAGIT);

‎src/main/java/gov/loc/repository/bagit/driver/CommandLineBagDriver.java‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import java.util.List;
1818
import java.util.Map;
1919

20+
import gov.loc.repository.bagit.filesystem.filter.NotHiddenFileSystemNodeFilter;
2021
import org.apache.commons.io.IOUtils;
2122
import org.apache.commons.logging.Log;
2223
import org.apache.commons.logging.LogFactory;
@@ -144,15 +145,16 @@ public class CommandLineBagDriver {
144145
public static final String PARAM_FAIL_MODE = "failmode";
145146
public static final String PARAM_COMPRESSION_LEVEL = "compressionlevel";
146147
public static final String PARAM_MOVE = "move";
147-
148+
public static final String PARAM_INCLUDE_HIDDEN = "includehiddenfiles";
149+
148150
public static final String VALUE_WRITER_FILESYSTEM = Format.FILESYSTEM.name().toLowerCase();
149151
public static final String VALUE_WRITER_ZIP = Format.ZIP.name().toLowerCase();
150152

151153
private static final Log log = LogFactory.getLog(CommandLineBagDriver.class);
152154

153155
private Map<String, Operation> operationMap = new HashMap<String, Operation>();
154-
private BagFactory bagFactory = new BagFactory();
155-
156+
157+
156158
public static void main(String[] args) throws Exception {
157159
CommandLineBagDriver driver = new CommandLineBagDriver();
158160
int ret = driver.execute(args);
@@ -372,6 +374,7 @@ private void addOperation(String name, String help, Parameter[] params, String[]
372374
jsap.registerParameter(new Switch( PARAM_HELP, JSAP.NO_SHORTFLAG, PARAM_HELP, "Prints help." ));
373375
jsap.registerParameter(new Switch(PARAM_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_VERBOSE, "Reports progress of the operation to the console."));
374376
jsap.registerParameter(new Switch(PARAM_LOG_VERBOSE, JSAP.NO_SHORTFLAG, PARAM_LOG_VERBOSE, "Reports progress of the operation to the log."));
377+
jsap.registerParameter(new Switch( PARAM_INCLUDE_HIDDEN, JSAP.NO_SHORTFLAG, PARAM_INCLUDE_HIDDEN, "Include hidden files." ));
375378

376379
this.operationMap.put(name, new Operation(name, jsap, help, examples));
377380
}
@@ -571,7 +574,8 @@ private static String argsToString(String[] args) {
571574
return sb.toString();
572575
}
573576

574-
private Bag getBag(File sourceFile, Version version, LoadOption loadOption) {
577+
private Bag getBag(BagFactory bagFactory, File sourceFile, Version version, LoadOption loadOption) {
578+
575579
if (version != null) {
576580
if (sourceFile != null) {
577581
return bagFactory.createBag(sourceFile, version, loadOption);
@@ -611,6 +615,15 @@ private int performOperation(Operation operation, JSAPResult config) {
611615
destFile = new File(config.getString(PARAM_DESTINATION));
612616
}
613617

618+
BagFactory bagFactory = null;
619+
if (config.getBoolean(PARAM_INCLUDE_HIDDEN)) {
620+
// null filter will include all nodes, including hidden ones
621+
bagFactory = new BagFactory(null);
622+
} else {
623+
// The default should filter out hidden files
624+
bagFactory = new BagFactory(new NotHiddenFileSystemNodeFilter());
625+
}
626+
614627
Writer writer = null;
615628
if (config.contains(PARAM_WRITER)) {
616629
Format format = Format.valueOf(config.getString(PARAM_WRITER).toUpperCase());
@@ -726,7 +739,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
726739
ValidVerifierImpl verifier = new ValidVerifierImpl(completeVerifier, checksumVerifier);
727740
verifier.addProgressListener(listener);
728741
verifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase()));
729-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
742+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
730743
try {
731744
SimpleResult result = verifier.verify(bag);
732745
log.info(result.toString());
@@ -745,7 +758,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
745758
completeVerifier.setIgnoreSymlinks(config.getBoolean(PARAM_EXCLUDE_SYMLINKS));
746759
completeVerifier.addProgressListener(listener);
747760
completeVerifier.setFailMode(FailMode.valueOf(config.getString(PARAM_FAIL_MODE).toUpperCase()));
748-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
761+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
749762
try {
750763
SimpleResult result = completeVerifier.verify(bag);
751764
log.info(result.toString());
@@ -758,7 +771,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
758771
bag.close();
759772
}
760773
} else if (OPERATION_VERIFY_TAGMANIFESTS.equals(operation.name)) {
761-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
774+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
762775
try {
763776
ParallelManifestChecksumVerifier verifier = new ParallelManifestChecksumVerifier();
764777
verifier.addProgressListener(listener);
@@ -774,7 +787,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
774787
bag.close();
775788
}
776789
} else if (OPERATION_VERIFY_PAYLOADMANIFESTS.equals(operation.name)) {
777-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
790+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
778791
try {
779792
SimpleResult result;
780793
if (bag.getPayloadManifests().size() == 0) {
@@ -795,7 +808,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
795808
bag.close();
796809
}
797810
} else if (OPERATION_MAKE_COMPLETE.equals(operation.name)) {
798-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
811+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
799812
try {
800813
Bag newBag = completer.complete(bag);
801814
try {
@@ -807,7 +820,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
807820
bag.close();
808821
}
809822
} else if (OPERATION_UPDATE.equals(operation.name)) {
810-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
823+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
811824
try {
812825
UpdateCompleter updateCompleter = new UpdateCompleter(bagFactory);
813826
updateCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm)));
@@ -825,7 +838,7 @@ else if (fetchRetryString.equalsIgnoreCase("retry")){
825838
}
826839
}
827840
else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
828-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
841+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
829842
try {
830843
TagManifestCompleter tagManifestCompleter = new TagManifestCompleter(bagFactory);
831844
tagManifestCompleter.setTagManifestAlgorithm(Algorithm.valueOfBagItAlgorithm(config.getString(PARAM_TAG_MANIFEST_ALGORITHM, Algorithm.MD5.bagItAlgorithm)));
@@ -842,8 +855,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
842855
bag.close();
843856
}
844857
} else if (OPERATION_UPDATE_PAYLOAD_OXUM.equals(operation.name)) {
845-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_FILES);
858+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_FILES);
846859
try {
860+
847861
UpdatePayloadOxumCompleter updatePayloadOxumCompleter = new UpdatePayloadOxumCompleter(bagFactory);
848862
Bag newBag = updatePayloadOxumCompleter.complete(bag);
849863
try {
@@ -860,7 +874,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
860874
bag.close();
861875
}
862876
} else if (OPERATION_BAG_IN_PLACE.equals(operation.name)) {
863-
PreBag preBag = this.bagFactory.createPreBag(sourceFile);
877+
PreBag preBag = bagFactory.createPreBag(sourceFile);
864878
if (config.contains(PARAM_BAGINFOTXT)) {
865879
File bagInfoTxtFile = config.getFile(PARAM_BAGINFOTXT);
866880
if (! bagInfoTxtFile.getName().equals(bagFactory.getBagConstants().getBagInfoTxt())) {
@@ -875,7 +889,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
875889
}
876890
preBag.makeBagInPlace(version != null ? version : BagFactory.LATEST, config.getBoolean(PARAM_RETAIN_BASE_DIR, false), config.getBoolean(PARAM_KEEP_EMPTY_DIRS, false), completer);
877891
} else if (OPERATION_CREATE.equals(operation.name)) {
878-
Bag bag = this.getBag(sourceFile, version, null);
892+
Bag bag = this.getBag(bagFactory, sourceFile, version, null);
879893
try {
880894
for(String filepath : config.getStringArray(PARAM_PAYLOAD)) {
881895
if (filepath.endsWith(File.separator + "*")) {
@@ -918,7 +932,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
918932

919933
} else if (OPERATION_MAKE_HOLEY.equals(operation.name)) {
920934
HolePuncherImpl puncher = new HolePuncherImpl(bagFactory);
921-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
935+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
922936
try {
923937
Bag newBag = puncher.makeHoley(bag, config.getString(PARAM_BASE_URL), ! config.getBoolean(PARAM_EXCLUDE_PAYLOAD_DIR, false), false, config.getBoolean(PARAM_RESUME));
924938
try {
@@ -930,7 +944,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
930944
bag.close();
931945
}
932946
} else if (OPERATION_GENERATE_PAYLOAD_OXUM.equals(operation.name)) {
933-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
947+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
934948
try {
935949
String oxum = BagHelper.generatePayloadOxum(bag);
936950
log.info("Payload-Oxum is " + oxum);
@@ -939,7 +953,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
939953
bag.close();
940954
}
941955
} else if (OPERATION_CHECK_PAYLOAD_OXUM.equals(operation.name)) {
942-
Bag bag = this.getBag(sourceFile, version, LoadOption.BY_MANIFESTS);
956+
Bag bag = this.getBag(bagFactory, sourceFile, version, LoadOption.BY_MANIFESTS);
943957
try {
944958
String genOxum = BagHelper.generatePayloadOxum(bag);
945959
BagInfoTxt bagInfo = bag.getBagInfoTxt();
@@ -971,7 +985,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
971985
}
972986
} else if (OPERATION_FILL_HOLEY.equals(operation.name)) {
973987
FileSystemFileDestination dest = new FileSystemFileDestination(sourceFile);
974-
Bag bag = this.getBag(sourceFile, version, null);
988+
Bag bag = this.getBag(bagFactory, sourceFile, version, null);
975989
try {
976990
SimpleResult result = fetcher.fetch(bag, dest, config.getBoolean(PARAM_RESUME), config.getBoolean(PARAM_VERIFY));
977991
log.info(result.toString());
@@ -996,7 +1010,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
9961010
|| OPERATION_SPLIT_BAG_BY_FILE_TYPE.equals(operation.name)
9971011
|| OPERATION_SPLIT_BAG_BY_SIZE_AND_FILE_TYPE.equals(operation.name)) {
9981012

999-
Bag srcBag = this.bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES);
1013+
Bag srcBag = bagFactory.createBag(sourceFile, BagFactory.LoadOption.BY_FILES);
10001014
try {
10011015
Double sourceBagSize = null;
10021016
if(srcBag.getBagInfoTxt() != null && srcBag.getBagInfoTxt().getPayloadOxum() != null){
@@ -1030,7 +1044,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
10301044
return RETURN_FAILURE;
10311045
}
10321046

1033-
Splitter splitter = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
1047+
Splitter splitter = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
10341048
List<Bag> splitBags = splitter.split(srcBag);
10351049
try {
10361050
this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, true);
@@ -1044,7 +1058,7 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
10441058
return RETURN_FAILURE;
10451059
}
10461060

1047-
Splitter splitter = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs);
1061+
Splitter splitter = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs);
10481062
List<Bag> splitBags = splitter.split(srcBag);
10491063
try {
10501064
this.completeAndWriteBagToDisk(splitBags, completer, writer, srcBag, destBagFile, false);
@@ -1058,9 +1072,9 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
10581072
return RETURN_FAILURE;
10591073
}
10601074

1061-
Splitter splitter1 = new SplitByFileType(this.bagFactory, fileExtensionsIn, excludeDirs);
1075+
Splitter splitter1 = new SplitByFileType(bagFactory, fileExtensionsIn, excludeDirs);
10621076
List<Bag> bags = splitter1.split(srcBag);
1063-
Splitter splitter2 = new SplitBySize(this.bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
1077+
Splitter splitter2 = new SplitBySize(bagFactory, maxBagSize, keepLowestLevelDir, excludeDirs);
10641078
try {
10651079
for(Bag bag : bags) {
10661080
List<Bag> bagsUnderMaxSize = new ArrayList<Bag>();
@@ -1104,7 +1118,8 @@ else if (OPERATION_UPDATE_TAGMANIFESTS.equals(operation.name)) {
11041118

11051119
private void completeAndWriteBagToDisk(List<Bag> bags, Completer completer, Writer writer, Bag srcBag, File destBagFile, boolean appendNumber){
11061120

1107-
int i = 0;
1121+
BagFactory bagFactory = srcBag.getBagFactory();
1122+
int i = 0;
11081123
for(Bag bag : bags) {
11091124
Bag newBag = completer.complete(bag);
11101125

‎src/main/java/gov/loc/repository/bagit/filesystem/FileSystem.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,14 @@
33
import java.io.Closeable;
44
import java.io.File;
55

6-
public interface FileSystem extends Closeable {
6+
public interface FileSystem extends Closeable {
77
DirNode getRoot();
88
/*
99
* The file that represents the file system.
1010
* This may be a file or directory depending on the type of file system.
1111
*/
1212
File getFile();
1313
FileNode resolve(String filepath);
14+
FileSystemNodeFilter getDefaultNodeFilter();
1415
void closeQuietly();
1516
}

‎src/main/java/gov/loc/repository/bagit/filesystem/FileSystemFactory.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package gov.loc.repository.bagit.filesystem;
22

33
import gov.loc.repository.bagit.Bag.Format;
4+
import gov.loc.repository.bagit.BagFactory;
45
import gov.loc.repository.bagit.filesystem.impl.FileFileSystem;
56
import gov.loc.repository.bagit.filesystem.impl.ZipFileSystem;
67
import gov.loc.repository.bagit.utilities.FormatHelper;
@@ -11,7 +12,7 @@
1112

1213
public class FileSystemFactory {
1314

14-
public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatException, UnsupportedFormatException {
15+
public static DirNode getDirNodeForBag(File fileForBag, BagFactory bagFactory) throws UnknownFormatException, UnsupportedFormatException {
1516
assert fileForBag != null;
1617

1718
if (! fileForBag.exists()) {
@@ -21,7 +22,7 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce
2122
Format format = FormatHelper.getFormat(fileForBag);
2223
FileSystem fs = null;
2324
if (Format.FILESYSTEM == format) {
24-
fs = new FileFileSystem(fileForBag);
25+
fs = new FileFileSystem(fileForBag, bagFactory.getDefaultNodeFilter());
2526
} else if (Format.ZIP == format) {
2627
fs = new ZipFileSystem(fileForBag);
2728
} else {
@@ -30,11 +31,11 @@ public static DirNode getDirNodeForBag(File fileForBag) throws UnknownFormatExce
3031

3132
DirNode root = fs.getRoot();
3233
if (format.isSerialized) {
33-
if (root.listChildren().size() != 1) {
34+
if (root.listChildren(fs.getDefaultNodeFilter()).size() != 1) {
3435
root.getFileSystem().closeQuietly();
3536
throw new RuntimeException("Unable to find bag_dir in serialized bag");
3637
}
37-
FileSystemNode bagDirNode = root.listChildren().iterator().next();
38+
FileSystemNode bagDirNode = root.listChildren(fs.getDefaultNodeFilter()).iterator().next();
3839
if (! (bagDirNode instanceof DirNode)) {
3940
root.getFileSystem().closeQuietly();
4041
throw new RuntimeException("Unable to find bag_dir in serialized bag");

‎src/main/java/gov/loc/repository/bagit/filesystem/filter/DirNodeFileSystemNodeFilter.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,17 @@
33
import gov.loc.repository.bagit.filesystem.DirNode;
44
import gov.loc.repository.bagit.filesystem.FileSystemNode;
55
import gov.loc.repository.bagit.filesystem.FileSystemNodeFilter;
6+
import gov.loc.repository.bagit.filesystem.impl.AbstractFileNode;
67

78
public class DirNodeFileSystemNodeFilter implements FileSystemNodeFilter {
8-
99
@Override
1010
public boolean accept(FileSystemNode fileSystemNode) {
11+
if (fileSystemNode.getFileSystem().getDefaultNodeFilter() != null &&
12+
fileSystemNode instanceof AbstractFileNode &&
13+
!fileSystemNode.getFileSystem().getDefaultNodeFilter().accept(fileSystemNode)) {
14+
// excluded by the default filter
15+
return false;
16+
}
1117
return fileSystemNode instanceof DirNode;
1218
}
13-
1419
}

0 commit comments

Comments
 (0)