feat: Change file metadata scanning to store and access metadata through an interface

- Extract generic implementation from FileSetRepository
- Implement FileMetadataSetRepository for storing FileMetadata sets
- Make controllers single-use
- Implement ArchivedFileMetadataSetRepository for storing ArchivedFileMetadata sets
- Implement repository for change status checking
- Integrate the new repositories with the backup and restore controllers
- Update tests

Resolves #752
{minor}

Signed-off-by: Esta Nagy <nagyesta@gmail.com>

Author: nagyesta

.idea/codeStyles/Project.xml

@@ -59,6 +59,7 @@ val copyLegalDocs = tasks.register<Copy>("copyLegalDocs") {
     rename("artifacts.json", "dependency-licenses.json")
     rename("bom.json", "SBOM.json")
 }.get()
+tasks.cyclonedxDirectBom.get().finalizedBy(":file-barj-stream-io:jar")
 copyLegalDocs.dependsOn(tasks.licensee)
 copyLegalDocs.dependsOn(tasks.cyclonedxDirectBom)
 tasks.javadoc.get().dependsOn(copyLegalDocs)

.idea/inspectionProfiles/Project_Default.xml

@@ -33,8 +33,9 @@ public class ParallelBackupPipeline extends BaseBackupPipeline<ParallelBarjCargo
      * @param threadCount The number of threads
      * @throws IOException When the stream cannot be created due to an I/O error
      */
-    public ParallelBackupPipeline(final @NotNull BackupIncrementManifest manifest,
-                                  final int threadCount) throws IOException {
+    public ParallelBackupPipeline(
+            final @NotNull BackupIncrementManifest manifest,
+            final int threadCount) throws IOException {
         super(manifest, convert(manifest, threadCount));
     }
 

file-barj-core/build.gradle.kts

@@ -38,10 +38,11 @@ public class PosixFileMetadataParser implements FileMetadataParser {
     public @NotNull FileMetadata parse(
             final @NonNull File file,
             final @NonNull BackupJobConfiguration configuration) {
+        final var absolutePath = BackupPath.of(file.toPath().toAbsolutePath());
         if (!Files.exists(file.toPath(), LinkOption.NOFOLLOW_LINKS)) {
             return FileMetadata.builder()
                     .id(UUID.randomUUID())
-                    .absolutePath(BackupPath.of(file.toPath().toAbsolutePath()))
+                    .absolutePath(absolutePath)
                     .fileType(FileType.MISSING)
                     .status(Change.DELETED)
                     .build();
@@ -52,7 +53,7 @@ public class PosixFileMetadataParser implements FileMetadataParser {
         return FileMetadata.builder()
                 .id(UUID.randomUUID())
                 .fileSystemKey(Optional.ofNullable(basicAttributes.fileKey()).map(String::valueOf).orElse(null))
-                .absolutePath(BackupPath.of(file.toPath().toAbsolutePath()))
+                .absolutePath(absolutePath)
                 .owner(posixFileAttributes.owner())
                 .group(posixFileAttributes.group())
                 .posixPermissions(posixFileAttributes.permissions())

file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/backup/pipeline/BackupController.java

@@ -3,8 +3,8 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.nagyesta.filebarj.core.config.BackupSource;
 import com.github.nagyesta.filebarj.core.model.BackupPath;
-import com.github.nagyesta.filebarj.core.persistence.FileSetRepository;
-import com.github.nagyesta.filebarj.core.persistence.entities.FileSetId;
+import com.github.nagyesta.filebarj.core.persistence.FilePathSetRepository;
+import com.github.nagyesta.filebarj.core.persistence.entities.FilePathSetId;
 import lombok.NonNull;
 import org.apache.commons.io.FilenameUtils;
 import org.jetbrains.annotations.NotNull;
@@ -32,17 +32,17 @@ public class BackupSourceScanner {
      */
     private static final Set<String> EXCLUDE_NO_FILES = Set.of();
 
-    private final FileSetRepository fileSetRepository;
+    private final FilePathSetRepository filePathSetRepository;
     private final BackupSource backupSource;
     private final Path backupSourceOsPath;
     private final SortedSet<String> includePatterns;
     private final SortedSet<String> excludePatterns;
     private final boolean shouldUsePatterns;
 
     public BackupSourceScanner(
-            final @NonNull FileSetRepository fileSetRepository,
+            final @NonNull FilePathSetRepository filePathSetRepository,
             final @NonNull BackupSource backupSource) {
-        this.fileSetRepository = fileSetRepository;
+        this.filePathSetRepository = filePathSetRepository;
         this.backupSource = backupSource;
         this.backupSourceOsPath = backupSource.getPath().toOsPath();
         final var originalIncludes = backupSource.getIncludePatterns();
@@ -55,11 +55,11 @@ public BackupSourceScanner(
     /**
      * Lists the matching {@link Path} entries.
      *
-     * @param resultFileSetId the Id of the file se where we should collect the results
+     * @param resultFilePathSetId the Id of the file se where we should collect the results
      */
     @JsonIgnore
-    public void listMatchingFilePaths(final @NonNull FileSetId resultFileSetId) {
-        try (var tempFileSetId = fileSetRepository.createFileSet()) {
+    public void listMatchingFilePaths(final @NonNull FilePathSetId resultFilePathSetId) {
+        try (var tempFileSetId = filePathSetRepository.createFileSet()) {
             var current = Optional.of(backupSourceOsPath);
             var parentDirsFromBackupSource = List.<Path>of();
             Path lastParent = null;
@@ -69,8 +69,8 @@ public void listMatchingFilePaths(final @NonNull FileSetId resultFileSetId) {
                     lastParent = currentPath.getParent();
                     parentDirsFromBackupSource = findParentsFromBackupSource(currentPath);
                 }
-                listRemainingFiles(currentPath, parentDirsFromBackupSource, resultFileSetId, tempFileSetId);
-                current = fileSetRepository.takeFirst(tempFileSetId);
+                listRemainingFiles(currentPath, parentDirsFromBackupSource, resultFilePathSetId, tempFileSetId);
+                current = filePathSetRepository.takeFirst(tempFileSetId);
             }
         }
     }
@@ -88,8 +88,8 @@ private List<Path> findParentsFromBackupSource(final @NotNull Path path) {
     private void listRemainingFiles(
             final @NotNull Path currentPath,
             final @NotNull List<Path> parentDirsFromBackupSource,
-            final @NotNull FileSetId sourceFileSetIt,
-            final @NotNull FileSetId tempFileSetId) {
+            final @NotNull FilePathSetId sourceFileSetIt,
+            final @NotNull FilePathSetId tempFilePathSetId) {
         if (!currentPath.toFile().exists()) {
             return;
         }
@@ -101,21 +101,21 @@ private void listRemainingFiles(
                 return;
             }
             //add all parents if they are not saved yet
-            fileSetRepository.appendTo(sourceFileSetIt, parentDirsFromBackupSource);
-            fileSetRepository.appendTo(sourceFileSetIt, currentPath);
+            filePathSetRepository.appendTo(sourceFileSetIt, parentDirsFromBackupSource);
+            filePathSetRepository.appendTo(sourceFileSetIt, currentPath);
         } else {
             if (isCurrentDirectoryInSourceSetByPatterns(currentPath)) {
                 //always add directories if the include patterns match and the exclude patterns do not match
-                fileSetRepository.appendTo(sourceFileSetIt, parentDirsFromBackupSource);
-                fileSetRepository.appendTo(sourceFileSetIt, currentPath);
+                filePathSetRepository.appendTo(sourceFileSetIt, parentDirsFromBackupSource);
+                filePathSetRepository.appendTo(sourceFileSetIt, currentPath);
             }
             if (hasChildren(currentPath)) {
                 Optional.ofNullable(currentPath.toFile().listFiles())
                         .stream()
                         .flatMap(Arrays::stream)
                         .map(File::toPath)
                         .filter(child -> !shouldIgnoreByExcludePattern(child))
-                        .forEach((Path child) -> fileSetRepository.appendTo(tempFileSetId, child));
+                        .forEach((Path child) -> filePathSetRepository.appendTo(tempFilePathSetId, child));
             }
         }
     }

file-barj-core/src/main/java/com/github/nagyesta/filebarj/core/backup/pipeline/ParallelBackupPipeline.java

@@ -4,7 +4,10 @@
 import com.github.nagyesta.filebarj.core.model.FileMetadata;
 import com.github.nagyesta.filebarj.core.model.enums.Change;
 import com.github.nagyesta.filebarj.core.model.enums.FileType;
+import com.github.nagyesta.filebarj.core.persistence.FileMetadataSetRepository;
+import com.github.nagyesta.filebarj.core.persistence.entities.FileMetadataSetId;
 import lombok.NonNull;
+import org.apache.commons.lang3.function.TriFunction;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -17,26 +20,23 @@
  */
 public abstract class BaseFileMetadataChangeDetector<T> implements FileMetadataChangeDetector {
 
-    private final SortedMap<String, Map<UUID, FileMetadata>> filesFromManifests;
-    private final SortedMap<String, Map<T, List<FileMetadata>>> contentIndex;
-    private final Map<String, FileMetadata> nameIndex;
+    private final FileMetadataSetRepository repository;
+    private final SortedMap<String, FileMetadataSetId> filesFromManifests;
     private final PermissionComparisonStrategy permissionComparisonStrategy;
 
     /**
      * Creates a new instance with the previous manifests.
      *
+     * @param repository         The repository we can use to access the metadata
      * @param filesFromManifests The files found in the previous manifests
      * @param permissionStrategy The permission comparison strategy
      */
     protected BaseFileMetadataChangeDetector(
-            final @NotNull Map<String, Map<UUID, FileMetadata>> filesFromManifests,
+            final @NotNull FileMetadataSetRepository repository,
+            final @NotNull Map<String, FileMetadataSetId> filesFromManifests,
             final @Nullable PermissionComparisonStrategy permissionStrategy) {
+        this.repository = repository;
         this.filesFromManifests = new TreeMap<>(filesFromManifests);
-        final SortedMap<String, Map<T, List<FileMetadata>>> contentIndexSet = new TreeMap<>();
-        final Map<String, FileMetadata> nameIndexMap = new TreeMap<>();
-        index(this.filesFromManifests, contentIndexSet, nameIndexMap);
-        this.contentIndex = contentIndexSet;
-        this.nameIndex = nameIndexMap;
         this.permissionComparisonStrategy = Objects.requireNonNullElse(permissionStrategy, PermissionComparisonStrategy.STRICT);
     }
 
@@ -54,23 +54,24 @@ public boolean hasMetadataChanged(
     @Override
     public boolean isFromLastIncrement(
             final @NonNull FileMetadata fileMetadata) {
-        return filesFromManifests.get(filesFromManifests.lastKey()).containsKey(fileMetadata.getId());
+        return repository.containsFileId(filesFromManifests.get(filesFromManifests.lastKey()), fileMetadata.getId());
     }
 
     @Override
     public @Nullable FileMetadata findMostRelevantPreviousVersion(
             final @NonNull FileMetadata currentMetadata) {
         final var increments = filesFromManifests.keySet().stream().sorted(Comparator.reverseOrder()).toList();
-        final var previousSamePath = nameIndex.getOrDefault(currentMetadata.getAbsolutePath().toString(), null);
+        final var previousSamePath = findPreviousVersionByAbsolutePath(currentMetadata.getAbsolutePath());
         if (previousSamePath != null && !hasContentChanged(previousSamePath, currentMetadata)) {
             return previousSamePath;
         }
         for (final var increment : increments) {
-            final var index = contentIndex.get(increment);
+            final var index = filesFromManifests.get(increment);
             final var key = getPrimaryContentCriteria(currentMetadata);
-            if (index.containsKey(key)) {
+            final var files = findFilesByPrimaryContentCriteria().apply(repository, index, key);
+            if (!files.isEmpty()) {
                 final var byPath = new TreeMap<BackupPath, FileMetadata>();
-                index.get(key).stream()
+                files.stream()
                         .filter(metadata -> !hasContentChanged(metadata, currentMetadata))
                         .forEach(metadata -> byPath.put(metadata.getAbsolutePath(), metadata));
                 if (!byPath.isEmpty()) {
@@ -84,7 +85,17 @@ public boolean isFromLastIncrement(
     @Override
     public @Nullable FileMetadata findPreviousVersionByAbsolutePath(
             final @NonNull BackupPath absolutePath) {
-        return nameIndex.get(absolutePath.toString());
+        return filesFromManifests.keySet()
+                .stream().sorted(Comparator.reverseOrder())
+                .map(increment -> {
+                    final var fileMetadataSetId = filesFromManifests.get(increment);
+                    return repository.findFileByPath(fileMetadataSetId, absolutePath)
+                            .filter(metadata -> metadata.getStatus() != Change.DELETED);
+                })
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .findFirst()
+                .orElse(null);
     }
 
     @Override
@@ -114,22 +125,11 @@ public boolean isFromLastIncrement(
      */
     protected abstract T getPrimaryContentCriteria(@NotNull FileMetadata metadata);
 
-    private void index(
-            final @NotNull SortedMap<String, Map<UUID, FileMetadata>> filesFromManifests,
-            final @NotNull SortedMap<String, Map<T, List<FileMetadata>>> contentIndexMap,
-            final @NotNull Map<String, FileMetadata> nameIndexMap) {
-        filesFromManifests.forEach((increment, files) -> files
-                .forEach((uuid, metadata) -> contentIndexMap
-                        .computeIfAbsent(increment, k -> new HashMap<>())
-                        .computeIfAbsent(getPrimaryContentCriteria(metadata), k -> new ArrayList<>())
-                        .add(metadata)));
-        //populate files in reverse manifest order to ensure each file has the latest metadata saved
-        filesFromManifests.keySet().stream()
-                .sorted(Comparator.reverseOrder())
-                .map(filesFromManifests::get)
-                .forEachOrdered(files -> files.entrySet().stream()
-                        .filter(entry -> entry.getValue().getStatus() != Change.DELETED)
-                        //put the file only if it is not already in the index
-                        .forEach(entry -> nameIndexMap.putIfAbsent(entry.getValue().getAbsolutePath().toString(), entry.getValue())));
-    }
+    /**
+     * Returns a function that can find the files from the repository by matching based on the primary content criteria.
+     *
+     * @return filter function
+     */
+    protected abstract
+    TriFunction<FileMetadataSetRepository, FileMetadataSetId, T, Set<FileMetadata>> findFilesByPrimaryContentCriteria();
 }