From b58234b450994b3ec6371db04427d137f1ab30f3 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 14 Oct 2025 13:34:10 +0200 Subject: [PATCH 01/43] feat: bump nugets --- src/Directory.Packages.props | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 549456d3..e1288e8b 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -11,17 +11,17 @@ https://github.com/woutervanranst/Arius - + - + - + @@ -36,16 +36,16 @@ - + - + - + From 36484dad8707dbe6d3cb794d79cb86546065a604 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Tue, 14 Oct 2025 20:02:47 +0200 Subject: [PATCH 02/43] Expand archive handler tests --- .../ArchiveCommandHandlerHandleTests.cs | 1174 +++++++++++++++++ .../Helpers/Fakes/FakeArchiveStorage.cs | 191 +++ 2 files changed, 1365 insertions(+) create mode 100644 src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs create mode 100644 src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs new file mode 100644 index 00000000..761c821b --- /dev/null +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -0,0 +1,1174 @@ +using Arius.Core.Features.Commands.Archive; +using Arius.Core.Shared.FileSystem; +using Arius.Core.Shared.Hashing; +using Arius.Core.Shared.StateRepositories; +using Arius.Core.Shared.Storage; +using Arius.Core.Tests.Helpers.Builders; +using Arius.Core.Tests.Helpers.Fakes; +using Arius.Core.Tests.Helpers.Fixtures; +using Arius.Core.Tests.Helpers.FakeLogger; +using FluentResults; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Testing; +using Shouldly; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Zio; + +namespace Arius.Core.Tests.Features.Commands.Archive; + +public class ArchiveCommandHandlerHandleTests : IClassFixture +{ + private readonly FixtureWithFileSystem fixture; + private readonly FakeLogger logger; + private readonly ArchiveCommandHandler handler; + + public ArchiveCommandHandlerHandleTests(FixtureWithFileSystem fixture) + { + this.fixture = fixture; + logger = new FakeLogger(); + handler = new ArchiveCommandHandler(logger, NullLoggerFactory.Instance, fixture.AriusConfiguration); + } + + private const int DefaultSmallFileBoundary = 1024; + + private static string ToRelativePointerPath(UPath binaryPath) => binaryPath.GetPointerFilePath().ToString(); + + private static string ToAbsolutePointerPath(FixtureWithFileSystem fixture, UPath binaryPath) => + Path.Combine(fixture.TestRunSourceFolder.FullName, binaryPath.GetPointerFilePath().ToString().TrimStart('/')); + + private static void ResetTestRoot(FixtureWithFileSystem fixture, string containerName) + { + foreach (var directory in Directory.EnumerateDirectories(fixture.TestRunSourceFolder.FullName)) + Directory.Delete(directory, recursive: true); + + foreach (var file in Directory.EnumerateFiles(fixture.TestRunSourceFolder.FullName)) + File.Delete(file); + + var stateCacheRoot = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "Arius", + "statecache", + fixture.RepositoryOptions?.AccountName ?? "testaccount", + containerName); + + if (Directory.Exists(stateCacheRoot)) + Directory.Delete(stateCacheRoot, recursive: true); + } + + private static async Task BuildHandlerContextAsync( + ArchiveCommand command, + FakeArchiveStorage archiveStorage, + FakeLoggerFactory loggerFactory) => + await new HandlerContextBuilder(command, loggerFactory) + .WithArchiveStorage(archiveStorage) + .BuildAsync(); + + private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistics(HandlerContext handlerContext) + { + var entries = handlerContext.FileSystem + .EnumerateFileEntries(UPath.Root, "*", SearchOption.AllDirectories) + .Select(FilePair.FromBinaryFileFileEntry) + .ToList(); + + var existingPointerFileCount = entries.Count(fp => fp.PointerFile.Exists); + + return (entries.Count, existingPointerFileCount); + } + + private async Task<(ArchiveCommand Command, HandlerContext Context, FakeArchiveStorage Storage, FakeLoggerFactory LoggerFactory)> + CreateHandlerContextAsync( + string containerName, + Action? configureCommand = null, + FakeArchiveStorage? archiveStorage = null, + FakeLoggerFactory? loggerFactory = null) + { + archiveStorage ??= new FakeArchiveStorage(); + loggerFactory ??= new FakeLoggerFactory(); + + var commandBuilder = new ArchiveCommandBuilder(fixture) + .WithContainerName(containerName) + .WithSmallFileBoundary(DefaultSmallFileBoundary) + .WithHashingParallelism(1) + .WithUploadParallelism(1); + + configureCommand?.Invoke(commandBuilder); + + var command = commandBuilder.Build(); + var handlerContext = await BuildHandlerContextAsync(command, archiveStorage, loggerFactory); + + return (command, handlerContext, archiveStorage, loggerFactory); + } + + [Fact] + public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() + { + // Arrange + var containerName = $"test-container-large-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var binaryPath = UPath.Root / "documents" / "presentation.pptx"; + var largeFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) + .WithRandomContent(4096, seed: 10) + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(1); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.ExistingPointerFiles.ShouldBe(expectedExistingPointerFile); + summary.BytesUploadedUncompressed.ShouldBe(largeFile.OriginalContent.LongLength); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + var chunk = archiveStorage.StoredChunks.Single().Value; + chunk.ContentType.ShouldBe("application/aes256cbc+gzip"); + chunk.Metadata.ShouldContainKey("OriginalContentLength"); + chunk.Metadata["OriginalContentLength"].ShouldBe(largeFile.OriginalContent.Length.ToString()); + + File.Exists(ToAbsolutePointerPath(fixture, binaryPath)).ShouldBeTrue(); + + var pointerEntry = handlerContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.Hash.ShouldBe(largeFile.OriginalHash); + pointerEntry.BinaryProperties.ShouldNotBeNull(); + pointerEntry.BinaryProperties.OriginalSize.ShouldBe(largeFile.OriginalContent.LongLength); + pointerEntry.BinaryProperties.ArchivedSize.ShouldBe(chunk.ContentLength); + pointerEntry.BinaryProperties.ParentHash.ShouldBeNull(); + + var binaryProperties = handlerContext.StateRepository.GetBinaryProperty(pointerEntry.Hash); + binaryProperties.ShouldNotBeNull(); + binaryProperties!.ArchivedSize.ShouldBe(chunk.ContentLength); + } + + [Fact] + public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBinaryProperties() + { + // Arrange + var containerName = $"test-container-small-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var binaryPath = UPath.Root / "notes" / "small.txt"; + var smallFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) + .WithRandomContent(512, seed: 2) + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(1); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.ExistingPointerFiles.ShouldBe(expectedExistingPointerFile); + summary.BytesUploadedUncompressed.ShouldBe(smallFile.OriginalContent.LongLength); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + var tarChunk = archiveStorage.StoredChunks.Single().Value; + tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); + tarChunk.Metadata.ShouldContainKey("OriginalContentLength"); + tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); + tarChunk.Metadata["OriginalContentLength"].ShouldBe(tarChunk.ContentLength.ToString()); + tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); + + File.Exists(ToAbsolutePointerPath(fixture, binaryPath)).ShouldBeTrue(); + + var pointerEntry = handlerContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.BinaryProperties.ShouldNotBeNull(); + pointerEntry.BinaryProperties.Hash.ShouldBe(smallFile.OriginalHash); + pointerEntry.BinaryProperties.ParentHash.ShouldNotBeNull(); + pointerEntry.BinaryProperties.OriginalSize.ShouldBe(smallFile.OriginalContent.LongLength); + pointerEntry.BinaryProperties.ArchivedSize.ShouldBeGreaterThan(0L); + + var parentHash = pointerEntry.BinaryProperties.ParentHash!; + var parentProperties = handlerContext.StateRepository.GetBinaryProperty(parentHash); + parentProperties.ShouldNotBeNull(); + parentProperties!.OriginalSize.ShouldBe(smallFile.OriginalContent.LongLength); + parentProperties.ArchivedSize.ShouldBe(tarChunk.ContentLength); + tarChunk.Metadata["OriginalContentLength"].ShouldBe(parentProperties.ArchivedSize.ToString()); + + handlerContext.StateRepository.GetBinaryProperty(pointerEntry.Hash).ShouldNotBeNull(); + } + + [Fact] + public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() + { + // Arrange + var containerName = $"test-container-empty-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var binaryPath = UPath.Root / "empty" / "file.bin"; + var emptyFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(1); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.ExistingPointerFiles.ShouldBe(expectedExistingPointerFile); + summary.BytesUploadedUncompressed.ShouldBe(0); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + var tarChunk = archiveStorage.StoredChunks.Single().Value; + tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); + tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); + tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); + + File.Exists(ToAbsolutePointerPath(fixture, binaryPath)).ShouldBeTrue(); + + var pointerEntry = handlerContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.BinaryProperties.ShouldNotBeNull(); + pointerEntry.BinaryProperties.OriginalSize.ShouldBe(0); + pointerEntry.BinaryProperties.ArchivedSize.ShouldBeGreaterThanOrEqualTo(0L); + + var parentHash = pointerEntry.BinaryProperties.ParentHash!; + handlerContext.StateRepository.GetBinaryProperty(parentHash).ShouldNotBeNull(); + } + + [Fact] + public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrackExistingCount() + { + // Arrange + var containerName = $"test-container-existing-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var binaryPath = UPath.Root / "existing" / "document.pdf"; + var binaryWithPointer = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) + .WithRandomContent(2048, seed: 5) + .Build(); + + var staleHash = FakeHashBuilder.GenerateValidHash(42); + staleHash.ShouldNotBe(binaryWithPointer.OriginalHash); + var stalePointer = binaryWithPointer.FilePair.CreatePointerFile(staleHash); + stalePointer.ReadHash().ShouldBe(staleHash); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(0); + summary.ExistingPointerFiles.ShouldBe(expectedExistingPointerFile); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.BytesUploadedUncompressed.ShouldBe(binaryWithPointer.OriginalContent.LongLength); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + + var pointerPath = ToAbsolutePointerPath(fixture, binaryPath); + File.Exists(pointerPath).ShouldBeTrue(); + + var updatedHash = binaryWithPointer.FilePair.PointerFile.ReadHash(); + updatedHash.ShouldBe(binaryWithPointer.OriginalHash); + + var pointerEntry = handlerContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.Hash.ShouldBe(binaryWithPointer.OriginalHash); + + var binaryProperties = handlerContext.StateRepository.GetBinaryProperty(pointerEntry.Hash); + binaryProperties.ShouldNotBeNull(); + binaryProperties!.ArchivedSize.ShouldBeGreaterThan(0L); + } + + [Fact] + public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches() + { + // Arrange + var containerName = $"test-container-mixed-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var smallFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "small.txt") + .WithRandomContent(512, seed: 1) + .Build(); + + var largeFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "large.bin") + .WithRandomContent(4096, seed: 2) + .Build(); + + var progressUpdates = new List(); + var progressReporter = new Progress(progressUpdates.Add); + + var (command, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + containerName, + builder => builder + .WithProgressReporter(progressReporter) + .WithHashingParallelism(1) + .WithUploadParallelism(1) + .WithSmallFileBoundary(DefaultSmallFileBoundary)); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(2); + summary.UniqueChunksUploaded.ShouldBe(2); + summary.PointerFilesCreated.ShouldBe(2); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.ExistingPointerFiles.ShouldBe(0); + summary.BytesUploadedUncompressed.ShouldBe(smallFile.OriginalContent.Length + largeFile.OriginalContent.Length); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(2); + archiveStorage.UploadedStates.ShouldContain(summary.NewStateName!); + + var tarChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); + tarChunk.Metadata.ShouldContainKey("OriginalContentLength"); + tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); + tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); + + var largeChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); + largeChunk.Metadata.ShouldContainKey("OriginalContentLength"); + largeChunk.Metadata["OriginalContentLength"].ShouldBe(largeFile.OriginalContent.Length.ToString()); + + var smallPointerPath = Path.Combine(fixture.TestRunSourceFolder.FullName, "small.txt.pointer.arius"); + File.Exists(smallPointerPath).ShouldBeTrue(); + + var largePointerPath = Path.Combine(fixture.TestRunSourceFolder.FullName, "large.bin.pointer.arius"); + File.Exists(largePointerPath).ShouldBeTrue(); + + handlerContext.StateRepository.GetPointerFileEntry("/small.txt.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry("/large.bin.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + } + + [Fact] + public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCreateMultiplePointers() + { + // Arrange + var containerName = $"test-container-duplicates-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var originalLargeFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "shared.bin") + .WithRandomContent(4096, seed: 42) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithDuplicate(originalLargeFile, UPath.Root / "duplicates" / "shared-copy.bin") + .Build(); + + var originalSmallFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "texts" / "note.txt") + .WithRandomContent(320, seed: 7) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithDuplicate(originalSmallFile, UPath.Root / "texts" / "archive" / "note-copy.txt") + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(2); // large owner + small owner + summary.UniqueChunksUploaded.ShouldBe(2); // large chunk + TAR + summary.PointerFilesCreated.ShouldBe(4); + summary.BytesUploadedUncompressed.ShouldBe( + originalLargeFile.OriginalContent.Length + originalSmallFile.OriginalContent.Length); + + archiveStorage.StoredChunks.Count.ShouldBe(2); + archiveStorage.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+gzip").ShouldBe(1); + archiveStorage.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+tar+gzip").ShouldBe(1); + + var largeChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); + largeChunk.Metadata.ShouldContainKey("OriginalContentLength"); + largeChunk.Metadata["OriginalContentLength"].ShouldBe(originalLargeFile.OriginalContent.Length.ToString()); + + var tarChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); + tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); + tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); + + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "shared.bin.pointer.arius")).ShouldBeTrue(); + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "duplicates", "shared-copy.bin.pointer.arius")).ShouldBeTrue(); + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "texts", "note.txt.pointer.arius")).ShouldBeTrue(); + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "texts", "archive", "note-copy.txt.pointer.arius")).ShouldBeTrue(); + + handlerContext.StateRepository.GetPointerFileEntry("/shared.bin.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry("/duplicates/shared-copy.bin.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry("/texts/note.txt.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry("/texts/archive/note-copy.txt.pointer.arius", includeBinaryProperties: true) + .ShouldNotBeNull(); + } + + [Fact] + public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChunk() + { + // Arrange + var containerName = $"test-container-small-batch-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var paths = new[] + { + UPath.Root / "tar" / "alpha.txt", + UPath.Root / "tar" / "beta.txt", + UPath.Root / "tar" / "gamma.txt" + }; + + var smallFiles = paths + .Select((path, index) => new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, path) + .WithRandomContent(256 + index * 10, seed: 100 + index) + .Build()) + .ToArray(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(paths.Length); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(paths.Length); + summary.BytesUploadedUncompressed.ShouldBe(smallFiles.Sum(f => f.OriginalContent.Length)); + summary.PointerFileEntriesDeleted.ShouldBe(0); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + var tarChunk = archiveStorage.StoredChunks.Single().Value; + tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); + tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); + tarChunk.Metadata["SmallChunkCount"].ShouldBe(paths.Length.ToString()); + + foreach (var path in paths) + { + var pointerPath = ToAbsolutePointerPath(fixture, path); + File.Exists(pointerPath).ShouldBeTrue(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(path), includeBinaryProperties: true) + .ShouldNotBeNull(); + } + } + + [Fact] + public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundaryExceeded() + { + // Arrange + var containerName = $"test-container-multi-tar-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var paths = new[] + { + UPath.Root / "tar" / "alpha.bin", + UPath.Root / "tar" / "beta.bin", + UPath.Root / "tar" / "gamma.bin" + }; + + var sizes = new[] { 900, 900, 500 }; + + var smallFiles = paths + .Select((path, index) => new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, path) + .WithRandomContent(sizes[index], seed: 200 + index) + .Build()) + .ToArray(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(paths.Length); + summary.UniqueChunksUploaded.ShouldBe(2); + summary.PointerFilesCreated.ShouldBe(paths.Length); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.BytesUploadedUncompressed.ShouldBe(smallFiles.Sum(f => f.OriginalContent.Length)); + + var tarChunks = archiveStorage.StoredChunks.Values + .Where(c => c.ContentType == "application/aes256cbc+tar+gzip") + .ToList(); + tarChunks.Count.ShouldBe(2); + tarChunks.Select(c => int.Parse(c.Metadata["SmallChunkCount"])) + .OrderBy(v => v) + .ShouldBe(new[] { 1, 2 }); + + foreach (var path in paths) + { + File.Exists(ToAbsolutePointerPath(fixture, path)).ShouldBeTrue(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(path), includeBinaryProperties: true) + .ShouldNotBeNull(); + } + } + + [Fact] + public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWriteDeferredPointers() + { + // Arrange + var containerName = $"test-container-small-duplicates-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var ownerAlpha = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "alpha-owner.txt") + .WithRandomContent(900, seed: 11) + .Build(); + + var ownerBeta = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "beta-owner.txt") + .WithRandomContent(920, seed: 12) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithDuplicate(ownerAlpha, UPath.Root / "tar" / "gamma-duplicate.txt") + .Build(); + + var ownerOmega = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "omega-owner.txt") + .WithRandomContent(480, seed: 13) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithDuplicate(ownerOmega, UPath.Root / "tar" / "zzz-duplicate.txt") + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.UniqueBinariesUploaded.ShouldBe(3); + summary.UniqueChunksUploaded.ShouldBe(2); + summary.PointerFilesCreated.ShouldBe(5); + summary.PointerFileEntriesDeleted.ShouldBe(0); + + var tarChunks = archiveStorage.StoredChunks.Values + .Where(c => c.ContentType == "application/aes256cbc+tar+gzip") + .ToList(); + tarChunks.Count.ShouldBe(2); + tarChunks.Select(c => int.Parse(c.Metadata["SmallChunkCount"])) + .OrderBy(v => v) + .ShouldBe(new[] { 1, 2 }); + + var duplicatePaths = new[] + { + UPath.Root / "tar" / "gamma-duplicate.txt", + UPath.Root / "tar" / "zzz-duplicate.txt" + }; + + foreach (var path in duplicatePaths) + { + File.Exists(ToAbsolutePointerPath(fixture, path)).ShouldBeTrue(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(path), includeBinaryProperties: true) + .ShouldNotBeNull(); + } + } + + [Fact] + public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() + { + // Arrange + var containerName = $"test-container-incremental-existing-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var binaryPath = UPath.Root / "incremental" / "presentation.pptx"; + var largeFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) + .WithRandomContent(4096, seed: 501) + .Build(); + + var archiveStorage = new FakeArchiveStorage(); + + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (initialFileCount, _) = GetInitialFileStatistics(initialContext); + + var firstResult = await handler.Handle(initialContext, CancellationToken.None); + firstResult.IsSuccess.ShouldBeTrue(); + + var originalHash = largeFile.OriginalHash; + + // Corrupt pointer file to ensure it is rewritten on incremental run + var staleHash = FakeHashBuilder.GenerateValidHash(999); + staleHash.ShouldNotBe(originalHash); + largeFile.FilePair.CreatePointerFile(staleHash); + + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + + // Act + var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + + // Assert + incrementalResult.IsSuccess.ShouldBeTrue(); + var summary = incrementalResult.Value; + + summary.TotalLocalFiles.ShouldBe(expectedFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(0); + summary.UniqueChunksUploaded.ShouldBe(0); + summary.PointerFilesCreated.ShouldBe(0); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.BytesUploadedUncompressed.ShouldBe(0); + summary.NewStateName.ShouldBeNull(); + + largeFile.FilePair.PointerFile.ReadHash().ShouldBe(originalHash); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + archiveStorage.UploadedStates.Count.ShouldBe(1); + + var pointerEntry = incrementalContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.Hash.ShouldBe(originalHash); + } + + [Fact] + public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() + { + // Arrange + var containerName = $"test-container-incremental-mixed-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var existingFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "existing.pdf") + .WithRandomContent(4096, seed: 2001) + .Build(); + + var archiveStorage = new FakeArchiveStorage(); + + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var firstResult = await handler.Handle(initialContext, CancellationToken.None); + firstResult.IsSuccess.ShouldBeTrue(); + + var newSmallFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "new-note.txt") + .WithRandomContent(512, seed: 2002) + .Build(); + + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + + // Act + var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + + // Assert + incrementalResult.IsSuccess.ShouldBeTrue(); + var summary = incrementalResult.Value; + + summary.TotalLocalFiles.ShouldBe(expectedFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(1); + summary.BytesUploadedUncompressed.ShouldBe(newSmallFile.OriginalContent.Length); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.NewStateName.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Values.Count.ShouldBe(2); + + var pointerEntry = incrementalContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(newSmallFile.OriginalPath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.BinaryProperties.ShouldNotBeNull(); + pointerEntry.BinaryProperties!.OriginalSize.ShouldBe(newSmallFile.OriginalContent.Length); + } + + [Fact] + public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry() + { + // Arrange + var containerName = $"test-container-incremental-deleted-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var deletedFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "to-delete.txt") + .WithRandomContent(2048, seed: 3001) + .Build(); + + var archiveStorage = new FakeArchiveStorage(); + + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var firstResult = await handler.Handle(initialContext, CancellationToken.None); + firstResult.IsSuccess.ShouldBeTrue(); + + File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt")); + File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt.pointer.arius")); + + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + + // Act + var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + + // Assert + incrementalResult.IsSuccess.ShouldBeTrue(); + var summary = incrementalResult.Value; + + summary.TotalLocalFiles.ShouldBe(expectedFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(0); + summary.PointerFileEntriesDeleted.ShouldBe(1); + summary.NewStateName.ShouldNotBeNull(); + + var pointerEntry = incrementalContext.StateRepository + .GetPointerFileEntry("/docs/to-delete.txt.pointer.arius", includeBinaryProperties: true); + pointerEntry.ShouldBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + archiveStorage.UploadedStates.Count.ShouldBe(2); + } + + [Fact] + public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBinaryProperties() + { + // Arrange + var containerName = $"test-container-incremental-modified-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var filePath = UPath.Root / "docs" / "mutable.bin"; + var originalFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, filePath) + .WithRandomContent(3072, seed: 4001) + .Build(); + + var archiveStorage = new FakeArchiveStorage(); + + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var firstResult = await handler.Handle(initialContext, CancellationToken.None); + firstResult.IsSuccess.ShouldBeTrue(); + + var originalHash = originalFile.OriginalHash; + var originalBinaryProperties = initialContext.StateRepository.GetBinaryProperty(originalHash); + originalBinaryProperties.ShouldNotBeNull(); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, filePath) + .WithRandomContent(3584, seed: 4002) + .Build(); + + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + + // Act + var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + + // Assert + incrementalResult.IsSuccess.ShouldBeTrue(); + var summary = incrementalResult.Value; + + summary.TotalLocalFiles.ShouldBe(expectedFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.UniqueChunksUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(0); // pointer already existed + summary.NewStateName.ShouldNotBeNull(); + + var pointerEntry = incrementalContext.StateRepository + .GetPointerFileEntry(ToRelativePointerPath(filePath), includeBinaryProperties: true); + pointerEntry.ShouldNotBeNull(); + pointerEntry!.Hash.ShouldNotBe(originalHash); + pointerEntry.BinaryProperties.ShouldNotBeNull(); + + var oldBinaryProperties = incrementalContext.StateRepository.GetBinaryProperty(originalHash); + oldBinaryProperties.ShouldNotBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(2); + } + + [Fact] + public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState() + { + // Arrange + var containerName = $"test-container-incremental-nochanges-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var baselineFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "baseline.txt") + .WithRandomContent(2048, seed: 5001) + .Build(); + + var archiveStorage = new FakeArchiveStorage(); + + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var firstResult = await handler.Handle(initialContext, CancellationToken.None); + firstResult.IsSuccess.ShouldBeTrue(); + + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + + // Act + var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + + // Assert + incrementalResult.IsSuccess.ShouldBeTrue(); + var summary = incrementalResult.Value; + + summary.TotalLocalFiles.ShouldBe(expectedFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(0); + summary.UniqueChunksUploaded.ShouldBe(0); + summary.PointerFilesCreated.ShouldBe(0); + summary.PointerFileEntriesDeleted.ShouldBe(0); + summary.NewStateName.ShouldBeNull(); + + archiveStorage.UploadedStates.Count.ShouldBe(1); + File.Exists(incrementalContext.StateRepository.StateDatabaseFile.FullName).ShouldBeFalse(); + } + + [Fact] + public async Task Error_CancellationByUser_ShouldReturnFailureResult() + { + // Arrange + var containerName = $"test-container-error-cancel-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large1.bin") + .WithRandomContent(4096, seed: 6001) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large2.bin") + .WithRandomContent(4096, seed: 6002) + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + // Act + var result = await handler.Handle(handlerContext, cts.Token); + + // Assert + result.IsFailed.ShouldBeTrue(); + result.Errors.ShouldNotBeEmpty(); + result.Errors.First().Message.ShouldContain("cancelled by user"); + + archiveStorage.StoredChunks.Count.ShouldBe(0); + } + + [Fact] + public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() + { + // Arrange + var containerName = $"test-container-error-hash-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + var failingPath = UPath.Root / "hash" / "will-fail.bin"; + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, failingPath) + .WithRandomContent(1024, seed: 6101) + .Build(); + + var successfulPath = UPath.Root / "hash" / "will-upload.bin"; + var successfulFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, successfulPath) + .WithRandomContent(2048, seed: 6102) + .Build(); + + var failingAbsolutePath = Path.Combine(fixture.TestRunSourceFolder.FullName, "hash", "will-fail.bin"); + + var deleted = false; + var progressUpdates = new List(); + void HandleProgress(ProgressUpdate update) + { + progressUpdates.Add(update); + if (!deleted && update is FileProgressUpdate fileUpdate && + fileUpdate.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && + fileUpdate.StatusMessage?.Contains("Hashing", StringComparison.OrdinalIgnoreCase) == true) + { + deleted = true; + File.Delete(failingAbsolutePath); + } + } + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + containerName, + builder => builder.WithProgressReporter(new Progress(HandleProgress))); + + var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(1); + summary.PointerFilesCreated.ShouldBe(1); + summary.BytesUploadedUncompressed.ShouldBe(successfulFile.OriginalContent.Length); + + File.Exists(failingAbsolutePath).ShouldBeFalse(); + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "hash", "will-fail.bin.pointer.arius")).ShouldBeFalse(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + + progressUpdates.OfType() + .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error", StringComparison.OrdinalIgnoreCase) == true) + .ShouldBeTrue(); + } + + [Fact] + public async Task Error_UploadTaskFails_ShouldReturnFailure() + { + // Arrange + var containerName = $"test-container-error-upload-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "uploads" / "large.bin") + .WithRandomContent(4096, seed: 6201) + .Build(); + + var archiveStorage = new ThrowingArchiveStorage(failureCount: 1); + + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsFailed.ShouldBeTrue(); + result.Errors.First().Message.ShouldContain("failed"); + archiveStorage.StoredChunks.Count.ShouldBe(0); + } + + [Fact] + public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() + { + // Arrange + var containerName = $"test-container-error-multi-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "multi" / "large.bin") + .WithRandomContent(4096, seed: 6301) + .Build(); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "multi" / "small.txt") + .WithRandomContent(256, seed: 6302) + .Build(); + + var archiveStorage = new ThrowingArchiveStorage(failureCount: 2); + + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsFailed.ShouldBeTrue(); + result.Errors.First().Message.ShouldContain("multiple tasks failed"); + archiveStorage.StoredChunks.Count.ShouldBe(0); + } + + [Fact] + public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() + { + // Arrange + var containerName = $"test-container-error-pointer-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.PointerFileOnly, UPath.Root / "orphans" / "lonely.bin") + .Build(); + + var progressUpdates = new List(); + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + containerName, + builder => builder.WithProgressReporter(new Progress(progressUpdates.Add))); + + var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary.UniqueBinariesUploaded.ShouldBe(0); + summary.PointerFilesCreated.ShouldBe(0); + + archiveStorage.StoredChunks.Count.ShouldBe(0); + + handlerContext.StateRepository.GetPointerFileEntry("/orphans/lonely.bin.pointer.arius", includeBinaryProperties: true) + .ShouldBeNull(); + + progressUpdates.OfType() + .Any(p => p.FileName.EndsWith("lonely.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("pointer file without binary", StringComparison.OrdinalIgnoreCase) == true) + .ShouldBeTrue(); + } + + [Fact] + public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() + { + // Arrange + var containerName = $"test-container-stale-{Guid.CreateVersion7()}"; + ResetTestRoot(fixture, containerName); + + _ = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "active.txt") + .WithRandomContent(256, seed: 7) + .Build(); + + var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + + var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); + + var staleHash = FakeHashBuilder.GenerateValidHash(99); + handlerContext.StateRepository.AddBinaryProperties(new BinaryProperties + { + Hash = staleHash, + OriginalSize = 1, + ArchivedSize = 1, + StorageTier = StorageTier.Cool + }); + handlerContext.StateRepository.UpsertPointerFileEntries(new PointerFileEntry + { + Hash = staleHash, + RelativeName = "/stale.bin.pointer.arius", + CreationTimeUtc = DateTime.UtcNow, + LastWriteTimeUtc = DateTime.UtcNow + }); + + // Act + var result = await handler.Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); + summary.PointerFileEntriesDeleted.ShouldBe(1); + handlerContext.StateRepository.GetPointerFileEntry("/stale.bin.pointer.arius") + .ShouldBeNull(); + + archiveStorage.StoredChunks.Count.ShouldBe(1); + archiveStorage.UploadedStates.ShouldNotBeEmpty(); + + File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "active.txt.pointer.arius")).ShouldBeTrue(); + } + + private sealed class ThrowingArchiveStorage : FakeArchiveStorage + { + private int remainingFailures; + private readonly Func? predicate; + + public ThrowingArchiveStorage(int failureCount, Func? predicate = null) + { + remainingFailures = failureCount; + this.predicate = predicate; + } + + public override Task> OpenWriteChunkAsync( + Hash h, + CompressionLevel compressionLevel, + string contentType, + IDictionary? metadata = null, + IProgress? progress = null, + bool overwrite = false, + CancellationToken cancellationToken = default) + { + if ((predicate is null || predicate(h)) && Interlocked.Decrement(ref remainingFailures) >= 0) + { + return Task.FromResult(Result.Fail(new ExceptionalError(new IOException("Simulated upload failure")))); + } + + return base.OpenWriteChunkAsync(h, compressionLevel, contentType, metadata, progress, overwrite, cancellationToken); + } + } +} diff --git a/src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs b/src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs new file mode 100644 index 00000000..f34c2e8a --- /dev/null +++ b/src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs @@ -0,0 +1,191 @@ +using Arius.Core.Shared.Hashing; +using Arius.Core.Shared.Storage; +using FluentResults; +using System.Collections.Concurrent; +using System.IO.Compression; +using Zio; + +namespace Arius.Core.Tests.Helpers.Fakes; + +internal class FakeArchiveStorage : IArchiveStorage +{ + private readonly ConcurrentDictionary chunks = new(); + private readonly ConcurrentDictionary hydratedChunks = new(); + private readonly ConcurrentDictionary states = new(StringComparer.OrdinalIgnoreCase); + + private int containerCreated; + + public IReadOnlyDictionary StoredChunks => chunks; + + public IReadOnlyDictionary StoredStates => states; + + public List UploadedStates { get; } = new(); + + public Task CreateContainerIfNotExistsAsync() + { + var created = Interlocked.Exchange(ref containerCreated, 1) == 0; + return Task.FromResult(created); + } + + public Task ContainerExistsAsync() => Task.FromResult(containerCreated == 1); + + public IAsyncEnumerable GetStates(CancellationToken cancellationToken = default) + { + var orderedStates = states.Keys + .OrderBy(s => s, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return orderedStates.ToAsyncEnumerable(); + } + + public Task DownloadStateAsync(string stateName, FileEntry targetFile, CancellationToken cancellationToken = default) + { + if (!states.TryGetValue(stateName, out var content)) + throw new InvalidOperationException($"State '{stateName}' does not exist in fake storage."); + + targetFile.Directory.Create(); + + using var stream = targetFile.Open(FileMode.Create, FileAccess.Write, FileShare.None); + stream.Write(content); + return Task.CompletedTask; + } + + public async Task UploadStateAsync(string stateName, FileEntry sourceFile, CancellationToken cancellationToken = default) + { + await using var sourceStream = sourceFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + using var memoryStream = new MemoryStream(); + await sourceStream.CopyToAsync(memoryStream, cancellationToken); + + states[stateName] = memoryStream.ToArray(); + UploadedStates.Add(stateName); + } + + public Task> OpenReadChunkAsync(Hash h, CancellationToken cancellationToken = default) + { + if (chunks.TryGetValue(h, out var chunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); + } + + if (hydratedChunks.TryGetValue(h, out var hydratedChunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(hydratedChunk.Content, writable: false))); + } + + return Task.FromResult(Result.Fail(new BlobNotFoundError(h.ToString()))); + } + + public Task> OpenReadHydratedChunkAsync(Hash h, CancellationToken cancellationToken = default) + { + if (hydratedChunks.TryGetValue(h, out var chunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); + } + + return Task.FromResult(Result.Fail(new BlobNotFoundError(h.ToString()))); + } + + public virtual Task> OpenWriteChunkAsync(Hash h, CompressionLevel compressionLevel, string contentType, IDictionary? metadata = null, IProgress? progress = null, bool overwrite = false, CancellationToken cancellationToken = default) + { + if (!overwrite && chunks.ContainsKey(h)) + { + return Task.FromResult(Result.Fail(new BlobAlreadyExistsError(h.ToString()))); + } + + var chunk = new FakeArchiveStorageChunk + { + ContentType = contentType, + CompressionLevel = compressionLevel, + Metadata = metadata != null ? new Dictionary(metadata, StringComparer.OrdinalIgnoreCase) : new Dictionary(StringComparer.OrdinalIgnoreCase), + StorageTier = StorageTier.Cool + }; + + var recordingStream = new RecordingMemoryStream(bytes => + { + chunk.Content = bytes; + chunk.ContentLength = bytes.LongLength; + chunks[h] = chunk; + }); + + return Task.FromResult(Result.Ok(recordingStream)); + } + + public Task GetChunkPropertiesAsync(Hash h, CancellationToken cancellationToken = default) + { + if (chunks.TryGetValue(h, out var chunk)) + { + return Task.FromResult(new StorageProperties( + h.ToString(), + chunk.ContentType, + chunk.Metadata.Count == 0 ? null : new Dictionary(chunk.Metadata, StringComparer.OrdinalIgnoreCase), + chunk.StorageTier, + chunk.ContentLength)); + } + + return Task.FromResult(null); + } + + public Task DeleteChunkAsync(Hash h, CancellationToken cancellationToken = default) + { + chunks.TryRemove(h, out _); + hydratedChunks.TryRemove(h, out _); + return Task.CompletedTask; + } + + public Task SetChunkMetadataAsync(Hash h, IDictionary metadata, CancellationToken cancellationToken = default) + { + if (chunks.TryGetValue(h, out var chunk)) + { + chunk.Metadata = new Dictionary(metadata, StringComparer.OrdinalIgnoreCase); + } + + return Task.CompletedTask; + } + + public Task SetChunkStorageTierPerPolicy(Hash h, long length, StorageTier targetTier) + { + if (chunks.TryGetValue(h, out var chunk)) + { + chunk.StorageTier = targetTier; + } + + return Task.FromResult(targetTier); + } + + public Task StartHydrationAsync(Hash hash, RehydratePriority priority) + { + if (chunks.TryGetValue(hash, out var chunk)) + { + hydratedChunks[hash] = chunk; + } + + return Task.CompletedTask; + } + + private sealed class RecordingMemoryStream : MemoryStream + { + private readonly Action onDispose; + + public RecordingMemoryStream(Action onDispose) => this.onDispose = onDispose; + + protected override void Dispose(bool disposing) + { + if (disposing) + { + onDispose(ToArray()); + } + + base.Dispose(disposing); + } + } +} + +internal sealed class FakeArchiveStorageChunk +{ + public byte[] Content { get; set; } = Array.Empty(); + public long ContentLength { get; set; } + public string ContentType { get; set; } = string.Empty; + public Dictionary Metadata { get; set; } = new(StringComparer.OrdinalIgnoreCase); + public StorageTier StorageTier { get; set; } = StorageTier.Cool; + public CompressionLevel CompressionLevel { get; set; } +} From 8f91fcb601dfcf149a13477f8d33aaa4e835646b Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 08:17:42 +0200 Subject: [PATCH 03/43] Update ArchiveCommandHandlerHandleTests.cs --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 761c821b..20636004 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -750,7 +750,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() summary.PointerFileEntriesDeleted.ShouldBe(0); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Values.Count.ShouldBe(2); + archiveStorage.StoredChunks.Values.Count().ShouldBe(2); var pointerEntry = incrementalContext.StateRepository .GetPointerFileEntry(ToRelativePointerPath(newSmallFile.OriginalPath), includeBinaryProperties: true); From b695ced60b57306d30d804cf611092b9d37595ad Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 17:37:45 +0200 Subject: [PATCH 04/43] chore: move MockArchiveStorageBuilder --- .../Commands/Restore}/MockArchiveStorageBuilder.cs | 2 +- .../PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename src/Arius.Core.Tests/{Helpers/Builders => Features/Commands/Restore}/MockArchiveStorageBuilder.cs (99%) diff --git a/src/Arius.Core.Tests/Helpers/Builders/MockArchiveStorageBuilder.cs b/src/Arius.Core.Tests/Features/Commands/Restore/MockArchiveStorageBuilder.cs similarity index 99% rename from src/Arius.Core.Tests/Helpers/Builders/MockArchiveStorageBuilder.cs rename to src/Arius.Core.Tests/Features/Commands/Restore/MockArchiveStorageBuilder.cs index 2eda7ee0..631c10c1 100644 --- a/src/Arius.Core.Tests/Helpers/Builders/MockArchiveStorageBuilder.cs +++ b/src/Arius.Core.Tests/Features/Commands/Restore/MockArchiveStorageBuilder.cs @@ -5,7 +5,7 @@ using NSubstitute; using System.Formats.Tar; -namespace Arius.Core.Tests.Helpers.Builders; +namespace Arius.Core.Tests.Features.Commands.Restore; internal class MockArchiveStorageBuilder { diff --git a/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs b/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs index e13d270b..d60e9dfd 100644 --- a/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs +++ b/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs @@ -2,6 +2,7 @@ using Arius.Core.Shared.FileSystem; using Arius.Core.Shared.StateRepositories; using Arius.Core.Shared.Storage; +using Arius.Core.Tests.Features.Commands.Restore; using Arius.Core.Tests.Helpers.Builders; using Arius.Core.Tests.Helpers.FakeLogger; using Arius.Core.Tests.Helpers.Fakes; From 919612849c92ea4e3c6f96d1426350ba44259045 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 17:45:11 +0200 Subject: [PATCH 05/43] chore: ordering --- .../ArchiveCommandHandlerHandleTests.cs | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 20636004..9d2cfb16 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -4,21 +4,14 @@ using Arius.Core.Shared.StateRepositories; using Arius.Core.Shared.Storage; using Arius.Core.Tests.Helpers.Builders; +using Arius.Core.Tests.Helpers.FakeLogger; using Arius.Core.Tests.Helpers.Fakes; using Arius.Core.Tests.Helpers.Fixtures; -using Arius.Core.Tests.Helpers.FakeLogger; using FluentResults; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Testing; using Shouldly; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; using System.IO.Compression; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Zio; namespace Arius.Core.Tests.Features.Commands.Archive; @@ -40,8 +33,7 @@ public ArchiveCommandHandlerHandleTests(FixtureWithFileSystem fixture) private static string ToRelativePointerPath(UPath binaryPath) => binaryPath.GetPointerFilePath().ToString(); - private static string ToAbsolutePointerPath(FixtureWithFileSystem fixture, UPath binaryPath) => - Path.Combine(fixture.TestRunSourceFolder.FullName, binaryPath.GetPointerFilePath().ToString().TrimStart('/')); + private static string ToAbsolutePointerPath(FixtureWithFileSystem fixture, UPath binaryPath) => Path.Combine(fixture.TestRunSourceFolder.FullName, binaryPath.GetPointerFilePath().ToString().TrimStart('/')); private static void ResetTestRoot(FixtureWithFileSystem fixture, string containerName) { @@ -51,21 +43,13 @@ private static void ResetTestRoot(FixtureWithFileSystem fixture, string containe foreach (var file in Directory.EnumerateFiles(fixture.TestRunSourceFolder.FullName)) File.Delete(file); - var stateCacheRoot = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - "Arius", - "statecache", - fixture.RepositoryOptions?.AccountName ?? "testaccount", - containerName); + var stateCacheRoot = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Arius", "statecache", fixture.RepositoryOptions?.AccountName ?? "testaccount", containerName); if (Directory.Exists(stateCacheRoot)) Directory.Delete(stateCacheRoot, recursive: true); } - private static async Task BuildHandlerContextAsync( - ArchiveCommand command, - FakeArchiveStorage archiveStorage, - FakeLoggerFactory loggerFactory) => + private static async Task BuildHandlerContextAsync(ArchiveCommand command, FakeArchiveStorage archiveStorage, FakeLoggerFactory loggerFactory) => await new HandlerContextBuilder(command, loggerFactory) .WithArchiveStorage(archiveStorage) .BuildAsync(); From b5494161d2636d1ea6bd128f8bd76c9c804e45ad Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 17:47:23 +0200 Subject: [PATCH 06/43] chore: move --- .../Fakes => Features/Commands/Archive}/FakeArchiveStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/Arius.Core.Tests/{Helpers/Fakes => Features/Commands/Archive}/FakeArchiveStorage.cs (99%) diff --git a/src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs b/src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs similarity index 99% rename from src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs rename to src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs index f34c2e8a..42eed70f 100644 --- a/src/Arius.Core.Tests/Helpers/Fakes/FakeArchiveStorage.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs @@ -5,7 +5,7 @@ using System.IO.Compression; using Zio; -namespace Arius.Core.Tests.Helpers.Fakes; +namespace Arius.Core.Tests.Features.Commands.Archive; internal class FakeArchiveStorage : IArchiveStorage { From e26cbc2bae6b70fede87a79c9b92ca70c007f62e Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 19:16:30 +0200 Subject: [PATCH 07/43] feat: refactor FakeArchiveStorage to MockArchiveStorageBuilder --- .../ArchiveCommandHandlerHandleTests.cs | 174 ++++------ .../Commands/Archive/FakeArchiveStorage.cs | 191 ----------- .../Archive/MockArchiveStorageBuilder.cs | 320 ++++++++++++++++++ 3 files changed, 394 insertions(+), 291 deletions(-) delete mode 100644 src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs create mode 100644 src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 9d2cfb16..f82ed730 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -49,7 +49,7 @@ private static void ResetTestRoot(FixtureWithFileSystem fixture, string containe Directory.Delete(stateCacheRoot, recursive: true); } - private static async Task BuildHandlerContextAsync(ArchiveCommand command, FakeArchiveStorage archiveStorage, FakeLoggerFactory loggerFactory) => + private static async Task BuildHandlerContextAsync(ArchiveCommand command, IArchiveStorage archiveStorage, FakeLoggerFactory loggerFactory) => await new HandlerContextBuilder(command, loggerFactory) .WithArchiveStorage(archiveStorage) .BuildAsync(); @@ -66,14 +66,14 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic return (entries.Count, existingPointerFileCount); } - private async Task<(ArchiveCommand Command, HandlerContext Context, FakeArchiveStorage Storage, FakeLoggerFactory LoggerFactory)> + private async Task<(ArchiveCommand Command, HandlerContext Context, MockArchiveStorageBuilder StorageBuilder, FakeLoggerFactory LoggerFactory)> CreateHandlerContextAsync( string containerName, Action? configureCommand = null, - FakeArchiveStorage? archiveStorage = null, + MockArchiveStorageBuilder? storageBuilder = null, FakeLoggerFactory? loggerFactory = null) { - archiveStorage ??= new FakeArchiveStorage(); + storageBuilder ??= new MockArchiveStorageBuilder(fixture); loggerFactory ??= new FakeLoggerFactory(); var commandBuilder = new ArchiveCommandBuilder(fixture) @@ -85,9 +85,10 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic configureCommand?.Invoke(commandBuilder); var command = commandBuilder.Build(); + var archiveStorage = storageBuilder.Build(); var handlerContext = await BuildHandlerContextAsync(command, archiveStorage, loggerFactory); - return (command, handlerContext, archiveStorage, loggerFactory); + return (command, handlerContext, storageBuilder, loggerFactory); } [Fact] @@ -103,7 +104,7 @@ public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() .WithRandomContent(4096, seed: 10) .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -123,8 +124,8 @@ public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() summary.BytesUploadedUncompressed.ShouldBe(largeFile.OriginalContent.LongLength); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); - var chunk = archiveStorage.StoredChunks.Single().Value; + storageBuilder.StoredChunks.Count.ShouldBe(1); + var chunk = storageBuilder.StoredChunks.Single().Value; chunk.ContentType.ShouldBe("application/aes256cbc+gzip"); chunk.Metadata.ShouldContainKey("OriginalContentLength"); chunk.Metadata["OriginalContentLength"].ShouldBe(largeFile.OriginalContent.Length.ToString()); @@ -158,7 +159,7 @@ public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBina .WithRandomContent(512, seed: 2) .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -178,8 +179,8 @@ public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBina summary.BytesUploadedUncompressed.ShouldBe(smallFile.OriginalContent.LongLength); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); - var tarChunk = archiveStorage.StoredChunks.Single().Value; + storageBuilder.StoredChunks.Count.ShouldBe(1); + var tarChunk = storageBuilder.StoredChunks.Single().Value; tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); tarChunk.Metadata.ShouldContainKey("OriginalContentLength"); tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); @@ -219,7 +220,7 @@ public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -239,8 +240,8 @@ public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() summary.BytesUploadedUncompressed.ShouldBe(0); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); - var tarChunk = archiveStorage.StoredChunks.Single().Value; + storageBuilder.StoredChunks.Count.ShouldBe(1); + var tarChunk = storageBuilder.StoredChunks.Single().Value; tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); @@ -276,7 +277,7 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac var stalePointer = binaryWithPointer.FilePair.CreatePointerFile(staleHash); stalePointer.ReadHash().ShouldBe(staleHash); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -296,7 +297,7 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac summary.BytesUploadedUncompressed.ShouldBe(binaryWithPointer.OriginalContent.LongLength); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); + storageBuilder.StoredChunks.Count.ShouldBe(1); var pointerPath = ToAbsolutePointerPath(fixture, binaryPath); File.Exists(pointerPath).ShouldBeTrue(); @@ -334,7 +335,7 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches var progressUpdates = new List(); var progressReporter = new Progress(progressUpdates.Add); - var (command, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + var (command, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( containerName, builder => builder .WithProgressReporter(progressReporter) @@ -360,15 +361,15 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches summary.BytesUploadedUncompressed.ShouldBe(smallFile.OriginalContent.Length + largeFile.OriginalContent.Length); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(2); - archiveStorage.UploadedStates.ShouldContain(summary.NewStateName!); + storageBuilder.StoredChunks.Count.ShouldBe(2); + storageBuilder.UploadedStates.ShouldContain(summary.NewStateName!); - var tarChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); + var tarChunk = storageBuilder.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); tarChunk.Metadata.ShouldContainKey("OriginalContentLength"); tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); - var largeChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); + var largeChunk = storageBuilder.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); largeChunk.Metadata.ShouldContainKey("OriginalContentLength"); largeChunk.Metadata["OriginalContentLength"].ShouldBe(largeFile.OriginalContent.Length.ToString()); @@ -409,7 +410,7 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre .WithDuplicate(originalSmallFile, UPath.Root / "texts" / "archive" / "note-copy.txt") .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -427,15 +428,15 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre summary.BytesUploadedUncompressed.ShouldBe( originalLargeFile.OriginalContent.Length + originalSmallFile.OriginalContent.Length); - archiveStorage.StoredChunks.Count.ShouldBe(2); - archiveStorage.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+gzip").ShouldBe(1); - archiveStorage.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+tar+gzip").ShouldBe(1); + storageBuilder.StoredChunks.Count.ShouldBe(2); + storageBuilder.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+gzip").ShouldBe(1); + storageBuilder.StoredChunks.Values.Count(c => c.ContentType == "application/aes256cbc+tar+gzip").ShouldBe(1); - var largeChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); + var largeChunk = storageBuilder.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+gzip"); largeChunk.Metadata.ShouldContainKey("OriginalContentLength"); largeChunk.Metadata["OriginalContentLength"].ShouldBe(originalLargeFile.OriginalContent.Length.ToString()); - var tarChunk = archiveStorage.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); + var tarChunk = storageBuilder.StoredChunks.Values.Single(c => c.ContentType == "application/aes256cbc+tar+gzip"); tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); tarChunk.Metadata["SmallChunkCount"].ShouldBe("1"); @@ -475,7 +476,7 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu .Build()) .ToArray(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -493,8 +494,8 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu summary.BytesUploadedUncompressed.ShouldBe(smallFiles.Sum(f => f.OriginalContent.Length)); summary.PointerFileEntriesDeleted.ShouldBe(0); - archiveStorage.StoredChunks.Count.ShouldBe(1); - var tarChunk = archiveStorage.StoredChunks.Single().Value; + storageBuilder.StoredChunks.Count.ShouldBe(1); + var tarChunk = storageBuilder.StoredChunks.Single().Value; tarChunk.ContentType.ShouldBe("application/aes256cbc+tar+gzip"); tarChunk.Metadata.ShouldContainKey("SmallChunkCount"); tarChunk.Metadata["SmallChunkCount"].ShouldBe(paths.Length.ToString()); @@ -531,7 +532,7 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary .Build()) .ToArray(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -549,7 +550,7 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary summary.PointerFileEntriesDeleted.ShouldBe(0); summary.BytesUploadedUncompressed.ShouldBe(smallFiles.Sum(f => f.OriginalContent.Length)); - var tarChunks = archiveStorage.StoredChunks.Values + var tarChunks = storageBuilder.StoredChunks.Values .Where(c => c.ContentType == "application/aes256cbc+tar+gzip") .ToList(); tarChunks.Count.ShouldBe(2); @@ -595,7 +596,7 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite .WithDuplicate(ownerOmega, UPath.Root / "tar" / "zzz-duplicate.txt") .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -612,7 +613,7 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite summary.PointerFilesCreated.ShouldBe(5); summary.PointerFileEntriesDeleted.ShouldBe(0); - var tarChunks = archiveStorage.StoredChunks.Values + var tarChunks = storageBuilder.StoredChunks.Values .Where(c => c.ContentType == "application/aes256cbc+tar+gzip") .ToList(); tarChunks.Count.ShouldBe(2); @@ -647,9 +648,9 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() .WithRandomContent(4096, seed: 501) .Build(); - var archiveStorage = new FakeArchiveStorage(); + var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (initialFileCount, _) = GetInitialFileStatistics(initialContext); var firstResult = await handler.Handle(initialContext, CancellationToken.None); @@ -662,7 +663,7 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() staleHash.ShouldNotBe(originalHash); largeFile.FilePair.CreatePointerFile(staleHash); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -683,8 +684,8 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() largeFile.FilePair.PointerFile.ReadHash().ShouldBe(originalHash); - archiveStorage.StoredChunks.Count.ShouldBe(1); - archiveStorage.UploadedStates.Count.ShouldBe(1); + storageBuilder.StoredChunks.Count.ShouldBe(1); + storageBuilder.UploadedStates.Count.ShouldBe(1); var pointerEntry = incrementalContext.StateRepository .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); @@ -704,9 +705,9 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() .WithRandomContent(4096, seed: 2001) .Build(); - var archiveStorage = new FakeArchiveStorage(); + var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); @@ -715,7 +716,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() .WithRandomContent(512, seed: 2002) .Build(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -734,7 +735,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() summary.PointerFileEntriesDeleted.ShouldBe(0); summary.NewStateName.ShouldNotBeNull(); - archiveStorage.StoredChunks.Values.Count().ShouldBe(2); + storageBuilder.StoredChunks.Values.Count().ShouldBe(2); var pointerEntry = incrementalContext.StateRepository .GetPointerFileEntry(ToRelativePointerPath(newSmallFile.OriginalPath), includeBinaryProperties: true); @@ -755,16 +756,16 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry .WithRandomContent(2048, seed: 3001) .Build(); - var archiveStorage = new FakeArchiveStorage(); + var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt")); File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt.pointer.arius")); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -784,8 +785,8 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry .GetPointerFileEntry("/docs/to-delete.txt.pointer.arius", includeBinaryProperties: true); pointerEntry.ShouldBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); - archiveStorage.UploadedStates.Count.ShouldBe(2); + storageBuilder.StoredChunks.Count.ShouldBe(1); + storageBuilder.UploadedStates.Count.ShouldBe(2); } [Fact] @@ -801,9 +802,9 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina .WithRandomContent(3072, seed: 4001) .Build(); - var archiveStorage = new FakeArchiveStorage(); + var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); @@ -816,7 +817,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina .WithRandomContent(3584, seed: 4002) .Build(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -842,7 +843,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina var oldBinaryProperties = incrementalContext.StateRepository.GetBinaryProperty(originalHash); oldBinaryProperties.ShouldNotBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(2); + storageBuilder.StoredChunks.Count.ShouldBe(2); } [Fact] @@ -857,13 +858,13 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState .WithRandomContent(2048, seed: 5001) .Build(); - var archiveStorage = new FakeArchiveStorage(); + var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -881,7 +882,7 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState summary.PointerFileEntriesDeleted.ShouldBe(0); summary.NewStateName.ShouldBeNull(); - archiveStorage.UploadedStates.Count.ShouldBe(1); + storageBuilder.UploadedStates.Count.ShouldBe(1); File.Exists(incrementalContext.StateRepository.StateDatabaseFile.FullName).ShouldBeFalse(); } @@ -902,7 +903,7 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() .WithRandomContent(4096, seed: 6002) .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); using var cts = new CancellationTokenSource(); cts.Cancel(); @@ -915,7 +916,7 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() result.Errors.ShouldNotBeEmpty(); result.Errors.First().Message.ShouldContain("cancelled by user"); - archiveStorage.StoredChunks.Count.ShouldBe(0); + storageBuilder.StoredChunks.Count.ShouldBe(0); } [Fact] @@ -953,7 +954,7 @@ void HandleProgress(ProgressUpdate update) } } - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( containerName, builder => builder.WithProgressReporter(new Progress(HandleProgress))); @@ -975,7 +976,7 @@ void HandleProgress(ProgressUpdate update) File.Exists(failingAbsolutePath).ShouldBeFalse(); File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "hash", "will-fail.bin.pointer.arius")).ShouldBeFalse(); - archiveStorage.StoredChunks.Count.ShouldBe(1); + storageBuilder.StoredChunks.Count.ShouldBe(1); progressUpdates.OfType() .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error", StringComparison.OrdinalIgnoreCase) == true) @@ -994,9 +995,10 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() .WithRandomContent(4096, seed: 6201) .Build(); - var archiveStorage = new ThrowingArchiveStorage(failureCount: 1); + var storageBuilder = new MockArchiveStorageBuilder(fixture) + .WithThrowOnWrite(failureCount: 1); - var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); // Act var result = await handler.Handle(handlerContext, CancellationToken.None); @@ -1004,7 +1006,7 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() // Assert result.IsFailed.ShouldBeTrue(); result.Errors.First().Message.ShouldContain("failed"); - archiveStorage.StoredChunks.Count.ShouldBe(0); + storageBuilder.StoredChunks.Count.ShouldBe(0); } [Fact] @@ -1024,9 +1026,10 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() .WithRandomContent(256, seed: 6302) .Build(); - var archiveStorage = new ThrowingArchiveStorage(failureCount: 2); + var storageBuilder = new MockArchiveStorageBuilder(fixture) + .WithThrowOnWrite(failureCount: 2); - var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, archiveStorage: archiveStorage); + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); // Act var result = await handler.Handle(handlerContext, CancellationToken.None); @@ -1034,7 +1037,7 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() // Assert result.IsFailed.ShouldBeTrue(); result.Errors.First().Message.ShouldContain("multiple tasks failed"); - archiveStorage.StoredChunks.Count.ShouldBe(0); + storageBuilder.StoredChunks.Count.ShouldBe(0); } [Fact] @@ -1049,7 +1052,7 @@ public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() .Build(); var progressUpdates = new List(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync( + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( containerName, builder => builder.WithProgressReporter(new Progress(progressUpdates.Add))); @@ -1067,7 +1070,7 @@ public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() summary.UniqueBinariesUploaded.ShouldBe(0); summary.PointerFilesCreated.ShouldBe(0); - archiveStorage.StoredChunks.Count.ShouldBe(0); + storageBuilder.StoredChunks.Count.ShouldBe(0); handlerContext.StateRepository.GetPointerFileEntry("/orphans/lonely.bin.pointer.arius", includeBinaryProperties: true) .ShouldBeNull(); @@ -1089,7 +1092,7 @@ public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() .WithRandomContent(256, seed: 7) .Build(); - var (_, handlerContext, archiveStorage, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -1121,38 +1124,9 @@ public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() handlerContext.StateRepository.GetPointerFileEntry("/stale.bin.pointer.arius") .ShouldBeNull(); - archiveStorage.StoredChunks.Count.ShouldBe(1); - archiveStorage.UploadedStates.ShouldNotBeEmpty(); + storageBuilder.StoredChunks.Count.ShouldBe(1); + storageBuilder.UploadedStates.ShouldNotBeEmpty(); File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "active.txt.pointer.arius")).ShouldBeTrue(); } - - private sealed class ThrowingArchiveStorage : FakeArchiveStorage - { - private int remainingFailures; - private readonly Func? predicate; - - public ThrowingArchiveStorage(int failureCount, Func? predicate = null) - { - remainingFailures = failureCount; - this.predicate = predicate; - } - - public override Task> OpenWriteChunkAsync( - Hash h, - CompressionLevel compressionLevel, - string contentType, - IDictionary? metadata = null, - IProgress? progress = null, - bool overwrite = false, - CancellationToken cancellationToken = default) - { - if ((predicate is null || predicate(h)) && Interlocked.Decrement(ref remainingFailures) >= 0) - { - return Task.FromResult(Result.Fail(new ExceptionalError(new IOException("Simulated upload failure")))); - } - - return base.OpenWriteChunkAsync(h, compressionLevel, contentType, metadata, progress, overwrite, cancellationToken); - } - } } diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs b/src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs deleted file mode 100644 index 42eed70f..00000000 --- a/src/Arius.Core.Tests/Features/Commands/Archive/FakeArchiveStorage.cs +++ /dev/null @@ -1,191 +0,0 @@ -using Arius.Core.Shared.Hashing; -using Arius.Core.Shared.Storage; -using FluentResults; -using System.Collections.Concurrent; -using System.IO.Compression; -using Zio; - -namespace Arius.Core.Tests.Features.Commands.Archive; - -internal class FakeArchiveStorage : IArchiveStorage -{ - private readonly ConcurrentDictionary chunks = new(); - private readonly ConcurrentDictionary hydratedChunks = new(); - private readonly ConcurrentDictionary states = new(StringComparer.OrdinalIgnoreCase); - - private int containerCreated; - - public IReadOnlyDictionary StoredChunks => chunks; - - public IReadOnlyDictionary StoredStates => states; - - public List UploadedStates { get; } = new(); - - public Task CreateContainerIfNotExistsAsync() - { - var created = Interlocked.Exchange(ref containerCreated, 1) == 0; - return Task.FromResult(created); - } - - public Task ContainerExistsAsync() => Task.FromResult(containerCreated == 1); - - public IAsyncEnumerable GetStates(CancellationToken cancellationToken = default) - { - var orderedStates = states.Keys - .OrderBy(s => s, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - return orderedStates.ToAsyncEnumerable(); - } - - public Task DownloadStateAsync(string stateName, FileEntry targetFile, CancellationToken cancellationToken = default) - { - if (!states.TryGetValue(stateName, out var content)) - throw new InvalidOperationException($"State '{stateName}' does not exist in fake storage."); - - targetFile.Directory.Create(); - - using var stream = targetFile.Open(FileMode.Create, FileAccess.Write, FileShare.None); - stream.Write(content); - return Task.CompletedTask; - } - - public async Task UploadStateAsync(string stateName, FileEntry sourceFile, CancellationToken cancellationToken = default) - { - await using var sourceStream = sourceFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read); - using var memoryStream = new MemoryStream(); - await sourceStream.CopyToAsync(memoryStream, cancellationToken); - - states[stateName] = memoryStream.ToArray(); - UploadedStates.Add(stateName); - } - - public Task> OpenReadChunkAsync(Hash h, CancellationToken cancellationToken = default) - { - if (chunks.TryGetValue(h, out var chunk)) - { - return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); - } - - if (hydratedChunks.TryGetValue(h, out var hydratedChunk)) - { - return Task.FromResult(Result.Ok(new MemoryStream(hydratedChunk.Content, writable: false))); - } - - return Task.FromResult(Result.Fail(new BlobNotFoundError(h.ToString()))); - } - - public Task> OpenReadHydratedChunkAsync(Hash h, CancellationToken cancellationToken = default) - { - if (hydratedChunks.TryGetValue(h, out var chunk)) - { - return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); - } - - return Task.FromResult(Result.Fail(new BlobNotFoundError(h.ToString()))); - } - - public virtual Task> OpenWriteChunkAsync(Hash h, CompressionLevel compressionLevel, string contentType, IDictionary? metadata = null, IProgress? progress = null, bool overwrite = false, CancellationToken cancellationToken = default) - { - if (!overwrite && chunks.ContainsKey(h)) - { - return Task.FromResult(Result.Fail(new BlobAlreadyExistsError(h.ToString()))); - } - - var chunk = new FakeArchiveStorageChunk - { - ContentType = contentType, - CompressionLevel = compressionLevel, - Metadata = metadata != null ? new Dictionary(metadata, StringComparer.OrdinalIgnoreCase) : new Dictionary(StringComparer.OrdinalIgnoreCase), - StorageTier = StorageTier.Cool - }; - - var recordingStream = new RecordingMemoryStream(bytes => - { - chunk.Content = bytes; - chunk.ContentLength = bytes.LongLength; - chunks[h] = chunk; - }); - - return Task.FromResult(Result.Ok(recordingStream)); - } - - public Task GetChunkPropertiesAsync(Hash h, CancellationToken cancellationToken = default) - { - if (chunks.TryGetValue(h, out var chunk)) - { - return Task.FromResult(new StorageProperties( - h.ToString(), - chunk.ContentType, - chunk.Metadata.Count == 0 ? null : new Dictionary(chunk.Metadata, StringComparer.OrdinalIgnoreCase), - chunk.StorageTier, - chunk.ContentLength)); - } - - return Task.FromResult(null); - } - - public Task DeleteChunkAsync(Hash h, CancellationToken cancellationToken = default) - { - chunks.TryRemove(h, out _); - hydratedChunks.TryRemove(h, out _); - return Task.CompletedTask; - } - - public Task SetChunkMetadataAsync(Hash h, IDictionary metadata, CancellationToken cancellationToken = default) - { - if (chunks.TryGetValue(h, out var chunk)) - { - chunk.Metadata = new Dictionary(metadata, StringComparer.OrdinalIgnoreCase); - } - - return Task.CompletedTask; - } - - public Task SetChunkStorageTierPerPolicy(Hash h, long length, StorageTier targetTier) - { - if (chunks.TryGetValue(h, out var chunk)) - { - chunk.StorageTier = targetTier; - } - - return Task.FromResult(targetTier); - } - - public Task StartHydrationAsync(Hash hash, RehydratePriority priority) - { - if (chunks.TryGetValue(hash, out var chunk)) - { - hydratedChunks[hash] = chunk; - } - - return Task.CompletedTask; - } - - private sealed class RecordingMemoryStream : MemoryStream - { - private readonly Action onDispose; - - public RecordingMemoryStream(Action onDispose) => this.onDispose = onDispose; - - protected override void Dispose(bool disposing) - { - if (disposing) - { - onDispose(ToArray()); - } - - base.Dispose(disposing); - } - } -} - -internal sealed class FakeArchiveStorageChunk -{ - public byte[] Content { get; set; } = Array.Empty(); - public long ContentLength { get; set; } - public string ContentType { get; set; } = string.Empty; - public Dictionary Metadata { get; set; } = new(StringComparer.OrdinalIgnoreCase); - public StorageTier StorageTier { get; set; } = StorageTier.Cool; - public CompressionLevel CompressionLevel { get; set; } -} diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs b/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs new file mode 100644 index 00000000..09c857d8 --- /dev/null +++ b/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs @@ -0,0 +1,320 @@ +using System.Collections.Concurrent; +using System.IO.Compression; +using Arius.Core.Shared.Hashing; +using Arius.Core.Shared.Storage; +using Arius.Core.Tests.Helpers.Fixtures; +using FluentResults; +using NSubstitute; +using Zio; + +namespace Arius.Core.Tests.Features.Commands.Archive; + +internal class MockArchiveStorageBuilder +{ + private readonly Fixture fixture; + + // Internal state for mock configuration + private readonly Dictionary chunks = new(); + private readonly Dictionary hydratedChunks = new(); + private readonly Dictionary states = new(StringComparer.OrdinalIgnoreCase); + private bool containerExists = true; + + // Track operations for potential assertions + private readonly ConcurrentDictionary writtenChunks = new(); + private readonly List uploadedStates = new(); + + // Error simulation + private int throwOnWriteFailureCount; + private Func? throwOnWritePredicate; + + // Expose internal state for test assertions + public IReadOnlyDictionary StoredChunks => chunks; + public IReadOnlyDictionary StoredStates => states; + public List UploadedStates => uploadedStates; + + public MockArchiveStorageBuilder(Fixture fixture) + { + this.fixture = fixture; + } + + public MockArchiveStorageBuilder WithThrowOnWrite(int failureCount, Func? predicate = null) + { + throwOnWriteFailureCount = failureCount; + throwOnWritePredicate = predicate; + return this; + } + + public MockArchiveStorageBuilder WithContainerExists(bool exists = true) + { + containerExists = exists; + return this; + } + + public MockArchiveStorageBuilder AddChunk(Hash hash, byte[] content, string contentType = "application/octet-stream", StorageTier tier = StorageTier.Cool, CompressionLevel compressionLevel = CompressionLevel.Optimal, IDictionary? metadata = null) + { + chunks[hash] = new FakeArchiveStorageChunk + { + Content = content, + ContentLength = content.Length, + ContentType = contentType, + StorageTier = tier, + CompressionLevel = compressionLevel, + Metadata = metadata != null ? new Dictionary(metadata, StringComparer.OrdinalIgnoreCase) : new Dictionary(StringComparer.OrdinalIgnoreCase) + }; + return this; + } + + public MockArchiveStorageBuilder AddHydratedChunk(Hash hash, byte[] content, string contentType = "application/octet-stream", StorageTier tier = StorageTier.Cool, CompressionLevel compressionLevel = CompressionLevel.Optimal, IDictionary? metadata = null) + { + hydratedChunks[hash] = new FakeArchiveStorageChunk + { + Content = content, + ContentLength = content.Length, + ContentType = contentType, + StorageTier = tier, + CompressionLevel = compressionLevel, + Metadata = metadata != null ? new Dictionary(metadata, StringComparer.OrdinalIgnoreCase) : new Dictionary(StringComparer.OrdinalIgnoreCase) + }; + return this; + } + + public MockArchiveStorageBuilder AddState(string name, byte[] content) + { + states[name] = content; + return this; + } + + public IArchiveStorage Build() + { + var mock = Substitute.For(); + + // Container operations + var containerCreated = containerExists; + mock.CreateContainerIfNotExistsAsync() + .Returns(callInfo => + { + var wasCreated = !containerCreated; + containerCreated = true; + return Task.FromResult(wasCreated); + }); + + mock.ContainerExistsAsync() + .Returns(_ => Task.FromResult(containerCreated)); + + // State operations + mock.GetStates(Arg.Any()) + .Returns(callInfo => + { + var orderedStates = states.Keys + .OrderBy(s => s, StringComparer.OrdinalIgnoreCase) + .ToArray(); + return orderedStates.ToAsyncEnumerable(); + }); + + mock.DownloadStateAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var stateName = callInfo.Arg(); + var targetFile = callInfo.ArgAt(1); + + if (!states.TryGetValue(stateName, out var content)) + throw new InvalidOperationException($"State '{stateName}' does not exist in fake storage."); + + targetFile.Directory.Create(); + using var stream = targetFile.Open(FileMode.Create, FileAccess.Write, FileShare.None); + stream.Write(content); + return Task.CompletedTask; + }); + + mock.UploadStateAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(async callInfo => + { + var stateName = callInfo.Arg(); + var sourceFile = callInfo.ArgAt(1); + var cancellationToken = callInfo.ArgAt(2); + + await using var sourceStream = sourceFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read); + using var memoryStream = new MemoryStream(); + await sourceStream.CopyToAsync(memoryStream, cancellationToken); + + states[stateName] = memoryStream.ToArray(); + uploadedStates.Add(stateName); + }); + + // Chunk read operations + mock.OpenReadChunkAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + + if (chunks.TryGetValue(hash, out var chunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); + } + + if (hydratedChunks.TryGetValue(hash, out var hydratedChunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(hydratedChunk.Content, writable: false))); + } + + return Task.FromResult(Result.Fail(new BlobNotFoundError(hash.ToString()))); + }); + + mock.OpenReadHydratedChunkAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + + if (hydratedChunks.TryGetValue(hash, out var chunk)) + { + return Task.FromResult(Result.Ok(new MemoryStream(chunk.Content, writable: false))); + } + + return Task.FromResult(Result.Fail(new BlobNotFoundError(hash.ToString()))); + }); + + // Chunk write operation + var remainingFailures = throwOnWriteFailureCount; + mock.OpenWriteChunkAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any>(), Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + var compressionLevel = callInfo.ArgAt(1); + var contentType = callInfo.ArgAt(2); + var metadata = callInfo.ArgAt>(3); + var overwrite = callInfo.ArgAt(5); + + // Simulate write failure if configured + if ((throwOnWritePredicate is null || throwOnWritePredicate(hash)) && Interlocked.Decrement(ref remainingFailures) >= 0) + { + return Task.FromResult(Result.Fail(new ExceptionalError(new IOException("Simulated upload failure")))); + } + + if (!overwrite && (chunks.ContainsKey(hash) || writtenChunks.ContainsKey(hash))) + { + return Task.FromResult(Result.Fail(new BlobAlreadyExistsError(hash.ToString()))); + } + + var chunk = new FakeArchiveStorageChunk + { + ContentType = contentType, + CompressionLevel = compressionLevel, + Metadata = metadata != null ? new Dictionary(metadata, StringComparer.OrdinalIgnoreCase) : new Dictionary(StringComparer.OrdinalIgnoreCase), + StorageTier = StorageTier.Cool + }; + + var recordingStream = new RecordingMemoryStream(bytes => + { + chunk.Content = bytes; + chunk.ContentLength = bytes.LongLength; + writtenChunks[hash] = chunk; + chunks[hash] = chunk; + }); + + return Task.FromResult(Result.Ok(recordingStream)); + }); + + // Get chunk properties + mock.GetChunkPropertiesAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + + if (chunks.TryGetValue(hash, out var chunk)) + { + return Task.FromResult(new StorageProperties( + hash.ToString(), + chunk.ContentType, + chunk.Metadata.Count == 0 ? null : new Dictionary(chunk.Metadata, StringComparer.OrdinalIgnoreCase), + chunk.StorageTier, + chunk.ContentLength)); + } + + return Task.FromResult(null); + }); + + // Delete chunk + mock.DeleteChunkAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + chunks.Remove(hash); + hydratedChunks.Remove(hash); + writtenChunks.TryRemove(hash, out _); + return Task.CompletedTask; + }); + + // Set chunk metadata + mock.SetChunkMetadataAsync(Arg.Any(), Arg.Any>(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + var metadata = callInfo.ArgAt>(1); + + if (chunks.TryGetValue(hash, out var chunk)) + { + chunk.Metadata = new Dictionary(metadata, StringComparer.OrdinalIgnoreCase); + } + + return Task.CompletedTask; + }); + + // Set storage tier + mock.SetChunkStorageTierPerPolicy(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + var targetTier = callInfo.ArgAt(2); + + if (chunks.TryGetValue(hash, out var chunk)) + { + chunk.StorageTier = targetTier; + } + + return Task.FromResult(targetTier); + }); + + // Start hydration + mock.StartHydrationAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var hash = callInfo.Arg(); + + if (chunks.TryGetValue(hash, out var chunk)) + { + hydratedChunks[hash] = chunk; + } + + return Task.CompletedTask; + }); + + return mock; + } + + private sealed class RecordingMemoryStream : MemoryStream + { + private readonly Action onDispose; + + public RecordingMemoryStream(Action onDispose) => this.onDispose = onDispose; + + protected override void Dispose(bool disposing) + { + if (disposing) + { + onDispose(ToArray()); + } + + base.Dispose(disposing); + } + } + + internal sealed class FakeArchiveStorageChunk + { + public byte[] Content { get; set; } = []; + public long ContentLength { get; set; } + public string ContentType { get; set; } = string.Empty; + public Dictionary Metadata { get; set; } = new(StringComparer.OrdinalIgnoreCase); + public StorageTier StorageTier { get; set; } = StorageTier.Cool; + public CompressionLevel CompressionLevel { get; set; } + } +} \ No newline at end of file From 7a032c88c02f01751c8052a2442b9b1917caab45 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 20:32:13 +0200 Subject: [PATCH 08/43] chore: format --- .../Archive/ArchiveCommandHandlerTests.cs | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs index f4549891..64cc1dc7 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs @@ -28,21 +28,21 @@ public ArchiveCommandHandlerTests(FixtureWithFileSystem fixture) } - [Fact] - [Trait("Category", "SkipCI")] - public async Task RunArchiveCommandTEMP() // NOTE TEMP this one is skipped in CI via the SkipCI category - { - var logger = new FakeLogger(); - - // TODO Make this better - var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - var c = new ArchiveCommandBuilder(fixture) - .WithLocalRoot(isWindows ? - new DirectoryInfo("C:\\Users\\WouterVanRanst\\Downloads\\Photos-001 (1)") : - new DirectoryInfo("/mnt/c/Users/WouterVanRanst/Downloads/Photos-001 (1)")) - .Build(); - await handler.Handle(c, CancellationToken.None); - } + //[Fact] + //[Trait("Category", "SkipCI")] + //public async Task RunArchiveCommandTEMP() // NOTE TEMP this one is skipped in CI via the SkipCI category + //{ + // var logger = new FakeLogger(); + + // // TODO Make this better + // var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + // var c = new ArchiveCommandBuilder(fixture) + // .WithLocalRoot(isWindows ? + // new DirectoryInfo("C:\\Users\\WouterVanRanst\\Downloads\\Photos-001 (1)") : + // new DirectoryInfo("/mnt/c/Users/WouterVanRanst/Downloads/Photos-001 (1)")) + // .Build(); + // await handler.Handle(c, CancellationToken.None); + //} [Fact(Skip = "TODO")] public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() @@ -61,30 +61,32 @@ public async Task UploadIfNotExistsAsync_WhenChunkDoesNotExist_ShouldUpload() var handlerContext = await CreateHandlerContextAsync(); + // Act var result = await handler.UploadIfNotExistsAsync(handlerContext, hash, sourceStream, compressionLevel, expectedContentType, null, CancellationToken.None); + // Assert result.OriginalSize.ShouldBeGreaterThan(0); result.ArchivedSize.ShouldBeGreaterThan(0); - // Verify the blob was actually created with correct properties and metadata + // Verify the blob was actually created with correct properties and metadata var properties = await handlerContext.ArchiveStorage.GetChunkPropertiesAsync(hash, CancellationToken.None); properties.ShouldNotBeNull(); properties.ContentType.ShouldBe(expectedContentType); - // Verify metadata is read from storage and matches returned values + // Verify metadata is read from storage and matches returned values properties.Metadata.ShouldNotBeNull(); properties.Metadata.ShouldContainKey("OriginalContentLength"); properties.Metadata["OriginalContentLength"].ShouldBe(result.OriginalSize.ToString()); - // Verify correct contentlength + // Verify correct contentlength properties.ContentLength.ShouldBe(result.ArchivedSize); - // Verify Storage Tier + // Verify Storage Tier properties.StorageTier.ShouldBe(StorageTier.Cool); - // Verify the stream was read to the end (ie the binary was uploaded) + // Verify the stream was read to the end (ie the binary was uploaded) sourceStream.Position.ShouldBe(sourceStream.Length); } @@ -100,38 +102,40 @@ public async Task UploadIfNotExistsAsync_WhenValidChunkExists_ShouldNotUploadAga var handlerContext = await CreateHandlerContextAsync(); - // First upload to create the blob + // First upload to create the blob await handler.UploadIfNotExistsAsync(handlerContext, hash, sourceStream, compressionLevel, expectedContentType, null, CancellationToken.None); await handlerContext.ArchiveStorage.SetChunkStorageTierPerPolicy(hash, 0, StorageTier.Hot); // Set to Hot tier to check if the correct storage tier was applied afterwards - // Reset stream for second call + // Reset stream for second call sourceStream.Seek(0, SeekOrigin.Begin); + // Act - Second call should detect existing blob var result = await handler.UploadIfNotExistsAsync(handlerContext, hash, sourceStream, compressionLevel, expectedContentType, null, CancellationToken.None); + // Assert result.OriginalSize.ShouldBeGreaterThan(0); result.ArchivedSize.ShouldBeGreaterThan(0); - // Verify properties are still correct and metadata is read from storage + // Verify properties are still correct and metadata is read from storage var properties = await handlerContext.ArchiveStorage.GetChunkPropertiesAsync(hash, CancellationToken.None); properties.ShouldNotBeNull(); properties.ContentType.ShouldBe(expectedContentType); - // Verify metadata is read from storage and matches returned values + // Verify metadata is read from storage and matches returned values properties.Metadata.ShouldNotBeNull(); properties.Metadata.ShouldContainKey("OriginalContentLength"); properties.Metadata["OriginalContentLength"].ShouldBe(result.OriginalSize.ToString()); - // Verify correct contentlength + // Verify correct contentlength properties.ContentLength.ShouldBe(result.ArchivedSize); - // Verify Storage Tier + // Verify Storage Tier properties.StorageTier.ShouldBe(StorageTier.Cool); - // Verify the stream was NOT read (ie the binary was NOT uploaded again) + // Verify the stream was NOT read (ie the binary was NOT uploaded again) sourceStream.Position.ShouldBe(0); } @@ -147,43 +151,45 @@ public async Task UploadIfNotExistsAsync_WhenInvalidChunk_ShouldDeleteAndReUploa var handlerContext = await CreateHandlerContextAsync(); - // Create a blob with wrong content type using BlobClient directly (simulating corruption) + // Create a blob with wrong content type using BlobClient directly (simulating corruption) var blobServiceClient = new BlobServiceClient(new Uri($"https://{fixture.RepositoryOptions.AccountName}.blob.core.windows.net"), new Azure.Storage.StorageSharedKeyCredential(fixture.RepositoryOptions.AccountName, fixture.RepositoryOptions.AccountKey)); var containerClient = blobServiceClient.GetBlobContainerClient(fixture.RepositoryOptions.ContainerName); var blobClient = containerClient.GetBlobClient($"chunks/{hash}"); - // Upload blob without metadata + // Upload blob without metadata var uploadOptions = new BlobUploadOptions { HttpHeaders = new BlobHttpHeaders { ContentType = correctContentType } }; await blobClient.UploadAsync(new MemoryStream("corrupted content"u8.ToArray()), uploadOptions, CancellationToken.None); - // Reset source stream + // Reset source stream sourceStream.Seek(0, SeekOrigin.Begin); + // Act - Should detect wrong content type, delete, and re-upload var result = await handler.UploadIfNotExistsAsync(handlerContext, hash, sourceStream, compressionLevel, correctContentType, null, CancellationToken.None); + // Assert result.OriginalSize.ShouldBeGreaterThan(0); result.ArchivedSize.ShouldBeGreaterThan(0); - // Verify the blob now has correct content type and metadata + // Verify the blob now has correct content type and metadata var properties = await handlerContext.ArchiveStorage.GetChunkPropertiesAsync(hash, CancellationToken.None); properties.ShouldNotBeNull(); properties.ContentType.ShouldBe(correctContentType); - // Verify metadata is read from storage and matches returned values + // Verify metadata is read from storage and matches returned values properties.Metadata.ShouldNotBeNull(); properties.Metadata.ShouldContainKey("OriginalContentLength"); properties.Metadata["OriginalContentLength"].ShouldBe(result.OriginalSize.ToString()); - // Verify correct contentlength + // Verify correct contentlength properties.ContentLength.ShouldBe(result.ArchivedSize); - // Verify Storage Tier + // Verify Storage Tier properties.StorageTier.ShouldBe(StorageTier.Cool); - // Verify the stream was read to the end (ie the binary was uploaded again) + // Verify the stream was read to the end (ie the binary was uploaded again) sourceStream.Position.ShouldBe(sourceStream.Length); } @@ -212,12 +218,12 @@ public async Task UploadIfNotExistsAsync_WhenTarArchive_ShouldIncludeSmallChunkC //result.OriginalSize.ShouldBeGreaterThan(0); //result.ArchivedSize.ShouldBeGreaterThan(0); - // Verify the blob was created with correct properties and metadata + // Verify the blob was created with correct properties and metadata var properties = await handlerContext.ArchiveStorage.GetChunkPropertiesAsync(hash, CancellationToken.None); properties.ShouldNotBeNull(); //properties.ContentType.ShouldBe(expectedContentType); - // Verify metadata includes both OriginalContentLength and SmallChunkCount + // Verify metadata includes both OriginalContentLength and SmallChunkCount properties.Metadata.ShouldNotBeNull(); //properties.Metadata.ShouldContainKey("OriginalContentLength"); //properties.Metadata["OriginalContentLength"].ShouldBe(result.OriginalSize.ToString()); @@ -225,10 +231,10 @@ public async Task UploadIfNotExistsAsync_WhenTarArchive_ShouldIncludeSmallChunkC properties.Metadata.ShouldContainKey("SmallChunkCount"); properties.Metadata["SmallChunkCount"].ShouldBe(expectedChunkCount.ToString()); - // Verify correct contentlength + // Verify correct contentlength properties.ContentLength.ShouldBe(result.ArchivedSize); - // Verify Storage Tier + // Verify Storage Tier properties.StorageTier.ShouldBe(StorageTier.Cool); } From aecc3c3ab5208834d071346701eb83a804615118 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 20:43:53 +0200 Subject: [PATCH 09/43] fix: addtl cleanup --- src/Arius.Core.Tests/Utils.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Arius.Core.Tests/Utils.cs b/src/Arius.Core.Tests/Utils.cs index 69548e50..19eb9672 100644 --- a/src/Arius.Core.Tests/Utils.cs +++ b/src/Arius.Core.Tests/Utils.cs @@ -27,7 +27,8 @@ public void CleanupLocalTemp() { var cutoff = DateTime.UtcNow.AddDays(-2); - foreach (var dir in Directory.EnumerateDirectories(Path.GetTempPath(), "Arius.Core.Tests*")) + foreach (var dir in Directory.EnumerateDirectories(Path.GetTempPath(), "Arius.Core.Tests*").Union( + Directory.EnumerateDirectories(Path.GetTempPath(), "arius-*"))) { var info = new DirectoryInfo(dir); From 11a77f94034b8069b738d98c5b4ea926ae2e85d0 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 20:44:00 +0200 Subject: [PATCH 10/43] chore --- .../Features/Commands/Archive/ArchiveCommandHandlerTests.cs | 3 ++- src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs index 64cc1dc7..01193693 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging.Testing; using Shouldly; using System.IO.Compression; -using System.Runtime.InteropServices; using System.Text; namespace Arius.Core.Tests.Features.Commands.Archive; @@ -49,6 +48,8 @@ public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() { } + // --- UPLOADIFNOTEXIST + [Fact] public async Task UploadIfNotExistsAsync_WhenChunkDoesNotExist_ShouldUpload() { diff --git a/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs b/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs index d86d4fc1..6d5bbbf1 100644 --- a/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs +++ b/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs @@ -37,7 +37,7 @@ public class FixtureWithFileSystem : Fixture, IDisposable public IFileSystem FileSystem { get; } public DirectoryInfo TestRunSourceFolder { get; } - public FixtureWithFileSystem() : base() + public FixtureWithFileSystem() { TestRunSourceFolder = Directory.CreateTempSubdirectory($"arius-core-tests-{DateTime.Now:yyyyMMddTHHmmss}_{Guid.CreateVersion7()}"); TestRunSourceFolder.Create(); From 94964a8ffd6c9084411bf5b3a77585ed7b1e06a9 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 20:54:46 +0200 Subject: [PATCH 11/43] chore: rename --- ...lerTests.cs => ArchiveCommandHandlerUploadIfNotExistsTests.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Arius.Core.Tests/Features/Commands/Archive/{ArchiveCommandHandlerTests.cs => ArchiveCommandHandlerUploadIfNotExistsTests.cs} (100%) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs similarity index 100% rename from src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerTests.cs rename to src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs From f853f6b4547e2e5b052f39ec59a70e7cff0298a2 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:01:40 +0200 Subject: [PATCH 12/43] chore: rename --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 5 +++++ .../ArchiveCommandHandlerUploadIfNotExistsTests.cs | 9 ++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index f82ed730..b78f23b8 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -91,6 +91,11 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic return (command, handlerContext, storageBuilder, loggerFactory); } + [Fact(Skip = "TODO")] + public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() + { + } + [Fact] public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() { diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs index 01193693..e8b8cb53 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs @@ -13,13 +13,13 @@ namespace Arius.Core.Tests.Features.Commands.Archive; -public class ArchiveCommandHandlerTests : IClassFixture +public class ArchiveCommandHandlerUploadIfNotExistsTests : IClassFixture { private readonly FixtureWithFileSystem fixture; private readonly FakeLogger logger; private readonly ArchiveCommandHandler handler; - public ArchiveCommandHandlerTests(FixtureWithFileSystem fixture) + public ArchiveCommandHandlerUploadIfNotExistsTests(FixtureWithFileSystem fixture) { this.fixture = fixture; logger = new(); @@ -43,12 +43,7 @@ public ArchiveCommandHandlerTests(FixtureWithFileSystem fixture) // await handler.Handle(c, CancellationToken.None); //} - [Fact(Skip = "TODO")] - public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() - { - } - // --- UPLOADIFNOTEXIST [Fact] public async Task UploadIfNotExistsAsync_WhenChunkDoesNotExist_ShouldUpload() From d5eb7c1452deb022719444bb2adc792fd5d102a4 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:02:09 +0200 Subject: [PATCH 13/43] chore: remove unused test --- ...hiveCommandHandlerUploadIfNotExistsTests.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs index e8b8cb53..12c4554b 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerUploadIfNotExistsTests.cs @@ -27,24 +27,6 @@ public ArchiveCommandHandlerUploadIfNotExistsTests(FixtureWithFileSystem fixture } - //[Fact] - //[Trait("Category", "SkipCI")] - //public async Task RunArchiveCommandTEMP() // NOTE TEMP this one is skipped in CI via the SkipCI category - //{ - // var logger = new FakeLogger(); - - // // TODO Make this better - // var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - // var c = new ArchiveCommandBuilder(fixture) - // .WithLocalRoot(isWindows ? - // new DirectoryInfo("C:\\Users\\WouterVanRanst\\Downloads\\Photos-001 (1)") : - // new DirectoryInfo("/mnt/c/Users/WouterVanRanst/Downloads/Photos-001 (1)")) - // .Build(); - // await handler.Handle(c, CancellationToken.None); - //} - - - [Fact] public async Task UploadIfNotExistsAsync_WhenChunkDoesNotExist_ShouldUpload() { From e08eec1426ff4ef3a1db1d69edcf3d140bc70c28 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:18:30 +0200 Subject: [PATCH 14/43] feat: remove ResetTestRoot --- .../ArchiveCommandHandlerHandleTests.cs | 53 +++---------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index b78f23b8..ac672654 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -1,30 +1,27 @@ using Arius.Core.Features.Commands.Archive; using Arius.Core.Shared.FileSystem; -using Arius.Core.Shared.Hashing; using Arius.Core.Shared.StateRepositories; using Arius.Core.Shared.Storage; using Arius.Core.Tests.Helpers.Builders; using Arius.Core.Tests.Helpers.FakeLogger; using Arius.Core.Tests.Helpers.Fakes; using Arius.Core.Tests.Helpers.Fixtures; -using FluentResults; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Testing; using Shouldly; -using System.IO.Compression; using Zio; namespace Arius.Core.Tests.Features.Commands.Archive; -public class ArchiveCommandHandlerHandleTests : IClassFixture +public class ArchiveCommandHandlerHandleTests { private readonly FixtureWithFileSystem fixture; private readonly FakeLogger logger; private readonly ArchiveCommandHandler handler; - public ArchiveCommandHandlerHandleTests(FixtureWithFileSystem fixture) + public ArchiveCommandHandlerHandleTests() { - this.fixture = fixture; + this.fixture = new(); logger = new FakeLogger(); handler = new ArchiveCommandHandler(logger, NullLoggerFactory.Instance, fixture.AriusConfiguration); } @@ -35,20 +32,6 @@ public ArchiveCommandHandlerHandleTests(FixtureWithFileSystem fixture) private static string ToAbsolutePointerPath(FixtureWithFileSystem fixture, UPath binaryPath) => Path.Combine(fixture.TestRunSourceFolder.FullName, binaryPath.GetPointerFilePath().ToString().TrimStart('/')); - private static void ResetTestRoot(FixtureWithFileSystem fixture, string containerName) - { - foreach (var directory in Directory.EnumerateDirectories(fixture.TestRunSourceFolder.FullName)) - Directory.Delete(directory, recursive: true); - - foreach (var file in Directory.EnumerateFiles(fixture.TestRunSourceFolder.FullName)) - File.Delete(file); - - var stateCacheRoot = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Arius", "statecache", fixture.RepositoryOptions?.AccountName ?? "testaccount", containerName); - - if (Directory.Exists(stateCacheRoot)) - Directory.Delete(stateCacheRoot, recursive: true); - } - private static async Task BuildHandlerContextAsync(ArchiveCommand command, IArchiveStorage archiveStorage, FakeLoggerFactory loggerFactory) => await new HandlerContextBuilder(command, loggerFactory) .WithArchiveStorage(archiveStorage) @@ -101,7 +84,6 @@ public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() { // Arrange var containerName = $"test-container-large-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var binaryPath = UPath.Root / "documents" / "presentation.pptx"; var largeFile = new FakeFileBuilder(fixture) @@ -156,7 +138,6 @@ public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBina { // Arrange var containerName = $"test-container-small-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var binaryPath = UPath.Root / "notes" / "small.txt"; var smallFile = new FakeFileBuilder(fixture) @@ -218,7 +199,6 @@ public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() { // Arrange var containerName = $"test-container-empty-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var binaryPath = UPath.Root / "empty" / "file.bin"; var emptyFile = new FakeFileBuilder(fixture) @@ -269,7 +249,6 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac { // Arrange var containerName = $"test-container-existing-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var binaryPath = UPath.Root / "existing" / "document.pdf"; var binaryWithPointer = new FakeFileBuilder(fixture) @@ -325,7 +304,6 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches { // Arrange var containerName = $"test-container-mixed-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var smallFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "small.txt") @@ -395,7 +373,6 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre { // Arrange var containerName = $"test-container-duplicates-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var originalLargeFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "shared.bin") @@ -465,7 +442,6 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu { // Arrange var containerName = $"test-container-small-batch-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var paths = new[] { @@ -519,7 +495,6 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary { // Arrange var containerName = $"test-container-multi-tar-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var paths = new[] { @@ -576,7 +551,6 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite { // Arrange var containerName = $"test-container-small-duplicates-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var ownerAlpha = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "alpha-owner.txt") @@ -645,7 +619,6 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() { // Arrange var containerName = $"test-container-incremental-existing-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var binaryPath = UPath.Root / "incremental" / "presentation.pptx"; var largeFile = new FakeFileBuilder(fixture) @@ -703,7 +676,6 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() { // Arrange var containerName = $"test-container-incremental-mixed-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var existingFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "existing.pdf") @@ -754,7 +726,6 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry { // Arrange var containerName = $"test-container-incremental-deleted-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var deletedFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "to-delete.txt") @@ -799,7 +770,6 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina { // Arrange var containerName = $"test-container-incremental-modified-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var filePath = UPath.Root / "docs" / "mutable.bin"; var originalFile = new FakeFileBuilder(fixture) @@ -856,7 +826,6 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState { // Arrange var containerName = $"test-container-incremental-nochanges-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var baselineFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "baseline.txt") @@ -896,7 +865,6 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() { // Arrange var containerName = $"test-container-error-cancel-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large1.bin") @@ -929,7 +897,6 @@ public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() { // Arrange var containerName = $"test-container-error-hash-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); var failingPath = UPath.Root / "hash" / "will-fail.bin"; _ = new FakeFileBuilder(fixture) @@ -993,7 +960,6 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() { // Arrange var containerName = $"test-container-error-upload-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "uploads" / "large.bin") @@ -1019,7 +985,6 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() { // Arrange var containerName = $"test-container-error-multi-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "multi" / "large.bin") @@ -1050,7 +1015,6 @@ public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() { // Arrange var containerName = $"test-container-error-pointer-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.PointerFileOnly, UPath.Root / "orphans" / "lonely.bin") @@ -1090,7 +1054,6 @@ public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() { // Arrange var containerName = $"test-container-stale-{Guid.CreateVersion7()}"; - ResetTestRoot(fixture, containerName); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "active.txt") @@ -1104,16 +1067,16 @@ public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() var staleHash = FakeHashBuilder.GenerateValidHash(99); handlerContext.StateRepository.AddBinaryProperties(new BinaryProperties { - Hash = staleHash, + Hash = staleHash, OriginalSize = 1, ArchivedSize = 1, - StorageTier = StorageTier.Cool + StorageTier = StorageTier.Cool }); handlerContext.StateRepository.UpsertPointerFileEntries(new PointerFileEntry { - Hash = staleHash, - RelativeName = "/stale.bin.pointer.arius", - CreationTimeUtc = DateTime.UtcNow, + Hash = staleHash, + RelativeName = "/stale.bin.pointer.arius", + CreationTimeUtc = DateTime.UtcNow, LastWriteTimeUtc = DateTime.UtcNow }); From 84192c97de3e10835cf8d055916cde157d9a51c8 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:24:48 +0200 Subject: [PATCH 15/43] feat: remove containername --- .../ArchiveCommandHandlerHandleTests.cs | 116 +++++------------- 1 file changed, 33 insertions(+), 83 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index ac672654..7cf494c9 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -49,18 +49,14 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic return (entries.Count, existingPointerFileCount); } - private async Task<(ArchiveCommand Command, HandlerContext Context, MockArchiveStorageBuilder StorageBuilder, FakeLoggerFactory LoggerFactory)> - CreateHandlerContextAsync( - string containerName, - Action? configureCommand = null, - MockArchiveStorageBuilder? storageBuilder = null, - FakeLoggerFactory? loggerFactory = null) + private async Task<(ArchiveCommand Command, HandlerContext Context, MockArchiveStorageBuilder StorageBuilder, FakeLoggerFactory LoggerFactory)> CreateHandlerContextAsync(Action? configureCommand = null, + MockArchiveStorageBuilder? storageBuilder = null, + FakeLoggerFactory? loggerFactory = null) { storageBuilder ??= new MockArchiveStorageBuilder(fixture); loggerFactory ??= new FakeLoggerFactory(); var commandBuilder = new ArchiveCommandBuilder(fixture) - .WithContainerName(containerName) .WithSmallFileBoundary(DefaultSmallFileBoundary) .WithHashingParallelism(1) .WithUploadParallelism(1); @@ -83,15 +79,13 @@ public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() { // Arrange - var containerName = $"test-container-large-{Guid.CreateVersion7()}"; - var binaryPath = UPath.Root / "documents" / "presentation.pptx"; var largeFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) .WithRandomContent(4096, seed: 10) .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -137,15 +131,13 @@ public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBinaryProperties() { // Arrange - var containerName = $"test-container-small-{Guid.CreateVersion7()}"; - var binaryPath = UPath.Root / "notes" / "small.txt"; var smallFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) .WithRandomContent(512, seed: 2) .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -198,14 +190,12 @@ public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBina public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() { // Arrange - var containerName = $"test-container-empty-{Guid.CreateVersion7()}"; - var binaryPath = UPath.Root / "empty" / "file.bin"; var emptyFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -248,8 +238,6 @@ public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrackExistingCount() { // Arrange - var containerName = $"test-container-existing-{Guid.CreateVersion7()}"; - var binaryPath = UPath.Root / "existing" / "document.pdf"; var binaryWithPointer = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) @@ -261,7 +249,7 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac var stalePointer = binaryWithPointer.FilePair.CreatePointerFile(staleHash); stalePointer.ReadHash().ShouldBe(staleHash); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); @@ -303,8 +291,6 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches() { // Arrange - var containerName = $"test-container-mixed-{Guid.CreateVersion7()}"; - var smallFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "small.txt") .WithRandomContent(512, seed: 1) @@ -318,13 +304,11 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches var progressUpdates = new List(); var progressReporter = new Progress(progressUpdates.Add); - var (command, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( - containerName, - builder => builder - .WithProgressReporter(progressReporter) - .WithHashingParallelism(1) - .WithUploadParallelism(1) - .WithSmallFileBoundary(DefaultSmallFileBoundary)); + var (command, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder + .WithProgressReporter(progressReporter) + .WithHashingParallelism(1) + .WithUploadParallelism(1) + .WithSmallFileBoundary(DefaultSmallFileBoundary)); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -372,8 +356,6 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCreateMultiplePointers() { // Arrange - var containerName = $"test-container-duplicates-{Guid.CreateVersion7()}"; - var originalLargeFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "shared.bin") .WithRandomContent(4096, seed: 42) @@ -392,7 +374,7 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre .WithDuplicate(originalSmallFile, UPath.Root / "texts" / "archive" / "note-copy.txt") .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -441,8 +423,6 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChunk() { // Arrange - var containerName = $"test-container-small-batch-{Guid.CreateVersion7()}"; - var paths = new[] { UPath.Root / "tar" / "alpha.txt", @@ -457,7 +437,7 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu .Build()) .ToArray(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -494,8 +474,6 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundaryExceeded() { // Arrange - var containerName = $"test-container-multi-tar-{Guid.CreateVersion7()}"; - var paths = new[] { UPath.Root / "tar" / "alpha.bin", @@ -512,7 +490,7 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary .Build()) .ToArray(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -550,8 +528,6 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWriteDeferredPointers() { // Arrange - var containerName = $"test-container-small-duplicates-{Guid.CreateVersion7()}"; - var ownerAlpha = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "alpha-owner.txt") .WithRandomContent(900, seed: 11) @@ -575,7 +551,7 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite .WithDuplicate(ownerOmega, UPath.Root / "tar" / "zzz-duplicate.txt") .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); @@ -618,8 +594,6 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() { // Arrange - var containerName = $"test-container-incremental-existing-{Guid.CreateVersion7()}"; - var binaryPath = UPath.Root / "incremental" / "presentation.pptx"; var largeFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, binaryPath) @@ -628,8 +602,8 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); - var (initialFileCount, _) = GetInitialFileStatistics(initialContext); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var (initialFileCount, _) = GetInitialFileStatistics(initialContext); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); @@ -641,7 +615,7 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() staleHash.ShouldNotBe(originalHash); largeFile.FilePair.CreatePointerFile(staleHash); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -675,8 +649,6 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() { // Arrange - var containerName = $"test-container-incremental-mixed-{Guid.CreateVersion7()}"; - var existingFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "existing.pdf") .WithRandomContent(4096, seed: 2001) @@ -684,7 +656,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); @@ -693,7 +665,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() .WithRandomContent(512, seed: 2002) .Build(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -725,8 +697,6 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry() { // Arrange - var containerName = $"test-container-incremental-deleted-{Guid.CreateVersion7()}"; - var deletedFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "to-delete.txt") .WithRandomContent(2048, seed: 3001) @@ -734,14 +704,14 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt")); File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt.pointer.arius")); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -769,8 +739,6 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBinaryProperties() { // Arrange - var containerName = $"test-container-incremental-modified-{Guid.CreateVersion7()}"; - var filePath = UPath.Root / "docs" / "mutable.bin"; var originalFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, filePath) @@ -779,7 +747,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); @@ -792,7 +760,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina .WithRandomContent(3584, seed: 4002) .Build(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -825,8 +793,6 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState() { // Arrange - var containerName = $"test-container-incremental-nochanges-{Guid.CreateVersion7()}"; - var baselineFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "baseline.txt") .WithRandomContent(2048, seed: 5001) @@ -834,11 +800,11 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var firstResult = await handler.Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act @@ -864,8 +830,6 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState public async Task Error_CancellationByUser_ShouldReturnFailureResult() { // Arrange - var containerName = $"test-container-error-cancel-{Guid.CreateVersion7()}"; - _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large1.bin") .WithRandomContent(4096, seed: 6001) @@ -876,7 +840,7 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() .WithRandomContent(4096, seed: 6002) .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); using var cts = new CancellationTokenSource(); cts.Cancel(); @@ -896,8 +860,6 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() { // Arrange - var containerName = $"test-container-error-hash-{Guid.CreateVersion7()}"; - var failingPath = UPath.Root / "hash" / "will-fail.bin"; _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, failingPath) @@ -926,9 +888,7 @@ void HandleProgress(ProgressUpdate update) } } - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( - containerName, - builder => builder.WithProgressReporter(new Progress(HandleProgress))); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(HandleProgress))); var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); @@ -959,8 +919,6 @@ void HandleProgress(ProgressUpdate update) public async Task Error_UploadTaskFails_ShouldReturnFailure() { // Arrange - var containerName = $"test-container-error-upload-{Guid.CreateVersion7()}"; - _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "uploads" / "large.bin") .WithRandomContent(4096, seed: 6201) @@ -969,7 +927,7 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() var storageBuilder = new MockArchiveStorageBuilder(fixture) .WithThrowOnWrite(failureCount: 1); - var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); // Act var result = await handler.Handle(handlerContext, CancellationToken.None); @@ -984,8 +942,6 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() { // Arrange - var containerName = $"test-container-error-multi-{Guid.CreateVersion7()}"; - _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "multi" / "large.bin") .WithRandomContent(4096, seed: 6301) @@ -999,7 +955,7 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() var storageBuilder = new MockArchiveStorageBuilder(fixture) .WithThrowOnWrite(failureCount: 2); - var (_, handlerContext, _, _) = await CreateHandlerContextAsync(containerName, storageBuilder: storageBuilder); + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); // Act var result = await handler.Handle(handlerContext, CancellationToken.None); @@ -1014,16 +970,12 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() { // Arrange - var containerName = $"test-container-error-pointer-{Guid.CreateVersion7()}"; - _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.PointerFileOnly, UPath.Root / "orphans" / "lonely.bin") .Build(); var progressUpdates = new List(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync( - containerName, - builder => builder.WithProgressReporter(new Progress(progressUpdates.Add))); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(progressUpdates.Add))); var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); @@ -1053,14 +1005,12 @@ public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() { // Arrange - var containerName = $"test-container-stale-{Guid.CreateVersion7()}"; - _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "active.txt") .WithRandomContent(256, seed: 7) .Build(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(containerName); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); From 04d0bb45fb6ed39409cbf9070168c1bcd2c7c49f Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:40:39 +0200 Subject: [PATCH 16/43] feat: enforce single use of command handlers --- .../Commands/Archive/ArchiveCommandHandler.cs | 16 +++++----------- .../Commands/Restore/RestoreCommandHandler.cs | 14 +++++--------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index f68fe48d..7397c622 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -23,6 +23,7 @@ internal class ArchiveCommandHandler : ICommandHandler logger; private readonly ILoggerFactory loggerFactory; + private int used; public ArchiveCommandHandler(ILogger logger, ILoggerFactory loggerFactory, IOptions config) { @@ -72,18 +73,11 @@ public async ValueTask> Handle(ArchiveCommand reque internal async ValueTask> Handle(HandlerContext handlerContext, CancellationToken cancellationToken) { - logger.LogInformation("Starting archive operation for path {LocalRoot} with hashing parallelism {HashingParallelism}, upload parallelism {UploadParallelism}", handlerContext.Request.LocalRoot, handlerContext.Request.HashingParallelism, handlerContext.Request.UploadParallelism); - - // Reset statistics for this operation - totalLocalFiles = 0; - existingPointerFiles = 0; + // Enforce single-use + if (Interlocked.Exchange(ref used, 1) != 0) + throw new InvalidOperationException($"{nameof(ArchiveCommandHandler)} can only be used once."); - uniqueBinariesUploaded = 0; - uniqueChunksUploaded = 0; - bytesUploadedUncompressed = 0; - bytesUploadedCompressed = 0; - pointerFilesCreated = 0; - pointerFileEntriesDeleted = 0; + logger.LogInformation("Starting archive operation for path {LocalRoot} with hashing parallelism {HashingParallelism}, upload parallelism {UploadParallelism}", handlerContext.Request.LocalRoot, handlerContext.Request.HashingParallelism, handlerContext.Request.UploadParallelism); using var errorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var errorCancellationToken = errorCancellationTokenSource.Token; diff --git a/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs b/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs index dd19fbff..83e5610f 100644 --- a/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs @@ -25,6 +25,7 @@ internal class RestoreCommandHandler : ICommandHandler logger; private readonly ILoggerFactory loggerFactory; + private int used; public RestoreCommandHandler(ILogger logger, ILoggerFactory loggerFactory, IOptions config) { @@ -57,16 +58,11 @@ public async ValueTask> Handle(RestoreCommand reque internal async ValueTask> Handle(HandlerContext handlerContext, CancellationToken cancellationToken) { - logger.LogInformation("Starting restore operation for {TargetCount} targets with hashing parallelism {HashParallelism}, download parallelism {DownloadParallelism}", handlerContext.Targets.Length, handlerContext.Request.HashParallelism, handlerContext.Request.DownloadParallelism); - - // Reset statistics for this operation - totalTargetFiles = 0; - verifiedFilesAlreadyExisting = 0; - chunksDownloaded = 0; - bytesDownloaded = 0; - filesWrittenToDisk = 0; - bytesWrittenToDisk = 0; + // Enforce single-use + if (Interlocked.Exchange(ref used, 1) != 0) + throw new InvalidOperationException($"{nameof(RestoreCommandHandler)} can only be used once."); + logger.LogInformation("Starting restore operation for {TargetCount} targets with hashing parallelism {HashParallelism}, download parallelism {DownloadParallelism}", handlerContext.Targets.Length, handlerContext.Request.HashParallelism, handlerContext.Request.DownloadParallelism); using var errorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var errorCancellationToken = errorCancellationTokenSource.Token; From aa9a2f27ec4312338b8e3fd699ebbe8700b59169 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 21:47:05 +0200 Subject: [PATCH 17/43] chore: inline handlercontextbuilder --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 7cf494c9..afc62113 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -32,11 +32,6 @@ public ArchiveCommandHandlerHandleTests() private static string ToAbsolutePointerPath(FixtureWithFileSystem fixture, UPath binaryPath) => Path.Combine(fixture.TestRunSourceFolder.FullName, binaryPath.GetPointerFilePath().ToString().TrimStart('/')); - private static async Task BuildHandlerContextAsync(ArchiveCommand command, IArchiveStorage archiveStorage, FakeLoggerFactory loggerFactory) => - await new HandlerContextBuilder(command, loggerFactory) - .WithArchiveStorage(archiveStorage) - .BuildAsync(); - private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistics(HandlerContext handlerContext) { var entries = handlerContext.FileSystem @@ -65,7 +60,9 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic var command = commandBuilder.Build(); var archiveStorage = storageBuilder.Build(); - var handlerContext = await BuildHandlerContextAsync(command, archiveStorage, loggerFactory); + var handlerContext = await new HandlerContextBuilder(command, loggerFactory) + .WithArchiveStorage(archiveStorage) + .BuildAsync(); return (command, handlerContext, storageBuilder, loggerFactory); } From cbb42c129d10dc036412984455504ee7147ed6e5 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 29 Oct 2025 14:30:28 +0100 Subject: [PATCH 18/43] feat: bump nugets --- src/Directory.Packages.props | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e1288e8b..81c2efcf 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -25,17 +25,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -43,7 +43,7 @@ - + From f7c6f5b64e5e1504e997858c0d8fc9edae599b7c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 29 Oct 2025 17:08:21 +0100 Subject: [PATCH 19/43] fix: state file name collision when two handlercontexts are created in rapid succession --- .../ArchiveCommandHandlerHandleTests.cs | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index afc62113..01281259 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -60,11 +60,23 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic var command = commandBuilder.Build(); var archiveStorage = storageBuilder.Build(); - var handlerContext = await new HandlerContextBuilder(command, loggerFactory) - .WithArchiveStorage(archiveStorage) - .BuildAsync(); - return (command, handlerContext, storageBuilder, loggerFactory); + Retry: + try + { + var handlerContext = await new HandlerContextBuilder(command, loggerFactory) + .WithArchiveStorage(archiveStorage) + .BuildAsync(); + + return (command, handlerContext, storageBuilder, loggerFactory); + } + catch (IOException) + { + await Task.Delay(100); // Delay until the statefile name ("yyyy-MM-ddTHH-mm-ss") is in different seconds + goto Retry; + } + + } [Fact(Skip = "TODO")] @@ -571,7 +583,7 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite tarChunks.Count.ShouldBe(2); tarChunks.Select(c => int.Parse(c.Metadata["SmallChunkCount"])) .OrderBy(v => v) - .ShouldBe(new[] { 1, 2 }); + .ShouldBe([1, 2]); var duplicatePaths = new[] { From 922a5889809f73edbadeb61c6a362e9647d7aaa8 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 29 Oct 2025 17:18:20 +0100 Subject: [PATCH 20/43] feat: refactored for single use handler --- .../ArchiveCommandHandlerHandleTests.cs | 58 +++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 01281259..3692742c 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -15,17 +15,15 @@ namespace Arius.Core.Tests.Features.Commands.Archive; public class ArchiveCommandHandlerHandleTests { - private readonly FixtureWithFileSystem fixture; - private readonly FakeLogger logger; - private readonly ArchiveCommandHandler handler; + private readonly FixtureWithFileSystem fixture; public ArchiveCommandHandlerHandleTests() { this.fixture = new(); - logger = new FakeLogger(); - handler = new ArchiveCommandHandler(logger, NullLoggerFactory.Instance, fixture.AriusConfiguration); } + private ArchiveCommandHandler CreateHandler() => new(new FakeLogger(), NullLoggerFactory.Instance, fixture.AriusConfiguration); + private const int DefaultSmallFileBoundary = 1024; private static string ToRelativePointerPath(UPath binaryPath) => binaryPath.GetPointerFilePath().ToString(); @@ -99,7 +97,7 @@ public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -151,7 +149,7 @@ public async Task Single_SmallFile_FirstUpload_ShouldCreateTarParentAndChildBina var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -209,7 +207,7 @@ public async Task Single_EmptyFile_ShouldUploadZeroLengthBinary() var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -263,7 +261,7 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac var (expectedInitialFileCount, expectedExistingPointerFile) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -322,7 +320,7 @@ public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -388,7 +386,7 @@ public async Task Multiple_WithDuplicates_InSameRun_ShouldUploadBinaryOnceAndCre var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -451,7 +449,7 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -504,7 +502,7 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -565,7 +563,7 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite var (expectedInitialFileCount, _) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -614,7 +612,7 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (initialFileCount, _) = GetInitialFileStatistics(initialContext); - var firstResult = await handler.Handle(initialContext, CancellationToken.None); + var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); var originalHash = largeFile.OriginalHash; @@ -628,7 +626,7 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert incrementalResult.IsSuccess.ShouldBeTrue(); @@ -666,7 +664,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() var storageBuilder = new MockArchiveStorageBuilder(fixture); var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await handler.Handle(initialContext, CancellationToken.None); + var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); var newSmallFile = new FakeFileBuilder(fixture) @@ -678,7 +676,7 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert incrementalResult.IsSuccess.ShouldBeTrue(); @@ -714,7 +712,7 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry var storageBuilder = new MockArchiveStorageBuilder(fixture); var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await handler.Handle(initialContext, CancellationToken.None); + var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt")); @@ -724,7 +722,7 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert incrementalResult.IsSuccess.ShouldBeTrue(); @@ -757,7 +755,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina var storageBuilder = new MockArchiveStorageBuilder(fixture); var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await handler.Handle(initialContext, CancellationToken.None); + var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); var originalHash = originalFile.OriginalHash; @@ -773,7 +771,7 @@ public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBina var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert incrementalResult.IsSuccess.ShouldBeTrue(); @@ -810,14 +808,14 @@ public async Task Incremental_NoChanges_ShouldSkipStateUploadAndDeleteLocalState var storageBuilder = new MockArchiveStorageBuilder(fixture); var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await handler.Handle(initialContext, CancellationToken.None); + var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); firstResult.IsSuccess.ShouldBeTrue(); var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await handler.Handle(incrementalContext, CancellationToken.None); + var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert incrementalResult.IsSuccess.ShouldBeTrue(); @@ -855,7 +853,7 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() cts.Cancel(); // Act - var result = await handler.Handle(handlerContext, cts.Token); + var result = await CreateHandler().Handle(handlerContext, cts.Token); // Assert result.IsFailed.ShouldBeTrue(); @@ -902,7 +900,7 @@ void HandleProgress(ProgressUpdate update) var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -939,7 +937,7 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() var (_, handlerContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsFailed.ShouldBeTrue(); @@ -967,7 +965,7 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() var (_, handlerContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsFailed.ShouldBeTrue(); @@ -989,7 +987,7 @@ public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); @@ -1040,7 +1038,7 @@ public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() }); // Act - var result = await handler.Handle(handlerContext, CancellationToken.None); + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); // Assert result.IsSuccess.ShouldBeTrue(); From 959ccb5fea65f03524493a76188ffbbfaddd63eb Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 29 Oct 2025 21:56:56 +0100 Subject: [PATCH 21/43] feat: fix Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundaryExceeded --- .../Archive/ArchiveCommandHandlerHandleTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 3692742c..b282b9aa 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -488,12 +488,12 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary UPath.Root / "tar" / "gamma.bin" }; - var sizes = new[] { 900, 900, 500 }; + var sizes = new[] { 600, 600, 600 }; // note: esp. for small binaries the TAR overhead is substantial var smallFiles = paths .Select((path, index) => new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, path) - .WithRandomContent(sizes[index], seed: 200 + index) + .WithRandomContent(sizes[index], seed: index) .Build()) .ToArray(); @@ -521,13 +521,13 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary tarChunks.Count.ShouldBe(2); tarChunks.Select(c => int.Parse(c.Metadata["SmallChunkCount"])) .OrderBy(v => v) - .ShouldBe(new[] { 1, 2 }); + .ShouldBe(new[] { 1, 2 }); // we expect a TAR with one small chunk and one with two small chunks foreach (var path in paths) { - File.Exists(ToAbsolutePointerPath(fixture, path)).ShouldBeTrue(); + File.Exists(ToAbsolutePointerPath(fixture, path)).ShouldBeTrue(); // the local binary still exists handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(path), includeBinaryProperties: true) - .ShouldNotBeNull(); + .ShouldNotBeNull(); // the pointerfileentry has been saved } } From e0a63f55906c50f0dd4f998087121ec8fa02be9c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 07:44:45 +0100 Subject: [PATCH 22/43] feat: fix Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWriteDeferredPointers --- .../ArchiveCommandHandlerHandleTests.cs | 56 ++++++++++--------- .../Helpers/Fixtures/Fixture.cs | 2 +- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index b282b9aa..a64a17f9 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -480,6 +480,8 @@ public async Task Multiple_SmallFiles_SingleTarBatch_ShouldUploadSingleParentChu [Fact] public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundaryExceeded() { + // We will hit the UploadSmallFileAsync / OWNER path with this test + // Arrange var paths = new[] { @@ -510,7 +512,7 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); summary.UniqueBinariesUploaded.ShouldBe(paths.Length); - summary.UniqueChunksUploaded.ShouldBe(2); + summary.UniqueChunksUploaded.ShouldBe(2); // <-- the Tar Writer flushed when the boundary exceeded summary.PointerFilesCreated.ShouldBe(paths.Length); summary.PointerFileEntriesDeleted.ShouldBe(0); summary.BytesUploadedUncompressed.ShouldBe(smallFiles.Sum(f => f.OriginalContent.Length)); @@ -534,28 +536,28 @@ public async Task Multiple_SmallFiles_MultipleTarBatches_ShouldFlushWhenBoundary [Fact] public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWriteDeferredPointers() { + // We will hit the UploadSmallFileAsync / NON-OWNER path with this test + // Arrange - var ownerAlpha = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "alpha-owner.txt") - .WithRandomContent(900, seed: 11) + var f10 = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "alpha.txt") + .WithRandomContent(600, seed: 1) .Build(); - - var ownerBeta = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "beta-owner.txt") - .WithRandomContent(920, seed: 12) + var f11 = new FakeFileBuilder(fixture) + .WithDuplicate(f10, UPath.Root / "tar" / "alpha-duplicate.txt") .Build(); - _ = new FakeFileBuilder(fixture) - .WithDuplicate(ownerAlpha, UPath.Root / "tar" / "gamma-duplicate.txt") + var f20 = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "beta.txt") + .WithRandomContent(600, seed: 2) .Build(); - var ownerOmega = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "omega-owner.txt") - .WithRandomContent(480, seed: 13) + var f30 = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "tar" / "omega.txt") + .WithRandomContent(600, seed: 3) .Build(); - - _ = new FakeFileBuilder(fixture) - .WithDuplicate(ownerOmega, UPath.Root / "tar" / "zzz-duplicate.txt") + var f31 = new FakeFileBuilder(fixture) + .WithDuplicate(f30, UPath.Root / "tar" / "omega-duplicate.txt") .Build(); var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); @@ -583,18 +585,18 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite .OrderBy(v => v) .ShouldBe([1, 2]); - var duplicatePaths = new[] - { - UPath.Root / "tar" / "gamma-duplicate.txt", - UPath.Root / "tar" / "zzz-duplicate.txt" - }; - foreach (var path in duplicatePaths) - { - File.Exists(ToAbsolutePointerPath(fixture, path)).ShouldBeTrue(); - handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(path), includeBinaryProperties: true) - .ShouldNotBeNull(); - } + f10.FilePair.BinaryFile.Exists.ShouldBeTrue(); + f11.FilePair.BinaryFile.Exists.ShouldBeTrue(); + f20.FilePair.BinaryFile.Exists.ShouldBeTrue(); + f30.FilePair.BinaryFile.Exists.ShouldBeTrue(); + f31.FilePair.BinaryFile.Exists.ShouldBeTrue(); + + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f10.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f11.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f20.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f30.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); + handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f31.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); } [Fact] diff --git a/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs b/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs index 6d5bbbf1..b8df0d89 100644 --- a/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs +++ b/src/Arius.Core.Tests/Helpers/Fixtures/Fixture.cs @@ -39,7 +39,7 @@ public class FixtureWithFileSystem : Fixture, IDisposable public FixtureWithFileSystem() { - TestRunSourceFolder = Directory.CreateTempSubdirectory($"arius-core-tests-{DateTime.Now:yyyyMMddTHHmmss}_{Guid.CreateVersion7()}"); + TestRunSourceFolder = new DirectoryInfo(Path.Join(Path.GetTempPath(), "Arius-Core-Tests", $"{DateTime.Now:yyyyMMddTHHmmss}_{Guid.CreateVersion7()}")); TestRunSourceFolder.Create(); var pfs = new PhysicalFileSystem(); From 9f6f4fe637001f0f9ec1b9f2ef7e05d62fffea88 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 08:30:48 +0100 Subject: [PATCH 23/43] feat: fix Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads --- .../ArchiveCommandHandlerHandleTests.cs | 59 +++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index a64a17f9..9b82937d 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -82,6 +82,9 @@ public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() { } + + // --- SINGLE FILE + [Fact] public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() { @@ -294,6 +297,9 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac binaryProperties!.ArchivedSize.ShouldBeGreaterThan(0L); } + + // --- MULTIPLE FILES + [Fact] public async Task Multiple_AllUnique_MixedSizes_ShouldUploadLargeAndSmallBatches() { @@ -599,6 +605,9 @@ public async Task Multiple_SmallFiles_WithDuplicates_CrossTarBatches_ShouldWrite handlerContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(f31.OriginalPath), includeBinaryProperties: true).ShouldNotBeNull(); } + + // --- INCREMENTAL RUNS + [Fact] public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() { @@ -614,44 +623,46 @@ public async Task Incremental_AllFilesAlreadyUploaded_ShouldSkipUploads() var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (initialFileCount, _) = GetInitialFileStatistics(initialContext); - var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); - firstResult.IsSuccess.ShouldBeTrue(); + var result1 = await CreateHandler().Handle(initialContext, CancellationToken.None); - var originalHash = largeFile.OriginalHash; + result1.IsSuccess.ShouldBeTrue(); + + storageBuilder.StoredChunks.Count.ShouldBe(1); + storageBuilder.UploadedStates.Count.ShouldBe(1); // Corrupt pointer file to ensure it is rewritten on incremental run var staleHash = FakeHashBuilder.GenerateValidHash(999); - staleHash.ShouldNotBe(originalHash); + staleHash.ShouldNotBe(largeFile.OriginalHash); largeFile.FilePair.CreatePointerFile(staleHash); var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); + var result2 = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert - incrementalResult.IsSuccess.ShouldBeTrue(); - var summary = incrementalResult.Value; - - summary.TotalLocalFiles.ShouldBe(expectedFileCount); - summary.ExistingPointerFiles.ShouldBe(existingPointerCount); - summary.UniqueBinariesUploaded.ShouldBe(0); - summary.UniqueChunksUploaded.ShouldBe(0); - summary.PointerFilesCreated.ShouldBe(0); - summary.PointerFileEntriesDeleted.ShouldBe(0); - summary.BytesUploadedUncompressed.ShouldBe(0); - summary.NewStateName.ShouldBeNull(); - - largeFile.FilePair.PointerFile.ReadHash().ShouldBe(originalHash); - - storageBuilder.StoredChunks.Count.ShouldBe(1); + result2.IsSuccess.ShouldBeTrue(); + var summary2 = result2.Value; + + summary2.TotalLocalFiles.ShouldBe(expectedFileCount); + summary2.ExistingPointerFiles.ShouldBe(existingPointerCount); + summary2.UniqueBinariesUploaded.ShouldBe(0); // <-- no additional binaries were uploaded + summary2.UniqueChunksUploaded.ShouldBe(0); // <-- etc + summary2.PointerFilesCreated.ShouldBe(0); // <-- etc + summary2.PointerFileEntriesDeleted.ShouldBe(0); // <-- etc + summary2.BytesUploadedUncompressed.ShouldBe(0); // <-- etc + + // No new state was created & uploaded and the (temporary) database file was deleted + summary2.NewStateName.ShouldBeNull(); + incrementalContext.StateRepository.HasChanges.ShouldBeFalse(); + incrementalContext.StateRepository.StateDatabaseFile.Exists.ShouldBeFalse(); storageBuilder.UploadedStates.Count.ShouldBe(1); - var pointerEntry = incrementalContext.StateRepository - .GetPointerFileEntry(ToRelativePointerPath(binaryPath), includeBinaryProperties: true); - pointerEntry.ShouldNotBeNull(); - pointerEntry!.Hash.ShouldBe(originalHash); + // Pointer file was corrected + largeFile.FilePair.PointerFile.ReadHash().ShouldBe(largeFile.OriginalHash); + + storageBuilder.StoredChunks.Count.ShouldBe(1); // <-- no additional chunks were uploaded } [Fact] From 49d927aecdc1690faf0e4f9640c5aee97c8c275e Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 14:37:26 +0100 Subject: [PATCH 24/43] feat: fix Incremental_FileAndPointerDeleted_PointerFileEntryDeleted, Incremental_FileDeleted_PointerRemains_ShouldStillExist --- .../ArchiveCommandHandlerHandleTests.cs | 105 +++++++++++++----- 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 9b82937d..bd613354 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -677,8 +677,8 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() var storageBuilder = new MockArchiveStorageBuilder(fixture); var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); - firstResult.IsSuccess.ShouldBeTrue(); + var result1 = await CreateHandler().Handle(initialContext, CancellationToken.None); + result1.IsSuccess.ShouldBeTrue(); var newSmallFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "new-note.txt") @@ -689,11 +689,11 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); // Act - var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); + var result2 = await CreateHandler().Handle(incrementalContext, CancellationToken.None); // Assert - incrementalResult.IsSuccess.ShouldBeTrue(); - var summary = incrementalResult.Value; + result2.IsSuccess.ShouldBeTrue(); + var summary = result2.Value; summary.TotalLocalFiles.ShouldBe(expectedFileCount); summary.ExistingPointerFiles.ShouldBe(existingPointerCount); @@ -702,59 +702,104 @@ public async Task Incremental_MixOfNewAndExisting_ShouldUploadOnlyNewFiles() summary.PointerFilesCreated.ShouldBe(1); summary.BytesUploadedUncompressed.ShouldBe(newSmallFile.OriginalContent.Length); summary.PointerFileEntriesDeleted.ShouldBe(0); + + // A new state was created & uploaded summary.NewStateName.ShouldNotBeNull(); + incrementalContext.StateRepository.HasChanges.ShouldBeTrue(); - storageBuilder.StoredChunks.Values.Count().ShouldBe(2); - - var pointerEntry = incrementalContext.StateRepository - .GetPointerFileEntry(ToRelativePointerPath(newSmallFile.OriginalPath), includeBinaryProperties: true); + var pointerEntry = incrementalContext.StateRepository.GetPointerFileEntry(ToRelativePointerPath(newSmallFile.OriginalPath), includeBinaryProperties: true); pointerEntry.ShouldNotBeNull(); pointerEntry!.BinaryProperties.ShouldNotBeNull(); pointerEntry.BinaryProperties!.OriginalSize.ShouldBe(newSmallFile.OriginalContent.Length); + + storageBuilder.StoredChunks.Values.Count().ShouldBe(2); } [Fact] - public async Task Incremental_FileDeleted_PointerRemains_ShouldCleanUpStateEntry() + public async Task Incremental_FileAndPointerDeleted_PointerFileEntryDeleted() { // Arrange var deletedFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "to-delete.txt") - .WithRandomContent(2048, seed: 3001) + .WithRandomContent(2048, seed: 1) .Build(); var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); - firstResult.IsSuccess.ShouldBeTrue(); + var (_, context1, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var result1 = await CreateHandler().Handle(context1, CancellationToken.None); + result1.IsSuccess.ShouldBeTrue(); - File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt")); - File.Delete(Path.Combine(fixture.TestRunSourceFolder.FullName, "docs", "to-delete.txt.pointer.arius")); + // Delete the pointer and the binary + deletedFile.FilePair.BinaryFile.Delete(); + deletedFile.FilePair.PointerFile.Delete(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + var (_, context2, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(context2); // Act - var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); + var result2 = await CreateHandler().Handle(context2, CancellationToken.None); // Assert - incrementalResult.IsSuccess.ShouldBeTrue(); - var summary = incrementalResult.Value; + result2.IsSuccess.ShouldBeTrue(); + var summary2 = result2.Value; - summary.TotalLocalFiles.ShouldBe(expectedFileCount); - summary.ExistingPointerFiles.ShouldBe(existingPointerCount); - summary.UniqueBinariesUploaded.ShouldBe(0); - summary.PointerFileEntriesDeleted.ShouldBe(1); - summary.NewStateName.ShouldNotBeNull(); + summary2.TotalLocalFiles.ShouldBe(0); + summary2.ExistingPointerFiles.ShouldBe(0); + summary2.UniqueBinariesUploaded.ShouldBe(0); + summary2.PointerFileEntriesDeleted.ShouldBe(1); - var pointerEntry = incrementalContext.StateRepository - .GetPointerFileEntry("/docs/to-delete.txt.pointer.arius", includeBinaryProperties: true); - pointerEntry.ShouldBeNull(); + // A new state was uploaded + summary2.NewStateName.ShouldNotBeNull(); + storageBuilder.UploadedStates.Count.ShouldBe(2); + // The PointerFileEntry should not exist + var pfe = context2.StateRepository.GetPointerFileEntry(deletedFile.FilePair.PointerFile.FullName, includeBinaryProperties: true); + pfe.ShouldBeNull(); + + context2.StateRepository.GetPointerFileEntries("/", false).ShouldBeEmpty(); + + // The deleted chunks should still be present storageBuilder.StoredChunks.Count.ShouldBe(1); - storageBuilder.UploadedStates.Count.ShouldBe(2); } + [Fact] + public async Task Incremental_FileDeleted_PointerRemains_ShouldStillExist() + { + // Arrange + var deletedBinary = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "docs" / "to-delete.txt") + .WithRandomContent(2048, seed: 1) + .Build(); + + var storageBuilder = new MockArchiveStorageBuilder(fixture); + + var (_, context1, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var result1 = await CreateHandler().Handle(context1, CancellationToken.None); + result1.IsSuccess.ShouldBeTrue(); + + // Delete the binary, the pointer remains + deletedBinary.FilePair.BinaryFile.Delete(); + + var (_, context2, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(context2); + + // Act + var result2 = await CreateHandler().Handle(context2, CancellationToken.None); + + // Assert + result2.IsSuccess.ShouldBeTrue(); + var summary2 = result2.Value; + + summary2.TotalLocalFiles.ShouldBe(1); + summary2.ExistingPointerFiles.ShouldBe(1); + summary2.UniqueBinariesUploaded.ShouldBe(0); + summary2.PointerFileEntriesDeleted.ShouldBe(0); + + // A new state was uploaded + summary2.NewStateName.ShouldBeNull(); + storageBuilder.UploadedStates.Count.ShouldBe(1); + } [Fact] public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBinaryProperties() { From d8e33a4154c5fe5d8202b73ff9329bc322bda0e0 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 22 Oct 2025 08:26:13 +0200 Subject: [PATCH 25/43] feat: add warnings --- .../Commands/Archive/ArchiveCommandHandler.cs | 31 +++++++++++++------ .../Commands/Archive/ArchiveCommandResult.cs | 4 +++ .../Commands/Restore/RestoreCommandHandler.cs | 23 +++++++++----- .../Commands/Restore/RestoreCommandResult.cs | 2 ++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index 7397c622..d061eba3 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -9,6 +9,7 @@ using Mediator; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using System.Collections.Concurrent; using System.IO.Compression; using System.Threading.Channels; using Zio; @@ -38,14 +39,16 @@ public ArchiveCommandHandler(ILogger logger, ILoggerFacto private readonly InFlightGate uploadGate = new(); // Statistics tracking - private int totalLocalFiles; - private int uniqueBinariesUploaded; - private int uniqueChunksUploaded; - private long bytesUploadedUncompressed; - private long bytesUploadedCompressed; - private int existingPointerFiles; - private int pointerFilesCreated; - private int pointerFileEntriesDeleted; + private int totalLocalFiles = 0; + private int existingPointerFiles = 0; + private int uniqueBinariesUploaded = 0; + private int uniqueChunksUploaded = 0; + private long bytesUploadedUncompressed = 0; + private long bytesUploadedCompressed = 0; + private int pointerFilesCreated = 0; + private int pointerFileEntriesDeleted = 0; + private readonly ConcurrentBag warnings = []; + private int filesSkipped = 0; // Pipeline channels: // @@ -129,7 +132,9 @@ internal async ValueTask> Handle(HandlerContext han BytesUploadedCompressed = bytesUploadedCompressed, PointerFilesCreated = pointerFilesCreated, PointerFileEntriesDeleted = pointerFileEntriesDeleted, - NewStateName = newStateName + NewStateName = newStateName, + Warnings = warnings.ToArray(), + FilesSkipped = filesSkipped }); } catch (OperationCanceledException) when (!errorCancellationToken.IsCancellationRequested && cancellationToken.IsCancellationRequested) @@ -252,6 +257,10 @@ private Task CreateHashTask(HandlerContext handlerContext, CancellationToken can logger.LogWarning("File {FileName} is a pointer file without an associated binary, skipping", filePair.FullName); handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(filePair.FullName, -1, "Error: pointer file without binary")); + + var warningMessage = $"File '{filePair.FullName}' is a pointer file without an associated binary, skipping"; + warnings.Add(warningMessage); + Interlocked.Increment(ref filesSkipped); } else { @@ -273,6 +282,10 @@ private Task CreateHashTask(HandlerContext handlerContext, CancellationToken can { logger.LogWarning("Error when hashing file {FileName}: {Message}, skipping.", filePair.FullName, e.Message); handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(filePair.FullName, -1, $"Error: {e.Message}")); + + var warningMessage = $"Error when hashing file '{filePair.FullName}': {e.Message}, skipping"; + warnings.Add(warningMessage); + Interlocked.Increment(ref filesSkipped); } catch (OperationCanceledException) { diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandResult.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandResult.cs index 1f1cc9c8..c907fd78 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandResult.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandResult.cs @@ -52,4 +52,8 @@ public sealed record ArchiveCommandResult /// The name of the new state file that was uploaded, or null if no state changes occurred /// public string? NewStateName { get; init; } + + + public IReadOnlyList Warnings { get; init; } = []; + public int FilesSkipped { get; init; } } diff --git a/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs b/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs index 83e5610f..2c1ae8f4 100644 --- a/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Restore/RestoreCommandHandler.cs @@ -34,12 +34,13 @@ public RestoreCommandHandler(ILogger logger, ILoggerFacto } // Statistics tracking - private int totalTargetFiles; - private int verifiedFilesAlreadyExisting; - private int chunksDownloaded; - private long bytesDownloaded; - private int filesWrittenToDisk; - private long bytesWrittenToDisk; + private readonly ConcurrentBag warnings = []; + private int totalTargetFiles = 0; + private int verifiedFilesAlreadyExisting = 0; + private int chunksDownloaded = 0; + private long bytesDownloaded = 0; + private int filesWrittenToDisk = 0; + private long bytesWrittenToDisk = 0; private readonly Channel filePairsToRestoreChannel = ChannelExtensions.CreateBounded(capacity: 25, singleWriter: true, singleReader: false); private readonly Channel filePairsToHashChannel = ChannelExtensions.CreateBounded(capacity: 25, singleWriter: true, singleReader: false); @@ -121,7 +122,8 @@ await Task.WhenAll(allTasks.Select(async t => BytesDownloaded = bytesDownloaded, BytesWrittenToDisk = bytesWrittenToDisk, ChunksDownloaded = chunksDownloaded, - Rehydrating = rehydratingFiles + Rehydrating = rehydratingFiles, + Warnings = warnings.ToArray() }); } @@ -148,6 +150,9 @@ private Task CreateIndexTask(HandlerContext handlerContext, CancellationToken ca { logger.LogWarning("Target {TargetPath} was specified but no matching PointerFileEntry found", targetPath); handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(targetPath.FullName, -1, "Error: no matching entry found")); + + var warningMessage = $"Target '{targetPath}' was specified but no matching PointerFileEntry found"; + warnings.Add(warningMessage); } else { @@ -241,6 +246,10 @@ private Task CreateHashTask(HandlerContext handlerContext, CancellationToken can // The hash does not match - we need to restore this binaryfile logger.LogWarning("File {FileName} hash mismatch (expected: {ExpectedHash}, actual: {ActualHash}), queued for restore", filePair.BinaryFile.FullName, pointerFileEntry.Hash.ToShortString(), h.ToShortString()); handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(filePair.BinaryFile.FullName, 50, "Hash mismatch, restoring...")); + + var warningMessage = $"File '{filePair.BinaryFile.FullName}' hash mismatch (expected: {pointerFileEntry.Hash.ToShortString()}, actual: {h.ToShortString()}), queued for restore"; + warnings.Add(warningMessage); + await filePairsToRestoreChannel.Writer.WriteAsync(filePairWithPointerFileEntry, innerCancellationToken); } } diff --git a/src/Arius.Core/Features/Commands/Restore/RestoreCommandResult.cs b/src/Arius.Core/Features/Commands/Restore/RestoreCommandResult.cs index 2d07a112..a06ba5c2 100644 --- a/src/Arius.Core/Features/Commands/Restore/RestoreCommandResult.cs +++ b/src/Arius.Core/Features/Commands/Restore/RestoreCommandResult.cs @@ -39,6 +39,8 @@ public sealed record RestoreCommandResult /// Details about the files that are still hydrating /// public IReadOnlyList Rehydrating { get; init; } = []; + + public IReadOnlyList Warnings { get; init; } = []; } public sealed record RehydrationDetail From cad42a554131ca7436608c4b526987cdb68bcbce Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 15:25:15 +0100 Subject: [PATCH 26/43] feat: add Single_LatentPointer_ShouldLogWarning --- .../ArchiveCommandHandlerHandleTests.cs | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index bd613354..33c7cbfa 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -86,7 +86,7 @@ public void UpdatedCreationTimeOrLastWriteTimeShouldBeUpdatedInStateDatabase() // --- SINGLE FILE [Fact] - public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndPointer() + public async Task Single_LargeFile_FirstUpload_ShouldUploadBinaryAndCreatePointer() { // Arrange var binaryPath = UPath.Root / "documents" / "presentation.pptx"; @@ -298,6 +298,35 @@ public async Task Single_BinaryWithExistingPointer_ShouldOverwritePointerAndTrac } + [Fact] + public async Task Single_LatentPointer_ShouldLogWarning() + { + // Arrange + var latentFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.PointerFileOnly, UPath.Root / "latent.txt") + .WithRandomContent(512, seed: 1) + .Build(); + + latentFile.FilePair.BinaryFile.Exists.ShouldBeFalse(); + latentFile.FilePair.PointerFile.Exists.ShouldBeTrue(); + + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); + + // Act + var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); + + // Assert + result.IsSuccess.ShouldBeTrue(); + var summary = result.Value; + + summary.FilesSkipped.ShouldBe(1); + summary.Warnings.ShouldContain("File '/latent.txt' is a pointer file without an associated binary, skipping"); + + handlerContext.StateRepository.HasChanges.ShouldBeFalse(); + handlerContext.StateRepository.StateDatabaseFile.Exists.ShouldBeFalse(); + } + + // --- MULTIPLE FILES [Fact] From f207d00b867a632cb95635d5d0d8639f16feec17 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 16:08:01 +0100 Subject: [PATCH 27/43] feat: update DbContext.PointerFileEntries to have only RelativeName as key --- ...FileEntryKeyIsOnlyRelativeName.Designer.cs | 91 +++++++++++++++++++ ...2_PointerFileEntryKeyIsOnlyRelativeName.cs | 45 +++++++++ .../StateRepositoryDbContextModelSnapshot.cs | 13 ++- .../StateRepositoryDbContext.cs | 4 +- 4 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.Designer.cs create mode 100644 src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs diff --git a/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.Designer.cs b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.Designer.cs new file mode 100644 index 00000000..cf5aea15 --- /dev/null +++ b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.Designer.cs @@ -0,0 +1,91 @@ +// +using System; +using Arius.Core.Shared.StateRepositories; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Arius.Core.Shared.StateRepositories.Migrations +{ + [DbContext(typeof(StateRepositoryDbContext))] + [Migration("20251031145332_PointerFileEntryKeyIsOnlyRelativeName")] + partial class PointerFileEntryKeyIsOnlyRelativeName + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.10"); + + modelBuilder.Entity("Arius.Core.Shared.StateRepositories.BinaryProperties", b => + { + b.Property("Hash") + .HasColumnType("BLOB"); + + b.Property("ArchivedSize") + .HasColumnType("INTEGER"); + + b.Property("OriginalSize") + .HasColumnType("INTEGER"); + + b.Property("ParentHash") + .HasColumnType("BLOB"); + + b.Property("StorageTier") + .HasColumnType("INTEGER"); + + b.HasKey("Hash"); + + b.HasIndex("Hash") + .IsUnique(); + + b.ToTable("BinaryProperties", (string)null); + }); + + modelBuilder.Entity("Arius.Core.Shared.StateRepositories.PointerFileEntry", b => + { + b.Property("RelativeName") + .HasColumnType("TEXT"); + + b.Property("CreationTimeUtc") + .HasColumnType("TEXT"); + + b.Property("Hash") + .IsRequired() + .HasColumnType("BLOB"); + + b.Property("LastWriteTimeUtc") + .HasColumnType("TEXT"); + + b.HasKey("RelativeName"); + + b.HasIndex("Hash"); + + b.HasIndex("RelativeName", "Hash") + .HasDatabaseName("IX_PointerFileEntries_RelativeName_Hash"); + + b.ToTable("PointerFileEntries", (string)null); + }); + + modelBuilder.Entity("Arius.Core.Shared.StateRepositories.PointerFileEntry", b => + { + b.HasOne("Arius.Core.Shared.StateRepositories.BinaryProperties", "BinaryProperties") + .WithMany("PointerFileEntries") + .HasForeignKey("Hash") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("BinaryProperties"); + }); + + modelBuilder.Entity("Arius.Core.Shared.StateRepositories.BinaryProperties", b => + { + b.Navigation("PointerFileEntries"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs new file mode 100644 index 00000000..9027bc70 --- /dev/null +++ b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs @@ -0,0 +1,45 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Arius.Core.Shared.StateRepositories.Migrations +{ + /// + public partial class PointerFileEntryKeyIsOnlyRelativeName : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_PointerFileEntries", + table: "PointerFileEntries"); + + migrationBuilder.DropIndex( + name: "IX_PointerFileEntries_RelativeName", + table: "PointerFileEntries"); + + migrationBuilder.AddPrimaryKey( + name: "PK_PointerFileEntries", + table: "PointerFileEntries", + column: "RelativeName"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropPrimaryKey( + name: "PK_PointerFileEntries", + table: "PointerFileEntries"); + + migrationBuilder.AddPrimaryKey( + name: "PK_PointerFileEntries", + table: "PointerFileEntries", + columns: new[] { "Hash", "RelativeName" }); + + migrationBuilder.CreateIndex( + name: "IX_PointerFileEntries_RelativeName", + table: "PointerFileEntries", + column: "RelativeName"); + } + } +} diff --git a/src/Arius.Core/Shared/StateRepositories/Migrations/StateRepositoryDbContextModelSnapshot.cs b/src/Arius.Core/Shared/StateRepositories/Migrations/StateRepositoryDbContextModelSnapshot.cs index 4e30c956..40117800 100644 --- a/src/Arius.Core/Shared/StateRepositories/Migrations/StateRepositoryDbContextModelSnapshot.cs +++ b/src/Arius.Core/Shared/StateRepositories/Migrations/StateRepositoryDbContextModelSnapshot.cs @@ -15,7 +15,7 @@ partial class StateRepositoryDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.9"); + modelBuilder.HasAnnotation("ProductVersion", "9.0.10"); modelBuilder.Entity("Arius.Core.Shared.StateRepositories.BinaryProperties", b => { @@ -44,24 +44,23 @@ protected override void BuildModel(ModelBuilder modelBuilder) modelBuilder.Entity("Arius.Core.Shared.StateRepositories.PointerFileEntry", b => { - b.Property("Hash") - .HasColumnType("BLOB"); - b.Property("RelativeName") .HasColumnType("TEXT"); b.Property("CreationTimeUtc") .HasColumnType("TEXT"); + b.Property("Hash") + .IsRequired() + .HasColumnType("BLOB"); + b.Property("LastWriteTimeUtc") .HasColumnType("TEXT"); - b.HasKey("Hash", "RelativeName"); + b.HasKey("RelativeName"); b.HasIndex("Hash"); - b.HasIndex("RelativeName"); - b.HasIndex("RelativeName", "Hash") .HasDatabaseName("IX_PointerFileEntries_RelativeName_Hash"); diff --git a/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContext.cs b/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContext.cs index 05e85e2f..b2971c9d 100644 --- a/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContext.cs +++ b/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContext.cs @@ -46,10 +46,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) var pfeb = modelBuilder.Entity(); pfeb.ToTable("PointerFileEntries"); - pfeb.HasKey(pfe => new { pfe.Hash, pfe.RelativeName }); + pfeb.HasKey(pfe => pfe.RelativeName); pfeb.HasIndex(pfe => pfe.Hash); // NOT unique - pfeb.HasIndex(pfe => pfe.RelativeName); // to facilitate GetPointerFileEntriesAtVersionAsync + //pfeb.HasIndex(pfe => pfe.RelativeName); // to facilitate GetPointerFileEntriesAtVersionAsync // Composite index for better query performance on prefix searches with joins (for GetPointerFileEntries / topdirectoryonly & GetPointerFileDirectories / topdirectoryonly) pfeb.HasIndex(pfe => new { pfe.RelativeName, pfe.Hash }) From 0abeb9707002ff0be0a8e232bb29b4ee174459a0 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 16:08:58 +0100 Subject: [PATCH 28/43] feat: fix Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBinaryProperties --- .../ArchiveCommandHandlerHandleTests.cs | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 33c7cbfa..c35fbb72 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -829,57 +829,61 @@ public async Task Incremental_FileDeleted_PointerRemains_ShouldStillExist() summary2.NewStateName.ShouldBeNull(); storageBuilder.UploadedStates.Count.ShouldBe(1); } + + [Fact] - public async Task Incremental_FileModified_ShouldUploadNewHashAndPreserveOldBinaryProperties() + public async Task Incremental_FileModified_ShouldUploadNewBinaryAndPreserveOldBinary() { // Arrange var filePath = UPath.Root / "docs" / "mutable.bin"; var originalFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, filePath) - .WithRandomContent(3072, seed: 4001) + .WithRandomContent(3072, seed: 1) .Build(); var storageBuilder = new MockArchiveStorageBuilder(fixture); - var (_, initialContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var firstResult = await CreateHandler().Handle(initialContext, CancellationToken.None); - firstResult.IsSuccess.ShouldBeTrue(); + var (_, context1, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var result1 = await CreateHandler().Handle(context1, CancellationToken.None); + result1.IsSuccess.ShouldBeTrue(); - var originalHash = originalFile.OriginalHash; - var originalBinaryProperties = initialContext.StateRepository.GetBinaryProperty(originalHash); - originalBinaryProperties.ShouldNotBeNull(); + var bp1 = context1.StateRepository.GetBinaryProperty(originalFile.OriginalHash); + bp1.ShouldNotBeNull(); - _ = new FakeFileBuilder(fixture) + // Overwrite the file with new content + var modifiedFile = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, filePath) - .WithRandomContent(3584, seed: 4002) + .WithRandomContent(4000, seed: 2) .Build(); - var (_, incrementalContext, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); - var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(incrementalContext); + var (_, context2, _, _) = await CreateHandlerContextAsync(storageBuilder: storageBuilder); + var (expectedFileCount, existingPointerCount) = GetInitialFileStatistics(context2); // Act - var incrementalResult = await CreateHandler().Handle(incrementalContext, CancellationToken.None); + var result2 = await CreateHandler().Handle(context2, CancellationToken.None); // Assert - incrementalResult.IsSuccess.ShouldBeTrue(); - var summary = incrementalResult.Value; + result2.IsSuccess.ShouldBeTrue(); + var summary = result2.Value; - summary.TotalLocalFiles.ShouldBe(expectedFileCount); - summary.ExistingPointerFiles.ShouldBe(existingPointerCount); - summary.UniqueBinariesUploaded.ShouldBe(1); + summary.TotalLocalFiles.ShouldBe(1); + summary.ExistingPointerFiles.ShouldBe(1); + summary.UniqueBinariesUploaded.ShouldBe(1); // one additional binary uploaded summary.UniqueChunksUploaded.ShouldBe(1); summary.PointerFilesCreated.ShouldBe(0); // pointer already existed summary.NewStateName.ShouldNotBeNull(); - var pointerEntry = incrementalContext.StateRepository - .GetPointerFileEntry(ToRelativePointerPath(filePath), includeBinaryProperties: true); - pointerEntry.ShouldNotBeNull(); - pointerEntry!.Hash.ShouldNotBe(originalHash); - pointerEntry.BinaryProperties.ShouldNotBeNull(); + var pfe = context2.StateRepository.GetPointerFileEntry(ToRelativePointerPath(filePath), includeBinaryProperties: true); + pfe.ShouldNotBeNull(); + pfe.Hash.ShouldBe(modifiedFile.OriginalHash); + pfe.BinaryProperties.ShouldNotBeNull(); + pfe.BinaryProperties.OriginalSize.ShouldBe(4000); - var oldBinaryProperties = incrementalContext.StateRepository.GetBinaryProperty(originalHash); - oldBinaryProperties.ShouldNotBeNull(); + // The BinaryProperties of the originalFile are still present + var originalBinaryProperties = context2.StateRepository.GetBinaryProperty(originalFile.OriginalHash); + originalBinaryProperties.ShouldNotBeNull(); + // The old Binary is still present storageBuilder.StoredChunks.Count.ShouldBe(2); } From 860151a0076792ed127a8f6206cf512f0f00b1fa Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 16:25:45 +0100 Subject: [PATCH 29/43] feat: update IStateRepository docstrings --- .../StateRepositories/IStateRepository.cs | 85 ++++++++++++++++--- 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/src/Arius.Core/Shared/StateRepositories/IStateRepository.cs b/src/Arius.Core/Shared/StateRepositories/IStateRepository.cs index 1f98e17f..f6025847 100644 --- a/src/Arius.Core/Shared/StateRepositories/IStateRepository.cs +++ b/src/Arius.Core/Shared/StateRepositories/IStateRepository.cs @@ -6,16 +6,77 @@ namespace Arius.Core.Shared.StateRepositories; internal interface IStateRepository { - FileEntry StateDatabaseFile { get; } - bool HasChanges { get; } - void Vacuum(); - void Delete(); - BinaryProperties? GetBinaryProperty(Hash h); - void SetBinaryPropertyArchiveTier(Hash h, StorageTier tier); - void AddBinaryProperties(params BinaryProperties[] bps); - void UpsertPointerFileEntries(params PointerFileEntry[] pfes); + /// + /// Gets the state repository database file that backs this repository instance. + /// + FileEntry StateDatabaseFile { get; } + + /// + /// Gets a value indicating whether the repository has tracked changes that are not yet persisted. + /// + bool HasChanges { get; } + + /// + /// Compacts the underlying state database to reclaim unused space. + /// + void Vacuum(); + + /// + /// Deletes the underlying state database and resets the repository. + /// + void Delete(); + + /// + /// Retrieves the binary properties for the specified hash, or null when absent. + /// + /// The hash that identifies the binary. + BinaryProperties? GetBinaryProperty(Hash h); + + /// + /// Updates the storage tier for the binary that matches the given hash. + /// + /// The hash that identifies the binary. + /// The new storage tier to apply. + void SetBinaryPropertyArchiveTier(Hash h, StorageTier tier); + + /// + /// Adds the provided binary property records to the repository. + /// + /// The binary property entries to persist. + void AddBinaryProperties(params BinaryProperties[] bps); + + /// + /// Inserts new pointer file entries or updates existing ones as needed. + /// + /// The pointer file entries to upsert. + void UpsertPointerFileEntries(params PointerFileEntry[] pfes); + + /// + /// Enumerates pointer file directories that match the specified prefix. + /// + /// A prefix that must start with '/' and is matched against stored entries. + /// When true, returns only directories one level below the prefix. IEnumerable GetPointerFileDirectories(string relativeNamePrefix, bool topDirectoryOnly); - IEnumerable GetPointerFileEntries(string relativeNamePrefix, bool topDirectoryOnly, bool includeBinaryProperties = false); - PointerFileEntry? GetPointerFileEntry(string relativeName, bool includeBinaryProperties = false); - int DeletePointerFileEntries(Func shouldBeDeleted); -} \ No newline at end of file + + /// + /// Enumerates pointer file entries that match the specified prefix. + /// + /// A prefix that must start with '/' and is matched against stored entries. + /// When true, limits results to the first level of the hierarchy. + /// When true, includes related binary properties in the results. + IEnumerable GetPointerFileEntries(string relativeNamePrefix, bool topDirectoryOnly, bool includeBinaryProperties = false); + + /// + /// Retrieves a single pointer file entry that matches the supplied relative name, or null if none exists. + /// + /// The full relative name, starting with '/'. + /// When true, includes the related binary properties. + PointerFileEntry? GetPointerFileEntry(string relativeName, bool includeBinaryProperties = false); + + /// + /// Deletes pointer file entries that satisfy the supplied predicate. + /// + /// A predicate used to determine which entries to remove. + /// The number of deleted entries. + int DeletePointerFileEntries(Func shouldBeDeleted); +} From 5252979b1ec8e05ea567bfbe231418ad4cbc4f10 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Fri, 31 Oct 2025 16:53:29 +0100 Subject: [PATCH 30/43] feat: fix cancellation in archivecommand --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 4 ++-- .../Features/Commands/Archive/ArchiveCommandHandler.cs | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index c35fbb72..24ced54d 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -930,12 +930,12 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() // Arrange _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large1.bin") - .WithRandomContent(4096, seed: 6001) + .WithRandomContent(4096, seed: 1) .Build(); _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "cancel" / "large2.bin") - .WithRandomContent(4096, seed: 6002) + .WithRandomContent(4096, seed: 2) .Build(); var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(); diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index d061eba3..8085de57 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -82,8 +82,9 @@ internal async ValueTask> Handle(HandlerContext han logger.LogInformation("Starting archive operation for path {LocalRoot} with hashing parallelism {HashingParallelism}, upload parallelism {UploadParallelism}", handlerContext.Request.LocalRoot, handlerContext.Request.HashingParallelism, handlerContext.Request.UploadParallelism); - using var errorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - var errorCancellationToken = errorCancellationTokenSource.Token; + using var errorCancellationTokenSource = new CancellationTokenSource(); + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCancellationTokenSource.Token); + var errorCancellationToken = linkedCancellationTokenSource.Token; var indexTask = CreateIndexTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); var hashTask = CreateHashTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); @@ -137,7 +138,7 @@ internal async ValueTask> Handle(HandlerContext han FilesSkipped = filesSkipped }); } - catch (OperationCanceledException) when (!errorCancellationToken.IsCancellationRequested && cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested && !errorCancellationTokenSource.IsCancellationRequested) { // User-triggered cancellation logger.LogInformation("Archive operation cancelled by user"); @@ -758,4 +759,4 @@ private async Task ProcessTarArchive(HandlerContext handlerContext, InMemoryGzip handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(entry.FilePair.FullName, 100, "Archive complete")); } -} \ No newline at end of file +} From ae8e14087fa3664a2c16d49c8fa6118f0f19a290 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 13:57:32 +0100 Subject: [PATCH 31/43] chore: change visiblity --- .../20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs index 9027bc70..c5b0b466 100644 --- a/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs +++ b/src/Arius.Core/Shared/StateRepositories/Migrations/20251031145332_PointerFileEntryKeyIsOnlyRelativeName.cs @@ -5,7 +5,7 @@ namespace Arius.Core.Shared.StateRepositories.Migrations { /// - public partial class PointerFileEntryKeyIsOnlyRelativeName : Migration + internal partial class PointerFileEntryKeyIsOnlyRelativeName : Migration { /// protected override void Up(MigrationBuilder migrationBuilder) From d8154c6f3aa16fe91f46f1b5820d7d7d5bacce89 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 13:59:10 +0100 Subject: [PATCH 32/43] feat: fix Error_HashTaskFails_ShouldSkipProblematicFileAndContinue --- .../ArchiveCommandHandlerHandleTests.cs | 34 ++++++++++--------- .../Commands/Archive/ArchiveCommandHandler.cs | 33 +++++++++--------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 24ced54d..9e0be94d 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -958,35 +958,33 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() { // Arrange - var failingPath = UPath.Root / "hash" / "will-fail.bin"; - _ = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, failingPath) - .WithRandomContent(1024, seed: 6101) + var failingFile = new FakeFileBuilder(fixture) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "hash" / "will-fail.bin") + .WithRandomContent(1024, seed: 1) .Build(); - var successfulPath = UPath.Root / "hash" / "will-upload.bin"; var successfulFile = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, successfulPath) - .WithRandomContent(2048, seed: 6102) + .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "hash" / "will-upload.bin") + .WithRandomContent(1024, seed: 2) .Build(); - var failingAbsolutePath = Path.Combine(fixture.TestRunSourceFolder.FullName, "hash", "will-fail.bin"); - var deleted = false; var progressUpdates = new List(); - void HandleProgress(ProgressUpdate update) + void ProgressHandler(ProgressUpdate update) { progressUpdates.Add(update); + + // Simulate a failure during hashing by deleting the file when we get to that point if (!deleted && update is FileProgressUpdate fileUpdate && fileUpdate.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && fileUpdate.StatusMessage?.Contains("Hashing", StringComparison.OrdinalIgnoreCase) == true) { deleted = true; - File.Delete(failingAbsolutePath); + failingFile.FilePair.BinaryFile.Delete(); } } - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(HandleProgress))); + var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(ProgressHandler))); var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); @@ -1003,14 +1001,18 @@ void HandleProgress(ProgressUpdate update) summary.PointerFilesCreated.ShouldBe(1); summary.BytesUploadedUncompressed.ShouldBe(successfulFile.OriginalContent.Length); - File.Exists(failingAbsolutePath).ShouldBeFalse(); - File.Exists(Path.Combine(fixture.TestRunSourceFolder.FullName, "hash", "will-fail.bin.pointer.arius")).ShouldBeFalse(); + // The Binary & Pointer do not exist + File.Exists(failingFile.FilePair.BinaryFile.FullName).ShouldBeFalse(); + File.Exists(failingFile.FilePair.PointerFile.FullName).ShouldBeFalse(); - storageBuilder.StoredChunks.Count.ShouldBe(1); + storageBuilder.StoredChunks.Count.ShouldBe(1); // only 1 chunk stored (not 2) progressUpdates.OfType() - .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error", StringComparison.OrdinalIgnoreCase) == true) + .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error: BinaryFile does not exist", StringComparison.OrdinalIgnoreCase) == true) .ShouldBeTrue(); + + summary.FilesSkipped.ShouldBe(1); + summary.Warnings.ShouldContain(w => w.StartsWith("Error when hashing file '/hash/will-fail.bin': BinaryFile does not exist")); } [Fact] diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index 8085de57..a54fb411 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -60,11 +60,13 @@ public ArchiveCommandHandler(ILogger logger, ILoggerFacto // - small files: a single consumer batches entries into a TAR; only the "owner" of a hash adds to the TAR. // duplicates (non-owners) are *deferred* — we DO NOT block the reader. // - large files: each owner uploads the blob directly; non-owners await the owner (safe here because it's in a parallel consumer). + private record FilePairWithHash(FilePair FilePair, Hash Hash); private readonly Channel indexedFilesChannel = ChannelExtensions.CreateBounded(capacity: 20, singleWriter: true, singleReader: false); private readonly Channel hashedLargeFilesChannel = ChannelExtensions.CreateBounded(capacity: 10, singleWriter: false, singleReader: false); private readonly Channel hashedSmallFilesChannel = ChannelExtensions.CreateBounded(capacity: 10, singleWriter: false, singleReader: true); - private record FilePairWithHash(FilePair FilePair, Hash Hash); + + // --- HANDLER public async ValueTask> Handle(ArchiveCommand request, CancellationToken cancellationToken) { @@ -199,6 +201,9 @@ await Task.WhenAll(allTasks.Select(async t => } } + + // --- HIGH LEVEL TASKS + private Task CreateIndexTask(HandlerContext handlerContext, CancellationToken cancellationToken, CancellationTokenSource errorCancellationTokenSource) => Task.Run(async () => { @@ -225,7 +230,7 @@ private Task CreateIndexTask(HandlerContext handlerContext, CancellationToken ca indexedFilesChannel.Writer.Complete(); throw; } - catch (Exception e) + catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue { logger.LogError(e, "File indexing failed with exception"); indexedFilesChannel.Writer.Complete(); @@ -279,7 +284,11 @@ private Task CreateHashTask(HandlerContext handlerContext, CancellationToken can await hashedLargeFilesChannel.Writer.WriteAsync(new(filePair, h), cancellationToken: innerCancellationToken); } } - catch (IOException e) + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) { logger.LogWarning("Error when hashing file {FileName}: {Message}, skipping.", filePair.FullName, e.Message); handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(filePair.FullName, -1, $"Error: {e.Message}")); @@ -288,16 +297,6 @@ private Task CreateHashTask(HandlerContext handlerContext, CancellationToken can warnings.Add(warningMessage); Interlocked.Increment(ref filesSkipped); } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - logger.LogError(e, "File hashing failed for {FileName}", filePair.FullName); - errorCancellationTokenSource.Cancel(); - throw; - } }); t.ContinueWith(_ => @@ -323,7 +322,7 @@ private Task CreateUploadLargeFilesTask(HandlerContext handlerContext, Cancellat { throw; } - catch (Exception e) + catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue { logger.LogError(e, "Large file upload task failed"); errorCancellationTokenSource.Cancel(); @@ -342,7 +341,7 @@ private Task CreateUploadSmallFilesTarArchiveTask(HandlerContext handlerContext, { throw; } - catch (Exception e) + catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue { logger.LogError(e, "Small files TAR archive task failed"); errorCancellationTokenSource.Cancel(); @@ -350,6 +349,9 @@ private Task CreateUploadSmallFilesTarArchiveTask(HandlerContext handlerContext, } }, cancellationToken); + + // --- HELPERS + private const string ChunkContentType = "application/aes256cbc+gzip"; private const string TarChunkContentType = "application/aes256cbc+tar+gzip"; @@ -758,5 +760,4 @@ private async Task ProcessTarArchive(HandlerContext handlerContext, InMemoryGzip foreach (var entry in tarWriter.TarredEntries) handlerContext.Request.ProgressReporter?.Report(new FileProgressUpdate(entry.FilePair.FullName, 100, "Archive complete")); } - } From 82dec0ba6a490716d0680a1402f1519d92b0d324 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 14:23:51 +0100 Subject: [PATCH 33/43] Update ArchiveCommandHandler.cs --- .../Features/Commands/Archive/ArchiveCommandHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index a54fb411..9a3b4f8c 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -230,7 +230,7 @@ private Task CreateIndexTask(HandlerContext handlerContext, CancellationToken ca indexedFilesChannel.Writer.Complete(); throw; } - catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue + catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write test Error_IndexTaskFails_ShouldSkipProblematicFileAndContinue { logger.LogError(e, "File indexing failed with exception"); indexedFilesChannel.Writer.Complete(); From 2eb184f5a38190d35419e2d3ffc42b7e5ac73875 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 14:32:23 +0100 Subject: [PATCH 34/43] chore: add placeholder Error_IndexTaskFails_ShouldSkipProblematicFileAndContinue --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 9e0be94d..2f704e47 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -954,6 +954,12 @@ public async Task Error_CancellationByUser_ShouldReturnFailureResult() storageBuilder.StoredChunks.Count.ShouldBe(0); } + [Fact(Skip = "TODO")] + public async Task Error_IndexTaskFails_ShouldSkipProblematicFileAndContinue() + { + // See example Error_HashTaskFails_ShouldSkipProblematicFileAndContinue + } + [Fact] public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() { From 46d28d7daf732a8f4e04e6c33ce55a2217b50cbd Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 14:32:52 +0100 Subject: [PATCH 35/43] feat: Error_HashTaskFails_ShouldSkipProblematicFileAndContinue tweaks --- .../ArchiveCommandHandlerHandleTests.cs | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 2f704e47..4d97c46c 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -969,7 +969,7 @@ public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() .WithRandomContent(1024, seed: 1) .Build(); - var successfulFile = new FakeFileBuilder(fixture) + _ = new FakeFileBuilder(fixture) .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "hash" / "will-upload.bin") .WithRandomContent(1024, seed: 2) .Build(); @@ -992,8 +992,6 @@ void ProgressHandler(ProgressUpdate update) var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(ProgressHandler))); - var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); - // Act var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); @@ -1001,24 +999,12 @@ void ProgressHandler(ProgressUpdate update) result.IsSuccess.ShouldBeTrue(); var summary = result.Value; - summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); - summary.ExistingPointerFiles.ShouldBe(existingPointerCount); - summary.UniqueBinariesUploaded.ShouldBe(1); - summary.PointerFilesCreated.ShouldBe(1); - summary.BytesUploadedUncompressed.ShouldBe(successfulFile.OriginalContent.Length); - - // The Binary & Pointer do not exist - File.Exists(failingFile.FilePair.BinaryFile.FullName).ShouldBeFalse(); - File.Exists(failingFile.FilePair.PointerFile.FullName).ShouldBeFalse(); - - storageBuilder.StoredChunks.Count.ShouldBe(1); // only 1 chunk stored (not 2) + summary.FilesSkipped.ShouldBe(1); + summary.Warnings.First().ShouldContain("Error when hashing file '/hash/will-fail.bin': BinaryFile does not exist"); progressUpdates.OfType() .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error: BinaryFile does not exist", StringComparison.OrdinalIgnoreCase) == true) .ShouldBeTrue(); - - summary.FilesSkipped.ShouldBe(1); - summary.Warnings.ShouldContain(w => w.StartsWith("Error when hashing file '/hash/will-fail.bin': BinaryFile does not exist")); } [Fact] From f47e4383ad229fbd677fbd36c8b92800f53a14a9 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 14:39:55 +0100 Subject: [PATCH 36/43] feat: refactor Handle task setup --- .../Commands/Archive/ArchiveCommandHandler.cs | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index 9a3b4f8c..1d350bb5 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -88,14 +88,17 @@ internal async ValueTask> Handle(HandlerContext han using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, errorCancellationTokenSource.Token); var errorCancellationToken = linkedCancellationTokenSource.Token; - var indexTask = CreateIndexTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); - var hashTask = CreateHashTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); - var uploadLargeFilesTask = CreateUploadLargeFilesTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); - var uploadSmallFilesTask = CreateUploadSmallFilesTarArchiveTask(handlerContext, errorCancellationToken, errorCancellationTokenSource); + var tasks = new Dictionary + { + ["indexTask"] = CreateIndexTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["hashTask"] = CreateHashTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["uploadLargeFilesTask"] = CreateUploadLargeFilesTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["uploadSmallFilesTask"] = CreateUploadSmallFilesTarArchiveTask(handlerContext, errorCancellationToken, errorCancellationTokenSource) + }; try { - await Task.WhenAll(indexTask, hashTask, uploadLargeFilesTask, uploadSmallFilesTask); + await Task.WhenAll(tasks.Values); // 6. Remove PointerFileEntries that do not exist on disk logger.LogDebug("Cleaning up pointer file entries that no longer exist on disk"); @@ -150,47 +153,36 @@ internal async ValueTask> Handle(HandlerContext han { // Either a task failed with an exception or error-triggered cancellation occurred // Wait for all tasks to complete gracefully - var allTasks = new[] { indexTask, hashTask, uploadLargeFilesTask, uploadSmallFilesTask }; - await Task.WhenAll(allTasks.Select(async t => + await Task.WhenAll(tasks.Values.Select(async t => { try { await t; } catch { /* Ignore exceptions during graceful shutdown */ } })); - // Map tasks to their names for logging - var taskNames = new Dictionary - { - { indexTask, nameof(indexTask) }, - { hashTask, nameof(hashTask) }, - { uploadLargeFilesTask, nameof(uploadLargeFilesTask) }, - { uploadSmallFilesTask, nameof(uploadSmallFilesTask) } - }; - // Log cancelled tasks (debug level) - var cancelledTasks = allTasks.Where(t => t.IsCanceled).ToArray(); - if (cancelledTasks.Any()) + var cancelledTaskNames = tasks.Where(kvp => kvp.Value.IsCanceled).Select(kvp => kvp.Key).ToArray(); + if (cancelledTaskNames.Any()) { - var cancelledTaskNames = cancelledTasks.Select(t => taskNames[t]).ToArray(); logger.LogDebug("Tasks cancelled during graceful shutdown: {TaskNames}", string.Join(", ", cancelledTaskNames)); } // Log and handle failed tasks (error level) - var faultedTasks = allTasks.Where(t => t.IsFaulted).ToArray(); + var faultedTasks = tasks.Where(kvp => kvp.Value.IsFaulted).ToArray(); if (faultedTasks.Any()) { - if (faultedTasks is { Length: 1 } && faultedTasks.Single() is var faultedTask) + if (faultedTasks is { Length: 1 } && faultedTasks.Single() is var faultedTaskEntry) { // Single faulted task - return the exception - var baseException = faultedTask.Exception!.GetBaseException(); - logger.LogError(baseException, "Task '{TaskName}' failed with exception '{Exception}'", taskNames[faultedTask], baseException.Message); + var baseException = faultedTaskEntry.Value.Exception!.GetBaseException(); + logger.LogError(baseException, "Task '{TaskName}' failed with exception '{Exception}'", faultedTaskEntry.Key, baseException.Message); return Result.Fail($"Archive operation failed: {baseException.Message}").WithError(new ExceptionalError(baseException)); } else { // Multiple faulted tasks - return aggregate exception - var exceptions = faultedTasks.Select(t => t.Exception!.GetBaseException()).ToArray(); + var exceptions = faultedTasks.Select(kvp => kvp.Value.Exception!.GetBaseException()).ToArray(); var aggregateException = new AggregateException("Multiple tasks failed during archive operation", exceptions); - logger.LogError(aggregateException, "Tasks failed: {TaskNames}", string.Join(", ", faultedTasks.Select(t => taskNames[t]))); + logger.LogError(aggregateException, "Tasks failed: {TaskNames}", string.Join(", ", faultedTasks.Select(kvp => kvp.Key))); return Result.Fail($"Archive operation failed: multiple tasks failed").WithError(new ExceptionalError(aggregateException)); } } From 14a5808c6be4c93794c863afab0eef14ea6c8bd6 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 15:18:23 +0100 Subject: [PATCH 37/43] feat: improve archivecommand cancellation handling & fix Error_UploadTaskFails_ShouldReturnFailure --- .../ArchiveCommandHandlerHandleTests.cs | 5 +- .../Commands/Archive/ArchiveCommandHandler.cs | 52 ++++++++----------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 4d97c46c..17d2faf7 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -73,8 +73,6 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic await Task.Delay(100); // Delay until the statefile name ("yyyy-MM-ddTHH-mm-ss") is in different seconds goto Retry; } - - } [Fact(Skip = "TODO")] @@ -1026,8 +1024,7 @@ public async Task Error_UploadTaskFails_ShouldReturnFailure() // Assert result.IsFailed.ShouldBeTrue(); - result.Errors.First().Message.ShouldContain("failed"); - storageBuilder.StoredChunks.Count.ShouldBe(0); + result.Errors.First().Message.ShouldStartWith("Archive operation failed: UploadLargeFilesTask failed with "); } [Fact] diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index 1d350bb5..663b3d68 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -90,10 +90,10 @@ internal async ValueTask> Handle(HandlerContext han var tasks = new Dictionary { - ["indexTask"] = CreateIndexTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), - ["hashTask"] = CreateHashTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), - ["uploadLargeFilesTask"] = CreateUploadLargeFilesTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), - ["uploadSmallFilesTask"] = CreateUploadSmallFilesTarArchiveTask(handlerContext, errorCancellationToken, errorCancellationTokenSource) + ["IndexTask"] = CreateIndexTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["HashTask"] = CreateHashTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["UploadLargeFilesTask"] = CreateUploadLargeFilesTask(handlerContext, errorCancellationToken, errorCancellationTokenSource), + ["UploadSmallFilesTask"] = CreateUploadSmallFilesTarArchiveTask(handlerContext, errorCancellationToken, errorCancellationTokenSource) }; try @@ -152,6 +152,10 @@ internal async ValueTask> Handle(HandlerContext han catch (Exception ex) { // Either a task failed with an exception or error-triggered cancellation occurred + + // Trigger error-driven cancellation to signal other tasks to stop gracefully + errorCancellationTokenSource.Cancel(); + // Wait for all tasks to complete gracefully await Task.WhenAll(tasks.Values.Select(async t => { @@ -167,29 +171,22 @@ await Task.WhenAll(tasks.Values.Select(async t => } // Log and handle failed tasks (error level) - var faultedTasks = tasks.Where(kvp => kvp.Value.IsFaulted).ToArray(); - if (faultedTasks.Any()) + var faultedTasks = tasks.Where(kvp => kvp.Value.IsFaulted).Select(kvp => (Name: kvp.Key, Exception: kvp.Value.Exception!.GetBaseException())).ToArray(); + if (faultedTasks is { Length: 1 } && faultedTasks.Single() is var faultedTask) { - if (faultedTasks is { Length: 1 } && faultedTasks.Single() is var faultedTaskEntry) - { - // Single faulted task - return the exception - var baseException = faultedTaskEntry.Value.Exception!.GetBaseException(); - logger.LogError(baseException, "Task '{TaskName}' failed with exception '{Exception}'", faultedTaskEntry.Key, baseException.Message); - return Result.Fail($"Archive operation failed: {baseException.Message}").WithError(new ExceptionalError(baseException)); - } - else - { - // Multiple faulted tasks - return aggregate exception - var exceptions = faultedTasks.Select(kvp => kvp.Value.Exception!.GetBaseException()).ToArray(); - var aggregateException = new AggregateException("Multiple tasks failed during archive operation", exceptions); - logger.LogError(aggregateException, "Tasks failed: {TaskNames}", string.Join(", ", faultedTasks.Select(kvp => kvp.Key))); - return Result.Fail($"Archive operation failed: multiple tasks failed").WithError(new ExceptionalError(aggregateException)); - } + // Single faulted task - return the exception + var msg = faultedTask.Exception?.GetBaseException().Message ?? "UNKNOWN"; + logger.LogError(faultedTask.Exception, "Task '{TaskName}' failed with exception '{Exception}'", faultedTask.Name, msg); + return Result.Fail($"Archive operation failed: {faultedTask.Name} failed with {msg}").WithError(new ExceptionalError(faultedTask.Exception)); + } + else + { + // Multiple faulted tasks - return aggregate exception + var exceptions = faultedTasks.Select(ft => ft.Exception).ToArray(); + var aggregateException = new AggregateException("Multiple tasks failed during archive operation", exceptions); + logger.LogError(aggregateException, "Tasks failed: {TaskNames}", string.Join(", ", faultedTasks.Select(ft => ft.Name))); + return Result.Fail($"Archive operation failed: multiple tasks failed").WithError(new ExceptionalError(aggregateException)); } - - // Unexpected error path - return generic failure - logger.LogError(ex, "Archive operation failed with unexpected error"); - return Result.Fail($"Archive operation failed: {ex.Message}").WithError(new ExceptionalError(ex)); } } @@ -226,7 +223,6 @@ private Task CreateIndexTask(HandlerContext handlerContext, CancellationToken ca { logger.LogError(e, "File indexing failed with exception"); indexedFilesChannel.Writer.Complete(); - errorCancellationTokenSource.Cancel(); throw; } }, cancellationToken); @@ -314,10 +310,9 @@ private Task CreateUploadLargeFilesTask(HandlerContext handlerContext, Cancellat { throw; } - catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue + catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, update test Error_UploadTaskFails_ShouldReturnFailure { logger.LogError(e, "Large file upload task failed"); - errorCancellationTokenSource.Cancel(); throw; } }); @@ -336,7 +331,6 @@ private Task CreateUploadSmallFilesTarArchiveTask(HandlerContext handlerContext, catch (Exception e) // TODO Align with approach of HashTask where we skip the file and log a warning instead of failing the entire task, write a test like Error_HashTaskFails_ShouldSkipProblematicFileAndContinue { logger.LogError(e, "Small files TAR archive task failed"); - errorCancellationTokenSource.Cancel(); throw; } }, cancellationToken); From 123e5823ccd6f4cd3c00775c6c2898c7fcdc0254 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 15:27:23 +0100 Subject: [PATCH 38/43] feat: fix Error_MultipleTasksFail_ShouldReturnAggregateException --- .../Commands/Archive/ArchiveCommandHandlerHandleTests.cs | 3 +-- .../Features/Commands/Archive/ArchiveCommandHandler.cs | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 17d2faf7..636e689a 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -1051,8 +1051,7 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() // Assert result.IsFailed.ShouldBeTrue(); - result.Errors.First().Message.ShouldContain("multiple tasks failed"); - storageBuilder.StoredChunks.Count.ShouldBe(0); + result.Errors.First().Message.ShouldMatch("Archive operation failed: .* tasks failed"); } [Fact] diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index 663b3d68..ae0370e8 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -184,8 +184,9 @@ await Task.WhenAll(tasks.Values.Select(async t => // Multiple faulted tasks - return aggregate exception var exceptions = faultedTasks.Select(ft => ft.Exception).ToArray(); var aggregateException = new AggregateException("Multiple tasks failed during archive operation", exceptions); - logger.LogError(aggregateException, "Tasks failed: {TaskNames}", string.Join(", ", faultedTasks.Select(ft => ft.Name))); - return Result.Fail($"Archive operation failed: multiple tasks failed").WithError(new ExceptionalError(aggregateException)); + var faultedTaskNames = string.Join(", ", faultedTasks.Select(ft => ft.Name)); + logger.LogError(aggregateException, "Tasks failed: {FaultedTaskNames}", faultedTaskNames); + return Result.Fail($"Archive operation failed: {faultedTaskNames} tasks failed").WithError(new ExceptionalError(aggregateException)); } } } From 415f1cf2205ace6e0c0bbb1897a6796f221e13ae Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 15:43:13 +0100 Subject: [PATCH 39/43] feat: robustness check --- .../StateRepositoryDbContextFactory.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContextFactory.cs b/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContextFactory.cs index 30d76eab..962e46fe 100644 --- a/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContextFactory.cs +++ b/src/Arius.Core/Shared/StateRepositories/StateRepositoryDbContextFactory.cs @@ -47,13 +47,19 @@ public StateRepositoryDbContextPool(FileEntry stateDatabaseFile, bool ensureCrea { lock (ensureCreatedLock) { - using var context = CreateContext(); + using var context = CreateContext(skipExistCheck: true); context.Database.Migrate(); } } } - public StateRepositoryDbContext CreateContext() => factory.CreateDbContext(); + public StateRepositoryDbContext CreateContext(bool skipExistCheck = false) + { + if (!skipExistCheck && !StateDatabaseFile.Exists) + throw new InvalidOperationException($"Database file {StateDatabaseFile} does not exist"); + + return factory.CreateDbContext(); + } public void SetHasChanges() => Interlocked.Exchange(ref hasChanges, true); From 05237c99b135f438720c02e4ff0b94103694ec6f Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 15:44:06 +0100 Subject: [PATCH 40/43] chore: remove obsolete test --- .../ArchiveCommandHandlerHandleTests.cs | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index 636e689a..a3e830b0 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -1054,41 +1054,6 @@ public async Task Error_MultipleTasksFail_ShouldReturnAggregateException() result.Errors.First().Message.ShouldMatch("Archive operation failed: .* tasks failed"); } - [Fact] - public async Task Error_PointerFileOnly_ShouldReportWarningAndSkip() - { - // Arrange - _ = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.PointerFileOnly, UPath.Root / "orphans" / "lonely.bin") - .Build(); - - var progressUpdates = new List(); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(progressUpdates.Add))); - - var (expectedInitialFileCount, existingPointerCount) = GetInitialFileStatistics(handlerContext); - - // Act - var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); - - // Assert - result.IsSuccess.ShouldBeTrue(); - var summary = result.Value; - - summary.TotalLocalFiles.ShouldBe(expectedInitialFileCount); - summary.ExistingPointerFiles.ShouldBe(existingPointerCount); - summary.UniqueBinariesUploaded.ShouldBe(0); - summary.PointerFilesCreated.ShouldBe(0); - - storageBuilder.StoredChunks.Count.ShouldBe(0); - - handlerContext.StateRepository.GetPointerFileEntry("/orphans/lonely.bin.pointer.arius", includeBinaryProperties: true) - .ShouldBeNull(); - - progressUpdates.OfType() - .Any(p => p.FileName.EndsWith("lonely.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("pointer file without binary", StringComparison.OrdinalIgnoreCase) == true) - .ShouldBeTrue(); - } - [Fact] public async Task StalePointerEntries_ShouldBeRemovedWhenMissingOnDisk() { From c6fbff18c3e87068f9640f9a18d0e67dbe9fc2a2 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 16:31:10 +0100 Subject: [PATCH 41/43] feat: introduce ISha256Hasher and simplify Error_HashTaskFails_ShouldSkipProblematicFileAndContinue test TEMP Update ArchiveCommandHandlerHandleTests.cs --- .../ArchiveCommandHandlerHandleTests.cs | 45 +++++++------------ .../Commands/Archive/ArchiveCommandHandler.cs | 9 +++- .../Commands/Archive/HandlerContext.cs | 2 +- .../Commands/Archive/HandlerContextBuilder.cs | 9 +++- src/Arius.Core/Shared/Hashing/Sha256Hasher.cs | 33 +++++++++++--- 5 files changed, 62 insertions(+), 36 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index a3e830b0..e6057f5c 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -1,5 +1,6 @@ using Arius.Core.Features.Commands.Archive; using Arius.Core.Shared.FileSystem; +using Arius.Core.Shared.Hashing; using Arius.Core.Shared.StateRepositories; using Arius.Core.Shared.Storage; using Arius.Core.Tests.Helpers.Builders; @@ -8,6 +9,7 @@ using Arius.Core.Tests.Helpers.Fixtures; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Testing; +using NSubstitute; using Shouldly; using Zio; @@ -44,7 +46,8 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic private async Task<(ArchiveCommand Command, HandlerContext Context, MockArchiveStorageBuilder StorageBuilder, FakeLoggerFactory LoggerFactory)> CreateHandlerContextAsync(Action? configureCommand = null, MockArchiveStorageBuilder? storageBuilder = null, - FakeLoggerFactory? loggerFactory = null) + FakeLoggerFactory? loggerFactory = null, + ISha256Hasher? hasher = null) { storageBuilder ??= new MockArchiveStorageBuilder(fixture); loggerFactory ??= new FakeLoggerFactory(); @@ -62,9 +65,13 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic Retry: try { - var handlerContext = await new HandlerContextBuilder(command, loggerFactory) - .WithArchiveStorage(archiveStorage) - .BuildAsync(); + var builder = new HandlerContextBuilder(command, loggerFactory) + .WithArchiveStorage(archiveStorage); + + if (hasher != null) + builder = builder.WithHasher(hasher); + + var handlerContext = await builder.BuildAsync(); return (command, handlerContext, storageBuilder, loggerFactory); } @@ -967,28 +974,14 @@ public async Task Error_HashTaskFails_ShouldSkipProblematicFileAndContinue() .WithRandomContent(1024, seed: 1) .Build(); - _ = new FakeFileBuilder(fixture) - .WithActualFile(FilePairType.BinaryFileOnly, UPath.Root / "hash" / "will-upload.bin") - .WithRandomContent(1024, seed: 2) - .Build(); + // Create a mock hasher that fails for will-fail.bin + var mockHasher = Substitute.For(); - var deleted = false; - var progressUpdates = new List(); - void ProgressHandler(ProgressUpdate update) - { - progressUpdates.Add(update); - - // Simulate a failure during hashing by deleting the file when we get to that point - if (!deleted && update is FileProgressUpdate fileUpdate && - fileUpdate.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && - fileUpdate.StatusMessage?.Contains("Hashing", StringComparison.OrdinalIgnoreCase) == true) - { - deleted = true; - failingFile.FilePair.BinaryFile.Delete(); - } - } + // Configure mock + mockHasher.GetHashAsync(Arg.Any()) + .Returns>(_ => throw new ArgumentException("BinaryFile does not exist")); - var (_, handlerContext, storageBuilder, _) = await CreateHandlerContextAsync(builder => builder.WithProgressReporter(new Progress(ProgressHandler))); + var (_, handlerContext, _, _) = await CreateHandlerContextAsync(hasher: mockHasher); // Act var result = await CreateHandler().Handle(handlerContext, CancellationToken.None); @@ -999,10 +992,6 @@ void ProgressHandler(ProgressUpdate update) summary.FilesSkipped.ShouldBe(1); summary.Warnings.First().ShouldContain("Error when hashing file '/hash/will-fail.bin': BinaryFile does not exist"); - - progressUpdates.OfType() - .Any(p => p.FileName.EndsWith("will-fail.bin", StringComparison.OrdinalIgnoreCase) && p.StatusMessage?.Contains("Error: BinaryFile does not exist", StringComparison.OrdinalIgnoreCase) == true) - .ShouldBeTrue(); } [Fact] diff --git a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs index ae0370e8..94b6564a 100644 --- a/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs +++ b/src/Arius.Core/Features/Commands/Archive/ArchiveCommandHandler.cs @@ -153,6 +153,8 @@ internal async ValueTask> Handle(HandlerContext han { // Either a task failed with an exception or error-triggered cancellation occurred + var faultedTasks = tasks.Where(kvp => kvp.Value.IsFaulted).Select(kvp => (Name: kvp.Key, Exception: kvp.Value.Exception!.GetBaseException())).ToArray(); + // Trigger error-driven cancellation to signal other tasks to stop gracefully errorCancellationTokenSource.Cancel(); @@ -163,6 +165,12 @@ await Task.WhenAll(tasks.Values.Select(async t => catch { /* Ignore exceptions during graceful shutdown */ } })); + // Observe all task exceptions to prevent UnobservedTaskException + foreach (var task in tasks.Values.Where(t => t.IsFaulted)) + { + _ = task.Exception; + } + // Log cancelled tasks (debug level) var cancelledTaskNames = tasks.Where(kvp => kvp.Value.IsCanceled).Select(kvp => kvp.Key).ToArray(); if (cancelledTaskNames.Any()) @@ -171,7 +179,6 @@ await Task.WhenAll(tasks.Values.Select(async t => } // Log and handle failed tasks (error level) - var faultedTasks = tasks.Where(kvp => kvp.Value.IsFaulted).Select(kvp => (Name: kvp.Key, Exception: kvp.Value.Exception!.GetBaseException())).ToArray(); if (faultedTasks is { Length: 1 } && faultedTasks.Single() is var faultedTask) { // Single faulted task - return the exception diff --git a/src/Arius.Core/Features/Commands/Archive/HandlerContext.cs b/src/Arius.Core/Features/Commands/Archive/HandlerContext.cs index 8013bda1..b9ebcfd2 100644 --- a/src/Arius.Core/Features/Commands/Archive/HandlerContext.cs +++ b/src/Arius.Core/Features/Commands/Archive/HandlerContext.cs @@ -10,6 +10,6 @@ internal record HandlerContext public required ArchiveCommand Request { get; init; } public required IArchiveStorage ArchiveStorage { get; init; } public required StateRepository StateRepository { get; init; } - public required Sha256Hasher Hasher { get; init; } + public required ISha256Hasher Hasher { get; init; } public required FilePairFileSystem FileSystem { get; init; } } \ No newline at end of file diff --git a/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs b/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs index 651bf8ec..5cfb2d7c 100644 --- a/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs +++ b/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs @@ -18,6 +18,7 @@ internal class HandlerContextBuilder private IArchiveStorage? archiveStorage; private StateRepository? stateRepository; private IFileSystem? baseFileSystem; + private ISha256Hasher? hasher; public HandlerContextBuilder(ArchiveCommand request, ILoggerFactory loggerFactory) { @@ -44,6 +45,12 @@ public HandlerContextBuilder WithBaseFileSystem(IFileSystem fileSystem) return this; } + public HandlerContextBuilder WithHasher(ISha256Hasher hasher) + { + this.hasher = hasher; + return this; + } + public async Task BuildAsync() { await new ArchiveCommandValidator().ValidateAndThrowAsync(request); @@ -65,7 +72,7 @@ public async Task BuildAsync() Request = request, ArchiveStorage = archiveStorage, StateRepository = stateRepository ?? await BuildStateRepositoryAsync(archiveStorage), - Hasher = new Sha256Hasher(request.Passphrase), + Hasher = hasher ?? new Sha256Hasher(request.Passphrase), FileSystem = GetFileSystem() }; diff --git a/src/Arius.Core/Shared/Hashing/Sha256Hasher.cs b/src/Arius.Core/Shared/Hashing/Sha256Hasher.cs index 6edae13d..a3203283 100644 --- a/src/Arius.Core/Shared/Hashing/Sha256Hasher.cs +++ b/src/Arius.Core/Shared/Hashing/Sha256Hasher.cs @@ -5,7 +5,33 @@ namespace Arius.Core.Shared.Hashing; -internal sealed class Sha256Hasher +internal interface ISha256Hasher +{ + /// + /// Gets the salted hash of a raw byte array. + /// + Task GetHashAsync(byte[] data); + + /// + /// Gets the salted hash of a Stream. + /// + /// + /// + Task GetHashAsync(Stream s); + + /// + /// Gets the salted hash of a FilePair. If it is PointerFileOnly, we simply + /// return the hash stored in the pointer file. Otherwise, we hash the BinaryFile. + /// + Task GetHashAsync(FilePair fp); + + /// + /// Gets the salted hash of a BinaryFile. Throws if the file does not exist. + /// + Task GetHashAsync(BinaryFile bf); +} + +internal sealed class Sha256Hasher : ISha256Hasher { private const int BufferSize = 81920; // 80 KB buffer private readonly byte[] saltBytes; @@ -22,10 +48,7 @@ internal sealed class Sha256Hasher ///// Raw salt bytes. //private Sha256Hasher(byte[] salt) => saltBytes = salt; - /// - /// Gets the salted hash of a raw byte array. Returns a Task for consistency. - /// - public Task GetHashAsync(byte[] data) + public Task GetHashAsync(byte[] data) // NOTE: returns a Task for consistency { // No I/O, so no real async needed. We keep a Task-based signature to match the rest of the code. var hashValue = ComputeSaltedHash(data); From 70c2d9d44838c2f4b3d9e28ffe207740f6fd157c Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 17:20:54 +0100 Subject: [PATCH 42/43] fix: better fix for identical time-based database naming during fast tests --- .../ArchiveCommandHandlerContextTests.cs | 6 +++--- .../ArchiveCommandHandlerHandleTests.cs | 21 ++++++------------- .../Commands/Archive/HandlerContextBuilder.cs | 2 +- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerContextTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerContextTests.cs index caf05856..b2c0ad88 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerContextTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerContextTests.cs @@ -52,7 +52,7 @@ public async Task CreateAsync_WhenNoRemoteStateExists_ShouldCreateNewStateFile() // Verify a new state file was created (with current timestamp format) var stateFiles = stateCache.GetStateFileEntries().ToArray(); stateFiles.Length.ShouldBe(1); - stateFiles[0].Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.db"); + stateFiles[0].Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\-\d{3}\.db"); // Verify no download was attempted since no remote state exists await mockArchiveStorage.DidNotReceive().DownloadStateAsync(Arg.Any(), Arg.Any(), Arg.Any()); @@ -101,7 +101,7 @@ await mockArchiveStorage.Received(1).DownloadStateAsync( var newStateFile = stateFiles.FirstOrDefault(f => f.Name != $"{existingStateName}.db"); newStateFile.ShouldNotBeNull("a new state file should be created"); - newStateFile.Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.db"); + newStateFile.Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\-\d{3}\.db"); } [Fact] @@ -139,6 +139,6 @@ public async Task CreateAsync_WhenRemoteStateExistsAndIsPresentLocally_ShouldNot var newStateFile = stateFiles.FirstOrDefault(f => f.Name != $"{existingStateName}.db"); newStateFile.ShouldNotBeNull("a new state file should be created"); - newStateFile.Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.db"); + newStateFile.Name.ShouldMatch(@"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\-\d{3}\.db"); } } \ No newline at end of file diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs index e6057f5c..e5173eaa 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/ArchiveCommandHandlerHandleTests.cs @@ -62,24 +62,15 @@ private static (int FileCount, int ExistingPointerCount) GetInitialFileStatistic var command = commandBuilder.Build(); var archiveStorage = storageBuilder.Build(); - Retry: - try - { - var builder = new HandlerContextBuilder(command, loggerFactory) - .WithArchiveStorage(archiveStorage); + var builder = new HandlerContextBuilder(command, loggerFactory) + .WithArchiveStorage(archiveStorage); - if (hasher != null) - builder = builder.WithHasher(hasher); + if (hasher != null) + builder = builder.WithHasher(hasher); - var handlerContext = await builder.BuildAsync(); + var handlerContext = await builder.BuildAsync(); - return (command, handlerContext, storageBuilder, loggerFactory); - } - catch (IOException) - { - await Task.Delay(100); // Delay until the statefile name ("yyyy-MM-ddTHH-mm-ss") is in different seconds - goto Retry; - } + return (command, handlerContext, storageBuilder, loggerFactory); } [Fact(Skip = "TODO")] diff --git a/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs b/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs index 5cfb2d7c..04c911ac 100644 --- a/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs +++ b/src/Arius.Core/Features/Commands/Archive/HandlerContextBuilder.cs @@ -82,7 +82,7 @@ async Task BuildStateRepositoryAsync(IArchiveStorage archiveSto var stateCache = new StateCache(request.AccountName, request.ContainerName); // Determine the version name for this run - var versionName = DateTime.UtcNow.ToString("yyyy-MM-ddTHH-mm-ss"); + var versionName = DateTime.UtcNow.ToString("yyyy-MM-ddTHH-mm-ss-fff"); request.ProgressReporter?.Report(new TaskProgressUpdate($"Determining version name '{versionName}'...", 0)); // Get the latest state from blob storage From a8cf53adb164eb608059f3c4893bf38621b01219 Mon Sep 17 00:00:00 2001 From: Wouter Van Ranst Date: Wed, 12 Nov 2025 17:32:47 +0100 Subject: [PATCH 43/43] chore: cleanup usings --- src/Arius.Cli.Tests/ArchiveCliCommandTests.cs | 1 - src/Arius.Cli.Tests/RestoreCliCommandTests.cs | 1 - src/Arius.Cli/Program.cs | 1 - .../Features/Commands/Archive/MockArchiveStorageBuilder.cs | 4 ++-- .../PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs | 1 - .../Shared/StateRepositories/StateRepositoryTests.cs | 1 - .../PointerFileEntries/PointerFileEntriesQueryHandler.cs | 4 ++-- .../Migrations/20250922100845_InitialCreate.cs | 3 +-- src/Arius.Core/Shared/StateRepositories/StateRepository.cs | 4 ++-- 9 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Arius.Cli.Tests/ArchiveCliCommandTests.cs b/src/Arius.Cli.Tests/ArchiveCliCommandTests.cs index d6b5f133..f862e841 100644 --- a/src/Arius.Cli.Tests/ArchiveCliCommandTests.cs +++ b/src/Arius.Cli.Tests/ArchiveCliCommandTests.cs @@ -4,7 +4,6 @@ using Mediator; using NSubstitute; using Shouldly; -using System.Threading.Tasks; namespace Arius.Cli.Tests; diff --git a/src/Arius.Cli.Tests/RestoreCliCommandTests.cs b/src/Arius.Cli.Tests/RestoreCliCommandTests.cs index 83925a26..d05f98fd 100644 --- a/src/Arius.Cli.Tests/RestoreCliCommandTests.cs +++ b/src/Arius.Cli.Tests/RestoreCliCommandTests.cs @@ -3,7 +3,6 @@ using Mediator; using NSubstitute; using Shouldly; -using System.Threading.Tasks; namespace Arius.Cli.Tests; diff --git a/src/Arius.Cli/Program.cs b/src/Arius.Cli/Program.cs index 48bb1cca..5034f2e8 100644 --- a/src/Arius.Cli/Program.cs +++ b/src/Arius.Cli/Program.cs @@ -6,7 +6,6 @@ using Serilog; using Serilog.Core; using Serilog.Events; -using System.IO; using System.Reflection; namespace Arius.Cli; diff --git a/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs b/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs index 09c857d8..609b9e90 100644 --- a/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs +++ b/src/Arius.Core.Tests/Features/Commands/Archive/MockArchiveStorageBuilder.cs @@ -1,10 +1,10 @@ -using System.Collections.Concurrent; -using System.IO.Compression; using Arius.Core.Shared.Hashing; using Arius.Core.Shared.Storage; using Arius.Core.Tests.Helpers.Fixtures; using FluentResults; using NSubstitute; +using System.Collections.Concurrent; +using System.IO.Compression; using Zio; namespace Arius.Core.Tests.Features.Commands.Archive; diff --git a/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs b/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs index d60e9dfd..ec695cd9 100644 --- a/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs +++ b/src/Arius.Core.Tests/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandlerTests.cs @@ -1,7 +1,6 @@ using Arius.Core.Features.Queries.PointerFileEntries; using Arius.Core.Shared.FileSystem; using Arius.Core.Shared.StateRepositories; -using Arius.Core.Shared.Storage; using Arius.Core.Tests.Features.Commands.Restore; using Arius.Core.Tests.Helpers.Builders; using Arius.Core.Tests.Helpers.FakeLogger; diff --git a/src/Arius.Core.Tests/Shared/StateRepositories/StateRepositoryTests.cs b/src/Arius.Core.Tests/Shared/StateRepositories/StateRepositoryTests.cs index 423d22e5..bfe4a4c3 100644 --- a/src/Arius.Core.Tests/Shared/StateRepositories/StateRepositoryTests.cs +++ b/src/Arius.Core.Tests/Shared/StateRepositories/StateRepositoryTests.cs @@ -1,5 +1,4 @@ using Arius.Core.Shared.FileSystem; -using Arius.Core.Shared.Hashing; using Arius.Core.Shared.StateRepositories; using Arius.Core.Shared.Storage; using Arius.Core.Tests.Helpers.Builders; diff --git a/src/Arius.Core/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandler.cs b/src/Arius.Core/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandler.cs index 9b5a8482..f2381bd5 100644 --- a/src/Arius.Core/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandler.cs +++ b/src/Arius.Core/Features/Queries/PointerFileEntries/PointerFileEntriesQueryHandler.cs @@ -1,10 +1,10 @@ +using Arius.Core.Shared.FileSystem; +using Arius.Core.Shared.Storage; using FluentValidation; using Mediator; using Microsoft.Extensions.Logging; using System.Runtime.CompilerServices; using System.Threading.Channels; -using Arius.Core.Shared.FileSystem; -using Arius.Core.Shared.Storage; using Zio; namespace Arius.Core.Features.Queries.PointerFileEntries; diff --git a/src/Arius.Core/Shared/StateRepositories/Migrations/20250922100845_InitialCreate.cs b/src/Arius.Core/Shared/StateRepositories/Migrations/20250922100845_InitialCreate.cs index a427bc9e..8faab9be 100644 --- a/src/Arius.Core/Shared/StateRepositories/Migrations/20250922100845_InitialCreate.cs +++ b/src/Arius.Core/Shared/StateRepositories/Migrations/20250922100845_InitialCreate.cs @@ -1,5 +1,4 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations; #nullable disable diff --git a/src/Arius.Core/Shared/StateRepositories/StateRepository.cs b/src/Arius.Core/Shared/StateRepositories/StateRepository.cs index c0d434f2..8949bc19 100644 --- a/src/Arius.Core/Shared/StateRepositories/StateRepository.cs +++ b/src/Arius.Core/Shared/StateRepositories/StateRepository.cs @@ -1,9 +1,9 @@ -using Arius.Core.Shared.Hashing; +using Arius.Core.Shared.FileSystem; +using Arius.Core.Shared.Hashing; using Arius.Core.Shared.Storage; using EFCore.BulkExtensions; using Microsoft.EntityFrameworkCore; using System.Runtime.CompilerServices; -using Arius.Core.Shared.FileSystem; using WouterVanRanst.Utils.Extensions; using Zio;