Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions src/Ramstack.FileSystem.Amazon/S3UploadStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,13 @@ public override void Write(ReadOnlySpan<byte> buffer)
try
{
_stream.Write(buffer);

if (_stream.Length >= MinPartSize)
UploadPart();
}
catch (Exception exception)
catch
{
Abort();
ExceptionDispatchInfo.Throw(exception);
throw;
}
}

Expand All @@ -124,10 +123,10 @@ public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, Cancella
if (_stream.Length >= MinPartSize)
await UploadPartAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception exception)
catch
{
await AbortAsync(cancellationToken).ConfigureAwait(false);
ExceptionDispatchInfo.Throw(exception);
throw;
}
}

Expand Down Expand Up @@ -254,10 +253,10 @@ private async ValueTask UploadPartAsync(CancellationToken cancellationToken)
_stream.Position = 0;
_stream.SetLength(0);
}
catch (Exception exception)
catch
{
await AbortAsync(cancellationToken).ConfigureAwait(false);
ExceptionDispatchInfo.Throw(exception);
throw;
}
}
}
Expand Down
27 changes: 13 additions & 14 deletions src/Ramstack.FileSystem.Google/GcsWriteStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
namespace Ramstack.FileSystem.Google;

/// <summary>
/// Represents a temporary write-only stream for Google Cloud Storage operations, redirecting all write operations to a temporary file.
/// Upon disposing or closing the stream, the data is transferred to the Azure Blob storage.
/// Represents a temporary write-only stream that buffers data to a temporary file before uploading it to Google Cloud Storage.
/// Data is committed to the storage bucket when the stream is disposed or closed.
/// </summary>
internal sealed class GcsWriteStream : Stream
{
private readonly GoogleFileSystem _fs;
private readonly string _objectName;
private readonly FileStream _stream = CreateTempFileStream();
private readonly FileStream _stream;
private bool _disposed;

/// <inheritdoc />
Expand Down Expand Up @@ -53,6 +53,16 @@ public GcsWriteStream(GoogleFileSystem fs, string objectName)
{
_fs = fs;
_objectName = objectName;
_stream = new FileStream(
Path.Combine(
Path.GetTempPath(),
Path.GetRandomFileName()),
FileMode.CreateNew,
FileAccess.ReadWrite,
FileShare.None,
bufferSize: 4096,
FileOptions.DeleteOnClose
| FileOptions.Asynchronous);
}

/// <inheritdoc />
Expand Down Expand Up @@ -84,7 +94,6 @@ public override void Write(ReadOnlySpan<byte> buffer)
{
_disposed = true;
_stream.Close();

throw;
}
}
Expand All @@ -104,7 +113,6 @@ public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, Cancella
{
_disposed = true;
_stream.Close();

throw;
}
}
Expand Down Expand Up @@ -176,15 +184,6 @@ await _fs.StorageClient
}
}

private static FileStream CreateTempFileStream()
{
const int BufferSize = 4096;
const FileOptions Options = FileOptions.DeleteOnClose | FileOptions.Asynchronous;

var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
return new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, BufferSize, Options);
}

[DoesNotReturn]
private static void Error_NotSupported() =>
throw new NotSupportedException();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace Ramstack.FileSystem.Amazon;
[Category("Cloud:Amazon")]
public class WritableAmazonFileSystemTests : VirtualFileSystemSpecificationTests
{
private readonly HashSet<string> _list = [];
private readonly HashSet<string> _buckets = [];
private readonly TempFileStorage _storage = new TempFileStorage();

[OneTimeSetUp]
Expand All @@ -35,17 +35,17 @@ public async Task Cleanup()
{
_storage.Dispose();

foreach (var name in _list.ToArray())
foreach (var name in _buckets.ToArray())
{
using var fs = CreateFileSystem(name);

try
{
await fs.DeleteDirectoryAsync("/");
}
catch (Exception exception)
catch (Exception e)
{
Console.WriteLine(exception);
Console.WriteLine(e);
}
}
}
Expand All @@ -61,14 +61,15 @@ public async Task File_OpenWrite_InternalBufferWriteError_DoesNotCreateFile()
Assert.That(underlying, Is.Not.Null);

// Write enough data to trigger automatic part upload (>= 5 MiB).
await stream.WriteAsync(new ReadOnlyMemory<byte>(new byte[6 * 1024 * 1024]));
await stream.WriteAsync(new byte[6 * 1024 * 1024]);

// Simulates an internal buffer write error.
await underlying.DisposeAsync();

try
{
await stream.WriteAsync(new ReadOnlyMemory<byte>(new byte[1024]));
await stream.WriteAsync(new byte[1024]);
Assert.Fail();
}
catch
{
Expand Down Expand Up @@ -239,7 +240,6 @@ public async Task File_OpenWrite_FlushWithMultipartUpload()
for (var i = 0; i < Count; i++)
await stream.WriteAsync(chunk);
}

{
var file = fs.GetFile(FileName);

Expand Down Expand Up @@ -314,7 +314,7 @@ protected override DirectoryInfo GetDirectoryInfo() =>

private AmazonS3FileSystem CreateFileSystem(string storageName)
{
_list.Add(storageName);
_buckets.Add(storageName);

return new AmazonS3FileSystem(
new BasicAWSCredentials("rustfsadmin", "rustfsadmin"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ public async Task Cleanup()
{
await fs.DeleteDirectoryAsync("/");
}
catch (Exception exception)
catch (Exception e)
{
Console.WriteLine(exception);
Console.WriteLine(e);
}
}
}
Expand All @@ -60,22 +60,19 @@ public async Task File_OpenWrite_InternalBufferWriteError_DoesNotCreateFile()
var underlying = (FileStream)stream.GetType().GetField("_stream", BindingFlags.NonPublic | BindingFlags.Instance)!.GetValue(stream)!;
Assert.That(underlying, Is.Not.Null);

await stream.WriteAsync(new ReadOnlyMemory<byte>(new byte[1024]));

// Forces to upload buffer.
await stream.FlushAsync();
await stream.WriteAsync(new byte[1024]);

// Simulates an internal buffer write error.
await underlying.DisposeAsync();

try
{
await stream.WriteAsync(new ReadOnlyMemory<byte>(new byte[1024]));
await stream.WriteAsync(new byte[1024]);
Assert.Fail();
}
catch (Exception exception)
catch
{
Console.WriteLine("Exception expected!");
Console.WriteLine(exception);
// Ignore
}
}

Expand Down Expand Up @@ -160,6 +157,9 @@ public async Task File_CopyTo_File_DifferentFileSystems()
Assert.That(
await reader.ReadToEndAsync(),
Is.EqualTo(content));

await source.DeleteAsync();
await destination.DeleteAsync();
}

[Test]
Expand Down Expand Up @@ -189,6 +189,9 @@ public async Task File_CopyTo_File_DifferentStorages()
Assert.That(
await reader.ReadToEndAsync(),
Is.EqualTo(content));

await source.DeleteAsync();
await destination.DeleteAsync();
}

protected override GoogleFileSystem GetFileSystem() =>
Expand Down
Loading