Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ namespace Amazon.S3.Transfer.Internal
internal class BufferedDownloadConfiguration : DownloadManagerConfiguration
{
/// <summary>
/// Maximum parts to keep in memory simultaneously.
/// Maximum parts to keep in memory simultaneously. When null, defaults to 1024.
/// </summary>
public int MaxInMemoryParts { get; set; }
public int? MaxInMemoryParts { get; set; }

/// <summary>
/// Buffer size for I/O operations.
Expand All @@ -48,20 +48,20 @@ internal class BufferedDownloadConfiguration : DownloadManagerConfiguration
/// Creates a BufferedDownloadConfiguration with the specified configuration values.
/// </summary>
/// <param name="concurrentServiceRequests">Maximum concurrent HTTP requests for downloading parts.</param>
/// <param name="maxInMemoryParts">Maximum number of parts to keep in memory simultaneously.</param>
/// <param name="maxInMemoryParts">Maximum number of parts to keep in memory simultaneously. When null, defaults to 1024.</param>
/// <param name="bufferSize">Buffer size used for optimal I/O operations.</param>
/// <param name="targetPartSizeBytes">Target size for each part in bytes.</param>
/// <param name="chunkBufferSize">Optional chunk buffer size for ChunkedBufferStream. When null, defaults to 64KB.</param>
/// <exception cref="ArgumentOutOfRangeException">Thrown when any parameter is less than or equal to 0.</exception>
public BufferedDownloadConfiguration(
int concurrentServiceRequests,
int maxInMemoryParts,
int? maxInMemoryParts,
int bufferSize,
long targetPartSizeBytes,
int? chunkBufferSize = null)
: base(concurrentServiceRequests, targetPartSizeBytes)
{
if (maxInMemoryParts <= 0)
if (maxInMemoryParts.HasValue && maxInMemoryParts.Value <= 0)
throw new ArgumentOutOfRangeException(nameof(maxInMemoryParts), "Must be greater than 0");
if (bufferSize <= 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize), "Must be greater than 0");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,14 +189,17 @@ public PartBufferManager(BufferedDownloadConfiguration config)
if (config == null)
throw new ArgumentNullException(nameof(config));

// Resolve null MaxInMemoryParts to default value of 1024
var maxInMemoryParts = config.MaxInMemoryParts ?? 1024;

_partDataSources = new ConcurrentDictionary<int, IPartDataSource>();
_bufferSpaceAvailable = new SemaphoreSlim(
config.MaxInMemoryParts, // initialCount
config.MaxInMemoryParts // maxCount - prevents exceeding configured limit
maxInMemoryParts, // initialCount
maxInMemoryParts // maxCount - prevents exceeding configured limit
);
_partAvailable = new AutoResetEvent(false);

_logger.DebugFormat("PartBufferManager initialized with MaxInMemoryParts={0}", config.MaxInMemoryParts);
_logger.DebugFormat("PartBufferManager initialized with MaxInMemoryParts={0}", maxInMemoryParts);
}

/// <inheritdoc/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,22 +32,17 @@ namespace Amazon.S3.Transfer
/// </summary>
public class TransferUtilityOpenStreamRequest : BaseDownloadRequest
{
private int _maxInMemoryParts = 1024;

/// <summary>
/// Gets or sets the maximum number of parts to buffer in memory during multipart downloads.
/// The default value is 1024.
/// When null, defaults to 1024.
/// </summary>
/// <remarks>
/// This property controls memory usage during streaming downloads. When combined with the
/// default part size of 8MB, the default value of 1024 parts allows up to 8GB of memory usage.
/// Adjust this value based on your application's memory constraints and performance requirements.
/// Set to null to use the default value of 1024.
/// </remarks>
public int MaxInMemoryParts
{
get { return this._maxInMemoryParts; }
set { this._maxInMemoryParts = value; }
}
public int? MaxInMemoryParts { get; set; }
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


/// <summary>
/// Gets or sets the chunk buffer size (in bytes) used for buffering out-of-order parts during multipart downloads.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public void Constructor_WithValidParameters_CreatesConfiguration()
{
// Arrange
int concurrentRequests = 10;
int maxInMemoryParts = 5;
int? maxInMemoryParts = 5;
int bufferSize = 8192;
long targetPartSize = 8 * 1024 * 1024; // 8MB

Expand All @@ -38,7 +38,7 @@ public void Constructor_WithChunkBufferSize_SetsProperty()
{
// Arrange
int concurrentRequests = 10;
int maxInMemoryParts = 5;
int? maxInMemoryParts = 5;
int bufferSize = 8192;
long targetPartSize = 8 * 1024 * 1024; // 8MB
int chunkBufferSize = 32 * 1024; // 32KB
Expand All @@ -59,7 +59,7 @@ public void Constructor_WithNullChunkBufferSize_LeavesPropertyNull()
{
// Arrange
int concurrentRequests = 10;
int maxInMemoryParts = 5;
int? maxInMemoryParts = 5;
int bufferSize = 8192;
long targetPartSize = 8 * 1024 * 1024; // 8MB

Expand Down Expand Up @@ -162,6 +162,22 @@ public void Constructor_WithZeroConcurrentRequests_ThrowsException()
var config = new BufferedDownloadConfiguration(0, 5, 8192, 8 * 1024 * 1024);
}

[TestMethod]
public void Constructor_WithNullMaxInMemoryParts_AcceptsValue()
{
// Arrange
int concurrentRequests = 10;
int? maxInMemoryParts = null;
int bufferSize = 8192;
long targetPartSize = 8 * 1024 * 1024; // 8MB

// Act
var config = new BufferedDownloadConfiguration(concurrentRequests, maxInMemoryParts, bufferSize, targetPartSize);

// Assert
Assert.IsNull(config.MaxInMemoryParts);
}

[TestMethod]
[ExpectedException(typeof(ArgumentOutOfRangeException))]
public void Constructor_WithNegativeMaxInMemoryParts_ThrowsException()
Expand Down