From 72b88b7083924084743940db5b40afcfd4406dbe Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Mon, 30 Aug 2021 20:10:27 +0200 Subject: [PATCH 1/4] Improve SFTP performance on medium-high latency connections - Increases maxPendingReads from 10 to 100 - Increases socket send/receive buffer to 10 SSH packets (320K) --- src/Renci.SshNet/Connection/ConnectorBase.cs | 2 +- src/Renci.SshNet/ServiceFactory.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Renci.SshNet/Connection/ConnectorBase.cs b/src/Renci.SshNet/Connection/ConnectorBase.cs index bdea64a6f..58b45960a 100644 --- a/src/Renci.SshNet/Connection/ConnectorBase.cs +++ b/src/Renci.SshNet/Connection/ConnectorBase.cs @@ -49,7 +49,7 @@ protected Socket SocketConnect(string host, int port, TimeSpan timeout) { SocketAbstraction.Connect(socket, ep, timeout); - const int socketBufferSize = 2 * Session.MaximumSshPacketSize; + const int socketBufferSize = 10 * Session.MaximumSshPacketSize; socket.SendBufferSize = socketBufferSize; socket.ReceiveBufferSize = socketBufferSize; return socket; diff --git a/src/Renci.SshNet/ServiceFactory.cs b/src/Renci.SshNet/ServiceFactory.cs index c41c3679e..961f0758b 100644 --- a/src/Renci.SshNet/ServiceFactory.cs +++ b/src/Renci.SshNet/ServiceFactory.cs @@ -143,7 +143,7 @@ public INetConfSession CreateNetConfSession(ISession session, int operationTimeo /// public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize) { - const int defaultMaxPendingReads = 3; + const int defaultMaxPendingReads = 10; // Issue #292: Avoid overlapping SSH_FXP_OPEN and SSH_FXP_LSTAT requests for the same file as this // causes a performance degradation on Sun SSH @@ -163,7 +163,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe { var fileAttributes = sftpSession.EndLStat(statAsyncResult); fileSize = fileAttributes.Size; - maxPendingReads = Math.Min(10, (int) Math.Ceiling((double) fileAttributes.Size / chunkSize) + 1); + maxPendingReads = Math.Min(100, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1); } catch (SshException ex) { From b32a422a2473f67a3879e712c861e718c479f197 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Mon, 30 Aug 2021 21:38:07 +0200 Subject: [PATCH 2/4] Fix merge --- src/Renci.SshNet/ServiceFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Renci.SshNet/ServiceFactory.cs b/src/Renci.SshNet/ServiceFactory.cs index 961f0758b..af30f11c8 100644 --- a/src/Renci.SshNet/ServiceFactory.cs +++ b/src/Renci.SshNet/ServiceFactory.cs @@ -163,7 +163,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe { var fileAttributes = sftpSession.EndLStat(statAsyncResult); fileSize = fileAttributes.Size; - maxPendingReads = Math.Min(100, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1); + maxPendingReads = Math.Min(100, (int)Math.Ceiling((double)fileAttributes.Size / chunkSize) + 1); } catch (SshException ex) { From 58c0b4f0cde69135459299914f20475a8fa42c8a Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Mon, 30 Aug 2021 22:11:38 +0200 Subject: [PATCH 3/4] Adjust SFTP FileReader testcases to accept new MaxPendingReads values --- ...ctoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs index ee3db7cba..6184a05d1 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs @@ -59,7 +59,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Throws(new SshException()); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 3, null)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, null)) .Returns(_sftpFileReaderMock.Object); } From cf08f18c8d114451bfa77f3daa74b67a303951b8 Mon Sep 17 00:00:00 2001 From: Pedro Fonseca Date: Wed, 1 Nov 2023 14:03:47 +0100 Subject: [PATCH 4/4] Fix CreateSftpFileReader testcase, make it dependant on MaxPendingReads constant --- ...er_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename test/Renci.SshNet.Tests/Classes/{ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs => ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs} (92%) diff --git a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs similarity index 92% rename from test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs rename to test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs index dad634472..8fe220c80 100644 --- a/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize.cs +++ b/test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs @@ -8,7 +8,7 @@ namespace Renci.SshNet.Tests.Classes { [TestClass] - public class ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesGreaterThanChunkSize + public class ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize { private ServiceFactory _serviceFactory; private Mock _sftpSessionMock; @@ -22,18 +22,20 @@ public class ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanTenTimesG private SftpFileAttributes _fileAttributes; private long _fileSize; private ISftpFileReader _actual; + private int _maxPendingReads; private void SetupData() { var random = new Random(); + _maxPendingReads = 100; _bufferSize = (uint)random.Next(1, int.MaxValue); _openAsyncResult = new SftpOpenAsyncResult(null, null); _handle = CryptoAbstraction.GenerateRandom(random.Next(1, 10)); _statAsyncResult = new SFtpStatAsyncResult(null, null); _fileName = random.Next().ToString(); _chunkSize = (uint) random.Next(1000, 5000); - _fileSize = _chunkSize * random.Next(11, 50); + _fileSize = _chunkSize * random.Next(_maxPendingReads + 1, _maxPendingReads * 2); _fileAttributes = new SftpFileAttributesBuilder().WithSize(_fileSize).Build(); } @@ -63,7 +65,7 @@ private void SetupMocks() .Setup(p => p.EndLStat(_statAsyncResult)) .Returns(_fileAttributes); _sftpSessionMock.InSequence(seq) - .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, _fileSize)) + .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, _maxPendingReads, _fileSize)) .Returns(_sftpFileReaderMock.Object); }