Skip to content

Commit 35021cc

Browse files
znamenapznamenap
committed
SftpClient Enumerates Rather Than Accumulates Directory Items (#395)
1 parent 43329ee commit 35021cc

File tree

3 files changed

+324
-4
lines changed

3 files changed

+324
-4
lines changed

‎src/Renci.SshNet.IntegrationTests/OldIntegrationTests/SftpClientTest.EnumerateDirectory.cs

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
using System.Diagnostics;
2+
3+
using Renci.SshNet.Common;
4+
5+
namespace Renci.SshNet.IntegrationTests.OldIntegrationTests
6+
{
7+
/// <summary>
8+
/// Implementation of the SSH File Transfer Protocol (SFTP) over SSH.
9+
/// </summary>
10+
public partial class SftpClientTest : IntegrationTestBase
11+
{
12+
[TestMethod]
13+
[TestCategory("Sftp")]
14+
[ExpectedException(typeof(SshConnectionException))]
15+
public void Test_Sftp_EnumerateDirectory_Without_Connecting()
16+
{
17+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
18+
{
19+
var files = sftp.EnumerateDirectory(".");
20+
foreach (var file in files)
21+
{
22+
Debug.WriteLine(file.FullName);
23+
}
24+
}
25+
}
26+
27+
[TestMethod]
28+
[TestCategory("Sftp")]
29+
[TestCategory("integration")]
30+
[ExpectedException(typeof(SftpPermissionDeniedException))]
31+
public void Test_Sftp_EnumerateDirectory_Permission_Denied()
32+
{
33+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
34+
{
35+
sftp.Connect();
36+
37+
var files = sftp.EnumerateDirectory("/root");
38+
foreach (var file in files)
39+
{
40+
Debug.WriteLine(file.FullName);
41+
}
42+
43+
sftp.Disconnect();
44+
}
45+
}
46+
47+
[TestMethod]
48+
[TestCategory("Sftp")]
49+
[TestCategory("integration")]
50+
[ExpectedException(typeof(SftpPathNotFoundException))]
51+
public void Test_Sftp_EnumerateDirectory_Not_Exists()
52+
{
53+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
54+
{
55+
sftp.Connect();
56+
57+
var files = sftp.EnumerateDirectory("/asdfgh");
58+
foreach (var file in files)
59+
{
60+
Debug.WriteLine(file.FullName);
61+
}
62+
63+
sftp.Disconnect();
64+
}
65+
}
66+
67+
[TestMethod]
68+
[TestCategory("Sftp")]
69+
[TestCategory("integration")]
70+
public void Test_Sftp_EnumerateDirectory_Current()
71+
{
72+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
73+
{
74+
sftp.Connect();
75+
76+
var files = sftp.EnumerateDirectory(".");
77+
78+
Assert.IsTrue(files.Count() > 0);
79+
80+
foreach (var file in files)
81+
{
82+
Debug.WriteLine(file.FullName);
83+
}
84+
85+
sftp.Disconnect();
86+
}
87+
}
88+
89+
[TestMethod]
90+
[TestCategory("Sftp")]
91+
[TestCategory("integration")]
92+
public void Test_Sftp_EnumerateDirectory_Empty()
93+
{
94+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
95+
{
96+
sftp.Connect();
97+
98+
var files = sftp.EnumerateDirectory(string.Empty);
99+
100+
Assert.IsTrue(files.Count() > 0);
101+
102+
foreach (var file in files)
103+
{
104+
Debug.WriteLine(file.FullName);
105+
}
106+
107+
sftp.Disconnect();
108+
}
109+
}
110+
111+
[TestMethod]
112+
[TestCategory("Sftp")]
113+
[TestCategory("integration")]
114+
[Description("Test passing null to EnumerateDirectory.")]
115+
[ExpectedException(typeof(ArgumentNullException))]
116+
public void Test_Sftp_EnumerateDirectory_Null()
117+
{
118+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
119+
{
120+
sftp.Connect();
121+
122+
var files = sftp.EnumerateDirectory(null);
123+
124+
Assert.IsTrue(files.Count() > 0);
125+
126+
foreach (var file in files)
127+
{
128+
Debug.WriteLine(file.FullName);
129+
}
130+
131+
sftp.Disconnect();
132+
}
133+
}
134+
135+
[TestMethod]
136+
[TestCategory("Sftp")]
137+
[TestCategory("integration")]
138+
public void Test_Sftp_EnumerateDirectory_HugeDirectory()
139+
{
140+
var stopwatch = Stopwatch.StartNew();
141+
try
142+
{
143+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
144+
{
145+
sftp.Connect();
146+
sftp.ChangeDirectory("/home/" + User.UserName);
147+
148+
var count = 10000;
149+
// Create 10000 directory items
150+
for (int i = 0; i < count; i++)
151+
{
152+
sftp.CreateDirectory(string.Format("test_{0}", i));
153+
}
154+
Debug.WriteLine(string.Format("Created {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
155+
156+
stopwatch.Reset();
157+
stopwatch.Start();
158+
var files = sftp.EnumerateDirectory(".");
159+
Debug.WriteLine(string.Format("Listed {0} directories within {1} seconds", count, stopwatch.Elapsed.TotalSeconds));
160+
161+
// Ensure that directory has at least 10000 items
162+
stopwatch.Reset();
163+
stopwatch.Start();
164+
var actualCount = files.Count();
165+
Assert.IsTrue(actualCount >= count);
166+
Debug.WriteLine(string.Format("Used {0} items within {1} seconds", actualCount, stopwatch.Elapsed.TotalSeconds));
167+
168+
sftp.Disconnect();
169+
}
170+
}
171+
finally
172+
{
173+
stopwatch.Reset();
174+
stopwatch.Start();
175+
RemoveAllFiles();
176+
stopwatch.Stop();
177+
Debug.WriteLine(string.Format("Removed all files within {0} seconds", stopwatch.Elapsed.TotalSeconds));
178+
}
179+
}
180+
181+
[TestMethod]
182+
[TestCategory("Sftp")]
183+
[TestCategory("integration")]
184+
[ExpectedException(typeof(SshConnectionException))]
185+
public void Test_Sftp_EnumerateDirectory_After_Disconnected()
186+
{
187+
try
188+
{
189+
using (var sftp = new SftpClient(SshServerHostName, SshServerPort, User.UserName, User.Password))
190+
{
191+
sftp.Connect();
192+
193+
sftp.CreateDirectory("test_at_dsiposed");
194+
195+
var files = sftp.EnumerateDirectory(".").Take(1);
196+
197+
sftp.Disconnect();
198+
199+
// Must fail on disconnected session.
200+
var count = files.Count();
201+
}
202+
}
203+
finally
204+
{
205+
RemoveAllFiles();
206+
}
207+
}
208+
}
209+
}

‎src/Renci.SshNet/ISftpClient.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,28 @@ public interface ISftpClient : IBaseClient, IDisposable
717717
IAsyncEnumerable<ISftpFile> ListDirectoryAsync(string path, CancellationToken cancellationToken);
718718
#endif //FEATURE_ASYNC_ENUMERABLE
719719

720+
/// <summary>
721+
/// Enumerates files and directories in remote directory.
722+
/// </summary>
723+
/// <remarks>
724+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
725+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
726+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
727+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
728+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
729+
/// </remarks>
730+
/// <param name="path">The path.</param>
731+
/// <param name="listCallback">The list callback.</param>
732+
/// <returns>
733+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
734+
/// </returns>
735+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
736+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
737+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
738+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
739+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
740+
IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null);
741+
720742
/// <summary>
721743
/// Opens a <see cref="SftpFileStream"/> on the specified path with read/write access.
722744
/// </summary>

‎src/Renci.SshNet/SftpClient.cs

Lines changed: 93 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Diagnostics.CodeAnalysis;
4-
using System.IO;
54
using System.Globalization;
5+
using System.IO;
66
using System.Net;
77
using System.Text;
88
using System.Threading;
9+
using System.Threading.Tasks;
10+
911
using Renci.SshNet.Abstractions;
1012
using Renci.SshNet.Common;
1113
using Renci.SshNet.Sftp;
12-
using System.Threading.Tasks;
1314
#if FEATURE_ASYNC_ENUMERABLE
1415
using System.Runtime.CompilerServices;
1516
#endif
@@ -706,6 +707,33 @@ public IEnumerable<ISftpFile> EndListDirectory(IAsyncResult asyncResult)
706707
return ar.EndInvoke();
707708
}
708709

710+
/// <summary>
711+
/// Enumerates files and directories in remote directory.
712+
/// </summary>
713+
/// <remarks>
714+
/// This method differs to <see cref="ListDirectory(string, Action{int})"/> in the way how the items are returned.
715+
/// It yields the items to the last moment for the enumerator to decide if it needs to continue or stop enumerating the items.
716+
/// It is handy in case of really huge directory contents at remote server - meaning really huge 65 thousand files and more.
717+
/// It also decrease the memory footprint and avoids LOH allocation as happen per call to <see cref="ListDirectory(string, Action{int})"/> method.
718+
/// There aren't asynchronous counterpart methods to this because enumerating should happen in your specific asynchronous block.
719+
/// </remarks>
720+
/// <param name="path">The path.</param>
721+
/// <param name="listCallback">The list callback.</param>
722+
/// <returns>
723+
/// An <see cref="System.Collections.Generic.IEnumerable{SftpFile}"/> of files and directories ready to be enumerated.
724+
/// </returns>
725+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
726+
/// <exception cref="SshConnectionException">Client is not connected.</exception>
727+
/// <exception cref="SftpPermissionDeniedException">Permission to list the contents of the directory was denied by the remote host. <para>-or-</para> A SSH command was denied by the server.</exception>
728+
/// <exception cref="SshException">A SSH error where <see cref="Exception.Message" /> is the message from the remote host.</exception>
729+
/// <exception cref="ObjectDisposedException">The method was called after the client was disposed.</exception>
730+
public IEnumerable<SftpFile> EnumerateDirectory(string path, Action<int> listCallback = null)
731+
{
732+
CheckDisposed();
733+
734+
return InternalEnumerateDirectory(path, listCallback);
735+
}
736+
709737
/// <summary>
710738
/// Gets reference to remote file or directory.
711739
/// </summary>
@@ -1613,7 +1641,7 @@ public Task<SftpFileStream> OpenAsync(string path, FileMode mode, FileAccess acc
16131641

16141642
cancellationToken.ThrowIfCancellationRequested();
16151643

1616-
return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int)_bufferSize, cancellationToken);
1644+
return SftpFileStream.OpenAsync(_sftpSession, path, mode, access, (int) _bufferSize, cancellationToken);
16171645
}
16181646

16191647
/// <summary>
@@ -2269,7 +2297,7 @@ private IEnumerable<FileInfo> InternalSynchronizeDirectories(string sourcePath,
22692297
/// <param name="path">The path.</param>
22702298
/// <param name="listCallback">The list callback.</param>
22712299
/// <returns>
2272-
/// A list of files in the specfied directory.
2300+
/// A list of files in the specified directory.
22732301
/// </returns>
22742302
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
22752303
/// <exception cref="SshConnectionException">Client not connected.</exception>
@@ -2324,6 +2352,67 @@ private IEnumerable<ISftpFile> InternalListDirectory(string path, Action<int> li
23242352
return result;
23252353
}
23262354

2355+
/// <summary>
2356+
/// Internals the list directory.
2357+
/// </summary>
2358+
/// <param name="path">The path.</param>
2359+
/// <param name="listCallback">The list callback.</param>
2360+
/// <returns>
2361+
/// A list of files in the specified directory.
2362+
/// </returns>
2363+
/// <exception cref="ArgumentNullException"><paramref name="path" /> is <b>null</b>.</exception>
2364+
/// <exception cref="SshConnectionException">Client not connected.</exception>
2365+
private IEnumerable<SftpFile> InternalEnumerateDirectory(string path, Action<int> listCallback)
2366+
{
2367+
if (path == null)
2368+
{
2369+
throw new ArgumentNullException("path");
2370+
}
2371+
2372+
if (_sftpSession == null)
2373+
{
2374+
throw new SshConnectionException("Client not connected.");
2375+
}
2376+
2377+
var fullPath = _sftpSession.GetCanonicalPath(path);
2378+
2379+
var handle = _sftpSession.RequestOpenDir(fullPath);
2380+
2381+
var basePath = fullPath;
2382+
2383+
if (!basePath.EndsWith("/"))
2384+
{
2385+
basePath = string.Format("{0}/", fullPath);
2386+
}
2387+
2388+
try
2389+
{
2390+
var count = 0;
2391+
var files = _sftpSession.RequestReadDir(handle);
2392+
2393+
while (files != null)
2394+
{
2395+
count += files.Length;
2396+
// Call callback to report number of files read
2397+
if (listCallback != null)
2398+
{
2399+
// Execute callback on different thread
2400+
ThreadAbstraction.ExecuteThread(() => listCallback(count));
2401+
}
2402+
foreach (var file in files)
2403+
{
2404+
var fullName = string.Format(CultureInfo.InvariantCulture, "{0}{1}", basePath, file.Key);
2405+
yield return new SftpFile(_sftpSession, fullName, file.Value);
2406+
}
2407+
files = _sftpSession.RequestReadDir(handle);
2408+
}
2409+
}
2410+
finally
2411+
{
2412+
_sftpSession.RequestClose(handle);
2413+
}
2414+
}
2415+
23272416
/// <summary>
23282417
/// Internals the download file.
23292418
/// </summary>

0 commit comments

Comments
 (0)