I have a scalability concern with using ListDirectory from ssh.net on a potentially very large list of files (hundreds or thousands of 3-5kb files). When getting a file, I need to open it, download it, create an internal object, and then delete it from the server. How can I make this code as scalable as possible?
Using code like this, I'm thinking it will choke once there are a large number of files.
{
var ftpInfo = await Task.WhenAll(config.url, config.port, config.userName, config.password);
var ftpUrl = ftpInfo[0];
var sftpPort = ftpInfo[1];
var ftpUsername = ftpInfo[2];
var ftpPassword = ftpInfo[3];
try
{
using (var sftp = new SftpClient(ftpUrl, int.Parse(sftpPort), ftpUsername, ftpPassword))
{
sftp.Connect();
var filesToDownload = sftp.ListDirectory(ReceiveFilesDir).Where(x => x.Name.StartsWith("example")).ToList();
var exampleFiles = filesToDownload.Select(x =>
new ExampleFile
{
Name = x.Name,
Content = sftp.ReadAllText(x.FullName),
FileIdentifiers = new S3FileIdentifiers()
{
FileId = Guid.NewGuid().ToString(),
BatchId = batchId,
BatchDate = batchDate
}
}
).ToList();
filesToDownload.ForEach(f => f.Delete());
return exampleFiles;
}
}
catch(SocketException ex)
{
Logger.Warning(ex, "Failed to retrieve files from ExampleFileSFTP");
return new List<ExampleFile>();
}
}