Add support for resuming SFTP file upload/download

src/Renci.SshNet/IServiceFactory.cs

@@ -90,10 +90,11 @@ internal partial interface IServiceFactory
  /// <param name="fileName">The file to read.</param>
  /// <param name="sftpSession">The SFTP session to use.</param>
  /// <param name="bufferSize">The size of buffer.</param>
+ /// <param name="offset">The offset to resume from.</param>
  /// <returns>
  /// An <see cref="ISftpFileReader"/>.
  /// </returns>
- ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize);
+ ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0);
 
  /// <summary>
  /// Creates a new <see cref="ISftpResponseFactory"/> instance.

src/Renci.SshNet/ServiceFactory.cs

@@ -127,10 +127,11 @@ public INetConfSession CreateNetConfSession(ISession session, int operationTimeo
  /// <param name="fileName">The file to read.</param>
  /// <param name="sftpSession">The SFTP session to use.</param>
  /// <param name="bufferSize">The size of buffer.</param>
+ /// <param name="offset">The offset to resume from.</param>
  /// <returns>
  /// An <see cref="ISftpFileReader"/>.
  /// </returns>
- public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize)
+ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSession, uint bufferSize, ulong offset = 0)
  {
  const int defaultMaxPendingReads = 10;
 
@@ -152,7 +153,9 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
  {
  var fileAttributes = sftpSession.EndLStat(statAsyncResult);
  fileSize = fileAttributes.Size;
- maxPendingReads = Math.Min(100, (int)Math.Ceiling((double)fileAttributes.Size / chunkSize) + 1);
+
+ // calculate maxPendingReads based on remaining size, not total filesize (needed for resume support)
+ maxPendingReads = Math.Min(100, (int) Math.Ceiling((double)(fileSize - (long)offset) / chunkSize) + 1);
  }
  catch (SshException ex)
  {
@@ -162,7 +165,7 @@ public ISftpFileReader CreateSftpFileReader(string fileName, ISftpSession sftpSe
  DiagnosticAbstraction.Log(string.Format("Failed to obtain size of file. Allowing maximum {0} pending reads: {1}", maxPendingReads, ex));
  }
 
- return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+ return sftpSession.CreateFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
  }
 
  /// <summary>

src/Renci.SshNet/Sftp/ISftpSession.cs

@@ -494,10 +494,11 @@ void RequestWrite(byte[] handle,
  /// <param name="chunkSize">The maximum number of bytes to read with each chunk.</param>
  /// <param name="maxPendingReads">The maximum number of pending reads.</param>
  /// <param name="fileSize">The size of the file or <see langword="null"/> when the size could not be determined.</param>
+ /// <param name="offset">The offset to resume from.</param>
  /// <returns>
  /// An <see cref="ISftpFileReader"/> for reading the content of the file represented by the
  /// specified <paramref name="handle"/>.
  /// </returns>
- ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize);
+ ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0);
  }
 }

src/Renci.SshNet/Sftp/SftpFileReader.cs

@@ -56,7 +56,8 @@ internal sealed class SftpFileReader : ISftpFileReader
  /// <param name="chunkSize">The size of a individual read-ahead chunk.</param>
  /// <param name="maxPendingReads">The maximum number of pending reads.</param>
  /// <param name="fileSize">The size of the file, if known; otherwise, <see langword="null"/>.</param>
- public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+ /// <param name="offset">The offset to resume from.</param>
+ public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
  {
  _handle = handle;
  _sftpSession = sftpSession;
@@ -69,6 +70,11 @@ public SftpFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, i
  _disposingWaitHandle = new ManualResetEvent(initialState: false);
  _waitHandles = _sftpSession.CreateWaitHandleArray(_disposingWaitHandle, _semaphore.AvailableWaitHandle);
 
+ // When resuming a download (offset > 0), set the initial offset of the remote file to
+ // the same offset as the local output file. Read-ahead also starts at the same offset.
+ _offset = offset;
+ _readAheadOffset = offset;
+
  StartReadAhead();
  }
 

src/Renci.SshNet/Sftp/SftpSession.cs

@@ -239,13 +239,14 @@ public async Task<string> GetCanonicalPathAsync(string path, CancellationToken c
  /// <param name="chunkSize">The maximum number of bytes to read with each chunk.</param>
  /// <param name="maxPendingReads">The maximum number of pending reads.</param>
  /// <param name="fileSize">The size of the file or <see langword="null"/> when the size could not be determined.</param>
+ /// <param name="offset">The offset to resume from.</param>
  /// <returns>
  /// An <see cref="ISftpFileReader"/> for reading the content of the file represented by the
  /// specified <paramref name="handle"/>.
  /// </returns>
- public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize)
+ public ISftpFileReader CreateFileReader(byte[] handle, ISftpSession sftpSession, uint chunkSize, int maxPendingReads, long? fileSize, ulong offset = 0)
  {
- return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize);
+ return new SftpFileReader(handle, sftpSession, chunkSize, maxPendingReads, fileSize, offset);
  }
 
  internal string GetFullRemotePath(string path)

src/Renci.SshNet/SftpClient.cs

@@ -979,7 +979,12 @@ public void UploadFile(Stream input, string path, bool canOverride, Action<ulong
 
  var flags = Flags.Write | Flags.Truncate;
 
- if (canOverride)
+ if (input.Position > 0)
+ {
+ // if the local stream position is not zero, open the remote file in APPEND mode to resume upload
+ flags = Flags.Write | Flags.Append;
+ }
+ else if (canOverride)
  {
  flags |= Flags.CreateNewOrOpen;
  }
@@ -1117,7 +1122,12 @@ public IAsyncResult BeginUploadFile(Stream input, string path, bool canOverride,
 
  var flags = Flags.Write | Flags.Truncate;
 
- if (canOverride)
+ if (input.Position > 0)
+ {
+ // if the local stream position is not zero, open the remote file in APPEND mode to resume upload
+ flags = Flags.Write | Flags.Append;
+ }
+ else if (canOverride)
  {
  flags |= Flags.CreateNewOrOpen;
  }
@@ -2432,7 +2442,8 @@ private void InternalUploadFile(Stream input, string path, Flags flags, SftpUplo
 
  var handle = _sftpSession.RequestOpen(fullPath, flags);
 
- ulong offset = 0;
+ // Set the initial offset of the remote file to the same as the local file to allow resuming
+ var offset = (ulong)input.Position;
 
  // create buffer of optimal length
  var buffer = new byte[_sftpSession.CalculateOptimalWriteLength(_bufferSize, handle)];

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_EndLStatThrowsSshException.cs

@@ -62,7 +62,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Throws(new SshException());
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, null))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 10, null, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsAlmostSixTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsEqualToChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsExactlyFiveTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 6, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLessThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 2, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsLittleMoreThanFiveTimesGreaterThanChunkSize.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 7, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsMoreThanMaxPendingReadsTimesChunkSize.cs

@@ -68,7 +68,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, _maxPendingReads, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, _maxPendingReads, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }
 

test/Renci.SshNet.Tests/Classes/ServiceFactoryTest_CreateSftpFileReader_FileSizeIsZero.cs

@@ -66,7 +66,7 @@ private void SetupMocks()
  .Setup(p => p.EndLStat(_statAsyncResult))
  .Returns(_fileAttributes);
  _sftpSessionMock.InSequence(seq)
- .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize))
+ .Setup(p => p.CreateFileReader(_handle, _sftpSessionMock.Object, _chunkSize, 1, _fileSize, 0))
  .Returns(_sftpFileReaderMock.Object);
  }