feature parity for update UploadFiles and DownloadFiles sync

FluentFTP/Client/AsyncClient/DownloadFiles.cs

@@ -21,9 +21,9 @@ public partial class AsyncFtpClient {
 		/// Supports very large files since it downloads data in chunks.
 		/// Same speed as <see cref="o:DownloadFile"/>.
 		/// </summary>
-		/// <param name="localDir">The full or relative path to the directory that files will be downloaded.</param>
+		/// <param name="localDir">The full or relative path to the directory that files will be downloaded into.</param>
 		/// <param name="remotePaths">The full or relative paths to the files on the server</param>
-		/// <param name="existsMode">Overwrite if you want the local file to be overwritten if it already exists. Append will also create a new file if it doesn't exists</param>
+		/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
 		/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
 		/// <param name="errorHandling">Used to determine how errors are handled</param>
 		/// <param name="token">The token that can be used to cancel the entire process</param>
@@ -142,7 +142,7 @@ public async Task<List<FtpResult>> DownloadFiles(string localDir, IEnumerable<st
 		}
 
 		/// <summary>
-		/// Remove successful downloads.
+		/// Remove successfully downloaded files.
 		/// </summary>
 		protected void PurgeSuccessfulDownloads(IEnumerable<string> localFiles) {
 			foreach (var localFile in localFiles) {

FluentFTP/Client/AsyncClient/UploadFiles.cs

@@ -23,7 +23,8 @@ public partial class AsyncFtpClient {
 		/// </summary>
 		/// <param name="localPaths">The full or relative paths to the files on the local file system. Files can be from multiple folders.</param>
 		/// <param name="remoteDir">The full or relative path to the directory that files will be uploaded on the server</param>
-		/// <param name="existsMode">What to do if the file already exists? Skip, overwrite or append? Set this to FtpExists.None for fastest performance but only if you are SURE that the files do not exist on the server.</param>
+		/// <param name="existsMode">What to do if the file already exists? Skip, overwrite or append? Set this to <see cref="FtpRemoteExists.NoCheck"/> for fastest performance,
+		///  but only if you are SURE that the files do not exist on the server.</param>
 		/// <param name="createRemoteDir">Create the remote directory if it does not exist.</param>
 		/// <param name="verifyOptions">Sets if checksum verification is required for a successful upload and what to do if it fails verification (See Remarks)</param>
 		/// <param name="errorHandling">Used to determine how errors are handled</param>
@@ -159,14 +160,48 @@ public async Task<List<FtpResult>> UploadFiles(IEnumerable<string> localPaths, s
 		}
 
 		/// <summary>
-		/// Remove the successfully uploaded files
+		/// Remove successfully uploaded files.
 		/// </summary>
 		protected async Task PurgeSuccessfulUploadsAsync(IEnumerable<string> remotePaths) {
 			foreach (var remotePath in remotePaths) {
 				await DeleteFile(remotePath);
 			}
 		}
 
+		/// <summary>
+		/// Uploads the given file paths to a single folder on the server asynchronously.
+		/// All files are placed directly into the given folder regardless of their path on the local filesystem.
+		/// High-level API that takes care of various edge cases internally.
+		/// Supports very large files since it uploads data in chunks.
+		/// Faster than uploading single files with <see cref="o:UploadFile"/> since it performs a single "file exists" check rather than one check per file.
+		/// </summary>
+		/// <param name="localFiles">The full or relative paths to the files on the local file system. Files can be from multiple folders.</param>
+		/// <param name="remoteDir">The full or relative path to the directory that files will be uploaded on the server</param>
+		/// <param name="existsMode">What to do if the file already exists? Skip, overwrite or append? Set this to <see cref="FtpRemoteExists.NoCheck"/> for fastest performance,
+		///  but only if you are SURE that the files do not exist on the server.</param>
+		/// <param name="createRemoteDir">Create the remote directory if it does not exist.</param>
+		/// <param name="verifyOptions">Sets if checksum verification is required for a successful upload and what to do if it fails verification (See Remarks)</param>
+		/// <param name="errorHandling">Used to determine how errors are handled</param>
+		/// <param name="token">The token that can be used to cancel the entire process</param>
+		/// <param name="progress">Provide an implementation of IProgress to track upload progress.</param>
+		/// <param name="rules">Only files that pass all these rules are uploaded, and the files that don't pass are skipped.</param>
+		/// <returns>
+		/// Returns a listing of all the local files, indicating if they were uploaded, skipped or overwritten.
+		/// Returns a blank list if nothing was transferred. Never returns null.
+		/// </returns>
+		/// <remarks>
+		/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server.  If the server does not support
+		/// any hash algorithm, then verification is ignored.  If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful 
+		/// upload &amp; verification.  Additionally, if any verify option is set and a retry is attempted the existsMode will automatically be set to <see cref="FtpRemoteExists.Overwrite"/>.
+		/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
+		/// to propagate from this method.
+		/// </remarks>
+		public async Task<List<FtpResult>> UploadFiles(IEnumerable<FileInfo> localFiles, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true,
+			FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, CancellationToken token = default(CancellationToken),
+			IProgress<FtpProgress> progress = null, List<FtpRule> rules = null) {
+			return await UploadFiles(localFiles.Select(f => f.FullName), remoteDir, existsMode, createRemoteDir, verifyOptions, errorHandling, token, progress, rules);
+		}
+
 		/// <summary>
 		/// Get a list of all the files that need to be uploaded
 		/// </summary>

FluentFTP/Client/BaseClient/GetFilesToDownload.cs

@@ -43,7 +43,7 @@ protected List<FtpResult> GetFilesToDownload2(string localFolder, IEnumerable<st
 			foreach (var remotePath in remotePaths) {
 
 				// calc local path
-				var localPath = localFolder + remotePath.GetFtpFileName();
+				var localPath = localFolder.CombineLocalPath(remotePath.GetFtpFileName());
 
 				RecordFileToDownload(rules, results, shouldExist, toDownload, null, localPath, remotePath);
 

FluentFTP/Client/Interfaces/IAsyncFtpClient.cs

@@ -89,6 +89,7 @@ public interface IAsyncFtpClient : IDisposable, IBaseFtpClient {
 		// HIGH LEVEL
 
 		Task<List<FtpResult>> UploadFiles(IEnumerable<string> localPaths, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, CancellationToken token = default(CancellationToken), IProgress<FtpProgress> progress = null, List<FtpRule> rules = null);
+		Task<List<FtpResult>> UploadFiles(IEnumerable<FileInfo> localFiles, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, CancellationToken token = default(CancellationToken), IProgress<FtpProgress> progress = null, List<FtpRule> rules = null);
 		Task<List<FtpResult>> DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, CancellationToken token = default(CancellationToken), IProgress<FtpProgress> progress = null, List<FtpRule> rules = null);
 
 		Task<FtpStatus> UploadFile(string localPath, string remotePath, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = false, FtpVerify verifyOptions = FtpVerify.None, IProgress<FtpProgress> progress = null, CancellationToken token = default(CancellationToken));

FluentFTP/Client/Interfaces/IFtpClient.cs

@@ -81,9 +81,9 @@ public interface IFtpClient : IDisposable, IBaseFtpClient {
 
 		// HIGH LEVEL
 
-		int UploadFiles(IEnumerable<string> localPaths, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null);
-		int UploadFiles(IEnumerable<FileInfo> localFiles, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null);
-		int DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null);
+		List<FtpResult> UploadFiles(IEnumerable<string> localPaths, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null, List<FtpRule> rules = null);
+		List<FtpResult> UploadFiles(IEnumerable<FileInfo> localFiles, string remoteDir, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = true, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null, List<FtpRule> rules = null);
+		List<FtpResult> DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null, List<FtpRule> rules = null);
 
 		FtpStatus UploadFile(string localPath, string remotePath, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = false, FtpVerify verifyOptions = FtpVerify.None, Action<FtpProgress> progress = null);
 		FtpStatus UploadStream(Stream fileStream, string remotePath, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, bool createRemoteDir = false, Action<FtpProgress> progress = null);

FluentFTP/Client/SyncClient/DownloadFiles.cs

@@ -9,6 +9,8 @@
 using FluentFTP.Client.Modules;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Data;
+using FluentFTP.Rules;
 
 namespace FluentFTP {
 	public partial class FtpClient {
@@ -24,17 +26,21 @@ public partial class FtpClient {
 		/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
 		/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
 		/// <param name="errorHandling">Used to determine how errors are handled</param>
-		/// <param name="progress">Provide a callback to track upload progress.</param>
-		/// <returns>The count of how many files were downloaded successfully. When existing files are skipped, they are not counted.</returns>
+		/// <param name="progress">Provide an implementation of IProgress to track upload progress.</param>
+		/// <param name="rules">Only files that pass all these rules are downloaded, and the files that don't pass are skipped.</param>
+		/// <returns>
+		/// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten.
+		/// Returns a blank list if nothing was transferred. Never returns null.
+		/// </returns>
 		/// <remarks>
 		/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server.  If the server does not support
 		/// any hash algorithm, then verification is ignored.  If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful 
 		/// upload &amp; verification.  Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
 		/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
 		/// to propagate from this method.
 		/// </remarks>
-		public int DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None,
-			FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null) {
+		public List<FtpResult> DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None,
+			FtpError errorHandling = FtpError.None, Action<FtpProgress> progress = null, List<FtpRule> rules = null) {
 
 			// verify args
 			if (!errorHandling.IsValidCombination()) {
@@ -47,36 +53,51 @@ public int DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLo
 
 			LogFunction(nameof(DownloadFiles), new object[] { localDir, remotePaths, existsMode, verifyOptions });
 
+
+			// result vars
 			var errorEncountered = false;
 			var successfulDownloads = new List<string>();
+			var results = new List<FtpResult>();
+			var shouldExist = new Dictionary<string, bool>(); // paths of the files that should exist (lowercase for CI checks)
 
 			// ensure ends with slash
 			localDir = !localDir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? localDir + Path.DirectorySeparatorChar.ToString() : localDir;
 
+			// check which files should be downloaded or filtered out based on rules
+			var filesToDownload = GetFilesToDownload2(localDir, remotePaths, rules, results, shouldExist);
+
 			// per remote file
 			var r = -1;
-			foreach (var remotePath in remotePaths) {
+			foreach (var result in filesToDownload) {
 				r++;
 
-				// calc local path
-				var localPath = localDir + remotePath.GetFtpFileName();
-
 				// create meta progress to store the file progress
 				var metaProgress = new FtpProgress(remotePaths.Count(), r);
 
 				// try to download it
 				try {
-					var ok = DownloadFileToFile(localPath, remotePath, existsMode, verifyOptions, progress, metaProgress);
+					var ok = DownloadFileToFile(result.LocalPath, result.RemotePath, existsMode, verifyOptions, progress, metaProgress);
+
+					// mark that the file succeeded
+					result.IsSuccess = ok.IsSuccess();
+					result.IsSkipped = ok == FtpStatus.Skipped;
+
 					if (ok.IsSuccess()) {
-						successfulDownloads.Add(localPath);
+						successfulDownloads.Add(result.LocalPath);
 					}
 					else if ((int)errorHandling > 1) {
 						errorEncountered = true;
 						break;
 					}
 				}
 				catch (Exception ex) {
-					LogWithPrefix(FtpTraceLevel.Error, "Failed to download " + remotePath, ex);
+
+					// mark that the file failed
+					result.IsFailed = true;
+					result.Exception = ex;
+
+					LogWithPrefix(FtpTraceLevel.Error, "Failed to download " + result.RemotePath, ex);
+
 					if (errorHandling.HasFlag(FtpError.Stop)) {
 						errorEncountered = true;
 						break;
@@ -105,13 +126,12 @@ public int DownloadFiles(string localDir, IEnumerable<string> remotePaths, FtpLo
 				}
 			}
 
-			return successfulDownloads.Count;
+			return results;
 		}
 
 		/// <summary>
-		/// Remove successfully downloaded files
+		/// Remove successfully downloaded files.
 		/// </summary>
-		/// <param name="localFiles"></param>
 		protected void PurgeSuccessfulDownloads(IEnumerable<string> localFiles) {
 			foreach (var localFile in localFiles) {
 				// absorb any errors because we don't want this to throw more errors!