diff --git a/.gitmodules b/.gitmodules index 50873ff..b4e1200 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "SSH.NET"] path = SSH.NET - url = https://github.com/hippiehunter/SSH.NET.git + url = git@github.com:hippiehunter/SSH.NET.git diff --git a/SFTPSync/Program.cs b/SFTPSync/Program.cs new file mode 100644 index 0000000..6c08a35 --- /dev/null +++ b/SFTPSync/Program.cs @@ -0,0 +1,32 @@ + +using Microsoft.Extensions.Configuration; +using System.CommandLine; +using System.IO; + +var configOption = new Option("--config") +{ + Arity = ArgumentArity.ZeroOrOne, + Description = "Path to the configuration file" +}; + +var rootCommand = SFTPSync.SFTPSync.GetRootCommand(); +rootCommand.Options.Add(configOption); + +// Parse args to get config path +var tempParseResult = rootCommand.Parse(args); + +var configPath = tempParseResult.GetValue(configOption) ?? "sftpsyncsettings.json"; +if (!Path.IsPathRooted(configPath)) +{ + configPath = Path.Combine(AppContext.BaseDirectory, configPath); +} + +var config = new ConfigurationBuilder() + .AddJsonFile(configPath, optional: true) + .Build(); + +// Now create the real rootCommand with config +rootCommand = SFTPSync.SFTPSync.GetRootCommand(config); +rootCommand.Options.Add(configOption); +var parseResult = rootCommand.Parse(args); +await parseResult.InvokeAsync(); diff --git a/SFTPSync/Properties/launchSettings.json b/SFTPSync/Properties/launchSettings.json index bde6ef9..1ededaa 100644 --- a/SFTPSync/Properties/launchSettings.json +++ b/SFTPSync/Properties/launchSettings.json @@ -2,7 +2,19 @@ "profiles": { "RemoteSFTPSyncCore": { "commandName": "Project", - "commandLineArgs": "VMSIT reliance1 reliance1 C:\\WINDEV\\WIN /DISK$VMS84/RELIANCE1/WIN/ \"*.DBL;*.C;*.H;*.COM;*.BAS;*.NME;*.MAR;*.HDF;*.FDL;*.OPT;*.EXP;*.HDR;*.MMS;*.FORM;*.SCR\"" + "commandLineArgs": "-h VMSIT -u reliance1 -l C:\\WINDEV\\WIN -r /DISK$VMS84/RELIANCE1/WIN/ -s \"*.DBL;*.C;*.H;*.COM;*.BAS;*.NME;*.MAR;*.HDF;*.FDL;*.OPT;*.EXP;*.HDR;*.MMS;*.FORM;*.SCR\" --password reliance1" + }, + "RemoteSFTPSyncCoreBrian": { + "commandName": "Project", + "commandLineArgs": "--host EMJDV1 -u 1brian -l D:\\synctest -r /DKC6/1Brian/synctest/ -s \"*.DBL;*.C;*.H;*.COM;*.BAS;*.NME;*.MAR;*.HDF;*.FDL;*.OPT;*.EXP;*.HDR;*.MMS;*.FORM;*.SCR\" --one-time -i c:\\Users\\bshelledy\\.ssh\\id_rsa" + }, + "RemoteSFTPSyncCoreEmpty": { + "commandName": "Project", + "commandLineArgs": "" + }, + "RemoteSFTPSyncCoreConfig": { + "commandName": "Project", + "commandLineArgs": "--config sftpsyncsettings.json --one-time -i c:\\Users\\bshelledy\\.ssh\\id_rsa" } } -} \ No newline at end of file + } \ No newline at end of file diff --git a/SFTPSync/SFTPSync.cs b/SFTPSync/SFTPSync.cs index a812003..f626920 100644 --- a/SFTPSync/SFTPSync.cs +++ b/SFTPSync/SFTPSync.cs @@ -1,30 +1,96 @@  +using Microsoft.Extensions.Configuration; using SFTPSyncLib; +using System.CommandLine; namespace SFTPSync { class SFTPSync { - static async Task Main(string[] args) + public static RootCommand GetRootCommand(Microsoft.Extensions.Configuration.IConfiguration? config = null) { - if (args.Length != 6) - { - Console.WriteLine("usage: SFTPSync host username password localRootDir remoteRootDir searchPattern"); - } - else + var rootCommand = new RootCommand("SFTP Sync is a utility application that can synchronize a local directory\nstructure and files to a remote OpenVMS system via the Secure FTP protocol. "); + var hostArg = new Option("--host", "-h") { Description = "The SFTP host to connect to" }; + var userNameArg = new Option("--username", "-u") { Description = "The username for SFTP authentication" }; + var passwordArg = new Option("--password") { Description = "The password for SFTP authentication" }; + var localRootDirArg = new Option("--localRootDir", "-l") { Description = "The local root directory to sync from" }; + var remoteRootDirArg = new Option("--remoteRootDir", "-r") { Description = "The remote root directory to sync to" }; + var searchPatternArg = new Option("--searchPattern", "-s") { Description = "The semicolon seperated search pattern for files to sync (e.g. *.txt or *.jpg;*.png)" }; + var oneTimeOption = new Option("--one-time", "-o") { Description = "Perform a one-time sync and exit" }; + var identityOption = new Option("--identity", "-i", "-id") { Description = "Path to the identity file for authentication" }; + rootCommand.Add(hostArg); + rootCommand.Add(userNameArg); + rootCommand.Add(passwordArg); + rootCommand.Add(localRootDirArg); + rootCommand.Add(remoteRootDirArg); + rootCommand.Add(searchPatternArg); + rootCommand.Add(oneTimeOption); + rootCommand.Add(identityOption); + + + + rootCommand.SetAction(async (parseResult) => { - var director = new SyncDirector(args[3]); - List remoteSyncWorkers = new List(); + // Map SFTPSyncUI config keys to SFTPSync expected keys + static string? GetConfigValue(Microsoft.Extensions.Configuration.IConfiguration? config, params string[] keys) + { + if (config == null) return null; + foreach (var key in keys) + { + var val = config[key]; + if (!string.IsNullOrEmpty(val)) return val; + } + return null; + } + + + string? GetOptionOrConfig(Option opt, params string[] configKeys) + { + var val = parseResult.GetValue(opt); + if (!string.IsNullOrEmpty(val)) return val!; + var configVal = GetConfigValue(config, configKeys.Length > 0 ? configKeys : new[] { opt.Name.TrimStart('-') }); + if (!string.IsNullOrEmpty(configVal)) return configVal!; + return null; + } + + // Map SFTPSyncUI keys to SFTPSync args/options + string? host = GetOptionOrConfig(hostArg, "host", "RemoteHost"); + if (string.IsNullOrWhiteSpace(host)) + { + throw new Exception("No Host specified."); + } + string? username = GetOptionOrConfig(userNameArg, "username", "RemoteUsername"); + if (string.IsNullOrWhiteSpace(username)) + { + throw new Exception("No Username specified."); + } + string? password = GetOptionOrConfig(passwordArg, "password", "RemotePassword"); + string? localRootDir = GetOptionOrConfig(localRootDirArg, "localRootDir", "LocalPath"); + if (string.IsNullOrWhiteSpace(localRootDir)) + { + throw new Exception("No Local Root Directory specified."); + } + string? remoteRootDir = GetOptionOrConfig(remoteRootDirArg, "remoteRootDir", "RemotePath"); + if (string.IsNullOrWhiteSpace(remoteRootDir)) + { + throw new Exception("No Remote Root Directory specified."); + } + string? searchPattern = GetOptionOrConfig(searchPatternArg, "searchPattern", "LocalSearchPattern"); + bool oneTime = parseResult.GetValue(oneTimeOption); + string? identityFile = GetOptionOrConfig(identityOption, "identity"); + var excludedDirs = config?.GetSection("ExcludedDirectories").Get() ?? []; + var director = new SyncDirector(localRootDir); + List remoteSyncWorkers = []; Logger.LogInfo("Starting initial sync..."); - foreach (var pattern in args[5].Split(';', StringSplitOptions.RemoveEmptyEntries)) + foreach (var pattern in searchPattern?.Split(';', StringSplitOptions.RemoveEmptyEntries) ?? []) { if (remoteSyncWorkers.Count > 0) { await remoteSyncWorkers[0].DoneMakingFolders; } - remoteSyncWorkers.Add(new RemoteSync(args[0], args[1], args[2], args[3], args[4], pattern, remoteSyncWorkers.Count == 0, director, null, false, remoteSyncWorkers.Count == 0)); + remoteSyncWorkers.Add(new RemoteSync(host, username, password, localRootDir, remoteRootDir, pattern, remoteSyncWorkers.Count == 0, director, [.. excludedDirs], false, remoteSyncWorkers.Count == 0, identityFile)); Logger.LogInfo($"Started sync worker {remoteSyncWorkers.Count} for pattern {pattern}"); } @@ -32,10 +98,14 @@ static async Task Main(string[] args) //Wait for all sync workers to finish initial sync then tell the user await Task.WhenAll(remoteSyncWorkers.Select(rsw => rsw.DoneInitialSync)); + if (oneTime) + { + Logger.LogInfo("Sync Complete"); + return; + } Logger.LogInfo("Initial sync complete, real-time sync active"); Console.Write("Press Ctrl+C to exit: "); - while (true) { var key = Console.ReadKey(intercept: true); @@ -54,7 +124,8 @@ static async Task Main(string[] args) } catch { } } - } + }); + return rootCommand; } } } diff --git a/SFTPSync/SFTPSync.csproj b/SFTPSync/SFTPSync.csproj index 5484e4c..7be90dd 100644 --- a/SFTPSync/SFTPSync.csproj +++ b/SFTPSync/SFTPSync.csproj @@ -11,8 +11,20 @@ 1.9 + + + + + + + + + PreserveNewest + + + diff --git a/SFTPSync/sftpsyncsettings.json b/SFTPSync/sftpsyncsettings.json new file mode 100644 index 0000000..5fb1ab6 --- /dev/null +++ b/SFTPSync/sftpsyncsettings.json @@ -0,0 +1,17 @@ +{ + "StartAtLogin": true, + "StartInTray": true, + "AutoStartSync": true, + "AccessVerified": true, + "DeleteEnabled": false, + "LocalPath": "D:\\WINDEV\\WIN", + "LocalSearchPattern": "*.DBL;*.C;*.H;*.COM;*.BAS;*.NME;*.MAR;*.HDF;*.FDL;*.OPT;*.EXP;*.HDR;*.MMS;*.FORM;*.SCR;*.SCM;*.FDL", + "RemoteHost": "EMJDV1", + "RemoteUsername": "1Brian", + "RemotePath": "/DKC6/1Brian/synctest/win/", + "ExcludedDirectories": [ + "D:\\WINDEV\\WIN\\EMJREL\\SRC\\REP\\deprecated", + "D:\\WINDEV\\WIN\\NOTES", + "D:\\WINDEV\\WIN\\WINDOWS" + ] +} \ No newline at end of file diff --git a/SFTPSyncLib/RemoteSync.cs b/SFTPSyncLib/RemoteSync.cs index 0973ba5..a6f9252 100644 --- a/SFTPSyncLib/RemoteSync.cs +++ b/SFTPSyncLib/RemoteSync.cs @@ -1,6 +1,7 @@ using Renci.SshNet; using Renci.SshNet.Sftp; using System.Collections.Concurrent; +using System.Text; using System.Text.RegularExpressions; namespace SFTPSyncLib @@ -9,7 +10,7 @@ public class RemoteSync : IDisposable { string _host; string _username; - string _password; + string? _password; string _searchPattern; string _localRootDirectory; string _remoteRootDirectory; @@ -35,9 +36,41 @@ public class RemoteSync : IDisposable public Task DoneInitialSync { get; } - public RemoteSync(string host, string username, string password, + private SftpClient GetSftpClient(string host, string username, string? password, string? identityFilePath = null) + { + if (string.IsNullOrWhiteSpace(password)) + { + var defPath = string.IsNullOrEmpty(identityFilePath); + if (defPath) + { + identityFilePath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + "/.ssh/id_rsa"; + } + if (!File.Exists(identityFilePath)) + { + string message; + if (defPath) + { + message = "No password or identity key provided."; + } + else + { + message = $"No password provided and identity file not found at {identityFilePath}"; + } + Logger.LogError(message); + throw new InvalidOperationException(message); + } + var key = new Renci.SshNet.PrivateKeyFile(identityFilePath); + return new SftpClient(host, username, key); + } + else + { + return new SftpClient(host, username, password); + } + } + + public RemoteSync(string host, string username, string? password, string localRootDirectory, string remoteRootDirectory, - string searchPattern, bool createFolders, SyncDirector director, List? excludedFolders, bool deleteEnabled, bool handleDirectoryDeletes) + string searchPattern, bool createFolders, SyncDirector director, List? excludedFolders, bool deleteEnabled, bool handleDirectoryDeletes, string? identityFile = null) { _host = host; _username = username; @@ -48,8 +81,7 @@ public RemoteSync(string host, string username, string password, _director = director; _excludedFolders = excludedFolders ?? new List(); _deleteEnabled = deleteEnabled; - _sftp = new SftpClient(host, username, password); - + _sftp = GetSftpClient(host, username, password, identityFile); //The first instance is responsible for creating ALL of the the directories. //Subsequent instances will not be created until this one completes. @@ -74,9 +106,9 @@ public RemoteSync(string host, string username, string password, } } - public RemoteSync(string host, string username, string password, + public RemoteSync(string host, string username, string? password, string localRootDirectory, string remoteRootDirectory, - string searchPattern, SyncDirector director, List? excludedFolders, Task initialSyncTask, bool deleteEnabled, bool handleDirectoryDeletes) + string searchPattern, SyncDirector director, List? excludedFolders, Task initialSyncTask, bool deleteEnabled, bool handleDirectoryDeletes, string? identityFile = null) { _host = host; _username = username; @@ -87,8 +119,7 @@ public RemoteSync(string host, string username, string password, _director = director; _excludedFolders = excludedFolders ?? new List(); _deleteEnabled = deleteEnabled; - _sftp = new SftpClient(host, username, password); - + _sftp = GetSftpClient(host, username, password, identityFile); DoneMakingFolders = Task.CompletedTask; DoneInitialSync = initialSyncTask; @@ -169,7 +200,8 @@ public static async Task RunSharedInitialSyncAsync( await Task.WhenAll(workers); } - + // regex to find \r\n at the end of a line + private static readonly Regex LineEndingRegex = new Regex(@"\r\n|\r|\n", RegexOptions.Compiled); /// /// Sync changes for a file. This is only used for changes AFTER the initial sync has completed. /// @@ -198,9 +230,20 @@ public static void SyncFile(SftpClient sftp, string sourcePath, string destinati // Read the local file content var localFileContent = File.ReadAllText(sourcePath); - - // Write the remote file - sftp.WriteAllText(destinationPath, localFileContent); + using var fs = File.OpenRead(sourcePath); + using var sr = new StreamReader(fs); + var sb = new StringBuilder(); + if (sftp.Exists(destinationPath)) sftp.Delete(destinationPath); + var sw = sftp.CreateText(destinationPath); + string? line; + while((line = sr.ReadLine()) is not null) + { + sw.WriteLine(line.Replace("\r\n", "\n").Replace("\r", "\n")); + } + sw.Flush(); + sw.Close(); + sr.Close(); + fs.Close(); return; }