Add retry to connection attempts for the PiscalSshClient

Core/Remote/PiscalSshClient.cs

@@ -5,7 +5,9 @@ using System.IO;
 using System.Linq;
 using log4net;
 using LeafWeb.Core.Utility;
+using Polly;
 using Renci.SshNet;
+using Renci.SshNet.Common;
 
 namespace LeafWeb.Core.Remote
 {
@@ -22,6 +24,9 @@ namespace LeafWeb.Core.Remote
 
 		private readonly ILog _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
 
+        private readonly int _connectionRetryCount = 3;
+        private readonly Policy _connectRetryPolicy;
+
 		public PiscalSshClient(string connectionString)
 		{
             if (string.IsNullOrEmpty(connectionString))
@@ -32,7 +37,12 @@ namespace LeafWeb.Core.Remote
 			var username = conn["username"] as string;
 			var password = conn["password"] as string;
 			_connectionInfo = new PasswordConnectionInfo(_host, username, password);
-		}
+
+            _connectRetryPolicy = Policy
+                .Handle<SshConnectionException>()
+                .Retry(_connectionRetryCount, 
+                    (ex, i) => _logger.Warn($"Retry {i} after exception: {ex.Message}"));
+        }
 
 		private SshClient GetSshClient()
 		{
@@ -43,12 +53,12 @@ namespace LeafWeb.Core.Remote
 		{
 			return new ScpClient(_connectionInfo);
 		}
-
-		private void Connect(PiscalLeafInput leafInput, BaseClient scp)
+        
+		private void Connect(PiscalLeafInput leafInput, BaseClient client)
 		{
 			try
             {
-                scp.Connect();
+                _connectRetryPolicy.Execute(client.Connect);
             }
 			catch (Exception ex)
 			{
@@ -62,7 +72,7 @@ namespace LeafWeb.Core.Remote
 		{
 			try
 			{
-				client.Disconnect();
+                _connectRetryPolicy.Execute(client.Disconnect);
 			}
 			catch (Exception ex)
 			{
@@ -77,9 +87,10 @@ namespace LeafWeb.Core.Remote
             // create directory
             using (var sshClient = GetSshClient())
             {
-                sshClient.Connect();
-                var runCommand = sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}");
+                Connect(leafInput, sshClient);
+                sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}");
                 sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}/input");
+                Disconnect(leafInput, sshClient);
             }
             // copy files
 			using (var scp = GetScpClient())
@@ -103,8 +114,6 @@ namespace LeafWeb.Core.Remote
 			if (string.IsNullOrEmpty(leafInput.PhotosyntheticType))
 				throw new PiscalClientException(leafInput.LeafInputId, "No PhotosyntheticType set");
 
-			//var inputDirectory = $"{BaseDirectory}/{leafInput.PiscalDirectoryName}";
-
 			CopyLeafInput(leafInput);
 
 			// begin processing
@@ -180,7 +189,7 @@ namespace LeafWeb.Core.Remote
 			// get output files
 			var status = GetLeafInputStatusRaw(leafInput);
 			if (status[0] != StatusComplete)
-				throw new PiscalClientException(leafInput.LeafInputId, "output not available, status is " + status[0]);
+				throw new PiscalClientException(leafInput.LeafInputId, "Output not available, status is not StatusComplete, instead is currently " + status[0]);
 
 			var filePaths = status.Skip(1);