From 129c0da4dd856b8a53a0a76663c10ac95ace9a5f Mon Sep 17 00:00:00 2001 From: James Kolpack Date: Thu, 21 Nov 2019 10:11:49 -0500 Subject: [PATCH] Add retry to connection attempts for the PiscalSshClient --- Core/App.config | 8 ++++++++ Core/Core.csproj | 14 +++++++++---- Core/Remote/PiscalSshClient.cs | 29 +++++++++++++++++---------- Core/packages.config | 36 ++++++++++++++++++++++++++++++++-- WebCms.Tests/app.config | 4 ++++ WebCms/Web.config | 4 ++++ WebCms/WebCms.csproj | 8 ++++---- WebCms/packages.config | 4 ++-- 8 files changed, 85 insertions(+), 22 deletions(-) diff --git a/Core/App.config b/Core/App.config index 01f6c9b..88b8ae7 100644 --- a/Core/App.config +++ b/Core/App.config @@ -10,4 +10,12 @@ + + + + + + + + \ No newline at end of file diff --git a/Core/Core.csproj b/Core/Core.csproj index afc4d6b..6a64deb 100644 --- a/Core/Core.csproj +++ b/Core/Core.csproj @@ -32,11 +32,9 @@ ..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.dll - True ..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll - True ..\packages\CsvHelper.12.2.1\lib\net45\CsvHelper.dll @@ -60,18 +58,26 @@ ..\packages\MlkPwgen.0.3.0\lib\net45\MlkPwgen.dll + + ..\packages\Polly.7.1.1\lib\netstandard1.1\Polly.dll + False ..\References\Renci.SshNet.dll + + + + ..\packages\System.Runtime.InteropServices.RuntimeInformation.4.3.0\lib\net45\System.Runtime.InteropServices.RuntimeInformation.dll + - - ..\packages\System.ValueTuple.4.4.0\lib\netstandard1.0\System.ValueTuple.dll + + ..\packages\System.ValueTuple.4.5.0\lib\netstandard1.0\System.ValueTuple.dll diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs index 21fb2c6..ebe1523 100644 --- a/Core/Remote/PiscalSshClient.cs +++ b/Core/Remote/PiscalSshClient.cs @@ -5,7 +5,9 @@ using System.IO; using System.Linq; using log4net; using LeafWeb.Core.Utility; +using Polly; using Renci.SshNet; +using Renci.SshNet.Common; namespace LeafWeb.Core.Remote { @@ -22,6 +24,9 @@ namespace LeafWeb.Core.Remote private readonly ILog _logger = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + private readonly int _connectionRetryCount = 3; + private readonly Policy _connectRetryPolicy; + public PiscalSshClient(string connectionString) { if (string.IsNullOrEmpty(connectionString)) @@ -32,7 +37,12 @@ namespace LeafWeb.Core.Remote var username = conn["username"] as string; var password = conn["password"] as string; _connectionInfo = new PasswordConnectionInfo(_host, username, password); - } + + _connectRetryPolicy = Policy + .Handle() + .Retry(_connectionRetryCount, + (ex, i) => _logger.Warn($"Retry {i} after exception: {ex.Message}")); + } private SshClient GetSshClient() { @@ -43,12 +53,12 @@ namespace LeafWeb.Core.Remote { return new ScpClient(_connectionInfo); } - - private void Connect(PiscalLeafInput leafInput, BaseClient scp) + + private void Connect(PiscalLeafInput leafInput, BaseClient client) { try { - scp.Connect(); + _connectRetryPolicy.Execute(client.Connect); } catch (Exception ex) { @@ -62,7 +72,7 @@ namespace LeafWeb.Core.Remote { try { - client.Disconnect(); + _connectRetryPolicy.Execute(client.Disconnect); } catch (Exception ex) { @@ -77,9 +87,10 @@ namespace LeafWeb.Core.Remote // create directory using (var sshClient = GetSshClient()) { - sshClient.Connect(); - var runCommand = sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}"); + Connect(leafInput, sshClient); + sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}"); sshClient.RunCommand($"mkdir {BaseDirectory}/{leafInput.PiscalDirectoryName}/input"); + Disconnect(leafInput, sshClient); } // copy files using (var scp = GetScpClient()) @@ -103,8 +114,6 @@ namespace LeafWeb.Core.Remote if (string.IsNullOrEmpty(leafInput.PhotosyntheticType)) throw new PiscalClientException(leafInput.LeafInputId, "No PhotosyntheticType set"); - //var inputDirectory = $"{BaseDirectory}/{leafInput.PiscalDirectoryName}"; - CopyLeafInput(leafInput); // begin processing @@ -180,7 +189,7 @@ namespace LeafWeb.Core.Remote // get output files var status = GetLeafInputStatusRaw(leafInput); if (status[0] != StatusComplete) - throw new PiscalClientException(leafInput.LeafInputId, "output not available, status is " + status[0]); + throw new PiscalClientException(leafInput.LeafInputId, "Output not available, status is not StatusComplete, instead is currently " + status[0]); var filePaths = status.Skip(1); diff --git a/Core/packages.config b/Core/packages.config index f6ccfa0..20380c0 100644 --- a/Core/packages.config +++ b/Core/packages.config @@ -6,7 +6,39 @@ - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/WebCms.Tests/app.config b/WebCms.Tests/app.config index 397b743..efd42e1 100644 --- a/WebCms.Tests/app.config +++ b/WebCms.Tests/app.config @@ -54,6 +54,10 @@ + + + + \ No newline at end of file diff --git a/WebCms/Web.config b/WebCms/Web.config index 7a48ccc..97ebe52 100644 --- a/WebCms/Web.config +++ b/WebCms/Web.config @@ -437,6 +437,10 @@ + + + + diff --git a/WebCms/WebCms.csproj b/WebCms/WebCms.csproj index 8c12cca..b067f59 100644 --- a/WebCms/WebCms.csproj +++ b/WebCms/WebCms.csproj @@ -203,8 +203,8 @@ ..\packages\Owin.1.0\lib\net40\Owin.dll True - - ..\packages\Polly.6.1.0\lib\netstandard1.1\Polly.dll + + ..\packages\Polly.7.1.1\lib\netstandard1.1\Polly.dll ..\packages\semver.1.1.2\lib\net451\Semver.dll @@ -241,8 +241,8 @@ ..\packages\System.Threading.Tasks.Dataflow.4.7.0\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll - - ..\packages\System.ValueTuple.4.4.0\lib\netstandard1.0\System.ValueTuple.dll + + ..\packages\System.ValueTuple.4.5.0\lib\netstandard1.0\System.ValueTuple.dll diff --git a/WebCms/packages.config b/WebCms/packages.config index adce933..533bf9c 100644 --- a/WebCms/packages.config +++ b/WebCms/packages.config @@ -58,7 +58,7 @@ - + @@ -92,7 +92,7 @@ - +