Add retry for background enqueue in PiscalQueueManager

Web/Services/PiscalQueueBase.cs

@@ -1,9 +1,11 @@
 using System;
+using System.Linq.Expressions;
 using Hangfire;
 using LeafWeb.Core.DAL;
 using LeafWeb.Core.Entities;
 using LeafWeb.Core.Remote;
 using NLog;
+using Polly;
 
 namespace LeafWeb.Web.Services
 {
@@ -12,12 +14,19 @@ namespace LeafWeb.Web.Services
 		protected readonly DataService DataService;
 		protected readonly PiscalService PiscalService;
 		protected readonly Logger Logger;
+		private readonly Policy _retryPolicy;
 
 		protected PiscalQueueBase(DataService dataService, PiscalService piscalService)
 		{
 			DataService = dataService;
 			PiscalService = piscalService;
 			Logger = LogManager.GetLogger(GetType().Name);
+
+			_retryPolicy = 
+				Policy
+				.Handle<TimeoutException>()
+				.Retry(3, 
+					(exception, i) => Logger.Warn($"Retry {i} after exception: {exception.Message}"));
 		}
 
 		protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
@@ -32,11 +41,11 @@ namespace LeafWeb.Web.Services
 				+ $"StackTrace: {ex.StackTrace}";
 		}
 
-		protected void PiscalExceptionNotify(PiscalClientException ex, LeafInput leafInput)
+		protected void PiscalExceptionHandle(PiscalClientException ex, LeafInput leafInput)
 		{
 			var errorMessage = FormatException(ex, ex.LeafInputId);
 			Logger.Error(errorMessage);
-			BackgroundJob.Enqueue<EmailNotificationService>(
+			BackgroundJobEnqueueRetry<EmailNotificationService>(
 				email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
 
 			// TODO send user email too
@@ -45,7 +54,6 @@ namespace LeafWeb.Web.Services
 			{
 				DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
 					ex.Message);
-				BackgroundJob.Enqueue<Cleanup>(s => s.DoWork(leafInput.Id));
 			}
 		}
 		
@@ -53,5 +61,10 @@ namespace LeafWeb.Web.Services
 		{
 			DataService.Dispose();
 		}
+
+		protected string BackgroundJobEnqueueRetry<T>(Expression<Action<T>> a)
+		{
+			return _retryPolicy.Execute(() => BackgroundJob.Enqueue(a));
+		}
 	}
 }