Add retry for background enqueue in PiscalQueueManager

This commit is contained in:
2016-04-19 07:33:26 -04:00
parent 3f12f19b70
commit 1359100670
8 changed files with 67 additions and 61 deletions
+32 -10
View File
@@ -1,9 +1,11 @@
using System;
using System.Linq;
using System.Linq.Expressions;
using System.Threading;
using Hangfire;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using Polly;
namespace LeafWeb.Web.Services
{
@@ -23,6 +25,8 @@ namespace LeafWeb.Web.Services
UpdateRunning();
StartNextPending();
// TODO: handle starting and finishing
}
finally
{
@@ -42,7 +46,7 @@ namespace LeafWeb.Web.Services
var runningLeafInputs = DataService.GetRunningLeafInputs().ToList();
if (runningLeafInputs.Any())
{
Logger.Trace("Leaf input currently running , don't enqueue any new");
Logger.Trace("Leaf input(s) currently running");
return;
}
@@ -60,10 +64,20 @@ namespace LeafWeb.Web.Services
var pendingInputId = pendingInput.Id;
Logger.Info("LeafInput: {0}, Starting", pendingInputId);
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
BackgroundJob.Enqueue<StartPending>(c => c.DoWork(pendingInputId));
try
{
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
BackgroundJobEnqueueRetry<StartPending>(c => c.DoWork(pendingInputId));
}
catch (Exception ex)
{
var errorMessage = FormatException(ex, pendingInputId);
Logger.Error(errorMessage);
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
}
}
private void UpdateRunning()
{
var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
@@ -86,20 +100,28 @@ namespace LeafWeb.Web.Services
break;
case PiscalStatus.Complete:
// TODO: change to "retrieving output"?
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing);
BackgroundJobEnqueueRetry<FinishComplete>(s => s.DoWork(leafInputId));
var retrieveFilesId = BackgroundJob.Enqueue<RetrieveOutputFiles>(s => s.DoWork(leafInputId));
BackgroundJob.ContinueWith<Cleanup>(retrieveFilesId, s => s.DoWork(leafInputId));
BackgroundJob.ContinueWith<EmailNotificationService>(retrieveFilesId,
email => email.SendLeafWebComplete(leafInputId));
BackgroundJob.ContinueWith<SetComplete>(retrieveFilesId, s => s.DoWork(leafInputId));
//try
//{
//}
//catch (PiscalClientException ex)
//{
// PiscalExceptionNotify(ex, leafInput);
//}
//catch (Exception ex)
//{
// var errorMessage = FormatException(ex, leafInput.Id);
// Logger.Error(errorMessage);
// DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
//}
break;
}
}
catch (PiscalClientException ex)
{
PiscalExceptionNotify(ex, leafInput);
PiscalExceptionHandle(ex, leafInput);
}
catch (Exception ex)
{