Add retry for background enqueue in PiscalQueueManager
This commit is contained in:
@@ -1,9 +1,11 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Threading;
|
||||
using Hangfire;
|
||||
using LeafWeb.Core.Entities;
|
||||
using LeafWeb.Core.Remote;
|
||||
using Polly;
|
||||
|
||||
namespace LeafWeb.Web.Services
|
||||
{
|
||||
@@ -23,6 +25,8 @@ namespace LeafWeb.Web.Services
|
||||
UpdateRunning();
|
||||
|
||||
StartNextPending();
|
||||
|
||||
// TODO: handle starting and finishing
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -42,7 +46,7 @@ namespace LeafWeb.Web.Services
|
||||
var runningLeafInputs = DataService.GetRunningLeafInputs().ToList();
|
||||
if (runningLeafInputs.Any())
|
||||
{
|
||||
Logger.Trace("Leaf input currently running , don't enqueue any new");
|
||||
Logger.Trace("Leaf input(s) currently running");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -60,10 +64,20 @@ namespace LeafWeb.Web.Services
|
||||
|
||||
var pendingInputId = pendingInput.Id;
|
||||
Logger.Info("LeafInput: {0}, Starting", pendingInputId);
|
||||
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
|
||||
BackgroundJob.Enqueue<StartPending>(c => c.DoWork(pendingInputId));
|
||||
try
|
||||
{
|
||||
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
|
||||
BackgroundJobEnqueueRetry<StartPending>(c => c.DoWork(pendingInputId));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var errorMessage = FormatException(ex, pendingInputId);
|
||||
Logger.Error(errorMessage);
|
||||
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void UpdateRunning()
|
||||
{
|
||||
var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
|
||||
@@ -86,20 +100,28 @@ namespace LeafWeb.Web.Services
|
||||
break;
|
||||
|
||||
case PiscalStatus.Complete:
|
||||
// TODO: change to "retrieving output"?
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing);
|
||||
BackgroundJobEnqueueRetry<FinishComplete>(s => s.DoWork(leafInputId));
|
||||
|
||||
var retrieveFilesId = BackgroundJob.Enqueue<RetrieveOutputFiles>(s => s.DoWork(leafInputId));
|
||||
BackgroundJob.ContinueWith<Cleanup>(retrieveFilesId, s => s.DoWork(leafInputId));
|
||||
BackgroundJob.ContinueWith<EmailNotificationService>(retrieveFilesId,
|
||||
email => email.SendLeafWebComplete(leafInputId));
|
||||
BackgroundJob.ContinueWith<SetComplete>(retrieveFilesId, s => s.DoWork(leafInputId));
|
||||
//try
|
||||
//{
|
||||
//}
|
||||
//catch (PiscalClientException ex)
|
||||
//{
|
||||
// PiscalExceptionNotify(ex, leafInput);
|
||||
//}
|
||||
//catch (Exception ex)
|
||||
//{
|
||||
// var errorMessage = FormatException(ex, leafInput.Id);
|
||||
// Logger.Error(errorMessage);
|
||||
// DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
|
||||
//}
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (PiscalClientException ex)
|
||||
{
|
||||
PiscalExceptionNotify(ex, leafInput);
|
||||
PiscalExceptionHandle(ex, leafInput);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user