Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Piscal Queue working more resilient to error

Browse Source
This commit is contained in:
James Kolpack
2016-04-18 10:44:36 -04:00
parent 6623aa48ea
commit ca1a235c1d
21 changed files with 418 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Web/HangfireStartup.cs
-2
View File
@@ -3,7 +3,6 @@ using Hangfire;
using Microsoft.Owin;
using LeafWeb.Web;
using LeafWeb.Web.Services;
using NLog.Internal;
using Owin;
using ConfigurationManager = System.Configuration.ConfigurationManager;
@@ -24,7 +23,6 @@ namespace LeafWeb.Web
private void SetupRecurringJobs()
{
// TODO: new SqlServerDistributedLock
var queueInterval = ConfigurationManager.AppSettings["ProcessQueueInterval"];
// https://discuss.hangfire.io/t/how-to-create-cron-job-that-is-executing-every-15-minutes/533
RecurringJob.AddOrUpdate<PiscalQueueManager>(PiscalProcessQueue, p => p.ProcessQueue(), queueInterval);
Web/Services/Cleanup.cs
+25
View File
@@ -0,0 +1,25 @@
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
public class Cleanup : PiscalQueueWorker
{
protected override void DoWorkInternal(LeafInput leafInput)
{
try
{
Logger.Info("LeafInput: {0}, Cleanup", leafInput.Id);
PiscalService.Cleanup(leafInput);
}
catch (PiscalClientException ex)
{
var errorMessage = FormatException(ex, ex.LeafInputId);
Logger.Error(errorMessage, ex);
// log the error, but ignore the cleanup issue for now
Logger.Info("LeafInput: {0}, Cleanup - likely has not occurred", leafInput.Id);
}
}
}
}
Web/Services/PiscalQueueBase.cs
+57
View File
@@ -0,0 +1,57 @@
using System;
using Hangfire;
using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using NLog;
namespace LeafWeb.Web.Services
{
public abstract class PiscalQueueBase : IDisposable
{
protected readonly DataService DataService;
protected readonly PiscalService PiscalService;
protected readonly Logger Logger;
protected PiscalQueueBase(DataService dataService, PiscalService piscalService)
{
DataService = dataService;
PiscalService = piscalService;
Logger = LogManager.GetLogger(GetType().Name);
}
protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
protected string FormatException(Exception ex, int leafInputId)
{
return
$"LeafInput: {leafInputId}{Environment.NewLine}" +
$"Class: {GetType().Name}{Environment.NewLine}" +
$"Exception: {ex.Message}{Environment.NewLine}" +
(ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty)
+ $"StackTrace: {ex.StackTrace}";
}
protected void PiscalExceptionNotify(PiscalClientException ex, LeafInput leafInput)
{
var errorMessage = FormatException(ex, ex.LeafInputId);
Logger.Error(errorMessage);
BackgroundJob.Enqueue<EmailNotificationService>(
email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
// TODO send user email too
if (leafInput != null)
{
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
ex.Message);
BackgroundJob.Enqueue<Cleanup>(s => s.DoWork(leafInput.Id));
}
}
public void Dispose()
{
DataService.Dispose();
}
}
}
Web/Services/PiscalQueueManager.cs
+39 -96
View File
@@ -2,168 +2,111 @@
using System.Linq;
using System.Threading;
using Hangfire;
using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using NLog;
namespace LeafWeb.Web.Services
{
public class PiscalQueueManager : IDisposable
public class PiscalQueueManager : PiscalQueueBase
{
private readonly DataService _dataService;
private readonly PiscalService _piscalService;
private readonly EmailNotificationService _emailService;
public PiscalQueueManager(DataService dataService, PiscalService piscalService, EmailNotificationService emailService)
{
_dataService = dataService;
_piscalService = piscalService;
_emailService = emailService;
}
public PiscalQueueManager() : this(new DataService(), new PiscalService(), new EmailNotificationService()) {}
private static readonly object ProcessQueueLock = new object();
public void ProcessQueue()
{
var logger = LogManager.GetCurrentClassLogger();
// prevent multiple entry into processing the queue
if (Monitor.TryEnter(ProcessQueueLock))
{
logger.Trace("ProcessQueue entered");
Logger.Trace("ProcessQueue entered");
try
{
ProcessRunning(logger);
UpdateRunning();
ProcessQueue(logger);
StartNextPending();
}
finally
{
logger.Trace("ProcessQueue exit");
Logger.Trace("ProcessQueue exit");
Monitor.Exit(ProcessQueueLock);
}
}
else
{
logger.Trace("ProcessQueue locked, queue already processing");
Logger.Trace("ProcessQueue locked, queue already processing");
}
}
private void ProcessQueue(ILogger logger)
private void StartNextPending()
{
var runningLeafInputs = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
var runningLeafInputs = DataService.GetRunningLeafInputs().ToList();
if (runningLeafInputs.Any())
{
logger.Trace("Leaf input currently running , don't enqueue any new");
Logger.Trace("Leaf input currently running , don't enqueue any new");
return;
}
var pending =
_dataService
var pendingInput =
DataService
.GetLeafInputs(LeafInputStatusType.Pending)
.OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime))
.FirstOrDefault();
if (pending == null)
if (pendingInput == null)
{
logger.Trace("No pending leaf input");
Logger.Trace("No pending leaf input");
return;
}
logger.Info("LeafInputFile: {0}, Start", pending.Id);
try
{
_piscalService.Run(pending);
}
catch (PiscalClientException ex)
{
var errorMessage =
$"LeafInputFile: {pending.Id}\r\nProcessRunning Exception: {ex.Message}"
+ $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}";
logger.Error(errorMessage);
BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessQueue Exception", errorMessage));
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Exception, "Error occurred starting LeafInput", ex.Message);
logger.Info("LeafInputFile: {0}, Cleanup", pending.Id);
_piscalService.Cleanup(pending);
// TODO: re-queue?
//_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput");
}
logger.Trace("LeafInputFile: {0}, Set Pending", pending.Id);
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running);
var pendingInputId = pendingInput.Id;
Logger.Info("LeafInput: {0}, Starting", pendingInputId);
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
BackgroundJob.Enqueue<StartPending>(c => c.DoWork(pendingInputId));
}
private void ProcessRunning(ILogger logger)
private void UpdateRunning()
{
var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
foreach (var leafInput in running)
{
try
{
var status = _piscalService.GetStatus(leafInput);
var status = PiscalService.GetStatus(leafInput);
var leafInputId = leafInput.Id;
switch (status)
{
case PiscalStatus.NotStarted:
logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", leafInput.Id);
// if it's not started, try to requeue the process - this is unusual state
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending);
// if it's not started - this is unusual state
Logger.Warn("LeafInput: {0}, Piscal Not Started", leafInput.Id);
break;
case PiscalStatus.Running:
logger.Trace("LeafInputFile: {0}, Running", leafInput.Id);
Logger.Trace("LeafInput: {0}, Piscal Running", leafInput.Id);
// continue running
break;
case PiscalStatus.Complete:
logger.Info("LeafInputFile: {0}, Complete", leafInput.Id);
// collect the leaf output
var leafOutputFiles = _piscalService.RetrieveOutputFiles(leafInput).ToList();
// TODO: change to "retrieving output"?
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing);
logger.Trace("LeafInputFile: {0}, saving output files", leafInput.Id);
foreach (var outputFile in leafOutputFiles)
_dataService.AddLeafOutputFile(outputFile);
logger.Info("LeafInputFile: {0}, output files: {1}", leafInput.Id,
string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
// update db
logger.Trace("LeafInputFile: {0}, Set Complete", leafInput.Id);
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
BackgroundJob.Enqueue(() => _emailService.SendLeafWebComplete(leafInput.Id));
// remove working data from the server
logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
_piscalService.Cleanup(leafInput);
var retrieveFilesId = BackgroundJob.Enqueue<RetrieveOutputFiles>(s => s.DoWork(leafInputId));
BackgroundJob.ContinueWith<Cleanup>(retrieveFilesId, s => s.DoWork(leafInputId));
BackgroundJob.ContinueWith<EmailNotificationService>(retrieveFilesId,
email => email.SendLeafWebComplete(leafInputId));
BackgroundJob.ContinueWith<SetComplete>(retrieveFilesId, s => s.DoWork(leafInputId));
break;
}
}
catch (PiscalClientException ex)
{
PiscalExceptionNotify(ex, leafInput);
}
catch (Exception ex)
{
var errorMessage =
$"LeafInputFile: {leafInput.Id}\r\nProcessRunning Exception: {ex.Message}"
+ $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}";
logger.Error(errorMessage);
BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessRunning Exception", errorMessage));
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", ex.Message);
// TODO: Send email
logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
_piscalService.Cleanup(leafInput);
// TODO: re-queue ?
var errorMessage = FormatException(ex, leafInput.Id);
Logger.Error(errorMessage);
}
}
}
public void Dispose()
{
_dataService.Dispose();
}
}
}
Web/Services/PiscalQueueWorker.cs
+39
View File
@@ -0,0 +1,39 @@
using System;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
public abstract class PiscalQueueWorker : PiscalQueueBase
{
public void DoWork(int leafInputId)
{
LeafInput leafInput = null;
try
{
leafInput = DataService.GetLeafInput(leafInputId);
DoWorkInternal(leafInput);
}
catch (PiscalClientException ex)
{
PiscalExceptionNotify(ex, leafInput);
if (leafInput != null)
{
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
ex.Message);
}
// signal to process next item
HangfireStartup.TriggerPiscalProcessQueue();
}
catch (Exception ex)
{
var errorMessage = FormatException(ex, leafInputId);
Logger.Error(errorMessage);
throw; // this will retry via HangFire
}
}
protected abstract void DoWorkInternal(LeafInput leafInputId);
}
}
Web/Services/PiscalService.cs
+3
View File
@@ -5,6 +5,9 @@ using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
/// <summary>
/// Thin layer over PiscalClient to translate Core entities to Piscal objects
/// </summary>
public class PiscalService
{
private readonly IPiscalClient _piscalClient;
Web/Services/RetrieveOutputFiles.cs
+28
View File
@@ -0,0 +1,28 @@
using System.Linq;
using LeafWeb.Core.Entities;
namespace LeafWeb.Web.Services
{
public class RetrieveOutputFiles : PiscalQueueWorker
{
protected override void DoWorkInternal(LeafInput leafInput)
{
Logger.Trace("LeafInput: {0}, RetrieveOutputFiles", leafInput.Id);
var leafOutputFiles = PiscalService.RetrieveOutputFiles(leafInput).ToList();
Logger.Trace("LeafInput: {0}, RetrieveOutputFiles saving output files", leafInput.Id);
foreach (var outputFile in leafOutputFiles)
{
if (leafInput.OutputFiles.All(file => file.Filename != outputFile.Filename))
DataService.AddLeafOutputFile(outputFile);
else
Logger.Warn("LeafInput: {0}, RetrieveOutputFiles duplicate file name: {1}", leafInput.Id, outputFile.Filename);
}
Logger.Info("LeafInput: {0}, RetrieveOutputFiles output files: {1}", leafInput.Id,
string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
}
}
}
Web/Services/SetComplete.cs
+16
View File
@@ -0,0 +1,16 @@
using LeafWeb.Core.Entities;
namespace LeafWeb.Web.Services
{
public class SetComplete : PiscalQueueWorker
{
protected override void DoWorkInternal(LeafInput leafInput)
{
Logger.Trace("LeafInput: {0}, Set Complete", leafInput.Id);
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
HangfireStartup.TriggerPiscalProcessQueue();
}
}
}
Web/Services/StartPending.cs
+16
View File
@@ -0,0 +1,16 @@
using LeafWeb.Core.Entities;
namespace LeafWeb.Web.Services
{
public class StartPending : PiscalQueueWorker
{
protected override void DoWorkInternal(LeafInput leafInput)
{
Logger.Trace("LeafInput: {0}, Run", leafInput.Id);
PiscalService.Run(leafInput);
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running);
}
}
}
Web/Web.csproj
+6
View File
@@ -960,10 +960,16 @@
</Compile>
<Compile Include="Attributes\HttpParamActionAttribute.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Services\Cleanup.cs" />
<Compile Include="Services\EmailNotificationService.cs" />
<Compile Include="Services\PiscalQueueBase.cs" />
<Compile Include="Services\PiscalQueueManager.cs" />
<Compile Include="Services\PiscalQueueWorker.cs" />
<Compile Include="Services\PiscalService.cs" />
<Compile Include="HangfireStartup.cs" />
<Compile Include="Services\RetrieveOutputFiles.cs" />
<Compile Include="Services\SetComplete.cs" />
<Compile Include="Services\StartPending.cs" />
<Compile Include="Utility\MarkdownHelper.cs" />
<Compile Include="Utility\Validation.cs" />
<Compile Include="ViewModels\LeafInput\ConfirmViewModel.cs" />