Code rearrange for services and charting
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
using System.Linq;
|
||||
using LeafWeb.Core.Entities;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
public class FinishComplete : PiscalQueueWorker
|
||||
{
|
||||
protected override void DoWorkInternal(LeafInput leafInput)
|
||||
{
|
||||
Logger.Trace("LeafInput: {0}, RetrieveOutputFiles", leafInput.Id);
|
||||
|
||||
var leafOutputFiles = PiscalService.RetrieveOutputFiles(leafInput).ToList();
|
||||
|
||||
Logger.Trace("LeafInput: {0}, RetrieveOutputFiles saving output files", leafInput.Id);
|
||||
|
||||
foreach (var outputFile in leafOutputFiles)
|
||||
{
|
||||
if (leafInput.OutputFiles.All(file => file.Filename != outputFile.Filename))
|
||||
DataService.AddLeafOutputFile(outputFile);
|
||||
else
|
||||
Logger.Warn("LeafInput: {0}, RetrieveOutputFiles duplicate file name: {1}", leafInput.Id, outputFile.Filename);
|
||||
}
|
||||
|
||||
Logger.Info("LeafInput: {0}, RetrieveOutputFiles output files: {1}", leafInput.Id,
|
||||
string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
|
||||
|
||||
Logger.Trace("LeafInput: {0}, Set Complete", leafInput.Id);
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
|
||||
|
||||
BackgroundJobEnqueueRetry<EmailNotificationService>(email => email.SendLeafWebComplete(leafInput.Id));
|
||||
|
||||
Logger.Info("LeafInput: {0}, Cleanup", leafInput.Id);
|
||||
PiscalService.Cleanup(leafInput);
|
||||
|
||||
HangfireStartup.TriggerPiscalProcessQueue();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using Hangfire;
|
||||
using LeafWeb.Core.DAL;
|
||||
using LeafWeb.Core.Entities;
|
||||
using LeafWeb.Core.Remote;
|
||||
using NLog;
|
||||
using Polly;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
public abstract class PiscalQueueBase : IDisposable
|
||||
{
|
||||
protected readonly DataService DataService;
|
||||
protected readonly PiscalService PiscalService;
|
||||
protected readonly Logger Logger;
|
||||
private readonly Policy _retryPolicy;
|
||||
|
||||
protected PiscalQueueBase(DataService dataService, PiscalService piscalService)
|
||||
{
|
||||
DataService = dataService;
|
||||
PiscalService = piscalService;
|
||||
Logger = LogManager.GetLogger(GetType().Name);
|
||||
|
||||
_retryPolicy =
|
||||
Policy
|
||||
.Handle<TimeoutException>()
|
||||
.Retry(3,
|
||||
(exception, i) => Logger.Warn($"Retry {i} after exception: {exception.Message}"));
|
||||
}
|
||||
|
||||
protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
|
||||
|
||||
protected string FormatException(Exception ex, int leafInputId)
|
||||
{
|
||||
return
|
||||
$"LeafInput: {leafInputId}{Environment.NewLine}" +
|
||||
$"Class: {GetType().Name}{Environment.NewLine}" +
|
||||
$"Exception: {ex.Message}{Environment.NewLine}" +
|
||||
(ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty)
|
||||
+ $"StackTrace: {ex.StackTrace}";
|
||||
}
|
||||
|
||||
protected void PiscalExceptionHandle(PiscalClientException ex, LeafInput leafInput)
|
||||
{
|
||||
var errorMessage = FormatException(ex, ex.LeafInputId);
|
||||
Logger.Error(errorMessage);
|
||||
BackgroundJobEnqueueRetry<EmailNotificationService>(
|
||||
email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
|
||||
|
||||
// TODO send user email too
|
||||
|
||||
if (leafInput != null)
|
||||
{
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
|
||||
ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
DataService.Dispose();
|
||||
}
|
||||
|
||||
protected string BackgroundJobEnqueueRetry<T>(Expression<Action<T>> a)
|
||||
{
|
||||
return _retryPolicy.Execute(() => BackgroundJob.Enqueue(a));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using LeafWeb.Core.Entities;
|
||||
using LeafWeb.Core.Remote;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
public class PiscalQueueManager : PiscalQueueBase
|
||||
{
|
||||
private static readonly object ProcessQueueLock = new object();
|
||||
|
||||
public void ProcessQueue()
|
||||
{
|
||||
// prevent multiple entry into processing the queue
|
||||
if (Monitor.TryEnter(ProcessQueueLock))
|
||||
{
|
||||
Logger.Trace("ProcessQueue entered");
|
||||
|
||||
try
|
||||
{
|
||||
UpdateRunning();
|
||||
|
||||
StartNextPending();
|
||||
|
||||
// TODO: handle starting and finishing
|
||||
}
|
||||
finally
|
||||
{
|
||||
Logger.Trace("ProcessQueue exit");
|
||||
|
||||
Monitor.Exit(ProcessQueueLock);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Trace("ProcessQueue locked, queue already processing");
|
||||
}
|
||||
}
|
||||
|
||||
private void ClearStalled()
|
||||
{
|
||||
var leafInputs =
|
||||
DataService.GetLeafInputs(
|
||||
LeafInputStatusType.Starting,
|
||||
LeafInputStatusType.Finishing
|
||||
)
|
||||
.Where(li =>
|
||||
li.StatusHistory.OrderBy(sh => sh.DateTime).First().DateTime
|
||||
> DateTime.Now.Subtract(TimeSpan.FromHours(1)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private void StartNextPending()
|
||||
{
|
||||
var runningLeafInputs =
|
||||
DataService.GetLeafInputs(
|
||||
LeafInputStatusType.Starting,
|
||||
LeafInputStatusType.Running,
|
||||
LeafInputStatusType.Finishing
|
||||
).ToList();
|
||||
|
||||
if (runningLeafInputs.Any())
|
||||
{
|
||||
Logger.Trace("Leaf input(s) currently running");
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingInput =
|
||||
DataService
|
||||
.GetLeafInputs(LeafInputStatusType.Pending)
|
||||
.OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime))
|
||||
.FirstOrDefault();
|
||||
|
||||
if (pendingInput == null)
|
||||
{
|
||||
Logger.Trace("No pending leaf input");
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingInputId = pendingInput.Id;
|
||||
Logger.Info("LeafInput: {0}, Starting", pendingInputId);
|
||||
try
|
||||
{
|
||||
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
|
||||
BackgroundJobEnqueueRetry<StartPending>(c => c.DoWork(pendingInputId));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var errorMessage = FormatException(ex, pendingInputId);
|
||||
Logger.Error(errorMessage);
|
||||
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateRunning()
|
||||
{
|
||||
var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
|
||||
foreach (var leafInput in running)
|
||||
{
|
||||
try
|
||||
{
|
||||
var status = PiscalService.GetStatus(leafInput);
|
||||
var leafInputId = leafInput.Id;
|
||||
switch (status)
|
||||
{
|
||||
case PiscalStatus.NotStarted:
|
||||
// if it's not started - this is unusual state
|
||||
Logger.Warn("LeafInput: {0}, Piscal Not Started", leafInput.Id);
|
||||
break;
|
||||
|
||||
case PiscalStatus.Running:
|
||||
Logger.Trace("LeafInput: {0}, Piscal Running", leafInput.Id);
|
||||
// continue running
|
||||
break;
|
||||
|
||||
case PiscalStatus.Complete:
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing);
|
||||
BackgroundJobEnqueueRetry<FinishComplete>(s => s.DoWork(leafInputId));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (PiscalClientException ex)
|
||||
{
|
||||
PiscalExceptionHandle(ex, leafInput);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var errorMessage = FormatException(ex, leafInput.Id);
|
||||
Logger.Error(errorMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using System;
|
||||
using LeafWeb.Core.Entities;
|
||||
using LeafWeb.Core.Remote;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
public abstract class PiscalQueueWorker : PiscalQueueBase
|
||||
{
|
||||
public void DoWork(int leafInputId)
|
||||
{
|
||||
LeafInput leafInput = null;
|
||||
try
|
||||
{
|
||||
leafInput = DataService.GetLeafInput(leafInputId);
|
||||
DoWorkInternal(leafInput);
|
||||
}
|
||||
catch (PiscalClientException ex)
|
||||
{
|
||||
PiscalExceptionHandle(ex, leafInput);
|
||||
|
||||
if (leafInput != null)
|
||||
{
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
|
||||
ex.Message);
|
||||
}
|
||||
// signal to process next item
|
||||
HangfireStartup.TriggerPiscalProcessQueue();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var errorMessage = FormatException(ex, leafInputId);
|
||||
Logger.Error(errorMessage);
|
||||
throw; // this will retry via HangFire
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract void DoWorkInternal(LeafInput leafInputId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Configuration;
|
||||
using LeafWeb.Core.Entities;
|
||||
using LeafWeb.Core.Remote;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
/// <summary>
|
||||
/// Thin layer over PiscalClient to translate Core entities to Piscal objects
|
||||
/// </summary>
|
||||
public class PiscalService
|
||||
{
|
||||
private readonly IPiscalClient _piscalClient;
|
||||
|
||||
public PiscalService(IPiscalClient piscalClient)
|
||||
{
|
||||
_piscalClient = piscalClient;
|
||||
}
|
||||
|
||||
public PiscalService() : this(new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString))
|
||||
{
|
||||
}
|
||||
|
||||
public void Run(LeafInput leafInput)
|
||||
{
|
||||
var inputFile = new PiscalLeafInput(leafInput);
|
||||
_piscalClient.RunLeafInput(inputFile);
|
||||
}
|
||||
|
||||
public PiscalStatus GetStatus(LeafInput leafInput)
|
||||
{
|
||||
var inputFile = new PiscalLeafInput(leafInput);
|
||||
return _piscalClient.GetLeafInputStatus(inputFile);
|
||||
}
|
||||
|
||||
public IEnumerable<LeafOutputFile> RetrieveOutputFiles(LeafInput leafInput)
|
||||
{
|
||||
var input = new PiscalLeafInput(leafInput);
|
||||
var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(input);
|
||||
foreach (var file in piscalLeafOutputFiles)
|
||||
{
|
||||
var leafOutputFile = file.GetLeafOutputFile();
|
||||
leafOutputFile.LeafInput = leafInput;
|
||||
yield return leafOutputFile;
|
||||
}
|
||||
}
|
||||
|
||||
public void Cleanup(LeafInput leafInput)
|
||||
{
|
||||
var input = new PiscalLeafInput(leafInput);
|
||||
_piscalClient.CleanupLeafProcess(input);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using LeafWeb.Core.Entities;
|
||||
|
||||
namespace LeafWeb.Web.Services.PiscalQueue
|
||||
{
|
||||
public class StartPending : PiscalQueueWorker
|
||||
{
|
||||
protected override void DoWorkInternal(LeafInput leafInput)
|
||||
{
|
||||
Logger.Trace("LeafInput: {0}, Run", leafInput.Id);
|
||||
|
||||
PiscalService.Run(leafInput);
|
||||
|
||||
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user