using System; using System.Linq; using System.Threading; using LeafWeb.Core.Entities; using LeafWeb.Core.Remote; using LeafWeb.WebCms.App_Start; namespace LeafWeb.WebCms.Services.PiscalQueue { public class PiscalQueueManager : PiscalQueueBase { private static readonly object Lock = new object(); public void ProcessQueue() { // prevent multiple entry into processing the queue if (Monitor.TryEnter(Lock)) { Logger.DebugFormat("ProcessQueue entered"); try { UpdateCancelling(); StartCancelPending(); UpdateRunning(); StartNextPending(); } finally { Logger.DebugFormat("ProcessQueue exit"); Monitor.Exit(Lock); } } else { Logger.DebugFormat("ProcessQueue locked, queue already processing"); } } public bool CancelPending(int leafInputId) { if (Monitor.TryEnter(Lock, TimeSpan.FromSeconds(1))) { Logger.DebugFormat("CancelLeafInput entered"); try { var leafInput = DataService.GetLeafInput(leafInputId); if (!leafInput.IsCancellable) return false; if (leafInput.IsPending) { Logger.DebugFormat("LeafInput: {0}, Set Cancelled from Pending", leafInput.Id); DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelled, "Emailing cancellation notification to user", $"Email: \'{leafInput.Email}\'"); // send notification immediately BackgroundJobEnqueueRetry(email => email.SendLeafWebCancelled(leafInput.Id)); } else if (leafInput.IsRunning) { DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.CancelPending); HangfireStartup.TriggerPiscalProcessQueue(); } return true; } finally { Logger.DebugFormat("CancelLeafInput exit"); Monitor.Exit(Lock); } } Logger.DebugFormat("CancelLeafInput locked"); return false; } private void StartCancelPending() { var cancelPendingLeafInputs = DataService.GetLeafInputs( LeafInputStatusType.CancelPending ).ToList(); foreach (var leafInput in cancelPendingLeafInputs) { try { var status = PiscalService.GetStatus(leafInput); switch (status) { case PiscalStatus.Running: Logger.DebugFormat("LeafInput: {0}, Set Cancelling", leafInput.Id); DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelling); Logger.InfoFormat("LeafInput: {0}, Kill", leafInput.Id); PiscalService.Kill(leafInput); break; case PiscalStatus.Complete: Logger.DebugFormat("LeafInput: {0}, Piscal Complete after cancelled - " + "setting to Running to copy output and notify user", leafInput.Id); DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running, "Piscal Complete after cancelled - setting to Running to copy output and notify user"); break; } } catch (PiscalClientException ex) { PiscalExceptionHandler(ex, leafInput); } catch (Exception ex) { var errorMessage = FormatException(ex); Logger.Error(errorMessage); } } } private void UpdateCancelling() { var cancellingLeafInputs = DataService.GetLeafInputs( LeafInputStatusType.Cancelling ).ToList(); foreach (var leafInput in cancellingLeafInputs) { try { var status = PiscalService.GetStatus(leafInput); switch (status) { case PiscalStatus.Running: Logger.InfoFormat("LeafInput: {0}, Piscal Running - still cancelling", leafInput.Id); // continue running break; case PiscalStatus.Complete: Logger.DebugFormat("LeafInput: {0}, Set Cancelled", leafInput.Id); DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelled, "Emailing cancellation notification to user and cleaning up files on Piscal", $"Email: \'{leafInput.Email}\'"); BackgroundJobEnqueueRetry(email => email.SendLeafWebCancelled(leafInput.Id)); Logger.InfoFormat("LeafInput: {0}, Cleanup", leafInput.Id); PiscalService.Cleanup(leafInput); break; } } catch (PiscalClientException ex) { PiscalExceptionHandler(ex, leafInput); } catch (Exception ex) { var errorMessage = FormatException(ex); Logger.Error(errorMessage); } } } private void StartNextPending() { var runningLeafInputs = DataService.GetLeafInputs( LeafInputStatusType.Starting, LeafInputStatusType.Running, LeafInputStatusType.Finishing ).ToList(); if (runningLeafInputs.Any()) { Logger.DebugFormat("Leaf input(s) currently running"); return; } var pendingInput = DataService .GetLeafInputsOrdered() .FirstOrDefault(li => li.CurrentStatus == LeafInputStatusType.Pending); if (pendingInput == null) { Logger.DebugFormat("No pending leaf input"); return; } var pendingInputId = pendingInput.Id; Logger.InfoFormat("LeafInput: {0}, Starting", pendingInputId); try { DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting, "Copying input to Piscal", $"File count: {pendingInput.InputFiles.Count}"); BackgroundJobEnqueueRetry(c => c.DoWork(pendingInputId)); } catch (Exception ex) { var errorMessage = FormatException(ex); Logger.Error(errorMessage); DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, "Exception while StartPending", errorMessage); } } private void UpdateRunning() { var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); foreach (var leafInput in running) { try { var status = PiscalService.GetStatus(leafInput); var leafInputId = leafInput.Id; switch (status) { case PiscalStatus.NotStarted: // if it's not started - this is unusual state var pendingToRetry = "Piscal reporting Not Started, setting to Pending to retry"; PiscalWarningHandler($"LeafInput: {leafInput.Id}, {pendingToRetry}", leafInput); DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending, details: pendingToRetry); break; case PiscalStatus.Running: Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id); // continue running break; case PiscalStatus.Complete: DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing, "Copying LeafOutput from Piscal"); BackgroundJobEnqueueRetry(s => s.DoWork(leafInputId)); break; } } catch (PiscalClientException ex) { PiscalExceptionHandler(ex, leafInput); } catch (Exception ex) { var errorMessage = FormatException(ex); Logger.Error(errorMessage); } } } } }