LeafWeb/WebCms/Services/PiscalQueue/PiscalQueueManager.cs

using System;
using System.Linq;
using System.Threading;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using LeafWeb.WebCms.App_Start;

namespace LeafWeb.WebCms.Services.PiscalQueue
{
	public class PiscalQueueManager : PiscalQueueBase
	{
		private static readonly object Lock = new object();

		public void ProcessQueue()
		{
			// prevent multiple entry into processing the queue
			if (Monitor.TryEnter(Lock))
			{
				Logger.DebugFormat("ProcessQueue entered");

				try
				{
					UpdateCancelling();

					StartCancelPending();

					UpdateRunning();

					StartNextPending();
				}
				finally
				{
					Logger.DebugFormat("ProcessQueue exit");

					Monitor.Exit(Lock);
				}
			}
			else
			{
				Logger.DebugFormat("ProcessQueue locked, queue already processing");
			}
		}

		public bool CancelPending(int leafInputId)
		{
			if (Monitor.TryEnter(Lock, TimeSpan.FromSeconds(1)))
			{
				Logger.DebugFormat("CancelLeafInput entered");

				try
				{
					var leafInput = DataService.GetLeafInput(leafInputId);
					if (!leafInput.IsCancellable)
						return false;
					if (leafInput.IsPending)
					{
						Logger.DebugFormat("LeafInput: {0}, Set Cancelled from Pending", leafInput.Id);
						DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelled,
							"Emailing cancellation notification to user",
							$"Email: \'{leafInput.Email}\'");

						// send notification immediately
						BackgroundJobEnqueueRetry<EmailNotificationService>(email => email.SendLeafWebCancelled(leafInput.Id));
					}
					else if (leafInput.IsRunning)
					{
						DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.CancelPending);
						HangfireStartup.TriggerPiscalProcessQueue();
					}
					return true;
				}
				finally
				{
					Logger.DebugFormat("CancelLeafInput exit");

					Monitor.Exit(Lock);
				}
			}

			Logger.DebugFormat("CancelLeafInput locked");
			return false;
		}

		private void StartCancelPending()
		{
			var cancelPendingLeafInputs =
				DataService.GetLeafInputs(
					LeafInputStatusType.CancelPending
					).ToList();

			foreach (var leafInput in cancelPendingLeafInputs)
			{
				try
				{
					var status = PiscalService.GetStatus(leafInput);
					switch (status)
					{
						case PiscalStatus.Running:
							Logger.DebugFormat("LeafInput: {0}, Set Cancelling", leafInput.Id);
							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelling);

							Logger.InfoFormat("LeafInput: {0}, Kill", leafInput.Id);
							PiscalService.Kill(leafInput);

							break;
						case PiscalStatus.Complete:
							Logger.DebugFormat("LeafInput: {0}, Piscal Complete after cancelled - " +
							                   "setting to Running to copy output and notify user", leafInput.Id);
							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running,
								"Piscal Complete after cancelled - setting to Running to copy output and notify user");
							break;
					}
				}
				catch (PiscalClientException ex)
				{
					PiscalExceptionHandler(ex, leafInput);
				}
				catch (Exception ex)
				{
					var errorMessage = FormatException(ex);
					Logger.Error(errorMessage);
				}
			}
		}

		private void UpdateCancelling()
		{
			var cancellingLeafInputs =
				DataService.GetLeafInputs(
					LeafInputStatusType.Cancelling
					).ToList();

			foreach (var leafInput in cancellingLeafInputs)
			{
				try
				{
					var status = PiscalService.GetStatus(leafInput);
					switch (status)
					{
						case PiscalStatus.Running:
							Logger.InfoFormat("LeafInput: {0}, Piscal Running - still cancelling", leafInput.Id);
							// continue running
							break;

						case PiscalStatus.Complete:
							Logger.DebugFormat("LeafInput: {0}, Set Cancelled", leafInput.Id);
							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Cancelled,
								"Emailing cancellation notification to user and cleaning up files on Piscal",
								$"Email: \'{leafInput.Email}\'");

							BackgroundJobEnqueueRetry<EmailNotificationService>(email => email.SendLeafWebCancelled(leafInput.Id));

							Logger.InfoFormat("LeafInput: {0}, Cleanup", leafInput.Id);
							PiscalService.Cleanup(leafInput);

							break;
					}
				}
				catch (PiscalClientException ex)
				{
					PiscalExceptionHandler(ex, leafInput);
				}
				catch (Exception ex)
				{
					var errorMessage = FormatException(ex);
					Logger.Error(errorMessage);
				}
			}
		}
		
		private void StartNextPending()
		{
			var runningLeafInputs = 
				DataService.GetLeafInputs(
					LeafInputStatusType.Starting,
					LeafInputStatusType.Running,
					LeafInputStatusType.Finishing
					).ToList();

			if (runningLeafInputs.Any())
			{
				Logger.DebugFormat("Leaf input(s) currently running");
				return;
			}

			var pendingInput = 
				DataService
					.GetLeafInputs(LeafInputStatusType.Pending)
					.OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime))
					.FirstOrDefault();

			if (pendingInput == null)
			{
				Logger.DebugFormat("No pending leaf input");
				return;
			}

			var pendingInputId = pendingInput.Id;
			Logger.InfoFormat("LeafInput: {0}, Starting", pendingInputId);
			try
			{
				DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting, "Copying input to Piscal",
					$"File count: {pendingInput.InputFiles.Count}");
				BackgroundJobEnqueueRetry<StartPending>(c => c.DoWork(pendingInputId));
			}
			catch (Exception ex)
			{
				var errorMessage = FormatException(ex);
				Logger.Error(errorMessage);
				DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, "Exception while StartPending", errorMessage);
			}
		}

		private void UpdateRunning()
		{
			var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
			foreach (var leafInput in running)
			{
				try
				{
					var status = PiscalService.GetStatus(leafInput);
					var leafInputId = leafInput.Id;
					switch (status)
					{
						case PiscalStatus.NotStarted:
							// if it's not started - this is unusual state
							var pendingToRetry = "Piscal reporting Not Started, setting to Pending to retry";
							PiscalWarningHandler($"LeafInput: {leafInput.Id}, {pendingToRetry}", leafInput);
							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending, details: pendingToRetry);
							break;

						case PiscalStatus.Running:
							Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id);
							// continue running
							break;

						case PiscalStatus.Complete:
							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing, "Copying LeafOutput from Piscal");
							BackgroundJobEnqueueRetry<FinishComplete>(s => s.DoWork(leafInputId));
							
							break;
					}
				}
				catch (PiscalClientException ex)
				{
					PiscalExceptionHandler(ex, leafInput);
				}
				catch (Exception ex)
				{
					var errorMessage = FormatException(ex);
					Logger.Error(errorMessage);
				}
			}
		}
	}
}