Add unresponsive status

WebCms/Services/PiscalQueue/PiscalQueueManager.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Configuration;
 using System.Linq;
 using System.Threading;
 using LeafWeb.Core.Entities;
 using LeafWeb.Core.Remote;
+using LeafWeb.Core.Utility;
 using LeafWeb.WebCms.App_Start;
 
 namespace LeafWeb.WebCms.Services.PiscalQueue
@@ -62,7 +64,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
 						// send notification immediately
 						BackgroundJobEnqueueRetry<EmailNotificationService>(email => email.SendLeafWebCancelled(leafInput.Id));
 					}
-					else if (leafInput.IsRunning)
+					else if (leafInput.IsRunning || leafInput.IsUnresponsive)
 					{
 						DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.CancelPending);
 						HangfireStartup.TriggerPiscalProcessQueue();
@@ -174,6 +176,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
 				DataService.GetLeafInputs(
 					LeafInputStatusType.Starting,
 					LeafInputStatusType.Running,
+                    LeafInputStatusType.Unresponsive,
 					LeafInputStatusType.Finishing
 					).ToList();
 
@@ -212,7 +215,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
 
 		private void UpdateRunning()
 		{
-			var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
+			var running = DataService.GetLeafInputs(LeafInputStatusType.Running, LeafInputStatusType.Unresponsive).ToList();
 			foreach (var leafInput in running)
 			{
 				try
@@ -227,12 +230,21 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
 							PiscalWarningHandler($"LeafInput: {leafInput.Id}, {pendingToRetry}", leafInput);
 							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending, details: pendingToRetry);
 							break;
-
+                        
 						case PiscalStatus.Running:
 							Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id);
-							//leafInput.EndTime;
-							// continue running
-							break;
+                            // it's exceeded the threshhold for being unresponsive but hasn't already been marked
+						    if (IsLeafInputUnresponsive(leafInput) && !leafInput.IsUnresponsive)
+						    {
+						        var warning =
+						            "Exceeded expected runtime, current elapsed time is " 
+						            + $"{leafInput.TimeInProgress.ToReadableString()}";
+
+						        PiscalWarningHandler($"LeafInput: {leafInput.Id}, {warning}", leafInput);
+                                DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Unresponsive, details: warning);
+                            }
+                            // continue running
+                            break;
 
 						case PiscalStatus.Complete:
 							DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing, "Copying LeafOutput from Piscal");
@@ -252,5 +264,14 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
 				}
 			}
 		}
+
+	    private static bool IsLeafInputUnresponsive(ILeafInput leafInput)
+	    {
+            // Test Unresponsive
+	        //return true;
+	        var piscalUnresponsiveHourCount = Convert.ToInt32(ConfigurationManager.AppSettings["PiscalUnresponsiveHourCount"]);
+
+	        return leafInput.TimeInProgress > TimeSpan.FromHours(piscalUnresponsiveHourCount);
+	    }
 	}
 }