Add unresponsive status

This commit is contained in:
2018-09-12 21:23:47 -04:00
parent 91d9efa7f7
commit 62e4e156ae
13 changed files with 106 additions and 61 deletions
+5 -2
View File
@@ -64,16 +64,19 @@ namespace LeafWeb.Core.Entities
public bool IsCancelPending => CurrentStatus == LeafInputStatusType.CancelPending;
public bool IsCancelling => CurrentStatus == LeafInputStatusType.Cancelling;
public bool IsCancelled => CurrentStatus == LeafInputStatusType.Cancelled;
public bool IsUnresponsive => CurrentStatus == LeafInputStatusType.Unresponsive;
public bool IsInProgress => IsStarting
|| IsRunning
|| IsFinishing
|| IsCancelPending
|| IsCancelling;
|| IsCancelling
|| IsUnresponsive;
public bool IsCancellable =>
IsRunning
|| IsPending;
|| IsPending
|| IsUnresponsive;
public bool IsDeletable => !IsInProgress;
public bool IsAtEndState => IsComplete || IsException || IsCancelled;
+2 -1
View File
@@ -10,6 +10,7 @@ namespace LeafWeb.Core.Entities
Exception = 5,
CancelPending = 6,
Cancelling = 7,
Cancelled = 8
Cancelled = 8,
Unresponsive = 9
}
}
+1 -1
View File
@@ -1 +1 @@
C:\Users\poprhythm\AppData\Local\Temp\Temporary ASP.NET Files\vs\f80e29bb\faae20bf\App_Web_all.generated.cs.8f9494c4.ulmxez9s.dll
C:\Users\poprhythm\AppData\Local\Temp\Temporary ASP.NET Files\vs\f80e29bb\faae20bf\App_Web_all.generated.cs.8f9494c4.kym3i-lp.dll
+4
View File
@@ -102,6 +102,10 @@ a.banner-link:hover {
color: #a94442; }
.status.status-exception:after {
content: "\e101"; }
.status.status-unresponsive {
color: #66512c; }
.status.status-unresponsive:after {
content: "\e085"; }
.status.status-running, .status.status-starting, .status.status-finishing {
color: #3c763d; }
.status.status-running:after {
+1 -1
View File
@@ -1 +1 @@
body.debug header .brand{position:relative;}body.debug header .brand:before{content:"";font-family:FontAwesome;font-style:normal;font-weight:normal;text-decoration:inherit;color:#f00;font-size:60px;padding-right:.5em;position:absolute;top:10px;left:0;}h1{padding:24px 0 12px 0;}p{padding:12px 0;}i{cursor:default;}footer{margin-top:24px !important;}.row-no-padding [class*="col-"]{padding-left:0 !important;padding-right:0 !important;}.home .dark .row:first-child .column:first-child h1{padding-top:0;}.home .blogarchive{padding-top:20px;}.top-buffer{margin-top:20px;}.detail-actions>a,.detail-actions>form>button{margin-top:20px;float:left;clear:left;}.banner-link{white-space:normal;padding:20px;background:#000;background:rgba(0,0,0,.5);-moz-border-radius:10px;border-radius:10px;}.banner-link .glyphicon{color:#8cc641;}a.banner-link:hover{text-decoration:none;background:rgba(0,0,0,.6);}a.banner-link:hover .glyphicon{color:#a8ed4f;}.headline-icon h1:after{color:rgba(172,214,118,.8);font-family:"Glyphicons Halflings";font-size:.8em;padding-left:10px;}.headline-icon.headline-icon-file h1:after{content:"";}.headline-icon.headline-icon-leaf h1:after{content:"";}.headline-icon.headline-icon-question h1:after{content:"";}.headline-icon.headline-icon-stats h1:after{content:"";}.headline-icon.headline-icon-user h1:after{content:"";}.headline-icon.headline-icon-list h1:after{content:"";}.status{white-space:nowrap;cursor:default;}.status:after{font-family:"Glyphicons Halflings";font-size:.8em;padding-left:5px;}.status.status-pending{color:#f0ad4e;}.status.status-pending:after{content:"";}.status.status-complete{color:#337ab7;}.status.status-complete:after{content:"";}.status.status-exception{color:#a94442;}.status.status-exception:after{content:"";}.status.status-running,.status.status-starting,.status.status-finishing{color:#3c763d;}.status.status-running:after{content:"";}.status.status-starting:after{content:"";}.status.status-finishing:after{content:"";}.status.status-cancelpending,.status.status-cancelling{color:#f0ad4e;}.status.status-cancelled{color:#ec971f;}.status.status-cancelpending:after{content:"";}.status.status-cancelling:after{content:"";}.status.status-cancelled:after{content:"";}#chart{padding-top:20px;}.btn-file{position:relative;overflow:hidden;}.btn-file input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;font-size:100px;text-align:right;filter:alpha(opacity=0);opacity:0;outline:none;background:#fff;cursor:inherit;display:block;}form .validation-summary-errors ul{list-style-type:none;}.autocomplete-suggestions{border:1px solid #999;background:#fff;overflow:auto;}.autocomplete-suggestion{padding:2px 5px;white-space:nowrap;overflow:hidden;}.autocomplete-selected{background:#f0f0f0;}.autocomplete-suggestions strong{font-weight:normal;color:#39f;}.autocomplete-group{padding:2px 5px;}.autocomplete-group strong{display:block;border-bottom:1px solid #000;}.toggle{width:15px;}.dropdown-menu li form .btn-link{display:block;color:#333;clear:both;float:left;font-size:1rem;font-weight:normal;line-height:1.42857;min-width:160px;padding:3px 20px;text-align:left;white-space:nowrap;}.dropdown-menu li form .btn-link:focus,.dropdown-menu li form .btn-link:hover{text-decoration:none;color:#262626;background-color:#f5f5f5;}.divider-right{border-right:1px dashed #333;}.read{overflow:hidden;font-size:16px !important;transition:all .3s;margin-bottom:10px;}.read-less{height:90px !important;}code,pre{line-height:1;}.alert{position:absolute;left:30%;top:10em;width:40%;z-index:1;}.copyright{font-size:.8em;color:#777 !important;cursor:default;}.withshadow{text-shadow:2px 2px 4px #000;}
body.debug header .brand{position:relative;}body.debug header .brand:before{content:"";font-family:FontAwesome;font-style:normal;font-weight:normal;text-decoration:inherit;color:#f00;font-size:60px;padding-right:.5em;position:absolute;top:10px;left:0;}h1{padding:24px 0 12px 0;}p{padding:12px 0;}i{cursor:default;}footer{margin-top:24px !important;}.row-no-padding [class*="col-"]{padding-left:0 !important;padding-right:0 !important;}.home .dark .row:first-child .column:first-child h1{padding-top:0;}.home .blogarchive{padding-top:20px;}.top-buffer{margin-top:20px;}.detail-actions>a,.detail-actions>form>button{margin-top:20px;float:left;clear:left;}.banner-link{white-space:normal;padding:20px;background:#000;background:rgba(0,0,0,.5);-moz-border-radius:10px;border-radius:10px;}.banner-link .glyphicon{color:#8cc641;}a.banner-link:hover{text-decoration:none;background:rgba(0,0,0,.6);}a.banner-link:hover .glyphicon{color:#a8ed4f;}.headline-icon h1:after{color:rgba(172,214,118,.8);font-family:"Glyphicons Halflings";font-size:.8em;padding-left:10px;}.headline-icon.headline-icon-file h1:after{content:"";}.headline-icon.headline-icon-leaf h1:after{content:"";}.headline-icon.headline-icon-question h1:after{content:"";}.headline-icon.headline-icon-stats h1:after{content:"";}.headline-icon.headline-icon-user h1:after{content:"";}.headline-icon.headline-icon-list h1:after{content:"";}.status{white-space:nowrap;cursor:default;}.status:after{font-family:"Glyphicons Halflings";font-size:.8em;padding-left:5px;}.status.status-pending{color:#f0ad4e;}.status.status-pending:after{content:"";}.status.status-complete{color:#337ab7;}.status.status-complete:after{content:"";}.status.status-exception{color:#a94442;}.status.status-exception:after{content:"";}.status.status-unresponsive{color:#66512c;}.status.status-unresponsive:after{content:"";}.status.status-running,.status.status-starting,.status.status-finishing{color:#3c763d;}.status.status-running:after{content:"";}.status.status-starting:after{content:"";}.status.status-finishing:after{content:"";}.status.status-cancelpending,.status.status-cancelling{color:#f0ad4e;}.status.status-cancelled{color:#ec971f;}.status.status-cancelpending:after{content:"";}.status.status-cancelling:after{content:"";}.status.status-cancelled:after{content:"";}#chart{padding-top:20px;}.btn-file{position:relative;overflow:hidden;}.btn-file input[type=file]{position:absolute;top:0;right:0;min-width:100%;min-height:100%;font-size:100px;text-align:right;filter:alpha(opacity=0);opacity:0;outline:none;background:#fff;cursor:inherit;display:block;}form .validation-summary-errors ul{list-style-type:none;}.autocomplete-suggestions{border:1px solid #999;background:#fff;overflow:auto;}.autocomplete-suggestion{padding:2px 5px;white-space:nowrap;overflow:hidden;}.autocomplete-selected{background:#f0f0f0;}.autocomplete-suggestions strong{font-weight:normal;color:#39f;}.autocomplete-group{padding:2px 5px;}.autocomplete-group strong{display:block;border-bottom:1px solid #000;}.toggle{width:15px;}.dropdown-menu li form .btn-link{display:block;color:#333;clear:both;float:left;font-size:1rem;font-weight:normal;line-height:1.42857;min-width:160px;padding:3px 20px;text-align:left;white-space:nowrap;}.dropdown-menu li form .btn-link:focus,.dropdown-menu li form .btn-link:hover{text-decoration:none;color:#262626;background-color:#f5f5f5;}.divider-right{border-right:1px dashed #333;}.read{overflow:hidden;font-size:16px !important;transition:all .3s;margin-bottom:10px;}.read-less{height:90px !important;}code,pre{line-height:1;}.alert{position:absolute;left:30%;top:10em;width:40%;z-index:1;}.copyright{font-size:.8em;color:#777 !important;cursor:default;}.withshadow{text-shadow:2px 2px 4px #000;}
+8
View File
@@ -143,6 +143,7 @@ a.banner-link:hover {
content:"\e084";
}
}
&.status-exception {
color: #a94442;
&:after {
@@ -150,6 +151,13 @@ a.banner-link:hover {
}
}
&.status-unresponsive {
color: #66512c;
&:after {
content:"\e085";
}
}
&.status-running, &.status-starting, &.status-finishing {
color: #3c763d;
}
+26 -26
View File
@@ -125,35 +125,35 @@ namespace LeafWeb.WebCms.Services
var message = new MailMessage(_emailFromAddress, leafInput.Email, FormatSubject(SuccessSubject, leafInput), body);
SendMessage(message);
}
else
{
body += "Please see the attached results.";
//else
//{
// body += "Please see the attached results.";
var message = new MailMessage(_emailFromAddress, leafInput.Email, SuccessSubject, body);
// var message = new MailMessage(_emailFromAddress, leafInput.Email, SuccessSubject, body);
var fileStreams =
(from outputFile in
leafInput.OutputFiles
select Tuple.Create(outputFile, new MemoryStream(outputFile.FileContents.Contents))).ToList();
try
{
foreach (var fileStream in fileStreams)
{
var attachment = new Attachment(fileStream.Item2, fileStream.Item1.Filename);
message.Attachments.Add(attachment);
}
// var fileStreams =
// (from outputFile in
// leafInput.OutputFiles
// select Tuple.Create(outputFile, new MemoryStream(outputFile.FileContents.Contents))).ToList();
// try
// {
// foreach (var fileStream in fileStreams)
// {
// var attachment = new Attachment(fileStream.Item2, fileStream.Item1.Filename);
// message.Attachments.Add(attachment);
// }
SendMessage(message);
}
finally
{
// can't dispose those memory streams until the message is sent
foreach (var stream in fileStreams.Select(f => f.Item2))
{
stream.Dispose();
}
}
}
// SendMessage(message);
// }
// finally
// {
// // can't dispose those memory streams until the message is sent
// foreach (var stream in fileStreams.Select(f => f.Item2))
// {
// stream.Dispose();
// }
// }
//}
}
private void SendLeafWebError(LeafInput leafInput, string errorMessage)
@@ -1,8 +1,10 @@
using System;
using System.Configuration;
using System.Linq;
using System.Threading;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using LeafWeb.Core.Utility;
using LeafWeb.WebCms.App_Start;
namespace LeafWeb.WebCms.Services.PiscalQueue
@@ -62,7 +64,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
// send notification immediately
BackgroundJobEnqueueRetry<EmailNotificationService>(email => email.SendLeafWebCancelled(leafInput.Id));
}
else if (leafInput.IsRunning)
else if (leafInput.IsRunning || leafInput.IsUnresponsive)
{
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.CancelPending);
HangfireStartup.TriggerPiscalProcessQueue();
@@ -174,6 +176,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
DataService.GetLeafInputs(
LeafInputStatusType.Starting,
LeafInputStatusType.Running,
LeafInputStatusType.Unresponsive,
LeafInputStatusType.Finishing
).ToList();
@@ -212,7 +215,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
private void UpdateRunning()
{
var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
var running = DataService.GetLeafInputs(LeafInputStatusType.Running, LeafInputStatusType.Unresponsive).ToList();
foreach (var leafInput in running)
{
try
@@ -227,12 +230,21 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
PiscalWarningHandler($"LeafInput: {leafInput.Id}, {pendingToRetry}", leafInput);
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending, details: pendingToRetry);
break;
case PiscalStatus.Running:
Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id);
//leafInput.EndTime;
// continue running
break;
// it's exceeded the threshhold for being unresponsive but hasn't already been marked
if (IsLeafInputUnresponsive(leafInput) && !leafInput.IsUnresponsive)
{
var warning =
"Exceeded expected runtime, current elapsed time is "
+ $"{leafInput.TimeInProgress.ToReadableString()}";
PiscalWarningHandler($"LeafInput: {leafInput.Id}, {warning}", leafInput);
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Unresponsive, details: warning);
}
// continue running
break;
case PiscalStatus.Complete:
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing, "Copying LeafOutput from Piscal");
@@ -252,5 +264,14 @@ namespace LeafWeb.WebCms.Services.PiscalQueue
}
}
}
private static bool IsLeafInputUnresponsive(ILeafInput leafInput)
{
// Test Unresponsive
//return true;
var piscalUnresponsiveHourCount = Convert.ToInt32(ConfigurationManager.AppSettings["PiscalUnresponsiveHourCount"]);
return leafInput.TimeInProgress > TimeSpan.FromHours(piscalUnresponsiveHourCount);
}
}
}
+18 -15
View File
@@ -4,22 +4,22 @@
<div class="row">
<div class="col-sm-3 divider-right detail-actions">
<h3>Actions</h3>
@Html.Partial("DisplayTemplates/_ChartButton", Model.LeafInputId, new ViewDataDictionary { { "Disabled", !Model.HasLeafChart } })
<a href="@Url.Action("DownloadInput", "Queue", new {id = Model.LeafInputId})"
class="btn btn-default" role="button">
class="btn btn-default" role="button">
<span class="glyphicon glyphicon-download"></span> Download Input
</a>
<a href="@Url.Action("DownloadOutputToUser", "Queue", new {id = Model.LeafInputId})"
class="btn btn-default@{if (!Model.HasOutputFiles)
class="btn btn-default@{if (!Model.HasOutputFiles)
{<text> disabled</text>}}" role="button">
<span class="glyphicon glyphicon-download"></span> Download ToUser
</a>
<a href="@Url.Action("DownloadOutputNotToUser", "Queue", new {id = Model.LeafInputId})"
class="btn btn-default@{if (!Model.HasOutputFiles)
class="btn btn-default@{if (!Model.HasOutputFiles)
{<text> disabled</text>}}" role="button">
<span class="glyphicon glyphicon-download"></span> Download NotToUser
</a>
@@ -32,22 +32,25 @@
<span class="glyphicon glyphicon-send"></span> Email User Download link
</button>
}
@if (Model.IsCancellable)
{
@CancelLink(Model)
}
@DeleteLink(Model.LeafInputId, Model.Identifier, Model.IsDeletable)
@DeleteLink(Model)
</div>
<div class="col-sm-9">
@Html.DisplayForModel()
</div>
</div>
@helper DeleteLink(int id, string identifier, bool deletable)
@helper DeleteLink(LeafInputDetails item)
{
using (Html.BeginUmbracoForm<QueueController>(
"Delete", null, new { @class = "confirm", confirm_msg = "Deletion cannot be undone! Confirm deleting '" + identifier + "'." }))
{
<input type="hidden" name="id" value="@id" />
<button type="submit" class="btn btn-default" @{if (!deletable) { <text> disabled="disabled" </text> }}>
<span class="glyphicon glyphicon-remove"></span> Delete
</button>
}
@Html.Partial("DisplayTemplates/_DeleteForm", Tuple.Create(item.LeafInputId, item.Identifier, item.IsDeletable, false))
}
@helper CancelLink(LeafInputDetails item)
{
@Html.Partial("DisplayTemplates/_CancelForm", Tuple.Create(item.LeafInputId, item.Identifier, false))
}
+5 -5
View File
@@ -146,17 +146,17 @@ else
}
@helper DeleteLink(LeafInput item)
{
@Html.Partial("DisplayTemplates/_DeleteForm", Tuple.Create(item.Id, item.Identifier, item.IsDeletable))
@Html.Partial("DisplayTemplates/_DeleteForm", Tuple.Create(item.Id, item.Identifier, item.IsDeletable, true))
}
@helper CancelLink(dynamic item)
@helper CancelLink(LeafInput item)
{
@Html.Partial("DisplayTemplates/_CancelForm", (Tuple<int, string>)Tuple.Create(item.Id, item.Identifier))
@Html.Partial("DisplayTemplates/_CancelForm", Tuple.Create(item.Id, item.Identifier, true))
}
@helper PriorityForm(dynamic item, Priority priority)
@helper PriorityForm(LeafInput item, Priority priority)
{
@Html.Partial("DisplayTemplates/_PriorityForm", (Tuple<int, Priority, Priority>)Tuple.Create(item.Id, item.PendingPriority, priority))
@Html.Partial("DisplayTemplates/_PriorityForm", Tuple.Create(item.Id, item.PendingPriority, priority))
}
@helper DisableItem(bool disabled)
@@ -1,14 +1,16 @@
@using LeafWeb.WebCms.Controllers
@model Tuple<int, string>
@model Tuple<int, string, bool>
@{
var leafInputId = Model.Item1;
var identifier = Model.Item2;
// true for a link, false for a button
var buttonType = Model.Item3 ? "btn-link" : "btn-default";
}
@using (Html.BeginUmbracoForm<QueueController>("Cancel", null,
new { @class = "confirm clearfix", confirm_msg = "Cancelling cannot be undone! Confirm cancelling '" + identifier + "'." }))
{
<input type="hidden" name="id" value="@leafInputId"/>
<button type="submit" class="btn btn-link">
<button type="submit" class="btn @buttonType">
<span class="glyphicon glyphicon-ban-circle"></span> Cancel
</button>
}
@@ -1,15 +1,17 @@
@using LeafWeb.WebCms.Controllers
@model Tuple<int, string, bool>
@model Tuple<int, string, bool, bool>
@{
var leafInputId = Model.Item1;
var identifier = Model.Item2;
var isDeletable = Model.Item3;
// true for a link, false for a button
var buttonType = Model.Item4 ? "btn-link" : "btn-default";
}
@using (Html.BeginUmbracoForm<QueueController>("Delete", null,
new { @class = "confirm clearfix", confirm_msg = "Deletion cannot be undone! Confirm deleting '" + identifier + "'." }))
{
<input type="hidden" name="id" value="@leafInputId"/>
<button type="submit" class="btn btn-link" @{if (!isDeletable) { <text> disabled="disabled" </text> }}>
<button type="submit" class="btn @buttonType" @{if (!isDeletable) { <text> disabled="disabled" </text> }}>
<span class="glyphicon glyphicon-remove"></span> Delete
</button>
}
+1
View File
@@ -69,6 +69,7 @@
<add key="LeafWebUrl" value="http://192.168.1.133:61755/" />
<add key="PiscalNotifyCompleteUrlPath" value="umbraco/surface/LeafInput/NotifyComplete" />
<add key="ResultsDownloadPath" value="Results/Download?token={0}" />
<add key="PiscalUnresponsiveHourCount" value="24" />
</appSettings>
<connectionStrings>
<remove name="umbracoDbDSN" />