diff --git a/WebCms/App_Start/HangfireStartup.cs b/WebCms/App_Start/HangfireStartup.cs new file mode 100644 index 0000000..71f16a4 --- /dev/null +++ b/WebCms/App_Start/HangfireStartup.cs @@ -0,0 +1,107 @@ +using System.Web.Hosting; +using Hangfire; +using LeafWeb.WebCms.App_Start; +using LeafWeb.WebCms.Services.PiscalQueue; +using Microsoft.Owin; +using Owin; +using Umbraco.Web; +using ConfigurationManager = System.Configuration.ConfigurationManager; + +[assembly: OwinStartup(typeof(LeafwebOwinStartup))] + +namespace LeafWeb.WebCms.App_Start +{ + public class LeafwebOwinStartup : UmbracoDefaultOwinStartup + { + public new void Configuration(IAppBuilder app) + { + base.Configuration(app); + new HangfireStartup().Configuration(app); + } + } + + // https://our.umbraco.org/forum/umbraco-7/using-umbraco-7/59500-Umbraco-and-Background-Tasks + public class HangfireStartup + { + private const string PiscalProcessQueue = "PiscalProcessQueue"; + + public void Configuration(IAppBuilder app) + { + app.UseHangfireDashboard(); + + SetupRecurringJobs(); + } + + private void SetupRecurringJobs() + { + var queueInterval = ConfigurationManager.AppSettings["ProcessQueueInterval"]; + // https://discuss.hangfire.io/t/how-to-create-cron-job-that-is-executing-every-15-minutes/533 + RecurringJob.AddOrUpdate(PiscalProcessQueue, p => p.ProcessQueue(), queueInterval); + } + + public static void TriggerPiscalProcessQueue() + { + RecurringJob.Trigger(PiscalProcessQueue); + } + } + + // http://docs.hangfire.io/en/latest/deployment-to-production/making-aspnet-app-always-running.html + public class ApplicationPreload : IProcessHostPreloadClient + { + public void Preload(string[] parameters) + { + HangfireBootstrapper.Instance.Start(); + } + } + + public class HangfireBootstrapper : IRegisteredObject + { + public static readonly HangfireBootstrapper Instance = new HangfireBootstrapper(); + + private readonly object _lockObject = new object(); + private bool _started; + + private BackgroundJobServer _backgroundJobServer; + + private HangfireBootstrapper() + { + } + + public void Start() + { + lock (_lockObject) + { + if (_started) return; + _started = true; + + HostingEnvironment.RegisterObject(this); + + // TODO: use Umbraco Connection string? + var cs = Umbraco.Core.ApplicationContext.Current.DatabaseContext.ConnectionString; + + GlobalConfiguration + .Configuration + .UseSqlServerStorage("LeafWebContext"); + // Specify other options here + + _backgroundJobServer = new BackgroundJobServer(); + } + } + + public void Stop() + { + lock (_lockObject) + { + + _backgroundJobServer?.Dispose(); + + HostingEnvironment.UnregisterObject(this); + } + } + + void IRegisteredObject.Stop(bool immediate) + { + Stop(); + } + } +} \ No newline at end of file diff --git a/WebCms/App_Start/RegisterDataService.cs b/WebCms/App_Start/RegisterServices.cs similarity index 57% rename from WebCms/App_Start/RegisterDataService.cs rename to WebCms/App_Start/RegisterServices.cs index 509f6c4..fd15057 100644 --- a/WebCms/App_Start/RegisterDataService.cs +++ b/WebCms/App_Start/RegisterServices.cs @@ -1,17 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Web; -using LeafWeb.Core.DAL; +using LeafWeb.Core.DAL; using Umbraco.Core; -namespace WebCms.App_Start +namespace LeafWeb.WebCms.App_Start { - public class RegisterDataService : ApplicationEventHandler + public class RegisterServices : ApplicationEventHandler { protected override void ApplicationStarted(UmbracoApplicationBase umbracoApplication, ApplicationContext applicationContext) { DataService.RegisterInitializer(); + HangfireBootstrapper.Instance.Start(); base.ApplicationStarted(umbracoApplication, applicationContext); } diff --git a/WebCms/Controllers/ControllerBase.cs b/WebCms/Controllers/ControllerBase.cs index 40ef984..de75120 100644 --- a/WebCms/Controllers/ControllerBase.cs +++ b/WebCms/Controllers/ControllerBase.cs @@ -5,7 +5,7 @@ using log4net; using LeafWeb.Core.DAL; using Umbraco.Web.Mvc; -namespace WebCms.Controllers +namespace LeafWeb.WebCms.Controllers { public class BaseController : SurfaceController { diff --git a/WebCms/Controllers/LeafInputController.cs b/WebCms/Controllers/LeafInputController.cs index fc4b536..f029f81 100644 --- a/WebCms/Controllers/LeafInputController.cs +++ b/WebCms/Controllers/LeafInputController.cs @@ -1,7 +1,7 @@ using System.Web.Mvc; -using WebCms.Models; +using LeafWeb.WebCms.Models; -namespace WebCms.Controllers +namespace LeafWeb.WebCms.Controllers { public class LeafInputController : BaseController { diff --git a/WebCms/Models/LeafInputConfirm.cs b/WebCms/Models/LeafInputConfirm.cs index c980ba3..0129b33 100644 --- a/WebCms/Models/LeafInputConfirm.cs +++ b/WebCms/Models/LeafInputConfirm.cs @@ -1,10 +1,7 @@ using System.ComponentModel.DataAnnotations; -using System.Globalization; using AutoMapper; -using Umbraco.Core.Models; -using Umbraco.Web.Models; -namespace WebCms.Models +namespace LeafWeb.WebCms.Models { public class LeafInputConfirm { diff --git a/WebCms/Models/LeafInputCreate.cs b/WebCms/Models/LeafInputCreate.cs index 95b4076..936545a 100644 --- a/WebCms/Models/LeafInputCreate.cs +++ b/WebCms/Models/LeafInputCreate.cs @@ -1,11 +1,7 @@ using System.ComponentModel.DataAnnotations; -using System.Globalization; using AutoMapper; -using LeafWeb.Core.DAL; -using Umbraco.Core.Models; -using Umbraco.Web.Models; -namespace WebCms.Models +namespace LeafWeb.WebCms.Models { public class LeafInputCreate { diff --git a/WebCms/Models/SelectListViewModel.cs b/WebCms/Models/SelectListViewModel.cs index 99809e6..d0fa08d 100644 --- a/WebCms/Models/SelectListViewModel.cs +++ b/WebCms/Models/SelectListViewModel.cs @@ -1,7 +1,7 @@ using System.Linq; using System.Web.Mvc; -namespace WebCms.Models +namespace LeafWeb.WebCms.Models { public class SelectListViewModel { diff --git a/WebCms/Services/DownloadUrlService.cs b/WebCms/Services/DownloadUrlService.cs new file mode 100644 index 0000000..37a9c2f --- /dev/null +++ b/WebCms/Services/DownloadUrlService.cs @@ -0,0 +1,22 @@ +using System.Configuration; +using LeafWeb.Core.Entities; + +namespace LeafWeb.WebCms.Services +{ + public class DownloadUrlService + { + private readonly string _downloadUrl; + + public DownloadUrlService() + { + _downloadUrl = + ConfigurationManager.AppSettings["LeafWebUrl"] + + ConfigurationManager.AppSettings["ResultsDownloadPath"]; + } + + public string GetDownloadUrl(LeafInput leafInput) + { + return string.Format(_downloadUrl, leafInput.UniqueToken); + } + } +} \ No newline at end of file diff --git a/WebCms/Services/EmailNotificationService.cs b/WebCms/Services/EmailNotificationService.cs new file mode 100644 index 0000000..332f688 --- /dev/null +++ b/WebCms/Services/EmailNotificationService.cs @@ -0,0 +1,177 @@ +using System; +using System.Configuration; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Mail; +using log4net; +using LeafWeb.Core.DAL; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Utility; + +namespace LeafWeb.WebCms.Services +{ + public class EmailNotificationService : IDisposable + { + private static readonly ILog Logger = LogManager.GetLogger(typeof(EmailNotificationService)); + + private readonly string _emailFromAddress; + + private const string EmailSuccessSubject = "LeafWeb Results"; + private const string EmailErrorSubject = "LeafWeb processing error"; + private const string EmailSystemErrorSubject = "LeafWeb system error"; + + /// + /// Comma separated values + /// + private readonly string _adminEmailAddresses; + + private readonly SmtpClient _smtpClient; + + private readonly DataService _dataService; + + private readonly DownloadUrlService _downloadUrlService; + + public EmailNotificationService(DataService dataService) + { + _dataService = dataService; + + _downloadUrlService = new DownloadUrlService(); + + _smtpClient = new SmtpClient( + ConfigurationManager.AppSettings["SmtpHost"], + Convert.ToInt32(ConfigurationManager.AppSettings["SmtpPort"])); + + if (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["SmtpUserName"])) + _smtpClient.Credentials = new NetworkCredential( + ConfigurationManager.AppSettings["SmtpUserName"], + ConfigurationManager.AppSettings["SmtpPassword"] + ); + + _emailFromAddress = ConfigurationManager.AppSettings["EmailFromAddress"]; + _adminEmailAddresses = ConfigurationManager.AppSettings["AdminEmailAddresses"]; + } + + public EmailNotificationService() : this(new DataService()) + { } + + public void SendLeafWebComplete(int leafInputId) + { + var leafInput = _dataService.GetLeafInput(leafInputId); + + var outputErrorMessage = leafInput.OutputErrorMessage; + if (outputErrorMessage != null) + SendLeafWebError(leafInput, outputErrorMessage.FileContents.Contents.GetString()); + else + SendLeafWebSuccess(leafInput); + } + + public void SendAdministratorMessage(string subject, string body) + { + var message = new MailMessage(_emailFromAddress, _adminEmailAddresses, subject, body); + SendMessage(message); + } + + private void SendLeafWebSuccess(LeafInput leafInput) + { + var body = $"Your leaf analysis job, {leafInput.Identifier}, has completed. "; + + body += FormatWarningMessage(leafInput); + + if (true) + { + var downloadUrl = _downloadUrlService.GetDownloadUrl(leafInput); + + body += + "Download results with the following link:" + + Environment.NewLine + Environment.NewLine + + downloadUrl; + + var message = new MailMessage(_emailFromAddress, leafInput.Email, EmailSuccessSubject, body); + SendMessage(message); + } + else + { + body += "Please see the attached results."; + + var message = new MailMessage(_emailFromAddress, leafInput.Email, EmailSuccessSubject, body); + + var fileStreams = + (from outputFile in + leafInput.OutputFiles + select Tuple.Create(outputFile, new MemoryStream(outputFile.FileContents.Contents))).ToList(); + try + { + foreach (var fileStream in fileStreams) + { + var attachment = new Attachment(fileStream.Item2, fileStream.Item1.Filename); + message.Attachments.Add(attachment); + } + + SendMessage(message); + } + finally + { + // can't dispose those memory streams until the message is sent + foreach (var stream in fileStreams.Select(f => f.Item2)) + { + stream.Dispose(); + } + } + } + } + + private void SendLeafWebError(LeafInput leafInput, string errorMessage) + { + var body = $"Your leaf analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine + + Environment.NewLine + Environment.NewLine + + errorMessage + + Environment.NewLine + Environment.NewLine + + "You will need to correct your input and resubmit."; + + body += FormatWarningMessage(leafInput); + + var message = new MailMessage(_emailFromAddress, leafInput.Email, EmailErrorSubject, body); + SendMessage(message); + } + + public void SendLeafWebSystemException(string leafInputIdentifier, string leafInputEmail) + { + var body = $"A system error occured while processing your leaf analysis job, {leafInputIdentifier}." + Environment.NewLine + + "System administrators have been notified. You will be notified again when the system error " + + "has been resolved and your data has been processed."; + + var message = new MailMessage(_emailFromAddress, leafInputEmail, EmailSystemErrorSubject, body); + SendMessage(message); + } + + private string FormatWarningMessage(LeafInput leafInput) + { + if (leafInput.OutputWarningMessage != null) + return Environment.NewLine + Environment.NewLine + + "The following warning message was generated." + + Environment.NewLine + Environment.NewLine + + leafInput.OutputWarningMessage.FileContents.Contents.GetString() + + Environment.NewLine; + return string.Empty; + } + + private void SendMessage(MailMessage mailMessage) + { + try + { + Logger.Debug("Email sending to " + mailMessage.To + ", subject: " + mailMessage.Subject); + _smtpClient.Send(mailMessage); + } + catch (SmtpException ex) + { + Logger.Error($"Failed to send mail: {ex.Message}", ex); + } + } + + public void Dispose() + { + _dataService.Dispose(); + } + } +} \ No newline at end of file diff --git a/WebCms/Services/LeafGasCharter.cs b/WebCms/Services/LeafGasCharter.cs new file mode 100644 index 0000000..0e2ac02 --- /dev/null +++ b/WebCms/Services/LeafGasCharter.cs @@ -0,0 +1,252 @@ +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Web.UI.DataVisualization.Charting; +using System.Web.UI.WebControls; +using LeafWeb.Core.Charter; +using LeafWeb.Core.Utility; + +namespace LeafWeb.WebCms.Services +{ + public static class LeafGasCharter + { + private static readonly Font TitleFont = new Font(new FontFamily("Times New Roman"), 10, FontStyle.Bold); + private static readonly Font AxisFont = new Font(new FontFamily("Times New Roman"), 10, FontStyle.Bold); + private static readonly Font Font = new Font(new FontFamily("Trebuchet MS"), 9, FontStyle.Bold); + public const int ChartWidth = 550; + public const int ChartHeight = 400; + + // cntrlcomparison + public static IEnumerable ProduceCharts(CurveData curve) + { + var curveId = curve.CurveId; + + var paramTitles = new[] + { + new {param = curve.FixedCndFixedCmp, title = ReflectionExtensions.GetPropertyDisplayName(c => c.FixedCndFixedCmp) }, + new {param = curve.FixedCndEstimatedCmp, title = ReflectionExtensions.GetPropertyDisplayName(c => c.FixedCndEstimatedCmp) }, + new {param = curve.EstimatedCndFixedCmp, title = ReflectionExtensions.GetPropertyDisplayName(c => c.EstimatedCndFixedCmp) }, + new {param = curve.EstimatedCndEstimatedCmp,title = ReflectionExtensions.GetPropertyDisplayName(c => c.EstimatedCndEstimatedCmp) }, + }; + + return paramTitles.SelectMany(item => CurveSeries(curveId, item.param, item.title)); + } + + private static IEnumerable CurveSeries(string curveId, CurveParamSet paramSet, string chartTitle) + { + var chloroChart = CreateEmptyChart("Chloroplastic CO2 partial pressure (Pa)", ChartWidth, ChartHeight); + var interChart = CreateEmptyChart("Intercellular CO2 partial pressure (Pa)", ChartWidth, ChartHeight); + + // Set the points for the symbol series for paramater set 1, chloroplastic + AddAnetMeasPoints(paramSet.AnetMeasChloro1Data, chloroChart.Series["Rubisco-limited"]); + AddAnetMeasPoints(paramSet.AnetMeasChloro2Data, chloroChart.Series["RuBP regeneration-limited"]); + + var tpuSeries = NewTpuSeries(paramSet.AnetMeasChloro3Data); + AddAnetMeasPoints(paramSet.AnetMeasChloro3Data, tpuSeries); + chloroChart.Series.Add(tpuSeries); + + // Set the points for the symbol series for paramater set 1, intercellular + AddAnetMeasPoints(paramSet.AnetMeasInter1Data, interChart.Series["Rubisco-limited"]); + AddAnetMeasPoints(paramSet.AnetMeasInter2Data, interChart.Series["RuBP regeneration-limited"]); + + tpuSeries = NewTpuSeries(paramSet.AnetMeasInter3Data); + AddAnetMeasPoints(paramSet.AnetMeasInter3Data, tpuSeries); + interChart.Series.Add(tpuSeries); + + // Set the points on the asymptote curve for parameter set 1, chloroplast + AddAsymptotePoints(paramSet.AcChloroData, chloroChart.Series["acCurve"]); + AddAsymptotePoints(paramSet.AjChloroData, chloroChart.Series["ajCurve"]); + AddAsymptotePoints(paramSet.AtChloroData, chloroChart.Series["atCurve"]); + + // Set the points on the asymptote curve for parameter set 1, intercellular + AddAsymptotePoints(paramSet.AcInterData, interChart.Series["acCurve"]); + AddAsymptotePoints(paramSet.AjInterData, interChart.Series["ajCurve"]); + AddAsymptotePoints(paramSet.AtInterData, interChart.Series["atCurve"]); + + var title = new Title($"LeafWeb curveID = {curveId}\n{chartTitle}"){Font = TitleFont}; + chloroChart.Titles.Add(title); + interChart.Titles.Add(title); + + chloroChart.ChartAreas["ChartArea"].AxisX.TitleFont = AxisFont; + chloroChart.ChartAreas["ChartArea"].AxisY.TitleFont = AxisFont; + + interChart.ChartAreas["ChartArea"].AxisX.TitleFont = AxisFont; + interChart.ChartAreas["ChartArea"].AxisY.TitleFont = AxisFont; + + yield return chloroChart; + yield return interChart; + } + + private static Series NewTpuSeries(IReadOnlyCollection data) + { + var seriesName = "TPU-limited"; + if (data.Count == 0) + seriesName = "Curve Asymptote"; + var series3 = new Series(seriesName) + { + MarkerSize = 9, + BorderWidth = 3, + XValueType = ChartValueType.Double, + ChartType = SeriesChartType.Point, + MarkerStyle = MarkerStyle.Square, + ShadowColor = Color.Black, + BorderColor = Color.Black, + Color = Color.Orange, + ShadowOffset = 0, + YValueType = ChartValueType.Double + }; + + return series3; + } + + private static void AddAnetMeasPoints(IEnumerable data, Series series) + { + // Set the points for the series from the ArrayList + foreach (var xy in data) + { + series.Points.AddXY(xy.X, xy.Y); + } + } + + private static void AddAsymptotePoints(IEnumerable data, Series series) + { + // Set the points for the series from the ArrayList + foreach (var xy in data.Where(xy => (xy.X != -9999) && (xy.Y != -9999))) + { + series.Points.AddXY(xy.X, xy.Y); + } + } + + private static Chart CreateEmptyChart(string axisXTitle, int width=700, int height=500) + { + var borderColor = Color.FromArgb(180, 26, 59, 105); + var chart = new Chart + { + BackColor = Color.White, + Width = Unit.Pixel(width), + Height = Unit.Pixel(height), + //BorderSkin = {SkinStyle = BorderSkinStyle.None}, + BorderColor = borderColor, + BorderlineColor = borderColor, + BorderlineWidth = 1, + BorderlineDashStyle = ChartDashStyle.Solid + }; + + chart.Legends.Add(new Legend + { + Enabled = true, + IsTextAutoFit = false, + Name = "Default", + Docking = Docking.Bottom, + BackColor = Color.Transparent, + Font = Font + }); + + chart.ChartAreas.Add(new ChartArea + { + Name = "ChartArea", + BorderColor = Color.FromArgb(64, 64, 64, 64), + BorderDashStyle = ChartDashStyle.Solid, + BackSecondaryColor = Color.White, + BackColor = Color.OldLace, + ShadowColor = Color.Transparent, + BackGradientStyle = GradientStyle.TopBottom, + AxisY = new Axis + { + LineColor = Color.FromArgb(64, 64, 64, 64), + Title = "Net assimilation rate (umol/m2/s)", + LabelStyle = { Font = Font }, + MajorGrid = new Grid { LineColor = Color.FromArgb(64, 64, 64, 64)} + }, + AxisX = new Axis + { + LineColor = Color.FromArgb(64, 64, 64, 64), + Minimum = 0, + Title = axisXTitle, + LabelStyle = { Font = Font }, + MajorGrid = new Grid { LineColor = Color.FromArgb(64, 64, 64, 64)} + } + }); + + chart.Series.Add(new Series + { + MarkerSize = 8, + BorderWidth = 3, + XValueType = ChartValueType.Double, + Name = "Rubisco-limited", + ChartType = SeriesChartType.Point, + MarkerStyle = MarkerStyle.Diamond, + ShadowColor = Color.Black, + BorderColor = borderColor, + Color = Color.Red, + ShadowOffset = 0, + YValueType = ChartValueType.Double + }); + + chart.Series.Add(new Series + { + MarkerSize = 9, + BorderWidth = 3, + XValueType = ChartValueType.Double, + Name = "RuBP regeneration-limited", + ChartType = SeriesChartType.Point, + MarkerStyle = MarkerStyle.Circle, + ShadowColor = Color.Black, + BorderColor = borderColor, + Color = Color.Blue, + ShadowOffset = 0, + YValueType = ChartValueType.Double + }); + + chart.Series.Add(new Series + { + MarkerSize = 2, + BorderWidth = 1, + XValueType = ChartValueType.Double, + Name = "acCurve", + ChartType = SeriesChartType.Line, + MarkerStyle = MarkerStyle.None, + ShadowColor = Color.Black, + BorderColor = borderColor, + Color = Color.Red, + ShadowOffset = 0, + YValueType = ChartValueType.Double, + IsVisibleInLegend = false + }); + + chart.Series.Add(new Series + { + MarkerSize = 2, + BorderWidth = 1, + XValueType = ChartValueType.Double, + Name = "ajCurve", + ChartType = SeriesChartType.Line, + MarkerStyle = MarkerStyle.None, + ShadowColor = Color.Black, + BorderColor = borderColor, + Color = Color.Blue, + ShadowOffset = 0, + YValueType = ChartValueType.Double, + IsVisibleInLegend = false + }); + + chart.Series.Add(new Series + { + MarkerSize = 2, + BorderWidth = 1, + XValueType = ChartValueType.Double, + Name = "atCurve", + ChartType = SeriesChartType.Line, + MarkerStyle = MarkerStyle.None, + ShadowColor = Color.Black, + BorderColor = borderColor, + Color = Color.Orange, + ShadowOffset = 0, + YValueType = ChartValueType.Double, + IsVisibleInLegend = false + }); + + return chart; + } + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/FinishComplete.cs b/WebCms/Services/PiscalQueue/FinishComplete.cs new file mode 100644 index 0000000..2ac8814 --- /dev/null +++ b/WebCms/Services/PiscalQueue/FinishComplete.cs @@ -0,0 +1,64 @@ +using System; +using System.Linq; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Parsers; +using LeafWeb.WebCms.App_Start; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + public class FinishComplete : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + Logger.DebugFormat("LeafInput: {0}, RetrieveOutputFiles", leafInput.Id); + + var leafOutputFiles = PiscalService.RetrieveOutputFiles(leafInput).ToList(); + + Logger.DebugFormat("LeafInput: {0}, RetrieveOutputFiles saving output files", leafInput.Id); + + foreach (var outputFile in leafOutputFiles) + { + if (leafInput.OutputFiles.All(file => file.Filename != outputFile.Filename)) + { + DataService.AddLeafOutputFile(outputFile); + + // parse cleaned input file data + if (outputFile.FileType == LeafOutputFileType.CleanedInput) + AddLeafInputData(outputFile, leafInput); + } + else + Logger.WarnFormat("LeafInput: {0}, RetrieveOutputFiles duplicate file name: {1}", leafInput.Id, outputFile.Filename); + } + + Logger.InfoFormat("LeafInput: {0}, RetrieveOutputFiles output files: {1}", leafInput.Id, + string.Join(", ", leafOutputFiles.Select(o => o.Filename))); + + Logger.DebugFormat("LeafInput: {0}, Set Complete", leafInput.Id); + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete); + + BackgroundJobEnqueueRetry(email => email.SendLeafWebComplete(leafInput.Id)); + + Logger.InfoFormat("LeafInput: {0}, Cleanup", leafInput.Id); + PiscalService.Cleanup(leafInput); + + HangfireStartup.TriggerPiscalProcessQueue(); + } + + private void AddLeafInputData(LeafOutputFile outputFile, LeafInput leafInput) + { + try + { + var parser = new LeafInputCsvParser(outputFile.FileContents.Contents); + var data = parser.Parse(); + data.LeafInput = leafInput; + data.LeafOutputFile = outputFile; + leafInput.LeafInputData.Add(data); + DataService.UpdateLeafInput(leafInput); + } + catch (Exception e) + { + Logger.Error($"LeafInput: {leafInput.Id}, while parsing CleanedInput file: {outputFile.Filename}", e); + } + } + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/PiscalQueueBase.cs b/WebCms/Services/PiscalQueue/PiscalQueueBase.cs new file mode 100644 index 0000000..2b79a98 --- /dev/null +++ b/WebCms/Services/PiscalQueue/PiscalQueueBase.cs @@ -0,0 +1,74 @@ +using System; +using System.Linq.Expressions; +using Hangfire; +using log4net; +using LeafWeb.Core.DAL; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; +using Polly; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + public abstract class PiscalQueueBase : IDisposable + { + protected readonly DataService DataService; + protected readonly PiscalService PiscalService; + protected readonly ILog Logger; + private readonly Policy _retryPolicy; + + protected PiscalQueueBase(DataService dataService, PiscalService piscalService) + { + DataService = dataService; + PiscalService = piscalService; + Logger = LogManager.GetLogger(GetType().Name); + + _retryPolicy = + Policy + .Handle() + .Retry(3, + (exception, i) => Logger.Warn($"Retry {i} after exception: {exception.Message}")); + } + + protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { } + + protected string FormatException(Exception ex) + { + return + (ex is PiscalClientException ? $"LeafInput: {((PiscalClientException) ex).LeafInputId}{Environment.NewLine}" : "") + + $"Class: {GetType().Name}{Environment.NewLine}" + + $"Exception message: {ex.Message}{Environment.NewLine}" + + (ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty) + + $"StackTrace: {ex.StackTrace}"; + } + + protected void PiscalExceptionHandler(PiscalClientException ex, LeafInput leafInput) + { + var errorMessage = FormatException(ex); + Logger.Error(errorMessage); + + // send admin an email + BackgroundJobEnqueueRetry( + email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage)); + + // send user email too + BackgroundJobEnqueueRetry( + email => email.SendLeafWebSystemException(leafInput.Identifier, leafInput.Email)); + + if (leafInput != null) + { + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", + ex.Message); + } + } + + public void Dispose() + { + DataService.Dispose(); + } + + protected string BackgroundJobEnqueueRetry(Expression> a) + { + return _retryPolicy.Execute(() => BackgroundJob.Enqueue(a)); + } + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/PiscalQueueManager.cs b/WebCms/Services/PiscalQueue/PiscalQueueManager.cs new file mode 100644 index 0000000..54f094a --- /dev/null +++ b/WebCms/Services/PiscalQueue/PiscalQueueManager.cs @@ -0,0 +1,137 @@ +using System; +using System.Linq; +using System.Threading; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + public class PiscalQueueManager : PiscalQueueBase + { + private static readonly object ProcessQueueLock = new object(); + + public void ProcessQueue() + { + // prevent multiple entry into processing the queue + if (Monitor.TryEnter(ProcessQueueLock)) + { + Logger.DebugFormat("ProcessQueue entered"); + + try + { + UpdateRunning(); + + StartNextPending(); + + // TODO: handle starting and finishing + } + finally + { + Logger.DebugFormat("ProcessQueue exit"); + + Monitor.Exit(ProcessQueueLock); + } + } + else + { + Logger.DebugFormat("ProcessQueue locked, queue already processing"); + } + } + + // TODO: clear any stalled processes + private void ClearStalled() + { + var leafInputs = + DataService.GetLeafInputs( + LeafInputStatusType.Starting, + LeafInputStatusType.Finishing + ) + .Where(li => + li.StatusHistory.OrderBy(sh => sh.DateTime).First().DateTime + > DateTime.Now.Subtract(TimeSpan.FromHours(1))) + .ToList(); + } + + private void StartNextPending() + { + var runningLeafInputs = + DataService.GetLeafInputs( + LeafInputStatusType.Starting, + LeafInputStatusType.Running, + LeafInputStatusType.Finishing + ).ToList(); + + if (runningLeafInputs.Any()) + { + Logger.DebugFormat("Leaf input(s) currently running"); + return; + } + + var pendingInput = + DataService + .GetLeafInputs(LeafInputStatusType.Pending) + .OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime)) + .FirstOrDefault(); + + if (pendingInput == null) + { + Logger.DebugFormat("No pending leaf input"); + return; + } + + var pendingInputId = pendingInput.Id; + Logger.InfoFormat("LeafInput: {0}, Starting", pendingInputId); + try + { + DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting); + BackgroundJobEnqueueRetry(c => c.DoWork(pendingInputId)); + } + catch (Exception ex) + { + var errorMessage = FormatException(ex); + Logger.Error(errorMessage); + DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage); + } + } + + private void UpdateRunning() + { + var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); + foreach (var leafInput in running) + { + try + { + var status = PiscalService.GetStatus(leafInput); + var leafInputId = leafInput.Id; + switch (status) + { + case PiscalStatus.NotStarted: + // if it's not started - this is unusual state + Logger.WarnFormat("LeafInput: {0}, Piscal Not Started", leafInput.Id); + break; + + case PiscalStatus.Running: + Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id); + // continue running + break; + + case PiscalStatus.Complete: + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing); + BackgroundJobEnqueueRetry(s => s.DoWork(leafInputId)); + + break; + } + } + catch (PiscalClientException ex) + { + PiscalExceptionHandler(ex, leafInput); + } + catch (Exception ex) + { + var errorMessage = FormatException(ex); + Logger.Error(errorMessage); + } + } + } + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/PiscalQueueWorker.cs b/WebCms/Services/PiscalQueue/PiscalQueueWorker.cs new file mode 100644 index 0000000..02bc3b0 --- /dev/null +++ b/WebCms/Services/PiscalQueue/PiscalQueueWorker.cs @@ -0,0 +1,35 @@ +using System; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; +using LeafWeb.WebCms.App_Start; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + public abstract class PiscalQueueWorker : PiscalQueueBase + { + public void DoWork(int leafInputId) + { + LeafInput leafInput = null; + try + { + leafInput = DataService.GetLeafInput(leafInputId); + DoWorkInternal(leafInput); + } + catch (PiscalClientException ex) + { + PiscalExceptionHandler(ex, leafInput); + + // signal to process next item + HangfireStartup.TriggerPiscalProcessQueue(); + } + catch (Exception ex) + { + var errorMessage = FormatException(ex); + Logger.Error(errorMessage); + throw; // this will retry via HangFire + } + } + + protected abstract void DoWorkInternal(LeafInput leafInputId); + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/PiscalService.cs b/WebCms/Services/PiscalQueue/PiscalService.cs new file mode 100644 index 0000000..2449036 --- /dev/null +++ b/WebCms/Services/PiscalQueue/PiscalService.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + /// + /// Thin layer over PiscalClient to translate Core entities to Piscal objects + /// + public class PiscalService + { + private readonly IPiscalClient _piscalClient; + private readonly string _notifyCompleteUrl; + + public PiscalService(IPiscalClient piscalClient) + { + _piscalClient = piscalClient; + _notifyCompleteUrl = + ConfigurationManager.AppSettings["LeafWebUrl"] + + ConfigurationManager.AppSettings["PiscalNotifyCompleteUrlPath"]; + } + + public PiscalService() : this(new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString)) + { + } + + public void Run(LeafInput leafInput) + { + var inputFile = new PiscalLeafInput(leafInput); + + if (!string.IsNullOrEmpty(_notifyCompleteUrl)) + inputFile.NotifyCompleteUrl = _notifyCompleteUrl; + + // TODO: remove this, just for testing + if (string.Equals(leafInput.Email, "james.kolpack@gmail.com", StringComparison.InvariantCultureIgnoreCase)) + inputFile.SuppressStorageCopy = true; + _piscalClient.RunLeafInput(inputFile); + } + + public PiscalStatus GetStatus(LeafInput leafInput) + { + var inputFile = new PiscalLeafInput(leafInput); + return _piscalClient.GetLeafInputStatus(inputFile); + } + + public IEnumerable RetrieveOutputFiles(LeafInput leafInput) + { + var input = new PiscalLeafInput(leafInput); + var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(input); + foreach (var file in piscalLeafOutputFiles) + { + var leafOutputFile = file.GetLeafOutputFile(); + leafOutputFile.LeafInput = leafInput; + yield return leafOutputFile; + } + } + + public void Cleanup(LeafInput leafInput) + { + var input = new PiscalLeafInput(leafInput); + _piscalClient.CleanupLeafProcess(input); + } + } +} \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/StartPending.cs b/WebCms/Services/PiscalQueue/StartPending.cs new file mode 100644 index 0000000..9acc1cd --- /dev/null +++ b/WebCms/Services/PiscalQueue/StartPending.cs @@ -0,0 +1,16 @@ +using LeafWeb.Core.Entities; + +namespace LeafWeb.WebCms.Services.PiscalQueue +{ + public class StartPending : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + Logger.DebugFormat("LeafInput: {0}, Run", leafInput.Id); + + PiscalService.Run(leafInput); + + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running); + } + } +} \ No newline at end of file diff --git a/WebCms/Utility/Validation.cs b/WebCms/Utility/Validation.cs index b232d44..d8e657f 100644 --- a/WebCms/Utility/Validation.cs +++ b/WebCms/Utility/Validation.cs @@ -2,7 +2,7 @@ using System; using System.Linq.Expressions; using System.Web.Mvc; -namespace WebCms.Utility +namespace LeafWeb.WebCms.Utility { public static class Validation { diff --git a/WebCms/Views/LeafInput/Create.cshtml b/WebCms/Views/LeafInput/Create.cshtml index e33ac94..b038e96 100644 --- a/WebCms/Views/LeafInput/Create.cshtml +++ b/WebCms/Views/LeafInput/Create.cshtml @@ -1,4 +1,4 @@ -@model WebCms.Models.LeafInputCreate +@model LeafWeb.WebCms.Models.LeafInputCreate
diff --git a/WebCms/Views/Shared/EditorTemplates/Boolean.cshtml b/WebCms/Views/Shared/EditorTemplates/Boolean.cshtml index e6f5fc4..175d9f0 100644 --- a/WebCms/Views/Shared/EditorTemplates/Boolean.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Boolean.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model Boolean? @{ diff --git a/WebCms/Views/Shared/EditorTemplates/DateTime.cshtml b/WebCms/Views/Shared/EditorTemplates/DateTime.cshtml index 2107690..e7a151c 100644 --- a/WebCms/Views/Shared/EditorTemplates/DateTime.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/DateTime.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model DateTime? @{ diff --git a/WebCms/Views/Shared/EditorTemplates/Decimal.cshtml b/WebCms/Views/Shared/EditorTemplates/Decimal.cshtml index 16a2f4e..e9d5600 100644 --- a/WebCms/Views/Shared/EditorTemplates/Decimal.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Decimal.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model decimal?
diff --git a/WebCms/Views/Shared/EditorTemplates/Enum.cshtml b/WebCms/Views/Shared/EditorTemplates/Enum.cshtml index 6632f62..b5beaca 100644 --- a/WebCms/Views/Shared/EditorTemplates/Enum.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Enum.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model object diff --git a/WebCms/Views/Shared/EditorTemplates/Int32.cshtml b/WebCms/Views/Shared/EditorTemplates/Int32.cshtml index 5aff9c0..83e615c 100644 --- a/WebCms/Views/Shared/EditorTemplates/Int32.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Int32.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model int? @{ diff --git a/WebCms/Views/Shared/EditorTemplates/Markdown.cshtml b/WebCms/Views/Shared/EditorTemplates/Markdown.cshtml index 498fb2d..7c31c60 100644 --- a/WebCms/Views/Shared/EditorTemplates/Markdown.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Markdown.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model object @{ diff --git a/WebCms/Views/Shared/EditorTemplates/Multiline.cshtml b/WebCms/Views/Shared/EditorTemplates/Multiline.cshtml index f9828b5..6358a92 100644 --- a/WebCms/Views/Shared/EditorTemplates/Multiline.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Multiline.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model object
diff --git a/WebCms/Views/Shared/EditorTemplates/Password.cshtml b/WebCms/Views/Shared/EditorTemplates/Password.cshtml index ecc8614..e314815 100644 --- a/WebCms/Views/Shared/EditorTemplates/Password.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Password.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model object @{ diff --git a/WebCms/Views/Shared/EditorTemplates/SelectListViewModel.cshtml b/WebCms/Views/Shared/EditorTemplates/SelectListViewModel.cshtml index 3c20e48..3442d04 100644 --- a/WebCms/Views/Shared/EditorTemplates/SelectListViewModel.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/SelectListViewModel.cshtml @@ -1,4 +1,4 @@ -@model WebCms.Models.SelectListViewModel +@model LeafWeb.WebCms.Models.SelectListViewModel @{ Layout = "~/Views/Shared/EditorTemplates/_FieldLayout.cshtml"; } diff --git a/WebCms/Views/Shared/EditorTemplates/Single.cshtml b/WebCms/Views/Shared/EditorTemplates/Single.cshtml index 6aba630..e22fcfb 100644 --- a/WebCms/Views/Shared/EditorTemplates/Single.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Single.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model float? @{ diff --git a/WebCms/Views/Shared/EditorTemplates/Text.cshtml b/WebCms/Views/Shared/EditorTemplates/Text.cshtml index 21d0e89..7852f80 100644 --- a/WebCms/Views/Shared/EditorTemplates/Text.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/Text.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model object
diff --git a/WebCms/Views/Shared/EditorTemplates/TimeSpan.cshtml b/WebCms/Views/Shared/EditorTemplates/TimeSpan.cshtml index dec631a..5e13015 100644 --- a/WebCms/Views/Shared/EditorTemplates/TimeSpan.cshtml +++ b/WebCms/Views/Shared/EditorTemplates/TimeSpan.cshtml @@ -1,4 +1,4 @@ -@using WebCms.Utility +@using LeafWeb.WebCms.Utility @model TimeSpan? @{ diff --git a/WebCms/Web.config b/WebCms/Web.config index 584875f..48587e4 100644 --- a/WebCms/Web.config +++ b/WebCms/Web.config @@ -35,7 +35,7 @@ --> - + @@ -47,7 +47,7 @@ - + diff --git a/WebCms/WebCms.csproj b/WebCms/WebCms.csproj index 04dff6f..1da6528 100644 --- a/WebCms/WebCms.csproj +++ b/WebCms/WebCms.csproj @@ -14,7 +14,7 @@ {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} Library Properties - WebCms + LeafWeb.WebCms WebCms v4.5.2 true @@ -88,6 +88,14 @@ ..\packages\Examine.0.1.70.0\lib\Examine.dll True + + ..\packages\Hangfire.Core.1.6.6\lib\net45\Hangfire.Core.dll + True + + + ..\packages\Hangfire.SqlServer.1.6.6\lib\net45\Hangfire.SqlServer.dll + True + ..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll True @@ -185,6 +193,10 @@ ..\packages\Owin.1.0\lib\net40\Owin.dll True + + ..\packages\Polly.4.3.0\lib\net45\Polly.dll + True + ..\packages\semver.1.1.2\lib\net451\Semver.dll True @@ -214,6 +226,7 @@ ..\packages\System.Reflection.Metadata.1.0.21\lib\portable-net45+win8\System.Reflection.Metadata.dll True + @@ -406,13 +419,23 @@ - + + + + + + + + + + + diff --git a/WebCms/WebCms.csproj.DotSettings b/WebCms/WebCms.csproj.DotSettings new file mode 100644 index 0000000..ab02106 --- /dev/null +++ b/WebCms/WebCms.csproj.DotSettings @@ -0,0 +1,2 @@ + + False \ No newline at end of file diff --git a/WebCms/packages.config b/WebCms/packages.config index 04bdd1c..c2780ab 100644 --- a/WebCms/packages.config +++ b/WebCms/packages.config @@ -5,6 +5,9 @@ + + + @@ -34,6 +37,7 @@ +