diff --git a/.gitignore b/.gitignore index 2ce2a7b..ea11c1c 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,4 @@ Notes/leafweb database work/* .vs Notes Web/Files/* +weblogs/* diff --git a/Core.Tests/Remote/PiscalSshClientTests.cs b/Core.Tests/Remote/PiscalSshClientTests.cs index bf6a7be..5d7774d 100644 --- a/Core.Tests/Remote/PiscalSshClientTests.cs +++ b/Core.Tests/Remote/PiscalSshClientTests.cs @@ -26,14 +26,14 @@ namespace LeafWeb.Core.Tests.Remote public void SubmitLeafInputFile() { var client = new PiscalSshClient(_piscalConnectionString); - client.SubmitLeafInputFile(_testInput); + client.RunLeafInputFile(_testInput); } [Test] public void GetLeafInputStatus() { var client = new PiscalSshClient(_piscalConnectionString); - var leafInputStatus = client.GetLeafInputStatus(_testInput); + var leafInputStatus = client.GetLeafInputFileStatus(_testInput); Console.WriteLine(leafInputStatus); } diff --git a/Core/DAL/DataService.cs b/Core/DAL/DataService.cs index 2badd81..55decd1 100644 --- a/Core/DAL/DataService.cs +++ b/Core/DAL/DataService.cs @@ -56,7 +56,7 @@ namespace LeafWeb.Core.DAL _db.LeafInputs.Add(leafInput); foreach (var leafInputFile in leafInput.Files) { - SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added); + SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Queued); } _db.SaveChanges(); } @@ -104,14 +104,6 @@ namespace LeafWeb.Core.DAL UpdateLeafInputFile(leafInputFile); } - public LeafInputFile GetNextUnprocessedLeafInputFile() - { - return - (from file in GetLeafInputFiles(LeafInputStatusType.Added) - orderby file.Id ascending - select file).FirstOrDefault(); - } - #endregion #region Photosynthesis Types @@ -127,5 +119,25 @@ namespace LeafWeb.Core.DAL } #endregion + + #region LeafOutputFile + + public void AddLeafOutputFile(LeafOutputFile leafOutput) + { + _db.LeafOutputFiles.Add(leafOutput); + _db.SaveChanges(); + } + + public IQueryable GetLeafOutputFiles() + { + return _db.LeafOutputFiles; + } + + public LeafOutputFile GetLeafOutputFile(int id) + { + return _db.LeafOutputFiles.Find(id); + } + + #endregion } } diff --git a/Core/Entities/LeafInputFile.cs b/Core/Entities/LeafInputFile.cs index 7ea7ca0..dedbce2 100644 --- a/Core/Entities/LeafInputFile.cs +++ b/Core/Entities/LeafInputFile.cs @@ -7,6 +7,10 @@ namespace LeafWeb.Core.Entities public int Id { get; set; } public virtual LeafInput LeafInput { get; set; } + public virtual ICollection LeafOutputFiles { get; set; } + + public LeafInputStatusType CurrentStatus { get; set; } + public virtual ICollection StatusHistory { get; set; } /// /// Parsed values from the LeafInput used in LeafWeb for filtering/searching @@ -17,7 +21,9 @@ namespace LeafWeb.Core.Entities public byte[] Contents { get; set; } - public LeafInputStatusType CurrentStatus { get; set; } - public virtual ICollection StatusHistory { get; set; } + public override string ToString() + { + return $"{Id}_{Filename}"; + } } } \ No newline at end of file diff --git a/Core/Entities/LeafInputStatusType.cs b/Core/Entities/LeafInputStatusType.cs index ff79410..3a1fde5 100644 --- a/Core/Entities/LeafInputStatusType.cs +++ b/Core/Entities/LeafInputStatusType.cs @@ -2,9 +2,9 @@ namespace LeafWeb.Core.Entities { public enum LeafInputStatusType { - Added, - ProcessStarted, - ProcessCompleted, - ProcessError + Queued, + Running, + Complete, + Error } } \ No newline at end of file diff --git a/Core/Remote/IPiscalClient.cs b/Core/Remote/IPiscalClient.cs index 195de8f..75389e3 100644 --- a/Core/Remote/IPiscalClient.cs +++ b/Core/Remote/IPiscalClient.cs @@ -4,8 +4,8 @@ namespace LeafWeb.Core.Remote { public interface IPiscalClient { - void SubmitLeafInputFile(PiscalLeafInputFile file); - PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file); + void RunLeafInputFile(PiscalLeafInputFile file); + PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file); IEnumerable RetrieveLeafOutput(PiscalLeafInputFile file); } } diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs index 1b2ff72..0d132af 100644 --- a/Core/Remote/PiscalSshClient.cs +++ b/Core/Remote/PiscalSshClient.cs @@ -34,7 +34,7 @@ namespace LeafWeb.Core.Remote return new ScpClient(_connectionInfo); } - public void SubmitLeafInputFile(PiscalLeafInputFile file) + public void RunLeafInputFile(PiscalLeafInputFile file) { var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}"; @@ -64,7 +64,7 @@ namespace LeafWeb.Core.Remote } } - public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file) + public PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file) { var statusRaw = GetLeafInputStatusRaw(file); @@ -99,6 +99,9 @@ namespace LeafWeb.Core.Remote } } + /// + /// Gets the leaf output from piscal, only run on if result status is success + /// public IEnumerable RetrieveLeafOutput(PiscalLeafInputFile file) { // get output files diff --git a/Web/Attributes/ActionLogAttribute.cs b/Web/Attributes/ActionLogAttribute.cs new file mode 100644 index 0000000..1a864be --- /dev/null +++ b/Web/Attributes/ActionLogAttribute.cs @@ -0,0 +1,27 @@ +using System.Linq; +using System.Web.Mvc; +using NLog; + +namespace LeafWeb.Web.Attributes +{ + public class ActionLogAttribute : ActionFilterAttribute + { + public override void OnActionExecuting(ActionExecutingContext filterContext) + { + if (filterContext != null) + { + var controller = filterContext.RouteData.Values["controller"].ToString(); + var action = filterContext.RouteData.Values["action"].ToString(); + var username = filterContext.HttpContext.User.Identity.Name; + var loggerName = $"{controller}Controller.{action}"; + var @params = string.Join(", ", filterContext.ActionParameters.Select(i => $"{i.Key}: {{{i.Value}}}")); + + var hostAddress = filterContext.HttpContext.Request.UserHostAddress; + + LogManager.GetLogger(loggerName) + .Info("UserHostAddress: {0}, username: {1}, params: {{{2}}}", hostAddress, username, @params); + } + base.OnActionExecuting(filterContext); + } + } +} \ No newline at end of file diff --git a/Web/Controllers/ControllerBase.cs b/Web/Controllers/ControllerBase.cs index 3d6de5f..6fe9734 100644 --- a/Web/Controllers/ControllerBase.cs +++ b/Web/Controllers/ControllerBase.cs @@ -1,9 +1,8 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Web; using System.Web.Mvc; using LeafWeb.Core.DAL; +using NLog; namespace LeafWeb.Web.Controllers { @@ -18,6 +17,20 @@ namespace LeafWeb.Web.Controllers base.Dispose(disposing); } + protected override void OnException(ExceptionContext filterContext) + { + if (filterContext?.Exception != null) + { + var controller = filterContext.RouteData.Values["controller"].ToString(); + var action = filterContext.RouteData.Values["action"].ToString(); + var loggerName = $"{controller}Controller.{action}"; + + LogManager.GetLogger(loggerName).Error(filterContext.Exception); + } + + base.OnException(filterContext); + } + protected bool IsHttpParamActionMatch() { return ControllerContext.RouteData.Values["action"].ToString() @@ -42,6 +55,10 @@ namespace LeafWeb.Web.Controllers case StatusType.Error: TempData["StatusMessage-Type"] = "alert-error"; break; + case StatusType.Info: + break; + default: + throw new ArgumentOutOfRangeException(nameof(statusType), statusType, null); } } diff --git a/Web/Controllers/LeafInputController.cs b/Web/Controllers/LeafInputController.cs index 9f797cf..7aca1f8 100644 --- a/Web/Controllers/LeafInputController.cs +++ b/Web/Controllers/LeafInputController.cs @@ -1,15 +1,13 @@ using System; -using System.Configuration; using System.IO; using System.Linq; using System.Web; using System.Web.Mvc; -using Hangfire; using LeafWeb.Core.Entities; -using LeafWeb.Core.Remote; using LeafWeb.Web.Attributes; using LeafWeb.Web.ViewModels; using LeafWeb.Web.ViewModels.LeafInput; +using NLog; namespace LeafWeb.Web.Controllers { @@ -57,6 +55,7 @@ namespace LeafWeb.Web.Controllers [HttpParamAction] [HttpPost] + [ActionLog] public ActionResult Confirm(CreateViewModel viewModel) { // directory name is the sessionID @@ -82,17 +81,15 @@ namespace LeafWeb.Web.Controllers DeleteBackloadDirectory(Session.SessionID); - foreach (var file in leafInput.Files.Select(f => new PiscalLeafInputFile(f))) - { - BackgroundJob.Enqueue(() => ProcessLeafInput(file)); - } - + var msg = $"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine + + $"When complete, an email will be delivered to {viewModel.Name} <{viewModel.Email}> with results."; + + SetStatusMessage(HttpUtility.HtmlEncode(msg),StatusType.Success); + + LogManager.GetCurrentClassLogger().Info(msg); + + HangfireStartup.TriggerPiscalProcessQueue(); - SetStatusMessage( - HttpUtility.HtmlEncode( - $"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine - + $"When complete, an email will be delivered to {viewModel.Name} <{viewModel.Email}> with results."), - StatusType.Success); return RedirectToAction("Index"); } @@ -100,12 +97,6 @@ namespace LeafWeb.Web.Controllers return View("Index", viewModel); } - public void ProcessLeafInput(PiscalLeafInputFile leafInputFile) - { - var piscalSshClient = new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString); - piscalSshClient.SubmitLeafInputFile(leafInputFile); - } - private FileInfo[] GetBackloadDirectoryFiles(string directoryName) { var path = Path.Combine(Server.MapPath("~/Files/"), directoryName + "\\"); diff --git a/Web/Controllers/LeafOutputController.cs b/Web/Controllers/LeafOutputController.cs new file mode 100644 index 0000000..1029c9f --- /dev/null +++ b/Web/Controllers/LeafOutputController.cs @@ -0,0 +1,13 @@ +using System.Web.Mvc; + +namespace LeafWeb.Web.Controllers +{ + public class LeafOutputController : ControllerBase + { + public ActionResult Index() + { + var viewModel = DataService.GetLeafOutputFiles(); + return View(viewModel); + } + } +} \ No newline at end of file diff --git a/Web/Startup.cs b/Web/HangfireStartup.cs similarity index 76% rename from Web/Startup.cs rename to Web/HangfireStartup.cs index 55a9d5f..5b4a8c7 100644 --- a/Web/Startup.cs +++ b/Web/HangfireStartup.cs @@ -2,14 +2,17 @@ using Hangfire; using Microsoft.Owin; using LeafWeb.Web; +using LeafWeb.Web.Services; using Owin; -[assembly: OwinStartup(typeof(Startup))] +[assembly: OwinStartup(typeof(HangfireStartup))] namespace LeafWeb.Web { - public class Startup + public class HangfireStartup { + private const string PiscalProcessQueue = "PiscalProcessQueue"; + public void Configuration(IAppBuilder app) { //GlobalConfiguration.Configuration @@ -23,8 +26,13 @@ namespace LeafWeb.Web private void SetupRecurringJobs() { - //RecurringJob.AddOrUpdate( - // "serviceReminderJob", s => s.SendAllNotificationEmails(), Cron.Weekly(DayOfWeek.Monday, 6)); + // TODO: new SqlServerDistributedLock + RecurringJob.AddOrUpdate(PiscalProcessQueue, p => p.ProcessQueue(), Cron.Minutely()); + } + + public static void TriggerPiscalProcessQueue() + { + RecurringJob.Trigger(PiscalProcessQueue); } } @@ -72,10 +80,7 @@ namespace LeafWeb.Web { lock (_lockObject) { - if (_backgroundJobServer != null) - { - _backgroundJobServer.Dispose(); - } + _backgroundJobServer?.Dispose(); HostingEnvironment.UnregisterObject(this); } diff --git a/Web/NLog.config b/Web/NLog.config new file mode 100644 index 0000000..dbc8f5e --- /dev/null +++ b/Web/NLog.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Web/NLog.xsd b/Web/NLog.xsd new file mode 100644 index 0000000..85019de --- /dev/null +++ b/Web/NLog.xsd @@ -0,0 +1,2479 @@ + + + + + + + + + + + + + + + Watch config file for changes and reload automatically. + + + + + Print internal NLog messages to the console. Default value is: false + + + + + Print internal NLog messages to the console error output. Default value is: false + + + + + Write internal NLog messages to the specified file. + + + + + Log level threshold for internal log messages. Default value is: Info. + + + + + Global log level threshold for application log messages. Messages below this level won't be logged.. + + + + + Pass NLog internal exceptions to the application. Default value is: false. + + + + + + + + + + + + + + Make all targets within this section asynchronous (creates additional threads but the calling thread isn't blocked by any target writes). + + + + + + + + + + + + + + + + + Prefix for targets/layout renderers/filters/conditions loaded from this assembly. + + + + + Load NLog extensions from the specified file (*.dll) + + + + + Load NLog extensions from the specified assembly. Assembly name should be fully qualified. + + + + + + + + + + Name of the logger. May include '*' character which acts like a wildcard. Allowed forms are: *, Name, *Name, Name* and *Name* + + + + + Comma separated list of levels that this rule matches. + + + + + Minimum level that this rule matches. + + + + + Maximum level that this rule matches. + + + + + Level that this rule matches. + + + + + Comma separated list of target names. + + + + + Ignore further rules if this one matches. + + + + + Enable or disable logging rule. Disabled rules are ignored. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the file to be included. The name is relative to the name of the current config file. + + + + + Ignore any errors in the include file. + + + + + + + Variable name. + + + + + Variable value. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + Indicates whether to add <!-- --> comments around all written texts. + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Number of log events that should be processed in a batch by the lazy writer thread. + + + + + Action to be taken when the lazy writer thread request queue count exceeds the set limit. + + + + + Limit on the number of requests in the lazy writer thread request queue. + + + + + Time in milliseconds to sleep between batches. + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + + + + + + + + + + + + + Name of the target. + + + + + Number of log events to be buffered. + + + + + Timeout (in milliseconds) after which the contents of buffer will be flushed if there's no write in the specified period of time. Use -1 to disable timed flushes. + + + + + Indicates whether to use sliding timeout. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Encoding to be used. + + + + + Instance of that is used to format log messages. + + + + + Maximum message size in bytes. + + + + + Indicates whether to append newline at the end of log message. + + + + + Action that should be taken if the message is larger than maxMessageSize. + + + + + Size of the connection cache (number of connections which are kept alive). + + + + + Network address. + + + + + Indicates whether to keep connection open whenever possible. + + + + + Maximum queue size. + + + + + Indicates whether to include NLog-specific extensions to log4j schema. + + + + + Indicates whether to include source info (file name and line number) in the information sent over the network. + + + + + NDC item separator. + + + + + Indicates whether to include call site (class and method name) in the information sent over the network. + + + + + AppInfo field. By default it's the friendly name of the current AppDomain. + + + + + Indicates whether to include stack contents. + + + + + Indicates whether to include dictionary contents. + + + + + + + + + + + + + + + + + + + + Layout that should be use to calcuate the value for the parameter. + + + + + Viewer parameter name. + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Text to be rendered. + + + + + Header. + + + + + Footer. + + + + + Indicates whether to use default row highlighting rules. + + + + + The encoding for writing messages to the . + + + + + Indicates whether the error stream (stderr) should be used instead of the output stream (stdout). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Condition that must be met in order to set the specified foreground and background color. + + + + + Background color. + + + + + Foreground color. + + + + + + + + + + + + + + + Indicates whether to ignore case when comparing texts. + + + + + Regular expression to be matched. You must specify either text or regex. + + + + + Text to be matched. You must specify either text or regex. + + + + + Indicates whether to match whole words only. + + + + + Background color. + + + + + Foreground color. + + + + + + + + + + + + + + + + + Name of the target. + + + + + Text to be rendered. + + + + + Header. + + + + + Footer. + + + + + Indicates whether to send the log messages to the standard error instead of the standard output. + + + + + The encoding for writing messages to the . + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Connection string. When provided, it overrides the values specified in DBHost, DBUserName, DBPassword, DBDatabase. + + + + + Name of the connection string (as specified in <connectionStrings> configuration section. + + + + + Database name. If the ConnectionString is not provided this value will be used to construct the "Database=" part of the connection string. + + + + + Database host name. If the ConnectionString is not provided this value will be used to construct the "Server=" part of the connection string. + + + + + Database password. If the ConnectionString is not provided this value will be used to construct the "Password=" part of the connection string. + + + + + Name of the database provider. + + + + + Database user name. If the ConnectionString is not provided this value will be used to construct the "User ID=" part of the connection string. + + + + + Indicates whether to keep the database connection open between the log events. + + + + + Connection string using for installation and uninstallation. If not provided, regular ConnectionString is being used. + + + + + Text of the SQL command to be run on each log level. + + + + + Type of the SQL command to be run on each log level. + + + + + + + + + + + + + + + + + + + + + + + Type of the command. + + + + + Connection string to run the command against. If not provided, connection string from the target is used. + + + + + Indicates whether to ignore failures. + + + + + Command text. + + + + + + + + + + + + + + Layout that should be use to calcuate the value for the parameter. + + + + + Database parameter name. + + + + + Database parameter precision. + + + + + Database parameter scale. + + + + + Database parameter size. + + + + + + + + + + + + + + + Name of the target. + + + + + Text to be rendered. + + + + + Header. + + + + + Footer. + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + Layout that renders event Category. + + + + + Layout that renders event ID. + + + + + Name of the Event Log to write to. This can be System, Application or any user-defined name. + + + + + Name of the machine on which Event Log service is running. + + + + + Value to be used as the event Source. + + + + + Optional entrytype. When not set, or when not convertable to then determined by + + + + + + + + + + + + + + + Name of the target. + + + + + Indicates whether to return to the first target after any successful write. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Text to be rendered. + + + + + Header. + + + + + Footer. + + + + + File encoding. + + + + + Line ending mode. + + + + + Maximum number of archive files that should be kept. + + + + + Way file archives are numbered. + + + + + Name of the file to be used for an archive. + + + + + Indicates whether to automatically archive log files every time the specified time passes. + + + + + Size in bytes above which log files will be automatically archived. + + + + + Indicates whether to compress archive files into the zip archive format. + + + + + Gets ors set a value indicating whether a managed file stream is forced, instead of used the native implementation. + + + + + File attributes (Windows only). + + + + + Indicates whether to replace file contents on each write instead of appending log message at the end. + + + + + Name of the file to write to. + + + + + Value specifying the date format to use when archving files. + + + + + Indicates whether to archive old log file on startup. + + + + + Indicates whether to create directories if they do not exist. + + + + + Indicates whether to delete old log file on startup. + + + + + Indicates whether to enable log file(s) to be deleted. + + + + + Maximum number of seconds that files are kept open. If this number is negative the files are not automatically closed after a period of inactivity. + + + + + Indicates whether concurrent writes to the log file by multiple processes on different network hosts. + + + + + Maximum number of log filenames that should be stored as existing. + + + + + Indicates whether to keep log file open instead of opening and closing it on each logging event. + + + + + Indicates whether concurrent writes to the log file by multiple processes on the same host. + + + + + Number of times the write is appended on the file before NLog discards the log message. + + + + + Delay in milliseconds to wait before attempting to write to the file again. + + + + + Indicates whether to automatically flush the file buffers after each log message. + + + + + Number of files to be kept open. Setting this to a higher value may improve performance in a situation where a single File target is writing to many files (such as splitting by level or by logger). + + + + + Log file buffer size in bytes. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Condition expression. Log events who meet this condition will be forwarded to the wrapped target. + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Windows domain name to change context to. + + + + + Required impersonation level. + + + + + Type of the logon provider. + + + + + Logon Type. + + + + + User account password. + + + + + Indicates whether to revert to the credentials of the process instead of impersonating another user. + + + + + Username to change context to. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Endpoint address. + + + + + Name of the endpoint configuration in WCF configuration file. + + + + + Indicates whether to use a WCF service contract that is one way (fire and forget) or two way (request-reply) + + + + + Client ID. + + + + + Indicates whether to include per-event properties in the payload sent to the server. + + + + + Indicates whether to use binary message encoding. + + + + + + + + + + + + + + Layout that should be use to calculate the value for the parameter. + + + + + Name of the parameter. + + + + + Type of the parameter. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Text to be rendered. + + + + + Header. + + + + + Footer. + + + + + Indicates whether to send message as HTML instead of plain text. + + + + + Encoding to be used for sending e-mail. + + + + + Indicates whether to add new lines between log entries. + + + + + BCC email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). + + + + + Recipients' email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). + + + + + CC email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). + + + + + Mail message body (repeated for each log message send in one mail). + + + + + Mail subject. + + + + + Sender's email address (e.g. joe@domain.com). + + + + + Indicates whether NewLine characters in the body should be replaced with tags. + + + + + Priority used for sending mails. + + + + + Indicates the SMTP client timeout. + + + + + SMTP Server to be used for sending. + + + + + SMTP Authentication mode. + + + + + Username used to connect to SMTP server (used when SmtpAuthentication is set to "basic"). + + + + + Password used to authenticate against SMTP server (used when SmtpAuthentication is set to "basic"). + + + + + Indicates whether SSL (secure sockets layer) should be used when communicating with SMTP server. + + + + + Port number that SMTP Server is listening on. + + + + + Indicates whether the default Settings from System.Net.MailSettings should be used. + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + Encoding to be used when writing text to the queue. + + + + + Indicates whether to use the XML format when serializing message. This will also disable creating queues. + + + + + Indicates whether to check if a queue exists before writing to it. + + + + + Indicates whether to create the queue if it doesn't exists. + + + + + Label to associate with each message. + + + + + Name of the queue to write to. + + + + + Indicates whether to use recoverable messages (with guaranteed delivery). + + + + + + + + + + + + + + + + + Name of the target. + + + + + Class name. + + + + + Method name. The method must be public and static. + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + Encoding to be used. + + + + + Maximum message size in bytes. + + + + + Indicates whether to append newline at the end of log message. + + + + + Action that should be taken if the message is larger than maxMessageSize. + + + + + Network address. + + + + + Size of the connection cache (number of connections which are kept alive). + + + + + Indicates whether to keep connection open whenever possible. + + + + + Maximum queue size. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Encoding to be used. + + + + + Instance of that is used to format log messages. + + + + + Maximum message size in bytes. + + + + + Indicates whether to append newline at the end of log message. + + + + + Action that should be taken if the message is larger than maxMessageSize. + + + + + Size of the connection cache (number of connections which are kept alive). + + + + + Network address. + + + + + Indicates whether to keep connection open whenever possible. + + + + + Maximum queue size. + + + + + Indicates whether to include NLog-specific extensions to log4j schema. + + + + + Indicates whether to include source info (file name and line number) in the information sent over the network. + + + + + NDC item separator. + + + + + Indicates whether to include call site (class and method name) in the information sent over the network. + + + + + AppInfo field. By default it's the friendly name of the current AppDomain. + + + + + Indicates whether to include stack contents. + + + + + Indicates whether to include dictionary contents. + + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + Indicates whether to perform layout calculation. + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Indicates whether performance counter should be automatically created. + + + + + Name of the performance counter category. + + + + + Counter help text. + + + + + Name of the performance counter. + + + + + Performance counter type. + + + + + Performance counter instance name. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Default filter to be applied when no specific rule matches. + + + + + + + + + + + + + Condition to be tested. + + + + + Resulting filter to be applied when the condition matches. + + + + + + + + + + + + Name of the target. + + + + + + + + + + + + + + + Name of the target. + + + + + Number of times to repeat each log message. + + + + + + + + + + + + + + + + Name of the target. + + + + + Number of retries that should be attempted on the wrapped target in case of a failure. + + + + + Time to wait between retries in milliseconds. + + + + + + + + + + + + + + Name of the target. + + + + + + + + + + + + + + Name of the target. + + + + + + + + + + + + + + + Name of the target. + + + + + Layout used to format log messages. + + + + + + + + + + + + + + + + + + + + + Name of the target. + + + + + Should we include the BOM (Byte-order-mark) for UTF? Influences the property. This will only work for UTF-8. + + + + + Encoding. + + + + + Web service method name. Only used with Soap. + + + + + Web service namespace. Only used with Soap. + + + + + Protocol to be used when calling web service. + + + + + Web service URL. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Footer layout. + + + + + Header layout. + + + + + Body layout (can be repeated multiple times). + + + + + Custom column delimiter value (valid when ColumnDelimiter is set to 'Custom'). + + + + + Column delimiter. + + + + + Quote Character. + + + + + Quoting mode. + + + + + Indicates whether CVS should include header. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Layout of the column. + + + + + Name of the column. + + + + + + + + + + + + + + + + + + + + Layout that will be rendered as the attribute's value. + + + + + Name of the attribute. + + + + + + + + + + + + + + Footer layout. + + + + + Header layout. + + + + + Body layout (can be repeated multiple times). + + + + + + + + + + + + + + + + + + + + + Layout text. + + + + + + + + + + + + + + + Action to be taken when filter matches. + + + + + Condition expression. + + + + + + + + + + + + + + + + + + + + + + + + + + Action to be taken when filter matches. + + + + + Indicates whether to ignore case when comparing strings. + + + + + Layout to be used to filter log messages. + + + + + Substring to be matched. + + + + + + + + + + + + + + + + + Action to be taken when filter matches. + + + + + String to compare the layout to. + + + + + Indicates whether to ignore case when comparing strings. + + + + + Layout to be used to filter log messages. + + + + + + + + + + + + + + + + + Action to be taken when filter matches. + + + + + Indicates whether to ignore case when comparing strings. + + + + + Layout to be used to filter log messages. + + + + + Substring to be matched. + + + + + + + + + + + + + + + + + Action to be taken when filter matches. + + + + + String to compare the layout to. + + + + + Indicates whether to ignore case when comparing strings. + + + + + Layout to be used to filter log messages. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Web/Services/JobService.cs b/Web/Services/JobService.cs deleted file mode 100644 index 7f9bb1c..0000000 --- a/Web/Services/JobService.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System; -using System.Configuration; -using System.Linq; -using LeafWeb.Core.DAL; -using LeafWeb.Core.Entities; -using LeafWeb.Core.Remote; - -namespace LeafWeb.Web.Services -{ - public class JobService : IDisposable - { - protected readonly DataService DataService = new DataService(); - - protected IPiscalClient GetPiscalClient() - { - return new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString); - } - - public void ProcessNextLeafInput() - { - var leafInputFile = DataService.GetNextUnprocessedLeafInputFile(); - if (leafInputFile == null) // no inputs, quit - return; - - var inputFile = new PiscalLeafInputFile(leafInputFile); - - var piscalSshClient = GetPiscalClient(); - piscalSshClient.SubmitLeafInputFile(inputFile); - - DataService.SetLeafInputFileStatus(leafInputFile, LeafInputStatusType.ProcessStarted); - } - - public void UpdateLeafInputStatus() - { - var leafInputFiles = - DataService - .GetLeafInputFiles(LeafInputStatusType.ProcessStarted) - .Select(f => new PiscalLeafInputFile(f)); - - var piscalClient = GetPiscalClient(); - foreach (var file in leafInputFiles) - { - var status = piscalClient.GetLeafInputStatus(file); - switch (status) - { - case PiscalStatus.Success: - // retrieve LeafOutput - var outputFile = piscalClient.RetrieveLeafOutput(file); - break; - case PiscalStatus.Error: - // record error - break; - case PiscalStatus.Running: - // do nothing - break; - } - } - } - - public void Dispose() - { - DataService.Dispose(); - } - } -} \ No newline at end of file diff --git a/Web/Services/PiscalQueueManager.cs b/Web/Services/PiscalQueueManager.cs new file mode 100644 index 0000000..78ca263 --- /dev/null +++ b/Web/Services/PiscalQueueManager.cs @@ -0,0 +1,78 @@ +using System; +using System.Linq; +using LeafWeb.Core.DAL; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; +using NLog; + +namespace LeafWeb.Web.Services +{ + public class PiscalQueueManager : IDisposable + { + private readonly DataService _dataService; + private readonly PiscalService _piscalService; + + public PiscalQueueManager(DataService dataService, PiscalService piscalService) + { + _dataService = dataService; + _piscalService = piscalService; + } + + public PiscalQueueManager() : this(new DataService(), new PiscalService()) {} + + public void ProcessQueue() + { + var logger = LogManager.GetCurrentClassLogger(); + + var runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList(); + foreach (var file in runningLeafInputFiles) + { + var status = _piscalService.GetLeafInputFileStatus(file); + switch (status) + { + case PiscalStatus.Running: + logger.Debug("LeafInputFile: {0}, {1}", file.Id, status); + // continue running + break; + case PiscalStatus.Success: + logger.Info("LeafInputFile: {0}, {1}", file.Id, status); + // collect the leaf output + var leafOutputFiles = _piscalService.RetrieveLeafOutputFile(file).ToList(); + foreach (var outputFile in leafOutputFiles) + { + _dataService.AddLeafOutputFile(outputFile); + } + logger.Info("LeafInputFile: {0}, output files: {1}", file.Id, string.Join(", ", leafOutputFiles.Select(o => o.Filename))); + _dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Complete); + break; + case PiscalStatus.Error: + logger.Error("LeafInputFile: {0}", file.Id); + _dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Error); + break; + } + } + + runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList(); + if (!runningLeafInputFiles.Any()) + { + var queuedFile = + _dataService + .GetLeafInputFiles(LeafInputStatusType.Queued) + .OrderBy(l => l.Id) + .FirstOrDefault(); + + if (queuedFile != null) + { + logger.Info("LeafInputFile: {0}, Start", queuedFile.Id); + _piscalService.RunLeafInputFile(queuedFile); + _dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Running); + } + } + } + + public void Dispose() + { + _dataService.Dispose(); + } + } +} \ No newline at end of file diff --git a/Web/Services/PiscalService.cs b/Web/Services/PiscalService.cs new file mode 100644 index 0000000..4bbc77d --- /dev/null +++ b/Web/Services/PiscalService.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Configuration; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; + +namespace LeafWeb.Web.Services +{ + public class PiscalService + { + private readonly IPiscalClient _piscalClient; + + public PiscalService(IPiscalClient piscalClient) + { + _piscalClient = piscalClient; + } + + public PiscalService() : this(new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString)) + { + } + + public void RunLeafInputFile(LeafInputFile leafInputFile) + { + var inputFile = new PiscalLeafInputFile(leafInputFile); + _piscalClient.RunLeafInputFile(inputFile); + } + + public PiscalStatus GetLeafInputFileStatus(LeafInputFile leafInputFile) + { + var inputFile = new PiscalLeafInputFile(leafInputFile); + return _piscalClient.GetLeafInputFileStatus(inputFile); + } + + public IEnumerable RetrieveLeafOutputFile(LeafInputFile leafInputFile) + { + var inputFile = new PiscalLeafInputFile(leafInputFile); + var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(inputFile); + foreach (var file in piscalLeafOutputFiles) + { + var leafOutputFile = file.GetLeafOutputFile(); + leafOutputFile.LeafInputFile = leafInputFile; + yield return leafOutputFile; + } + } + } +} \ No newline at end of file diff --git a/Web/Web.csproj b/Web/Web.csproj index f3b491b..24ecb7b 100644 --- a/Web/Web.csproj +++ b/Web/Web.csproj @@ -102,6 +102,10 @@ ..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll True + + ..\packages\NLog.4.2.3\lib\net45\NLog.dll + True + ..\packages\Owin.1.0\lib\net40\Owin.dll True @@ -513,6 +517,12 @@ + + Always + + + Designer + @@ -905,6 +915,7 @@ + @@ -915,14 +926,16 @@ + Global.asax - - + + + diff --git a/Web/packages.config b/Web/packages.config index 1284e2c..61802f3 100644 --- a/Web/packages.config +++ b/Web/packages.config @@ -29,6 +29,9 @@ + + +