Organize Piscal Service items

Add nlog, PiscalQueueManager

	modified:   Core.Tests/Remote/PiscalSshClientTests.cs
	modified:   Core/DAL/DataService.cs
	modified:   Core/Remote/IPiscalClient.cs
	modified:   Core/Remote/PiscalSshClient.cs
	new file:   Web/Attributes/ActionLogAttribute.cs
	modified:   Web/Controllers/ControllerBase.cs
	modified:   Web/Controllers/LeafInputController.cs
	modified:   Web/Controllers/LeafOutputController.cs
	new file:   Web/NLog.config
	new file:   Web/NLog.xsd
	new file:   Web/Services/PiscalQueueManager.cs
	modified:   Web/Services/PiscalService.cs
	modified:   Web/Startup.cs
	modified:   Web/Web.csproj
	modified:   Web/packages.config

Ignore logs

Organize piscal queue manager

	modified:   Core/Entities/LeafInputFile.cs
	modified:   Web/Controllers/LeafInputController.cs
	renamed:    Web/Startup.cs -> Web/HangfireStartup.cs
	modified:   Web/Services/PiscalQueueManager.cs
	modified:   Web/Web.csproj

cleanup usings in leafinputcontroller
This commit is contained in:
2016-02-29 10:14:07 -05:00
parent 59e2f9d8bd
commit 10cd2986cf
19 changed files with 2768 additions and 117 deletions
+1
View File
@@ -176,3 +176,4 @@ Notes/leafweb database work/*
.vs
Notes
Web/Files/*
weblogs/*
+2 -2
View File
@@ -26,14 +26,14 @@ namespace LeafWeb.Core.Tests.Remote
public void SubmitLeafInputFile()
{
var client = new PiscalSshClient(_piscalConnectionString);
client.SubmitLeafInputFile(_testInput);
client.RunLeafInputFile(_testInput);
}
[Test]
public void GetLeafInputStatus()
{
var client = new PiscalSshClient(_piscalConnectionString);
var leafInputStatus = client.GetLeafInputStatus(_testInput);
var leafInputStatus = client.GetLeafInputFileStatus(_testInput);
Console.WriteLine(leafInputStatus);
}
+21 -9
View File
@@ -56,7 +56,7 @@ namespace LeafWeb.Core.DAL
_db.LeafInputs.Add(leafInput);
foreach (var leafInputFile in leafInput.Files)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added);
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Queued);
}
_db.SaveChanges();
}
@@ -104,14 +104,6 @@ namespace LeafWeb.Core.DAL
UpdateLeafInputFile(leafInputFile);
}
public LeafInputFile GetNextUnprocessedLeafInputFile()
{
return
(from file in GetLeafInputFiles(LeafInputStatusType.Added)
orderby file.Id ascending
select file).FirstOrDefault();
}
#endregion
#region Photosynthesis Types
@@ -127,5 +119,25 @@ namespace LeafWeb.Core.DAL
}
#endregion
#region LeafOutputFile
public void AddLeafOutputFile(LeafOutputFile leafOutput)
{
_db.LeafOutputFiles.Add(leafOutput);
_db.SaveChanges();
}
public IQueryable<LeafOutputFile> GetLeafOutputFiles()
{
return _db.LeafOutputFiles;
}
public LeafOutputFile GetLeafOutputFile(int id)
{
return _db.LeafOutputFiles.Find(id);
}
#endregion
}
}
+8 -2
View File
@@ -7,6 +7,10 @@ namespace LeafWeb.Core.Entities
public int Id { get; set; }
public virtual LeafInput LeafInput { get; set; }
public virtual ICollection<LeafOutputFile> LeafOutputFiles { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
/// <summary>
/// Parsed values from the LeafInput used in LeafWeb for filtering/searching
@@ -17,7 +21,9 @@ namespace LeafWeb.Core.Entities
public byte[] Contents { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
public override string ToString()
{
return $"{Id}_{Filename}";
}
}
}
+4 -4
View File
@@ -2,9 +2,9 @@ namespace LeafWeb.Core.Entities
{
public enum LeafInputStatusType
{
Added,
ProcessStarted,
ProcessCompleted,
ProcessError
Queued,
Running,
Complete,
Error
}
}
+2 -2
View File
@@ -4,8 +4,8 @@ namespace LeafWeb.Core.Remote
{
public interface IPiscalClient
{
void SubmitLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
void RunLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
}
}
+5 -2
View File
@@ -34,7 +34,7 @@ namespace LeafWeb.Core.Remote
return new ScpClient(_connectionInfo);
}
public void SubmitLeafInputFile(PiscalLeafInputFile file)
public void RunLeafInputFile(PiscalLeafInputFile file)
{
var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}";
@@ -64,7 +64,7 @@ namespace LeafWeb.Core.Remote
}
}
public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
public PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file)
{
var statusRaw = GetLeafInputStatusRaw(file);
@@ -99,6 +99,9 @@ namespace LeafWeb.Core.Remote
}
}
/// <summary>
/// Gets the leaf output from piscal, only run on if result status is success
/// </summary>
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
{
// get output files
+27
View File
@@ -0,0 +1,27 @@
using System.Linq;
using System.Web.Mvc;
using NLog;
namespace LeafWeb.Web.Attributes
{
public class ActionLogAttribute : ActionFilterAttribute
{
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
if (filterContext != null)
{
var controller = filterContext.RouteData.Values["controller"].ToString();
var action = filterContext.RouteData.Values["action"].ToString();
var username = filterContext.HttpContext.User.Identity.Name;
var loggerName = $"{controller}Controller.{action}";
var @params = string.Join(", ", filterContext.ActionParameters.Select(i => $"{i.Key}: {{{i.Value}}}"));
var hostAddress = filterContext.HttpContext.Request.UserHostAddress;
LogManager.GetLogger(loggerName)
.Info("UserHostAddress: {0}, username: {1}, params: {{{2}}}", hostAddress, username, @params);
}
base.OnActionExecuting(filterContext);
}
}
}
+19 -2
View File
@@ -1,9 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using LeafWeb.Core.DAL;
using NLog;
namespace LeafWeb.Web.Controllers
{
@@ -18,6 +17,20 @@ namespace LeafWeb.Web.Controllers
base.Dispose(disposing);
}
protected override void OnException(ExceptionContext filterContext)
{
if (filterContext?.Exception != null)
{
var controller = filterContext.RouteData.Values["controller"].ToString();
var action = filterContext.RouteData.Values["action"].ToString();
var loggerName = $"{controller}Controller.{action}";
LogManager.GetLogger(loggerName).Error(filterContext.Exception);
}
base.OnException(filterContext);
}
protected bool IsHttpParamActionMatch()
{
return ControllerContext.RouteData.Values["action"].ToString()
@@ -42,6 +55,10 @@ namespace LeafWeb.Web.Controllers
case StatusType.Error:
TempData["StatusMessage-Type"] = "alert-error";
break;
case StatusType.Info:
break;
default:
throw new ArgumentOutOfRangeException(nameof(statusType), statusType, null);
}
}
+10 -19
View File
@@ -1,15 +1,13 @@
using System;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using Hangfire;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using LeafWeb.Web.Attributes;
using LeafWeb.Web.ViewModels;
using LeafWeb.Web.ViewModels.LeafInput;
using NLog;
namespace LeafWeb.Web.Controllers
{
@@ -57,6 +55,7 @@ namespace LeafWeb.Web.Controllers
[HttpParamAction]
[HttpPost]
[ActionLog]
public ActionResult Confirm(CreateViewModel viewModel)
{
// directory name is the sessionID
@@ -82,17 +81,15 @@ namespace LeafWeb.Web.Controllers
DeleteBackloadDirectory(Session.SessionID);
foreach (var file in leafInput.Files.Select(f => new PiscalLeafInputFile(f)))
{
BackgroundJob.Enqueue(() => ProcessLeafInput(file));
}
var msg = $"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine
+ $"When complete, an email will be delivered to {viewModel.Name} <{viewModel.Email}> with results.";
SetStatusMessage(HttpUtility.HtmlEncode(msg),StatusType.Success);
LogManager.GetCurrentClassLogger().Info(msg);
HangfireStartup.TriggerPiscalProcessQueue();
SetStatusMessage(
HttpUtility.HtmlEncode(
$"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine
+ $"When complete, an email will be delivered to {viewModel.Name} <{viewModel.Email}> with results."),
StatusType.Success);
return RedirectToAction("Index");
}
@@ -100,12 +97,6 @@ namespace LeafWeb.Web.Controllers
return View("Index", viewModel);
}
public void ProcessLeafInput(PiscalLeafInputFile leafInputFile)
{
var piscalSshClient = new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
piscalSshClient.SubmitLeafInputFile(leafInputFile);
}
private FileInfo[] GetBackloadDirectoryFiles(string directoryName)
{
var path = Path.Combine(Server.MapPath("~/Files/"), directoryName + "\\");
+13
View File
@@ -0,0 +1,13 @@
using System.Web.Mvc;
namespace LeafWeb.Web.Controllers
{
public class LeafOutputController : ControllerBase
{
public ActionResult Index()
{
var viewModel = DataService.GetLeafOutputFiles();
return View(viewModel);
}
}
}
+13 -8
View File
@@ -2,14 +2,17 @@
using Hangfire;
using Microsoft.Owin;
using LeafWeb.Web;
using LeafWeb.Web.Services;
using Owin;
[assembly: OwinStartup(typeof(Startup))]
[assembly: OwinStartup(typeof(HangfireStartup))]
namespace LeafWeb.Web
{
public class Startup
public class HangfireStartup
{
private const string PiscalProcessQueue = "PiscalProcessQueue";
public void Configuration(IAppBuilder app)
{
//GlobalConfiguration.Configuration
@@ -23,8 +26,13 @@ namespace LeafWeb.Web
private void SetupRecurringJobs()
{
//RecurringJob.AddOrUpdate<ServiceReminderEmailService>(
// "serviceReminderJob", s => s.SendAllNotificationEmails(), Cron.Weekly(DayOfWeek.Monday, 6));
// TODO: new SqlServerDistributedLock
RecurringJob.AddOrUpdate<PiscalQueueManager>(PiscalProcessQueue, p => p.ProcessQueue(), Cron.Minutely());
}
public static void TriggerPiscalProcessQueue()
{
RecurringJob.Trigger(PiscalProcessQueue);
}
}
@@ -72,10 +80,7 @@ namespace LeafWeb.Web
{
lock (_lockObject)
{
if (_backgroundJobServer != null)
{
_backgroundJobServer.Dispose();
}
_backgroundJobServer?.Dispose();
HostingEnvironment.UnregisterObject(this);
}
+23
View File
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8" ?>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.nlog-project.org/schemas/NLog.xsd NLog.xsd"
autoReload="true"
throwExceptions="false"
internalLogLevel="Off" internalLogFile="c:\temp\nlog-internal.log" >
<!--
See https://github.com/nlog/nlog/wiki/Configuration-file
for information on customizing logging rules and outputs.
-->
<targets>
<target name="debugLogger" xsi:type="File" fileName="${basedir}/../weblogs/LeafWeb.Debug.${shortdate}.txt" />
<target name="exceptionLogger" xsi:type="File" fileName="${basedir}/../weblogs/LeafWeb.Error.${shortdate}.txt" />
</targets>
<rules>
<logger name="Hangfire.*" minlevel="Debug" maxlevel="Info" final="true"/>
<logger name="*" minlevel="Debug" writeTo="debugLogger"/>
<logger name="*" minlevel="Error" writeTo="exceptionLogger" />
</rules>
</nlog>
+2479
View File
File diff suppressed because it is too large Load Diff
-65
View File
@@ -1,65 +0,0 @@
using System;
using System.Configuration;
using System.Linq;
using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
public class JobService : IDisposable
{
protected readonly DataService DataService = new DataService();
protected IPiscalClient GetPiscalClient()
{
return new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
}
public void ProcessNextLeafInput()
{
var leafInputFile = DataService.GetNextUnprocessedLeafInputFile();
if (leafInputFile == null) // no inputs, quit
return;
var inputFile = new PiscalLeafInputFile(leafInputFile);
var piscalSshClient = GetPiscalClient();
piscalSshClient.SubmitLeafInputFile(inputFile);
DataService.SetLeafInputFileStatus(leafInputFile, LeafInputStatusType.ProcessStarted);
}
public void UpdateLeafInputStatus()
{
var leafInputFiles =
DataService
.GetLeafInputFiles(LeafInputStatusType.ProcessStarted)
.Select(f => new PiscalLeafInputFile(f));
var piscalClient = GetPiscalClient();
foreach (var file in leafInputFiles)
{
var status = piscalClient.GetLeafInputStatus(file);
switch (status)
{
case PiscalStatus.Success:
// retrieve LeafOutput
var outputFile = piscalClient.RetrieveLeafOutput(file);
break;
case PiscalStatus.Error:
// record error
break;
case PiscalStatus.Running:
// do nothing
break;
}
}
}
public void Dispose()
{
DataService.Dispose();
}
}
}
+78
View File
@@ -0,0 +1,78 @@
using System;
using System.Linq;
using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using NLog;
namespace LeafWeb.Web.Services
{
public class PiscalQueueManager : IDisposable
{
private readonly DataService _dataService;
private readonly PiscalService _piscalService;
public PiscalQueueManager(DataService dataService, PiscalService piscalService)
{
_dataService = dataService;
_piscalService = piscalService;
}
public PiscalQueueManager() : this(new DataService(), new PiscalService()) {}
public void ProcessQueue()
{
var logger = LogManager.GetCurrentClassLogger();
var runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList();
foreach (var file in runningLeafInputFiles)
{
var status = _piscalService.GetLeafInputFileStatus(file);
switch (status)
{
case PiscalStatus.Running:
logger.Debug("LeafInputFile: {0}, {1}", file.Id, status);
// continue running
break;
case PiscalStatus.Success:
logger.Info("LeafInputFile: {0}, {1}", file.Id, status);
// collect the leaf output
var leafOutputFiles = _piscalService.RetrieveLeafOutputFile(file).ToList();
foreach (var outputFile in leafOutputFiles)
{
_dataService.AddLeafOutputFile(outputFile);
}
logger.Info("LeafInputFile: {0}, output files: {1}", file.Id, string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Complete);
break;
case PiscalStatus.Error:
logger.Error("LeafInputFile: {0}", file.Id);
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Error);
break;
}
}
runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList();
if (!runningLeafInputFiles.Any())
{
var queuedFile =
_dataService
.GetLeafInputFiles(LeafInputStatusType.Queued)
.OrderBy(l => l.Id)
.FirstOrDefault();
if (queuedFile != null)
{
logger.Info("LeafInputFile: {0}, Start", queuedFile.Id);
_piscalService.RunLeafInputFile(queuedFile);
_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Running);
}
}
}
public void Dispose()
{
_dataService.Dispose();
}
}
}
+45
View File
@@ -0,0 +1,45 @@
using System.Collections.Generic;
using System.Configuration;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
public class PiscalService
{
private readonly IPiscalClient _piscalClient;
public PiscalService(IPiscalClient piscalClient)
{
_piscalClient = piscalClient;
}
public PiscalService() : this(new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString))
{
}
public void RunLeafInputFile(LeafInputFile leafInputFile)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
_piscalClient.RunLeafInputFile(inputFile);
}
public PiscalStatus GetLeafInputFileStatus(LeafInputFile leafInputFile)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
return _piscalClient.GetLeafInputFileStatus(inputFile);
}
public IEnumerable<LeafOutputFile> RetrieveLeafOutputFile(LeafInputFile leafInputFile)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(inputFile);
foreach (var file in piscalLeafOutputFiles)
{
var leafOutputFile = file.GetLeafOutputFile();
leafOutputFile.LeafInputFile = leafInputFile;
yield return leafOutputFile;
}
}
}
}
+15 -2
View File
@@ -102,6 +102,10 @@
<HintPath>..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
<HintPath>..\packages\NLog.4.2.3\lib\net45\NLog.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Owin, Version=1.0.0.0, Culture=neutral, PublicKeyToken=f0ebd12fd5e55cc5, processorArchitecture=MSIL">
<HintPath>..\packages\Owin.1.0\lib\net40\Owin.dll</HintPath>
<Private>True</Private>
@@ -513,6 +517,12 @@
<Content Include="Content\bootstrap\bootstrap.less" />
<Content Include="Content\bootstrap\badges.less" />
<Content Include="Content\bootstrap\alerts.less" />
<Content Include="NLog.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<None Include="NLog.xsd">
<SubType>Designer</SubType>
</None>
<None Include="Properties\PublishProfiles\LeafWeb - Web Deploy.pubxml" />
<None Include="Scripts\jquery-1.9.1.intellisense.js" />
<Content Include="Scripts\angular-mocks.js" />
@@ -905,6 +915,7 @@
<Compile Include="App_Start\BootstrapEditorTemplatesConfig.cs" />
<Compile Include="App_Start\BundleConfig.cs" />
<Compile Include="App_Start\RouteConfig.cs" />
<Compile Include="Attributes\ActionLogAttribute.cs" />
<Compile Include="Backload\Bundles\BackloadBundles.cs" />
<Compile Include="Backload\Controller\BackloadController.cs" />
<Compile Include="Backload\Controller\BackloadController.obsolete.cs" />
@@ -915,14 +926,16 @@
<Compile Include="Controllers\FluxnetSiteController.cs" />
<Compile Include="Controllers\LeafCharterController.cs" />
<Compile Include="Controllers\LeafInputController.cs" />
<Compile Include="Controllers\LeafOutputController.cs" />
<Compile Include="Controllers\PagesController.cs" />
<Compile Include="Global.asax.cs">
<DependentUpon>Global.asax</DependentUpon>
</Compile>
<Compile Include="Attributes\HttpParamActionAttribute.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Services\JobService.cs" />
<Compile Include="Startup.cs" />
<Compile Include="Services\PiscalQueueManager.cs" />
<Compile Include="Services\PiscalService.cs" />
<Compile Include="HangfireStartup.cs" />
<Compile Include="Utility\MarkdownHelper.cs" />
<Compile Include="Utility\Validation.cs" />
<Compile Include="ViewModels\LeafInput\ConfirmViewModel.cs" />
+3
View File
@@ -29,6 +29,9 @@
<package id="Microsoft.Owin.Host.SystemWeb" version="3.0.0" targetFramework="net45" />
<package id="Microsoft.Web.Infrastructure" version="1.0.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="8.0.2" targetFramework="net45" />
<package id="NLog" version="4.2.3" targetFramework="net45" />
<package id="NLog.Config" version="4.2.3" targetFramework="net45" />
<package id="NLog.Schema" version="4.0.0" targetFramework="net45" />
<package id="Owin" version="1.0" targetFramework="net45" />
<package id="Twitter.Bootstrap.Less" version="3.3.6" targetFramework="net45" />
<package id="WebGrease" version="1.6.0" targetFramework="net45" />