diff --git a/Core.Tests/Remote/PiscalSshClientTests.cs b/Core.Tests/Remote/PiscalSshClientTests.cs index 724ed4e..4db4bae 100644 --- a/Core.Tests/Remote/PiscalSshClientTests.cs +++ b/Core.Tests/Remote/PiscalSshClientTests.cs @@ -1,5 +1,6 @@ using System; using System.Configuration; +using System.IO; using System.Linq; using LeafWeb.Core.Remote; using LeafWeb.Core.Utility; @@ -10,17 +11,19 @@ namespace LeafWeb.Core.Tests.Remote [TestFixture] public class PiscalSshClientTests { + private const string ContentDirectory = @"Parsers\LeafInputData\"; + private readonly PiscalLeafInput _testInput = new PiscalLeafInput { LeafInputId = 1, - DirectoryName = "TestDirectory2", + DirectoryName = "TestDirectory3", PhotosyntheticType = "C4_photosynthesis_leafweb", InputFiles = new[] { new PiscalLeafInputFile { - Filename = "blah", Contents = "test".GetBytes() + Filename = "valid.csv", Contents = File.ReadAllBytes(FileUtility.GetContentFile(ContentDirectory, "LeafInput-valid.csv").FullName) } } }; @@ -28,17 +31,22 @@ namespace LeafWeb.Core.Tests.Remote private readonly string _piscalConnectionString = ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString; + private PiscalSshClient GetTestClient() + { + return new PiscalSshClient(_piscalConnectionString); + } + [Test] public void SubmitLeafInputFile() { - var client = new PiscalSshClient(_piscalConnectionString); + var client = GetTestClient(); client.RunLeafInput(_testInput); } [Test] public void GetLeafInputStatus() { - var client = new PiscalSshClient(_piscalConnectionString); + var client = GetTestClient(); var leafInputStatus = client.GetLeafInputStatus(_testInput); Console.WriteLine(leafInputStatus); } @@ -46,10 +54,17 @@ namespace LeafWeb.Core.Tests.Remote [Test] public void RetrieveLeafOutput() { - var client = new PiscalSshClient(_piscalConnectionString); + var client = GetTestClient(); var result = client.RetrieveLeafOutput(_testInput).ToList(); Console.WriteLine(string.Join(", ", result.Select(o => o.Filename))); //Console.WriteLine(result[0].Contents.GetString()); } + + [Test] + public void CleanLeafOutput() + { + var client = GetTestClient(); + client.CleanupLeafProcess(_testInput); + } } } diff --git a/Core/Core.csproj b/Core/Core.csproj index 611281f..44a436f 100644 --- a/Core/Core.csproj +++ b/Core/Core.csproj @@ -50,6 +50,10 @@ ..\packages\fasterflect.2.1.3\lib\net40\Fasterflect.dll True + + ..\packages\NLog.4.2.3\lib\net45\NLog.dll + True + ..\packages\SSH.NET.2013.4.7\lib\net40\Renci.SshNet.dll True @@ -88,6 +92,7 @@ + diff --git a/Core/DAL/DataService.cs b/Core/DAL/DataService.cs index 604ccb4..37c2f47 100644 --- a/Core/DAL/DataService.cs +++ b/Core/DAL/DataService.cs @@ -66,7 +66,7 @@ namespace LeafWeb.Core.DAL _db.SaveChanges(); } - private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null) + private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null, string details = null) { leafInputFile.CurrentStatus = status; var leafInputFileStatus = new LeafInputStatus @@ -74,6 +74,7 @@ namespace LeafWeb.Core.DAL Status = status, DateTime = DateTime.Now, Description = description, + Details = details, LeafInput = leafInputFile }; if (leafInputFile.StatusHistory == null) @@ -81,9 +82,9 @@ namespace LeafWeb.Core.DAL leafInputFile.StatusHistory.Add(leafInputFileStatus); } - public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null) + public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null, string details = null) { - SetLeafInputStatusNoUpdate(leafInput, status, description); + SetLeafInputStatusNoUpdate(leafInput, status, description, details); UpdateLeafInput(leafInput); } diff --git a/Core/Entities/LeafInputStatus.cs b/Core/Entities/LeafInputStatus.cs index a42d579..6469c86 100644 --- a/Core/Entities/LeafInputStatus.cs +++ b/Core/Entities/LeafInputStatus.cs @@ -11,6 +11,7 @@ namespace LeafWeb.Core.Entities public virtual LeafInput LeafInput { get; set; } public LeafInputStatusType Status { get; set; } public string Description { get; set; } + public string Details { get; set; } public DateTime DateTime { get; set; } } } \ No newline at end of file diff --git a/Core/Entities/LeafInputStatusType.cs b/Core/Entities/LeafInputStatusType.cs index 6d648b4..0348aa3 100644 --- a/Core/Entities/LeafInputStatusType.cs +++ b/Core/Entities/LeafInputStatusType.cs @@ -5,6 +5,6 @@ namespace LeafWeb.Core.Entities Pending = 0, Running = 1, Complete = 2, - Error = 3 + Exception = 3 } } \ No newline at end of file diff --git a/Core/Remote/IPiscalClient.cs b/Core/Remote/IPiscalClient.cs index 76c578d..124789a 100644 --- a/Core/Remote/IPiscalClient.cs +++ b/Core/Remote/IPiscalClient.cs @@ -8,6 +8,5 @@ namespace LeafWeb.Core.Remote PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput); IEnumerable RetrieveLeafOutput(PiscalLeafInput leafInput); void CleanupLeafProcess(PiscalLeafInput leafInput); - string GetErrorMessage(PiscalLeafInput leafInput); } } diff --git a/Core/Remote/PiscalLeafInput.cs b/Core/Remote/PiscalLeafInput.cs new file mode 100644 index 0000000..3d0e09b --- /dev/null +++ b/Core/Remote/PiscalLeafInput.cs @@ -0,0 +1,40 @@ +using System.Linq; +using AutoMapper; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Utility; + +namespace LeafWeb.Core.Remote +{ + public class PiscalLeafInput + { + private static readonly IMapper Mapper; + public int LeafInputId { get; set; } + public string PhotosyntheticType { get; set; } + public string DirectoryName { get; set; } + public PiscalLeafInputFile[] InputFiles { get; set; } + + static PiscalLeafInput() + { + var config = + new MapperConfiguration(cfg => + { + cfg.CreateMap() + .ForMember(dest => dest.DirectoryName, + opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src))) + .ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id)) + .ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray())) + .ForMember( + dest => dest.PhotosyntheticType, + opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore())); + }); + Mapper = config.CreateMapper(); + } + + public PiscalLeafInput() { } + + public PiscalLeafInput(LeafInput leafInput) + { + Mapper.Map(leafInput, this); + } + } +} \ No newline at end of file diff --git a/Core/Remote/PiscalLeafInputFile.cs b/Core/Remote/PiscalLeafInputFile.cs index 8925512..4e7a9e0 100644 --- a/Core/Remote/PiscalLeafInputFile.cs +++ b/Core/Remote/PiscalLeafInputFile.cs @@ -1,43 +1,9 @@ -using System.Linq; -using AutoMapper; +using AutoMapper; using LeafWeb.Core.Entities; using LeafWeb.Core.Utility; namespace LeafWeb.Core.Remote { - public class PiscalLeafInput - { - private static readonly IMapper Mapper; - public int LeafInputId { get; set; } - public string PhotosyntheticType { get; set; } - public string DirectoryName { get; set; } - public PiscalLeafInputFile[] InputFiles { get; set; } - - static PiscalLeafInput() - { - var config = - new MapperConfiguration(cfg => - { - cfg.CreateMap() - .ForMember(dest => dest.DirectoryName, - opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src))) - .ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id)) - .ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray())) - .ForMember( - dest => dest.PhotosyntheticType, - opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore())); - }); - Mapper = config.CreateMapper(); - } - - public PiscalLeafInput() { } - - public PiscalLeafInput(LeafInput leafInput) - { - Mapper.Map(leafInput, this); - } - } - public class PiscalLeafInputFile { private static readonly IMapper Mapper; diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs index e655e91..a9c5412 100644 --- a/Core/Remote/PiscalSshClient.cs +++ b/Core/Remote/PiscalSshClient.cs @@ -4,6 +4,7 @@ using System.Data.Common; using System.IO; using System.Linq; using LeafWeb.Core.Utility; +using NLog; using Renci.SshNet; namespace LeafWeb.Core.Remote @@ -14,10 +15,11 @@ namespace LeafWeb.Core.Remote private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh"; private readonly PasswordConnectionInfo _connectionInfo; - private const string StatusSuccess = "success"; + private const string StatusComplete = "complete"; private const string StatusRunning = "running"; private const string StatusNotStarted = "not started"; - private const string StatusError = "error"; + + Logger _logger = LogManager.GetCurrentClassLogger(); public PiscalSshClient(string connectionString) { @@ -45,10 +47,10 @@ namespace LeafWeb.Core.Remote using (var scp = GetScpClient()) foreach (var file in leafInput.InputFiles) { - var inputPath = $"{directory}/{file.Filename}"; + var inputPath = $"{directory}/input/{file.Filename}"; using (var stream = new MemoryStream(file.Contents)) { - Console.WriteLine(inputPath); + _logger.Trace("Copying " + inputPath); scp.Connect(); scp.Upload(stream, inputPath); scp.Disconnect(); @@ -66,8 +68,7 @@ namespace LeafWeb.Core.Remote using (var ssh = GetSshClient()) { ssh.Connect(); - var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType}"; - Console.Write(commandText); + var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType} -s"; var command = ssh.CreateCommand(commandText); command.Execute(); ssh.Disconnect(); @@ -75,7 +76,7 @@ namespace LeafWeb.Core.Remote if (command.ExitStatus != 0) throw new PiscalClientException(command.Result); - Console.Write(command.Result); + _logger.Debug(command.Result); } } @@ -87,12 +88,12 @@ namespace LeafWeb.Core.Remote { case StatusRunning: return PiscalStatus.Running; - case StatusSuccess: - return PiscalStatus.Success; + case StatusComplete: + return PiscalStatus.Complete; case StatusNotStarted: return PiscalStatus.NotStarted; default: - return PiscalStatus.Error; + throw new PiscalClientException("Unknown status: " + statusRaw[0]); } } @@ -101,7 +102,7 @@ namespace LeafWeb.Core.Remote using (var ssh = GetSshClient()) { ssh.Connect(); - var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -s"; + var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName}"; var command = ssh.CreateCommand(commandText); command.Execute(); ssh.Disconnect(); @@ -123,7 +124,7 @@ namespace LeafWeb.Core.Remote { // get output files var status = GetLeafInputStatusRaw(leafInput); - if (status[0] != StatusSuccess) + if (status[0] != StatusComplete) throw new PiscalClientException("output not available, status is " + status[0]); var filePaths = status.Skip(1); @@ -149,23 +150,9 @@ namespace LeafWeb.Core.Remote scp.Disconnect(); } } - - public string GetErrorMessage(PiscalLeafInput leafInput) - { - var status = GetLeafInputStatusRaw(leafInput); - if (status[0] != StatusError) - return string.Empty; - - var errorLines = status.Skip(1).ToArray(); - return errorLines.Join(Environment.NewLine); - } - + public void CleanupLeafProcess(PiscalLeafInput leafInput) { - var status = GetLeafInputStatusRaw(leafInput); - if (status[0] == StatusRunning) - throw new PiscalClientException("Trying to cleanup a running process"); - using (var ssh = GetSshClient()) { ssh.Connect(); diff --git a/Core/Remote/PiscalStatus.cs b/Core/Remote/PiscalStatus.cs index 47f77a0..fda6bd3 100644 --- a/Core/Remote/PiscalStatus.cs +++ b/Core/Remote/PiscalStatus.cs @@ -2,9 +2,8 @@ namespace LeafWeb.Core.Remote { public enum PiscalStatus { - Running, - Success, - NotStarted, - Error + NotStarted, // Process has not been started + Running, // Process is currently executing + Complete, // Process has completed } } \ No newline at end of file diff --git a/Core/packages.config b/Core/packages.config index 139f4ea..37b09c6 100644 --- a/Core/packages.config +++ b/Core/packages.config @@ -4,5 +4,6 @@ + \ No newline at end of file diff --git a/Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs b/Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs index 62c7590..d304122 100644 --- a/Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs +++ b/Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs @@ -61,7 +61,7 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus public void CanConstructFromLeafInputFile_Error() { var leafInput = GetLeafInput(); - leafInput.CurrentStatus = LeafInputStatusType.Error; + leafInput.CurrentStatus = LeafInputStatusType.Exception; leafInput.StatusHistory = new [] { new LeafInputStatus @@ -69,12 +69,12 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus DateTime = DateTime.Today, LeafInput = leafInput, Description = "My Error", - Status = LeafInputStatusType.Error + Status = LeafInputStatusType.Exception } }; var viewModel = new ResultStatusViewModel(leafInput); - Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Error.ToString())); + Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Exception.ToString())); Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description)); } } diff --git a/Web.Tests/Web.Tests.csproj b/Web.Tests/Web.Tests.csproj index 838b639..6a8cebf 100644 --- a/Web.Tests/Web.Tests.csproj +++ b/Web.Tests/Web.Tests.csproj @@ -47,6 +47,7 @@ + diff --git a/Web.Tests/app.config b/Web.Tests/app.config new file mode 100644 index 0000000..3a4dafb --- /dev/null +++ b/Web.Tests/app.config @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Web/Services/EmailNotificationService.cs b/Web/Services/EmailNotificationService.cs index ba0ccb1..9eb3012 100644 --- a/Web/Services/EmailNotificationService.cs +++ b/Web/Services/EmailNotificationService.cs @@ -34,17 +34,17 @@ namespace LeafWeb.Web.Services public void SendLeafWebError(int leafInputId, string errorMessage) { var leafInput = _dataService.GetLeafInput(leafInputId); - var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine + var body = $"Your leaf analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine + "You will need to correct your input and resubmit." + Environment.NewLine + Environment.NewLine + errorMessage; var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb processing error", body); SendMessage(message); } - public void SendLeafWebSuccess(int leafInputId) + public void SendLeafWebComplete(int leafInputId) { var leafInput = _dataService.GetLeafInput(leafInputId); - var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, has completed." + Environment.NewLine; + var body = $"Your leaf analysis job, {leafInput.Identifier}, has completed." + Environment.NewLine; var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb results", body); var fileStreams = diff --git a/Web/Services/PiscalQueueManager.cs b/Web/Services/PiscalQueueManager.cs index fd546ba..40eae92 100644 --- a/Web/Services/PiscalQueueManager.cs +++ b/Web/Services/PiscalQueueManager.cs @@ -76,9 +76,11 @@ namespace LeafWeb.Web.Services catch (PiscalClientException ex) { logger.Error("LeafInputFile: {0}, ProcessQueue Exception: {1}", pending.Id, ex.Message); - _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Error, "Error occurred submitting LeafInput"); + _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Exception, "Error occurred starting LeafInput", ex.Message); + logger.Info("LeafInputFile: {0}, Cleanup", pending.Id); + _piscalService.Cleanup(pending); - // TODO: re-queue + // TODO: re-queue? //_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput"); } _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running); @@ -87,66 +89,54 @@ namespace LeafWeb.Web.Services private void ProcessRunning(ILogger logger) { var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); - foreach (var file in running) + foreach (var leafInput in running) { - var status = _piscalService.GetStatus(file); try { + var status = _piscalService.GetStatus(leafInput); switch (status) { + case PiscalStatus.NotStarted: + logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", leafInput.Id); + // if it's not started, try to requeue the process - unusual state + _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending); + break; + case PiscalStatus.Running: - logger.Trace("LeafInputFile: {0}, Running", file.Id); + logger.Trace("LeafInputFile: {0}, Running", leafInput.Id); // continue running break; - case PiscalStatus.Success: - logger.Info("LeafInputFile: {0}, Success", file.Id); + case PiscalStatus.Complete: + logger.Info("LeafInputFile: {0}, Complete", leafInput.Id); // collect the leaf output - var leafOutputFiles = _piscalService.RetrieveOutputFiles(file).ToList(); + var leafOutputFiles = _piscalService.RetrieveOutputFiles(leafInput).ToList(); foreach (var outputFile in leafOutputFiles) _dataService.AddLeafOutputFile(outputFile); - logger.Info("LeafInputFile: {0}, output files: {1}", file.Id, + logger.Info("LeafInputFile: {0}, output files: {1}", leafInput.Id, string.Join(", ", leafOutputFiles.Select(o => o.Filename))); // update db - _dataService.SetLeafInputStatus(file, LeafInputStatusType.Complete); + _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete); - BackgroundJob.Enqueue(() => _emailService.SendLeafWebSuccess(file.Id)); + BackgroundJob.Enqueue(() => _emailService.SendLeafWebComplete(leafInput.Id)); // remove working data from the server - logger.Info("LeafInputFile: {0}, Cleanup", file.Id); - _piscalService.Cleanup(file); - break; - - case PiscalStatus.NotStarted: - logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", file.Id); - // if it's not started, try to requeue the process - unusual state - _dataService.SetLeafInputStatus(file, LeafInputStatusType.Pending); - break; - - case PiscalStatus.Error: - logger.Info("LeafInputFile: {0}, Error", file.Id); - - var errorMessage = _piscalService.GetErrorMessage(file); - logger.Info("LeafInputFile: {0}, Error Message: {1}", file.Id, errorMessage); - - _dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, errorMessage); - - BackgroundJob.Enqueue(() => _emailService.SendLeafWebError(file.Id, errorMessage)); - - // remove working data from the server - logger.Info("LeafInputFile: {0}, Cleanup", file.Id); - _piscalService.Cleanup(file); + logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id); + _piscalService.Cleanup(leafInput); break; } } catch (PiscalClientException ex) { - logger.Error("LeafInputFile: {0}, ProcessRunning Exception: {1}", file.Id, ex.Message); - _dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, "Error occurred processing LeafInput"); + logger.Error("LeafInputFile: {0}, ProcessRunning Exception: {1}", leafInput.Id, ex.Message); + _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", ex.Message); - // TODO: re-queue + // TODO: Send email + logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id); + _piscalService.Cleanup(leafInput); + // TODO: re-queue ? } } } diff --git a/Web/Services/PiscalService.cs b/Web/Services/PiscalService.cs index 1a83154..71153f3 100644 --- a/Web/Services/PiscalService.cs +++ b/Web/Services/PiscalService.cs @@ -41,13 +41,7 @@ namespace LeafWeb.Web.Services yield return leafOutputFile; } } - - public string GetErrorMessage(LeafInput leafInput) - { - var inputFile = new PiscalLeafInput(leafInput); - return _piscalClient.GetErrorMessage(inputFile); - } - + public void Cleanup(LeafInput leafInput) { var input = new PiscalLeafInput(leafInput); diff --git a/Web/ViewModels/ResultStatus/ResultStatusViewModel.cs b/Web/ViewModels/ResultStatus/ResultStatusViewModel.cs index 06fff69..e5db4d8 100644 --- a/Web/ViewModels/ResultStatus/ResultStatusViewModel.cs +++ b/Web/ViewModels/ResultStatus/ResultStatusViewModel.cs @@ -43,7 +43,7 @@ namespace LeafWeb.Web.ViewModels.ResultStatus opt => opt.ResolveUsing( src => src.StatusHistory? - .Where(sh => sh.Status == LeafInputStatusType.Error) + .Where(sh => sh.Status == LeafInputStatusType.Exception) .Select(sh => sh.Description) .ToArray() ?? new string[] {}));