Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Piscal processing work

Browse Source
This commit is contained in:
poprhythm
2016-04-06 13:00:26 -04:00
parent 214199fed5
commit 151f17943f
18 changed files with 141 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core.Tests/Remote/PiscalSshClientTests.cs
+20 -5
View File
@@ -1,5 +1,6 @@
using System;
using System.Configuration;
using System.IO;
using System.Linq;
using LeafWeb.Core.Remote;
using LeafWeb.Core.Utility;
@@ -10,17 +11,19 @@ namespace LeafWeb.Core.Tests.Remote
[TestFixture]
public class PiscalSshClientTests
{
private const string ContentDirectory = @"Parsers\LeafInputData\";
private readonly PiscalLeafInput _testInput =
new PiscalLeafInput
{
LeafInputId = 1,
DirectoryName = "TestDirectory2",
DirectoryName = "TestDirectory3",
PhotosyntheticType = "C4_photosynthesis_leafweb",
InputFiles = new[]
{
new PiscalLeafInputFile
{
Filename = "blah", Contents = "test".GetBytes()
Filename = "valid.csv", Contents = File.ReadAllBytes(FileUtility.GetContentFile(ContentDirectory, "LeafInput-valid.csv").FullName)
}
}
};
@@ -28,17 +31,22 @@ namespace LeafWeb.Core.Tests.Remote
private readonly string _piscalConnectionString =
ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString;
private PiscalSshClient GetTestClient()
{
return new PiscalSshClient(_piscalConnectionString);
}
[Test]
public void SubmitLeafInputFile()
{
var client = new PiscalSshClient(_piscalConnectionString);
var client = GetTestClient();
client.RunLeafInput(_testInput);
}
[Test]
public void GetLeafInputStatus()
{
var client = new PiscalSshClient(_piscalConnectionString);
var client = GetTestClient();
var leafInputStatus = client.GetLeafInputStatus(_testInput);
Console.WriteLine(leafInputStatus);
}
@@ -46,10 +54,17 @@ namespace LeafWeb.Core.Tests.Remote
[Test]
public void RetrieveLeafOutput()
{
var client = new PiscalSshClient(_piscalConnectionString);
var client = GetTestClient();
var result = client.RetrieveLeafOutput(_testInput).ToList();
Console.WriteLine(string.Join(", ", result.Select(o => o.Filename)));
//Console.WriteLine(result[0].Contents.GetString());
}
[Test]
public void CleanLeafOutput()
{
var client = GetTestClient();
client.CleanupLeafProcess(_testInput);
}
}
}
Core/Core.csproj
+5
View File
@@ -50,6 +50,10 @@
<HintPath>..\packages\fasterflect.2.1.3\lib\net40\Fasterflect.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
<HintPath>..\packages\NLog.4.2.3\lib\net45\NLog.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Renci.SshNet, Version=2013.4.7.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\SSH.NET.2013.4.7\lib\net40\Renci.SshNet.dll</HintPath>
<Private>True</Private>
@@ -88,6 +92,7 @@
<Compile Include="Entities\LeafInputStatus.cs" />
<Compile Include="Parsers\FluxnetSiteCsvParser.cs" />
<Compile Include="Remote\PiscalClientException.cs" />
<Compile Include="Remote\PiscalLeafInput.cs" />
<Compile Include="Remote\PiscalLeafInputFile.cs" />
<Compile Include="Remote\PiscalLeafOutputFile.cs" />
<Compile Include="Remote\PiscalSshClient.cs" />
Core/DAL/DataService.cs
+4 -3
View File
@@ -66,7 +66,7 @@ namespace LeafWeb.Core.DAL
_db.SaveChanges();
}
private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null)
private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null, string details = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputStatus
@@ -74,6 +74,7 @@ namespace LeafWeb.Core.DAL
Status = status,
DateTime = DateTime.Now,
Description = description,
Details = details,
LeafInput = leafInputFile
};
if (leafInputFile.StatusHistory == null)
@@ -81,9 +82,9 @@ namespace LeafWeb.Core.DAL
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null)
public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null, string details = null)
{
SetLeafInputStatusNoUpdate(leafInput, status, description);
SetLeafInputStatusNoUpdate(leafInput, status, description, details);
UpdateLeafInput(leafInput);
}
Core/Entities/LeafInputStatus.cs
+1
View File
@@ -11,6 +11,7 @@ namespace LeafWeb.Core.Entities
public virtual LeafInput LeafInput { get; set; }
public LeafInputStatusType Status { get; set; }
public string Description { get; set; }
public string Details { get; set; }
public DateTime DateTime { get; set; }
}
}
Core/Entities/LeafInputStatusType.cs
+1 -1
View File
@@ -5,6 +5,6 @@ namespace LeafWeb.Core.Entities
Pending = 0,
Running = 1,
Complete = 2,
Error = 3
Exception = 3
}
}
Core/Remote/IPiscalClient.cs
-1
View File
@@ -8,6 +8,5 @@ namespace LeafWeb.Core.Remote
PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInput leafInput);
void CleanupLeafProcess(PiscalLeafInput leafInput);
string GetErrorMessage(PiscalLeafInput leafInput);
}
}
Core/Remote/PiscalLeafInput.cs
+40
View File
@@ -0,0 +1,40 @@
using System.Linq;
using AutoMapper;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Utility;
namespace LeafWeb.Core.Remote
{
public class PiscalLeafInput
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; }
static PiscalLeafInput()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore()));
});
Mapper = config.CreateMapper();
}
public PiscalLeafInput() { }
public PiscalLeafInput(LeafInput leafInput)
{
Mapper.Map(leafInput, this);
}
}
}
Core/Remote/PiscalLeafInputFile.cs
+1 -35
View File
@@ -1,43 +1,9 @@
using System.Linq;
using AutoMapper;
using AutoMapper;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Utility;
namespace LeafWeb.Core.Remote
{
public class PiscalLeafInput
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; }
static PiscalLeafInput()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore()));
});
Mapper = config.CreateMapper();
}
public PiscalLeafInput() { }
public PiscalLeafInput(LeafInput leafInput)
{
Mapper.Map(leafInput, this);
}
}
public class PiscalLeafInputFile
{
private static readonly IMapper Mapper;
Core/Remote/PiscalSshClient.cs
+13 -26
View File
@@ -4,6 +4,7 @@ using System.Data.Common;
using System.IO;
using System.Linq;
using LeafWeb.Core.Utility;
using NLog;
using Renci.SshNet;
namespace LeafWeb.Core.Remote
@@ -14,10 +15,11 @@ namespace LeafWeb.Core.Remote
private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh";
private readonly PasswordConnectionInfo _connectionInfo;
private const string StatusSuccess = "success";
private const string StatusComplete = "complete";
private const string StatusRunning = "running";
private const string StatusNotStarted = "not started";
private const string StatusError = "error";
Logger _logger = LogManager.GetCurrentClassLogger();
public PiscalSshClient(string connectionString)
{
@@ -45,10 +47,10 @@ namespace LeafWeb.Core.Remote
using (var scp = GetScpClient())
foreach (var file in leafInput.InputFiles)
{
var inputPath = $"{directory}/{file.Filename}";
var inputPath = $"{directory}/input/{file.Filename}";
using (var stream = new MemoryStream(file.Contents))
{
Console.WriteLine(inputPath);
_logger.Trace("Copying " + inputPath);
scp.Connect();
scp.Upload(stream, inputPath);
scp.Disconnect();
@@ -66,8 +68,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType}";
Console.Write(commandText);
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType} -s";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -75,7 +76,7 @@ namespace LeafWeb.Core.Remote
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Result);
Console.Write(command.Result);
_logger.Debug(command.Result);
}
}
@@ -87,12 +88,12 @@ namespace LeafWeb.Core.Remote
{
case StatusRunning:
return PiscalStatus.Running;
case StatusSuccess:
return PiscalStatus.Success;
case StatusComplete:
return PiscalStatus.Complete;
case StatusNotStarted:
return PiscalStatus.NotStarted;
default:
return PiscalStatus.Error;
throw new PiscalClientException("Unknown status: " + statusRaw[0]);
}
}
@@ -101,7 +102,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -s";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName}";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -123,7 +124,7 @@ namespace LeafWeb.Core.Remote
{
// get output files
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusSuccess)
if (status[0] != StatusComplete)
throw new PiscalClientException("output not available, status is " + status[0]);
var filePaths = status.Skip(1);
@@ -150,22 +151,8 @@ namespace LeafWeb.Core.Remote
}
}
public string GetErrorMessage(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusError)
return string.Empty;
var errorLines = status.Skip(1).ToArray();
return errorLines.Join(Environment.NewLine);
}
public void CleanupLeafProcess(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] == StatusRunning)
throw new PiscalClientException("Trying to cleanup a running process");
using (var ssh = GetSshClient())
{
ssh.Connect();
Core/Remote/PiscalStatus.cs
+3 -4
View File
@@ -2,9 +2,8 @@ namespace LeafWeb.Core.Remote
{
public enum PiscalStatus
{
Running,
Success,
NotStarted,
Error
NotStarted, // Process has not been started
Running, // Process is currently executing
Complete, // Process has completed
}
}
Core/packages.config
+1
View File
@@ -4,5 +4,6 @@
<package id="CsvHelper" version="2.13.2.0" targetFramework="net45" />
<package id="EntityFramework" version="6.1.3" targetFramework="net45" />
<package id="fasterflect" version="2.1.3" targetFramework="net45" />
<package id="NLog" version="4.2.3" targetFramework="net45" />
<package id="SSH.NET" version="2013.4.7" targetFramework="net45" />
</packages>
Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs
+3 -3
View File
@@ -61,7 +61,7 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus
public void CanConstructFromLeafInputFile_Error()
{
var leafInput = GetLeafInput();
leafInput.CurrentStatus = LeafInputStatusType.Error;
leafInput.CurrentStatus = LeafInputStatusType.Exception;
leafInput.StatusHistory = new []
{
new LeafInputStatus
@@ -69,12 +69,12 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus
DateTime = DateTime.Today,
LeafInput = leafInput,
Description = "My Error",
Status = LeafInputStatusType.Error
Status = LeafInputStatusType.Exception
}
};
var viewModel = new ResultStatusViewModel(leafInput);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Error.ToString()));
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Exception.ToString()));
Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description));
}
}
Web.Tests/Web.Tests.csproj
+1
View File
@@ -47,6 +47,7 @@
<Compile Include="ViewModels\ResultStatus\ResultStatusViewModelTests.cs" />
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
Web.Tests/app.config
+15
View File
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-8.0.0.0" newVersion="8.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="WebGrease" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.6.5135.21930" newVersion="1.6.5135.21930" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
Web/Services/EmailNotificationService.cs
+3 -3
View File
@@ -34,17 +34,17 @@ namespace LeafWeb.Web.Services
public void SendLeafWebError(int leafInputId, string errorMessage)
{
var leafInput = _dataService.GetLeafInput(leafInputId);
var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine
var body = $"Your leaf analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine
+ "You will need to correct your input and resubmit." + Environment.NewLine + Environment.NewLine
+ errorMessage;
var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb processing error", body);
SendMessage(message);
}
public void SendLeafWebSuccess(int leafInputId)
public void SendLeafWebComplete(int leafInputId)
{
var leafInput = _dataService.GetLeafInput(leafInputId);
var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, has completed." + Environment.NewLine;
var body = $"Your leaf analysis job, {leafInput.Identifier}, has completed." + Environment.NewLine;
var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb results", body);
var fileStreams =
Web/Services/PiscalQueueManager.cs
+27 -37
View File
@@ -76,9 +76,11 @@ namespace LeafWeb.Web.Services
catch (PiscalClientException ex)
{
logger.Error("LeafInputFile: {0}, ProcessQueue Exception: {1}", pending.Id, ex.Message);
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Error, "Error occurred submitting LeafInput");
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Exception, "Error occurred starting LeafInput", ex.Message);
logger.Info("LeafInputFile: {0}, Cleanup", pending.Id);
_piscalService.Cleanup(pending);
// TODO: re-queue
// TODO: re-queue?
//_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput");
}
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running);
@@ -87,66 +89,54 @@ namespace LeafWeb.Web.Services
private void ProcessRunning(ILogger logger)
{
var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
foreach (var file in running)
foreach (var leafInput in running)
{
var status = _piscalService.GetStatus(file);
try
{
var status = _piscalService.GetStatus(leafInput);
switch (status)
{
case PiscalStatus.NotStarted:
logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", leafInput.Id);
// if it's not started, try to requeue the process - unusual state
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending);
break;
case PiscalStatus.Running:
logger.Trace("LeafInputFile: {0}, Running", file.Id);
logger.Trace("LeafInputFile: {0}, Running", leafInput.Id);
// continue running
break;
case PiscalStatus.Success:
logger.Info("LeafInputFile: {0}, Success", file.Id);
case PiscalStatus.Complete:
logger.Info("LeafInputFile: {0}, Complete", leafInput.Id);
// collect the leaf output
var leafOutputFiles = _piscalService.RetrieveOutputFiles(file).ToList();
var leafOutputFiles = _piscalService.RetrieveOutputFiles(leafInput).ToList();
foreach (var outputFile in leafOutputFiles)
_dataService.AddLeafOutputFile(outputFile);
logger.Info("LeafInputFile: {0}, output files: {1}", file.Id,
logger.Info("LeafInputFile: {0}, output files: {1}", leafInput.Id,
string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
// update db
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Complete);
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
BackgroundJob.Enqueue(() => _emailService.SendLeafWebSuccess(file.Id));
BackgroundJob.Enqueue(() => _emailService.SendLeafWebComplete(leafInput.Id));
// remove working data from the server
logger.Info("LeafInputFile: {0}, Cleanup", file.Id);
_piscalService.Cleanup(file);
break;
case PiscalStatus.NotStarted:
logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", file.Id);
// if it's not started, try to requeue the process - unusual state
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Pending);
break;
case PiscalStatus.Error:
logger.Info("LeafInputFile: {0}, Error", file.Id);
var errorMessage = _piscalService.GetErrorMessage(file);
logger.Info("LeafInputFile: {0}, Error Message: {1}", file.Id, errorMessage);
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, errorMessage);
BackgroundJob.Enqueue(() => _emailService.SendLeafWebError(file.Id, errorMessage));
// remove working data from the server
logger.Info("LeafInputFile: {0}, Cleanup", file.Id);
_piscalService.Cleanup(file);
logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
_piscalService.Cleanup(leafInput);
break;
}
}
catch (PiscalClientException ex)
{
logger.Error("LeafInputFile: {0}, ProcessRunning Exception: {1}", file.Id, ex.Message);
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, "Error occurred processing LeafInput");
logger.Error("LeafInputFile: {0}, ProcessRunning Exception: {1}", leafInput.Id, ex.Message);
_dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", ex.Message);
// TODO: re-queue
// TODO: Send email
logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
_piscalService.Cleanup(leafInput);
// TODO: re-queue ?
}
}
}
Web/Services/PiscalService.cs
-6
View File
@@ -42,12 +42,6 @@ namespace LeafWeb.Web.Services
}
}
public string GetErrorMessage(LeafInput leafInput)
{
var inputFile = new PiscalLeafInput(leafInput);
return _piscalClient.GetErrorMessage(inputFile);
}
public void Cleanup(LeafInput leafInput)
{
var input = new PiscalLeafInput(leafInput);
Web/ViewModels/ResultStatus/ResultStatusViewModel.cs
+1 -1
View File
@@ -43,7 +43,7 @@ namespace LeafWeb.Web.ViewModels.ResultStatus
opt => opt.ResolveUsing(
src =>
src.StatusHistory?
.Where(sh => sh.Status == LeafInputStatusType.Error)
.Where(sh => sh.Status == LeafInputStatusType.Exception)
.Select(sh => sh.Description)
.ToArray()
?? new string[] {}));