Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Submit all LeafInputFiles together

Browse Source
This commit is contained in:
James Kolpack
2016-03-28 10:20:50 -04:00
parent 4b2757b19a
commit 9e86b92f08
29 changed files with 353 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core.Tests/Parsers/LeafInputCsvParserTests.cs
+1 -1
View File
@@ -135,7 +135,7 @@ namespace LeafWeb.Core.Tests.Parsers
var input = parser.Parse();
}
[Test, Explicit]
[Explicit]
public void Parse_FindIssues()
{
var dir = @"C:\Users\poprhythm\Documents\code\LeafWeb\Notes\leafweb database work\newcurves\RemovableDisk\curves";
Core.Tests/Remote/PiscalSshClientTests.cs
+13 -7
View File
@@ -10,13 +10,19 @@ namespace LeafWeb.Core.Tests.Remote
[TestFixture]
public class PiscalSshClientTests
{
private readonly PiscalLeafInputFile _testInput =
new PiscalLeafInputFile
private readonly PiscalLeafInput _testInput =
new PiscalLeafInput
{
Filename = "blah",
LeafInputId = 1,
Contents = "test".GetBytes(),
DirectoryName = "TestDirectory2"
DirectoryName = "TestDirectory2",
PhotosyntheticType = "C4_photosynthesis_leafweb",
InputFiles = new[]
{
new PiscalLeafInputFile
{
Filename = "blah", Contents = "test".GetBytes()
}
}
};
private readonly string _piscalConnectionString =
@@ -26,14 +32,14 @@ namespace LeafWeb.Core.Tests.Remote
public void SubmitLeafInputFile()
{
var client = new PiscalSshClient(_piscalConnectionString);
client.RunLeafInputFile(_testInput);
client.RunLeafInput(_testInput);
}
[Test]
public void GetLeafInputStatus()
{
var client = new PiscalSshClient(_piscalConnectionString);
var leafInputStatus = client.GetLeafInputFileStatus(_testInput);
var leafInputStatus = client.GetLeafInputStatus(_testInput);
Console.WriteLine(leafInputStatus);
}
Core/Core.csproj
+1 -1
View File
@@ -85,7 +85,7 @@
<Compile Include="Entities\LeafInputFile.cs" />
<Compile Include="Entities\LeafInputDataPhotosynthetic.cs" />
<Compile Include="Entities\LeafInputDataSite.cs" />
<Compile Include="Entities\LeafInputFileStatus.cs" />
<Compile Include="Entities\LeafInputStatus.cs" />
<Compile Include="Parsers\FluxnetSiteCsvParser.cs" />
<Compile Include="Remote\PiscalClientException.cs" />
<Compile Include="Remote\PiscalLeafInputFile.cs" />
Core/DAL/DataService.cs
+37 -32
View File
@@ -50,16 +50,49 @@ namespace LeafWeb.Core.DAL
return _db.LeafInputs.FirstOrDefault(li => li.Id == id);
}
public IQueryable<LeafInput> GetLeafInputs(LeafInputStatusType status)
{
return
from file in _db.LeafInputs
where file.CurrentStatus == status
select file;
}
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
_db.LeafInputs.Add(leafInput);
foreach (var leafInputFile in leafInput.Files)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Queued);
}
SetLeafInputStatusNoUpdate(leafInput, LeafInputStatusType.Pending);
_db.SaveChanges();
}
private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInput = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null)
{
SetLeafInputStatusNoUpdate(leafInput, status, description);
UpdateLeafInput(leafInput);
}
public void UpdateLeafInput(LeafInput leafInput)
{
_db.Entry(leafInput).State = EntityState.Modified;
_db.SaveChanges();
}
#endregion
#region LeafInputFile
@@ -74,40 +107,12 @@ namespace LeafWeb.Core.DAL
return _db.LeafInputFiles.Find(id);
}
public IQueryable<LeafInputFile> GetLeafInputFiles(LeafInputStatusType status)
{
return
from file in _db.LeafInputFiles
where file.CurrentStatus == status
select file;
}
public void UpdateLeafInputFile(LeafInputFile leafInputFile)
{
_db.Entry(leafInputFile).State = EntityState.Modified;
_db.SaveChanges();
}
private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputFileStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInputFile = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputFileStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
UpdateLeafInputFile(leafInputFile);
}
#endregion
Core/DAL/LeafWebContext.cs
+1 -1
View File
@@ -8,7 +8,7 @@ namespace LeafWeb.Core.DAL
{
public DbSet<LeafInput> LeafInputs { get; set; }
public DbSet<LeafInputFile> LeafInputFiles { get; set; }
public DbSet<LeafInputFileStatus> LeafInputStatus { get; set; }
public DbSet<LeafInputStatus> LeafInputStatus { get; set; }
public DbSet<FluxnetSite> FluxnetSites { get; set; }
public DbSet<PhotosynthesisType> PhotosynthesisTypes { get; set; }
public DbSet<LeafOutputFile> LeafOutputFiles { get; set; }
Core/Entities/LeafInput.cs
+10 -1
View File
@@ -8,7 +8,11 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual ICollection<LeafInputFile> Files { get; set; }
public virtual ICollection<LeafInputFile> InputFiles { get; set; }
public virtual ICollection<LeafOutputFile> OutputFiles { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputStatus> StatusHistory { get; set; }
[Required(ErrorMessage = "Name required")]
public string Name { get; set; }
@@ -28,5 +32,10 @@ namespace LeafWeb.Core.Entities
[DataType(DataType.Date)]
[Required]
public DateTime Added { get; set; }
public override string ToString()
{
return $"{Id}_{Identifier}";
}
}
}
Core/Entities/LeafInputFile.cs
-9
View File
@@ -7,10 +7,6 @@ namespace LeafWeb.Core.Entities
public int Id { get; set; }
public virtual LeafInput LeafInput { get; set; }
public virtual ICollection<LeafOutputFile> LeafOutputFiles { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
/// <summary>
/// Parsed values from the LeafInput used in LeafWeb for filtering/searching
@@ -20,10 +16,5 @@ namespace LeafWeb.Core.Entities
public string Filename { get; set; }
public byte[] Contents { get; set; }
public override string ToString()
{
return $"{Id}_{Filename}";
}
}
}
Core/Entities/LeafInputFileStatus.cs → Core/Entities/LeafInputStatus.cs
+2 -2
View File
@@ -5,10 +5,10 @@ namespace LeafWeb.Core.Entities
/// <summary>
/// Status of processing LeafInput
/// </summary>
public class LeafInputFileStatus
public class LeafInputStatus
{
public int Id { get; set; }
public virtual LeafInputFile LeafInputFile { get; set; }
public virtual LeafInput LeafInput { get; set; }
public LeafInputStatusType Status { get; set; }
public string Description { get; set; }
public DateTime DateTime { get; set; }
Core/Entities/LeafInputStatusType.cs
+4 -4
View File
@@ -2,9 +2,9 @@ namespace LeafWeb.Core.Entities
{
public enum LeafInputStatusType
{
Queued,
Running,
Complete,
Error
Pending = 0,
Running = 1,
Complete = 2,
Error = 3
}
}
Core/Entities/LeafOutputFile.cs
+1 -1
View File
@@ -5,7 +5,7 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual LeafInputFile LeafInputFile { get; set; }
public virtual LeafInput LeafInput { get; set; }
public string Filename { get; set; }
Core/Remote/IPiscalClient.cs
+5 -5
View File
@@ -4,10 +4,10 @@ namespace LeafWeb.Core.Remote
{
public interface IPiscalClient
{
void RunLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
void CleanupLeafProcess(PiscalLeafInputFile file);
string GetErrorMessage(PiscalLeafInputFile file);
void RunLeafInput(PiscalLeafInput leafInput);
PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInput leafInput);
void CleanupLeafProcess(PiscalLeafInput leafInput);
string GetErrorMessage(PiscalLeafInput leafInput);
}
}
Core/Remote/PiscalLeafInputFile.cs
+37 -13
View File
@@ -1,18 +1,48 @@
using AutoMapper;
using System.Linq;
using AutoMapper;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Utility;
namespace LeafWeb.Core.Remote
{
public class PiscalLeafInput
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; }
static PiscalLeafInput()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore()));
});
Mapper = config.CreateMapper();
}
public PiscalLeafInput() { }
public PiscalLeafInput(LeafInput leafInput)
{
Mapper.Map(leafInput, this);
}
}
public class PiscalLeafInputFile
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string Filename { get; set; }
public byte[] Contents { get; set; }
public string DirectoryName { get; set; }
public string PhotosyntheticType { get; set; }
static PiscalLeafInputFile()
{
@@ -20,14 +50,8 @@ namespace LeafWeb.Core.Remote
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInputFile, PiscalLeafInputFile>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.Filename, opt =>
opt.MapFrom(src => src.Filename.WhitespaceToUnderscore().FilterValidFilename()))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.LeafInput.PhotosynthesisType.Id.WhitespaceToUnderscore()));
.ForMember(dest => dest.Filename, opt =>
opt.MapFrom(src => src.Filename.WhitespaceToUnderscore().FilterValidFilename()));
});
Mapper = config.CreateMapper();
}
Core/Remote/PiscalSshClient.cs
+32 -22
View File
@@ -39,25 +39,35 @@ namespace LeafWeb.Core.Remote
return new ScpClient(_connectionInfo);
}
public void RunLeafInputFile(PiscalLeafInputFile file)
private void CopyLeafInput(PiscalLeafInput leafInput, string directory)
{
var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}";
// copy file
// copy files
using (var scp = GetScpClient())
using (var stream = new MemoryStream(file.Contents))
foreach (var file in leafInput.InputFiles)
{
Console.WriteLine(inputPath);
scp.Connect();
scp.Upload(stream, inputPath);
scp.Disconnect();
var inputPath = $"{directory}/{file.Filename}";
using (var stream = new MemoryStream(file.Contents))
{
Console.WriteLine(inputPath);
scp.Connect();
scp.Upload(stream, inputPath);
scp.Disconnect();
}
}
}
public void RunLeafInput(PiscalLeafInput leafInput)
{
var inputDirectory = $"{BaseDirectory}/{leafInput.DirectoryName}";
CopyLeafInput(leafInput, inputDirectory);
// begin processing
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename} -p {file.PhotosyntheticType}";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType}";
Console.Write(commandText);
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -69,9 +79,9 @@ namespace LeafWeb.Core.Remote
}
}
public PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file)
public PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput)
{
var statusRaw = GetLeafInputStatusRaw(file);
var statusRaw = GetLeafInputStatusRaw(leafInput);
switch (statusRaw[0])
{
@@ -86,12 +96,12 @@ namespace LeafWeb.Core.Remote
}
}
private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
private string[] GetLeafInputStatusRaw(PiscalLeafInput leafInput)
{
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -s";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -109,10 +119,10 @@ namespace LeafWeb.Core.Remote
/// <summary>
/// Gets the leaf output from piscal, only run on if result status is success
/// </summary>
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInput leafInput)
{
// get output files
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusSuccess)
throw new PiscalClientException("output not available, status is " + status[0]);
@@ -131,7 +141,7 @@ namespace LeafWeb.Core.Remote
{
Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(),
DirectoryName = file.DirectoryName
DirectoryName = leafInput.DirectoryName
};
}
}
@@ -140,9 +150,9 @@ namespace LeafWeb.Core.Remote
}
}
public string GetErrorMessage(PiscalLeafInputFile file)
public string GetErrorMessage(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusError)
return string.Empty;
@@ -150,16 +160,16 @@ namespace LeafWeb.Core.Remote
return errorLines.Join(Environment.NewLine);
}
public void CleanupLeafProcess(PiscalLeafInputFile file)
public void CleanupLeafProcess(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] == StatusRunning)
throw new PiscalClientException("Trying to cleanup a running process");
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -c";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -c";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Core/Remote/PiscalUtility.cs
+5 -5
View File
@@ -7,21 +7,21 @@ namespace LeafWeb.Core.Remote
{
public static class PiscalUtility
{
public static string GetPiscalDirectoryName(int id, string name, string identifier)
public static string GetPiscalDirectoryName(int id, string identifier)
{
return $"{id}_{name.FilterAlphaNumeric()}_{identifier.FilterAlphaNumeric()}";
return $"{id}_{identifier.FilterAlphaNumeric()}";
}
public static string GetPiscalDirectoryName(LeafInputFile leafInputFile)
public static string GetPiscalDirectoryName(LeafInput leafInput)
{
return GetPiscalDirectoryName(leafInputFile.Id, leafInputFile.LeafInput.Name, leafInputFile.LeafInput.Identifier);
return GetPiscalDirectoryName(leafInput.Id, leafInput.Identifier);
}
public static int GetIdFromDirectoryName(string directoryName)
{
var match = Regex.Match(directoryName, @"\d_");
if (!match.Success)
throw new FormatException("DirectoryName expected to be formatted {number}_{name}_{identifier}: " + directoryName);
throw new FormatException("DirectoryName expected to be formatted {number}_{identifier}: " + directoryName);
return int.Parse(match.Captures[0].Value);
}
}
Web.Tests/ViewModels/LeafOutput/LeafOutputViewModelTests.cs
-79
View File
@@ -1,79 +0,0 @@
using System;
using System.Linq;
using LeafWeb.Core.Entities;
using LeafWeb.Web.ViewModels.LeafOutput;
using NUnit.Framework;
namespace LeafWeb.Web.Tests.ViewModels.LeafOutput
{
[TestFixture]
public class LeafOutputViewModelTests
{
private LeafInputFile GetLeafInputFile()
{
return new LeafInputFile
{
Filename = "MyFilename.ext",
Id = 3,
CurrentStatus = LeafInputStatusType.Complete,
LeafInput = new LeafInput
{
Added = DateTime.Today,
Email = "test@email.com",
Identifier = "Ident I Fier",
Name = "My Name",
PhotosynthesisType = new PhotosynthesisType {Id = "1", Name = "1", SortOrder = 1}
},
LeafOutputFiles = new[] {new LeafOutputFile {Filename = "OutputFilename.txt"}}
};
}
[Test]
public void CanConstructFromLeafInputFile()
{
var file = GetLeafInputFile();
var viewModel = new LeafOutputViewModel(file);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(file.CurrentStatus.ToString()));
Assert.That(viewModel.LeafInputFileId, Is.EqualTo(file.Id));
Assert.That(viewModel.LeafInputFilename, Is.EqualTo(file.Filename));
Assert.That(viewModel.LeafOutputFilenames, Has.Length.EqualTo(1));
Assert.That(viewModel.LeafInputIdentifier, Is.EqualTo(file.LeafInput.Identifier));
Assert.That(viewModel.LeafInputSiteId, Is.EqualTo(file.LeafInput.SiteId));
Assert.That(viewModel.LeafInputPhotosynthesisType, Is.EqualTo(file.LeafInput.PhotosynthesisType.Name));
}
[Test]
public void CanConstructFromLeafInputFile_Running()
{
var file = GetLeafInputFile();
file.CurrentStatus = LeafInputStatusType.Running;
file.LeafOutputFiles = new LeafOutputFile[0];
var viewModel = new LeafOutputViewModel(file);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Running.ToString()));
Assert.That(viewModel.LeafOutputFilenames, Has.Length.EqualTo(0));
}
[Test]
public void CanConstructFromLeafInputFile_Error()
{
var file = GetLeafInputFile();
file.CurrentStatus = LeafInputStatusType.Error;
file.StatusHistory = new []
{
new LeafInputFileStatus
{
DateTime = DateTime.Today,
LeafInputFile = file,
Description = "My Error",
Status = LeafInputStatusType.Error
}
};
var viewModel = new LeafOutputViewModel(file);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Error.ToString()));
Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(file.StatusHistory.First().Description));
}
}
}
Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs
+81
View File
@@ -0,0 +1,81 @@
using System;
using System.Linq;
using LeafWeb.Core.Entities;
using LeafWeb.Web.ViewModels.ResultStatus;
using NUnit.Framework;
namespace LeafWeb.Web.Tests.ViewModels.ResultStatus
{
[TestFixture]
public class ResultStatusViewModelTests
{
private LeafInput GetLeafInput()
{
return new LeafInput
{
CurrentStatus = LeafInputStatusType.Complete,
OutputFiles = new[] {new LeafOutputFile {Filename = "OutputFilename.txt"}},
Added = DateTime.Today,
Email = "test@email.com",
Identifier = "Ident I Fier",
Name = "My Name",
PhotosynthesisType = new PhotosynthesisType {Id = "1", Name = "1", SortOrder = 1},
InputFiles = new[]
{
new LeafInputFile
{
Filename = "MyFilename.ext",
Id = 3
}
}
};
}
[Test]
public void CanConstructFromLeafInputFile()
{
var leafInput = GetLeafInput();
var viewModel = new ResultStatusViewModel(leafInput);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(leafInput.CurrentStatus.ToString()));
Assert.That(viewModel.LeafInputId, Is.EqualTo(leafInput.Id));
Assert.That(viewModel.LeafOutputFilenames, Has.Length.EqualTo(1));
Assert.That(viewModel.LeafInputIdentifier, Is.EqualTo(leafInput.Identifier));
Assert.That(viewModel.LeafInputSiteId, Is.EqualTo(leafInput.SiteId));
Assert.That(viewModel.LeafInputPhotosynthesisType, Is.EqualTo(leafInput.PhotosynthesisType.Name));
}
[Test]
public void CanConstructFromLeafInputFile_Running()
{
var leafInput = GetLeafInput();
leafInput.CurrentStatus = LeafInputStatusType.Running;
leafInput.OutputFiles = new LeafOutputFile[0];
var viewModel = new ResultStatusViewModel(leafInput);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Running.ToString()));
Assert.That(viewModel.LeafOutputFilenames, Has.Length.EqualTo(0));
}
[Test]
public void CanConstructFromLeafInputFile_Error()
{
var leafInput = GetLeafInput();
leafInput.CurrentStatus = LeafInputStatusType.Error;
leafInput.StatusHistory = new []
{
new LeafInputStatus
{
DateTime = DateTime.Today,
LeafInput = leafInput,
Description = "My Error",
Status = LeafInputStatusType.Error
}
};
var viewModel = new ResultStatusViewModel(leafInput);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Error.ToString()));
Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description));
}
}
}
Web.Tests/Web.Tests.csproj
+1 -1
View File
@@ -44,7 +44,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ViewModels\LeafOutput\LeafOutputViewModelTests.cs" />
<Compile Include="ViewModels\ResultStatus\ResultStatusViewModelTests.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
Web/Controllers/LeafCharterController.cs
+7 -7
View File
@@ -14,23 +14,23 @@ namespace LeafWeb.Web.Controllers
{
public class LeafCharterController : ControllerBase
{
public ActionResult Index(int leafInputFileId)
public ActionResult Index(int leafInputId)
{
var hasLeafOutputFile =
DataService
.GetLeafInputFile(leafInputFileId)?
.LeafOutputFiles?
.GetLeafInput(leafInputId)?
.OutputFiles?
.FirstOrDefault(f => f.IsLeafChartFile) != null;
return View(leafInputFileId);
return View(leafInputId);
}
public ActionResult LeafCharts(int leafInputFileId, int number)
public ActionResult LeafCharts(int leafInputId, int number)
{
var leafOutputFile =
DataService
.GetLeafInputFile(leafInputFileId)?
.LeafOutputFiles?
.GetLeafInput(leafInputId)?
.OutputFiles?
.FirstOrDefault(f => f.IsLeafChartFile);
if (leafOutputFile == null)
Web/Controllers/LeafInputController.cs
+2 -2
View File
@@ -72,7 +72,7 @@ namespace LeafWeb.Web.Controllers
// convert viewModel into Model
var leafInput = viewModel.GetFileInput(DataService);
// load files into LeafInputFile
leafInput.Files =
leafInput.InputFiles =
(from f in files
let bytes = System.IO.File.ReadAllBytes(f.FullName)
select new LeafInputFile {Filename = f.Name, Contents = bytes}).ToList();
@@ -89,7 +89,7 @@ namespace LeafWeb.Web.Controllers
var logger = LogManager.GetCurrentClassLogger();
logger.Info("LeafInput: {0} Added, {1}, {2}, {3}", leafInput.Id, leafInput.Identifier, leafInput.SiteId, leafInput.Email);
logger.Info("LeafInputFiles: {0}, Queued", leafInput.Files.Select(f => f.Id.ToString()).Join(", "));
logger.Info("LeafInputFiles: {0}, Queued", leafInput.InputFiles.Select(f => f.Id.ToString()).Join(", "));
HangfireStartup.TriggerPiscalProcessQueue();
Web/Controllers/LeafOutputController.cs → Web/Controllers/ResultStatusController.cs
+4 -4
View File
@@ -1,18 +1,18 @@
using System.Linq;
using System.Web.Mvc;
using LeafWeb.Web.ViewModels.LeafOutput;
using LeafWeb.Web.ViewModels.ResultStatus;
namespace LeafWeb.Web.Controllers
{
public class LeafOutputController : ControllerBase
public class ResultStatusController : ControllerBase
{
public ActionResult Index()
{
var viewModel =
DataService.GetLeafInputFiles()
DataService.GetLeafInputs()
.OrderByDescending(f => f.Id)
.ToList()
.Select(f => new LeafOutputViewModel(f));
.Select(leafInput => new ResultStatusViewModel(leafInput));
return View(viewModel);
}
}
Web/Properties/PublishProfiles/LeafWeb.pubxml
+39
View File
@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
This file is used by the publish/package process of your Web project. You can customize the behavior of this process
by editing this MSBuild file. In order to learn more about this please visit http://go.microsoft.com/fwlink/?LinkID=208121.
-->
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<WebPublishMethod>MSDeploy</WebPublishMethod>
<LastUsedBuildConfiguration>Release</LastUsedBuildConfiguration>
<LastUsedPlatform>Any CPU</LastUsedPlatform>
<SiteUrlToLaunchAfterPublish>http://leafweb.azurewebsites.net</SiteUrlToLaunchAfterPublish>
<LaunchSiteAfterPublish>True</LaunchSiteAfterPublish>
<ExcludeApp_Data>False</ExcludeApp_Data>
<MSDeployServiceURL>leafweb.scm.azurewebsites.net:443</MSDeployServiceURL>
<DeployIisAppPath>LeafWeb</DeployIisAppPath>
<RemoteSitePhysicalPath />
<SkipExtraFilesOnServer>True</SkipExtraFilesOnServer>
<MSDeployPublishMethod>WMSVC</MSDeployPublishMethod>
<EnableMSDeployBackup>True</EnableMSDeployBackup>
<UserName>$LeafWeb</UserName>
<_SavePWD>True</_SavePWD>
<_DestinationType>AzureWebSite</_DestinationType>
<PublishDatabaseSettings>
<Objects xmlns="">
<ObjectGroup Name="LeafWebContext" Order="1" Enabled="False">
<Destination Path="Data Source=tcp:leafweb.database.windows.net;Initial Catalog=leafweb;Integrated Security=False;User ID=lwadmin@leafweb;Password=j4f1a2e!;Encrypt=True" Name="Server=tcp:leafweb.database.windows.net;Database=leafweb;User ID=lwadmin@leafweb;Password=j4f1a2e!;Trusted_Connection=False;Encrypt=True" />
<Object Type="DbCodeFirst">
<Source Path="DBContext" DbContext="LeafWeb.Core.DAL.LeafWebContext, Core" Origin="Configuration" />
</Object>
</ObjectGroup>
</Objects>
</PublishDatabaseSettings>
</PropertyGroup>
<ItemGroup>
<MSDeployParameterValue Include="$(DeployParameterPrefix)LeafWebContext-Web.config Connection String">
<ParameterValue>Server=tcp:leafweb.database.windows.net;Database=leafweb;User ID=lwadmin@leafweb;Password=j4f1a2e!;Trusted_Connection=False;Encrypt=True</ParameterValue>
</MSDeployParameterValue>
</ItemGroup>
</Project>
Web/Services/EmailNotificationService.cs
+9 -9
View File
@@ -31,25 +31,25 @@ namespace LeafWeb.Web.Services
public EmailNotificationService() : this(new DataService())
{ }
public void SendLeafWebError(int leafWebFileId, string errorMessage)
public void SendLeafWebError(int leafInputId, string errorMessage)
{
var file = _dataService.GetLeafInputFile(leafWebFileId);
var body = $"Your LeafWeb analysis job, {file.LeafInput.Identifier} (with filename {file.Filename}), encountered the following errors." + Environment.NewLine
var leafInput = _dataService.GetLeafInput(leafInputId);
var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, encountered the following errors." + Environment.NewLine
+ "You will need to correct your input and resubmit." + Environment.NewLine + Environment.NewLine
+ errorMessage;
var message = new MailMessage(_emaialFromAddress, file.LeafInput.Email, "LeafWeb processing error", body);
var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb processing error", body);
SendMessage(message);
}
public void SendLeafWebSuccess(int leafWebFileId)
public void SendLeafWebSuccess(int leafInputId)
{
var file = _dataService.GetLeafInputFile(leafWebFileId);
var body = $"Your LeafWeb analysis job, {file.LeafInput.Identifier} (with filename {file.Filename}), has completed." + Environment.NewLine;
var message = new MailMessage(_emaialFromAddress, file.LeafInput.Email, "LeafWeb results", body);
var leafInput = _dataService.GetLeafInput(leafInputId);
var body = $"Your LeafWeb analysis job, {leafInput.Identifier}, has completed." + Environment.NewLine;
var message = new MailMessage(_emaialFromAddress, leafInput.Email, "LeafWeb results", body);
var fileStreams =
(from outputFile in
file.LeafOutputFiles
leafInput.OutputFiles
select Tuple.Create(outputFile, new MemoryStream(outputFile.Contents))).ToList();
try
Web/Services/PiscalQueueManager.cs
+16 -16
View File
@@ -55,39 +55,39 @@ namespace LeafWeb.Web.Services
private void ProcessQueue(ILogger logger)
{
var runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList();
if (runningLeafInputFiles.Any())
var runningLeafInputs = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
if (runningLeafInputs.Any())
return;
var queuedFile =
var pending =
_dataService
.GetLeafInputFiles(LeafInputStatusType.Queued)
.GetLeafInputs(LeafInputStatusType.Pending)
.OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime))
.FirstOrDefault();
if (queuedFile == null)
if (pending == null)
return;
logger.Info("LeafInputFile: {0}, Start", queuedFile.Id);
logger.Info("LeafInputFile: {0}, Start", pending.Id);
try
{
_piscalService.Run(queuedFile);
_piscalService.Run(pending);
}
catch (PiscalClientException ex)
{
logger.Error("LeafInputFile: {0}, ProcessQueue Exception: {1}", queuedFile.Id, ex.Message);
_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Error, "Error occurred submitting LeafInput");
logger.Error("LeafInputFile: {0}, ProcessQueue Exception: {1}", pending.Id, ex.Message);
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Error, "Error occurred submitting LeafInput");
// TODO: re-queue
//_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput");
}
_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Running);
_dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running);
}
private void ProcessRunning(ILogger logger)
{
var runningLeafInputFiles = _dataService.GetLeafInputFiles(LeafInputStatusType.Running).ToList();
foreach (var file in runningLeafInputFiles)
var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
foreach (var file in running)
{
var status = _piscalService.GetStatus(file);
try
@@ -110,7 +110,7 @@ namespace LeafWeb.Web.Services
string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
// update db
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Complete);
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Complete);
BackgroundJob.Enqueue(() => _emailService.SendLeafWebSuccess(file.Id));
@@ -122,7 +122,7 @@ namespace LeafWeb.Web.Services
case PiscalStatus.NotStarted:
logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", file.Id);
// if it's not started, try to requeue the process - unusual state
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Queued);
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Pending);
break;
case PiscalStatus.Error:
@@ -131,7 +131,7 @@ namespace LeafWeb.Web.Services
var errorMessage = _piscalService.GetErrorMessage(file);
logger.Info("LeafInputFile: {0}, Error Message: {1}", file.Id, errorMessage);
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Error, errorMessage);
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, errorMessage);
BackgroundJob.Enqueue(() => _emailService.SendLeafWebError(file.Id, errorMessage));
@@ -144,7 +144,7 @@ namespace LeafWeb.Web.Services
catch (PiscalClientException ex)
{
logger.Error("LeafInputFile: {0}, ProcessRunning Exception: {1}", file.Id, ex.Message);
_dataService.SetLeafInputFileStatus(file, LeafInputStatusType.Error, "Error occurred processing LeafInput");
_dataService.SetLeafInputStatus(file, LeafInputStatusType.Error, "Error occurred processing LeafInput");
// TODO: re-queue
}
Web/Services/PiscalService.cs
+15 -15
View File
@@ -18,40 +18,40 @@ namespace LeafWeb.Web.Services
{
}
public void Run(LeafInputFile leafInputFile)
public void Run(LeafInput leafInput)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
_piscalClient.RunLeafInputFile(inputFile);
var inputFile = new PiscalLeafInput(leafInput);
_piscalClient.RunLeafInput(inputFile);
}
public PiscalStatus GetStatus(LeafInputFile leafInputFile)
public PiscalStatus GetStatus(LeafInput leafInput)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
return _piscalClient.GetLeafInputFileStatus(inputFile);
var inputFile = new PiscalLeafInput(leafInput);
return _piscalClient.GetLeafInputStatus(inputFile);
}
public IEnumerable<LeafOutputFile> RetrieveOutputFiles(LeafInputFile leafInputFile)
public IEnumerable<LeafOutputFile> RetrieveOutputFiles(LeafInput leafInput)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(inputFile);
var input = new PiscalLeafInput(leafInput);
var piscalLeafOutputFiles = _piscalClient.RetrieveLeafOutput(input);
foreach (var file in piscalLeafOutputFiles)
{
var leafOutputFile = file.GetLeafOutputFile();
leafOutputFile.LeafInputFile = leafInputFile;
leafOutputFile.LeafInput = leafInput;
yield return leafOutputFile;
}
}
public string GetErrorMessage(LeafInputFile leafInputFile)
public string GetErrorMessage(LeafInput leafInput)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
var inputFile = new PiscalLeafInput(leafInput);
return _piscalClient.GetErrorMessage(inputFile);
}
public void Cleanup(LeafInputFile leafInputFile)
public void Cleanup(LeafInput leafInput)
{
var inputFile = new PiscalLeafInputFile(leafInputFile);
_piscalClient.CleanupLeafProcess(inputFile);
var input = new PiscalLeafInput(leafInput);
_piscalClient.CleanupLeafProcess(input);
}
}
}
Web/ViewModels/LeafOutput/LeafOutputViewModel.cs → Web/ViewModels/ResultStatus/ResultStatusViewModel.cs
+18 -20
View File
@@ -2,45 +2,43 @@
using AutoMapper;
using LeafWeb.Core.Entities;
namespace LeafWeb.Web.ViewModels.LeafOutput
namespace LeafWeb.Web.ViewModels.ResultStatus
{
public class LeafOutputViewModel
public class ResultStatusViewModel
{
private static readonly IMapper Mapper;
public int LeafInputFileId { get; set; }
public string LeafInputFilename { get; set; }
public string CurrentStatus { get; set; }
public string[] ErrorMessages { get; set; }
public string[] LeafOutputFilenames { get; set; }
public bool HasLeafChartOutputFile { get; set; }
public int LeafInputId { get; set; }
public string LeafInputName { get; set; }
public string LeafInputIdentifier { get; set; }
public string LeafInputSiteId { get; set; }
public string LeafInputPhotosynthesisType { get; set; }
public string CurrentStatus { get; set; }
public string[] ErrorMessages { get; set; }
public string[] LeafOutputFilenames { get; set; }
public bool HasLeafChartOutputFile { get; set; }
static LeafOutputViewModel()
static ResultStatusViewModel()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInputFile, LeafOutputViewModel>()
.ForMember(dest => dest.LeafInputFileId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.LeafInputFilename, opt => opt.MapFrom(src => src.Filename))
cfg.CreateMap<Core.Entities.LeafInput, ResultStatusViewModel>()
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.LeafOutputFilenames,
opt => opt.ResolveUsing(
file =>
file.LeafOutputFiles?
src =>
src.OutputFiles?
.Select(o => o.Filename)
.ToArray()
?? new string[] {}))
.ForMember(dest => dest.HasLeafChartOutputFile,
opt => opt.ResolveUsing(
file => file.LeafOutputFiles?.Any(o => o.IsLeafChartFile)))
.ForMember(dest => dest.LeafInputName, opt => opt.MapFrom(src => src.LeafInput.Name))
.ForMember(dest => dest.LeafInputIdentifier, opt => opt.MapFrom(src => src.LeafInput.Identifier))
.ForMember(dest => dest.LeafInputSiteId, opt => opt.MapFrom(src => src.LeafInput.SiteId))
.ForMember(dest => dest.LeafInputPhotosynthesisType, opt => opt.MapFrom(src => src.LeafInput.PhotosynthesisType.Name))
file => file.OutputFiles?.Any(o => o.IsLeafChartFile)))
.ForMember(dest => dest.LeafInputName, opt => opt.MapFrom(src => src.Name))
.ForMember(dest => dest.LeafInputIdentifier, opt => opt.MapFrom(src => src.Identifier))
.ForMember(dest => dest.LeafInputSiteId, opt => opt.MapFrom(src => src.SiteId))
.ForMember(dest => dest.LeafInputPhotosynthesisType, opt => opt.MapFrom(src => src.PhotosynthesisType.Name))
.ForMember(dest => dest.ErrorMessages,
opt => opt.ResolveUsing(
src =>
@@ -53,7 +51,7 @@ namespace LeafWeb.Web.ViewModels.LeafOutput
Mapper = config.CreateMapper();
}
public LeafOutputViewModel(LeafInputFile leafInput)
public ResultStatusViewModel(Core.Entities.LeafInput leafInput)
{
Mapper.Map(leafInput, this);
}
Web/Views/LeafInput/Index.cshtml
+5 -3
View File
@@ -4,13 +4,15 @@
@Styles.Render("~/backload/blueimp/bootstrap/BasicPlusUI/css")
}
@Html.Partial("_StatusMessage")
<h1>Submitting Data and Retrieving EDO Results</h1>
<p>
There is no limit on the number of files you may submit for analysis. Keep selecting files and hitting the Add button until all of the files you need to upload are shown in the list. Then enter an identifier for this set of data and click the Upload button.
There is no limit on the number of files you may submit for analysis.
Keep selecting files and hitting the Add button until all of the files you need to upload are shown in the list.
Then enter an identifier for this set of data and click the Upload button.
</p>
@Html.Partial("_StatusMessage")
<div class="row">
<div class="col-md-7 well">
@Html.Partial("_ValidationSummary")
Web/Views/LeafOutput/Index.cshtml → Web/Views/ResultStatus/Index.cshtml
+2 -4
View File
@@ -1,4 +1,4 @@
@model IEnumerable<LeafWeb.Web.ViewModels.LeafOutput.LeafOutputViewModel>
@model IEnumerable<LeafWeb.Web.ViewModels.ResultStatus.ResultStatusViewModel>
@{
ViewBag.Title = "Results";
@@ -11,10 +11,8 @@
grid.Columns(
grid.Column("LeafInputIdentifier", "Identifier"),
grid.Column("LeafInputSiteId", "Site Id"),
grid.Column("LeafInputFilename", "Filename"),
grid.Column("LeafInputName", "Submitted By"),
grid.Column("CurrentStatus", "Status"),
grid.Column("Chart", "Chart", item => @Html.ActionLink("Chart", "Index", "LeafCharter", new { leafInputFileId = item.LeafInputFileId}, new {}))
grid.Column("CurrentStatus", "Status")
),
htmlAttributes: new { @class = "table table-striped table-bordered table-hover table-condensed" }
)
Web/Views/Shared/_Layout.cshtml
+1 -1
View File
@@ -31,7 +31,7 @@
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Leaf Data <span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="/LeafInput">Submit Data</a></li>
<li><a href="/LeafOutput">Result Status</a></li>
<li><a href="/ResultStatus">Result Status</a></li>
</ul>
</li>
<li>
Web/Web.csproj
+4 -3
View File
@@ -524,6 +524,7 @@
<SubType>Designer</SubType>
</None>
<None Include="Properties\PublishProfiles\LeafWeb - Web Deploy.pubxml" />
<None Include="Properties\PublishProfiles\LeafWeb.pubxml" />
<None Include="Scripts\jquery-1.9.1.intellisense.js" />
<Content Include="Scripts\angular-mocks.js" />
<Content Include="Scripts\bootstrap-datepicker.js" />
@@ -926,7 +927,7 @@
<Compile Include="Controllers\FluxnetSiteController.cs" />
<Compile Include="Controllers\LeafCharterController.cs" />
<Compile Include="Controllers\LeafInputController.cs" />
<Compile Include="Controllers\LeafOutputController.cs" />
<Compile Include="Controllers\ResultStatusController.cs" />
<Compile Include="Controllers\PagesController.cs" />
<Compile Include="Global.asax.cs">
<DependentUpon>Global.asax</DependentUpon>
@@ -941,7 +942,7 @@
<Compile Include="Utility\Validation.cs" />
<Compile Include="ViewModels\LeafInput\ConfirmViewModel.cs" />
<Compile Include="ViewModels\LeafInput\CreateViewModel.cs" />
<Compile Include="ViewModels\LeafOutput\LeafOutputViewModel.cs" />
<Compile Include="ViewModels\ResultStatus\ResultStatusViewModel.cs" />
<Compile Include="ViewModels\SelectListViewModel.cs" />
</ItemGroup>
<ItemGroup>
@@ -974,7 +975,7 @@
<Content Include="Views\Shared\_StatusMessage.cshtml" />
<Content Include="Views\Shared\_ValidationField.cshtml" />
<Content Include="Views\Shared\EditorTemplates\SelectListViewModel.cshtml" />
<Content Include="Views\LeafOutput\Index.cshtml" />
<Content Include="Views\ResultStatus\Index.cshtml" />
<Content Include="Views\Pages\Information.cshtml" />
<None Include="Web.Debug.config">
<DependentUpon>Web.config</DependentUpon>