Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Submit all LeafInputFiles together

Browse Source
This commit is contained in:
poprhythm
2016-03-28 10:20:50 -04:00
parent 4b2757b19a
commit 9e86b92f08
29 changed files with 353 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core/Core.csproj
+1 -1
View File
@@ -85,7 +85,7 @@
<Compile Include="Entities\LeafInputFile.cs" />
<Compile Include="Entities\LeafInputDataPhotosynthetic.cs" />
<Compile Include="Entities\LeafInputDataSite.cs" />
<Compile Include="Entities\LeafInputFileStatus.cs" />
<Compile Include="Entities\LeafInputStatus.cs" />
<Compile Include="Parsers\FluxnetSiteCsvParser.cs" />
<Compile Include="Remote\PiscalClientException.cs" />
<Compile Include="Remote\PiscalLeafInputFile.cs" />
Core/DAL/DataService.cs
+37 -32
View File
@@ -50,16 +50,49 @@ namespace LeafWeb.Core.DAL
return _db.LeafInputs.FirstOrDefault(li => li.Id == id);
}
public IQueryable<LeafInput> GetLeafInputs(LeafInputStatusType status)
{
return
from file in _db.LeafInputs
where file.CurrentStatus == status
select file;
}
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
_db.LeafInputs.Add(leafInput);
foreach (var leafInputFile in leafInput.Files)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Queued);
}
SetLeafInputStatusNoUpdate(leafInput, LeafInputStatusType.Pending);
_db.SaveChanges();
}
private void SetLeafInputStatusNoUpdate(LeafInput leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInput = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputStatus(LeafInput leafInput, LeafInputStatusType status, string description = null)
{
SetLeafInputStatusNoUpdate(leafInput, status, description);
UpdateLeafInput(leafInput);
}
public void UpdateLeafInput(LeafInput leafInput)
{
_db.Entry(leafInput).State = EntityState.Modified;
_db.SaveChanges();
}
#endregion
#region LeafInputFile
@@ -74,40 +107,12 @@ namespace LeafWeb.Core.DAL
return _db.LeafInputFiles.Find(id);
}
public IQueryable<LeafInputFile> GetLeafInputFiles(LeafInputStatusType status)
{
return
from file in _db.LeafInputFiles
where file.CurrentStatus == status
select file;
}
public void UpdateLeafInputFile(LeafInputFile leafInputFile)
{
_db.Entry(leafInputFile).State = EntityState.Modified;
_db.SaveChanges();
}
private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputFileStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInputFile = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputFileStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
UpdateLeafInputFile(leafInputFile);
}
#endregion
Core/DAL/LeafWebContext.cs
+1 -1
View File
@@ -8,7 +8,7 @@ namespace LeafWeb.Core.DAL
{
public DbSet<LeafInput> LeafInputs { get; set; }
public DbSet<LeafInputFile> LeafInputFiles { get; set; }
public DbSet<LeafInputFileStatus> LeafInputStatus { get; set; }
public DbSet<LeafInputStatus> LeafInputStatus { get; set; }
public DbSet<FluxnetSite> FluxnetSites { get; set; }
public DbSet<PhotosynthesisType> PhotosynthesisTypes { get; set; }
public DbSet<LeafOutputFile> LeafOutputFiles { get; set; }
Core/Entities/LeafInput.cs
+10 -1
View File
@@ -8,7 +8,11 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual ICollection<LeafInputFile> Files { get; set; }
public virtual ICollection<LeafInputFile> InputFiles { get; set; }
public virtual ICollection<LeafOutputFile> OutputFiles { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputStatus> StatusHistory { get; set; }
[Required(ErrorMessage = "Name required")]
public string Name { get; set; }
@@ -28,5 +32,10 @@ namespace LeafWeb.Core.Entities
[DataType(DataType.Date)]
[Required]
public DateTime Added { get; set; }
public override string ToString()
{
return $"{Id}_{Identifier}";
}
}
}
Core/Entities/LeafInputFile.cs
-9
View File
@@ -7,10 +7,6 @@ namespace LeafWeb.Core.Entities
public int Id { get; set; }
public virtual LeafInput LeafInput { get; set; }
public virtual ICollection<LeafOutputFile> LeafOutputFiles { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
/// <summary>
/// Parsed values from the LeafInput used in LeafWeb for filtering/searching
@@ -20,10 +16,5 @@ namespace LeafWeb.Core.Entities
public string Filename { get; set; }
public byte[] Contents { get; set; }
public override string ToString()
{
return $"{Id}_{Filename}";
}
}
}
Core/Entities/LeafInputFileStatus.cs → Core/Entities/LeafInputStatus.cs
+2 -2
View File
@@ -5,10 +5,10 @@ namespace LeafWeb.Core.Entities
/// <summary>
/// Status of processing LeafInput
/// </summary>
public class LeafInputFileStatus
public class LeafInputStatus
{
public int Id { get; set; }
public virtual LeafInputFile LeafInputFile { get; set; }
public virtual LeafInput LeafInput { get; set; }
public LeafInputStatusType Status { get; set; }
public string Description { get; set; }
public DateTime DateTime { get; set; }
Core/Entities/LeafInputStatusType.cs
+4 -4
View File
@@ -2,9 +2,9 @@ namespace LeafWeb.Core.Entities
{
public enum LeafInputStatusType
{
Queued,
Running,
Complete,
Error
Pending = 0,
Running = 1,
Complete = 2,
Error = 3
}
}
Core/Entities/LeafOutputFile.cs
+1 -1
View File
@@ -5,7 +5,7 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual LeafInputFile LeafInputFile { get; set; }
public virtual LeafInput LeafInput { get; set; }
public string Filename { get; set; }
Core/Remote/IPiscalClient.cs
+5 -5
View File
@@ -4,10 +4,10 @@ namespace LeafWeb.Core.Remote
{
public interface IPiscalClient
{
void RunLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
void CleanupLeafProcess(PiscalLeafInputFile file);
string GetErrorMessage(PiscalLeafInputFile file);
void RunLeafInput(PiscalLeafInput leafInput);
PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInput leafInput);
void CleanupLeafProcess(PiscalLeafInput leafInput);
string GetErrorMessage(PiscalLeafInput leafInput);
}
}
Core/Remote/PiscalLeafInputFile.cs
+37 -13
View File
@@ -1,18 +1,48 @@
using AutoMapper;
using System.Linq;
using AutoMapper;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Utility;
namespace LeafWeb.Core.Remote
{
public class PiscalLeafInput
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; }
static PiscalLeafInput()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.PhotosynthesisType.Id.WhitespaceToUnderscore()));
});
Mapper = config.CreateMapper();
}
public PiscalLeafInput() { }
public PiscalLeafInput(LeafInput leafInput)
{
Mapper.Map(leafInput, this);
}
}
public class PiscalLeafInputFile
{
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string Filename { get; set; }
public byte[] Contents { get; set; }
public string DirectoryName { get; set; }
public string PhotosyntheticType { get; set; }
static PiscalLeafInputFile()
{
@@ -20,14 +50,8 @@ namespace LeafWeb.Core.Remote
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInputFile, PiscalLeafInputFile>()
.ForMember(dest => dest.DirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.Filename, opt =>
opt.MapFrom(src => src.Filename.WhitespaceToUnderscore().FilterValidFilename()))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(
dest => dest.PhotosyntheticType,
opt => opt.MapFrom(src => src.LeafInput.PhotosynthesisType.Id.WhitespaceToUnderscore()));
.ForMember(dest => dest.Filename, opt =>
opt.MapFrom(src => src.Filename.WhitespaceToUnderscore().FilterValidFilename()));
});
Mapper = config.CreateMapper();
}
Core/Remote/PiscalSshClient.cs
+32 -22
View File
@@ -39,25 +39,35 @@ namespace LeafWeb.Core.Remote
return new ScpClient(_connectionInfo);
}
public void RunLeafInputFile(PiscalLeafInputFile file)
private void CopyLeafInput(PiscalLeafInput leafInput, string directory)
{
var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}";
// copy file
// copy files
using (var scp = GetScpClient())
using (var stream = new MemoryStream(file.Contents))
foreach (var file in leafInput.InputFiles)
{
Console.WriteLine(inputPath);
scp.Connect();
scp.Upload(stream, inputPath);
scp.Disconnect();
var inputPath = $"{directory}/{file.Filename}";
using (var stream = new MemoryStream(file.Contents))
{
Console.WriteLine(inputPath);
scp.Connect();
scp.Upload(stream, inputPath);
scp.Disconnect();
}
}
}
public void RunLeafInput(PiscalLeafInput leafInput)
{
var inputDirectory = $"{BaseDirectory}/{leafInput.DirectoryName}";
CopyLeafInput(leafInput, inputDirectory);
// begin processing
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename} -p {file.PhotosyntheticType}";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType}";
Console.Write(commandText);
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -69,9 +79,9 @@ namespace LeafWeb.Core.Remote
}
}
public PiscalStatus GetLeafInputFileStatus(PiscalLeafInputFile file)
public PiscalStatus GetLeafInputStatus(PiscalLeafInput leafInput)
{
var statusRaw = GetLeafInputStatusRaw(file);
var statusRaw = GetLeafInputStatusRaw(leafInput);
switch (statusRaw[0])
{
@@ -86,12 +96,12 @@ namespace LeafWeb.Core.Remote
}
}
private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
private string[] GetLeafInputStatusRaw(PiscalLeafInput leafInput)
{
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -s";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
@@ -109,10 +119,10 @@ namespace LeafWeb.Core.Remote
/// <summary>
/// Gets the leaf output from piscal, only run on if result status is success
/// </summary>
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInput leafInput)
{
// get output files
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusSuccess)
throw new PiscalClientException("output not available, status is " + status[0]);
@@ -131,7 +141,7 @@ namespace LeafWeb.Core.Remote
{
Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(),
DirectoryName = file.DirectoryName
DirectoryName = leafInput.DirectoryName
};
}
}
@@ -140,9 +150,9 @@ namespace LeafWeb.Core.Remote
}
}
public string GetErrorMessage(PiscalLeafInputFile file)
public string GetErrorMessage(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusError)
return string.Empty;
@@ -150,16 +160,16 @@ namespace LeafWeb.Core.Remote
return errorLines.Join(Environment.NewLine);
}
public void CleanupLeafProcess(PiscalLeafInputFile file)
public void CleanupLeafProcess(PiscalLeafInput leafInput)
{
var status = GetLeafInputStatusRaw(file);
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] == StatusRunning)
throw new PiscalClientException("Trying to cleanup a running process");
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -c";
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -c";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Core/Remote/PiscalUtility.cs
+5 -5
View File
@@ -7,21 +7,21 @@ namespace LeafWeb.Core.Remote
{
public static class PiscalUtility
{
public static string GetPiscalDirectoryName(int id, string name, string identifier)
public static string GetPiscalDirectoryName(int id, string identifier)
{
return $"{id}_{name.FilterAlphaNumeric()}_{identifier.FilterAlphaNumeric()}";
return $"{id}_{identifier.FilterAlphaNumeric()}";
}
public static string GetPiscalDirectoryName(LeafInputFile leafInputFile)
public static string GetPiscalDirectoryName(LeafInput leafInput)
{
return GetPiscalDirectoryName(leafInputFile.Id, leafInputFile.LeafInput.Name, leafInputFile.LeafInput.Identifier);
return GetPiscalDirectoryName(leafInput.Id, leafInput.Identifier);
}
public static int GetIdFromDirectoryName(string directoryName)
{
var match = Regex.Match(directoryName, @"\d_");
if (!match.Success)
throw new FormatException("DirectoryName expected to be formatted {number}_{name}_{identifier}: " + directoryName);
throw new FormatException("DirectoryName expected to be formatted {number}_{identifier}: " + directoryName);
return int.Parse(match.Captures[0].Value);
}
}