Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add output file type, collect all files

Browse Source
This commit is contained in:
James Kolpack
2016-05-18 13:05:24 -04:00
parent 4343f08733
commit 77ae03534e
19 changed files with 288 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core.Tests/Remote/PiscalSshClientTests.cs
+1 -1
View File
@@ -17,7 +17,7 @@ namespace LeafWeb.Core.Tests.Remote
new PiscalLeafInput new PiscalLeafInput
{ {
LeafInputId = 1, LeafInputId = 1,
DirectoryName = "TestDirectory3", PiscalDirectoryName = "TestDirectory3",
PhotosyntheticType = "C4_photosynthesis_leafweb", PhotosyntheticType = "C4_photosynthesis_leafweb",
InputFiles = new[] InputFiles = new[]
{ {
Core/Core.csproj
+8
View File
@@ -85,6 +85,7 @@
<Compile Include="Entities\LeafGasComparison.cs" /> <Compile Include="Entities\LeafGasComparison.cs" />
<Compile Include="Entities\LeafInputStatusType.cs" /> <Compile Include="Entities\LeafInputStatusType.cs" />
<Compile Include="Entities\LeafOutputFile.cs" /> <Compile Include="Entities\LeafOutputFile.cs" />
<Compile Include="Entities\LeafOutputFileType.cs" />
<Compile Include="Entities\PhotosynthesisType.cs" /> <Compile Include="Entities\PhotosynthesisType.cs" />
<Compile Include="Migrations\201604151603282_InitialCreate.cs" /> <Compile Include="Migrations\201604151603282_InitialCreate.cs" />
<Compile Include="Migrations\201604151603282_InitialCreate.Designer.cs"> <Compile Include="Migrations\201604151603282_InitialCreate.Designer.cs">
@@ -94,6 +95,10 @@
<Compile Include="Migrations\201605061427009_LeafInputUniqueToken.Designer.cs"> <Compile Include="Migrations\201605061427009_LeafInputUniqueToken.Designer.cs">
<DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon> <DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon>
</Compile> </Compile>
<Compile Include="Migrations\201605161458209_LeafOutputFileType.cs" />
<Compile Include="Migrations\201605161458209_LeafOutputFileType.Designer.cs">
<DependentUpon>201605161458209_LeafOutputFileType.cs</DependentUpon>
</Compile>
<Compile Include="Migrations\Configuration.cs" /> <Compile Include="Migrations\Configuration.cs" />
<Compile Include="Parsers\LeafGasComparisonParser.cs" /> <Compile Include="Parsers\LeafGasComparisonParser.cs" />
<Compile Include="Remote\IPiscalClient.cs" /> <Compile Include="Remote\IPiscalClient.cs" />
@@ -144,6 +149,9 @@
<EmbeddedResource Include="Migrations\201605061427009_LeafInputUniqueToken.resx"> <EmbeddedResource Include="Migrations\201605061427009_LeafInputUniqueToken.resx">
<DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon> <DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon>
</EmbeddedResource> </EmbeddedResource>
<EmbeddedResource Include="Migrations\201605161458209_LeafOutputFileType.resx">
<DependentUpon>201605161458209_LeafOutputFileType.cs</DependentUpon>
</EmbeddedResource>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Core/Entities/LeafInput.cs
+2 -3
View File
@@ -2,7 +2,6 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema; using System.ComponentModel.DataAnnotations.Schema;
using System.Data;
using System.IO; using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Linq; using System.Linq;
@@ -55,12 +54,12 @@ namespace LeafWeb.Core.Entities
/// <summary> /// <summary>
/// Contains all output files in a zip /// Contains all output files in a zip
/// </summary> /// </summary>
public byte[] GetOutputFileZip() public byte[] GetOutputFileZip(LeafOutputFileType? fileType)
{ {
using (var compressedFileStream = new MemoryStream()) using (var compressedFileStream = new MemoryStream())
using (var archive = new ZipArchive(compressedFileStream, ZipArchiveMode.Create, true)) using (var archive = new ZipArchive(compressedFileStream, ZipArchiveMode.Create, true))
{ {
foreach (var outputFile in OutputFiles) foreach (var outputFile in OutputFiles.Where(f => !fileType.HasValue || f.FileType==fileType.Value))
{ {
var entry = archive.CreateEntry(outputFile.Filename); var entry = archive.CreateEntry(outputFile.Filename);
using (var originalFileStream = new MemoryStream(outputFile.Contents)) using (var originalFileStream = new MemoryStream(outputFile.Contents))
Core/Entities/LeafOutputFile.cs
+2
View File
@@ -14,6 +14,8 @@ namespace LeafWeb.Core.Entities
public string Filename { get; set; } public string Filename { get; set; }
public LeafOutputFileType FileType { get; set; }
public byte[] Contents { get; set; } public byte[] Contents { get; set; }
public bool IsLeafChartFile => Filename?.Contains(Filename_LeafChart) ?? false; public bool IsLeafChartFile => Filename?.Contains(Filename_LeafChart) ?? false;
Core/Entities/LeafOutputFileType.cs
+9
View File
@@ -0,0 +1,9 @@
namespace LeafWeb.Core.Entities
{
public enum LeafOutputFileType
{
ToUser,
NotToUser,
CleanedInput
}
}
Core/Migrations/201605161458209_LeafOutputFileType.Designer.cs
Generated
+29
View File
@@ -0,0 +1,29 @@
// <auto-generated />
namespace LeafWeb.Core.Migrations
{
using System.CodeDom.Compiler;
using System.Data.Entity.Migrations;
using System.Data.Entity.Migrations.Infrastructure;
using System.Resources;
[GeneratedCode("EntityFramework.Migrations", "6.1.3-40302")]
public sealed partial class LeafOutputFileType : IMigrationMetadata
{
private readonly ResourceManager Resources = new ResourceManager(typeof(LeafOutputFileType));
string IMigrationMetadata.Id
{
get { return "201605161458209_LeafOutputFileType"; }
}
string IMigrationMetadata.Source
{
get { return null; }
}
string IMigrationMetadata.Target
{
get { return Resources.GetString("Target"); }
}
}
}
Core/Migrations/201605161458209_LeafOutputFileType.cs
+19
View File
@@ -0,0 +1,19 @@
namespace LeafWeb.Core.Migrations
{
using System;
using System.Data.Entity.Migrations;
public partial class LeafOutputFileType : DbMigration
{
public override void Up()
{
AddColumn("dbo.LeafOutputFile", "FileType", c => c.Int(nullable: false));
Sql(@"UPDATE dbo.LeafOutputFile SET FileType = 0"); // default to "ToUser"
}
public override void Down()
{
DropColumn("dbo.LeafOutputFile", "FileType");
}
}
}
Core/Migrations/201605161458209_LeafOutputFileType.resx
+126
View File
@@ -0,0 +1,126 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" use="required" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string" />
<xsd:attribute name="name" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="Target" xml:space="preserve">
<value>H4sIAAAAAAAEAO1c3W4rtxG+L9B3WOiqKRzJ9mmBxLATOLLdCPUfLJ+0dwatpWSiu1xlyTUkFH2yXvSR+goh95fkklzujxT5JDg3x1zyIzn8ODMccvT///7v/PtNGHjvMCYowhejk/HxyIN4EfkIry5GCV1+/c3o++/++Ifzaz/ceD8V9T7xeqwlJhejN0rXZ5MJWbzBEJBxiBZxRKIlHS+icAL8aHJ6fPzt5ORkAhnEiGF53vlTgikKYfoH+3Ma4QVc0wQEd5EPA5KXsy/zFNW7ByEka7CAF6NbCJb/gK/jaRTD8dXl7ci7DBBg45jDYDnyAMYRBZSN8uwzgXMaR3g1X7MCEDxv15DVW4KAwHz0Z1V114kcn/KJTKqGBdQiITQKWwKefMolM1Gbd5LvqJQck901kzHd8lmn8rsY3QTJBkM6R5TNX+3vbBrEvK4i4RQEQTIWGh95/NNRSQfGGv6PFScBTWJ4gWFCYxAceY/Ja4AWf4fb5+hfEF/gJAjEIbJBsm9SASt6jKM1jOn2CS7lgc/8kTeRm0/U9mXretNsdowSjNsj7w5sbiFe0TfG+tNvRt4N2kC/KMk58hkjthVYIxon7M97NnzwGsDyuyC9OWUi+RvEMAYU+o+AUhgzCtxHGNYGrQyRi5T/zzJC9l+nEdo7mkZs28XbnfdzC7DPGtCBOzqfVIy28pwTeIbXCb1BQQemS833zvUuJK/YPcP002knps58mMqxaXG5UPB+2IopGxIpOvoBYcDJ69KRgnwP3tEqnb3K1GKpR94TDNIa5A2tM2NSEeGlpAMbzU0chU9RILYWvr88g3gFOfUjS6V5lMSL3vTuQe3faa2yLYljVnPO0JKScjILsm/8i2YkdvSB1HvLXq9DgIL9d5uJfIlgvP++uS212vod9cta/5zAdItYHY1d9H3p+7Cc8hVj/zMKHVCMSlFUdkNpxULhWbVioTpdB/qQ0MaRVnVeBL0pj1ZXR6vHtRV1utw26se3iEZki+kbJChXJ1Yx6xroxF2vZxG7pnJb8Wf68EfETBC3y7Y5ZFXNK6CtZDaltZq97Wm1st2MatX+d8v6azmMvKNsdyjWu1qcTsb7MBzRzopMpwOsGq/TJtIpqbYbqY7xwTbT4cUW9hJXmEcxfYj9yt8zKZV+x53CN+986MkAPhindq+gd3TmuYJkEaN1FsvcMQOvIGVHHbL7fkrvejBv2zEE0c2DMvp/Rl/LbYsmoXmDZmSZkZsArKpwe89N298YsIVmGirYMmKIG0tepTsYvsK4MGkQ+ymNfgJBwv4+ri2qVJ0NNKZi/RN7/acEY7H6qb36DcKIEUNo8MneYBqF6wDyUH9e/y/2+tcbfgeT7ta8wV/rdMgWvoEMqr81BBdkzH1T4Tn6TLh5c2PCfUSVBg1UmAYQYOjnG1vPB730LwmJFiiVorobxeO53Pc19j2Xs7rGHjCdyiSM1kymTDEwSYzH9bk14JfHOg1+dgKT+/izKghh0s2y0PrNtvHanWjTyaJh0G692CTTLPmWgjEGF5oX0xZpcBh6k2waIxRCJ7pzxy4EVbO/jRMwG2Ojn9VJUOawybAsynwBfhgGCJeqK9fY6Rl5o7uRYHowV+8k97HUGXHYOaT1S2omj8oByWci3WHXhCNDSVpFB6aoHVc4K5QTjBS51GKJmqUBsL4FdKC6jeI64YKd5lkXNRREgUoaWCmALFS1BZrV45ajLSunJi/jpCVcsbV0cPk81JOeLAJH8eiDPXoRNRu4diZOmZtE1QZ5WY2a0xJ0lJc5UG6jVZPta2/9+tDMYu8EWO1mH06G9WOmRYB2m9jSKupEp9cqLc1gX94Vp+DS8lWv0ybZ87TiGdvE8I7t/A6s1+ywIbxry0u8efaobfr1vP17rzDDmCyI5tlXOdqyJxrFYAWVr/yY6MMbFBN6BSh4Bfx8NvXDWjXFzhsMR9GZxpTXl7CwJUUj/n/JqSjf+ElWXwGqhHnD5hdCTNOpQmHZra3TR4YgALH54dg0CpIQNzxFs2FVL7xEqKrUHal8wiUClYXuONUTLRGoKq0jnU8UMasLOqmtqLLDVJY4cUgxr71YJLt77XnU0N4ka5VC7bhTXZ9JNCxL23CnuMeSyVOUHt6aD7XefdZ6X+usPECSlkj+5I5Z1zlt9U3+hEiEyIvcMcT3QLJ8qvJ2ulSVdFHmjiK91xGhpA/uePkbHBEpLzqoXSU68723lnBE7ba/bACHrkyr6LaKpDta25A+jFrWHT16kaj5/OVAJBeQ3ZCpv3YVLs4ldVYVH8ziq2eyYQyzHqyNeTYh7GbJdda5vVmWbsZFKOlDG7z89lvGygtb4JS32RJQWbp/MsqHcgMjpTCiC+ukBq2cfB54MN3fa8KOdXk5cVKA1LCTy6wcRMfx5bGRjuNrPSpmzHyU3qrMCH8oUd4Eu85aDc90Ios+qOrs5IiN2nkzhlWxRUsPhTm2SPFviz3mELOTytE1dD5zWre1OW7ccYE0gIPqIXP8/KAY5SSG4ahVj7y7O1BNqsnuuBlXyhRSPxTlZL1QOCgu9dNOtRsItUrphpU3EcqNw3ke/W9Or69dB2RVRh6b/zvy+VXAfEsoDMe8wnj+czANEHP8qgp3AKMlJDR7cDU6PeZZT1KO/uHky08I8QPN7YkhaV5et72nruN3EC/eQFx/YN4hSy47q8rAfwrB5isRrUNWeS8sNXO8BVjH7PAOS9rvPTfChZK2vdhuuaBqjk3PBZXzXRjWa5ry0m1JJdUnyKD2UmiGfbi5GP07bXjmzf75IrY98tKQyJl37P1naCZ8ISzQ5jCnPfVIVzZSqX0Kcj+oelpxPzw5Vbgflib9t9TVDjwXmpc0PzliDkX24cx7ZiTntFdyifskCvuMc9QpdaExf/VFZ6Icpq1Fkve52dD13Pem0/qH3fyDmgA1lbKLEvnNmJHGJ9R7oNRgruFgbmEtKbCZQ51TAr+MHdzfbmtS7nqtoZJW1w9LSZ3rbH8ORxMMlXwzbKpN9ma8Y3INg2R7BHI3EoGAqXBCY4Dq9zqPMcILtAZBbS71uIvL5uNCLjHVL1dwDTHfVdqpunTYFHoqe1A0QpMsdp2LNCw9xJyCHolMH4Mg1tcpB8uQ1klZpguJHulRfVg4EFucXobsTq84deZ8TbJP6rRPUxvW+DT/lsCXol/s71cOQMHo8/Xqz/3VNTWm4zVl42U3Bcyze40YATJnqnO6XnO2nq47pYprh/Z8PmtHLdP9HLL9TN31ygh0TAjU9b2fxEGrkA8pu1CfUOSWoyQyQPPY6EPkDvYRQn0f6F7PfJiUQPM2M71MdWKJm3APOsuvzyaRdYZLQuAwyXv1K3JmfIXfqWdeAEGrCoLf/GO4kMxuWWeGl1HhAygjKqooYY07SIHPbPJlTNESLCj7vICEiD/4ch2+8h8JybYKmzIMXwMp64t7Ebb+0wxFecznD2m4iAwxBTZMxMM6D/iHBAV+Oe4bTajPAMHdkzykxteS8tDaalsi1X/yzASUi6/0qp5huA4YGHnAc/AOu4ztM4G3cAUW2+KlgxmkeSFksZ9fIbCKQUhyjKo9+5Nx2A833/0CJnntaK5hAAA=</value>
</data>
<data name="DefaultSchema" xml:space="preserve">
<value>dbo</value>
</data>
</root>
Core/Remote/PiscalLeafInput.cs
+2 -2
View File
@@ -10,7 +10,7 @@ namespace LeafWeb.Core.Remote
private static readonly IMapper Mapper; private static readonly IMapper Mapper;
public int LeafInputId { get; set; } public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; } public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; } public string PiscalDirectoryName { get; set; }
public bool SuppressStorageCopy { get; set; } public bool SuppressStorageCopy { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; } public PiscalLeafInputFile[] InputFiles { get; set; }
public string NotifyCompleteUrl { get; set; } public string NotifyCompleteUrl { get; set; }
@@ -21,7 +21,7 @@ namespace LeafWeb.Core.Remote
new MapperConfiguration(cfg => new MapperConfiguration(cfg =>
{ {
cfg.CreateMap<LeafInput, PiscalLeafInput>() cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName, .ForMember(dest => dest.PiscalDirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src))) opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id)) .ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray())) .ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
Core/Remote/PiscalLeafOutputFile.cs
+22 -3
View File
@@ -1,3 +1,4 @@
using System;
using AutoMapper; using AutoMapper;
using LeafWeb.Core.Entities; using LeafWeb.Core.Entities;
@@ -9,15 +10,33 @@ namespace LeafWeb.Core.Remote
public string Filename { get; set; } public string Filename { get; set; }
public byte[] Contents { get; set; } public byte[] Contents { get; set; }
public string DirectoryName { get; set; } public string PiscalDirectoryName { get; set; }
public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName); public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(PiscalDirectoryName);
// valid values: "touser", "nottouser", "clninput"
public string OutputFileType { get; set; }
static PiscalLeafOutputFile() static PiscalLeafOutputFile()
{ {
var config = var config =
new MapperConfiguration(cfg => new MapperConfiguration(cfg =>
{ {
cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>(); cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>()
.ForMember(dest => dest.FileType, opt =>
opt.ResolveUsing(src =>
{
switch (src.OutputFileType)
{
case "touser":
return LeafOutputFileType.ToUser;
case "nottouser":
return LeafOutputFileType.NotToUser;
case "clninput":
return LeafOutputFileType.CleanedInput;
default:
throw new ArgumentException();
}
}));
}); });
Mapper = config.CreateMapper(); Mapper = config.CreateMapper();
} }
Core/Remote/PiscalSshClient.cs
+14 -7
View File
@@ -59,7 +59,7 @@ namespace LeafWeb.Core.Remote
public void RunLeafInput(PiscalLeafInput leafInput) public void RunLeafInput(PiscalLeafInput leafInput)
{ {
var inputDirectory = $"{BaseDirectory}/{leafInput.DirectoryName}"; var inputDirectory = $"{BaseDirectory}/{leafInput.PiscalDirectoryName}";
CopyLeafInput(leafInput, inputDirectory); CopyLeafInput(leafInput, inputDirectory);
@@ -67,7 +67,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient()) using (var ssh = GetSshClient())
{ {
ssh.Connect(); ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType} -s"; var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -p {leafInput.PhotosyntheticType} -s";
if (leafInput.SuppressStorageCopy) if (leafInput.SuppressStorageCopy)
commandText += " -t"; commandText += " -t";
@@ -80,7 +80,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect(); ssh.Disconnect();
if (command.ExitStatus != 0) if (command.ExitStatus != 0)
throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine()); throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
_logger.Debug("RunLeafInput result: " + command.Result.TrimEndNewLine()); _logger.Debug("RunLeafInput result: " + command.Result.TrimEndNewLine());
} }
@@ -108,13 +108,13 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient()) using (var ssh = GetSshClient())
{ {
ssh.Connect(); ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName}"; var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName}";
var command = ssh.CreateCommand(commandText); var command = ssh.CreateCommand(commandText);
command.Execute(); command.Execute();
ssh.Disconnect(); ssh.Disconnect();
if (command.ExitStatus != 0) if (command.ExitStatus != 0)
throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine()); throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
return command.Result return command.Result
.SplitNewLine() .SplitNewLine()
@@ -138,8 +138,14 @@ namespace LeafWeb.Core.Remote
using (var scp = GetScpClient()) using (var scp = GetScpClient())
{ {
scp.Connect(); scp.Connect();
string outputFileType = string.Empty;
foreach (var filePath in filePaths) foreach (var filePath in filePaths)
{ {
if (filePath.StartsWith("#"))
{
outputFileType = filePath.Substring(1);
continue;
}
using (var stream = new MemoryStream()) using (var stream = new MemoryStream())
{ {
scp.Download(filePath, stream); scp.Download(filePath, stream);
@@ -148,7 +154,8 @@ namespace LeafWeb.Core.Remote
{ {
Contents = stream.ToArray(), Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(), Filename = filePath.FilenameFromPath(),
DirectoryName = leafInput.DirectoryName PiscalDirectoryName = leafInput.PiscalDirectoryName,
OutputFileType = outputFileType
}; };
} }
} }
@@ -162,7 +169,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient()) using (var ssh = GetSshClient())
{ {
ssh.Connect(); ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -c"; var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -c";
var command = ssh.CreateCommand(commandText); var command = ssh.CreateCommand(commandText);
command.Execute(); command.Execute();
ssh.Disconnect(); ssh.Disconnect();
Core/Remote/piscal_launcher.sh
+2 -2
View File
@@ -89,8 +89,7 @@ pushd $run_directory >> /dev/null
if [ -z "$test_output_directory" ]; then if [ -z "$test_output_directory" ]; then
eval $piscal_executable eval $piscal_executable
else else
cp "$test_output_directory"/* "$output_directory_touser"/ cp -r "$test_output_directory"/* "$output_directory_base"/
cp "$test_output_directory"/* "$output_directory_nottouser"/
fi fi
popd >> /dev/null popd >> /dev/null
@@ -99,6 +98,7 @@ popd >> /dev/null
if [ -z "$suppress_storage_copy" ]; then if [ -z "$suppress_storage_copy" ]; then
cp "$output_directory_touser"/* "$storage_directory"/ cp "$output_directory_touser"/* "$storage_directory"/
cp "$output_directory_nottouser"/* "$storage_directory"/ cp "$output_directory_nottouser"/* "$storage_directory"/
cp "$cleaned_input_directory"/* "$storage_directory"/
fi fi
# notify given url of completion # notify given url of completion
Core/Remote/piscal_manager.sh
+15 -4
View File
@@ -27,8 +27,9 @@ task="get_status" # default task
input_directory_name="input" input_directory_name="input"
pid_filename="piscal.pid" pid_filename="piscal.pid"
stderr_filename="piscal.err" stderr_filename="piscal.err"
output_directory_name="output/fitresult/touser"
cleaned_input_directory_name="output/clninput" cleaned_input_directory_name="output/clninput"
output_directory_name="output/fitresult/touser"
nottouser_directory_name="output/fitresult/nottouser"
launcher="$base_directory/piscal_launcher.sh" launcher="$base_directory/piscal_launcher.sh"
# http://stackoverflow.com/a/14203146/99492 # http://stackoverflow.com/a/14203146/99492
@@ -111,7 +112,7 @@ if [ "$task" = "launch" ]; then
# write the PID to a temp file to check for completion later # write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename echo $! > $working_directory/$pid_filename
echo started echo started
elif [ "$task" = "get_status" ]; then elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then
# if the pid doesn't exist, then process hasn't started # if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then if [ ! -f "$pid_path" ]; then
echo "not started" echo "not started"
@@ -137,11 +138,21 @@ elif [ "$task" = "get_status" ]; then
fi fi
echo complete echo complete
find "$output_directory"/* if [ "$task" = "get_status" ]; then
echo "#touser"
find "$output_directory"/*
cleaned_input_directory="$working_directory/$cleaned_input_directory_name"
echo "#clninput"
find "$cleaned_input_directory"/*
nottouser_directory="$working_directory/$nottouser_directory_name"
echo "#nottouser"
find "$nottouser_directory"/*
fi
fi fi
elif [ "$task" = "cleanup" ]; then elif [ "$task" = "cleanup" ]; then
rm -rf "$working_directory" rm -rf "$working_directory"
echo hi
elif [ "$task" = "kill" ]; then elif [ "$task" = "kill" ]; then
# if the pid doesn't exist, then process hasn't started # if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then if [ ! -f "$pid_path" ]; then
Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs
+1 -1
View File
@@ -75,7 +75,7 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus
var viewModel = new ResultStatusViewModel(leafInput); var viewModel = new ResultStatusViewModel(leafInput);
Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Exception.ToString())); Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Exception.ToString()));
Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description)); //Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description));
} }
} }
} }
Web/Services/EmailNotificationService.cs
+11
View File
@@ -19,6 +19,7 @@ namespace LeafWeb.Web.Services
private const string EmailSuccessSubject = "LeafWeb Results"; private const string EmailSuccessSubject = "LeafWeb Results";
private const string EmailErrorSubject = "LeafWeb processing error"; private const string EmailErrorSubject = "LeafWeb processing error";
private const string EmailSystemErrorSubject = "LeafWeb system error";
/// <summary> /// <summary>
/// Comma separated values /// Comma separated values
@@ -134,6 +135,16 @@ namespace LeafWeb.Web.Services
SendMessage(message); SendMessage(message);
} }
public void SendLeafWebSystemException(string leafInputIdentifier, string leafInputEmail)
{
var body = $"A system error occured while processing your leaf analysis job, {leafInputIdentifier}." + Environment.NewLine
+ "System administrators have been notified. You will be notified again when the system error "
+ "has been resolved and your data has been processed.";
var message = new MailMessage(_emailFromAddress, leafInputEmail, EmailSystemErrorSubject, body);
SendMessage(message);
}
private string FormatWarningMessage(LeafInput leafInput) private string FormatWarningMessage(LeafInput leafInput)
{ {
if (leafInput.OutputWarningMessage != null) if (leafInput.OutputWarningMessage != null)
Web/Services/PiscalQueue/PiscalQueueBase.cs
+10 -6
View File
@@ -31,24 +31,28 @@ namespace LeafWeb.Web.Services.PiscalQueue
protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { } protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
protected string FormatException(Exception ex, int leafInputId) protected string FormatException(Exception ex)
{ {
return return
$"LeafInput: {leafInputId}{Environment.NewLine}" + (ex is PiscalClientException ? $"LeafInput: {((PiscalClientException) ex).LeafInputId}{Environment.NewLine}" : "") +
$"Class: {GetType().Name}{Environment.NewLine}" + $"Class: {GetType().Name}{Environment.NewLine}" +
$"Exception: {ex.Message}{Environment.NewLine}" + $"Exception message: {ex.Message}{Environment.NewLine}" +
(ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty) (ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty)
+ $"StackTrace: {ex.StackTrace}"; + $"StackTrace: {ex.StackTrace}";
} }
protected void PiscalExceptionHandle(PiscalClientException ex, LeafInput leafInput) protected void PiscalExceptionHandler(PiscalClientException ex, LeafInput leafInput)
{ {
var errorMessage = FormatException(ex, ex.LeafInputId); var errorMessage = FormatException(ex);
Logger.Error(errorMessage); Logger.Error(errorMessage);
// send admin an email
BackgroundJobEnqueueRetry<EmailNotificationService>( BackgroundJobEnqueueRetry<EmailNotificationService>(
email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage)); email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
// TODO send user email too // send user email too
BackgroundJobEnqueueRetry<EmailNotificationService>(
email => email.SendLeafWebSystemException(leafInput.Identifier, leafInput.Email));
if (leafInput != null) if (leafInput != null)
{ {
Web/Services/PiscalQueue/PiscalQueueManager.cs
+3 -3
View File
@@ -88,7 +88,7 @@ namespace LeafWeb.Web.Services.PiscalQueue
} }
catch (Exception ex) catch (Exception ex)
{ {
var errorMessage = FormatException(ex, pendingInputId); var errorMessage = FormatException(ex);
Logger.Error(errorMessage); Logger.Error(errorMessage);
DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage); DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
} }
@@ -124,11 +124,11 @@ namespace LeafWeb.Web.Services.PiscalQueue
} }
catch (PiscalClientException ex) catch (PiscalClientException ex)
{ {
PiscalExceptionHandle(ex, leafInput); PiscalExceptionHandler(ex, leafInput);
} }
catch (Exception ex) catch (Exception ex)
{ {
var errorMessage = FormatException(ex, leafInput.Id); var errorMessage = FormatException(ex);
Logger.Error(errorMessage); Logger.Error(errorMessage);
} }
} }
Web/Services/PiscalQueue/PiscalQueueWorker.cs
+2 -7
View File
@@ -16,19 +16,14 @@ namespace LeafWeb.Web.Services.PiscalQueue
} }
catch (PiscalClientException ex) catch (PiscalClientException ex)
{ {
PiscalExceptionHandle(ex, leafInput); PiscalExceptionHandler(ex, leafInput);
if (leafInput != null)
{
DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
ex.Message);
}
// signal to process next item // signal to process next item
HangfireStartup.TriggerPiscalProcessQueue(); HangfireStartup.TriggerPiscalProcessQueue();
} }
catch (Exception ex) catch (Exception ex)
{ {
var errorMessage = FormatException(ex, leafInputId); var errorMessage = FormatException(ex);
Logger.Error(errorMessage); Logger.Error(errorMessage);
throw; // this will retry via HangFire throw; // this will retry via HangFire
} }
Web/ViewModels/Results/ResultStatusViewModel.cs
+10 -9
View File
@@ -15,7 +15,7 @@ namespace LeafWeb.Web.ViewModels.Results
public string LeafInputPhotosynthesisType { get; set; } public string LeafInputPhotosynthesisType { get; set; }
public bool HasLeafChart { get; set; } public bool HasLeafChart { get; set; }
public string CurrentStatus { get; set; } public string CurrentStatus { get; set; }
public string[] ErrorMessages { get; set; } //public string[] ErrorMessages { get; set; }
//public string[] LeafOutputFilenames { get; set; } //public string[] LeafOutputFilenames { get; set; }
//public bool HasLeafChartOutputFile { get; set; } //public bool HasLeafChartOutputFile { get; set; }
@@ -41,14 +41,15 @@ namespace LeafWeb.Web.ViewModels.Results
.ForMember(dest => dest.LeafInputIdentifier, opt => opt.MapFrom(src => src.Identifier)) .ForMember(dest => dest.LeafInputIdentifier, opt => opt.MapFrom(src => src.Identifier))
.ForMember(dest => dest.LeafInputSiteId, opt => opt.MapFrom(src => src.SiteId)) .ForMember(dest => dest.LeafInputSiteId, opt => opt.MapFrom(src => src.SiteId))
.ForMember(dest => dest.LeafInputPhotosynthesisType, opt => opt.MapFrom(src => src.PhotosynthesisType.Name)) .ForMember(dest => dest.LeafInputPhotosynthesisType, opt => opt.MapFrom(src => src.PhotosynthesisType.Name))
.ForMember(dest => dest.ErrorMessages, //.ForMember(dest => dest.ErrorMessages,
opt => opt.ResolveUsing( // opt => opt.ResolveUsing(
src => // src =>
src.StatusHistory? // src.StatusHistory?
.Where(sh => sh.Status == LeafInputStatusType.Exception) // .Where(sh => sh.Status == LeafInputStatusType.Exception)
.Select(sh => sh.Description) // .Select(sh => sh.Description)
.ToArray() // .ToArray()
?? new string[] {})); // ?? new string[] {}))
;
}); });
Mapper = config.CreateMapper(); Mapper = config.CreateMapper();
} }