Add output file type, collect all files

Core.Tests/Remote/PiscalSshClientTests.cs

@@ -17,7 +17,7 @@ namespace LeafWeb.Core.Tests.Remote
 			new PiscalLeafInput
 			{
 				LeafInputId = 1,
-				DirectoryName = "TestDirectory3",
+				PiscalDirectoryName = "TestDirectory3",
 				PhotosyntheticType = "C4_photosynthesis_leafweb",
 				InputFiles = new[]
 				{

Core/Core.csproj

@@ -85,6 +85,7 @@
     <Compile Include="Entities\LeafGasComparison.cs" />
     <Compile Include="Entities\LeafInputStatusType.cs" />
     <Compile Include="Entities\LeafOutputFile.cs" />
+    <Compile Include="Entities\LeafOutputFileType.cs" />
     <Compile Include="Entities\PhotosynthesisType.cs" />
     <Compile Include="Migrations\201604151603282_InitialCreate.cs" />
     <Compile Include="Migrations\201604151603282_InitialCreate.Designer.cs">
@@ -94,6 +95,10 @@
     <Compile Include="Migrations\201605061427009_LeafInputUniqueToken.Designer.cs">
       <DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon>
     </Compile>
+    <Compile Include="Migrations\201605161458209_LeafOutputFileType.cs" />
+    <Compile Include="Migrations\201605161458209_LeafOutputFileType.Designer.cs">
+      <DependentUpon>201605161458209_LeafOutputFileType.cs</DependentUpon>
+    </Compile>
     <Compile Include="Migrations\Configuration.cs" />
     <Compile Include="Parsers\LeafGasComparisonParser.cs" />
     <Compile Include="Remote\IPiscalClient.cs" />
@@ -144,6 +149,9 @@
     <EmbeddedResource Include="Migrations\201605061427009_LeafInputUniqueToken.resx">
       <DependentUpon>201605061427009_LeafInputUniqueToken.cs</DependentUpon>
     </EmbeddedResource>
+    <EmbeddedResource Include="Migrations\201605161458209_LeafOutputFileType.resx">
+      <DependentUpon>201605161458209_LeafOutputFileType.cs</DependentUpon>
+    </EmbeddedResource>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 

Core/Entities/LeafInput.cs

@@ -2,7 +2,6 @@ using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 using System.ComponentModel.DataAnnotations.Schema;
-using System.Data;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
@@ -55,12 +54,12 @@ namespace LeafWeb.Core.Entities
 		/// <summary>
 		/// Contains all output files in a zip
 		/// </summary>
-		public byte[] GetOutputFileZip()
+		public byte[] GetOutputFileZip(LeafOutputFileType? fileType)
 		{
 			using (var compressedFileStream = new MemoryStream())
 			using (var archive = new ZipArchive(compressedFileStream, ZipArchiveMode.Create, true))
 			{
-				foreach (var outputFile in OutputFiles)
+				foreach (var outputFile in OutputFiles.Where(f => !fileType.HasValue || f.FileType==fileType.Value))
 				{
 					var entry = archive.CreateEntry(outputFile.Filename);
 					using (var originalFileStream = new MemoryStream(outputFile.Contents))

Core/Entities/LeafOutputFile.cs

@@ -14,6 +14,8 @@ namespace LeafWeb.Core.Entities
 
 		public string Filename { get; set; }
 
+		public LeafOutputFileType FileType { get; set; }
+
 		public byte[] Contents { get; set; }
 
 		public bool IsLeafChartFile => Filename?.Contains(Filename_LeafChart) ?? false;

Core/Entities/LeafOutputFileType.cs

@@ -0,0 +1,9 @@
+namespace LeafWeb.Core.Entities
+{
+	public enum LeafOutputFileType
+	{
+		ToUser,
+		NotToUser,
+		CleanedInput
+	}
+}

Core/Migrations/201605161458209_LeafOutputFileType.Designer.cs

@@ -0,0 +1,29 @@
+// <auto-generated />
+namespace LeafWeb.Core.Migrations
+{
+    using System.CodeDom.Compiler;
+    using System.Data.Entity.Migrations;
+    using System.Data.Entity.Migrations.Infrastructure;
+    using System.Resources;
+    
+    [GeneratedCode("EntityFramework.Migrations", "6.1.3-40302")]
+    public sealed partial class LeafOutputFileType : IMigrationMetadata
+    {
+        private readonly ResourceManager Resources = new ResourceManager(typeof(LeafOutputFileType));
+        
+        string IMigrationMetadata.Id
+        {
+            get { return "201605161458209_LeafOutputFileType"; }
+        }
+        
+        string IMigrationMetadata.Source
+        {
+            get { return null; }
+        }
+        
+        string IMigrationMetadata.Target
+        {
+            get { return Resources.GetString("Target"); }
+        }
+    }
+}

Core/Migrations/201605161458209_LeafOutputFileType.cs

@@ -0,0 +1,19 @@
+namespace LeafWeb.Core.Migrations
+{
+    using System;
+    using System.Data.Entity.Migrations;
+    
+    public partial class LeafOutputFileType : DbMigration
+    {
+        public override void Up()
+        {
+            AddColumn("dbo.LeafOutputFile", "FileType", c => c.Int(nullable: false));
+			Sql(@"UPDATE dbo.LeafOutputFile SET FileType = 0"); // default to "ToUser"
+		}
+
+		public override void Down()
+        {
+            DropColumn("dbo.LeafOutputFile", "FileType");
+        }
+    }
+}

Core/Migrations/201605161458209_LeafOutputFileType.resx

@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="utf-8"?>
+<root>
+  <!-- 
+    Microsoft ResX Schema 
+    
+    Version 2.0
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
+    associated with the data types.
+    
+    Example:
+    
+    ... ado.net/XML headers & schema ...
+    <resheader name="resmimetype">text/microsoft-resx</resheader>
+    <resheader name="version">2.0</resheader>
+    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+        <value>[base64 mime encoded serialized .NET Framework object]</value>
+    </data>
+    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+        <comment>This is a comment</comment>
+    </data>
+                
+    There are any number of "resheader" rows that contain simple 
+    name/value pairs.
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
+    mimetype set.
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
+    extensible. For a given mimetype the value must be set accordingly:
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
+    read any of the formats listed below.
+    
+    mimetype: application/x-microsoft.net.object.binary.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+            : and then encoded with base64 encoding.
+    
+    mimetype: application/x-microsoft.net.object.soap.base64
+    value   : The object must be serialized with 
+            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.bytearray.base64
+    value   : The object must be serialized into a byte array 
+            : using a System.ComponentModel.TypeConverter
+            : and then encoded with base64 encoding.
+    -->
+  <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
+    <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
+    <xsd:element name="root" msdata:IsDataSet="true">
+      <xsd:complexType>
+        <xsd:choice maxOccurs="unbounded">
+          <xsd:element name="metadata">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" />
+              </xsd:sequence>
+              <xsd:attribute name="name" use="required" type="xsd:string" />
+              <xsd:attribute name="type" type="xsd:string" />
+              <xsd:attribute name="mimetype" type="xsd:string" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="assembly">
+            <xsd:complexType>
+              <xsd:attribute name="alias" type="xsd:string" />
+              <xsd:attribute name="name" type="xsd:string" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="data">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+                <xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
+              <xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
+              <xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
+              <xsd:attribute ref="xml:space" />
+            </xsd:complexType>
+          </xsd:element>
+          <xsd:element name="resheader">
+            <xsd:complexType>
+              <xsd:sequence>
+                <xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
+              </xsd:sequence>
+              <xsd:attribute name="name" type="xsd:string" use="required" />
+            </xsd:complexType>
+          </xsd:element>
+        </xsd:choice>
+      </xsd:complexType>
+    </xsd:element>
+  </xsd:schema>
+  <resheader name="resmimetype">
+    <value>text/microsoft-resx</value>
+  </resheader>
+  <resheader name="version">
+    <value>2.0</value>
+  </resheader>
+  <resheader name="reader">
+    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <resheader name="writer">
+    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </resheader>
+  <data name="Target" xml:space="preserve">
+    <value>H4sIAAAAAAAEAO1c3W4rtxG+L9B3WOiqKRzJ9mmBxLATOLLdCPUfLJ+0dwatpWSiu1xlyTUkFH2yXvSR+goh95fkklzujxT5JDg3x1zyIzn8ODMccvT///7v/PtNGHjvMCYowhejk/HxyIN4EfkIry5GCV1+/c3o++/++Ifzaz/ceD8V9T7xeqwlJhejN0rXZ5MJWbzBEJBxiBZxRKIlHS+icAL8aHJ6fPzt5ORkAhnEiGF53vlTgikKYfoH+3Ma4QVc0wQEd5EPA5KXsy/zFNW7ByEka7CAF6NbCJb/gK/jaRTD8dXl7ci7DBBg45jDYDnyAMYRBZSN8uwzgXMaR3g1X7MCEDxv15DVW4KAwHz0Z1V114kcn/KJTKqGBdQiITQKWwKefMolM1Gbd5LvqJQck901kzHd8lmn8rsY3QTJBkM6R5TNX+3vbBrEvK4i4RQEQTIWGh95/NNRSQfGGv6PFScBTWJ4gWFCYxAceY/Ja4AWf4fb5+hfEF/gJAjEIbJBsm9SASt6jKM1jOn2CS7lgc/8kTeRm0/U9mXretNsdowSjNsj7w5sbiFe0TfG+tNvRt4N2kC/KMk58hkjthVYIxon7M97NnzwGsDyuyC9OWUi+RvEMAYU+o+AUhgzCtxHGNYGrQyRi5T/zzJC9l+nEdo7mkZs28XbnfdzC7DPGtCBOzqfVIy28pwTeIbXCb1BQQemS833zvUuJK/YPcP002knps58mMqxaXG5UPB+2IopGxIpOvoBYcDJ69KRgnwP3tEqnb3K1GKpR94TDNIa5A2tM2NSEeGlpAMbzU0chU9RILYWvr88g3gFOfUjS6V5lMSL3vTuQe3faa2yLYljVnPO0JKScjILsm/8i2YkdvSB1HvLXq9DgIL9d5uJfIlgvP++uS212vod9cta/5zAdItYHY1d9H3p+7Cc8hVj/zMKHVCMSlFUdkNpxULhWbVioTpdB/qQ0MaRVnVeBL0pj1ZXR6vHtRV1utw26se3iEZki+kbJChXJ1Yx6xroxF2vZxG7pnJb8Wf68EfETBC3y7Y5ZFXNK6CtZDaltZq97Wm1st2MatX+d8v6azmMvKNsdyjWu1qcTsb7MBzRzopMpwOsGq/TJtIpqbYbqY7xwTbT4cUW9hJXmEcxfYj9yt8zKZV+x53CN+986MkAPhindq+gd3TmuYJkEaN1FsvcMQOvIGVHHbL7fkrvejBv2zEE0c2DMvp/Rl/LbYsmoXmDZmSZkZsArKpwe89N298YsIVmGirYMmKIG0tepTsYvsK4MGkQ+ymNfgJBwv4+ri2qVJ0NNKZi/RN7/acEY7H6qb36DcKIEUNo8MneYBqF6wDyUH9e/y/2+tcbfgeT7ta8wV/rdMgWvoEMqr81BBdkzH1T4Tn6TLh5c2PCfUSVBg1UmAYQYOjnG1vPB730LwmJFiiVorobxeO53Pc19j2Xs7rGHjCdyiSM1kymTDEwSYzH9bk14JfHOg1+dgKT+/izKghh0s2y0PrNtvHanWjTyaJh0G692CTTLPmWgjEGF5oX0xZpcBh6k2waIxRCJ7pzxy4EVbO/jRMwG2Ojn9VJUOawybAsynwBfhgGCJeqK9fY6Rl5o7uRYHowV+8k97HUGXHYOaT1S2omj8oByWci3WHXhCNDSVpFB6aoHVc4K5QTjBS51GKJmqUBsL4FdKC6jeI64YKd5lkXNRREgUoaWCmALFS1BZrV45ajLSunJi/jpCVcsbV0cPk81JOeLAJH8eiDPXoRNRu4diZOmZtE1QZ5WY2a0xJ0lJc5UG6jVZPta2/9+tDMYu8EWO1mH06G9WOmRYB2m9jSKupEp9cqLc1gX94Vp+DS8lWv0ybZ87TiGdvE8I7t/A6s1+ywIbxry0u8efaobfr1vP17rzDDmCyI5tlXOdqyJxrFYAWVr/yY6MMbFBN6BSh4Bfx8NvXDWjXFzhsMR9GZxpTXl7CwJUUj/n/JqSjf+ElWXwGqhHnD5hdCTNOpQmHZra3TR4YgALH54dg0CpIQNzxFs2FVL7xEqKrUHal8wiUClYXuONUTLRGoKq0jnU8UMasLOqmtqLLDVJY4cUgxr71YJLt77XnU0N4ka5VC7bhTXZ9JNCxL23CnuMeSyVOUHt6aD7XefdZ6X+usPECSlkj+5I5Z1zlt9U3+hEiEyIvcMcT3QLJ8qvJ2ulSVdFHmjiK91xGhpA/uePkbHBEpLzqoXSU68723lnBE7ba/bACHrkyr6LaKpDta25A+jFrWHT16kaj5/OVAJBeQ3ZCpv3YVLs4ldVYVH8ziq2eyYQyzHqyNeTYh7GbJdda5vVmWbsZFKOlDG7z89lvGygtb4JS32RJQWbp/MsqHcgMjpTCiC+ukBq2cfB54MN3fa8KOdXk5cVKA1LCTy6wcRMfx5bGRjuNrPSpmzHyU3qrMCH8oUd4Eu85aDc90Ios+qOrs5IiN2nkzhlWxRUsPhTm2SPFviz3mELOTytE1dD5zWre1OW7ccYE0gIPqIXP8/KAY5SSG4ahVj7y7O1BNqsnuuBlXyhRSPxTlZL1QOCgu9dNOtRsItUrphpU3EcqNw3ke/W9Or69dB2RVRh6b/zvy+VXAfEsoDMe8wnj+czANEHP8qgp3AKMlJDR7cDU6PeZZT1KO/uHky08I8QPN7YkhaV5et72nruN3EC/eQFx/YN4hSy47q8rAfwrB5isRrUNWeS8sNXO8BVjH7PAOS9rvPTfChZK2vdhuuaBqjk3PBZXzXRjWa5ry0m1JJdUnyKD2UmiGfbi5GP07bXjmzf75IrY98tKQyJl37P1naCZ8ISzQ5jCnPfVIVzZSqX0Kcj+oelpxPzw5Vbgflib9t9TVDjwXmpc0PzliDkX24cx7ZiTntFdyifskCvuMc9QpdaExf/VFZ6Icpq1Fkve52dD13Pem0/qH3fyDmgA1lbKLEvnNmJHGJ9R7oNRgruFgbmEtKbCZQ51TAr+MHdzfbmtS7nqtoZJW1w9LSZ3rbH8ORxMMlXwzbKpN9ma8Y3INg2R7BHI3EoGAqXBCY4Dq9zqPMcILtAZBbS71uIvL5uNCLjHVL1dwDTHfVdqpunTYFHoqe1A0QpMsdp2LNCw9xJyCHolMH4Mg1tcpB8uQ1klZpguJHulRfVg4EFucXobsTq84deZ8TbJP6rRPUxvW+DT/lsCXol/s71cOQMHo8/Xqz/3VNTWm4zVl42U3Bcyze40YATJnqnO6XnO2nq47pYprh/Z8PmtHLdP9HLL9TN31ygh0TAjU9b2fxEGrkA8pu1CfUOSWoyQyQPPY6EPkDvYRQn0f6F7PfJiUQPM2M71MdWKJm3APOsuvzyaRdYZLQuAwyXv1K3JmfIXfqWdeAEGrCoLf/GO4kMxuWWeGl1HhAygjKqooYY07SIHPbPJlTNESLCj7vICEiD/4ch2+8h8JybYKmzIMXwMp64t7Ebb+0wxFecznD2m4iAwxBTZMxMM6D/iHBAV+Oe4bTajPAMHdkzykxteS8tDaalsi1X/yzASUi6/0qp5huA4YGHnAc/AOu4ztM4G3cAUW2+KlgxmkeSFksZ9fIbCKQUhyjKo9+5Nx2A833/0CJnntaK5hAAA=</value>
+  </data>
+  <data name="DefaultSchema" xml:space="preserve">
+    <value>dbo</value>
+  </data>
+</root>

Core/Remote/PiscalLeafInput.cs

@@ -10,7 +10,7 @@ namespace LeafWeb.Core.Remote
 		private static readonly IMapper Mapper;
 		public int LeafInputId { get; set; }
 		public string PhotosyntheticType { get; set; }
-		public string DirectoryName { get; set; }
+		public string PiscalDirectoryName { get; set; }
 		public bool SuppressStorageCopy { get; set; }
 		public PiscalLeafInputFile[] InputFiles { get; set; }
 		public string NotifyCompleteUrl { get; set; }
@@ -21,7 +21,7 @@ namespace LeafWeb.Core.Remote
 				new MapperConfiguration(cfg =>
 				{
 					cfg.CreateMap<LeafInput, PiscalLeafInput>()
-						.ForMember(dest => dest.DirectoryName,
+						.ForMember(dest => dest.PiscalDirectoryName,
 							opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
 						.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
 						.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))

Core/Remote/PiscalLeafOutputFile.cs

@@ -1,3 +1,4 @@
+using System;
 using AutoMapper;
 using LeafWeb.Core.Entities;
 
@@ -9,15 +10,33 @@ namespace LeafWeb.Core.Remote
 
 		public string Filename { get; set; }
 		public byte[] Contents { get; set; }
-		public string DirectoryName { get; set; }
-		public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName);
+		public string PiscalDirectoryName { get; set; }
+		public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(PiscalDirectoryName);
+		// valid values: "touser", "nottouser", "clninput"
+		public string OutputFileType { get; set; }
 
 		static PiscalLeafOutputFile()
 		{
 			var config =
 				new MapperConfiguration(cfg =>
 				{
-					cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>();
+					cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>()
+						.ForMember(dest => dest.FileType, opt =>
+							opt.ResolveUsing(src =>
+							{
+								switch (src.OutputFileType)
+								{
+									case "touser":
+										return LeafOutputFileType.ToUser;
+									case "nottouser":
+										return LeafOutputFileType.NotToUser;
+									case "clninput":
+										return LeafOutputFileType.CleanedInput;
+									default:
+										throw new ArgumentException();
+								}
+								
+							}));
 				});
 			Mapper = config.CreateMapper();
 		}

Core/Remote/PiscalSshClient.cs

@@ -59,7 +59,7 @@ namespace LeafWeb.Core.Remote
 
 		public void RunLeafInput(PiscalLeafInput leafInput)
 		{
-			var inputDirectory = $"{BaseDirectory}/{leafInput.DirectoryName}";
+			var inputDirectory = $"{BaseDirectory}/{leafInput.PiscalDirectoryName}";
 
 			CopyLeafInput(leafInput, inputDirectory);
 
@@ -67,7 +67,7 @@ namespace LeafWeb.Core.Remote
 			using (var ssh = GetSshClient())
 			{
 				ssh.Connect();
-				var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType} -s";
+				var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -p {leafInput.PhotosyntheticType} -s";
 
 				if (leafInput.SuppressStorageCopy)
 					commandText += " -t";
@@ -80,7 +80,7 @@ namespace LeafWeb.Core.Remote
 				ssh.Disconnect();
 
 				if (command.ExitStatus != 0)
-					throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine());
+					throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
 
 				_logger.Debug("RunLeafInput result: " + command.Result.TrimEndNewLine());
 			}
@@ -108,13 +108,13 @@ namespace LeafWeb.Core.Remote
 			using (var ssh = GetSshClient())
 			{
 				ssh.Connect();
-				var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName}";
+				var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName}";
 				var command = ssh.CreateCommand(commandText);
 				command.Execute();
 				ssh.Disconnect();
 
 				if (command.ExitStatus != 0)
-					throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine());
+					throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
 
 				return command.Result
 					.SplitNewLine()
@@ -138,8 +138,14 @@ namespace LeafWeb.Core.Remote
 			using (var scp = GetScpClient())
 			{
 				scp.Connect();
+				string outputFileType = string.Empty;
 				foreach (var filePath in filePaths)
 				{
+					if (filePath.StartsWith("#"))
+					{
+						outputFileType = filePath.Substring(1);
+						continue;
+					}
 					using (var stream = new MemoryStream())
 					{
 						scp.Download(filePath, stream);
@@ -148,7 +154,8 @@ namespace LeafWeb.Core.Remote
 							{
 								Contents = stream.ToArray(),
 								Filename = filePath.FilenameFromPath(),
-								DirectoryName = leafInput.DirectoryName
+								PiscalDirectoryName = leafInput.PiscalDirectoryName,
+								OutputFileType = outputFileType
 							};
 					}
 				}
@@ -162,7 +169,7 @@ namespace LeafWeb.Core.Remote
 			using (var ssh = GetSshClient())
 			{
 				ssh.Connect();
-				var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -c";
+				var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -c";
 				var command = ssh.CreateCommand(commandText);
 				command.Execute();
 				ssh.Disconnect();

Core/Remote/piscal_launcher.sh

@@ -89,8 +89,7 @@ pushd $run_directory >> /dev/null
 if [ -z "$test_output_directory" ]; then
     eval $piscal_executable
 else
-    cp "$test_output_directory"/* "$output_directory_touser"/
-    cp "$test_output_directory"/* "$output_directory_nottouser"/
+    cp -r "$test_output_directory"/* "$output_directory_base"/
 fi
 
 popd >> /dev/null
@@ -99,6 +98,7 @@ popd >> /dev/null
 if [ -z "$suppress_storage_copy" ]; then
     cp "$output_directory_touser"/* "$storage_directory"/
     cp "$output_directory_nottouser"/* "$storage_directory"/
+    cp "$cleaned_input_directory"/* "$storage_directory"/
 fi
 
 # notify given url of completion

Core/Remote/piscal_manager.sh

@@ -27,8 +27,9 @@ task="get_status" # default task
 input_directory_name="input"
 pid_filename="piscal.pid"
 stderr_filename="piscal.err"
-output_directory_name="output/fitresult/touser"
 cleaned_input_directory_name="output/clninput"
+output_directory_name="output/fitresult/touser"
+nottouser_directory_name="output/fitresult/nottouser"
 launcher="$base_directory/piscal_launcher.sh"
 
 # http://stackoverflow.com/a/14203146/99492
@@ -111,7 +112,7 @@ if [ "$task" = "launch" ]; then
     # write the PID to a temp file to check for completion later
     echo $! > $working_directory/$pid_filename
     echo started
-elif [ "$task" = "get_status" ]; then
+elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then
     # if the pid doesn't exist, then process hasn't started
     if [ ! -f "$pid_path" ]; then
         echo "not started"
@@ -137,11 +138,21 @@ elif [ "$task" = "get_status" ]; then
         fi
         
         echo complete
-        find "$output_directory"/*
+        if [ "$task" = "get_status" ]; then
+            echo "#touser"
+            find "$output_directory"/*
+            
+            cleaned_input_directory="$working_directory/$cleaned_input_directory_name"
+            echo "#clninput"
+            find "$cleaned_input_directory"/*
+            
+            nottouser_directory="$working_directory/$nottouser_directory_name"
+            echo "#nottouser"
+            find "$nottouser_directory"/*
+       fi
     fi
 elif [ "$task" = "cleanup" ]; then
     rm -rf "$working_directory"
-    echo hi
 elif [ "$task" = "kill" ]; then
     # if the pid doesn't exist, then process hasn't started
     if [ ! -f "$pid_path" ]; then

Web.Tests/ViewModels/ResultStatus/ResultStatusViewModelTests.cs

@@ -75,7 +75,7 @@ namespace LeafWeb.Web.Tests.ViewModels.ResultStatus
 			var viewModel = new ResultStatusViewModel(leafInput);
 			
 			Assert.That(viewModel.CurrentStatus, Is.EqualTo(LeafInputStatusType.Exception.ToString()));
-			Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description));
+			//Assert.That(viewModel.ErrorMessages[0], Is.EqualTo(leafInput.StatusHistory.First().Description));
 		}
 	}
 }

Web/Services/EmailNotificationService.cs

@@ -19,6 +19,7 @@ namespace LeafWeb.Web.Services
 
 		private const string EmailSuccessSubject = "LeafWeb Results";
 		private const string EmailErrorSubject = "LeafWeb processing error";
+		private const string EmailSystemErrorSubject = "LeafWeb system error";
 
 		/// <summary>
 		/// Comma separated values
@@ -134,6 +135,16 @@ namespace LeafWeb.Web.Services
 			SendMessage(message);
 		}
 
+		public void SendLeafWebSystemException(string leafInputIdentifier, string leafInputEmail)
+		{
+			var body = $"A system error occured while processing your leaf analysis job, {leafInputIdentifier}." + Environment.NewLine
+					   + "System administrators have been notified. You will be notified again when the system error "
+					   + "has been resolved and your data has been processed.";
+
+			var message = new MailMessage(_emailFromAddress, leafInputEmail, EmailSystemErrorSubject, body);
+			SendMessage(message);
+		}
+
 		private string FormatWarningMessage(LeafInput leafInput)
 		{
 			if (leafInput.OutputWarningMessage != null)

Web/Services/PiscalQueue/PiscalQueueBase.cs

@@ -31,24 +31,28 @@ namespace LeafWeb.Web.Services.PiscalQueue
 
 		protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
 
-		protected string FormatException(Exception ex, int leafInputId)
+		protected string FormatException(Exception ex)
 		{
 			return
-				$"LeafInput: {leafInputId}{Environment.NewLine}" +
+				(ex is PiscalClientException ? $"LeafInput: {((PiscalClientException) ex).LeafInputId}{Environment.NewLine}" : "") +
 				$"Class: {GetType().Name}{Environment.NewLine}" + 
-				$"Exception: {ex.Message}{Environment.NewLine}" +
+				$"Exception message: {ex.Message}{Environment.NewLine}" +
 				(ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty)
 				+ $"StackTrace: {ex.StackTrace}";
 		}
 
-		protected void PiscalExceptionHandle(PiscalClientException ex, LeafInput leafInput)
+		protected void PiscalExceptionHandler(PiscalClientException ex, LeafInput leafInput)
 		{
-			var errorMessage = FormatException(ex, ex.LeafInputId);
+			var errorMessage = FormatException(ex);
 			Logger.Error(errorMessage);
+
+			// send admin an email
 			BackgroundJobEnqueueRetry<EmailNotificationService>(
 				email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
 
-			// TODO send user email too
+			// send user email too
+			BackgroundJobEnqueueRetry<EmailNotificationService>(
+				email => email.SendLeafWebSystemException(leafInput.Identifier, leafInput.Email));
 
 			if (leafInput != null)
 			{

Web/Services/PiscalQueue/PiscalQueueManager.cs

@@ -88,7 +88,7 @@ namespace LeafWeb.Web.Services.PiscalQueue
 			}
 			catch (Exception ex)
 			{
-				var errorMessage = FormatException(ex, pendingInputId);
+				var errorMessage = FormatException(ex);
 				Logger.Error(errorMessage);
 				DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Exception, ex.Message, errorMessage);
 			}
@@ -124,11 +124,11 @@ namespace LeafWeb.Web.Services.PiscalQueue
 				}
 				catch (PiscalClientException ex)
 				{
-					PiscalExceptionHandle(ex, leafInput);
+					PiscalExceptionHandler(ex, leafInput);
 				}
 				catch (Exception ex)
 				{
-					var errorMessage = FormatException(ex, leafInput.Id);
+					var errorMessage = FormatException(ex);
 					Logger.Error(errorMessage);
 				}
 			}

Web/Services/PiscalQueue/PiscalQueueWorker.cs

@@ -16,19 +16,14 @@ namespace LeafWeb.Web.Services.PiscalQueue
 			}
 			catch (PiscalClientException ex)
 			{
-				PiscalExceptionHandle(ex, leafInput);
+				PiscalExceptionHandler(ex, leafInput);
 
-				if (leafInput != null)
-				{
-					DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
-						ex.Message);
-				}
 				// signal to process next item
 				HangfireStartup.TriggerPiscalProcessQueue();
 			}
 			catch (Exception ex)
 			{
-				var errorMessage = FormatException(ex, leafInputId);
+				var errorMessage = FormatException(ex);
 				Logger.Error(errorMessage);
 				throw; // this will retry via HangFire
 			}

Web/ViewModels/Results/ResultStatusViewModel.cs

@@ -15,7 +15,7 @@ namespace LeafWeb.Web.ViewModels.Results
 		public string LeafInputPhotosynthesisType { get; set; }
 		public bool HasLeafChart { get; set; }
 		public string CurrentStatus { get; set; }
-		public string[] ErrorMessages { get; set; }
+		//public string[] ErrorMessages { get; set; }
 		//public string[] LeafOutputFilenames { get; set; }
 		//public bool HasLeafChartOutputFile { get; set; }
 
@@ -41,14 +41,15 @@ namespace LeafWeb.Web.ViewModels.Results
 					.ForMember(dest => dest.LeafInputIdentifier, opt => opt.MapFrom(src => src.Identifier))
 					.ForMember(dest => dest.LeafInputSiteId, opt => opt.MapFrom(src => src.SiteId))
 					.ForMember(dest => dest.LeafInputPhotosynthesisType, opt => opt.MapFrom(src => src.PhotosynthesisType.Name))
-					.ForMember(dest => dest.ErrorMessages, 
-						opt => opt.ResolveUsing(
-							src =>
-								src.StatusHistory?
-								.Where(sh => sh.Status == LeafInputStatusType.Exception)
-								.Select(sh => sh.Description)
-								.ToArray() 
-								?? new string[] {}));
+					//.ForMember(dest => dest.ErrorMessages, 
+					//	opt => opt.ResolveUsing(
+					//		src =>
+					//			src.StatusHistory?
+					//			.Where(sh => sh.Status == LeafInputStatusType.Exception)
+					//			.Select(sh => sh.Description)
+					//			.ToArray() 
+					//			?? new string[] {}))
+								;
 			});
 			Mapper = config.CreateMapper();
 		}