Piscal client - check status and retrieve file skeleton in place

Core.Tests/Remote/PiscalSshClientTests.cs

@@ -1,5 +1,6 @@
-using System.Configuration;
+using System;
-using LeafWeb.Core.Entities;
+using System.Configuration;
+using System.Linq;
 using LeafWeb.Core.Remote;
 using LeafWeb.Core.Utility;
 using NUnit.Framework;
@@ -9,13 +10,38 @@ namespace LeafWeb.Core.Tests.Remote
 	[TestFixture]
 	public class PiscalSshClientTests
 	{
+		private readonly PiscalLeafInputFile _testInput =
+			new PiscalLeafInputFile
+			{
+				Filename = "blah",
+				LeafInputId = 1,
+				Contents = "test".GetBytes(),
+				DirectoryName = "TestDirectory"
+			};
+
+		private readonly string _piscalConnectionString =
+			ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString;
+
 		[Test]
-		public void Client()
+		public void SubmitLeafInputFile()
 		{
-			var leafInputFile = new PiscalLeafInputFile{Filename = "blah", Id = 1, Contents = "test".GetBytes(), DirectoryName = "TestDirectory"};
+			var client = new PiscalSshClient(_piscalConnectionString);
-			var piscalConnectionString = ConfigurationManager.ConnectionStrings["PiscalServer"];
+			client.SubmitLeafInputFile(_testInput);
-			var client = new PiscalSshClient(piscalConnectionString.ConnectionString);
+		}
-			client.SubmitLeafInputFile(leafInputFile);
+
+		[Test]
+		public void GetLeafInputStatus()
+		{
+			var client = new PiscalSshClient(_piscalConnectionString);
+			client.GetLeafInputStatus(_testInput);
+		}
+
+		[Test]
+		public void RetrieveLeafInputResult()
+		{
+			var client = new PiscalSshClient(_piscalConnectionString);
+			var result = client.RetrieveLeafInputResult(_testInput).ToList();
+			Console.WriteLine(result[0].Contents.GetString());
 		}
 	}
 }

Core/Core.csproj

@@ -86,7 +86,12 @@
     <Compile Include="Entities\LeafInputDataSite.cs" />
     <Compile Include="Entities\LeafInputStatus.cs" />
     <Compile Include="Parsers\FluxnetSiteCsvParser.cs" />
+    <Compile Include="Remote\PiscalClientException.cs" />
     <Compile Include="Remote\PiscalLeafInputFile.cs" />
+    <Compile Include="Remote\PiscalLeafOutputFile.cs" />
+    <Compile Include="Remote\PiscalSshClient.cs" />
+    <Compile Include="Remote\PiscalStatus.cs" />
+    <Compile Include="Remote\PiscalUtility.cs" />
     <Compile Include="Utility\BoolTypeConverter.cs" />
     <Compile Include="Parsers\CntrlComparisonParser.cs" />
     <Compile Include="Parsers\CsvParserBase.cs" />
@@ -105,6 +110,7 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <None Include="packages.config" />
+    <None Include="Remote\piscal_manager.sh" />
   </ItemGroup>
   <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

Core/Entities/LeafInput.cs

@@ -33,10 +33,5 @@ namespace LeafWeb.Core.Entities
 
 		[DataType(DataType.Date)]
 		public DateTime? Processed { get; set; }
-
-		/// <summary>
-		/// An cleaned up identifier to be used for file naming, etc
-		/// </summary>
-		public string CombinedIdentifier => $"{Name.FilterAlphaNumeric()}_{Identifier.FilterAlphaNumeric()}";
 	}
 }

Core/Entities/LeafInputFile.cs

@@ -14,7 +14,5 @@
 		public string Filename { get; set; }
 		
 		public byte[] Contents { get; set; }
-
-		public string CombinedIdentifier => $"{LeafInput.CombinedIdentifier}_{Id}";
 	}
 }

Core/Remote/IPiscalClient.cs

@@ -1,82 +1,11 @@
-using System;
+using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using LeafWeb.Core.Utility;
-using Renci.SshNet;
 
 namespace LeafWeb.Core.Remote
 {
 	public interface IPiscalClient
 	{
 		void SubmitLeafInputFile(PiscalLeafInputFile file);
-		void RetrieveLeafInputResult(PiscalLeafInputFile file);
+		PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
-	}
+		IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file);
-
-	public class PiscalSshClient : IPiscalClient
-	{
-		private const string BaseDirectory = "LeafInput";
-		private readonly PasswordConnectionInfo _connectionInfo;
-
-		public PiscalSshClient(string connectionString)
-		{
-			var conn = connectionString.SplitConnectionString();
-
-			var host = conn["host"];
-			var username = conn["username"];
-			var password = conn["password"];
-			_connectionInfo = new PasswordConnectionInfo(host, username, password);
-		}
-
-		private SshClient GetSshClient()
-		{
-			return new SshClient(_connectionInfo);
-		}
-
-		private ScpClient GetScpClient()
-		{
-			return new ScpClient(_connectionInfo);
-		}
-
-		public void SubmitLeafInputFile(PiscalLeafInputFile file)
-		{
-			var inputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename;
-			var outputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename;
-
-			using (var scp = GetScpClient())
-			using (var stream = new MemoryStream(file.Contents))
-			{
-				scp.Connect();
-				scp.Upload(stream, inputPath);
-			}
-
-			using (var ssh = GetSshClient())
-			{
-				ssh.Connect();
-				var lsCommend = ssh.CreateCommand("ls -r");
-				lsCommend.Execute();
-				//var extendedData = Encoding.ASCII.GetString(cmd.ExtendedOutputStream.ToArray());
-				var extendedData = new StreamReader(lsCommend.ExtendedOutputStream, Encoding.ASCII).ReadToEnd();
-				ssh.Disconnect();
-
-				Console.Write(lsCommend.Result);
-				Console.Write(extendedData);
-				//Assert.AreEqual("12345\n", cmd.Result);
-				//Assert.AreEqual("654321\n", extendedData);
-			}
-			// copy file
-
-			// begin processing
-		}
-
-		public void RetrieveLeafInputResult(PiscalLeafInputFile file)
-		{
-			using (var scp = GetScpClient())
-			using (var stream = new MemoryStream(file.Contents))
-			{
-				scp.Connect();
-				scp.Upload(stream, BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename);
-			}
-
-		}
 	}
 }

Core/Remote/PiscalClientException.cs

@@ -0,0 +1,11 @@
+using System;
+
+namespace LeafWeb.Core.Remote
+{
+	public class PiscalClientException : Exception
+	{
+		public PiscalClientException(string error) : base(error)
+		{
+		}
+	}
+}

Core/Remote/PiscalLeafInputFile.cs

@@ -1,6 +1,5 @@
 using AutoMapper;
 using LeafWeb.Core.Entities;
-using LeafWeb.Core.Utility;
 
 namespace LeafWeb.Core.Remote
 {
@@ -8,7 +7,7 @@ namespace LeafWeb.Core.Remote
 	{
 		private static readonly IMapper Mapper;
 
-		public int Id { get; set; }
+		public int LeafInputId { get; set; }
 		public string Filename { get; set; }
 		public byte[] Contents { get; set; }
 		public string DirectoryName { get; set; }
@@ -20,9 +19,8 @@ namespace LeafWeb.Core.Remote
 			{
 				cfg.CreateMap<LeafInputFile, PiscalLeafInputFile>()
 					.ForMember(dest => dest.DirectoryName, opt => opt.MapFrom(
-						src =>
+						src => PiscalUtility.GetPiscalDirectoryName(src)))
-						$"{src.Id}_{src.LeafInput.Name.FilterAlphaNumeric()}_{src.LeafInput.Identifier.FilterAlphaNumeric()}"
+					.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id));
-					));
 			});
 			Mapper = config.CreateMapper();
 		}

Core/Remote/PiscalLeafOutputFile.cs

@@ -0,0 +1,34 @@
+using AutoMapper;
+using LeafWeb.Core.Entities;
+
+namespace LeafWeb.Core.Remote
+{
+	public class PiscalLeafOutputFile
+	{
+		private static readonly IMapper Mapper;
+
+		public string Filename { get; set; }
+		public byte[] Contents { get; set; }
+		public string DirectoryName { get; set; }
+		public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName);
+
+		static PiscalLeafOutputFile()
+		{
+			var config =
+				new MapperConfiguration(cfg =>
+				{
+					cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>();
+				});
+			Mapper = config.CreateMapper();
+		}
+
+		public PiscalLeafOutputFile() { }
+
+		public LeafOutputFile GetLeafOutputFile()
+		{
+			var leafOutputFile = new LeafOutputFile();
+			Mapper.Map(this, leafOutputFile);
+			return leafOutputFile;
+		}
+	}
+}

Core/Remote/PiscalSshClient.cs

@@ -0,0 +1,118 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.IO;
+using System.Linq;
+using LeafWeb.Core.Utility;
+using Renci.SshNet;
+
+namespace LeafWeb.Core.Remote
+{
+	public class PiscalSshClient : IPiscalClient
+	{
+		private const string BaseDirectory = "./LeafWeb";
+		private const string RemoteScriptPath = "./piscal_manager.sh";
+		private readonly PasswordConnectionInfo _connectionInfo;
+
+		public PiscalSshClient(string connectionString)
+		{
+			var conn = new DbConnectionStringBuilder {ConnectionString = connectionString};
+
+			var host = conn["host"] as string;
+			var username = conn["username"] as string;
+			var password = conn["password"] as string;
+			_connectionInfo = new PasswordConnectionInfo(host, username, password);
+		}
+
+		private SshClient GetSshClient()
+		{
+			return new SshClient(_connectionInfo);
+		}
+
+		private ScpClient GetScpClient()
+		{
+			return new ScpClient(_connectionInfo);
+		}
+
+		public void SubmitLeafInputFile(PiscalLeafInputFile file)
+		{
+			var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}";
+
+			// copy file
+			using (var scp = GetScpClient())
+			using (var stream = new MemoryStream(file.Contents))
+			{
+				Console.WriteLine(inputPath);
+				scp.Connect();
+				scp.Upload(stream, inputPath);
+				scp.Disconnect();
+			}
+
+			// begin processing
+			using (var ssh = GetSshClient())
+			{
+				ssh.Connect();
+				var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
+				var exeCommand = ssh.CreateCommand(commandText);
+				exeCommand.Execute();
+				ssh.Disconnect();
+
+				Console.Write(exeCommand.Result);
+
+				if (exeCommand.ExitStatus != 0)
+					throw new PiscalClientException(exeCommand.Error);
+			}
+		}
+
+		public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
+		{
+			using (var ssh = GetSshClient())
+			{
+				ssh.Connect();
+				var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
+				var exeCommand = ssh.CreateCommand(commandText);
+				exeCommand.Execute();
+				ssh.Disconnect();
+
+				Console.Write(exeCommand.Result);
+			}
+			return PiscalStatus.Success;
+		}
+
+		public IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file)
+		{
+			var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
+
+			string[] filenames;
+
+			// get output files
+			using (var ssh = GetSshClient())
+			{
+				ssh.Connect();
+				var commandText = $"ls -1 {outputDirectory}";
+				var lsCommand = ssh.CreateCommand(commandText);
+				lsCommand.Execute();
+				ssh.Disconnect();
+
+				filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
+				Console.Write(lsCommand.Result);
+			}
+
+			using (var scp = GetScpClient())
+			{
+				scp.Connect();
+				foreach (var filename in filenames)
+				{
+					using (var stream = new MemoryStream())
+					{
+						var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
+						scp.Download(filePath, stream);
+						yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
+					}
+				}
+
+				scp.Disconnect();
+			}
+		}
+	}
+}

Core/Remote/PiscalStatus.cs

@@ -0,0 +1,9 @@
+namespace LeafWeb.Core.Remote
+{
+	public enum PiscalStatus
+	{
+		Running,
+		Success,
+		Error
+	}
+}

Core/Remote/PiscalUtility.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Text.RegularExpressions;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Utility;
+
+namespace LeafWeb.Core.Remote
+{
+	public static class PiscalUtility
+	{
+		public static string GetPiscalDirectoryName(int id, string name, string identifier)
+		{
+			return $"{id}_{name.FilterAlphaNumeric()}_{identifier.FilterAlphaNumeric()}";
+		}
+
+		public static string GetPiscalDirectoryName(LeafInputFile leafInputFile)
+		{
+			return GetPiscalDirectoryName(leafInputFile.Id, leafInputFile.LeafInput.Name, leafInputFile.LeafInput.Identifier);
+		}
+
+		public static int GetIdFromDirectoryName(string directoryName)
+		{
+			var match = Regex.Match(directoryName, @"\d_");
+			if (!match.Success)
+				throw new FormatException("DirectoryName expected to be formatted {number}_{name}_{identifier}: " + directoryName);
+			return int.Parse(match.Captures[0].Value);
+		}
+	}
+}

Core/Remote/piscal_manager.sh

@@ -0,0 +1,93 @@
+#~/bin/bash
+# piscal manager script
+
+usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
+
+where:
+    -h  show this help text
+    -d  working directory name
+    -o  input filename
+    -s  job status"
+
+# http://stackoverflow.com/a/14203146/99492
+# http://wiki.bash-hackers.org/howto/getopts_tutorial
+
+# Initialize variables:
+base_directory="/home/poprhythm/LeafWeb"
+directory_name=""
+input_filename=""
+get_status=false
+pid_filename="piscal.pid"
+out_filename="piscal.out"
+
+while getopts "hd:f:s" opt; do
+    #echo "$opt = $OPTARG"
+    case "$opt" in
+    h )
+        echo "$usage"
+        exit
+        ;;
+    d )
+        directory_name=$OPTARG
+        ;;
+    f )
+        input_filename=$OPTARG
+        ;;
+    s )
+        get_status=true
+        ;;
+    \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
+    esac
+done
+if [ -z "$directory_name" ]; then
+    echo "directory name required (-d)"
+    exit 1
+fi
+if [ ! -d "$base_directory/$directory_name" ]; then
+    echo "directory $base_directory/$directory_name not found"
+    exit 1
+fi
+if [ "$get_status" = false ]; then
+    if [ -z "$input_filename" ]; then
+        echo "input filename required (-f)"
+        exit 1
+    fi
+    
+    if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
+        echo "input filename $base_directory/$directory_name/$input_filename not found"
+        exit 1
+    fi
+fi
+
+if [ "$get_status" = true ]; then
+    pid_path="$base_directory/$directory_name/$pid_filename"
+    
+    # if the pid doesn't exist, then process never started
+    if [ ! -f "$pid_path" ]; then
+        echo "no pid file found"
+        exit 1
+    fi
+    
+    pid=$(head -n 1 $pid_path)
+    # if the pid exists, check the process status using ps
+    if ps -p $pid > /dev/null; then
+        # if it is in ps, then it's still running
+        echo running
+    else
+        # otherwise, it is complete, check the output for success/error
+        # TODO: examine output for errors, etc
+        echo complete
+    fi
+
+else
+    
+    command="sleep 100s"
+    nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
+
+    # write the PID to a temp file to check for completion later
+    echo $! > $base_directory/$directory_name/$pid_filename
+
+fi

Core/Utility/StringExtensions.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
@@ -16,6 +15,11 @@ namespace LeafWeb.Core.Utility
 				current + (char.IsUpper(c) && current.Length > 0 ? " " + c : c.ToString(CultureInfo.InvariantCulture)));
 		}
 
+		public static string[] SplitNewLine(this string str)
+		{
+			return str.Split(new[] {"\n", "\r\n"}, StringSplitOptions.None);
+		}
+
 		public static string LowercaseFirst(string s)
 		{
 			// Check for empty string.
@@ -29,23 +33,12 @@ namespace LeafWeb.Core.Utility
 
 		public static byte[] GetBytes(this string str)
 		{
-			var bytes = new byte[str.Length * sizeof(char)];
+			return System.Text.Encoding.Default.GetBytes(str);
-			Buffer.BlockCopy(str.ToCharArray(), 0, bytes, 0, bytes.Length);
-			return bytes;
 		}
 
 		public static string GetString(this byte[] bytes)
 		{
-			var chars = new char[bytes.Length / sizeof(char)];
+			return System.Text.Encoding.Default.GetString(bytes);
-			Buffer.BlockCopy(bytes, 0, chars, 0, bytes.Length);
-			return new string(chars);
-		}
-
-		public static Dictionary<string, string> SplitConnectionString(this string connectionString)
-		{
-			return connectionString.Split(';')
-				.Select(t => t.Split(new[] { '=' }, 2))
-				.ToDictionary(t => t[0].Trim(), t => t[1].Trim(), StringComparer.InvariantCultureIgnoreCase);
 		}
 
 		public static string FilterAlphaNumeric(this string input)