From 75fa448a0f476679650c44affc0fc35a4b005145 Mon Sep 17 00:00:00 2001 From: James Kolpack Date: Fri, 19 Feb 2016 11:45:16 -0500 Subject: [PATCH] Piscal client - check status and retrieve file skeleton in place --- Core.Tests/Remote/PiscalSshClientTests.cs | 40 ++++++-- Core/Core.csproj | 6 ++ Core/Entities/LeafInput.cs | 5 - Core/Entities/LeafInputFile.cs | 2 - Core/Remote/IPiscalClient.cs | 77 +------------- Core/Remote/PiscalClientException.cs | 11 ++ Core/Remote/PiscalLeafInputFile.cs | 8 +- Core/Remote/PiscalLeafOutputFile.cs | 34 +++++++ Core/Remote/PiscalSshClient.cs | 118 ++++++++++++++++++++++ Core/Remote/PiscalStatus.cs | 9 ++ Core/Remote/PiscalUtility.cs | 28 +++++ Core/Remote/piscal_manager.sh | 93 +++++++++++++++++ Core/Utility/StringExtensions.cs | 21 ++-- 13 files changed, 345 insertions(+), 107 deletions(-) create mode 100644 Core/Remote/PiscalClientException.cs create mode 100644 Core/Remote/PiscalLeafOutputFile.cs create mode 100644 Core/Remote/PiscalSshClient.cs create mode 100644 Core/Remote/PiscalStatus.cs create mode 100644 Core/Remote/PiscalUtility.cs create mode 100644 Core/Remote/piscal_manager.sh diff --git a/Core.Tests/Remote/PiscalSshClientTests.cs b/Core.Tests/Remote/PiscalSshClientTests.cs index 674f366..85650e0 100644 --- a/Core.Tests/Remote/PiscalSshClientTests.cs +++ b/Core.Tests/Remote/PiscalSshClientTests.cs @@ -1,5 +1,6 @@ -using System.Configuration; -using LeafWeb.Core.Entities; +using System; +using System.Configuration; +using System.Linq; using LeafWeb.Core.Remote; using LeafWeb.Core.Utility; using NUnit.Framework; @@ -9,13 +10,38 @@ namespace LeafWeb.Core.Tests.Remote [TestFixture] public class PiscalSshClientTests { + private readonly PiscalLeafInputFile _testInput = + new PiscalLeafInputFile + { + Filename = "blah", + LeafInputId = 1, + Contents = "test".GetBytes(), + DirectoryName = "TestDirectory" + }; + + private readonly string _piscalConnectionString = + ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString; + [Test] - public void Client() + public void SubmitLeafInputFile() { - var leafInputFile = new PiscalLeafInputFile{Filename = "blah", Id = 1, Contents = "test".GetBytes(), DirectoryName = "TestDirectory"}; - var piscalConnectionString = ConfigurationManager.ConnectionStrings["PiscalServer"]; - var client = new PiscalSshClient(piscalConnectionString.ConnectionString); - client.SubmitLeafInputFile(leafInputFile); + var client = new PiscalSshClient(_piscalConnectionString); + client.SubmitLeafInputFile(_testInput); + } + + [Test] + public void GetLeafInputStatus() + { + var client = new PiscalSshClient(_piscalConnectionString); + client.GetLeafInputStatus(_testInput); + } + + [Test] + public void RetrieveLeafInputResult() + { + var client = new PiscalSshClient(_piscalConnectionString); + var result = client.RetrieveLeafInputResult(_testInput).ToList(); + Console.WriteLine(result[0].Contents.GetString()); } } } diff --git a/Core/Core.csproj b/Core/Core.csproj index 300864d..247b711 100644 --- a/Core/Core.csproj +++ b/Core/Core.csproj @@ -86,7 +86,12 @@ + + + + + @@ -105,6 +110,7 @@ PreserveNewest + diff --git a/Core/Entities/LeafInput.cs b/Core/Entities/LeafInput.cs index 4634337..ea712cf 100644 --- a/Core/Entities/LeafInput.cs +++ b/Core/Entities/LeafInput.cs @@ -33,10 +33,5 @@ namespace LeafWeb.Core.Entities [DataType(DataType.Date)] public DateTime? Processed { get; set; } - - /// - /// An cleaned up identifier to be used for file naming, etc - /// - public string CombinedIdentifier => $"{Name.FilterAlphaNumeric()}_{Identifier.FilterAlphaNumeric()}"; } } \ No newline at end of file diff --git a/Core/Entities/LeafInputFile.cs b/Core/Entities/LeafInputFile.cs index 28d75ed..3726235 100644 --- a/Core/Entities/LeafInputFile.cs +++ b/Core/Entities/LeafInputFile.cs @@ -14,7 +14,5 @@ public string Filename { get; set; } public byte[] Contents { get; set; } - - public string CombinedIdentifier => $"{LeafInput.CombinedIdentifier}_{Id}"; } } \ No newline at end of file diff --git a/Core/Remote/IPiscalClient.cs b/Core/Remote/IPiscalClient.cs index 4f5c780..a92edb9 100644 --- a/Core/Remote/IPiscalClient.cs +++ b/Core/Remote/IPiscalClient.cs @@ -1,82 +1,11 @@ -using System; -using System.IO; -using System.Text; -using LeafWeb.Core.Utility; -using Renci.SshNet; +using System.Collections.Generic; namespace LeafWeb.Core.Remote { public interface IPiscalClient { void SubmitLeafInputFile(PiscalLeafInputFile file); - void RetrieveLeafInputResult(PiscalLeafInputFile file); - } - - public class PiscalSshClient : IPiscalClient - { - private const string BaseDirectory = "LeafInput"; - private readonly PasswordConnectionInfo _connectionInfo; - - public PiscalSshClient(string connectionString) - { - var conn = connectionString.SplitConnectionString(); - - var host = conn["host"]; - var username = conn["username"]; - var password = conn["password"]; - _connectionInfo = new PasswordConnectionInfo(host, username, password); - } - - private SshClient GetSshClient() - { - return new SshClient(_connectionInfo); - } - - private ScpClient GetScpClient() - { - return new ScpClient(_connectionInfo); - } - - public void SubmitLeafInputFile(PiscalLeafInputFile file) - { - var inputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename; - var outputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename; - - using (var scp = GetScpClient()) - using (var stream = new MemoryStream(file.Contents)) - { - scp.Connect(); - scp.Upload(stream, inputPath); - } - - using (var ssh = GetSshClient()) - { - ssh.Connect(); - var lsCommend = ssh.CreateCommand("ls -r"); - lsCommend.Execute(); - //var extendedData = Encoding.ASCII.GetString(cmd.ExtendedOutputStream.ToArray()); - var extendedData = new StreamReader(lsCommend.ExtendedOutputStream, Encoding.ASCII).ReadToEnd(); - ssh.Disconnect(); - - Console.Write(lsCommend.Result); - Console.Write(extendedData); - //Assert.AreEqual("12345\n", cmd.Result); - //Assert.AreEqual("654321\n", extendedData); - } - // copy file - - // begin processing - } - - public void RetrieveLeafInputResult(PiscalLeafInputFile file) - { - using (var scp = GetScpClient()) - using (var stream = new MemoryStream(file.Contents)) - { - scp.Connect(); - scp.Upload(stream, BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename); - } - - } + PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file); + IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file); } } diff --git a/Core/Remote/PiscalClientException.cs b/Core/Remote/PiscalClientException.cs new file mode 100644 index 0000000..cdb6b8d --- /dev/null +++ b/Core/Remote/PiscalClientException.cs @@ -0,0 +1,11 @@ +using System; + +namespace LeafWeb.Core.Remote +{ + public class PiscalClientException : Exception + { + public PiscalClientException(string error) : base(error) + { + } + } +} \ No newline at end of file diff --git a/Core/Remote/PiscalLeafInputFile.cs b/Core/Remote/PiscalLeafInputFile.cs index 3c4120a..422d41e 100644 --- a/Core/Remote/PiscalLeafInputFile.cs +++ b/Core/Remote/PiscalLeafInputFile.cs @@ -1,6 +1,5 @@ using AutoMapper; using LeafWeb.Core.Entities; -using LeafWeb.Core.Utility; namespace LeafWeb.Core.Remote { @@ -8,7 +7,7 @@ namespace LeafWeb.Core.Remote { private static readonly IMapper Mapper; - public int Id { get; set; } + public int LeafInputId { get; set; } public string Filename { get; set; } public byte[] Contents { get; set; } public string DirectoryName { get; set; } @@ -20,9 +19,8 @@ namespace LeafWeb.Core.Remote { cfg.CreateMap() .ForMember(dest => dest.DirectoryName, opt => opt.MapFrom( - src => - $"{src.Id}_{src.LeafInput.Name.FilterAlphaNumeric()}_{src.LeafInput.Identifier.FilterAlphaNumeric()}" - )); + src => PiscalUtility.GetPiscalDirectoryName(src))) + .ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id)); }); Mapper = config.CreateMapper(); } diff --git a/Core/Remote/PiscalLeafOutputFile.cs b/Core/Remote/PiscalLeafOutputFile.cs new file mode 100644 index 0000000..a7d8d3d --- /dev/null +++ b/Core/Remote/PiscalLeafOutputFile.cs @@ -0,0 +1,34 @@ +using AutoMapper; +using LeafWeb.Core.Entities; + +namespace LeafWeb.Core.Remote +{ + public class PiscalLeafOutputFile + { + private static readonly IMapper Mapper; + + public string Filename { get; set; } + public byte[] Contents { get; set; } + public string DirectoryName { get; set; } + public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName); + + static PiscalLeafOutputFile() + { + var config = + new MapperConfiguration(cfg => + { + cfg.CreateMap(); + }); + Mapper = config.CreateMapper(); + } + + public PiscalLeafOutputFile() { } + + public LeafOutputFile GetLeafOutputFile() + { + var leafOutputFile = new LeafOutputFile(); + Mapper.Map(this, leafOutputFile); + return leafOutputFile; + } + } +} \ No newline at end of file diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs new file mode 100644 index 0000000..5310ba1 --- /dev/null +++ b/Core/Remote/PiscalSshClient.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.IO; +using System.Linq; +using LeafWeb.Core.Utility; +using Renci.SshNet; + +namespace LeafWeb.Core.Remote +{ + public class PiscalSshClient : IPiscalClient + { + private const string BaseDirectory = "./LeafWeb"; + private const string RemoteScriptPath = "./piscal_manager.sh"; + private readonly PasswordConnectionInfo _connectionInfo; + + public PiscalSshClient(string connectionString) + { + var conn = new DbConnectionStringBuilder {ConnectionString = connectionString}; + + var host = conn["host"] as string; + var username = conn["username"] as string; + var password = conn["password"] as string; + _connectionInfo = new PasswordConnectionInfo(host, username, password); + } + + private SshClient GetSshClient() + { + return new SshClient(_connectionInfo); + } + + private ScpClient GetScpClient() + { + return new ScpClient(_connectionInfo); + } + + public void SubmitLeafInputFile(PiscalLeafInputFile file) + { + var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}"; + + // copy file + using (var scp = GetScpClient()) + using (var stream = new MemoryStream(file.Contents)) + { + Console.WriteLine(inputPath); + scp.Connect(); + scp.Upload(stream, inputPath); + scp.Disconnect(); + } + + // begin processing + using (var ssh = GetSshClient()) + { + ssh.Connect(); + var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}"; + var exeCommand = ssh.CreateCommand(commandText); + exeCommand.Execute(); + ssh.Disconnect(); + + Console.Write(exeCommand.Result); + + if (exeCommand.ExitStatus != 0) + throw new PiscalClientException(exeCommand.Error); + } + } + + public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file) + { + using (var ssh = GetSshClient()) + { + ssh.Connect(); + var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s"; + var exeCommand = ssh.CreateCommand(commandText); + exeCommand.Execute(); + ssh.Disconnect(); + + Console.Write(exeCommand.Result); + } + return PiscalStatus.Success; + } + + public IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file) + { + var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}"; + + string[] filenames; + + // get output files + using (var ssh = GetSshClient()) + { + ssh.Connect(); + var commandText = $"ls -1 {outputDirectory}"; + var lsCommand = ssh.CreateCommand(commandText); + lsCommand.Execute(); + ssh.Disconnect(); + + filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray(); + Console.Write(lsCommand.Result); + } + + using (var scp = GetScpClient()) + { + scp.Connect(); + foreach (var filename in filenames) + { + using (var stream = new MemoryStream()) + { + var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}"; + scp.Download(filePath, stream); + yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename}; + } + } + + scp.Disconnect(); + } + } + } +} \ No newline at end of file diff --git a/Core/Remote/PiscalStatus.cs b/Core/Remote/PiscalStatus.cs new file mode 100644 index 0000000..c06582c --- /dev/null +++ b/Core/Remote/PiscalStatus.cs @@ -0,0 +1,9 @@ +namespace LeafWeb.Core.Remote +{ + public enum PiscalStatus + { + Running, + Success, + Error + } +} \ No newline at end of file diff --git a/Core/Remote/PiscalUtility.cs b/Core/Remote/PiscalUtility.cs new file mode 100644 index 0000000..fda32d0 --- /dev/null +++ b/Core/Remote/PiscalUtility.cs @@ -0,0 +1,28 @@ +using System; +using System.Text.RegularExpressions; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Utility; + +namespace LeafWeb.Core.Remote +{ + public static class PiscalUtility + { + public static string GetPiscalDirectoryName(int id, string name, string identifier) + { + return $"{id}_{name.FilterAlphaNumeric()}_{identifier.FilterAlphaNumeric()}"; + } + + public static string GetPiscalDirectoryName(LeafInputFile leafInputFile) + { + return GetPiscalDirectoryName(leafInputFile.Id, leafInputFile.LeafInput.Name, leafInputFile.LeafInput.Identifier); + } + + public static int GetIdFromDirectoryName(string directoryName) + { + var match = Regex.Match(directoryName, @"\d_"); + if (!match.Success) + throw new FormatException("DirectoryName expected to be formatted {number}_{name}_{identifier}: " + directoryName); + return int.Parse(match.Captures[0].Value); + } + } +} \ No newline at end of file diff --git a/Core/Remote/piscal_manager.sh b/Core/Remote/piscal_manager.sh new file mode 100644 index 0000000..cedb8bf --- /dev/null +++ b/Core/Remote/piscal_manager.sh @@ -0,0 +1,93 @@ +#~/bin/bash +# piscal manager script + +usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal + +where: + -h show this help text + -d working directory name + -o input filename + -s job status" + +# http://stackoverflow.com/a/14203146/99492 +# http://wiki.bash-hackers.org/howto/getopts_tutorial + +# Initialize variables: +base_directory="/home/poprhythm/LeafWeb" +directory_name="" +input_filename="" +get_status=false +pid_filename="piscal.pid" +out_filename="piscal.out" + +while getopts "hd:f:s" opt; do + #echo "$opt = $OPTARG" + case "$opt" in + h ) + echo "$usage" + exit + ;; + d ) + directory_name=$OPTARG + ;; + f ) + input_filename=$OPTARG + ;; + s ) + get_status=true + ;; + \?) printf "illegal option: -%s\n" "$OPTARG" >&2 + echo "$usage" >&2 + exit 1 + ;; + esac +done +if [ -z "$directory_name" ]; then + echo "directory name required (-d)" + exit 1 +fi +if [ ! -d "$base_directory/$directory_name" ]; then + echo "directory $base_directory/$directory_name not found" + exit 1 +fi +if [ "$get_status" = false ]; then + if [ -z "$input_filename" ]; then + echo "input filename required (-f)" + exit 1 + fi + + if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then + echo "input filename $base_directory/$directory_name/$input_filename not found" + exit 1 + fi +fi + +if [ "$get_status" = true ]; then + pid_path="$base_directory/$directory_name/$pid_filename" + + # if the pid doesn't exist, then process never started + if [ ! -f "$pid_path" ]; then + echo "no pid file found" + exit 1 + fi + + pid=$(head -n 1 $pid_path) + # if the pid exists, check the process status using ps + if ps -p $pid > /dev/null; then + # if it is in ps, then it's still running + echo running + else + # otherwise, it is complete, check the output for success/error + # TODO: examine output for errors, etc + echo complete + fi + +else + + command="sleep 100s" + nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 & + + # write the PID to a temp file to check for completion later + echo $! > $base_directory/$directory_name/$pid_filename + +fi \ No newline at end of file diff --git a/Core/Utility/StringExtensions.cs b/Core/Utility/StringExtensions.cs index 946b055..0b2ca8c 100644 --- a/Core/Utility/StringExtensions.cs +++ b/Core/Utility/StringExtensions.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Text.RegularExpressions; @@ -16,6 +15,11 @@ namespace LeafWeb.Core.Utility current + (char.IsUpper(c) && current.Length > 0 ? " " + c : c.ToString(CultureInfo.InvariantCulture))); } + public static string[] SplitNewLine(this string str) + { + return str.Split(new[] {"\n", "\r\n"}, StringSplitOptions.None); + } + public static string LowercaseFirst(string s) { // Check for empty string. @@ -29,23 +33,12 @@ namespace LeafWeb.Core.Utility public static byte[] GetBytes(this string str) { - var bytes = new byte[str.Length * sizeof(char)]; - Buffer.BlockCopy(str.ToCharArray(), 0, bytes, 0, bytes.Length); - return bytes; + return System.Text.Encoding.Default.GetBytes(str); } public static string GetString(this byte[] bytes) { - var chars = new char[bytes.Length / sizeof(char)]; - Buffer.BlockCopy(bytes, 0, chars, 0, bytes.Length); - return new string(chars); - } - - public static Dictionary SplitConnectionString(this string connectionString) - { - return connectionString.Split(';') - .Select(t => t.Split(new[] { '=' }, 2)) - .ToDictionary(t => t[0].Trim(), t => t[1].Trim(), StringComparer.InvariantCultureIgnoreCase); + return System.Text.Encoding.Default.GetString(bytes); } public static string FilterAlphaNumeric(this string input)