diff --git a/Core.Tests/Remote/PiscalSshClientTests.cs b/Core.Tests/Remote/PiscalSshClientTests.cs
index 674f366..85650e0 100644
--- a/Core.Tests/Remote/PiscalSshClientTests.cs
+++ b/Core.Tests/Remote/PiscalSshClientTests.cs
@@ -1,5 +1,6 @@
-using System.Configuration;
-using LeafWeb.Core.Entities;
+using System;
+using System.Configuration;
+using System.Linq;
using LeafWeb.Core.Remote;
using LeafWeb.Core.Utility;
using NUnit.Framework;
@@ -9,13 +10,38 @@ namespace LeafWeb.Core.Tests.Remote
[TestFixture]
public class PiscalSshClientTests
{
+ private readonly PiscalLeafInputFile _testInput =
+ new PiscalLeafInputFile
+ {
+ Filename = "blah",
+ LeafInputId = 1,
+ Contents = "test".GetBytes(),
+ DirectoryName = "TestDirectory"
+ };
+
+ private readonly string _piscalConnectionString =
+ ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString;
+
[Test]
- public void Client()
+ public void SubmitLeafInputFile()
{
- var leafInputFile = new PiscalLeafInputFile{Filename = "blah", Id = 1, Contents = "test".GetBytes(), DirectoryName = "TestDirectory"};
- var piscalConnectionString = ConfigurationManager.ConnectionStrings["PiscalServer"];
- var client = new PiscalSshClient(piscalConnectionString.ConnectionString);
- client.SubmitLeafInputFile(leafInputFile);
+ var client = new PiscalSshClient(_piscalConnectionString);
+ client.SubmitLeafInputFile(_testInput);
+ }
+
+ [Test]
+ public void GetLeafInputStatus()
+ {
+ var client = new PiscalSshClient(_piscalConnectionString);
+ client.GetLeafInputStatus(_testInput);
+ }
+
+ [Test]
+ public void RetrieveLeafInputResult()
+ {
+ var client = new PiscalSshClient(_piscalConnectionString);
+ var result = client.RetrieveLeafInputResult(_testInput).ToList();
+ Console.WriteLine(result[0].Contents.GetString());
}
}
}
diff --git a/Core/Core.csproj b/Core/Core.csproj
index 300864d..247b711 100644
--- a/Core/Core.csproj
+++ b/Core/Core.csproj
@@ -86,7 +86,12 @@
+
+
+
+
+
@@ -105,6 +110,7 @@
PreserveNewest
+
diff --git a/Core/Entities/LeafInput.cs b/Core/Entities/LeafInput.cs
index 4634337..ea712cf 100644
--- a/Core/Entities/LeafInput.cs
+++ b/Core/Entities/LeafInput.cs
@@ -33,10 +33,5 @@ namespace LeafWeb.Core.Entities
[DataType(DataType.Date)]
public DateTime? Processed { get; set; }
-
- ///
- /// An cleaned up identifier to be used for file naming, etc
- ///
- public string CombinedIdentifier => $"{Name.FilterAlphaNumeric()}_{Identifier.FilterAlphaNumeric()}";
}
}
\ No newline at end of file
diff --git a/Core/Entities/LeafInputFile.cs b/Core/Entities/LeafInputFile.cs
index 28d75ed..3726235 100644
--- a/Core/Entities/LeafInputFile.cs
+++ b/Core/Entities/LeafInputFile.cs
@@ -14,7 +14,5 @@
public string Filename { get; set; }
public byte[] Contents { get; set; }
-
- public string CombinedIdentifier => $"{LeafInput.CombinedIdentifier}_{Id}";
}
}
\ No newline at end of file
diff --git a/Core/Remote/IPiscalClient.cs b/Core/Remote/IPiscalClient.cs
index 4f5c780..a92edb9 100644
--- a/Core/Remote/IPiscalClient.cs
+++ b/Core/Remote/IPiscalClient.cs
@@ -1,82 +1,11 @@
-using System;
-using System.IO;
-using System.Text;
-using LeafWeb.Core.Utility;
-using Renci.SshNet;
+using System.Collections.Generic;
namespace LeafWeb.Core.Remote
{
public interface IPiscalClient
{
void SubmitLeafInputFile(PiscalLeafInputFile file);
- void RetrieveLeafInputResult(PiscalLeafInputFile file);
- }
-
- public class PiscalSshClient : IPiscalClient
- {
- private const string BaseDirectory = "LeafInput";
- private readonly PasswordConnectionInfo _connectionInfo;
-
- public PiscalSshClient(string connectionString)
- {
- var conn = connectionString.SplitConnectionString();
-
- var host = conn["host"];
- var username = conn["username"];
- var password = conn["password"];
- _connectionInfo = new PasswordConnectionInfo(host, username, password);
- }
-
- private SshClient GetSshClient()
- {
- return new SshClient(_connectionInfo);
- }
-
- private ScpClient GetScpClient()
- {
- return new ScpClient(_connectionInfo);
- }
-
- public void SubmitLeafInputFile(PiscalLeafInputFile file)
- {
- var inputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename;
- var outputPath = BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename;
-
- using (var scp = GetScpClient())
- using (var stream = new MemoryStream(file.Contents))
- {
- scp.Connect();
- scp.Upload(stream, inputPath);
- }
-
- using (var ssh = GetSshClient())
- {
- ssh.Connect();
- var lsCommend = ssh.CreateCommand("ls -r");
- lsCommend.Execute();
- //var extendedData = Encoding.ASCII.GetString(cmd.ExtendedOutputStream.ToArray());
- var extendedData = new StreamReader(lsCommend.ExtendedOutputStream, Encoding.ASCII).ReadToEnd();
- ssh.Disconnect();
-
- Console.Write(lsCommend.Result);
- Console.Write(extendedData);
- //Assert.AreEqual("12345\n", cmd.Result);
- //Assert.AreEqual("654321\n", extendedData);
- }
- // copy file
-
- // begin processing
- }
-
- public void RetrieveLeafInputResult(PiscalLeafInputFile file)
- {
- using (var scp = GetScpClient())
- using (var stream = new MemoryStream(file.Contents))
- {
- scp.Connect();
- scp.Upload(stream, BaseDirectory + "/" + file.DirectoryName + "/" + file.Filename);
- }
-
- }
+ PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
+ IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file);
}
}
diff --git a/Core/Remote/PiscalClientException.cs b/Core/Remote/PiscalClientException.cs
new file mode 100644
index 0000000..cdb6b8d
--- /dev/null
+++ b/Core/Remote/PiscalClientException.cs
@@ -0,0 +1,11 @@
+using System;
+
+namespace LeafWeb.Core.Remote
+{
+ public class PiscalClientException : Exception
+ {
+ public PiscalClientException(string error) : base(error)
+ {
+ }
+ }
+}
\ No newline at end of file
diff --git a/Core/Remote/PiscalLeafInputFile.cs b/Core/Remote/PiscalLeafInputFile.cs
index 3c4120a..422d41e 100644
--- a/Core/Remote/PiscalLeafInputFile.cs
+++ b/Core/Remote/PiscalLeafInputFile.cs
@@ -1,6 +1,5 @@
using AutoMapper;
using LeafWeb.Core.Entities;
-using LeafWeb.Core.Utility;
namespace LeafWeb.Core.Remote
{
@@ -8,7 +7,7 @@ namespace LeafWeb.Core.Remote
{
private static readonly IMapper Mapper;
- public int Id { get; set; }
+ public int LeafInputId { get; set; }
public string Filename { get; set; }
public byte[] Contents { get; set; }
public string DirectoryName { get; set; }
@@ -20,9 +19,8 @@ namespace LeafWeb.Core.Remote
{
cfg.CreateMap()
.ForMember(dest => dest.DirectoryName, opt => opt.MapFrom(
- src =>
- $"{src.Id}_{src.LeafInput.Name.FilterAlphaNumeric()}_{src.LeafInput.Identifier.FilterAlphaNumeric()}"
- ));
+ src => PiscalUtility.GetPiscalDirectoryName(src)))
+ .ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id));
});
Mapper = config.CreateMapper();
}
diff --git a/Core/Remote/PiscalLeafOutputFile.cs b/Core/Remote/PiscalLeafOutputFile.cs
new file mode 100644
index 0000000..a7d8d3d
--- /dev/null
+++ b/Core/Remote/PiscalLeafOutputFile.cs
@@ -0,0 +1,34 @@
+using AutoMapper;
+using LeafWeb.Core.Entities;
+
+namespace LeafWeb.Core.Remote
+{
+ public class PiscalLeafOutputFile
+ {
+ private static readonly IMapper Mapper;
+
+ public string Filename { get; set; }
+ public byte[] Contents { get; set; }
+ public string DirectoryName { get; set; }
+ public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName);
+
+ static PiscalLeafOutputFile()
+ {
+ var config =
+ new MapperConfiguration(cfg =>
+ {
+ cfg.CreateMap();
+ });
+ Mapper = config.CreateMapper();
+ }
+
+ public PiscalLeafOutputFile() { }
+
+ public LeafOutputFile GetLeafOutputFile()
+ {
+ var leafOutputFile = new LeafOutputFile();
+ Mapper.Map(this, leafOutputFile);
+ return leafOutputFile;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs
new file mode 100644
index 0000000..5310ba1
--- /dev/null
+++ b/Core/Remote/PiscalSshClient.cs
@@ -0,0 +1,118 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.IO;
+using System.Linq;
+using LeafWeb.Core.Utility;
+using Renci.SshNet;
+
+namespace LeafWeb.Core.Remote
+{
+ public class PiscalSshClient : IPiscalClient
+ {
+ private const string BaseDirectory = "./LeafWeb";
+ private const string RemoteScriptPath = "./piscal_manager.sh";
+ private readonly PasswordConnectionInfo _connectionInfo;
+
+ public PiscalSshClient(string connectionString)
+ {
+ var conn = new DbConnectionStringBuilder {ConnectionString = connectionString};
+
+ var host = conn["host"] as string;
+ var username = conn["username"] as string;
+ var password = conn["password"] as string;
+ _connectionInfo = new PasswordConnectionInfo(host, username, password);
+ }
+
+ private SshClient GetSshClient()
+ {
+ return new SshClient(_connectionInfo);
+ }
+
+ private ScpClient GetScpClient()
+ {
+ return new ScpClient(_connectionInfo);
+ }
+
+ public void SubmitLeafInputFile(PiscalLeafInputFile file)
+ {
+ var inputPath = $"{BaseDirectory}/{file.DirectoryName}/{file.Filename}";
+
+ // copy file
+ using (var scp = GetScpClient())
+ using (var stream = new MemoryStream(file.Contents))
+ {
+ Console.WriteLine(inputPath);
+ scp.Connect();
+ scp.Upload(stream, inputPath);
+ scp.Disconnect();
+ }
+
+ // begin processing
+ using (var ssh = GetSshClient())
+ {
+ ssh.Connect();
+ var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
+ var exeCommand = ssh.CreateCommand(commandText);
+ exeCommand.Execute();
+ ssh.Disconnect();
+
+ Console.Write(exeCommand.Result);
+
+ if (exeCommand.ExitStatus != 0)
+ throw new PiscalClientException(exeCommand.Error);
+ }
+ }
+
+ public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
+ {
+ using (var ssh = GetSshClient())
+ {
+ ssh.Connect();
+ var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
+ var exeCommand = ssh.CreateCommand(commandText);
+ exeCommand.Execute();
+ ssh.Disconnect();
+
+ Console.Write(exeCommand.Result);
+ }
+ return PiscalStatus.Success;
+ }
+
+ public IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file)
+ {
+ var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
+
+ string[] filenames;
+
+ // get output files
+ using (var ssh = GetSshClient())
+ {
+ ssh.Connect();
+ var commandText = $"ls -1 {outputDirectory}";
+ var lsCommand = ssh.CreateCommand(commandText);
+ lsCommand.Execute();
+ ssh.Disconnect();
+
+ filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
+ Console.Write(lsCommand.Result);
+ }
+
+ using (var scp = GetScpClient())
+ {
+ scp.Connect();
+ foreach (var filename in filenames)
+ {
+ using (var stream = new MemoryStream())
+ {
+ var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
+ scp.Download(filePath, stream);
+ yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
+ }
+ }
+
+ scp.Disconnect();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Core/Remote/PiscalStatus.cs b/Core/Remote/PiscalStatus.cs
new file mode 100644
index 0000000..c06582c
--- /dev/null
+++ b/Core/Remote/PiscalStatus.cs
@@ -0,0 +1,9 @@
+namespace LeafWeb.Core.Remote
+{
+ public enum PiscalStatus
+ {
+ Running,
+ Success,
+ Error
+ }
+}
\ No newline at end of file
diff --git a/Core/Remote/PiscalUtility.cs b/Core/Remote/PiscalUtility.cs
new file mode 100644
index 0000000..fda32d0
--- /dev/null
+++ b/Core/Remote/PiscalUtility.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Text.RegularExpressions;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Utility;
+
+namespace LeafWeb.Core.Remote
+{
+ public static class PiscalUtility
+ {
+ public static string GetPiscalDirectoryName(int id, string name, string identifier)
+ {
+ return $"{id}_{name.FilterAlphaNumeric()}_{identifier.FilterAlphaNumeric()}";
+ }
+
+ public static string GetPiscalDirectoryName(LeafInputFile leafInputFile)
+ {
+ return GetPiscalDirectoryName(leafInputFile.Id, leafInputFile.LeafInput.Name, leafInputFile.LeafInput.Identifier);
+ }
+
+ public static int GetIdFromDirectoryName(string directoryName)
+ {
+ var match = Regex.Match(directoryName, @"\d_");
+ if (!match.Success)
+ throw new FormatException("DirectoryName expected to be formatted {number}_{name}_{identifier}: " + directoryName);
+ return int.Parse(match.Captures[0].Value);
+ }
+ }
+}
\ No newline at end of file
diff --git a/Core/Remote/piscal_manager.sh b/Core/Remote/piscal_manager.sh
new file mode 100644
index 0000000..cedb8bf
--- /dev/null
+++ b/Core/Remote/piscal_manager.sh
@@ -0,0 +1,93 @@
+#~/bin/bash
+# piscal manager script
+
+usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
+
+where:
+ -h show this help text
+ -d working directory name
+ -o input filename
+ -s job status"
+
+# http://stackoverflow.com/a/14203146/99492
+# http://wiki.bash-hackers.org/howto/getopts_tutorial
+
+# Initialize variables:
+base_directory="/home/poprhythm/LeafWeb"
+directory_name=""
+input_filename=""
+get_status=false
+pid_filename="piscal.pid"
+out_filename="piscal.out"
+
+while getopts "hd:f:s" opt; do
+ #echo "$opt = $OPTARG"
+ case "$opt" in
+ h )
+ echo "$usage"
+ exit
+ ;;
+ d )
+ directory_name=$OPTARG
+ ;;
+ f )
+ input_filename=$OPTARG
+ ;;
+ s )
+ get_status=true
+ ;;
+ \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+ echo "$usage" >&2
+ exit 1
+ ;;
+ esac
+done
+if [ -z "$directory_name" ]; then
+ echo "directory name required (-d)"
+ exit 1
+fi
+if [ ! -d "$base_directory/$directory_name" ]; then
+ echo "directory $base_directory/$directory_name not found"
+ exit 1
+fi
+if [ "$get_status" = false ]; then
+ if [ -z "$input_filename" ]; then
+ echo "input filename required (-f)"
+ exit 1
+ fi
+
+ if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
+ echo "input filename $base_directory/$directory_name/$input_filename not found"
+ exit 1
+ fi
+fi
+
+if [ "$get_status" = true ]; then
+ pid_path="$base_directory/$directory_name/$pid_filename"
+
+ # if the pid doesn't exist, then process never started
+ if [ ! -f "$pid_path" ]; then
+ echo "no pid file found"
+ exit 1
+ fi
+
+ pid=$(head -n 1 $pid_path)
+ # if the pid exists, check the process status using ps
+ if ps -p $pid > /dev/null; then
+ # if it is in ps, then it's still running
+ echo running
+ else
+ # otherwise, it is complete, check the output for success/error
+ # TODO: examine output for errors, etc
+ echo complete
+ fi
+
+else
+
+ command="sleep 100s"
+ nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
+
+ # write the PID to a temp file to check for completion later
+ echo $! > $base_directory/$directory_name/$pid_filename
+
+fi
\ No newline at end of file
diff --git a/Core/Utility/StringExtensions.cs b/Core/Utility/StringExtensions.cs
index 946b055..0b2ca8c 100644
--- a/Core/Utility/StringExtensions.cs
+++ b/Core/Utility/StringExtensions.cs
@@ -1,5 +1,4 @@
using System;
-using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
@@ -16,6 +15,11 @@ namespace LeafWeb.Core.Utility
current + (char.IsUpper(c) && current.Length > 0 ? " " + c : c.ToString(CultureInfo.InvariantCulture)));
}
+ public static string[] SplitNewLine(this string str)
+ {
+ return str.Split(new[] {"\n", "\r\n"}, StringSplitOptions.None);
+ }
+
public static string LowercaseFirst(string s)
{
// Check for empty string.
@@ -29,23 +33,12 @@ namespace LeafWeb.Core.Utility
public static byte[] GetBytes(this string str)
{
- var bytes = new byte[str.Length * sizeof(char)];
- Buffer.BlockCopy(str.ToCharArray(), 0, bytes, 0, bytes.Length);
- return bytes;
+ return System.Text.Encoding.Default.GetBytes(str);
}
public static string GetString(this byte[] bytes)
{
- var chars = new char[bytes.Length / sizeof(char)];
- Buffer.BlockCopy(bytes, 0, chars, 0, bytes.Length);
- return new string(chars);
- }
-
- public static Dictionary SplitConnectionString(this string connectionString)
- {
- return connectionString.Split(';')
- .Select(t => t.Split(new[] { '=' }, 2))
- .ToDictionary(t => t[0].Trim(), t => t[1].Trim(), StringComparer.InvariantCultureIgnoreCase);
+ return System.Text.Encoding.Default.GetString(bytes);
}
public static string FilterAlphaNumeric(this string input)