diff --git a/Core.Tests/Remote/PiscalSshClientTests.cs b/Core.Tests/Remote/PiscalSshClientTests.cs
index 85650e0..bf6a7be 100644
--- a/Core.Tests/Remote/PiscalSshClientTests.cs
+++ b/Core.Tests/Remote/PiscalSshClientTests.cs
@@ -16,7 +16,7 @@ namespace LeafWeb.Core.Tests.Remote
Filename = "blah",
LeafInputId = 1,
Contents = "test".GetBytes(),
- DirectoryName = "TestDirectory"
+ DirectoryName = "TestDirectory2"
};
private readonly string _piscalConnectionString =
@@ -33,15 +33,17 @@ namespace LeafWeb.Core.Tests.Remote
public void GetLeafInputStatus()
{
var client = new PiscalSshClient(_piscalConnectionString);
- client.GetLeafInputStatus(_testInput);
+ var leafInputStatus = client.GetLeafInputStatus(_testInput);
+ Console.WriteLine(leafInputStatus);
}
[Test]
- public void RetrieveLeafInputResult()
+ public void RetrieveLeafOutput()
{
var client = new PiscalSshClient(_piscalConnectionString);
- var result = client.RetrieveLeafInputResult(_testInput).ToList();
- Console.WriteLine(result[0].Contents.GetString());
+ var result = client.RetrieveLeafOutput(_testInput).ToList();
+ Console.WriteLine(string.Join(", ", result.Select(o => o.Filename)));
+ //Console.WriteLine(result[0].Contents.GetString());
}
}
}
diff --git a/Core.Tests/Utility/StringExtensionsTests.cs b/Core.Tests/Utility/StringExtensionsTests.cs
index cfb8537..c7652dc 100644
--- a/Core.Tests/Utility/StringExtensionsTests.cs
+++ b/Core.Tests/Utility/StringExtensionsTests.cs
@@ -38,5 +38,13 @@ namespace LeafWeb.Core.Tests.Utility
var result = str.FilterAlphaNumeric();
Assert.That(result, Is.EqualTo("newline"));
}
+
+ [Test]
+ public void FilenameFromPath()
+ {
+ var str = "/full/path/to/file.ext";
+ var result = str.FilenameFromPath();
+ Assert.That(result, Is.EqualTo("file.ext"));
+ }
}
}
diff --git a/Core/Core.csproj b/Core/Core.csproj
index 761d44c..9d83762 100644
--- a/Core/Core.csproj
+++ b/Core/Core.csproj
@@ -111,6 +111,7 @@
PreserveNewest
+
diff --git a/Core/DAL/DataService.cs b/Core/DAL/DataService.cs
index b8c4e17..2badd81 100644
--- a/Core/DAL/DataService.cs
+++ b/Core/DAL/DataService.cs
@@ -38,8 +38,7 @@ namespace LeafWeb.Core.DAL
}
#endregion
-
- #region LeafInput Sites
+ #region LeafInput
public IQueryable GetLeafInputs()
{
@@ -54,13 +53,69 @@ namespace LeafWeb.Core.DAL
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
-
_db.LeafInputs.Add(leafInput);
+ foreach (var leafInputFile in leafInput.Files)
+ {
+ SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added);
+ }
+ _db.SaveChanges();
+ }
+ #endregion
+
+ #region LeafInputFile
+
+ public IQueryable GetLeafInputFiles()
+ {
+ return _db.LeafInputFiles;
+ }
+
+ public IQueryable GetLeafInputFiles(LeafInputStatusType status)
+ {
+ return
+ from file in _db.LeafInputFiles
+ where file.CurrentStatus == status
+ select file;
+ }
+
+ public void UpdateLeafInputFile(LeafInputFile leafInputFile)
+ {
+ _db.Entry(leafInputFile).State = EntityState.Modified;
_db.SaveChanges();
}
+ private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
+ {
+ leafInputFile.CurrentStatus = status;
+ var leafInputFileStatus = new LeafInputFileStatus
+ {
+ Status = status,
+ DateTime = DateTime.Now,
+ Description = description,
+ LeafInputFile = leafInputFile
+ };
+ if (leafInputFile.StatusHistory == null)
+ leafInputFile.StatusHistory = new List();
+ leafInputFile.StatusHistory.Add(leafInputFileStatus);
+ }
+
+ public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
+ {
+ SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
+ UpdateLeafInputFile(leafInputFile);
+ }
+
+ public LeafInputFile GetNextUnprocessedLeafInputFile()
+ {
+ return
+ (from file in GetLeafInputFiles(LeafInputStatusType.Added)
+ orderby file.Id ascending
+ select file).FirstOrDefault();
+ }
+
#endregion
+ #region Photosynthesis Types
+
public IQueryable GetPhotosynthesisTypes()
{
return _db.PhotosynthesisTypes.OrderBy(pt => pt.SortOrder);
@@ -70,5 +125,7 @@ namespace LeafWeb.Core.DAL
{
return _db.PhotosynthesisTypes.Find(id);
}
+
+ #endregion
}
}
diff --git a/Core/Entities/LeafInput.cs b/Core/Entities/LeafInput.cs
index 8205fcb..452bf9a 100644
--- a/Core/Entities/LeafInput.cs
+++ b/Core/Entities/LeafInput.cs
@@ -8,7 +8,7 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
- public virtual ICollection LeafInputFiles { get; set; }
+ public virtual ICollection Files { get; set; }
[Required(ErrorMessage = "Name required")]
public string Name { get; set; }
diff --git a/Core/Entities/LeafInputFile.cs b/Core/Entities/LeafInputFile.cs
index 645ff6d..7ea7ca0 100644
--- a/Core/Entities/LeafInputFile.cs
+++ b/Core/Entities/LeafInputFile.cs
@@ -18,6 +18,6 @@ namespace LeafWeb.Core.Entities
public byte[] Contents { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
- public virtual ICollection LeafInputStatuses { get; set; }
+ public virtual ICollection StatusHistory { get; set; }
}
}
\ No newline at end of file
diff --git a/Core/Remote/IPiscalClient.cs b/Core/Remote/IPiscalClient.cs
index a92edb9..195de8f 100644
--- a/Core/Remote/IPiscalClient.cs
+++ b/Core/Remote/IPiscalClient.cs
@@ -6,6 +6,6 @@ namespace LeafWeb.Core.Remote
{
void SubmitLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
- IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file);
+ IEnumerable RetrieveLeafOutput(PiscalLeafInputFile file);
}
}
diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs
index 5310ba1..1b2ff72 100644
--- a/Core/Remote/PiscalSshClient.cs
+++ b/Core/Remote/PiscalSshClient.cs
@@ -11,7 +11,7 @@ namespace LeafWeb.Core.Remote
public class PiscalSshClient : IPiscalClient
{
private const string BaseDirectory = "./LeafWeb";
- private const string RemoteScriptPath = "./piscal_manager.sh";
+ private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh";
private readonly PasswordConnectionInfo _connectionInfo;
public PiscalSshClient(string connectionString)
@@ -53,61 +53,76 @@ namespace LeafWeb.Core.Remote
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
- var exeCommand = ssh.CreateCommand(commandText);
- exeCommand.Execute();
+ var command = ssh.CreateCommand(commandText);
+ command.Execute();
ssh.Disconnect();
- Console.Write(exeCommand.Result);
+ if (command.ExitStatus != 0)
+ throw new PiscalClientException(command.Error);
- if (exeCommand.ExitStatus != 0)
- throw new PiscalClientException(exeCommand.Error);
+ Console.Write(command.Result);
}
}
public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
+ {
+ var statusRaw = GetLeafInputStatusRaw(file);
+
+ switch (statusRaw[0])
+ {
+ case "running":
+ return PiscalStatus.Running;
+ case "success":
+ return PiscalStatus.Success;
+ default:
+ return PiscalStatus.Error;
+ }
+ }
+
+ private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
{
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
- var exeCommand = ssh.CreateCommand(commandText);
- exeCommand.Execute();
+ var command = ssh.CreateCommand(commandText);
+ command.Execute();
ssh.Disconnect();
- Console.Write(exeCommand.Result);
+ if (command.ExitStatus != 0)
+ throw new PiscalClientException(command.Error);
+
+ return command.Result
+ .SplitNewLine()
+ .Where(s => s.Length > 0)
+ .Select(s => s.Trim()).ToArray();
}
- return PiscalStatus.Success;
}
- public IEnumerable RetrieveLeafInputResult(PiscalLeafInputFile file)
+ public IEnumerable RetrieveLeafOutput(PiscalLeafInputFile file)
{
- var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
-
- string[] filenames;
-
// get output files
- using (var ssh = GetSshClient())
- {
- ssh.Connect();
- var commandText = $"ls -1 {outputDirectory}";
- var lsCommand = ssh.CreateCommand(commandText);
- lsCommand.Execute();
- ssh.Disconnect();
+ var status = GetLeafInputStatusRaw(file);
+ if (status[0] != "success")
+ throw new PiscalClientException("output not available, status is " + status[0]);
- filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
- Console.Write(lsCommand.Result);
- }
+ var filePaths = status.Skip(1);
using (var scp = GetScpClient())
{
scp.Connect();
- foreach (var filename in filenames)
+ foreach (var filePath in filePaths)
{
using (var stream = new MemoryStream())
{
- var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
scp.Download(filePath, stream);
- yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
+ yield return
+ new PiscalLeafOutputFile
+ {
+ Contents = stream.ToArray(),
+ Filename = filePath.FilenameFromPath(),
+ DirectoryName = file.DirectoryName
+ };
}
}
diff --git a/Core/Remote/piscal_launcher.sh b/Core/Remote/piscal_launcher.sh
new file mode 100644
index 0000000..a9c4fe5
--- /dev/null
+++ b/Core/Remote/piscal_launcher.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# piscal launcher
+
+usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
+
+where:
+ -h show this help text
+ -d working directory
+ -f input filename"
+
+directory=""
+input_filename=""
+
+while getopts "hd:f:" opt; do
+ case "$opt" in
+ h )
+ echo "$usage"
+ exit
+ ;;
+ d )
+ directory=$OPTARG
+ ;;
+ f )
+ input_filename=$OPTARG
+ task="process"
+ ;;
+ \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+ echo "$usage" >&2
+ exit 1
+ ;;
+ esac
+done
+if [ -z "$directory" ]; then
+ echo "working directory required (-d)"
+ exit 1
+fi
+if [ ! -d "$directory" ]; then
+ echo "working directory $directory not found"
+ exit 1
+fi
+if [ -z "$input_filename" ]; then
+ echo "input filename required (-f)"
+ exit 1
+fi
+if [ ! -f "$directory/$input_filename" ]; then
+ echo "input filename $directory/$input_filename not found"
+ exit 1
+fi
+
+output_directory="$directory/output"
+
+sleep 1m
+if [ ! -d "$output_directory" ]; then
+ mkdir "$output_directory"
+fi
+
+# TODO: run actual command
+cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"
diff --git a/Core/Remote/piscal_manager.sh b/Core/Remote/piscal_manager.sh
index cedb8bf..37dd71c 100644
--- a/Core/Remote/piscal_manager.sh
+++ b/Core/Remote/piscal_manager.sh
@@ -1,12 +1,12 @@
-#~/bin/bash
+#!/bin/bash
# piscal manager script
-usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
+usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
where:
-h show this help text
-d working directory name
- -o input filename
+ -f input filename
-s job status"
# http://stackoverflow.com/a/14203146/99492
@@ -16,9 +16,10 @@ where:
base_directory="/home/poprhythm/LeafWeb"
directory_name=""
input_filename=""
-get_status=false
+task="start" # default task
pid_filename="piscal.pid"
out_filename="piscal.out"
+launcher="$base_directory/piscal_launcher.sh"
while getopts "hd:f:s" opt; do
#echo "$opt = $OPTARG"
@@ -32,9 +33,10 @@ while getopts "hd:f:s" opt; do
;;
f )
input_filename=$OPTARG
+ task="start"
;;
s )
- get_status=true
+ task="get_status"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -46,24 +48,30 @@ if [ -z "$directory_name" ]; then
echo "directory name required (-d)"
exit 1
fi
-if [ ! -d "$base_directory/$directory_name" ]; then
- echo "directory $base_directory/$directory_name not found"
+working_directory="$base_directory/$directory_name"
+if [ ! -d "$working_directory" ]; then
+ echo "directory $working_directory not found"
exit 1
fi
-if [ "$get_status" = false ]; then
+if [ "$task" = "start" ]; then
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
- if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
- echo "input filename $base_directory/$directory_name/$input_filename not found"
+ if [ ! -f "$working_directory/$input_filename" ]; then
+ echo "input filename $working_directory/$input_filename not found"
exit 1
fi
-fi
+
+ command="$launcher -d $working_directory -f $input_filename"
+ nohup ${command} > $working_directory/$out_filename 2>&1 &
-if [ "$get_status" = true ]; then
- pid_path="$base_directory/$directory_name/$pid_filename"
+ # write the PID to a temp file to check for completion later
+ echo $! > $working_directory/$pid_filename
+ echo started
+elif [ "$task" = "get_status" ]; then
+ pid_path="$working_directory/$pid_filename"
# if the pid doesn't exist, then process never started
if [ ! -f "$pid_path" ]; then
@@ -79,15 +87,12 @@ if [ "$get_status" = true ]; then
else
# otherwise, it is complete, check the output for success/error
# TODO: examine output for errors, etc
- echo complete
+ #cat piscal.out
+ if [ ! -d "$working_directory/output" ]; then
+ echo "output directory $working_directory/output not found"
+ exit 1
+ fi
+ echo success
+ find "$working_directory/output"/*
fi
-
-else
-
- command="sleep 100s"
- nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
-
- # write the PID to a temp file to check for completion later
- echo $! > $base_directory/$directory_name/$pid_filename
-
fi
\ No newline at end of file
diff --git a/Core/Utility/StringExtensions.cs b/Core/Utility/StringExtensions.cs
index 0b2ca8c..9545f76 100644
--- a/Core/Utility/StringExtensions.cs
+++ b/Core/Utility/StringExtensions.cs
@@ -45,5 +45,10 @@ namespace LeafWeb.Core.Utility
{
return Regex.Replace(input, @"[^\w_]", "");
}
+
+ public static string FilenameFromPath(this string path)
+ {
+ return Regex.Replace(path, @".*/([^/]*$)", "$1");
+ }
}
}
diff --git a/Web/Controllers/LeafInputController.cs b/Web/Controllers/LeafInputController.cs
index 678f353..9f797cf 100644
--- a/Web/Controllers/LeafInputController.cs
+++ b/Web/Controllers/LeafInputController.cs
@@ -1,9 +1,12 @@
using System;
+using System.Configuration;
using System.IO;
using System.Linq;
using System.Web;
using System.Web.Mvc;
+using Hangfire;
using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
using LeafWeb.Web.Attributes;
using LeafWeb.Web.ViewModels;
using LeafWeb.Web.ViewModels.LeafInput;
@@ -69,7 +72,7 @@ namespace LeafWeb.Web.Controllers
// convert viewModel into Model
var leafInput = viewModel.GetFileInput(DataService);
// load files into LeafInputFile
- leafInput.LeafInputFiles =
+ leafInput.Files =
(from f in files
let bytes = System.IO.File.ReadAllBytes(f.FullName)
select new LeafInputFile {Filename = f.Name, Contents = bytes}).ToList();
@@ -78,7 +81,13 @@ namespace LeafWeb.Web.Controllers
DataService.AddLeafInput(leafInput);
DeleteBackloadDirectory(Session.SessionID);
-
+
+ foreach (var file in leafInput.Files.Select(f => new PiscalLeafInputFile(f)))
+ {
+ BackgroundJob.Enqueue(() => ProcessLeafInput(file));
+ }
+
+
SetStatusMessage(
HttpUtility.HtmlEncode(
$"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine
@@ -91,6 +100,12 @@ namespace LeafWeb.Web.Controllers
return View("Index", viewModel);
}
+ public void ProcessLeafInput(PiscalLeafInputFile leafInputFile)
+ {
+ var piscalSshClient = new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
+ piscalSshClient.SubmitLeafInputFile(leafInputFile);
+ }
+
private FileInfo[] GetBackloadDirectoryFiles(string directoryName)
{
var path = Path.Combine(Server.MapPath("~/Files/"), directoryName + "\\");
diff --git a/Web/Services/JobService.cs b/Web/Services/JobService.cs
new file mode 100644
index 0000000..7f9bb1c
--- /dev/null
+++ b/Web/Services/JobService.cs
@@ -0,0 +1,65 @@
+using System;
+using System.Configuration;
+using System.Linq;
+using LeafWeb.Core.DAL;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
+
+namespace LeafWeb.Web.Services
+{
+ public class JobService : IDisposable
+ {
+ protected readonly DataService DataService = new DataService();
+
+ protected IPiscalClient GetPiscalClient()
+ {
+ return new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
+ }
+
+ public void ProcessNextLeafInput()
+ {
+ var leafInputFile = DataService.GetNextUnprocessedLeafInputFile();
+ if (leafInputFile == null) // no inputs, quit
+ return;
+
+ var inputFile = new PiscalLeafInputFile(leafInputFile);
+
+ var piscalSshClient = GetPiscalClient();
+ piscalSshClient.SubmitLeafInputFile(inputFile);
+
+ DataService.SetLeafInputFileStatus(leafInputFile, LeafInputStatusType.ProcessStarted);
+ }
+
+ public void UpdateLeafInputStatus()
+ {
+ var leafInputFiles =
+ DataService
+ .GetLeafInputFiles(LeafInputStatusType.ProcessStarted)
+ .Select(f => new PiscalLeafInputFile(f));
+
+ var piscalClient = GetPiscalClient();
+ foreach (var file in leafInputFiles)
+ {
+ var status = piscalClient.GetLeafInputStatus(file);
+ switch (status)
+ {
+ case PiscalStatus.Success:
+ // retrieve LeafOutput
+ var outputFile = piscalClient.RetrieveLeafOutput(file);
+ break;
+ case PiscalStatus.Error:
+ // record error
+ break;
+ case PiscalStatus.Running:
+ // do nothing
+ break;
+ }
+ }
+ }
+
+ public void Dispose()
+ {
+ DataService.Dispose();
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Web.csproj b/Web/Web.csproj
index c1ebe8c..f3b491b 100644
--- a/Web/Web.csproj
+++ b/Web/Web.csproj
@@ -921,6 +921,7 @@
+