Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retrieve LeafOutput operational

Browse Source
This commit is contained in:
James Kolpack
2016-02-26 11:54:07 -05:00
parent 76722345a8
commit 59e2f9d8bd
14 changed files with 296 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core.Tests/Remote/PiscalSshClientTests.cs
+7 -5
View File
@@ -16,7 +16,7 @@ namespace LeafWeb.Core.Tests.Remote
Filename = "blah",
LeafInputId = 1,
Contents = "test".GetBytes(),
DirectoryName = "TestDirectory"
DirectoryName = "TestDirectory2"
};
private readonly string _piscalConnectionString =
@@ -33,15 +33,17 @@ namespace LeafWeb.Core.Tests.Remote
public void GetLeafInputStatus()
{
var client = new PiscalSshClient(_piscalConnectionString);
client.GetLeafInputStatus(_testInput);
var leafInputStatus = client.GetLeafInputStatus(_testInput);
Console.WriteLine(leafInputStatus);
}
[Test]
public void RetrieveLeafInputResult()
public void RetrieveLeafOutput()
{
var client = new PiscalSshClient(_piscalConnectionString);
var result = client.RetrieveLeafInputResult(_testInput).ToList();
Console.WriteLine(result[0].Contents.GetString());
var result = client.RetrieveLeafOutput(_testInput).ToList();
Console.WriteLine(string.Join(", ", result.Select(o => o.Filename)));
//Console.WriteLine(result[0].Contents.GetString());
}
}
}
Core.Tests/Utility/StringExtensionsTests.cs
+8
View File
@@ -38,5 +38,13 @@ namespace LeafWeb.Core.Tests.Utility
var result = str.FilterAlphaNumeric();
Assert.That(result, Is.EqualTo("newline"));
}
[Test]
public void FilenameFromPath()
{
var str = "/full/path/to/file.ext";
var result = str.FilenameFromPath();
Assert.That(result, Is.EqualTo("file.ext"));
}
}
}
Core/Core.csproj
+1
View File
@@ -111,6 +111,7 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="packages.config" />
<None Include="Remote\piscal_launcher.sh" />
<None Include="Remote\piscal_manager.sh" />
</ItemGroup>
<ItemGroup />
Core/DAL/DataService.cs
+60 -3
View File
@@ -38,8 +38,7 @@ namespace LeafWeb.Core.DAL
}
#endregion
#region LeafInput Sites
#region LeafInput
public IQueryable<LeafInput> GetLeafInputs()
{
@@ -54,13 +53,69 @@ namespace LeafWeb.Core.DAL
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
_db.LeafInputs.Add(leafInput);
foreach (var leafInputFile in leafInput.Files)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added);
}
_db.SaveChanges();
}
#endregion
#region LeafInputFile
public IQueryable<LeafInputFile> GetLeafInputFiles()
{
return _db.LeafInputFiles;
}
public IQueryable<LeafInputFile> GetLeafInputFiles(LeafInputStatusType status)
{
return
from file in _db.LeafInputFiles
where file.CurrentStatus == status
select file;
}
public void UpdateLeafInputFile(LeafInputFile leafInputFile)
{
_db.Entry(leafInputFile).State = EntityState.Modified;
_db.SaveChanges();
}
private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputFileStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInputFile = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputFileStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
UpdateLeafInputFile(leafInputFile);
}
public LeafInputFile GetNextUnprocessedLeafInputFile()
{
return
(from file in GetLeafInputFiles(LeafInputStatusType.Added)
orderby file.Id ascending
select file).FirstOrDefault();
}
#endregion
#region Photosynthesis Types
public IQueryable<PhotosynthesisType> GetPhotosynthesisTypes()
{
return _db.PhotosynthesisTypes.OrderBy(pt => pt.SortOrder);
@@ -70,5 +125,7 @@ namespace LeafWeb.Core.DAL
{
return _db.PhotosynthesisTypes.Find(id);
}
#endregion
}
}
Core/Entities/LeafInput.cs
+1 -1
View File
@@ -8,7 +8,7 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual ICollection<LeafInputFile> LeafInputFiles { get; set; }
public virtual ICollection<LeafInputFile> Files { get; set; }
[Required(ErrorMessage = "Name required")]
public string Name { get; set; }
Core/Entities/LeafInputFile.cs
+1 -1
View File
@@ -18,6 +18,6 @@ namespace LeafWeb.Core.Entities
public byte[] Contents { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> LeafInputStatuses { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
}
}
Core/Remote/IPiscalClient.cs
+1 -1
View File
@@ -6,6 +6,6 @@ namespace LeafWeb.Core.Remote
{
void SubmitLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
}
}
Core/Remote/PiscalSshClient.cs
+43 -28
View File
@@ -11,7 +11,7 @@ namespace LeafWeb.Core.Remote
public class PiscalSshClient : IPiscalClient
{
private const string BaseDirectory = "./LeafWeb";
private const string RemoteScriptPath = "./piscal_manager.sh";
private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh";
private readonly PasswordConnectionInfo _connectionInfo;
public PiscalSshClient(string connectionString)
@@ -53,61 +53,76 @@ namespace LeafWeb.Core.Remote
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
var exeCommand = ssh.CreateCommand(commandText);
exeCommand.Execute();
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Console.Write(exeCommand.Result);
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Error);
if (exeCommand.ExitStatus != 0)
throw new PiscalClientException(exeCommand.Error);
Console.Write(command.Result);
}
}
public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
{
var statusRaw = GetLeafInputStatusRaw(file);
switch (statusRaw[0])
{
case "running":
return PiscalStatus.Running;
case "success":
return PiscalStatus.Success;
default:
return PiscalStatus.Error;
}
}
private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
{
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
var exeCommand = ssh.CreateCommand(commandText);
exeCommand.Execute();
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Console.Write(exeCommand.Result);
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Error);
return command.Result
.SplitNewLine()
.Where(s => s.Length > 0)
.Select(s => s.Trim()).ToArray();
}
return PiscalStatus.Success;
}
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file)
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
{
var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
string[] filenames;
// get output files
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"ls -1 {outputDirectory}";
var lsCommand = ssh.CreateCommand(commandText);
lsCommand.Execute();
ssh.Disconnect();
var status = GetLeafInputStatusRaw(file);
if (status[0] != "success")
throw new PiscalClientException("output not available, status is " + status[0]);
filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
Console.Write(lsCommand.Result);
}
var filePaths = status.Skip(1);
using (var scp = GetScpClient())
{
scp.Connect();
foreach (var filename in filenames)
foreach (var filePath in filePaths)
{
using (var stream = new MemoryStream())
{
var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
scp.Download(filePath, stream);
yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
yield return
new PiscalLeafOutputFile
{
Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(),
DirectoryName = file.DirectoryName
};
}
}
Core/Remote/piscal_launcher.sh
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
# piscal launcher
usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
where:
-h show this help text
-d working directory
-f input filename"
directory=""
input_filename=""
while getopts "hd:f:" opt; do
case "$opt" in
h )
echo "$usage"
exit
;;
d )
directory=$OPTARG
;;
f )
input_filename=$OPTARG
task="process"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
exit 1
;;
esac
done
if [ -z "$directory" ]; then
echo "working directory required (-d)"
exit 1
fi
if [ ! -d "$directory" ]; then
echo "working directory $directory not found"
exit 1
fi
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
if [ ! -f "$directory/$input_filename" ]; then
echo "input filename $directory/$input_filename not found"
exit 1
fi
output_directory="$directory/output"
sleep 1m
if [ ! -d "$output_directory" ]; then
mkdir "$output_directory"
fi
# TODO: run actual command
cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"
Core/Remote/piscal_manager.sh
+28 -23
View File
@@ -1,12 +1,12 @@
#~/bin/bash
#!/bin/bash
# piscal manager script
usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
where:
-h show this help text
-d working directory name
-o input filename
-f input filename
-s job status"
# http://stackoverflow.com/a/14203146/99492
@@ -16,9 +16,10 @@ where:
base_directory="/home/poprhythm/LeafWeb"
directory_name=""
input_filename=""
get_status=false
task="start" # default task
pid_filename="piscal.pid"
out_filename="piscal.out"
launcher="$base_directory/piscal_launcher.sh"
while getopts "hd:f:s" opt; do
#echo "$opt = $OPTARG"
@@ -32,9 +33,10 @@ while getopts "hd:f:s" opt; do
;;
f )
input_filename=$OPTARG
task="start"
;;
s )
get_status=true
task="get_status"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -46,24 +48,30 @@ if [ -z "$directory_name" ]; then
echo "directory name required (-d)"
exit 1
fi
if [ ! -d "$base_directory/$directory_name" ]; then
echo "directory $base_directory/$directory_name not found"
working_directory="$base_directory/$directory_name"
if [ ! -d "$working_directory" ]; then
echo "directory $working_directory not found"
exit 1
fi
if [ "$get_status" = false ]; then
if [ "$task" = "start" ]; then
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
echo "input filename $base_directory/$directory_name/$input_filename not found"
if [ ! -f "$working_directory/$input_filename" ]; then
echo "input filename $working_directory/$input_filename not found"
exit 1
fi
fi
if [ "$get_status" = true ]; then
pid_path="$base_directory/$directory_name/$pid_filename"
command="$launcher -d $working_directory -f $input_filename"
nohup ${command} > $working_directory/$out_filename 2>&1 &
# write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename
echo started
elif [ "$task" = "get_status" ]; then
pid_path="$working_directory/$pid_filename"
# if the pid doesn't exist, then process never started
if [ ! -f "$pid_path" ]; then
@@ -79,15 +87,12 @@ if [ "$get_status" = true ]; then
else
# otherwise, it is complete, check the output for success/error
# TODO: examine output for errors, etc
echo complete
#cat piscal.out
if [ ! -d "$working_directory/output" ]; then
echo "output directory $working_directory/output not found"
exit 1
fi
echo success
find "$working_directory/output"/*
fi
else
command="sleep 100s"
nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
# write the PID to a temp file to check for completion later
echo $! > $base_directory/$directory_name/$pid_filename
fi
Core/Utility/StringExtensions.cs
+5
View File
@@ -45,5 +45,10 @@ namespace LeafWeb.Core.Utility
{
return Regex.Replace(input, @"[^\w_]", "");
}
public static string FilenameFromPath(this string path)
{
return Regex.Replace(path, @".*/([^/]*$)", "$1");
}
}
}
Web/Controllers/LeafInputController.cs
+16 -1
View File
@@ -1,9 +1,12 @@
using System;
using System.Configuration;
using System.IO;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using Hangfire;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
using LeafWeb.Web.Attributes;
using LeafWeb.Web.ViewModels;
using LeafWeb.Web.ViewModels.LeafInput;
@@ -69,7 +72,7 @@ namespace LeafWeb.Web.Controllers
// convert viewModel into Model
var leafInput = viewModel.GetFileInput(DataService);
// load files into LeafInputFile
leafInput.LeafInputFiles =
leafInput.Files =
(from f in files
let bytes = System.IO.File.ReadAllBytes(f.FullName)
select new LeafInputFile {Filename = f.Name, Contents = bytes}).ToList();
@@ -79,6 +82,12 @@ namespace LeafWeb.Web.Controllers
DeleteBackloadDirectory(Session.SessionID);
foreach (var file in leafInput.Files.Select(f => new PiscalLeafInputFile(f)))
{
BackgroundJob.Enqueue(() => ProcessLeafInput(file));
}
SetStatusMessage(
HttpUtility.HtmlEncode(
$"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine
@@ -91,6 +100,12 @@ namespace LeafWeb.Web.Controllers
return View("Index", viewModel);
}
public void ProcessLeafInput(PiscalLeafInputFile leafInputFile)
{
var piscalSshClient = new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
piscalSshClient.SubmitLeafInputFile(leafInputFile);
}
private FileInfo[] GetBackloadDirectoryFiles(string directoryName)
{
var path = Path.Combine(Server.MapPath("~/Files/"), directoryName + "\\");
Web/Services/JobService.cs
+65
View File
@@ -0,0 +1,65 @@
using System;
using System.Configuration;
using System.Linq;
using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
public class JobService : IDisposable
{
protected readonly DataService DataService = new DataService();
protected IPiscalClient GetPiscalClient()
{
return new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
}
public void ProcessNextLeafInput()
{
var leafInputFile = DataService.GetNextUnprocessedLeafInputFile();
if (leafInputFile == null) // no inputs, quit
return;
var inputFile = new PiscalLeafInputFile(leafInputFile);
var piscalSshClient = GetPiscalClient();
piscalSshClient.SubmitLeafInputFile(inputFile);
DataService.SetLeafInputFileStatus(leafInputFile, LeafInputStatusType.ProcessStarted);
}
public void UpdateLeafInputStatus()
{
var leafInputFiles =
DataService
.GetLeafInputFiles(LeafInputStatusType.ProcessStarted)
.Select(f => new PiscalLeafInputFile(f));
var piscalClient = GetPiscalClient();
foreach (var file in leafInputFiles)
{
var status = piscalClient.GetLeafInputStatus(file);
switch (status)
{
case PiscalStatus.Success:
// retrieve LeafOutput
var outputFile = piscalClient.RetrieveLeafOutput(file);
break;
case PiscalStatus.Error:
// record error
break;
case PiscalStatus.Running:
// do nothing
break;
}
}
}
public void Dispose()
{
DataService.Dispose();
}
}
}
Web/Web.csproj
+1
View File
@@ -921,6 +921,7 @@
</Compile>
<Compile Include="Attributes\HttpParamActionAttribute.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Services\JobService.cs" />
<Compile Include="Startup.cs" />
<Compile Include="Utility\MarkdownHelper.cs" />
<Compile Include="Utility\Validation.cs" />