Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retrieve LeafOutput operational

Browse Source
This commit is contained in:
poprhythm
2016-02-26 11:54:07 -05:00
parent 76722345a8
commit 59e2f9d8bd
14 changed files with 296 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core/Core.csproj
+1
View File
@@ -111,6 +111,7 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="packages.config" />
<None Include="Remote\piscal_launcher.sh" />
<None Include="Remote\piscal_manager.sh" />
</ItemGroup>
<ItemGroup />
Core/DAL/DataService.cs
+60 -3
View File
@@ -38,8 +38,7 @@ namespace LeafWeb.Core.DAL
}
#endregion
#region LeafInput Sites
#region LeafInput
public IQueryable<LeafInput> GetLeafInputs()
{
@@ -54,13 +53,69 @@ namespace LeafWeb.Core.DAL
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
_db.LeafInputs.Add(leafInput);
foreach (var leafInputFile in leafInput.Files)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added);
}
_db.SaveChanges();
}
#endregion
#region LeafInputFile
public IQueryable<LeafInputFile> GetLeafInputFiles()
{
return _db.LeafInputFiles;
}
public IQueryable<LeafInputFile> GetLeafInputFiles(LeafInputStatusType status)
{
return
from file in _db.LeafInputFiles
where file.CurrentStatus == status
select file;
}
public void UpdateLeafInputFile(LeafInputFile leafInputFile)
{
_db.Entry(leafInputFile).State = EntityState.Modified;
_db.SaveChanges();
}
private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
leafInputFile.CurrentStatus = status;
var leafInputFileStatus = new LeafInputFileStatus
{
Status = status,
DateTime = DateTime.Now,
Description = description,
LeafInputFile = leafInputFile
};
if (leafInputFile.StatusHistory == null)
leafInputFile.StatusHistory = new List<LeafInputFileStatus>();
leafInputFile.StatusHistory.Add(leafInputFileStatus);
}
public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
{
SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
UpdateLeafInputFile(leafInputFile);
}
public LeafInputFile GetNextUnprocessedLeafInputFile()
{
return
(from file in GetLeafInputFiles(LeafInputStatusType.Added)
orderby file.Id ascending
select file).FirstOrDefault();
}
#endregion
#region Photosynthesis Types
public IQueryable<PhotosynthesisType> GetPhotosynthesisTypes()
{
return _db.PhotosynthesisTypes.OrderBy(pt => pt.SortOrder);
@@ -70,5 +125,7 @@ namespace LeafWeb.Core.DAL
{
return _db.PhotosynthesisTypes.Find(id);
}
#endregion
}
}
Core/Entities/LeafInput.cs
+1 -1
View File
@@ -8,7 +8,7 @@ namespace LeafWeb.Core.Entities
{
public int Id { get; set; }
public virtual ICollection<LeafInputFile> LeafInputFiles { get; set; }
public virtual ICollection<LeafInputFile> Files { get; set; }
[Required(ErrorMessage = "Name required")]
public string Name { get; set; }
Core/Entities/LeafInputFile.cs
+1 -1
View File
@@ -18,6 +18,6 @@ namespace LeafWeb.Core.Entities
public byte[] Contents { get; set; }
public LeafInputStatusType CurrentStatus { get; set; }
public virtual ICollection<LeafInputFileStatus> LeafInputStatuses { get; set; }
public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
}
}
Core/Remote/IPiscalClient.cs
+1 -1
View File
@@ -6,6 +6,6 @@ namespace LeafWeb.Core.Remote
{
void SubmitLeafInputFile(PiscalLeafInputFile file);
PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file);
IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
}
}
Core/Remote/PiscalSshClient.cs
+43 -28
View File
@@ -11,7 +11,7 @@ namespace LeafWeb.Core.Remote
public class PiscalSshClient : IPiscalClient
{
private const string BaseDirectory = "./LeafWeb";
private const string RemoteScriptPath = "./piscal_manager.sh";
private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh";
private readonly PasswordConnectionInfo _connectionInfo;
public PiscalSshClient(string connectionString)
@@ -53,61 +53,76 @@ namespace LeafWeb.Core.Remote
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
var exeCommand = ssh.CreateCommand(commandText);
exeCommand.Execute();
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Console.Write(exeCommand.Result);
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Error);
if (exeCommand.ExitStatus != 0)
throw new PiscalClientException(exeCommand.Error);
Console.Write(command.Result);
}
}
public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
{
var statusRaw = GetLeafInputStatusRaw(file);
switch (statusRaw[0])
{
case "running":
return PiscalStatus.Running;
case "success":
return PiscalStatus.Success;
default:
return PiscalStatus.Error;
}
}
private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
{
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
var exeCommand = ssh.CreateCommand(commandText);
exeCommand.Execute();
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Console.Write(exeCommand.Result);
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Error);
return command.Result
.SplitNewLine()
.Where(s => s.Length > 0)
.Select(s => s.Trim()).ToArray();
}
return PiscalStatus.Success;
}
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file)
public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
{
var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
string[] filenames;
// get output files
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"ls -1 {outputDirectory}";
var lsCommand = ssh.CreateCommand(commandText);
lsCommand.Execute();
ssh.Disconnect();
var status = GetLeafInputStatusRaw(file);
if (status[0] != "success")
throw new PiscalClientException("output not available, status is " + status[0]);
filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
Console.Write(lsCommand.Result);
}
var filePaths = status.Skip(1);
using (var scp = GetScpClient())
{
scp.Connect();
foreach (var filename in filenames)
foreach (var filePath in filePaths)
{
using (var stream = new MemoryStream())
{
var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
scp.Download(filePath, stream);
yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
yield return
new PiscalLeafOutputFile
{
Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(),
DirectoryName = file.DirectoryName
};
}
}
Core/Remote/piscal_launcher.sh
+58
View File
@@ -0,0 +1,58 @@
#!/bin/bash
# piscal launcher
usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
where:
-h show this help text
-d working directory
-f input filename"
directory=""
input_filename=""
while getopts "hd:f:" opt; do
case "$opt" in
h )
echo "$usage"
exit
;;
d )
directory=$OPTARG
;;
f )
input_filename=$OPTARG
task="process"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
exit 1
;;
esac
done
if [ -z "$directory" ]; then
echo "working directory required (-d)"
exit 1
fi
if [ ! -d "$directory" ]; then
echo "working directory $directory not found"
exit 1
fi
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
if [ ! -f "$directory/$input_filename" ]; then
echo "input filename $directory/$input_filename not found"
exit 1
fi
output_directory="$directory/output"
sleep 1m
if [ ! -d "$output_directory" ]; then
mkdir "$output_directory"
fi
# TODO: run actual command
cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"
Core/Remote/piscal_manager.sh
+28 -23
View File
@@ -1,12 +1,12 @@
#~/bin/bash
#!/bin/bash
# piscal manager script
usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
where:
-h show this help text
-d working directory name
-o input filename
-f input filename
-s job status"
# http://stackoverflow.com/a/14203146/99492
@@ -16,9 +16,10 @@ where:
base_directory="/home/poprhythm/LeafWeb"
directory_name=""
input_filename=""
get_status=false
task="start" # default task
pid_filename="piscal.pid"
out_filename="piscal.out"
launcher="$base_directory/piscal_launcher.sh"
while getopts "hd:f:s" opt; do
#echo "$opt = $OPTARG"
@@ -32,9 +33,10 @@ while getopts "hd:f:s" opt; do
;;
f )
input_filename=$OPTARG
task="start"
;;
s )
get_status=true
task="get_status"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -46,24 +48,30 @@ if [ -z "$directory_name" ]; then
echo "directory name required (-d)"
exit 1
fi
if [ ! -d "$base_directory/$directory_name" ]; then
echo "directory $base_directory/$directory_name not found"
working_directory="$base_directory/$directory_name"
if [ ! -d "$working_directory" ]; then
echo "directory $working_directory not found"
exit 1
fi
if [ "$get_status" = false ]; then
if [ "$task" = "start" ]; then
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
echo "input filename $base_directory/$directory_name/$input_filename not found"
if [ ! -f "$working_directory/$input_filename" ]; then
echo "input filename $working_directory/$input_filename not found"
exit 1
fi
fi
command="$launcher -d $working_directory -f $input_filename"
nohup ${command} > $working_directory/$out_filename 2>&1 &
if [ "$get_status" = true ]; then
pid_path="$base_directory/$directory_name/$pid_filename"
# write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename
echo started
elif [ "$task" = "get_status" ]; then
pid_path="$working_directory/$pid_filename"
# if the pid doesn't exist, then process never started
if [ ! -f "$pid_path" ]; then
@@ -79,15 +87,12 @@ if [ "$get_status" = true ]; then
else
# otherwise, it is complete, check the output for success/error
# TODO: examine output for errors, etc
echo complete
#cat piscal.out
if [ ! -d "$working_directory/output" ]; then
echo "output directory $working_directory/output not found"
exit 1
fi
echo success
find "$working_directory/output"/*
fi
else
command="sleep 100s"
nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
# write the PID to a temp file to check for completion later
echo $! > $base_directory/$directory_name/$pid_filename
fi
Core/Utility/StringExtensions.cs
+5
View File
@@ -45,5 +45,10 @@ namespace LeafWeb.Core.Utility
{
return Regex.Replace(input, @"[^\w_]", "");
}
public static string FilenameFromPath(this string path)
{
return Regex.Replace(path, @".*/([^/]*$)", "$1");
}
}
}