Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add output file type, collect all files

Browse Source
This commit is contained in:
James Kolpack
2016-05-18 13:05:24 -04:00
parent 4343f08733
commit 77ae03534e
19 changed files with 288 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core/Remote/PiscalLeafInput.cs
+2 -2
View File
@@ -10,7 +10,7 @@ namespace LeafWeb.Core.Remote
private static readonly IMapper Mapper;
public int LeafInputId { get; set; }
public string PhotosyntheticType { get; set; }
public string DirectoryName { get; set; }
public string PiscalDirectoryName { get; set; }
public bool SuppressStorageCopy { get; set; }
public PiscalLeafInputFile[] InputFiles { get; set; }
public string NotifyCompleteUrl { get; set; }
@@ -21,7 +21,7 @@ namespace LeafWeb.Core.Remote
new MapperConfiguration(cfg =>
{
cfg.CreateMap<LeafInput, PiscalLeafInput>()
.ForMember(dest => dest.DirectoryName,
.ForMember(dest => dest.PiscalDirectoryName,
opt => opt.MapFrom(src => PiscalUtility.GetPiscalDirectoryName(src)))
.ForMember(dest => dest.LeafInputId, opt => opt.MapFrom(src => src.Id))
.ForMember(dest => dest.InputFiles, opt => opt.MapFrom(src => src.InputFiles.Select(f => new PiscalLeafInputFile(f)).ToArray()))
Core/Remote/PiscalLeafOutputFile.cs
+22 -3
View File
@@ -1,3 +1,4 @@
using System;
using AutoMapper;
using LeafWeb.Core.Entities;
@@ -9,15 +10,33 @@ namespace LeafWeb.Core.Remote
public string Filename { get; set; }
public byte[] Contents { get; set; }
public string DirectoryName { get; set; }
public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(DirectoryName);
public string PiscalDirectoryName { get; set; }
public int LeafInputId => PiscalUtility.GetIdFromDirectoryName(PiscalDirectoryName);
// valid values: "touser", "nottouser", "clninput"
public string OutputFileType { get; set; }
static PiscalLeafOutputFile()
{
var config =
new MapperConfiguration(cfg =>
{
cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>();
cfg.CreateMap<PiscalLeafOutputFile, LeafOutputFile>()
.ForMember(dest => dest.FileType, opt =>
opt.ResolveUsing(src =>
{
switch (src.OutputFileType)
{
case "touser":
return LeafOutputFileType.ToUser;
case "nottouser":
return LeafOutputFileType.NotToUser;
case "clninput":
return LeafOutputFileType.CleanedInput;
default:
throw new ArgumentException();
}
}));
});
Mapper = config.CreateMapper();
}
Core/Remote/PiscalSshClient.cs
+14 -7
View File
@@ -59,7 +59,7 @@ namespace LeafWeb.Core.Remote
public void RunLeafInput(PiscalLeafInput leafInput)
{
var inputDirectory = $"{BaseDirectory}/{leafInput.DirectoryName}";
var inputDirectory = $"{BaseDirectory}/{leafInput.PiscalDirectoryName}";
CopyLeafInput(leafInput, inputDirectory);
@@ -67,7 +67,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -p {leafInput.PhotosyntheticType} -s";
var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -p {leafInput.PhotosyntheticType} -s";
if (leafInput.SuppressStorageCopy)
commandText += " -t";
@@ -80,7 +80,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine());
throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
_logger.Debug("RunLeafInput result: " + command.Result.TrimEndNewLine());
}
@@ -108,13 +108,13 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName}";
var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName}";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
if (command.ExitStatus != 0)
throw new PiscalClientException(leafInput.LeafInputId, command.Result.TrimEndNewLine());
throw new PiscalClientException(leafInput.LeafInputId, command.Error.TrimEndNewLine());
return command.Result
.SplitNewLine()
@@ -138,8 +138,14 @@ namespace LeafWeb.Core.Remote
using (var scp = GetScpClient())
{
scp.Connect();
string outputFileType = string.Empty;
foreach (var filePath in filePaths)
{
if (filePath.StartsWith("#"))
{
outputFileType = filePath.Substring(1);
continue;
}
using (var stream = new MemoryStream())
{
scp.Download(filePath, stream);
@@ -148,7 +154,8 @@ namespace LeafWeb.Core.Remote
{
Contents = stream.ToArray(),
Filename = filePath.FilenameFromPath(),
DirectoryName = leafInput.DirectoryName
PiscalDirectoryName = leafInput.PiscalDirectoryName,
OutputFileType = outputFileType
};
}
}
@@ -162,7 +169,7 @@ namespace LeafWeb.Core.Remote
using (var ssh = GetSshClient())
{
ssh.Connect();
var commandText = $"{RemoteScriptPath} -d {leafInput.DirectoryName} -c";
var commandText = $"{RemoteScriptPath} -d {leafInput.PiscalDirectoryName} -c";
var command = ssh.CreateCommand(commandText);
command.Execute();
ssh.Disconnect();
Core/Remote/piscal_launcher.sh
+2 -2
View File
@@ -89,8 +89,7 @@ pushd $run_directory >> /dev/null
if [ -z "$test_output_directory" ]; then
eval $piscal_executable
else
cp "$test_output_directory"/* "$output_directory_touser"/
cp "$test_output_directory"/* "$output_directory_nottouser"/
cp -r "$test_output_directory"/* "$output_directory_base"/
fi
popd >> /dev/null
@@ -99,6 +98,7 @@ popd >> /dev/null
if [ -z "$suppress_storage_copy" ]; then
cp "$output_directory_touser"/* "$storage_directory"/
cp "$output_directory_nottouser"/* "$storage_directory"/
cp "$cleaned_input_directory"/* "$storage_directory"/
fi
# notify given url of completion
Core/Remote/piscal_manager.sh
+15 -4
View File
@@ -27,8 +27,9 @@ task="get_status" # default task
input_directory_name="input"
pid_filename="piscal.pid"
stderr_filename="piscal.err"
output_directory_name="output/fitresult/touser"
cleaned_input_directory_name="output/clninput"
output_directory_name="output/fitresult/touser"
nottouser_directory_name="output/fitresult/nottouser"
launcher="$base_directory/piscal_launcher.sh"
# http://stackoverflow.com/a/14203146/99492
@@ -111,7 +112,7 @@ if [ "$task" = "launch" ]; then
# write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename
echo started
elif [ "$task" = "get_status" ]; then
elif [ "$task" = "get_status" ] || [ "$task" = "get_status_cleaned_input" ]; then
# if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then
echo "not started"
@@ -137,11 +138,21 @@ elif [ "$task" = "get_status" ]; then
fi
echo complete
find "$output_directory"/*
if [ "$task" = "get_status" ]; then
echo "#touser"
find "$output_directory"/*
cleaned_input_directory="$working_directory/$cleaned_input_directory_name"
echo "#clninput"
find "$cleaned_input_directory"/*
nottouser_directory="$working_directory/$nottouser_directory_name"
echo "#nottouser"
find "$nottouser_directory"/*
fi
fi
elif [ "$task" = "cleanup" ]; then
rm -rf "$working_directory"
echo hi
elif [ "$task" = "kill" ]; then
# if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then