Kolpack-Software-Consulting/LeafWeb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Piscal Queue working more resilient to error

Browse Source
This commit is contained in:
James Kolpack
2016-04-18 10:44:36 -04:00
parent 6623aa48ea
commit ca1a235c1d
21 changed files with 418 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Core/Core.csproj
+1
View File
@@ -116,6 +116,7 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="packages.config" />
<None Include="Remote\piscal_launcher.cfg" />
<None Include="Remote\piscal_launcher.sh" />
<None Include="Remote\piscal_manager.sh" />
</ItemGroup>
Core/DAL/DataService.cs
+11
View File
@@ -58,6 +58,17 @@ namespace LeafWeb.Core.DAL
select file;
}
public IQueryable<LeafInput> GetRunningLeafInputs()
{
return
from file in _db.LeafInputs
where
file.CurrentStatus == LeafInputStatusType.Running ||
file.CurrentStatus == LeafInputStatusType.Starting ||
file.CurrentStatus == LeafInputStatusType.Finishing
select file;
}
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
Core/DAL/LeafWebContext.cs
+3
View File
@@ -2,6 +2,7 @@
using System.Data.Entity.ModelConfiguration.Conventions;
using System.Data.Entity.SqlServer;
using LeafWeb.Core.Entities;
using NLog;
namespace LeafWeb.Core.DAL
{
@@ -16,6 +17,8 @@ namespace LeafWeb.Core.DAL
protected override void OnModelCreating(DbModelBuilder modelBuilder)
{
LogManager.GetCurrentClassLogger().Debug("OnModelCreating");
modelBuilder.Conventions.Remove<PluralizingTableNameConvention>();
base.OnModelCreating(modelBuilder);
}
Core/DAL/LeafWebInitializer.cs
+3
View File
@@ -3,6 +3,7 @@ using System.Linq;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Parsers;
using LeafWeb.Core.Utility;
using NLog;
namespace LeafWeb.Core.DAL
{
@@ -12,6 +13,8 @@ namespace LeafWeb.Core.DAL
protected override void Seed(LeafWebContext context)
{
LogManager.GetCurrentClassLogger().Debug("Seed");
// get fluxnet sites from file
var fileInfo = FileUtility.GetContentFile(ContentDirectory, "fluxnet_site_list_all_October2015_with_joins_corrected.csv", true);
var fluxnetSiteCsvParser = new FluxnetSiteCsvParser(fileInfo);
Core/Entities/LeafInput.cs
+2 -1
View File
@@ -29,7 +29,8 @@ namespace LeafWeb.Core.Entities
[Required(ErrorMessage = "Site Id required")]
public string SiteId { get; set; }
[Required(ErrorMessage = "PhotosynthesisType required")]
// [Required(ErrorMessage = "PhotosynthesisType required")]
// http://stackoverflow.com/questions/6038541/ef-validation-failing-on-update-when-using-lazy-loaded-required-properties
public virtual PhotosynthesisType PhotosynthesisType { get; set; }
[DataType(DataType.Date)]
Core/Entities/LeafInputStatusType.cs
+5 -3
View File
@@ -3,8 +3,10 @@ namespace LeafWeb.Core.Entities
public enum LeafInputStatusType
{
Pending = 0,
Running = 1,
Complete = 2,
Exception = 3
Starting = 1,
Running = 2,
Finishing = 3,
Complete = 4,
Exception = 5
}
}
Core/Remote/PiscalClientException.cs
+3 -1
View File
@@ -4,8 +4,10 @@ namespace LeafWeb.Core.Remote
{
public class PiscalClientException : Exception
{
public PiscalClientException(string error) : base(error)
public int LeafInputId { get; private set; }
public PiscalClientException(int leafInputId, string error) : base(error)
{
LeafInputId = leafInputId;
}
}
}
Core/Remote/PiscalSshClient.cs
+5 -5
View File
@@ -74,7 +74,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Result);
throw new PiscalClientException(leafInput.LeafInputId, command.Result);
_logger.Debug("RunLeafInput result: " + command.Result);
}
@@ -93,7 +93,7 @@ namespace LeafWeb.Core.Remote
case StatusNotStarted:
return PiscalStatus.NotStarted;
default:
throw new PiscalClientException("Unknown status: " + statusRaw[0]);
throw new PiscalClientException(leafInput.LeafInputId, "Unknown status: " + statusRaw[0]);
}
}
@@ -108,7 +108,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Result);
throw new PiscalClientException(leafInput.LeafInputId, command.Result);
return command.Result
.SplitNewLine()
@@ -125,7 +125,7 @@ namespace LeafWeb.Core.Remote
// get output files
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusComplete)
throw new PiscalClientException("output not available, status is " + status[0]);
throw new PiscalClientException(leafInput.LeafInputId, "output not available, status is " + status[0]);
var filePaths = status.Skip(1);
@@ -162,7 +162,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
throw new PiscalClientException(command.Error);
throw new PiscalClientException(leafInput.LeafInputId, command.Error);
}
}
}
Core/Remote/piscal_launcher.cfg
+5
View File
@@ -0,0 +1,5 @@
piscal_executable="/home/poprhythm/Code/piscal/leafres/testrun/piscal"
storage_directory="/home/poprhythm/LeafWebStorage"
# uncomment to use test output instead of running Piscal
# test_output_directory="/home/poprhythm/LeafWebTestOutput"
Core/Remote/piscal_launcher.sh
+68 -24
View File
@@ -1,28 +1,39 @@
#!/bin/bash
# piscal launcher
usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal
where:
-h show this help text
-d working directory
-f input filename"
-f input filename
-u url to notify on completion"
directory=""
input_filename=""
# http://stackoverflow.com/a/246128/99492
base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
working_directory=""
output_directory_base_name="output"
input_directory_name="input"
cleaned_input_directory_name="clninput"
touser_directory_name="fitresult/touser"
nottouser_directory_name="fitresult/nottouser"
notify_url=""
while getopts "hd:f:" opt; do
# import the settings from piscal.cfg
# $piscal_executable and $storage_directory
. "$base_directory/piscal_launcher.cfg"
while getopts "hd:f:u:" opt; do
case "$opt" in
h )
echo "$usage"
exit
;;
d )
directory=$OPTARG
working_directory=$OPTARG
;;
f )
input_filename=$OPTARG
task="process"
u )
notify_url=$OPTARG
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -30,29 +41,62 @@ while getopts "hd:f:" opt; do
;;
esac
done
if [ -z "$directory" ]; then
if [ -z "$working_directory" ]; then
echo "working directory required (-d)"
exit 1
fi
if [ ! -d "$directory" ]; then
echo "working directory $directory not found"
if [ ! -d "$working_directory" ]; then
echo "working directory $working_directory not found"
exit 1
fi
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
output_directory_base="$working_directory/$output_directory_base_name"
output_directory_touser="$output_directory_base/$touser_directory_name"
output_directory_nottouser="$output_directory_base/$nottouser_directory_name"
cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name"
run_directory="$working_directory/run"
# setup output directories
if [ ! -d "$output_directory_base" ]; then
mkdir "$output_directory_base"
fi
if [ ! -f "$directory/$input_filename" ]; then
echo "input filename $directory/$input_filename not found"
exit 1
if [ ! -d "$cleaned_input_directory" ]; then
mkdir "$cleaned_input_directory"
fi
output_directory="$directory/output"
#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\
# -exec cp {} "$cleaned_input_directory" \;
sleep 1m
if [ ! -d "$output_directory" ]; then
mkdir "$output_directory"
if [ ! -d "$output_directory_base/fitresult" ]; then
mkdir "$output_directory_base/fitresult"
fi
if [ ! -d "$output_directory_touser" ]; then
mkdir "$output_directory_touser"
fi
if [ ! -d "$output_directory_nottouser" ]; then
mkdir "$output_directory_nottouser"
fi
if [ ! -d "$run_directory" ]; then
mkdir "$run_directory"
fi
# TODO: run actual command
cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"
# run piscal
pushd $run_directory >> /dev/null
if [ -z "$test_output_directory" ]; then
eval $piscal_executable
else
cp "$test_output_directory"/* "$output_directory_touser"/
cp "$test_output_directory"/* "$output_directory_nottouser"/
fi
popd >> /dev/null
# copy output to storage
cp "$output_directory_touser"/*.csv "$storage_directory"/
cp "$output_directory_nottouser"/*.csv "$storage_directory"/
# notify given url of completion
if [ -z "$notify_url" ]; then
wget -qO- "$notify_url" &> /dev/null
fi
Core/Remote/piscal_manager.sh
+83 -44
View File
@@ -1,27 +1,33 @@
#!/bin/bash
# piscal manager script
usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
usage="$(basename "$0") [-h] -d directory_name -p photosynthetic_type [-s|-c|-k] -- script to manage Piscal
where:
-h show this help text
-d working directory name
-f input filename
-s job status"
-p photosynthetic type
-s start job
-c cleanup directory
-k kill job"
# Initialize variables:
# http://stackoverflow.com/a/246128/99492
base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
directory_name=""
photosynthetic_type="C3_photosynthesis_leafweb" #default
task="get_status" # default task
input_directory_name="input"
pid_filename="piscal.pid"
stderr_filename="piscal.err"
output_directory_name="output/fitresult/touser"
cleaned_input_directory_name="output/clninput"
launcher="$base_directory/piscal_launcher.sh"
# http://stackoverflow.com/a/14203146/99492
# http://wiki.bash-hackers.org/howto/getopts_tutorial
# Initialize variables:
base_directory="/home/poprhythm/LeafWeb"
directory_name=""
input_filename=""
task="start" # default task
pid_filename="piscal.pid"
out_filename="piscal.out"
launcher="$base_directory/piscal_launcher.sh"
while getopts "hd:f:s" opt; do
while getopts "hd:f:p:sck" opt; do
#echo "$opt = $OPTARG"
case "$opt" in
h )
@@ -31,12 +37,17 @@ while getopts "hd:f:s" opt; do
d )
directory_name=$OPTARG
;;
f )
input_filename=$OPTARG
task="start"
p )
photosynthetic_type=$OPTARG
;;
s )
task="get_status"
task="launch"
;;
c )
task="cleanup"
;;
k )
task="kill"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -44,55 +55,83 @@ while getopts "hd:f:s" opt; do
;;
esac
done
## Examine directories
if [ -z "$directory_name" ]; then
echo "directory name required (-d)"
exit 1
fi
working_directory="$base_directory/$directory_name"
if [ ! -d "$working_directory" ]; then
echo "directory $working_directory not found"
echo "working directory $working_directory not found"
exit 1
fi
if [ "$task" = "start" ]; then
if [ -z "$input_filename" ]; then
echo "input filename required (-f)"
exit 1
fi
input_directory="$working_directory/$input_directory_name"
if [ ! -d "$input_directory" ]; then
echo "input directory $input_directory not found"
exit 1
fi
pid_path="$working_directory/$pid_filename"
## Process task
if [ "$task" = "launch" ]; then
piscal_config_file="$working_directory"/piscal.cfg
# write config file for piscal
echo $photosynthetic_type > "$piscal_config_file"
find "$input_directory" -maxdepth 1 -type f\
-printf '%P\n'\
>> "$piscal_config_file"
if [ ! -f "$working_directory/$input_filename" ]; then
echo "input filename $working_directory/$input_filename not found"
exit 1
fi
command="$launcher -d $working_directory -f $input_filename"
nohup ${command} > $working_directory/$out_filename 2>&1 &
command="$launcher -d $working_directory -f piscal.cfg"
# launch it, sending error output to file
nohup ${command} > $working_directory/$stderr_filename 2>&1 &
# write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename
echo started
elif [ "$task" = "get_status" ]; then
pid_path="$working_directory/$pid_filename"
# if the pid doesn't exist, then process never started
# if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then
echo "no pid file found"
exit 1
echo "not started"
exit
fi
pid=$(head -n 1 $pid_path)
# if the pid exists, check the process status using ps
if ps -p $pid > /dev/null; then
# if it is in ps, then it's still running
echo running
else
# otherwise, it is complete, check the output for success/error
# TODO: examine output for errors, etc
#cat piscal.out
if [ ! -d "$working_directory/output" ]; then
echo "output directory $working_directory/output not found"
# otherwise, it is complete, check for runtime errors
if [ -s "$working_directory/$stderr_filename" ]; then
cat "$working_directory/$stderr_filename"
exit 1
fi
echo success
find "$working_directory/output"/*
output_directory="$working_directory/$output_directory_name"
if [ ! -d "$output_directory" ]; then
echo "output directory $output_directory not found"
exit 1
fi
echo complete
find "$output_directory"/*
fi
fi
elif [ "$task" = "cleanup" ]; then
rm -rf "$working_directory"
echo hi
elif [ "$task" = "kill" ]; then
# if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then
echo "not started"
exit
fi
pid=$(head -n 1 $pid_path)
# if the pid exists, check the process status using ps
if ps -p $pid > /dev/null; then
# if it is in ps, then it's still running
kill $pid
fi
echo killed
fi