diff --git a/Core/Core.csproj b/Core/Core.csproj
index 44a436f..1335eeb 100644
--- a/Core/Core.csproj
+++ b/Core/Core.csproj
@@ -116,6 +116,7 @@
PreserveNewest
+
diff --git a/Core/DAL/DataService.cs b/Core/DAL/DataService.cs
index 37c2f47..b481565 100644
--- a/Core/DAL/DataService.cs
+++ b/Core/DAL/DataService.cs
@@ -58,6 +58,17 @@ namespace LeafWeb.Core.DAL
select file;
}
+ public IQueryable GetRunningLeafInputs()
+ {
+ return
+ from file in _db.LeafInputs
+ where
+ file.CurrentStatus == LeafInputStatusType.Running ||
+ file.CurrentStatus == LeafInputStatusType.Starting ||
+ file.CurrentStatus == LeafInputStatusType.Finishing
+ select file;
+ }
+
public void AddLeafInput(LeafInput leafInput)
{
leafInput.Added = DateTime.Now;
diff --git a/Core/DAL/LeafWebContext.cs b/Core/DAL/LeafWebContext.cs
index d5359fb..72b5b8b 100644
--- a/Core/DAL/LeafWebContext.cs
+++ b/Core/DAL/LeafWebContext.cs
@@ -2,6 +2,7 @@
using System.Data.Entity.ModelConfiguration.Conventions;
using System.Data.Entity.SqlServer;
using LeafWeb.Core.Entities;
+using NLog;
namespace LeafWeb.Core.DAL
{
@@ -16,6 +17,8 @@ namespace LeafWeb.Core.DAL
protected override void OnModelCreating(DbModelBuilder modelBuilder)
{
+ LogManager.GetCurrentClassLogger().Debug("OnModelCreating");
+
modelBuilder.Conventions.Remove();
base.OnModelCreating(modelBuilder);
}
diff --git a/Core/DAL/LeafWebInitializer.cs b/Core/DAL/LeafWebInitializer.cs
index 709473f..a6582fd 100644
--- a/Core/DAL/LeafWebInitializer.cs
+++ b/Core/DAL/LeafWebInitializer.cs
@@ -3,6 +3,7 @@ using System.Linq;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Parsers;
using LeafWeb.Core.Utility;
+using NLog;
namespace LeafWeb.Core.DAL
{
@@ -12,6 +13,8 @@ namespace LeafWeb.Core.DAL
protected override void Seed(LeafWebContext context)
{
+ LogManager.GetCurrentClassLogger().Debug("Seed");
+
// get fluxnet sites from file
var fileInfo = FileUtility.GetContentFile(ContentDirectory, "fluxnet_site_list_all_October2015_with_joins_corrected.csv", true);
var fluxnetSiteCsvParser = new FluxnetSiteCsvParser(fileInfo);
diff --git a/Core/Entities/LeafInput.cs b/Core/Entities/LeafInput.cs
index 1c35af4..e75118c 100644
--- a/Core/Entities/LeafInput.cs
+++ b/Core/Entities/LeafInput.cs
@@ -29,7 +29,8 @@ namespace LeafWeb.Core.Entities
[Required(ErrorMessage = "Site Id required")]
public string SiteId { get; set; }
- [Required(ErrorMessage = "PhotosynthesisType required")]
+ // [Required(ErrorMessage = "PhotosynthesisType required")]
+ // http://stackoverflow.com/questions/6038541/ef-validation-failing-on-update-when-using-lazy-loaded-required-properties
public virtual PhotosynthesisType PhotosynthesisType { get; set; }
[DataType(DataType.Date)]
diff --git a/Core/Entities/LeafInputStatusType.cs b/Core/Entities/LeafInputStatusType.cs
index 0348aa3..12492bb 100644
--- a/Core/Entities/LeafInputStatusType.cs
+++ b/Core/Entities/LeafInputStatusType.cs
@@ -3,8 +3,10 @@ namespace LeafWeb.Core.Entities
public enum LeafInputStatusType
{
Pending = 0,
- Running = 1,
- Complete = 2,
- Exception = 3
+ Starting = 1,
+ Running = 2,
+ Finishing = 3,
+ Complete = 4,
+ Exception = 5
}
}
\ No newline at end of file
diff --git a/Core/Remote/PiscalClientException.cs b/Core/Remote/PiscalClientException.cs
index cdb6b8d..9ba1cef 100644
--- a/Core/Remote/PiscalClientException.cs
+++ b/Core/Remote/PiscalClientException.cs
@@ -4,8 +4,10 @@ namespace LeafWeb.Core.Remote
{
public class PiscalClientException : Exception
{
- public PiscalClientException(string error) : base(error)
+ public int LeafInputId { get; private set; }
+ public PiscalClientException(int leafInputId, string error) : base(error)
{
+ LeafInputId = leafInputId;
}
}
}
\ No newline at end of file
diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs
index c9cd001..664aef1 100644
--- a/Core/Remote/PiscalSshClient.cs
+++ b/Core/Remote/PiscalSshClient.cs
@@ -74,7 +74,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
- throw new PiscalClientException(command.Result);
+ throw new PiscalClientException(leafInput.LeafInputId, command.Result);
_logger.Debug("RunLeafInput result: " + command.Result);
}
@@ -93,7 +93,7 @@ namespace LeafWeb.Core.Remote
case StatusNotStarted:
return PiscalStatus.NotStarted;
default:
- throw new PiscalClientException("Unknown status: " + statusRaw[0]);
+ throw new PiscalClientException(leafInput.LeafInputId, "Unknown status: " + statusRaw[0]);
}
}
@@ -108,7 +108,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
- throw new PiscalClientException(command.Result);
+ throw new PiscalClientException(leafInput.LeafInputId, command.Result);
return command.Result
.SplitNewLine()
@@ -125,7 +125,7 @@ namespace LeafWeb.Core.Remote
// get output files
var status = GetLeafInputStatusRaw(leafInput);
if (status[0] != StatusComplete)
- throw new PiscalClientException("output not available, status is " + status[0]);
+ throw new PiscalClientException(leafInput.LeafInputId, "output not available, status is " + status[0]);
var filePaths = status.Skip(1);
@@ -162,7 +162,7 @@ namespace LeafWeb.Core.Remote
ssh.Disconnect();
if (command.ExitStatus != 0)
- throw new PiscalClientException(command.Error);
+ throw new PiscalClientException(leafInput.LeafInputId, command.Error);
}
}
}
diff --git a/Core/Remote/piscal_launcher.cfg b/Core/Remote/piscal_launcher.cfg
new file mode 100644
index 0000000..2bf8bff
--- /dev/null
+++ b/Core/Remote/piscal_launcher.cfg
@@ -0,0 +1,5 @@
+piscal_executable="/home/poprhythm/Code/piscal/leafres/testrun/piscal"
+storage_directory="/home/poprhythm/LeafWebStorage"
+
+# uncomment to use test output instead of running Piscal
+# test_output_directory="/home/poprhythm/LeafWebTestOutput"
\ No newline at end of file
diff --git a/Core/Remote/piscal_launcher.sh b/Core/Remote/piscal_launcher.sh
index a9c4fe5..4f2e5e0 100644
--- a/Core/Remote/piscal_launcher.sh
+++ b/Core/Remote/piscal_launcher.sh
@@ -1,28 +1,39 @@
#!/bin/bash
# piscal launcher
-usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
+usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal
where:
-h show this help text
-d working directory
- -f input filename"
+ -f input filename
+ -u url to notify on completion"
-directory=""
-input_filename=""
+# http://stackoverflow.com/a/246128/99492
+base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+working_directory=""
+output_directory_base_name="output"
+input_directory_name="input"
+cleaned_input_directory_name="clninput"
+touser_directory_name="fitresult/touser"
+nottouser_directory_name="fitresult/nottouser"
+notify_url=""
-while getopts "hd:f:" opt; do
+# import the settings from piscal.cfg
+# $piscal_executable and $storage_directory
+. "$base_directory/piscal_launcher.cfg"
+
+while getopts "hd:f:u:" opt; do
case "$opt" in
h )
echo "$usage"
exit
;;
d )
- directory=$OPTARG
+ working_directory=$OPTARG
;;
- f )
- input_filename=$OPTARG
- task="process"
+ u )
+ notify_url=$OPTARG
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -30,29 +41,62 @@ while getopts "hd:f:" opt; do
;;
esac
done
-if [ -z "$directory" ]; then
+
+if [ -z "$working_directory" ]; then
echo "working directory required (-d)"
exit 1
fi
-if [ ! -d "$directory" ]; then
- echo "working directory $directory not found"
+if [ ! -d "$working_directory" ]; then
+ echo "working directory $working_directory not found"
exit 1
fi
-if [ -z "$input_filename" ]; then
- echo "input filename required (-f)"
- exit 1
+output_directory_base="$working_directory/$output_directory_base_name"
+output_directory_touser="$output_directory_base/$touser_directory_name"
+output_directory_nottouser="$output_directory_base/$nottouser_directory_name"
+cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name"
+run_directory="$working_directory/run"
+
+# setup output directories
+if [ ! -d "$output_directory_base" ]; then
+ mkdir "$output_directory_base"
fi
-if [ ! -f "$directory/$input_filename" ]; then
- echo "input filename $directory/$input_filename not found"
- exit 1
+if [ ! -d "$cleaned_input_directory" ]; then
+ mkdir "$cleaned_input_directory"
fi
-output_directory="$directory/output"
+#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\
+# -exec cp {} "$cleaned_input_directory" \;
-sleep 1m
-if [ ! -d "$output_directory" ]; then
- mkdir "$output_directory"
+if [ ! -d "$output_directory_base/fitresult" ]; then
+ mkdir "$output_directory_base/fitresult"
+fi
+if [ ! -d "$output_directory_touser" ]; then
+ mkdir "$output_directory_touser"
+fi
+if [ ! -d "$output_directory_nottouser" ]; then
+ mkdir "$output_directory_nottouser"
+fi
+if [ ! -d "$run_directory" ]; then
+ mkdir "$run_directory"
fi
-# TODO: run actual command
-cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"
+# run piscal
+pushd $run_directory >> /dev/null
+
+if [ -z "$test_output_directory" ]; then
+ eval $piscal_executable
+else
+ cp "$test_output_directory"/* "$output_directory_touser"/
+ cp "$test_output_directory"/* "$output_directory_nottouser"/
+fi
+
+popd >> /dev/null
+
+# copy output to storage
+cp "$output_directory_touser"/*.csv "$storage_directory"/
+cp "$output_directory_nottouser"/*.csv "$storage_directory"/
+
+# notify given url of completion
+if [ -z "$notify_url" ]; then
+ wget -qO- "$notify_url" &> /dev/null
+fi
diff --git a/Core/Remote/piscal_manager.sh b/Core/Remote/piscal_manager.sh
index 37dd71c..6c68038 100644
--- a/Core/Remote/piscal_manager.sh
+++ b/Core/Remote/piscal_manager.sh
@@ -1,27 +1,33 @@
#!/bin/bash
# piscal manager script
-usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
+usage="$(basename "$0") [-h] -d directory_name -p photosynthetic_type [-s|-c|-k] -- script to manage Piscal
where:
-h show this help text
-d working directory name
- -f input filename
- -s job status"
+ -p photosynthetic type
+ -s start job
+ -c cleanup directory
+ -k kill job"
+
+# Initialize variables:
+# http://stackoverflow.com/a/246128/99492
+base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+directory_name=""
+photosynthetic_type="C3_photosynthesis_leafweb" #default
+task="get_status" # default task
+input_directory_name="input"
+pid_filename="piscal.pid"
+stderr_filename="piscal.err"
+output_directory_name="output/fitresult/touser"
+cleaned_input_directory_name="output/clninput"
+launcher="$base_directory/piscal_launcher.sh"
# http://stackoverflow.com/a/14203146/99492
# http://wiki.bash-hackers.org/howto/getopts_tutorial
-# Initialize variables:
-base_directory="/home/poprhythm/LeafWeb"
-directory_name=""
-input_filename=""
-task="start" # default task
-pid_filename="piscal.pid"
-out_filename="piscal.out"
-launcher="$base_directory/piscal_launcher.sh"
-
-while getopts "hd:f:s" opt; do
+while getopts "hd:f:p:sck" opt; do
#echo "$opt = $OPTARG"
case "$opt" in
h )
@@ -31,12 +37,17 @@ while getopts "hd:f:s" opt; do
d )
directory_name=$OPTARG
;;
- f )
- input_filename=$OPTARG
- task="start"
+ p )
+ photosynthetic_type=$OPTARG
;;
s )
- task="get_status"
+ task="launch"
+ ;;
+ c )
+ task="cleanup"
+ ;;
+ k )
+ task="kill"
;;
\?) printf "illegal option: -%s\n" "$OPTARG" >&2
echo "$usage" >&2
@@ -44,55 +55,83 @@ while getopts "hd:f:s" opt; do
;;
esac
done
+
+## Examine directories
if [ -z "$directory_name" ]; then
echo "directory name required (-d)"
exit 1
fi
working_directory="$base_directory/$directory_name"
if [ ! -d "$working_directory" ]; then
- echo "directory $working_directory not found"
+ echo "working directory $working_directory not found"
exit 1
fi
-if [ "$task" = "start" ]; then
- if [ -z "$input_filename" ]; then
- echo "input filename required (-f)"
- exit 1
- fi
+input_directory="$working_directory/$input_directory_name"
+if [ ! -d "$input_directory" ]; then
+ echo "input directory $input_directory not found"
+ exit 1
+fi
+pid_path="$working_directory/$pid_filename"
+
+## Process task
+if [ "$task" = "launch" ]; then
+ piscal_config_file="$working_directory"/piscal.cfg
+ # write config file for piscal
+ echo $photosynthetic_type > "$piscal_config_file"
+ find "$input_directory" -maxdepth 1 -type f\
+ -printf '%P\n'\
+ >> "$piscal_config_file"
- if [ ! -f "$working_directory/$input_filename" ]; then
- echo "input filename $working_directory/$input_filename not found"
- exit 1
- fi
-
- command="$launcher -d $working_directory -f $input_filename"
- nohup ${command} > $working_directory/$out_filename 2>&1 &
+ command="$launcher -d $working_directory -f piscal.cfg"
+ # launch it, sending error output to file
+ nohup ${command} > $working_directory/$stderr_filename 2>&1 &
# write the PID to a temp file to check for completion later
echo $! > $working_directory/$pid_filename
echo started
elif [ "$task" = "get_status" ]; then
- pid_path="$working_directory/$pid_filename"
-
- # if the pid doesn't exist, then process never started
+ # if the pid doesn't exist, then process hasn't started
if [ ! -f "$pid_path" ]; then
- echo "no pid file found"
- exit 1
+ echo "not started"
+ exit
fi
-
+
pid=$(head -n 1 $pid_path)
# if the pid exists, check the process status using ps
if ps -p $pid > /dev/null; then
# if it is in ps, then it's still running
echo running
else
- # otherwise, it is complete, check the output for success/error
- # TODO: examine output for errors, etc
- #cat piscal.out
- if [ ! -d "$working_directory/output" ]; then
- echo "output directory $working_directory/output not found"
+ # otherwise, it is complete, check for runtime errors
+ if [ -s "$working_directory/$stderr_filename" ]; then
+ cat "$working_directory/$stderr_filename"
exit 1
fi
- echo success
- find "$working_directory/output"/*
+
+ output_directory="$working_directory/$output_directory_name"
+ if [ ! -d "$output_directory" ]; then
+ echo "output directory $output_directory not found"
+ exit 1
+ fi
+
+ echo complete
+ find "$output_directory"/*
fi
-fi
\ No newline at end of file
+elif [ "$task" = "cleanup" ]; then
+ rm -rf "$working_directory"
+ echo hi
+elif [ "$task" = "kill" ]; then
+ # if the pid doesn't exist, then process hasn't started
+ if [ ! -f "$pid_path" ]; then
+ echo "not started"
+ exit
+ fi
+
+ pid=$(head -n 1 $pid_path)
+ # if the pid exists, check the process status using ps
+ if ps -p $pid > /dev/null; then
+ # if it is in ps, then it's still running
+ kill $pid
+ fi
+ echo killed
+fi
diff --git a/Web/HangfireStartup.cs b/Web/HangfireStartup.cs
index 8342bc5..5a41206 100644
--- a/Web/HangfireStartup.cs
+++ b/Web/HangfireStartup.cs
@@ -3,7 +3,6 @@ using Hangfire;
using Microsoft.Owin;
using LeafWeb.Web;
using LeafWeb.Web.Services;
-using NLog.Internal;
using Owin;
using ConfigurationManager = System.Configuration.ConfigurationManager;
@@ -24,7 +23,6 @@ namespace LeafWeb.Web
private void SetupRecurringJobs()
{
- // TODO: new SqlServerDistributedLock
var queueInterval = ConfigurationManager.AppSettings["ProcessQueueInterval"];
// https://discuss.hangfire.io/t/how-to-create-cron-job-that-is-executing-every-15-minutes/533
RecurringJob.AddOrUpdate(PiscalProcessQueue, p => p.ProcessQueue(), queueInterval);
diff --git a/Web/Services/Cleanup.cs b/Web/Services/Cleanup.cs
new file mode 100644
index 0000000..6e25af3
--- /dev/null
+++ b/Web/Services/Cleanup.cs
@@ -0,0 +1,25 @@
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
+
+namespace LeafWeb.Web.Services
+{
+ public class Cleanup : PiscalQueueWorker
+ {
+ protected override void DoWorkInternal(LeafInput leafInput)
+ {
+ try
+ {
+ Logger.Info("LeafInput: {0}, Cleanup", leafInput.Id);
+
+ PiscalService.Cleanup(leafInput);
+ }
+ catch (PiscalClientException ex)
+ {
+ var errorMessage = FormatException(ex, ex.LeafInputId);
+ Logger.Error(errorMessage, ex);
+ // log the error, but ignore the cleanup issue for now
+ Logger.Info("LeafInput: {0}, Cleanup - likely has not occurred", leafInput.Id);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Services/PiscalQueueBase.cs b/Web/Services/PiscalQueueBase.cs
new file mode 100644
index 0000000..7b9870f
--- /dev/null
+++ b/Web/Services/PiscalQueueBase.cs
@@ -0,0 +1,57 @@
+using System;
+using Hangfire;
+using LeafWeb.Core.DAL;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
+using NLog;
+
+namespace LeafWeb.Web.Services
+{
+ public abstract class PiscalQueueBase : IDisposable
+ {
+ protected readonly DataService DataService;
+ protected readonly PiscalService PiscalService;
+ protected readonly Logger Logger;
+
+ protected PiscalQueueBase(DataService dataService, PiscalService piscalService)
+ {
+ DataService = dataService;
+ PiscalService = piscalService;
+ Logger = LogManager.GetLogger(GetType().Name);
+ }
+
+ protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { }
+
+ protected string FormatException(Exception ex, int leafInputId)
+ {
+ return
+ $"LeafInput: {leafInputId}{Environment.NewLine}" +
+ $"Class: {GetType().Name}{Environment.NewLine}" +
+ $"Exception: {ex.Message}{Environment.NewLine}" +
+ (ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty)
+ + $"StackTrace: {ex.StackTrace}";
+ }
+
+ protected void PiscalExceptionNotify(PiscalClientException ex, LeafInput leafInput)
+ {
+ var errorMessage = FormatException(ex, ex.LeafInputId);
+ Logger.Error(errorMessage);
+ BackgroundJob.Enqueue(
+ email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage));
+
+ // TODO send user email too
+
+ if (leafInput != null)
+ {
+ DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
+ ex.Message);
+ BackgroundJob.Enqueue(s => s.DoWork(leafInput.Id));
+ }
+ }
+
+ public void Dispose()
+ {
+ DataService.Dispose();
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Services/PiscalQueueManager.cs b/Web/Services/PiscalQueueManager.cs
index a2a77e8..57bb4da 100644
--- a/Web/Services/PiscalQueueManager.cs
+++ b/Web/Services/PiscalQueueManager.cs
@@ -2,168 +2,111 @@
using System.Linq;
using System.Threading;
using Hangfire;
-using LeafWeb.Core.DAL;
using LeafWeb.Core.Entities;
using LeafWeb.Core.Remote;
-using NLog;
namespace LeafWeb.Web.Services
{
- public class PiscalQueueManager : IDisposable
+ public class PiscalQueueManager : PiscalQueueBase
{
- private readonly DataService _dataService;
- private readonly PiscalService _piscalService;
- private readonly EmailNotificationService _emailService;
-
- public PiscalQueueManager(DataService dataService, PiscalService piscalService, EmailNotificationService emailService)
- {
- _dataService = dataService;
- _piscalService = piscalService;
- _emailService = emailService;
- }
-
- public PiscalQueueManager() : this(new DataService(), new PiscalService(), new EmailNotificationService()) {}
-
private static readonly object ProcessQueueLock = new object();
public void ProcessQueue()
{
- var logger = LogManager.GetCurrentClassLogger();
-
+ // prevent multiple entry into processing the queue
if (Monitor.TryEnter(ProcessQueueLock))
{
- logger.Trace("ProcessQueue entered");
+ Logger.Trace("ProcessQueue entered");
try
{
- ProcessRunning(logger);
+ UpdateRunning();
- ProcessQueue(logger);
+ StartNextPending();
}
finally
{
- logger.Trace("ProcessQueue exit");
+ Logger.Trace("ProcessQueue exit");
Monitor.Exit(ProcessQueueLock);
}
}
else
{
- logger.Trace("ProcessQueue locked, queue already processing");
+ Logger.Trace("ProcessQueue locked, queue already processing");
}
}
-
- private void ProcessQueue(ILogger logger)
+
+ private void StartNextPending()
{
- var runningLeafInputs = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
+ var runningLeafInputs = DataService.GetRunningLeafInputs().ToList();
if (runningLeafInputs.Any())
{
- logger.Trace("Leaf input currently running , don't enqueue any new");
+ Logger.Trace("Leaf input currently running , don't enqueue any new");
return;
}
- var pending =
- _dataService
+ var pendingInput =
+ DataService
.GetLeafInputs(LeafInputStatusType.Pending)
.OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime))
.FirstOrDefault();
- if (pending == null)
+ if (pendingInput == null)
{
- logger.Trace("No pending leaf input");
+ Logger.Trace("No pending leaf input");
return;
}
- logger.Info("LeafInputFile: {0}, Start", pending.Id);
- try
- {
- _piscalService.Run(pending);
- }
- catch (PiscalClientException ex)
- {
- var errorMessage =
- $"LeafInputFile: {pending.Id}\r\nProcessRunning Exception: {ex.Message}"
- + $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}";
- logger.Error(errorMessage);
- BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessQueue Exception", errorMessage));
-
- _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Exception, "Error occurred starting LeafInput", ex.Message);
- logger.Info("LeafInputFile: {0}, Cleanup", pending.Id);
- _piscalService.Cleanup(pending);
-
- // TODO: re-queue?
- //_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput");
- }
- logger.Trace("LeafInputFile: {0}, Set Pending", pending.Id);
- _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running);
+ var pendingInputId = pendingInput.Id;
+ Logger.Info("LeafInput: {0}, Starting", pendingInputId);
+ DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting);
+ BackgroundJob.Enqueue(c => c.DoWork(pendingInputId));
}
- private void ProcessRunning(ILogger logger)
+ private void UpdateRunning()
{
- var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
+ var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList();
foreach (var leafInput in running)
{
try
{
- var status = _piscalService.GetStatus(leafInput);
+ var status = PiscalService.GetStatus(leafInput);
+ var leafInputId = leafInput.Id;
switch (status)
{
case PiscalStatus.NotStarted:
- logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", leafInput.Id);
- // if it's not started, try to requeue the process - this is unusual state
- _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending);
+ // if it's not started - this is unusual state
+ Logger.Warn("LeafInput: {0}, Piscal Not Started", leafInput.Id);
break;
case PiscalStatus.Running:
- logger.Trace("LeafInputFile: {0}, Running", leafInput.Id);
+ Logger.Trace("LeafInput: {0}, Piscal Running", leafInput.Id);
// continue running
break;
case PiscalStatus.Complete:
- logger.Info("LeafInputFile: {0}, Complete", leafInput.Id);
- // collect the leaf output
- var leafOutputFiles = _piscalService.RetrieveOutputFiles(leafInput).ToList();
+ // TODO: change to "retrieving output"?
+ DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing);
- logger.Trace("LeafInputFile: {0}, saving output files", leafInput.Id);
- foreach (var outputFile in leafOutputFiles)
- _dataService.AddLeafOutputFile(outputFile);
-
- logger.Info("LeafInputFile: {0}, output files: {1}", leafInput.Id,
- string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
-
- // update db
- logger.Trace("LeafInputFile: {0}, Set Complete", leafInput.Id);
- _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
-
- BackgroundJob.Enqueue(() => _emailService.SendLeafWebComplete(leafInput.Id));
-
- // remove working data from the server
- logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
- _piscalService.Cleanup(leafInput);
+ var retrieveFilesId = BackgroundJob.Enqueue(s => s.DoWork(leafInputId));
+ BackgroundJob.ContinueWith(retrieveFilesId, s => s.DoWork(leafInputId));
+ BackgroundJob.ContinueWith(retrieveFilesId,
+ email => email.SendLeafWebComplete(leafInputId));
+ BackgroundJob.ContinueWith(retrieveFilesId, s => s.DoWork(leafInputId));
break;
}
}
+ catch (PiscalClientException ex)
+ {
+ PiscalExceptionNotify(ex, leafInput);
+ }
catch (Exception ex)
{
- var errorMessage =
- $"LeafInputFile: {leafInput.Id}\r\nProcessRunning Exception: {ex.Message}"
- + $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}";
- logger.Error(errorMessage);
- BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessRunning Exception", errorMessage));
-
- _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", ex.Message);
-
- // TODO: Send email
- logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id);
- _piscalService.Cleanup(leafInput);
- // TODO: re-queue ?
+ var errorMessage = FormatException(ex, leafInput.Id);
+ Logger.Error(errorMessage);
}
}
}
-
- public void Dispose()
- {
- _dataService.Dispose();
- }
}
}
\ No newline at end of file
diff --git a/Web/Services/PiscalQueueWorker.cs b/Web/Services/PiscalQueueWorker.cs
new file mode 100644
index 0000000..16a323a
--- /dev/null
+++ b/Web/Services/PiscalQueueWorker.cs
@@ -0,0 +1,39 @@
+using System;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
+
+namespace LeafWeb.Web.Services
+{
+ public abstract class PiscalQueueWorker : PiscalQueueBase
+ {
+ public void DoWork(int leafInputId)
+ {
+ LeafInput leafInput = null;
+ try
+ {
+ leafInput = DataService.GetLeafInput(leafInputId);
+ DoWorkInternal(leafInput);
+ }
+ catch (PiscalClientException ex)
+ {
+ PiscalExceptionNotify(ex, leafInput);
+
+ if (leafInput != null)
+ {
+ DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput",
+ ex.Message);
+ }
+ // signal to process next item
+ HangfireStartup.TriggerPiscalProcessQueue();
+ }
+ catch (Exception ex)
+ {
+ var errorMessage = FormatException(ex, leafInputId);
+ Logger.Error(errorMessage);
+ throw; // this will retry via HangFire
+ }
+ }
+
+ protected abstract void DoWorkInternal(LeafInput leafInputId);
+ }
+}
\ No newline at end of file
diff --git a/Web/Services/PiscalService.cs b/Web/Services/PiscalService.cs
index 71153f3..8b6d74b 100644
--- a/Web/Services/PiscalService.cs
+++ b/Web/Services/PiscalService.cs
@@ -5,6 +5,9 @@ using LeafWeb.Core.Remote;
namespace LeafWeb.Web.Services
{
+ ///
+ /// Thin layer over PiscalClient to translate Core entities to Piscal objects
+ ///
public class PiscalService
{
private readonly IPiscalClient _piscalClient;
diff --git a/Web/Services/RetrieveOutputFiles.cs b/Web/Services/RetrieveOutputFiles.cs
new file mode 100644
index 0000000..b6fc083
--- /dev/null
+++ b/Web/Services/RetrieveOutputFiles.cs
@@ -0,0 +1,28 @@
+using System.Linq;
+using LeafWeb.Core.Entities;
+
+namespace LeafWeb.Web.Services
+{
+ public class RetrieveOutputFiles : PiscalQueueWorker
+ {
+ protected override void DoWorkInternal(LeafInput leafInput)
+ {
+ Logger.Trace("LeafInput: {0}, RetrieveOutputFiles", leafInput.Id);
+
+ var leafOutputFiles = PiscalService.RetrieveOutputFiles(leafInput).ToList();
+
+ Logger.Trace("LeafInput: {0}, RetrieveOutputFiles saving output files", leafInput.Id);
+
+ foreach (var outputFile in leafOutputFiles)
+ {
+ if (leafInput.OutputFiles.All(file => file.Filename != outputFile.Filename))
+ DataService.AddLeafOutputFile(outputFile);
+ else
+ Logger.Warn("LeafInput: {0}, RetrieveOutputFiles duplicate file name: {1}", leafInput.Id, outputFile.Filename);
+ }
+
+ Logger.Info("LeafInput: {0}, RetrieveOutputFiles output files: {1}", leafInput.Id,
+ string.Join(", ", leafOutputFiles.Select(o => o.Filename)));
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Services/SetComplete.cs b/Web/Services/SetComplete.cs
new file mode 100644
index 0000000..535579a
--- /dev/null
+++ b/Web/Services/SetComplete.cs
@@ -0,0 +1,16 @@
+using LeafWeb.Core.Entities;
+
+namespace LeafWeb.Web.Services
+{
+ public class SetComplete : PiscalQueueWorker
+ {
+ protected override void DoWorkInternal(LeafInput leafInput)
+ {
+ Logger.Trace("LeafInput: {0}, Set Complete", leafInput.Id);
+
+ DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete);
+
+ HangfireStartup.TriggerPiscalProcessQueue();
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Services/StartPending.cs b/Web/Services/StartPending.cs
new file mode 100644
index 0000000..9e98f97
--- /dev/null
+++ b/Web/Services/StartPending.cs
@@ -0,0 +1,16 @@
+using LeafWeb.Core.Entities;
+
+namespace LeafWeb.Web.Services
+{
+ public class StartPending : PiscalQueueWorker
+ {
+ protected override void DoWorkInternal(LeafInput leafInput)
+ {
+ Logger.Trace("LeafInput: {0}, Run", leafInput.Id);
+
+ PiscalService.Run(leafInput);
+
+ DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running);
+ }
+ }
+}
\ No newline at end of file
diff --git a/Web/Web.csproj b/Web/Web.csproj
index 33a9831..18fbb75 100644
--- a/Web/Web.csproj
+++ b/Web/Web.csproj
@@ -960,10 +960,16 @@
+
+
+
+
+
+