From badf048c49321d88801825f9c7f7cc88134b0b2a Mon Sep 17 00:00:00 2001 From: James Kolpack Date: Mon, 18 Apr 2016 10:44:36 -0400 Subject: [PATCH] Piscal Queue working more resilient to error --- Core/DAL/DataService.cs | 11 +++ Core/DAL/LeafWebContext.cs | 3 + Core/DAL/LeafWebInitializer.cs | 3 + Core/Entities/LeafInput.cs | 3 +- Core/Entities/LeafInputStatusType.cs | 8 +- Core/Remote/PiscalClientException.cs | 4 +- Core/Remote/PiscalSshClient.cs | 10 +- Core/Remote/piscal_launcher.cfg | 5 + Core/Remote/piscal_launcher.sh | 92 +++++++++++++----- Core/Remote/piscal_manager.sh | 127 ++++++++++++++++--------- Web/HangfireStartup.cs | 2 - Web/Services/Cleanup.cs | 25 +++++ Web/Services/PiscalQueueBase.cs | 57 +++++++++++ Web/Services/PiscalQueueManager.cs | 135 ++++++++------------------- Web/Services/PiscalQueueWorker.cs | 39 ++++++++ Web/Services/PiscalService.cs | 3 + Web/Services/RetrieveOutputFiles.cs | 28 ++++++ Web/Services/SetComplete.cs | 16 ++++ Web/Services/StartPending.cs | 16 ++++ Web/Web.csproj | 6 ++ 20 files changed, 417 insertions(+), 176 deletions(-) create mode 100644 Core/Remote/piscal_launcher.cfg create mode 100644 Web/Services/Cleanup.cs create mode 100644 Web/Services/PiscalQueueBase.cs create mode 100644 Web/Services/PiscalQueueWorker.cs create mode 100644 Web/Services/RetrieveOutputFiles.cs create mode 100644 Web/Services/SetComplete.cs create mode 100644 Web/Services/StartPending.cs diff --git a/Core/DAL/DataService.cs b/Core/DAL/DataService.cs index 37c2f47..b481565 100644 --- a/Core/DAL/DataService.cs +++ b/Core/DAL/DataService.cs @@ -58,6 +58,17 @@ namespace LeafWeb.Core.DAL select file; } + public IQueryable GetRunningLeafInputs() + { + return + from file in _db.LeafInputs + where + file.CurrentStatus == LeafInputStatusType.Running || + file.CurrentStatus == LeafInputStatusType.Starting || + file.CurrentStatus == LeafInputStatusType.Finishing + select file; + } + public void AddLeafInput(LeafInput leafInput) { leafInput.Added = DateTime.Now; diff --git a/Core/DAL/LeafWebContext.cs b/Core/DAL/LeafWebContext.cs index d5359fb..72b5b8b 100644 --- a/Core/DAL/LeafWebContext.cs +++ b/Core/DAL/LeafWebContext.cs @@ -2,6 +2,7 @@ using System.Data.Entity.ModelConfiguration.Conventions; using System.Data.Entity.SqlServer; using LeafWeb.Core.Entities; +using NLog; namespace LeafWeb.Core.DAL { @@ -16,6 +17,8 @@ namespace LeafWeb.Core.DAL protected override void OnModelCreating(DbModelBuilder modelBuilder) { + LogManager.GetCurrentClassLogger().Debug("OnModelCreating"); + modelBuilder.Conventions.Remove(); base.OnModelCreating(modelBuilder); } diff --git a/Core/DAL/LeafWebInitializer.cs b/Core/DAL/LeafWebInitializer.cs index 709473f..a6582fd 100644 --- a/Core/DAL/LeafWebInitializer.cs +++ b/Core/DAL/LeafWebInitializer.cs @@ -3,6 +3,7 @@ using System.Linq; using LeafWeb.Core.Entities; using LeafWeb.Core.Parsers; using LeafWeb.Core.Utility; +using NLog; namespace LeafWeb.Core.DAL { @@ -12,6 +13,8 @@ namespace LeafWeb.Core.DAL protected override void Seed(LeafWebContext context) { + LogManager.GetCurrentClassLogger().Debug("Seed"); + // get fluxnet sites from file var fileInfo = FileUtility.GetContentFile(ContentDirectory, "fluxnet_site_list_all_October2015_with_joins_corrected.csv", true); var fluxnetSiteCsvParser = new FluxnetSiteCsvParser(fileInfo); diff --git a/Core/Entities/LeafInput.cs b/Core/Entities/LeafInput.cs index 1c35af4..e75118c 100644 --- a/Core/Entities/LeafInput.cs +++ b/Core/Entities/LeafInput.cs @@ -29,7 +29,8 @@ namespace LeafWeb.Core.Entities [Required(ErrorMessage = "Site Id required")] public string SiteId { get; set; } - [Required(ErrorMessage = "PhotosynthesisType required")] + // [Required(ErrorMessage = "PhotosynthesisType required")] + // http://stackoverflow.com/questions/6038541/ef-validation-failing-on-update-when-using-lazy-loaded-required-properties public virtual PhotosynthesisType PhotosynthesisType { get; set; } [DataType(DataType.Date)] diff --git a/Core/Entities/LeafInputStatusType.cs b/Core/Entities/LeafInputStatusType.cs index 0348aa3..12492bb 100644 --- a/Core/Entities/LeafInputStatusType.cs +++ b/Core/Entities/LeafInputStatusType.cs @@ -3,8 +3,10 @@ namespace LeafWeb.Core.Entities public enum LeafInputStatusType { Pending = 0, - Running = 1, - Complete = 2, - Exception = 3 + Starting = 1, + Running = 2, + Finishing = 3, + Complete = 4, + Exception = 5 } } \ No newline at end of file diff --git a/Core/Remote/PiscalClientException.cs b/Core/Remote/PiscalClientException.cs index cdb6b8d..9ba1cef 100644 --- a/Core/Remote/PiscalClientException.cs +++ b/Core/Remote/PiscalClientException.cs @@ -4,8 +4,10 @@ namespace LeafWeb.Core.Remote { public class PiscalClientException : Exception { - public PiscalClientException(string error) : base(error) + public int LeafInputId { get; private set; } + public PiscalClientException(int leafInputId, string error) : base(error) { + LeafInputId = leafInputId; } } } \ No newline at end of file diff --git a/Core/Remote/PiscalSshClient.cs b/Core/Remote/PiscalSshClient.cs index c9cd001..664aef1 100644 --- a/Core/Remote/PiscalSshClient.cs +++ b/Core/Remote/PiscalSshClient.cs @@ -74,7 +74,7 @@ namespace LeafWeb.Core.Remote ssh.Disconnect(); if (command.ExitStatus != 0) - throw new PiscalClientException(command.Result); + throw new PiscalClientException(leafInput.LeafInputId, command.Result); _logger.Debug("RunLeafInput result: " + command.Result); } @@ -93,7 +93,7 @@ namespace LeafWeb.Core.Remote case StatusNotStarted: return PiscalStatus.NotStarted; default: - throw new PiscalClientException("Unknown status: " + statusRaw[0]); + throw new PiscalClientException(leafInput.LeafInputId, "Unknown status: " + statusRaw[0]); } } @@ -108,7 +108,7 @@ namespace LeafWeb.Core.Remote ssh.Disconnect(); if (command.ExitStatus != 0) - throw new PiscalClientException(command.Result); + throw new PiscalClientException(leafInput.LeafInputId, command.Result); return command.Result .SplitNewLine() @@ -125,7 +125,7 @@ namespace LeafWeb.Core.Remote // get output files var status = GetLeafInputStatusRaw(leafInput); if (status[0] != StatusComplete) - throw new PiscalClientException("output not available, status is " + status[0]); + throw new PiscalClientException(leafInput.LeafInputId, "output not available, status is " + status[0]); var filePaths = status.Skip(1); @@ -162,7 +162,7 @@ namespace LeafWeb.Core.Remote ssh.Disconnect(); if (command.ExitStatus != 0) - throw new PiscalClientException(command.Error); + throw new PiscalClientException(leafInput.LeafInputId, command.Error); } } } diff --git a/Core/Remote/piscal_launcher.cfg b/Core/Remote/piscal_launcher.cfg new file mode 100644 index 0000000..2bf8bff --- /dev/null +++ b/Core/Remote/piscal_launcher.cfg @@ -0,0 +1,5 @@ +piscal_executable="/home/poprhythm/Code/piscal/leafres/testrun/piscal" +storage_directory="/home/poprhythm/LeafWebStorage" + +# uncomment to use test output instead of running Piscal +# test_output_directory="/home/poprhythm/LeafWebTestOutput" \ No newline at end of file diff --git a/Core/Remote/piscal_launcher.sh b/Core/Remote/piscal_launcher.sh index a9c4fe5..4f2e5e0 100644 --- a/Core/Remote/piscal_launcher.sh +++ b/Core/Remote/piscal_launcher.sh @@ -1,28 +1,39 @@ #!/bin/bash # piscal launcher -usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal +usage="$(basename "$0") [-h] -d working_directory [-u notify_url] -- script to launch Piscal where: -h show this help text -d working directory - -f input filename" + -f input filename + -u url to notify on completion" -directory="" -input_filename="" +# http://stackoverflow.com/a/246128/99492 +base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +working_directory="" +output_directory_base_name="output" +input_directory_name="input" +cleaned_input_directory_name="clninput" +touser_directory_name="fitresult/touser" +nottouser_directory_name="fitresult/nottouser" +notify_url="" -while getopts "hd:f:" opt; do +# import the settings from piscal.cfg +# $piscal_executable and $storage_directory +. "$base_directory/piscal_launcher.cfg" + +while getopts "hd:f:u:" opt; do case "$opt" in h ) echo "$usage" exit ;; d ) - directory=$OPTARG + working_directory=$OPTARG ;; - f ) - input_filename=$OPTARG - task="process" + u ) + notify_url=$OPTARG ;; \?) printf "illegal option: -%s\n" "$OPTARG" >&2 echo "$usage" >&2 @@ -30,29 +41,62 @@ while getopts "hd:f:" opt; do ;; esac done -if [ -z "$directory" ]; then + +if [ -z "$working_directory" ]; then echo "working directory required (-d)" exit 1 fi -if [ ! -d "$directory" ]; then - echo "working directory $directory not found" +if [ ! -d "$working_directory" ]; then + echo "working directory $working_directory not found" exit 1 fi -if [ -z "$input_filename" ]; then - echo "input filename required (-f)" - exit 1 +output_directory_base="$working_directory/$output_directory_base_name" +output_directory_touser="$output_directory_base/$touser_directory_name" +output_directory_nottouser="$output_directory_base/$nottouser_directory_name" +cleaned_input_directory="$output_directory_base/$cleaned_input_directory_name" +run_directory="$working_directory/run" + +# setup output directories +if [ ! -d "$output_directory_base" ]; then + mkdir "$output_directory_base" fi -if [ ! -f "$directory/$input_filename" ]; then - echo "input filename $directory/$input_filename not found" - exit 1 +if [ ! -d "$cleaned_input_directory" ]; then + mkdir "$cleaned_input_directory" fi -output_directory="$directory/output" +#find "$working_directory/$input_directory_name" -maxdepth 1 -type f\ +# -exec cp {} "$cleaned_input_directory" \; -sleep 1m -if [ ! -d "$output_directory" ]; then - mkdir "$output_directory" +if [ ! -d "$output_directory_base/fitresult" ]; then + mkdir "$output_directory_base/fitresult" +fi +if [ ! -d "$output_directory_touser" ]; then + mkdir "$output_directory_touser" +fi +if [ ! -d "$output_directory_nottouser" ]; then + mkdir "$output_directory_nottouser" +fi +if [ ! -d "$run_directory" ]; then + mkdir "$run_directory" fi -# TODO: run actual command -cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory" +# run piscal +pushd $run_directory >> /dev/null + +if [ -z "$test_output_directory" ]; then + eval $piscal_executable +else + cp "$test_output_directory"/* "$output_directory_touser"/ + cp "$test_output_directory"/* "$output_directory_nottouser"/ +fi + +popd >> /dev/null + +# copy output to storage +cp "$output_directory_touser"/*.csv "$storage_directory"/ +cp "$output_directory_nottouser"/*.csv "$storage_directory"/ + +# notify given url of completion +if [ -z "$notify_url" ]; then + wget -qO- "$notify_url" &> /dev/null +fi diff --git a/Core/Remote/piscal_manager.sh b/Core/Remote/piscal_manager.sh index 37dd71c..6c68038 100644 --- a/Core/Remote/piscal_manager.sh +++ b/Core/Remote/piscal_manager.sh @@ -1,27 +1,33 @@ #!/bin/bash # piscal manager script -usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal +usage="$(basename "$0") [-h] -d directory_name -p photosynthetic_type [-s|-c|-k] -- script to manage Piscal where: -h show this help text -d working directory name - -f input filename - -s job status" + -p photosynthetic type + -s start job + -c cleanup directory + -k kill job" + +# Initialize variables: +# http://stackoverflow.com/a/246128/99492 +base_directory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +directory_name="" +photosynthetic_type="C3_photosynthesis_leafweb" #default +task="get_status" # default task +input_directory_name="input" +pid_filename="piscal.pid" +stderr_filename="piscal.err" +output_directory_name="output/fitresult/touser" +cleaned_input_directory_name="output/clninput" +launcher="$base_directory/piscal_launcher.sh" # http://stackoverflow.com/a/14203146/99492 # http://wiki.bash-hackers.org/howto/getopts_tutorial -# Initialize variables: -base_directory="/home/poprhythm/LeafWeb" -directory_name="" -input_filename="" -task="start" # default task -pid_filename="piscal.pid" -out_filename="piscal.out" -launcher="$base_directory/piscal_launcher.sh" - -while getopts "hd:f:s" opt; do +while getopts "hd:f:p:sck" opt; do #echo "$opt = $OPTARG" case "$opt" in h ) @@ -31,12 +37,17 @@ while getopts "hd:f:s" opt; do d ) directory_name=$OPTARG ;; - f ) - input_filename=$OPTARG - task="start" + p ) + photosynthetic_type=$OPTARG ;; s ) - task="get_status" + task="launch" + ;; + c ) + task="cleanup" + ;; + k ) + task="kill" ;; \?) printf "illegal option: -%s\n" "$OPTARG" >&2 echo "$usage" >&2 @@ -44,55 +55,83 @@ while getopts "hd:f:s" opt; do ;; esac done + +## Examine directories if [ -z "$directory_name" ]; then echo "directory name required (-d)" exit 1 fi working_directory="$base_directory/$directory_name" if [ ! -d "$working_directory" ]; then - echo "directory $working_directory not found" + echo "working directory $working_directory not found" exit 1 fi -if [ "$task" = "start" ]; then - if [ -z "$input_filename" ]; then - echo "input filename required (-f)" - exit 1 - fi +input_directory="$working_directory/$input_directory_name" +if [ ! -d "$input_directory" ]; then + echo "input directory $input_directory not found" + exit 1 +fi +pid_path="$working_directory/$pid_filename" + +## Process task +if [ "$task" = "launch" ]; then + piscal_config_file="$working_directory"/piscal.cfg + # write config file for piscal + echo $photosynthetic_type > "$piscal_config_file" + find "$input_directory" -maxdepth 1 -type f\ + -printf '%P\n'\ + >> "$piscal_config_file" - if [ ! -f "$working_directory/$input_filename" ]; then - echo "input filename $working_directory/$input_filename not found" - exit 1 - fi - - command="$launcher -d $working_directory -f $input_filename" - nohup ${command} > $working_directory/$out_filename 2>&1 & + command="$launcher -d $working_directory -f piscal.cfg" + # launch it, sending error output to file + nohup ${command} > $working_directory/$stderr_filename 2>&1 & # write the PID to a temp file to check for completion later echo $! > $working_directory/$pid_filename echo started elif [ "$task" = "get_status" ]; then - pid_path="$working_directory/$pid_filename" - - # if the pid doesn't exist, then process never started + # if the pid doesn't exist, then process hasn't started if [ ! -f "$pid_path" ]; then - echo "no pid file found" - exit 1 + echo "not started" + exit fi - + pid=$(head -n 1 $pid_path) # if the pid exists, check the process status using ps if ps -p $pid > /dev/null; then # if it is in ps, then it's still running echo running else - # otherwise, it is complete, check the output for success/error - # TODO: examine output for errors, etc - #cat piscal.out - if [ ! -d "$working_directory/output" ]; then - echo "output directory $working_directory/output not found" + # otherwise, it is complete, check for runtime errors + if [ -s "$working_directory/$stderr_filename" ]; then + cat "$working_directory/$stderr_filename" exit 1 fi - echo success - find "$working_directory/output"/* + + output_directory="$working_directory/$output_directory_name" + if [ ! -d "$output_directory" ]; then + echo "output directory $output_directory not found" + exit 1 + fi + + echo complete + find "$output_directory"/* fi -fi \ No newline at end of file +elif [ "$task" = "cleanup" ]; then + rm -rf "$working_directory" + echo hi +elif [ "$task" = "kill" ]; then + # if the pid doesn't exist, then process hasn't started + if [ ! -f "$pid_path" ]; then + echo "not started" + exit + fi + + pid=$(head -n 1 $pid_path) + # if the pid exists, check the process status using ps + if ps -p $pid > /dev/null; then + # if it is in ps, then it's still running + kill $pid + fi + echo killed +fi diff --git a/Web/HangfireStartup.cs b/Web/HangfireStartup.cs index 8342bc5..5a41206 100644 --- a/Web/HangfireStartup.cs +++ b/Web/HangfireStartup.cs @@ -3,7 +3,6 @@ using Hangfire; using Microsoft.Owin; using LeafWeb.Web; using LeafWeb.Web.Services; -using NLog.Internal; using Owin; using ConfigurationManager = System.Configuration.ConfigurationManager; @@ -24,7 +23,6 @@ namespace LeafWeb.Web private void SetupRecurringJobs() { - // TODO: new SqlServerDistributedLock var queueInterval = ConfigurationManager.AppSettings["ProcessQueueInterval"]; // https://discuss.hangfire.io/t/how-to-create-cron-job-that-is-executing-every-15-minutes/533 RecurringJob.AddOrUpdate(PiscalProcessQueue, p => p.ProcessQueue(), queueInterval); diff --git a/Web/Services/Cleanup.cs b/Web/Services/Cleanup.cs new file mode 100644 index 0000000..6e25af3 --- /dev/null +++ b/Web/Services/Cleanup.cs @@ -0,0 +1,25 @@ +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; + +namespace LeafWeb.Web.Services +{ + public class Cleanup : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + try + { + Logger.Info("LeafInput: {0}, Cleanup", leafInput.Id); + + PiscalService.Cleanup(leafInput); + } + catch (PiscalClientException ex) + { + var errorMessage = FormatException(ex, ex.LeafInputId); + Logger.Error(errorMessage, ex); + // log the error, but ignore the cleanup issue for now + Logger.Info("LeafInput: {0}, Cleanup - likely has not occurred", leafInput.Id); + } + } + } +} \ No newline at end of file diff --git a/Web/Services/PiscalQueueBase.cs b/Web/Services/PiscalQueueBase.cs new file mode 100644 index 0000000..7b9870f --- /dev/null +++ b/Web/Services/PiscalQueueBase.cs @@ -0,0 +1,57 @@ +using System; +using Hangfire; +using LeafWeb.Core.DAL; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; +using NLog; + +namespace LeafWeb.Web.Services +{ + public abstract class PiscalQueueBase : IDisposable + { + protected readonly DataService DataService; + protected readonly PiscalService PiscalService; + protected readonly Logger Logger; + + protected PiscalQueueBase(DataService dataService, PiscalService piscalService) + { + DataService = dataService; + PiscalService = piscalService; + Logger = LogManager.GetLogger(GetType().Name); + } + + protected PiscalQueueBase() : this(new DataService(), new PiscalService()) { } + + protected string FormatException(Exception ex, int leafInputId) + { + return + $"LeafInput: {leafInputId}{Environment.NewLine}" + + $"Class: {GetType().Name}{Environment.NewLine}" + + $"Exception: {ex.Message}{Environment.NewLine}" + + (ex.InnerException != null ? $"InnerException: {ex.InnerException}{Environment.NewLine}" : string.Empty) + + $"StackTrace: {ex.StackTrace}"; + } + + protected void PiscalExceptionNotify(PiscalClientException ex, LeafInput leafInput) + { + var errorMessage = FormatException(ex, ex.LeafInputId); + Logger.Error(errorMessage); + BackgroundJob.Enqueue( + email => email.SendAdministratorMessage($"LeafWeb: PiscalQueue {GetType().Name} Exception", errorMessage)); + + // TODO send user email too + + if (leafInput != null) + { + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", + ex.Message); + BackgroundJob.Enqueue(s => s.DoWork(leafInput.Id)); + } + } + + public void Dispose() + { + DataService.Dispose(); + } + } +} \ No newline at end of file diff --git a/Web/Services/PiscalQueueManager.cs b/Web/Services/PiscalQueueManager.cs index a2a77e8..57bb4da 100644 --- a/Web/Services/PiscalQueueManager.cs +++ b/Web/Services/PiscalQueueManager.cs @@ -2,168 +2,111 @@ using System.Linq; using System.Threading; using Hangfire; -using LeafWeb.Core.DAL; using LeafWeb.Core.Entities; using LeafWeb.Core.Remote; -using NLog; namespace LeafWeb.Web.Services { - public class PiscalQueueManager : IDisposable + public class PiscalQueueManager : PiscalQueueBase { - private readonly DataService _dataService; - private readonly PiscalService _piscalService; - private readonly EmailNotificationService _emailService; - - public PiscalQueueManager(DataService dataService, PiscalService piscalService, EmailNotificationService emailService) - { - _dataService = dataService; - _piscalService = piscalService; - _emailService = emailService; - } - - public PiscalQueueManager() : this(new DataService(), new PiscalService(), new EmailNotificationService()) {} - private static readonly object ProcessQueueLock = new object(); public void ProcessQueue() { - var logger = LogManager.GetCurrentClassLogger(); - + // prevent multiple entry into processing the queue if (Monitor.TryEnter(ProcessQueueLock)) { - logger.Trace("ProcessQueue entered"); + Logger.Trace("ProcessQueue entered"); try { - ProcessRunning(logger); + UpdateRunning(); - ProcessQueue(logger); + StartNextPending(); } finally { - logger.Trace("ProcessQueue exit"); + Logger.Trace("ProcessQueue exit"); Monitor.Exit(ProcessQueueLock); } } else { - logger.Trace("ProcessQueue locked, queue already processing"); + Logger.Trace("ProcessQueue locked, queue already processing"); } } - - private void ProcessQueue(ILogger logger) + + private void StartNextPending() { - var runningLeafInputs = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); + var runningLeafInputs = DataService.GetRunningLeafInputs().ToList(); if (runningLeafInputs.Any()) { - logger.Trace("Leaf input currently running , don't enqueue any new"); + Logger.Trace("Leaf input currently running , don't enqueue any new"); return; } - var pending = - _dataService + var pendingInput = + DataService .GetLeafInputs(LeafInputStatusType.Pending) .OrderBy(l => l.StatusHistory.Min(sh => sh.DateTime)) .FirstOrDefault(); - if (pending == null) + if (pendingInput == null) { - logger.Trace("No pending leaf input"); + Logger.Trace("No pending leaf input"); return; } - logger.Info("LeafInputFile: {0}, Start", pending.Id); - try - { - _piscalService.Run(pending); - } - catch (PiscalClientException ex) - { - var errorMessage = - $"LeafInputFile: {pending.Id}\r\nProcessRunning Exception: {ex.Message}" - + $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}"; - logger.Error(errorMessage); - BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessQueue Exception", errorMessage)); - - _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Exception, "Error occurred starting LeafInput", ex.Message); - logger.Info("LeafInputFile: {0}, Cleanup", pending.Id); - _piscalService.Cleanup(pending); - - // TODO: re-queue? - //_dataService.SetLeafInputFileStatus(queuedFile, LeafInputStatusType.Queued, "Re-queuing LeafInput"); - } - logger.Trace("LeafInputFile: {0}, Set Pending", pending.Id); - _dataService.SetLeafInputStatus(pending, LeafInputStatusType.Running); + var pendingInputId = pendingInput.Id; + Logger.Info("LeafInput: {0}, Starting", pendingInputId); + DataService.SetLeafInputStatus(pendingInput, LeafInputStatusType.Starting); + BackgroundJob.Enqueue(c => c.DoWork(pendingInputId)); } - private void ProcessRunning(ILogger logger) + private void UpdateRunning() { - var running = _dataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); + var running = DataService.GetLeafInputs(LeafInputStatusType.Running).ToList(); foreach (var leafInput in running) { try { - var status = _piscalService.GetStatus(leafInput); + var status = PiscalService.GetStatus(leafInput); + var leafInputId = leafInput.Id; switch (status) { case PiscalStatus.NotStarted: - logger.Warn("LeafInputFile: {0}, Not Started, re-queueing", leafInput.Id); - // if it's not started, try to requeue the process - this is unusual state - _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Pending); + // if it's not started - this is unusual state + Logger.Warn("LeafInput: {0}, Piscal Not Started", leafInput.Id); break; case PiscalStatus.Running: - logger.Trace("LeafInputFile: {0}, Running", leafInput.Id); + Logger.Trace("LeafInput: {0}, Piscal Running", leafInput.Id); // continue running break; case PiscalStatus.Complete: - logger.Info("LeafInputFile: {0}, Complete", leafInput.Id); - // collect the leaf output - var leafOutputFiles = _piscalService.RetrieveOutputFiles(leafInput).ToList(); + // TODO: change to "retrieving output"? + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Finishing); - logger.Trace("LeafInputFile: {0}, saving output files", leafInput.Id); - foreach (var outputFile in leafOutputFiles) - _dataService.AddLeafOutputFile(outputFile); - - logger.Info("LeafInputFile: {0}, output files: {1}", leafInput.Id, - string.Join(", ", leafOutputFiles.Select(o => o.Filename))); - - // update db - logger.Trace("LeafInputFile: {0}, Set Complete", leafInput.Id); - _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete); - - BackgroundJob.Enqueue(() => _emailService.SendLeafWebComplete(leafInput.Id)); - - // remove working data from the server - logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id); - _piscalService.Cleanup(leafInput); + var retrieveFilesId = BackgroundJob.Enqueue(s => s.DoWork(leafInputId)); + BackgroundJob.ContinueWith(retrieveFilesId, s => s.DoWork(leafInputId)); + BackgroundJob.ContinueWith(retrieveFilesId, + email => email.SendLeafWebComplete(leafInputId)); + BackgroundJob.ContinueWith(retrieveFilesId, s => s.DoWork(leafInputId)); break; } } + catch (PiscalClientException ex) + { + PiscalExceptionNotify(ex, leafInput); + } catch (Exception ex) { - var errorMessage = - $"LeafInputFile: {leafInput.Id}\r\nProcessRunning Exception: {ex.Message}" - + $"\r\nInnerException: {ex.InnerException}\r\nStackTrace: {ex.StackTrace}"; - logger.Error(errorMessage); - BackgroundJob.Enqueue(() => _emailService.SendAdministratorMessage("LeafWeb ProcessRunning Exception", errorMessage)); - - _dataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", ex.Message); - - // TODO: Send email - logger.Info("LeafInputFile: {0}, Cleanup", leafInput.Id); - _piscalService.Cleanup(leafInput); - // TODO: re-queue ? + var errorMessage = FormatException(ex, leafInput.Id); + Logger.Error(errorMessage); } } } - - public void Dispose() - { - _dataService.Dispose(); - } } } \ No newline at end of file diff --git a/Web/Services/PiscalQueueWorker.cs b/Web/Services/PiscalQueueWorker.cs new file mode 100644 index 0000000..16a323a --- /dev/null +++ b/Web/Services/PiscalQueueWorker.cs @@ -0,0 +1,39 @@ +using System; +using LeafWeb.Core.Entities; +using LeafWeb.Core.Remote; + +namespace LeafWeb.Web.Services +{ + public abstract class PiscalQueueWorker : PiscalQueueBase + { + public void DoWork(int leafInputId) + { + LeafInput leafInput = null; + try + { + leafInput = DataService.GetLeafInput(leafInputId); + DoWorkInternal(leafInput); + } + catch (PiscalClientException ex) + { + PiscalExceptionNotify(ex, leafInput); + + if (leafInput != null) + { + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Exception, "Error occurred processing LeafInput", + ex.Message); + } + // signal to process next item + HangfireStartup.TriggerPiscalProcessQueue(); + } + catch (Exception ex) + { + var errorMessage = FormatException(ex, leafInputId); + Logger.Error(errorMessage); + throw; // this will retry via HangFire + } + } + + protected abstract void DoWorkInternal(LeafInput leafInputId); + } +} \ No newline at end of file diff --git a/Web/Services/PiscalService.cs b/Web/Services/PiscalService.cs index 71153f3..8b6d74b 100644 --- a/Web/Services/PiscalService.cs +++ b/Web/Services/PiscalService.cs @@ -5,6 +5,9 @@ using LeafWeb.Core.Remote; namespace LeafWeb.Web.Services { + /// + /// Thin layer over PiscalClient to translate Core entities to Piscal objects + /// public class PiscalService { private readonly IPiscalClient _piscalClient; diff --git a/Web/Services/RetrieveOutputFiles.cs b/Web/Services/RetrieveOutputFiles.cs new file mode 100644 index 0000000..b6fc083 --- /dev/null +++ b/Web/Services/RetrieveOutputFiles.cs @@ -0,0 +1,28 @@ +using System.Linq; +using LeafWeb.Core.Entities; + +namespace LeafWeb.Web.Services +{ + public class RetrieveOutputFiles : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + Logger.Trace("LeafInput: {0}, RetrieveOutputFiles", leafInput.Id); + + var leafOutputFiles = PiscalService.RetrieveOutputFiles(leafInput).ToList(); + + Logger.Trace("LeafInput: {0}, RetrieveOutputFiles saving output files", leafInput.Id); + + foreach (var outputFile in leafOutputFiles) + { + if (leafInput.OutputFiles.All(file => file.Filename != outputFile.Filename)) + DataService.AddLeafOutputFile(outputFile); + else + Logger.Warn("LeafInput: {0}, RetrieveOutputFiles duplicate file name: {1}", leafInput.Id, outputFile.Filename); + } + + Logger.Info("LeafInput: {0}, RetrieveOutputFiles output files: {1}", leafInput.Id, + string.Join(", ", leafOutputFiles.Select(o => o.Filename))); + } + } +} \ No newline at end of file diff --git a/Web/Services/SetComplete.cs b/Web/Services/SetComplete.cs new file mode 100644 index 0000000..535579a --- /dev/null +++ b/Web/Services/SetComplete.cs @@ -0,0 +1,16 @@ +using LeafWeb.Core.Entities; + +namespace LeafWeb.Web.Services +{ + public class SetComplete : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + Logger.Trace("LeafInput: {0}, Set Complete", leafInput.Id); + + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Complete); + + HangfireStartup.TriggerPiscalProcessQueue(); + } + } +} \ No newline at end of file diff --git a/Web/Services/StartPending.cs b/Web/Services/StartPending.cs new file mode 100644 index 0000000..9e98f97 --- /dev/null +++ b/Web/Services/StartPending.cs @@ -0,0 +1,16 @@ +using LeafWeb.Core.Entities; + +namespace LeafWeb.Web.Services +{ + public class StartPending : PiscalQueueWorker + { + protected override void DoWorkInternal(LeafInput leafInput) + { + Logger.Trace("LeafInput: {0}, Run", leafInput.Id); + + PiscalService.Run(leafInput); + + DataService.SetLeafInputStatus(leafInput, LeafInputStatusType.Running); + } + } +} \ No newline at end of file diff --git a/Web/Web.csproj b/Web/Web.csproj index 33a9831..18fbb75 100644 --- a/Web/Web.csproj +++ b/Web/Web.csproj @@ -960,10 +960,16 @@ + + + + + +