Retrieve LeafOutput operational

Core.Tests/Remote/PiscalSshClientTests.cs

@@ -16,7 +16,7 @@ namespace LeafWeb.Core.Tests.Remote
 				Filename = "blah",
 				LeafInputId = 1,
 				Contents = "test".GetBytes(),
-				DirectoryName = "TestDirectory"
+				DirectoryName = "TestDirectory2"
 			};
 
 		private readonly string _piscalConnectionString =
@@ -33,15 +33,17 @@ namespace LeafWeb.Core.Tests.Remote
 		public void GetLeafInputStatus()
 		{
 			var client = new PiscalSshClient(_piscalConnectionString);
-			client.GetLeafInputStatus(_testInput);
+			var leafInputStatus = client.GetLeafInputStatus(_testInput);
+			Console.WriteLine(leafInputStatus);
 		}
 
 		[Test]
-		public void RetrieveLeafInputResult()
+		public void RetrieveLeafOutput()
 		{
 			var client = new PiscalSshClient(_piscalConnectionString);
-			var result = client.RetrieveLeafInputResult(_testInput).ToList();
+			var result = client.RetrieveLeafOutput(_testInput).ToList();
-			Console.WriteLine(result[0].Contents.GetString());
+			Console.WriteLine(string.Join(", ", result.Select(o => o.Filename)));
+			//Console.WriteLine(result[0].Contents.GetString());
 		}
 	}
 }

Core.Tests/Utility/StringExtensionsTests.cs

@@ -38,5 +38,13 @@ namespace LeafWeb.Core.Tests.Utility
 			var result = str.FilterAlphaNumeric();
 			Assert.That(result, Is.EqualTo("newline"));
 		}
+
+		[Test]
+		public void FilenameFromPath()
+		{
+			var str = "/full/path/to/file.ext";
+			var result = str.FilenameFromPath();
+			Assert.That(result, Is.EqualTo("file.ext"));
+		}
 	}
 }

Core/Core.csproj

@@ -111,6 +111,7 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <None Include="packages.config" />
+    <None Include="Remote\piscal_launcher.sh" />
     <None Include="Remote\piscal_manager.sh" />
   </ItemGroup>
   <ItemGroup />

Core/DAL/DataService.cs

@@ -38,8 +38,7 @@ namespace LeafWeb.Core.DAL
 		}
 		#endregion
 
-
+		#region LeafInput
-		#region LeafInput Sites
 
 		public IQueryable<LeafInput> GetLeafInputs()
 		{
@@ -54,13 +53,69 @@ namespace LeafWeb.Core.DAL
 		public void AddLeafInput(LeafInput leafInput)
 		{
 			leafInput.Added = DateTime.Now;
-
 			_db.LeafInputs.Add(leafInput);
+			foreach (var leafInputFile in leafInput.Files)
+			{
+				SetLeafInputFileStatusNoUpdate(leafInputFile, LeafInputStatusType.Added);
+			}
+			_db.SaveChanges();
+		}
+		#endregion
+
+		#region LeafInputFile
+
+		public IQueryable<LeafInputFile> GetLeafInputFiles()
+		{
+			return _db.LeafInputFiles;
+		}
+
+		public IQueryable<LeafInputFile> GetLeafInputFiles(LeafInputStatusType status)
+		{
+			return
+				from file in _db.LeafInputFiles
+				where file.CurrentStatus == status
+				select file;
+		}
+
+		public void UpdateLeafInputFile(LeafInputFile leafInputFile)
+		{
+			_db.Entry(leafInputFile).State = EntityState.Modified;
 			_db.SaveChanges();
 		}
 
+		private void SetLeafInputFileStatusNoUpdate(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
+		{
+			leafInputFile.CurrentStatus = status;
+			var leafInputFileStatus = new LeafInputFileStatus
+			{
+				Status = status,
+				DateTime = DateTime.Now,
+				Description = description,
+				LeafInputFile = leafInputFile
+			};
+			if (leafInputFile.StatusHistory == null)
+				leafInputFile.StatusHistory = new List<LeafInputFileStatus>();
+			leafInputFile.StatusHistory.Add(leafInputFileStatus);
+		}
+
+		public void SetLeafInputFileStatus(LeafInputFile leafInputFile, LeafInputStatusType status, string description = null)
+		{
+			SetLeafInputFileStatusNoUpdate(leafInputFile, status, description);
+			UpdateLeafInputFile(leafInputFile);
+		}
+
+		public LeafInputFile GetNextUnprocessedLeafInputFile()
+		{
+			return
+				(from file in GetLeafInputFiles(LeafInputStatusType.Added)
+				orderby file.Id ascending
+				select file).FirstOrDefault();
+		}
+
 		#endregion
 
+		#region Photosynthesis Types
+
 		public IQueryable<PhotosynthesisType> GetPhotosynthesisTypes()
 		{
 			return _db.PhotosynthesisTypes.OrderBy(pt => pt.SortOrder);
@@ -70,5 +125,7 @@ namespace LeafWeb.Core.DAL
 		{
 			return _db.PhotosynthesisTypes.Find(id);
 		}
+
+		#endregion
 	}
 }

Core/Entities/LeafInput.cs

@@ -8,7 +8,7 @@ namespace LeafWeb.Core.Entities
 	{
 		public int Id { get; set; }
 
-		public virtual ICollection<LeafInputFile> LeafInputFiles { get; set; }
+		public virtual ICollection<LeafInputFile> Files { get; set; }
 
 		[Required(ErrorMessage = "Name required")]
 		public string Name { get; set; }

Core/Entities/LeafInputFile.cs

@@ -18,6 +18,6 @@ namespace LeafWeb.Core.Entities
 		public byte[] Contents { get; set; }
 
 		public LeafInputStatusType CurrentStatus { get; set; }
-		public virtual ICollection<LeafInputFileStatus> LeafInputStatuses { get; set; }
+		public virtual ICollection<LeafInputFileStatus> StatusHistory { get; set; }
 	}
 }

Core/Remote/IPiscalClient.cs

@@ -6,6 +6,6 @@ namespace LeafWeb.Core.Remote
 	{
 		void SubmitLeafInputFile(PiscalLeafInputFile file);
 		PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file);
-		IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file);
+		IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file);
 	}
 }

Core/Remote/PiscalSshClient.cs

@@ -11,7 +11,7 @@ namespace LeafWeb.Core.Remote
 	public class PiscalSshClient : IPiscalClient
 	{
 		private const string BaseDirectory = "./LeafWeb";
-		private const string RemoteScriptPath = "./piscal_manager.sh";
+		private const string RemoteScriptPath = BaseDirectory + "/piscal_manager.sh";
 		private readonly PasswordConnectionInfo _connectionInfo;
 
 		public PiscalSshClient(string connectionString)
@@ -53,61 +53,76 @@ namespace LeafWeb.Core.Remote
 			{
 				ssh.Connect();
 				var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -f {file.Filename}";
-				var exeCommand = ssh.CreateCommand(commandText);
+				var command = ssh.CreateCommand(commandText);
-				exeCommand.Execute();
+				command.Execute();
 				ssh.Disconnect();
 
-				Console.Write(exeCommand.Result);
+				if (command.ExitStatus != 0)
+					throw new PiscalClientException(command.Error);
 
-				if (exeCommand.ExitStatus != 0)
+				Console.Write(command.Result);
-					throw new PiscalClientException(exeCommand.Error);
 			}
 		}
 
 		public PiscalStatus GetLeafInputStatus(PiscalLeafInputFile file)
+		{
+			var statusRaw = GetLeafInputStatusRaw(file);
+
+			switch (statusRaw[0])
+			{
+				case "running":
+					return PiscalStatus.Running;
+				case "success":
+					return PiscalStatus.Success;
+				default:
+					return PiscalStatus.Error;
+			}
+		}
+
+		private string[] GetLeafInputStatusRaw(PiscalLeafInputFile file)
 		{
 			using (var ssh = GetSshClient())
 			{
 				ssh.Connect();
 				var commandText = $"{RemoteScriptPath} -d {file.DirectoryName} -s";
-				var exeCommand = ssh.CreateCommand(commandText);
+				var command = ssh.CreateCommand(commandText);
-				exeCommand.Execute();
+				command.Execute();
 				ssh.Disconnect();
 
-				Console.Write(exeCommand.Result);
+				if (command.ExitStatus != 0)
+					throw new PiscalClientException(command.Error);
+
+				return command.Result
+					.SplitNewLine()
+					.Where(s => s.Length > 0)
+					.Select(s => s.Trim()).ToArray();
 			}
-			return PiscalStatus.Success;
 		}
 
-		public IEnumerable<PiscalLeafOutputFile> RetrieveLeafInputResult(PiscalLeafInputFile file)
+		public IEnumerable<PiscalLeafOutputFile> RetrieveLeafOutput(PiscalLeafInputFile file)
 		{
-			var outputDirectory = $"{BaseDirectory}/{file.DirectoryName}";
-
-			string[] filenames;
-
 			// get output files
-			using (var ssh = GetSshClient())
+			var status = GetLeafInputStatusRaw(file);
-			{
+			if (status[0] != "success")
-				ssh.Connect();
+				throw new PiscalClientException("output not available, status is " + status[0]);
-				var commandText = $"ls -1 {outputDirectory}";
-				var lsCommand = ssh.CreateCommand(commandText);
-				lsCommand.Execute();
-				ssh.Disconnect();
 
-				filenames = lsCommand.Result.SplitNewLine().Where(s => s.Length > 0).Select(s => s.Trim()).ToArray();
+			var filePaths = status.Skip(1);
-				Console.Write(lsCommand.Result);
-			}
 
 			using (var scp = GetScpClient())
 			{
 				scp.Connect();
-				foreach (var filename in filenames)
+				foreach (var filePath in filePaths)
 				{
 					using (var stream = new MemoryStream())
 					{
-						var filePath = $"{BaseDirectory}/{file.DirectoryName}/{filename}";
 						scp.Download(filePath, stream);
-						yield return new PiscalLeafOutputFile {Contents = stream.ToArray(), Filename = filename};
+						yield return 
+							new PiscalLeafOutputFile
+							{
+								Contents = stream.ToArray(),
+								Filename = filePath.FilenameFromPath(),
+								DirectoryName = file.DirectoryName
+							};
 					}
 				}
 

Core/Remote/piscal_launcher.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# piscal launcher
+
+usage="$(basename "$0") [-h] -d working_directory [-f input_filename] -- script to launch Piscal
+
+where:
+    -h  show this help text
+    -d  working directory
+    -f  input filename"
+
+directory=""
+input_filename=""
+
+while getopts "hd:f:" opt; do
+    case "$opt" in
+    h )
+        echo "$usage"
+        exit
+        ;;
+    d )
+        directory=$OPTARG
+        ;;
+    f )
+        input_filename=$OPTARG
+        task="process"
+        ;;
+    \?) printf "illegal option: -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
+    esac
+done
+if [ -z "$directory" ]; then
+    echo "working directory required (-d)"
+    exit 1
+fi
+if [ ! -d "$directory" ]; then
+    echo "working directory $directory not found"
+    exit 1
+fi
+if [ -z "$input_filename" ]; then
+    echo "input filename required (-f)"
+    exit 1
+fi
+if [ ! -f "$directory/$input_filename" ]; then
+    echo "input filename $directory/$input_filename not found"
+    exit 1
+fi
+
+output_directory="$directory/output"
+
+sleep 1m
+if [ ! -d "$output_directory" ]; then
+    mkdir "$output_directory"
+fi
+
+# TODO: run actual command
+cp "/home/poprhythm/LeafWebTestOutput"/* "$output_directory"

Core/Remote/piscal_manager.sh

@@ -1,12 +1,12 @@
-#~/bin/bash
+#!/bin/bash
 # piscal manager script
 
-usage="$(basename "$0") [-h] -d directory_name -f input_filename -- script to manage Piscal
+usage="$(basename "$0") [-h] -d directory_name [-f input_filename|-s] -- script to manage Piscal
 
 where:
     -h  show this help text
     -d  working directory name
-    -o  input filename
+    -f  input filename
     -s  job status"
 
 # http://stackoverflow.com/a/14203146/99492
@@ -16,9 +16,10 @@ where:
 base_directory="/home/poprhythm/LeafWeb"
 directory_name=""
 input_filename=""
-get_status=false
+task="start" # default task
 pid_filename="piscal.pid"
 out_filename="piscal.out"
+launcher="$base_directory/piscal_launcher.sh"
 
 while getopts "hd:f:s" opt; do
     #echo "$opt = $OPTARG"
@@ -32,9 +33,10 @@ while getopts "hd:f:s" opt; do
         ;;
     f )
         input_filename=$OPTARG
+        task="start"
         ;;
     s )
-        get_status=true
+        task="get_status"
         ;;
     \?) printf "illegal option: -%s\n" "$OPTARG" >&2
         echo "$usage" >&2
@@ -46,24 +48,30 @@ if [ -z "$directory_name" ]; then
     echo "directory name required (-d)"
     exit 1
 fi
-if [ ! -d "$base_directory/$directory_name" ]; then
+working_directory="$base_directory/$directory_name"
-    echo "directory $base_directory/$directory_name not found"
+if [ ! -d "$working_directory" ]; then
+    echo "directory $working_directory not found"
     exit 1
 fi
-if [ "$get_status" = false ]; then
+if [ "$task" = "start" ]; then
     if [ -z "$input_filename" ]; then
         echo "input filename required (-f)"
         exit 1
     fi
     
-    if [ ! -f "$base_directory/$directory_name/$input_filename" ]; then
+    if [ ! -f "$working_directory/$input_filename" ]; then
-        echo "input filename $base_directory/$directory_name/$input_filename not found"
+        echo "input filename $working_directory/$input_filename not found"
         exit 1
     fi
-fi
+    
+    command="$launcher -d $working_directory -f $input_filename"
+    nohup ${command} > $working_directory/$out_filename 2>&1 &
 
-if [ "$get_status" = true ]; then
+    # write the PID to a temp file to check for completion later
-    pid_path="$base_directory/$directory_name/$pid_filename"
+    echo $! > $working_directory/$pid_filename
+    echo started
+elif [ "$task" = "get_status" ]; then
+    pid_path="$working_directory/$pid_filename"
     
     # if the pid doesn't exist, then process never started
     if [ ! -f "$pid_path" ]; then
@@ -79,15 +87,12 @@ if [ "$get_status" = true ]; then
     else
         # otherwise, it is complete, check the output for success/error
         # TODO: examine output for errors, etc
-        echo complete
+        #cat piscal.out
+        if [ ! -d "$working_directory/output" ]; then
+            echo "output directory $working_directory/output not found"
+            exit 1
+        fi
+        echo success
+        find "$working_directory/output"/*
     fi
-
-else
-    
-    command="sleep 100s"
-    nohup ${command} > $base_directory/$directory_name/$out_filename 2>&1 &
-
-    # write the PID to a temp file to check for completion later
-    echo $! > $base_directory/$directory_name/$pid_filename
-
 fi

Core/Utility/StringExtensions.cs

@@ -45,5 +45,10 @@ namespace LeafWeb.Core.Utility
 		{
 			return Regex.Replace(input, @"[^\w_]", "");
 		}
+
+		public static string FilenameFromPath(this string path)
+		{
+			return Regex.Replace(path, @".*/([^/]*$)", "$1");
+		}
 	}
 }

Web/Controllers/LeafInputController.cs

@@ -1,9 +1,12 @@
 using System;
+using System.Configuration;
 using System.IO;
 using System.Linq;
 using System.Web;
 using System.Web.Mvc;
+using Hangfire;
 using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
 using LeafWeb.Web.Attributes;
 using LeafWeb.Web.ViewModels;
 using LeafWeb.Web.ViewModels.LeafInput;
@@ -69,7 +72,7 @@ namespace LeafWeb.Web.Controllers
 				// convert viewModel into Model
 				var leafInput = viewModel.GetFileInput(DataService);
 				// load files into LeafInputFile
-				leafInput.LeafInputFiles =
+				leafInput.Files =
 					(from f in files
 					let bytes = System.IO.File.ReadAllBytes(f.FullName)
 					select new LeafInputFile {Filename = f.Name, Contents = bytes}).ToList();
@@ -78,7 +81,13 @@ namespace LeafWeb.Web.Controllers
 				DataService.AddLeafInput(leafInput);
 
 				DeleteBackloadDirectory(Session.SessionID);
-			
+
+				foreach (var file in leafInput.Files.Select(f => new PiscalLeafInputFile(f)))
+				{
+					BackgroundJob.Enqueue(() => ProcessLeafInput(file));
+				}
+				
+
 				SetStatusMessage(
 					HttpUtility.HtmlEncode(
 					$"A data set has submitted for '{viewModel.Identifier}' from '{viewModel.SiteId}'. " + Environment.NewLine
@@ -91,6 +100,12 @@ namespace LeafWeb.Web.Controllers
 			return View("Index", viewModel);
 		}
 
+		public void ProcessLeafInput(PiscalLeafInputFile leafInputFile)
+		{
+			var piscalSshClient = new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
+			piscalSshClient.SubmitLeafInputFile(leafInputFile);
+		}
+
 		private FileInfo[] GetBackloadDirectoryFiles(string directoryName)
 		{
 			var path = Path.Combine(Server.MapPath("~/Files/"), directoryName + "\\");

Web/Services/JobService.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Configuration;
+using System.Linq;
+using LeafWeb.Core.DAL;
+using LeafWeb.Core.Entities;
+using LeafWeb.Core.Remote;
+
+namespace LeafWeb.Web.Services
+{
+	public class JobService : IDisposable
+	{
+		protected readonly DataService DataService = new DataService();
+
+		protected IPiscalClient GetPiscalClient()
+		{
+			return new PiscalSshClient(ConfigurationManager.ConnectionStrings["PiscalServer"].ConnectionString);
+		}
+
+		public void ProcessNextLeafInput()
+		{
+			var leafInputFile = DataService.GetNextUnprocessedLeafInputFile();
+			if (leafInputFile == null) // no inputs, quit
+				return;
+
+			var inputFile = new PiscalLeafInputFile(leafInputFile);
+
+			var piscalSshClient = GetPiscalClient();
+			piscalSshClient.SubmitLeafInputFile(inputFile);
+
+			DataService.SetLeafInputFileStatus(leafInputFile, LeafInputStatusType.ProcessStarted);
+		}
+
+		public void UpdateLeafInputStatus()
+		{
+			var leafInputFiles =
+				DataService
+				.GetLeafInputFiles(LeafInputStatusType.ProcessStarted)
+				.Select(f => new PiscalLeafInputFile(f));
+
+			var piscalClient = GetPiscalClient();
+			foreach (var file in leafInputFiles)
+			{
+				var status = piscalClient.GetLeafInputStatus(file);
+				switch (status)
+				{
+					case PiscalStatus.Success:
+						// retrieve LeafOutput
+						var outputFile = piscalClient.RetrieveLeafOutput(file);
+						break;
+					case PiscalStatus.Error:
+						// record error
+						break;
+					case PiscalStatus.Running:
+						// do nothing
+						break;
+				}
+			}
+		}
+
+		public void Dispose()
+		{
+			DataService.Dispose();
+		}
+	}
+}

Web/Web.csproj

@@ -921,6 +921,7 @@
     </Compile>
     <Compile Include="Attributes\HttpParamActionAttribute.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Services\JobService.cs" />
     <Compile Include="Startup.cs" />
     <Compile Include="Utility\MarkdownHelper.cs" />
     <Compile Include="Utility\Validation.cs" />