From 19d09c88835182373fb2f781218e93051795af51 Mon Sep 17 00:00:00 2001 From: James Kolpack Date: Thu, 16 Nov 2017 09:36:35 -0500 Subject: [PATCH] Fix parser issue when HHMMSS is too long --- Core.Tests/Core.Tests.csproj | 3 ++ Core.Tests/Parsers/LeafInputCsvParserTests.cs | 22 ++++++++++++++ .../20171115002141clean10c_day3_high.csv | 30 +++++++++++++++++++ Core/Core.csproj | 1 + Core/Parsers/LeafInputCleaner.cs | 18 +++++++++++ WebCms/App_Data/Models/all.dll.path | 2 +- WebCms/Services/PiscalQueue/FinishComplete.cs | 12 ++++++++ .../PiscalQueue/PiscalQueueManager.cs | 1 + 8 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 Core.Tests/Parsers/LeafInputData/20171115002141clean10c_day3_high.csv create mode 100644 Core/Parsers/LeafInputCleaner.cs diff --git a/Core.Tests/Core.Tests.csproj b/Core.Tests/Core.Tests.csproj index 2f7ff9b..7a97431 100644 --- a/Core.Tests/Core.Tests.csproj +++ b/Core.Tests/Core.Tests.csproj @@ -83,6 +83,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/Core.Tests/Parsers/LeafInputCsvParserTests.cs b/Core.Tests/Parsers/LeafInputCsvParserTests.cs index 2507f4a..3f97f39 100644 --- a/Core.Tests/Parsers/LeafInputCsvParserTests.cs +++ b/Core.Tests/Parsers/LeafInputCsvParserTests.cs @@ -36,6 +36,28 @@ namespace LeafWeb.Core.Tests.Parsers Assert.That(dataItem.Photo, Is.EqualTo(10.49195882)); } + [Test] + public void Parse_Cleaned_AnnaDay1() + { + var fileInfo = FileUtility.GetContentFile(ContentDirectory, "20171115002141clean10c_day3_high.csv"); + var leafInput = new LeafInputCsvParser(fileInfo); + var input = leafInput.Parse(); + Assert.That(input.InvestigatorName, Is.EqualTo("Anna Schyman")); + + var site = input.Site; + Assert.That(site.SiteId, Is.EqualTo("LNU")); + Assert.That(site.Latitude, Is.EqualTo(56d)); + + var photosynthetic = input.Photosynthetic; + Assert.That(photosynthetic.GammaStar, Is.EqualTo(0)); + + var data = input.Data; + Assert.That(data.Count, Is.GreaterThan(2)); + var dataItem = data.First(); + Assert.That(dataItem.Obs, Is.EqualTo(85d)); + //Assert.That(dataItem.Photo, Is.EqualTo(10.49195882)); + } + [Test] public void Parse_Valid() { diff --git a/Core.Tests/Parsers/LeafInputData/20171115002141clean10c_day3_high.csv b/Core.Tests/Parsers/LeafInputData/20171115002141clean10c_day3_high.csv new file mode 100644 index 0000000..ba5192d --- /dev/null +++ b/Core.Tests/Parsers/LeafInputData/20171115002141clean10c_day3_high.csv @@ -0,0 +1,30 @@ +Investigator name: Anna Schyman +Contact information: ss222uu@studen.lnu.se +Site name in full: Linneaus university. Växjö. Sweden +Vegetation type: Mainly conifers +Soil type: +Major species: Picea abies. +Sample leaf light environment: full sun. 1000 PAR +Water stress assessment: +Instrument used: LI6400 +Extra info: LICOR filenames: 2017-09-27-Anna-day-1-LI6800-extracurves-31-40-onLI6400branch-Low-fertility-Picea-Asa +SiteID,Latitude(Degrees),Longitude(Degrees),Elevation,SampleYear,SampleDayOfYear,GrowSeasonStart,GrowSeasonEnd,StandAge,CanopyHeight,LeafAreaIndex,SpeciesSampled,AveTimeResolution,SampleHeight,LeafAge,SpecificLeafArea,LfNitrogenContent,LfCarbonContent,LfPhosphContent +NoUnit,NorthPositive,EastPositive,m,NoUnit,DayOfYear,DayOfYear,DayOfYear,Year,m,m2/m2,NoBlankSpace,Minutes,m,days,cm2/g,%,%,% +LNU,56,14,173,2017,200,-9999,-9999,-9999,-9999,-9999,piceaabies,3,2,-9999,-9999,-9999,-9999,-9999 +Gamma*,Kc,Ko,Alpha,Rd,gi +Pa,Pa,Pa,NoUnit,umol/m2/s,umol/m2/s/Pa +-9999,-9999,-9999,-9999,-9999,-9999 +Obs,HHMMSS,FTime,Photo,!AdjPhoto,!StomCond,!Ci,!Trmmol,!VpdL,Area,StmRat,BLCond,Tair,!Tleaf,TBlk,CO2R,CO2S,H2OR,H2OS,RH_R,RH_S,Flow,!PARi,PARo,Press,CsMch,HsMch,StableF,Status,PhiPS2,OxygenPress +NoUnit,HHMMSS,Second,umol/m2/s,umol/m2/s,mol/m2/s,umol/mol,mmol/m2/s,kPa,cm2,NA,mol/m2/s,oC,oC,oC,umol/mol,umol/mol,mmol/mol,mmol/mol,%,%,umol/s,umol/m2/s,umol/m2/s,Kpa,umol/mol,mmol/mol,NA,NA,NA,KPA +85,20170930_08:11:59,1506751918,-9999,6.136374004,0.104548655,296.6792788,0.702591832,0.702942496,3.312,1,1.231735026,12.3824,13.9593,9.99909,400.027,398.256,8.67641,8.86862,62.14847711,62.14847711,1199.91,1000.31,0.215183,101.085,0.015,-1.36,1,111115,-9999,-9999 +86,20170930_08:13:40,1506752018,-9999,5.229101434,0.106245968,213.6473413,0.724975456,0.714143328,3.312,1,1.236378876,12.3247,13.9381,9.99919,300.016,298.513,8.5385,8.73685,61.45270846,61.45270846,1199.97,999.915,0.215183,101.077,0.013,-1.303,1,111115,-9999,-9999 +87,20170930_08:15:20,1506752119,-9999,3.618049084,0.106568105,140.4742504,0.736507702,0.72337472,3.312,1,1.239513716,12.2737,13.9075,9.99934,200.002,198.963,8.41281,8.61433,60.79377925,60.79377925,1200.03,999.897,0.215183,101.076,0.014,-1.256,1,111115,-9999,-9999 +88,20170930_08:17:01,1506752219,-9999,1.536044008,0.105817722,74.397357,0.735848133,0.727757148,3.312,1,1.238598257,12.2177,13.8444,9.99919,100.015,99.5708,8.30458,8.50596,60.25455594,60.25455594,1199.92,1000.18,0.215183,101.083,0.013,-1.205,1,111115,-9999,-9999 +89,20170930_08:18:41,1506752320,-9999,0.384430912,0.10645484,43.31962971,0.747904008,0.735721631,3.312,1,1.234816227,12.1776,13.8155,9.99921,49.9954,49.879,8.19178,8.39647,59.64536779,59.64536779,1199.99,1000.22,0.215183,101.099,0.013,-1.239,1,111115,-9999,-9999 +90,20170930_08:20:22,1506752420,-9999,6.946106925,0.10574346,284.6871084,0.736374895,0.729077567,3.312,1,1.236228158,12.0969,13.7089,9.99932,400.007,398.009,8.15211,8.35365,59.65948755,59.65948755,1200.01,999.922,0.215183,101.103,0.013,-1.302,1,111115,-9999,-9999 +91,20170930_08:22:02,1506752521,-9999,8.651450163,0.104120681,403.8341017,0.716273029,0.719671214,3.312,1,1.23776346,12.0551,13.6498,9.99969,550.023,547.527,8.19123,8.38726,60.06229735,60.06229735,1200.02,999.856,0.215183,101.099,0.012,-1.468,1,111115,-9999,-9999 +92,20170930_08:23:43,1506752621,-9999,9.218194654,0.107063182,497.292019,0.737619663,0.721810254,3.312,1,1.232623781,12.0393,13.5999,9.99951,649.971,647.295,8.11426,8.31615,59.61308615,59.61308615,1200,1000.29,0.215183,101.096,0.011,-1.622,1,111115,-9999,-9999 +93,20170930_08:25:23,1506752722,-9999,10.03431942,0.108732171,634.9655259,0.753451304,0.726378135,3.312,1,1.23650011,11.9848,13.5856,9.99952,799.982,797.047,8.05092,8.25714,59.39932867,59.39932867,1200.09,999.942,0.215183,101.09,0.01,-1.745,1,111115,-9999,-9999 +94,20170930_08:27:04,1506752822,-9999,10.35195782,0.109296219,779.0174427,0.761084273,0.73012856,3.312,1,1.237249011,11.9558,13.5335,9.9996,949.982,946.926,7.95977,8.16811,58.86903548,58.86903548,1200.02,999.933,0.215183,101.086,0.011,-1.759,1,111115,-9999,-9999 +95,20170930_08:28:44,1506752923,-9999,10.89016509,0.109569654,1018.188992,0.769796283,0.736665215,3.312,1,1.239551132,11.9306,13.504,9.99955,1200.11,1196.85,7.86371,8.07446,58.2872515,58.2872515,1199.99,999.986,0.215183,101.08,0.01,-1.742,1,111115,-9999,-9999 +96,20170930_08:30:26,1506753025,-9999,11.08669435,0.109312286,1409.96255,0.771356692,0.739858196,3.312,1,1.238941384,11.8885,13.4613,9.99954,1600.11,1596.71,7.78923,8.00043,57.91202718,57.91202718,1199.95,999.987,0.215183,101.078,0.009,-1.65,1,111115,-9999,-9999 diff --git a/Core/Core.csproj b/Core/Core.csproj index e24f3f9..6a1a1eb 100644 --- a/Core/Core.csproj +++ b/Core/Core.csproj @@ -92,6 +92,7 @@ + diff --git a/Core/Parsers/LeafInputCleaner.cs b/Core/Parsers/LeafInputCleaner.cs new file mode 100644 index 0000000..2ac31ef --- /dev/null +++ b/Core/Parsers/LeafInputCleaner.cs @@ -0,0 +1,18 @@ +using LeafWeb.Core.Entities; + +namespace LeafWeb.Core.Parsers +{ + public static class LeafInputCleaner + { + public static void Clean(LeafInputData leafInput) + { + foreach (var data in leafInput.Data) + { + if (!string.IsNullOrEmpty(data.HHMMSS) && data.HHMMSS.Length > 8) + { + data.HHMMSS = data.HHMMSS.Substring(data.HHMMSS.Length - 8, 8); + } + } + } + } +} \ No newline at end of file diff --git a/WebCms/App_Data/Models/all.dll.path b/WebCms/App_Data/Models/all.dll.path index daa4bc9..cc6fe8a 100644 --- a/WebCms/App_Data/Models/all.dll.path +++ b/WebCms/App_Data/Models/all.dll.path @@ -1 +1 @@ -C:\Users\poprhythm\AppData\Local\Temp\Temporary ASP.NET Files\vs\f80e29bb\faae20bf\App_Web_all.generated.cs.8f9494c4.isytfmyx.dll \ No newline at end of file +C:\Users\poprhythm\AppData\Local\Temp\Temporary ASP.NET Files\vs\f80e29bb\faae20bf\App_Web_all.generated.cs.8f9494c4.v5ghxtap.dll \ No newline at end of file diff --git a/WebCms/Services/PiscalQueue/FinishComplete.cs b/WebCms/Services/PiscalQueue/FinishComplete.cs index 6da3448..caca86b 100644 --- a/WebCms/Services/PiscalQueue/FinishComplete.cs +++ b/WebCms/Services/PiscalQueue/FinishComplete.cs @@ -1,4 +1,5 @@ using System; +using System.Data.Entity.Validation; using System.Linq; using LeafWeb.Core.Entities; using LeafWeb.Core.Parsers; @@ -53,11 +54,22 @@ namespace LeafWeb.WebCms.Services.PiscalQueue { var parser = new LeafInputCsvParser(outputFile.FileContents.Contents); var data = parser.Parse(); + LeafInputCleaner.Clean(data); data.LeafInput = leafInput; data.LeafOutputFile = outputFile; leafInput.LeafInputData.Add(data); DataService.UpdateLeafInput(leafInput); } + catch (DbEntityValidationException ex) + { + var valErrors = + string.Join("", + ex.EntityValidationErrors.Select( + e => string.Join(Environment.NewLine, + e.ValidationErrors.Select(v => $"{v.PropertyName} - {v.ErrorMessage}")))); + Logger.Error($"LeafInput: {leafInput.Id}, while parsing CleanedInput file: {outputFile.Filename}"+ + $"{Environment.NewLine}Validation Errors:{valErrors}"); + } catch (Exception e) { Logger.Error($"LeafInput: {leafInput.Id}, while parsing CleanedInput file: {outputFile.Filename}", e); diff --git a/WebCms/Services/PiscalQueue/PiscalQueueManager.cs b/WebCms/Services/PiscalQueue/PiscalQueueManager.cs index cfdf761..b261535 100644 --- a/WebCms/Services/PiscalQueue/PiscalQueueManager.cs +++ b/WebCms/Services/PiscalQueue/PiscalQueueManager.cs @@ -230,6 +230,7 @@ namespace LeafWeb.WebCms.Services.PiscalQueue case PiscalStatus.Running: Logger.DebugFormat("LeafInput: {0}, Piscal Running", leafInput.Id); + //leafInput.EndTime; // continue running break;