Handle blank lines in LeafInput
This commit is contained in:
@@ -104,6 +104,9 @@
|
||||
<Content Include="Parsers\LeafInputData\LeafInput-cleaned-columns-moved.csv">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="Parsers\LeafInputData\LeafInput-cleaned-newlines.csv">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<None Include="Parsers\LeafInputData\LeafInput-tabSeparated.csv" />
|
||||
<Content Include="Parsers\LeafOutputData\cntrlbestparameters_Wild Capsicum.csv">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
|
||||
@@ -62,6 +62,32 @@ namespace LeafWeb.Core.Tests.Parsers
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Parse_NewLines()
|
||||
{
|
||||
var originalFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned.csv");
|
||||
var columnsMovedFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned-newlines.csv");
|
||||
var originalParser = new LeafInputCsvParser(originalFileInfo);
|
||||
var columnsMovedParser = new LeafInputCsvParser(columnsMovedFileInfo);
|
||||
|
||||
var original = originalParser.Parse();
|
||||
var columnsMoved = columnsMovedParser.Parse();
|
||||
Assert.That(columnsMoved.ContactInformation, Is.EqualTo("validcleaned@yahoo.com.br"));
|
||||
|
||||
Assert.AreEqual(original.Site.Elevation, columnsMoved.Site.Elevation);
|
||||
Assert.AreEqual(original.Site.Latitude, columnsMoved.Site.Latitude);
|
||||
Assert.AreEqual(original.Site.AverageTimeResolution, columnsMoved.Site.AverageTimeResolution);
|
||||
Assert.AreEqual(original.Site.SampleYear, columnsMoved.Site.SampleYear);
|
||||
|
||||
Assert.AreEqual(original.Photosynthetic.GammaStar, columnsMoved.Photosynthetic.GammaStar);
|
||||
|
||||
for (var i = 0; i < original.Data.Count; i++)
|
||||
{
|
||||
Assert.AreEqual(original.Data.Skip(i).First().CO2S, columnsMoved.Data.Skip(i).First().CO2S);
|
||||
Assert.AreEqual(original.Data.Skip(i).First().AdjPhoto, columnsMoved.Data.Skip(i).First().AdjPhoto);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Parse_Cleaned_AnnaDay1()
|
||||
{
|
||||
@@ -205,7 +231,7 @@ namespace LeafWeb.Core.Tests.Parsers
|
||||
var input = parser.Parse();
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Test, Ignore("TODO: Look at why this file is not parsed correctly")]
|
||||
public void Parse_JN1()
|
||||
{
|
||||
var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-JN_1.csv");
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
Investigator name: Valid Cleaned
|
||||
Contact information: validcleaned@yahoo.com.br
|
||||
Site name in full: NA
|
||||
Vegetation type: NA
|
||||
|
||||
Soil type: NA
|
||||
Major species: NA
|
||||
Sample leaf light environment: Full Sun
|
||||
"Water_stress_assessment:_No_water_stress, plants_irrigated_every_day"
|
||||
Instrument used: Licor-6400XT
|
||||
Extra info:
|
||||
|
||||
SiteID,Latitude(Degrees),Longitude(Degrees),Elevation,SampleYear,SampleDayOfYear,GrowSeasonStart,GrowSeasonEnd,StandAge,CanopyHeight,LeafAreaIndex,SpeciesSampled,AveTimeResolution,SampleHeight,LeafAge,SpecificLeafArea,LfNitrogenContent,LfCarbonContent,LfPhosphContent
|
||||
NoUnit,NorthPositive,EastPositive,m,NoUnit,DayOfYear,DayOfYear,DayOfYear,Year,m,m2/m2,NoBlankSpace,Minutes,m,days,cm2/g,%,%,%
|
||||
|
||||
-9999,-20.75,-42.88,648,2009,-9999,-9999,-9999,-9999,-9999,-9999,CoffeaArabica,-9999,-9999,-9999,140,3.48,-9999,-9999
|
||||
Gamma*,Kc,Ko,Alpha,Rd,gi
|
||||
Pa,Pa,Pa,NoUnit,umol/m2/s,umol/m2/s/Pa
|
||||
10,-9999,-9999,-9999,-9999,-9999
|
||||
|
||||
Obs,HHMMSS,FTime,Photo,!AdjPhoto,!StomCond,!Ci,!Trmmol,!VpdL,Area,StmRat,BLCond,Tair,!Tleaf,TBlk,CO2R,CO2S,H2OR,H2OS,RH_R,RH_S,Flow,!PARi,PARo,Press,CsMch,HsMch,StableF,Status,PhiPS2,OxygenPress
|
||||
NoUnit,HHMMSS,Second,umol/m2/s,umol/m2/s,mol/m2/s,umol/mol,mmol/m2/s,kPa,cm2,NA,mol/m2/s,oC,oC,oC,umol/mol,umol/mol,mmol/mol,mmol/mol,%,%,umol/s,umol/m2/s,umol/m2/s,Kpa,umol/mol,mmol/mol,NA,NA,NA,KPA
|
||||
|
||||
10,-9999,-9999,10.49195882,11.05941043,0.244883469,212.5041051,2.63076876,1.036135144,2,0,4.644859791,23.52121258,23.53769627,23.3545414,300.0483063,292.5618317,18.24994297,19.96389828,58.95560608,64.49247513,300.8535187,993.0888855,0.049617826,94.00339508,-2.052112579,-0.246214688,0.425,111115,0.180769687,21
|
||||
10,-9999,-9999,6.613751647,7.414357534,0.200000749,130.8730299,2.24618742,1.072960558,2,0,4.644859791,23.57428112,23.68962097,23.40673904,200.1022598,195.4142136,18.39681854,19.86024399,59.23092155,63.94262123,300.8801514,992.5410278,0.049735996,93.98853455,-1.348773956,-0.243325055,0.475,111115,0.163288942,21
|
||||
10,-9999,-9999,4.431079608,5.349146841,0.170652088,92.42684697,2.00393304,1.114947396,2,0,4.644859791,23.58453674,23.90329285,23.41538334,149.6127335,146.4719864,18.51102257,19.8167675,59.56615295,63.76787758,300.8583466,992.1625671,0.054449931,93.9954071,-1.367557526,-0.230450451,0.375,111115,0.147352806,21
|
||||
10,-9999,-9999,1.523393138,2.556557599,0.159880602,70.4656295,1.965014975,1.163792845,2,0,4.644859791,23.59998169,24.21637478,23.43008366,99.656147,98.51480789,18.61892796,19.89914742,59.85464211,63.97018967,300.8723236,991.761908,0.039717828,93.9906868,-0.692935944,-0.232964337,0.575,111115,0.130597749,21
|
||||
10,-9999,-9999,0.894293187,1.987936699,0.167650406,52.50296582,2.025541888,1.145843431,2,0,4.644859791,23.62822838,24.19524651,23.45776844,74.00842819,73.31520309,18.72938271,20.0488924,60.108358,64.34307251,300.8590363,991.5204651,0.045964496,93.99226608,-0.604091644,-0.238077939,0.575,111115,0.122154858,21
|
||||
10,-9999,-9999,-0.495843139,0.655970767,0.176641615,42.05656346,2.078428704,1.118051438,2,0,4.644859791,23.63969841,24.11649742,23.46931267,48.81573753,49.07753906,18.83879395,20.19250221,60.41828194,64.75978699,300.871344,991.50896,0.042428688,93.99311447,-0.591835022,-0.238228619,0.7,111115,0.11011836,21
|
||||
10,-9999,-9999,14.01214221,14.34608078,0.188886895,258.1956724,2.081065547,1.052338661,2,0,4.644859791,23.63486118,23.77918701,23.46806164,399.7125275,389.8589264,18.8980257,20.25335598,60.6206089,64.96820526,300.8738037,991.5494934,0.06246434,93.98490219,-3.991065979,-0.239916623,0.35,111115,0.183856114,21
|
||||
10,-9999,-9999,19.20963114,19.06936549,0.167346311,390.2278007,1.923908779,1.090216329,2,0,4.644859791,23.6730854,23.97962074,23.50691929,600.9658813,587.4440186,18.97605591,20.22919121,60.72654266,64.73679276,300.8435883,990.6557251,0.06564644,93.97809906,-4.053321838,-0.263443768,0.6,111115,0.224674063,21
|
||||
10,-9999,-9999,22.68315779,22.07166985,0.140898278,513.7169863,1.709378251,1.143940773,2,0,4.644859791,23.72432709,24.24497566,23.55486546,799.7560669,783.7866455,19.05482311,20.1682272,60.79165916,64.34382095,300.861908,990.5349609,0.044432433,93.97946396,-6.789344788,-0.258801281,0.625,111115,0.241833394,21
|
||||
10,-9999,-9999,25.21838506,24.1313968,0.128254365,657.3239246,1.569373475,1.150702993,2,0,4.644859791,23.77119751,24.27973061,23.60147991,999.7003906,981.9117737,19.14113102,20.16335564,60.89594879,64.14807434,300.8594788,990.3832764,0.037125352,93.98093872,-10.22812653,-0.260886014,0.65,111115,0.250284039,21
|
||||
10,-9999,-9999,27.03077557,25.46461001,0.120765245,816.0761447,1.52318325,1.183928533,2,0,4.644859791,23.80861893,24.49090347,23.63921146,1200.733826,1181.568982,19.23064556,20.22269268,61.04207878,64.19104309,300.8688538,990.2308533,0.058457612,93.97930527,-14.02487946,-0.262095273,0.575,111115,0.252074273,21
|
||||
10,-9999,-9999,28.6322564,26.59017955,0.117258539,984.2999029,1.510964689,1.208454852,2,0,4.644859791,23.86426582,24.66598072,23.69386177,1400.285156,1379.865356,19.32225647,20.30630074,61.13061714,64.24388199,300.8569305,990.0950989,0.060696683,93.9834198,-17.98886871,-0.264069378,0.65,111115,0.25113703,21
|
||||
10,-9999,-9999,29.59434444,27.07367747,0.111042426,1152.036476,1.451859088,1.224292598,2,0,4.644859791,23.92603722,24.77762184,23.75417137,1600.474329,1579.277905,19.41603336,20.36147785,61.19426117,64.17406006,300.8736847,989.9327454,0.052446734,93.97519684,-22.0407486,-0.271241009,0.65,111115,0.252735204,21
|
||||
10,-9999,-9999,30.43679512,26.95968124,0.103627708,1515.947025,1.36954251,1.235421377,2,0,4.644859791,23.97047672,24.84818077,23.79694042,1999.831714,1977.797449,19.49336185,20.38523922,61.27052002,64.07382736,300.8539337,989.8097778,0.061404274,93.96966782,-31.06003571,-0.273886502,0.625,111115,0.248197252,21
|
||||
|
@@ -10,25 +10,25 @@ namespace LeafWeb.Core.Parsers
|
||||
private readonly MemoryStream _memoryStream;
|
||||
protected readonly CsvReader CsvReader;
|
||||
|
||||
protected CsvParserBase(FileSystemInfo csvFile)
|
||||
protected CsvParserBase(FileSystemInfo csvFile, bool ignoreBlankLines)
|
||||
{
|
||||
_reader = OpenCsv(csvFile);
|
||||
CsvReader = InitCsvReader(_reader);
|
||||
CsvReader = InitCsvReader(_reader, ignoreBlankLines);
|
||||
}
|
||||
|
||||
protected CsvParserBase(byte[] fileContents)
|
||||
protected CsvParserBase(byte[] fileContents, bool ignoreBlankLines)
|
||||
{
|
||||
_memoryStream = new MemoryStream(fileContents);
|
||||
_reader = new StreamReader(_memoryStream);
|
||||
|
||||
CsvReader = InitCsvReader(_reader);
|
||||
CsvReader = InitCsvReader(_reader, ignoreBlankLines);
|
||||
}
|
||||
|
||||
private static CsvReader InitCsvReader(StreamReader reader)
|
||||
private static CsvReader InitCsvReader(TextReader reader, bool ignoreBlankLines)
|
||||
{
|
||||
var csvReader = new CsvReader(reader);
|
||||
csvReader.Configuration.HasHeaderRecord = false;
|
||||
csvReader.Configuration.IgnoreBlankLines = false;
|
||||
csvReader.Configuration.IgnoreBlankLines = ignoreBlankLines;
|
||||
csvReader.Configuration.ReadingExceptionOccurred = exception => false;
|
||||
return csvReader;
|
||||
}
|
||||
@@ -53,8 +53,7 @@ namespace LeafWeb.Core.Parsers
|
||||
public void Dispose()
|
||||
{
|
||||
_reader.Dispose();
|
||||
if (_memoryStream != null)
|
||||
_memoryStream.Dispose();
|
||||
}
|
||||
_memoryStream?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using System.Linq;
|
||||
using CsvHelper;
|
||||
using CsvHelper.Configuration;
|
||||
using LeafWeb.Core.Entities;
|
||||
// ReSharper disable IdentifierTypo
|
||||
|
||||
namespace LeafWeb.Core.Parsers
|
||||
{
|
||||
|
||||
@@ -8,11 +8,11 @@ namespace LeafWeb.Core.Parsers
|
||||
{
|
||||
public class LeafGasComparisonParser : CsvParserBase
|
||||
{
|
||||
public LeafGasComparisonParser(FileSystemInfo csvFile) : base(csvFile)
|
||||
public LeafGasComparisonParser(FileSystemInfo csvFile) : base(csvFile, false)
|
||||
{
|
||||
}
|
||||
|
||||
public LeafGasComparisonParser(byte[] fileContents) : base(fileContents)
|
||||
public LeafGasComparisonParser(byte[] fileContents) : base(fileContents, false)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -9,11 +9,11 @@ namespace LeafWeb.Core.Parsers
|
||||
{
|
||||
public class LeafInputCsvParser : CsvParserBase
|
||||
{
|
||||
public LeafInputCsvParser(FileSystemInfo csvFile) : base(csvFile)
|
||||
public LeafInputCsvParser(FileSystemInfo csvFile) : base(csvFile, true)
|
||||
{
|
||||
}
|
||||
|
||||
public LeafInputCsvParser(byte[] fileContents) : base(fileContents)
|
||||
public LeafInputCsvParser(byte[] fileContents) : base(fileContents, true)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -45,9 +45,16 @@ namespace LeafWeb.Core.Parsers
|
||||
if (!CsvReader.Read())
|
||||
throw new ParseException("Could not read line number " + CsvReader.Context.Row);
|
||||
|
||||
string field;
|
||||
if (!CsvReader.TryGetField(0, out field))
|
||||
if (!CsvReader.TryGetField(0, out string field))
|
||||
throw new ParseException("Could not read first field on line number " + CsvReader.Context.Row);
|
||||
|
||||
// skip empty lines
|
||||
if (string.IsNullOrEmpty(field))
|
||||
{
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
|
||||
items.Add(field);
|
||||
}
|
||||
return ParsedObjectFactory<LeafInputData>.Create(items.ToArray());
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace LeafWeb.Core.Utility
|
||||
{
|
||||
public static void Register()
|
||||
{
|
||||
TypeDescriptor.AddAttributes(typeof(Boolean),
|
||||
TypeDescriptor.AddAttributes(typeof(bool),
|
||||
new TypeConverterAttribute(typeof(BoolTypeConverter)));
|
||||
}
|
||||
|
||||
@@ -24,26 +24,29 @@ namespace LeafWeb.Core.Utility
|
||||
|
||||
public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType)
|
||||
{
|
||||
if (value is bool)
|
||||
return value;
|
||||
if (value is string)
|
||||
{
|
||||
var s = value as string;
|
||||
if (string.IsNullOrEmpty(s))
|
||||
return false;
|
||||
switch (s.Trim().ToUpper())
|
||||
{
|
||||
case "TRUE":
|
||||
case "YES":
|
||||
case "1":
|
||||
case "-1":
|
||||
return true;
|
||||
switch (value)
|
||||
{
|
||||
case bool _:
|
||||
return value;
|
||||
case string s:
|
||||
{
|
||||
if (string.IsNullOrEmpty(s))
|
||||
return false;
|
||||
switch (s.Trim().ToUpper())
|
||||
{
|
||||
case "TRUE":
|
||||
case "YES":
|
||||
case "1":
|
||||
case "-1":
|
||||
return true;
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return base.ConvertTo(context, culture, value, destinationType);
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
default:
|
||||
return base.ConvertTo(context, culture, value, destinationType);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,8 +52,7 @@ namespace LeafWeb.Core.Utility
|
||||
|
||||
if (property != null)
|
||||
{
|
||||
object convertedVal;
|
||||
if (!TryConvertValue(property, value, out convertedVal))
|
||||
if (!TryConvertValue(property, value, out var convertedVal))
|
||||
throw new ParseException($"Cannot convert value '{value}' for {property.Name} at line number {lineNumber}");
|
||||
property.Set(obj, convertedVal);
|
||||
}
|
||||
@@ -83,8 +82,7 @@ namespace LeafWeb.Core.Utility
|
||||
|
||||
if (property != null)
|
||||
{
|
||||
object convertedVal;
|
||||
if (!TryConvertValue(property, value, out convertedVal))
|
||||
if (!TryConvertValue(property, value, out var convertedVal))
|
||||
throw new ParseException($"Cannot convert value '{value}' for {property.Name} in position {position}");
|
||||
property.Set(obj, convertedVal);
|
||||
}
|
||||
@@ -113,8 +111,7 @@ namespace LeafWeb.Core.Utility
|
||||
|
||||
if (property != null)
|
||||
{
|
||||
object convertedVal;
|
||||
if (!TryConvertValue(property, value, out convertedVal))
|
||||
if (!TryConvertValue(property, value, out var convertedVal))
|
||||
throw new ParseException($"Cannot convert value '{value}' for {property.Name} in position {position}");
|
||||
property.Set(obj, convertedVal);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user