using System; using System.Collections.Generic; using System.IO; using System.Linq; using LeafWeb.Core.Entities; using LeafWeb.Core.Parsers; using LeafWeb.Core.Utility; using NUnit.Framework; namespace LeafWeb.Core.Tests.Parsers { [TestFixture] public class LeafInputCsvParserTests { private const string ContentDirectory = @"Parsers\LeafInputData\"; [Test] public void Parse_Cleaned() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned.csv"); var leafInput = new LeafInputCsvParser(fileInfo); var input = leafInput.Parse(); Assert.That(input.InvestigatorName, Is.EqualTo("Valid Cleaned")); var site = input.Site; Assert.That(site.SiteId, Is.Null); Assert.That(site.Latitude, Is.EqualTo(-20.75d)); var photosynthetic = input.Photosynthetic; Assert.That(photosynthetic.GammaStar, Is.EqualTo(10)); var data = input.Data; Assert.That(data.Count(), Is.GreaterThan(2)); var dataItem = data.First(); Assert.That(dataItem.Obs, Is.EqualTo(10)); Assert.That(dataItem.Photo, Is.EqualTo(10.49195882)); } [Test] public void Parse_Columns_Moved() { var originalFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned.csv"); var columnsMovedFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned-columns-moved.csv"); var originalParser = new LeafInputCsvParser(originalFileInfo); var columnsMovedParser = new LeafInputCsvParser(columnsMovedFileInfo); var original = originalParser.Parse(); var columnsMoved = columnsMovedParser.Parse(); Assert.That(columnsMoved.ContactInformation, Is.EqualTo("validcleaned@yahoo.com.br")); Assert.AreEqual(original.Site.Elevation, columnsMoved.Site.Elevation); Assert.AreEqual(original.Site.Latitude, columnsMoved.Site.Latitude); Assert.AreEqual(original.Site.AverageTimeResolution, columnsMoved.Site.AverageTimeResolution); Assert.AreEqual(original.Site.SampleYear, columnsMoved.Site.SampleYear); Assert.AreEqual(original.Photosynthetic.GammaStar, columnsMoved.Photosynthetic.GammaStar); for (var i = 0; i < original.Data.Count; i++) { Assert.AreEqual(original.Data.Skip(i).First().CO2S, columnsMoved.Data.Skip(i).First().CO2S); Assert.AreEqual(original.Data.Skip(i).First().AdjPhoto, columnsMoved.Data.Skip(i).First().AdjPhoto); } } [Test] public void Parse_NewLines() { var originalFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned.csv"); var columnsMovedFileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-cleaned-newlines.csv"); var originalParser = new LeafInputCsvParser(originalFileInfo); var columnsMovedParser = new LeafInputCsvParser(columnsMovedFileInfo); var original = originalParser.Parse(); var columnsMoved = columnsMovedParser.Parse(); Assert.That(columnsMoved.ContactInformation, Is.EqualTo("validcleaned@yahoo.com.br")); Assert.AreEqual(original.Site.Elevation, columnsMoved.Site.Elevation); Assert.AreEqual(original.Site.Latitude, columnsMoved.Site.Latitude); Assert.AreEqual(original.Site.AverageTimeResolution, columnsMoved.Site.AverageTimeResolution); Assert.AreEqual(original.Site.SampleYear, columnsMoved.Site.SampleYear); Assert.AreEqual(original.Photosynthetic.GammaStar, columnsMoved.Photosynthetic.GammaStar); for (var i = 0; i < original.Data.Count; i++) { Assert.AreEqual(original.Data.Skip(i).First().CO2S, columnsMoved.Data.Skip(i).First().CO2S); Assert.AreEqual(original.Data.Skip(i).First().AdjPhoto, columnsMoved.Data.Skip(i).First().AdjPhoto); } } [Test] public void Parse_Cleaned_AnnaDay1() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "20171115002141clean10c_day3_high.csv"); var leafInput = new LeafInputCsvParser(fileInfo); var input = leafInput.Parse(); Assert.That(input.InvestigatorName, Is.EqualTo("Anna Schyman")); var site = input.Site; Assert.That(site.SiteId, Is.EqualTo("LNU")); Assert.That(site.Latitude, Is.EqualTo(56d)); var photosynthetic = input.Photosynthetic; Assert.That(photosynthetic.GammaStar, Is.EqualTo(0)); var data = input.Data; Assert.That(data.Count, Is.GreaterThan(2)); var dataItem = data.First(); Assert.That(dataItem.Obs, Is.EqualTo(85d)); //Assert.That(dataItem.Photo, Is.EqualTo(10.49195882)); } [Test] public void Parse_Valid() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-valid.csv"); var leafInput = new LeafInputCsvParser(fileInfo); var input = leafInput.Parse(); Assert.That(input.InvestigatorName, Is.EqualTo("Valid Name")); Assert.That(input.ContactInformation, Is.EqualTo("validname@ornl.gov")); Assert.That(input.SiteName, Is.EqualTo("FACE")); Assert.That(input.VegetationType, Is.EqualTo("tropical forest")); Assert.That(input.SoilType, Is.EqualTo("dense soil")); Assert.That(input.MajorSpecies, Is.EqualTo("Tulip Poplar, Oak")); Assert.That(input.SampleLeafLightEnv, Is.EqualTo("heavily shaded from above")); Assert.That(input.WaterStressAssessment, Is.EqualTo("Little to no stress")); Assert.That(input.InstrumentUsed, Is.EqualTo("Licor-6400")); Assert.That(input.ExtraInfo, Is.EqualTo("dear01")); var site = input.Site; Assert.That(site.SiteId, Is.EqualTo("Metropolitano")); Assert.That(site.Latitude, Is.EqualTo(8.98333)); Assert.That(site.Longitude, Is.EqualTo(-79.55)); Assert.That(site.Elevation, Is.Null); Assert.That(site.SampleYear, Is.EqualTo(2013)); Assert.That(site.SampleDayOfYear, Is.EqualTo(22)); Assert.That(site.GrowSeasonStart, Is.Null); Assert.That(site.GrowSeasonEnd, Is.Null); Assert.That(site.StandAge, Is.Null); Assert.That(site.CanopyHeight, Is.Null); Assert.That(site.LeafAreaIndex, Is.Null); Assert.That(site.SpeciesSampled, Is.EqualTo("Dendropanax_arboreus")); Assert.That(site.AverageTimeResolution, Is.Null); Assert.That(site.SampleHeight, Is.Null); Assert.That(site.LeafAge, Is.Null); Assert.That(site.SpecificLeafArea, Is.EqualTo(245.5609363)); Assert.That(site.LfNitrogenContent, Is.EqualTo(2.19)); Assert.That(site.LfCarbonContent, Is.EqualTo(46.03)); Assert.That(site.LfPhosphContent, Is.EqualTo(0.129929353)); //Assert.That(site.WoodPorosity, Is.Null); //Assert.That(site.SapWoodDensity, Is.Null); var photosynthetic = input.Photosynthetic; Assert.That(photosynthetic.GammaStar, Is.EqualTo(1.2)); Assert.That(photosynthetic.Kc, Is.EqualTo(2.3)); Assert.That(photosynthetic.Ko, Is.EqualTo(3.4)); Assert.That(photosynthetic.Alpha, Is.EqualTo(4.5)); Assert.That(photosynthetic.Rd, Is.EqualTo(6.5)); Assert.That(photosynthetic.gi, Is.EqualTo(4.3)); var data = input.Data; Assert.That(data.Count(), Is.GreaterThan(2)); var dataItem = data.First(); Assert.That(dataItem.Obs, Is.EqualTo(2)); Assert.That(dataItem.HHMMSS, Is.EqualTo("11:26:17")); Assert.That(dataItem.FTime, Is.EqualTo(681.5)); Assert.That(dataItem.Photo, Is.EqualTo(6.707855907)); Assert.That(dataItem.AdjPhoto, Is.EqualTo(6.707855907)); Assert.That(dataItem.StomCond, Is.EqualTo(0.066161416)); Assert.That(dataItem.Ci, Is.EqualTo(185.9999057)); Assert.That(dataItem.Trmmol, Is.EqualTo(0.936751155)); Assert.That(dataItem.VpdL, Is.EqualTo(1.386027641)); Assert.That(dataItem.Area, Is.EqualTo(2)); Assert.That(dataItem.StmRat, Is.EqualTo(1)); Assert.That(dataItem.BLCond, Is.EqualTo(9.289719582)); Assert.That(dataItem.Tair, Is.EqualTo(29.09575272)); Assert.That(dataItem.Tleaf, Is.EqualTo(29.91356087)); Assert.That(dataItem.TBlk, Is.EqualTo(28.63459015)); Assert.That(dataItem.CO2R, Is.EqualTo(360.0646362)); Assert.That(dataItem.CO2S, Is.EqualTo(355.3765869)); Assert.That(dataItem.H2OR, Is.EqualTo(27.71545029)); Assert.That(dataItem.H2OS, Is.EqualTo(28.32151604)); Assert.That(dataItem.RH_R, Is.EqualTo(69.04498291)); Assert.That(dataItem.RH_S, Is.EqualTo(70.5548172)); Assert.That(dataItem.Flow, Is.EqualTo(300.3703613)); Assert.That(dataItem.PARi, Is.EqualTo(500.4228821)); Assert.That(dataItem.PARo, Is.EqualTo(23.31687355)); Assert.That(dataItem.Press, Is.EqualTo(100.7472458)); Assert.That(dataItem.CsMch, Is.EqualTo(1.491710782)); Assert.That(dataItem.HsMch, Is.EqualTo(10.9474802)); Assert.That(dataItem.StableF, Is.EqualTo(0.5)); Assert.That(dataItem.Status, Is.EqualTo("111115")); Assert.That(dataItem.PhiPS2, Is.Null); Assert.That(dataItem.OxygenPress, Is.Null); } [Test] public void Parse_TitlesRemoved() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-titlesRemoved.csv"); var leafInput = new LeafInputCsvParser(fileInfo); var input = leafInput.Parse(); Assert.That(input.InvestigatorName, Is.EqualTo("Valid Name")); Assert.That(input.ContactInformation, Is.EqualTo("validname@ornl.gov")); Assert.That(input.SiteName, Is.EqualTo("FACE")); Assert.That(input.VegetationType, Is.EqualTo("tropical forest")); Assert.That(input.SoilType, Is.EqualTo("dense soil")); Assert.That(input.MajorSpecies, Is.EqualTo("Tulip Poplar, Oak")); Assert.That(input.SampleLeafLightEnv, Is.EqualTo("heavily shaded from above")); Assert.That(input.WaterStressAssessment, Is.EqualTo("Little to no stress")); Assert.That(input.InstrumentUsed, Is.EqualTo("Licor-6400")); Assert.That(input.ExtraInfo, Is.EqualTo("dear01")); } [Test] public void Parse_IncompleteRows() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-incompleteRows.csv"); var parser = new LeafInputCsvParser(fileInfo); Assert.That(() => parser.Parse(), Throws.TypeOf()); } [Test] public void Parse_NoData() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-noData.csv"); var parser = new LeafInputCsvParser(fileInfo); var input = parser.Parse(); } [Test, Ignore("TODO: Look at why this file is not parsed correctly")] public void Parse_JN1() { var fileInfo = FileUtility.GetContentFile(ContentDirectory, "LeafInput-JN_1.csv"); var parser = new LeafInputCsvParser(fileInfo); var input = parser.Parse(); } [Explicit] public void Parse_FindIssues() { var dir = @"C:\Users\poprhythm\Documents\code\LeafWeb\Notes\leafweb database work\newcurves\RemovableDisk\curves"; var files = Directory.GetFiles(dir, "*.csv").Select(f => new FileInfo(f)).ToList(); var leafInputs = new List(); var num = 1; foreach (var file in files) { try { var parser = new LeafInputCsvParser(file); var leafInput = parser.Parse(); leafInputs.Add(leafInput); Console.WriteLine("{3}/{4}, {0}, {1}, {2}", file.Name, leafInput.InvestigatorName, leafInput.ContactInformation, num++, files.Count); } catch (Exception ex) { File.AppendAllLines(@"C:\Temp\errors.txt", new[] { string.Format("{1} : {0}", ex.Message, file.Name) }); Console.WriteLine("{1} : {0}", ex.Message, file.Name); } } LeafInputCsvParser.ExportCsv(@"C:\Temp\test.csv", leafInputs); } [Explicit] [Test] public void FailedParseData_FindIssues() { const string dir = @"C:\temp\LeafWeb_FailedParseData\"; var files = Directory.GetFiles(dir, "*.csv", SearchOption.AllDirectories).Select(f => new FileInfo(f)).ToList(); var leafInputs = new List(); var num = 1; foreach (var file in files) { try { var parser = new LeafInputCsvParser(file); var leafInput = parser.Parse(); leafInputs.Add(leafInput); Console.WriteLine("{3}/{4}, {0}, {1}, {2}", file.Name, leafInput.InvestigatorName, leafInput.ContactInformation, num++, files.Count); } catch (Exception ex) { File.AppendAllLines(@"C:\Temp\errors.txt", new[] { string.Format("{2}\\{1} : {0}", ex.Message, file.Name, file.Directory?.Name) }); Console.WriteLine("{2}\\{1} : {0}", ex.Message, file.Name, file.Directory?.Name); } } LeafInputCsvParser.ExportCsv(@"C:\Temp\test.csv", leafInputs); } } }