From aea82da8972a3edab25549f467ff5905f8515ed2 Mon Sep 17 00:00:00 2001 From: Bobbie Hodgetts Date: Mon, 27 Jan 2020 16:50:42 +0000 Subject: [PATCH] Added parsing of Xero invoice export file --- .../BealeEngineering.Core.csproj | 21 ++- .../Data/Xero/SaleInvoiceGet.cs | 158 ++++++++++++++++ .../Logic/Import/wipXeroInvoiceFlatFile.cs | 54 ++++++ .../Logic/Utilities/CSVGetRFC4180Compliant.cs | 178 ++++++++++++++++++ .../Model/Import/XeroInvoiceFlatFile.cs | 57 ++++++ .../Model/Import/XeroInvoiceFlatFileDTO.cs | 53 ++++++ .../BealeEngineering.Core/Test/AUtoexec.cs | 8 +- .../Test/Import/ImportFlatfile.cs | 21 +++ .../BealeEngineering.Core/packages.config | 6 + 9 files changed, 551 insertions(+), 5 deletions(-) create mode 100644 BealeEngineering/BealeEngineering.Core/Data/Xero/SaleInvoiceGet.cs create mode 100644 BealeEngineering/BealeEngineering.Core/Logic/Import/wipXeroInvoiceFlatFile.cs create mode 100644 BealeEngineering/BealeEngineering.Core/Logic/Utilities/CSVGetRFC4180Compliant.cs create mode 100644 BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFile.cs create mode 100644 BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFileDTO.cs create mode 100644 BealeEngineering/BealeEngineering.Core/Test/Import/ImportFlatfile.cs diff --git a/BealeEngineering/BealeEngineering.Core/BealeEngineering.Core.csproj b/BealeEngineering/BealeEngineering.Core/BealeEngineering.Core.csproj index b42d248..6bdac92 100644 --- a/BealeEngineering/BealeEngineering.Core/BealeEngineering.Core.csproj +++ b/BealeEngineering/BealeEngineering.Core/BealeEngineering.Core.csproj @@ -31,11 +31,24 @@ 4 + + ..\packages\CsvHelper.13.0.0\lib\net47\CsvHelper.dll + ..\packages\Dapper.2.0.30\lib\net461\Dapper.dll + + ..\packages\Microsoft.Bcl.AsyncInterfaces.1.1.0\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll + + + + ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll + + + ..\packages\System.Threading.Tasks.Extensions.4.5.2\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll + @@ -50,26 +63,32 @@ + + + + + + - + diff --git a/BealeEngineering/BealeEngineering.Core/Data/Xero/SaleInvoiceGet.cs b/BealeEngineering/BealeEngineering.Core/Data/Xero/SaleInvoiceGet.cs new file mode 100644 index 0000000..dfbb25f --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Data/Xero/SaleInvoiceGet.cs @@ -0,0 +1,158 @@ +using CsvHelper; +using Microsoft.VisualBasic.FileIO; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BealeEngineering.Core.Data.Xero.FlatFile +{ + public class ImportInvoice + { + public ImportInvoice() + { + FileInputPath = + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + + @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv"; + FileOutputPath = + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + + @"\Downloads\MyNewTextFile.txt"; + } + private StringBuilder IntermediateCsv { get; set; } + public string FileInputPath { get; set; } + public string FileOutputPath { get; set; } + private List> MsVbTextParserResult { get; set; } + private StringBuilder CsvContent { get; set; } + /// + /// Imports Xero invoice flat-file into model class. + /// + /// + /// Dictionary, Invoice numbers against data. + public Dictionary ByFilePath(string filePath) + { + /* So here's the rub. Any field in a CSV doc that has a double quote wihtin, must be enclosed by double quotes and + * the double quote within must be 'escaped' by a double quote. + * However, in that situation, Xero flat file doesn't enclose the field or escape the double quote. + * CsvHelper cannot handle this situation. + * However, the MS VB TextFieldParser can. + * Therefore, parse with TextFieldParser, from this create file in the correct CSV format and the + * feed that into CsvHelper to map to a class. + * + * Long winded but easier than reinventing the wheel. + * + * Keep an eye on + * https://github.com/JoshClose/CsvHelper/issues/989 + * CsvHelper may nativily support when this feature is complete. + */ + + // first off, get a RFC4180 compliant csv string + var csvRFC = new Logic.Utilities.CSVGetRFC4180Compliant(); + csvRFC.ByFilePath(filePath); + if (!csvRFC.OutputStringIsSet) + { throw new Exception("CSV Error."); } + + // parse intermediate csv into data class + var dto = new List(); + using (TextReader reader = new StringReader(csvRFC.OutputString)) + using (var csv = new CsvReader(reader, CultureInfo.CurrentUICulture)) + { + csv.Configuration.DetectColumnCountChanges = true; + + dto = csv.GetRecords().ToList(); + } + + return ConvertFlatDTO(ref dto); + } + + private Dictionary ConvertFlatDTO(ref List flatData) + { + // ensure flat data is in invoice number order + var invDictionary = new Dictionary(); + string lastNumber = null; + foreach (var line in flatData) + { + if (line.InvoiceNumber != lastNumber) + { + lastNumber = line.InvoiceNumber; + if (invDictionary.ContainsKey(lastNumber)) + { + throw new Exception("Invoices are not grouped in CSV flatfile."); + } + else + { + invDictionary.Add(lastNumber, 0); + } + } + } + + // convert to one to many class data + var dictionaryList = new Dictionary(); + foreach (var line in flatData) + { + string invoiceNumber = line.InvoiceNumber; + + if (!dictionaryList.ContainsKey(invoiceNumber)) + { + var invoice = new Model.Import.XeroInvoiceFlatFile(); + + invoice.ContactName = line.ContactName; + invoice.Currency = line.Currency; + invoice.DueDate = line.DueDate; + invoice.EmailAddress = line.EmailAddress; + invoice.InvoiceAmountDue = line.InvoiceAmountDue; + invoice.InvoiceAmountPaid = line.InvoiceAmountPaid; + invoice.InvoiceDate = line.InvoiceDate; + invoice.InvoiceNumber = line.InvoiceNumber; + invoice.PlannedDate = line.PlannedDate; + invoice.POAddressLine1 = line.POAddressLine1; + invoice.POAddressLine2 = line.POAddressLine2; + invoice.POAddressLine3 = line.POAddressLine3; + invoice.POAddressLine4 = line.POAddressLine4; + invoice.POCity = line.POCity; + invoice.POCountry = line.POCountry; + invoice.POPostalCode = line.POPostalCode; + invoice.PORegion = line.PORegion; + invoice.Reference = line.Reference; + invoice.SAAddressLine1 = line.SAAddressLine1; + invoice.SAAddressLine2 = line.SAAddressLine2; + invoice.SAAddressLine3 = line.SAAddressLine3; + invoice.SAAddressLine4 = line.SAAddressLine4; + invoice.SACity = line.SACity; + invoice.SACountry = line.SACountry; + invoice.SAPostalCode = line.SAPostalCode; + invoice.SARegion = line.SARegion; + invoice.Sent = line.Sent; + invoice.Status = line.Status; + invoice.TaxTotal = line.TaxTotal; + invoice.Total = line.Total; + invoice.Type = line.Type; + + dictionaryList.Add(invoice.InvoiceNumber, invoice); + } + + var item = new Model.Import.XeroInvoiceFlatFile.LineItem(); + + item.AccountCode = line.AccountCode; + item.Description = line.Description; + item.Discount = line.Discount; + item.InventoryItemCode = line.InventoryItemCode; + item.LineAmount = line.LineAmount; + item.Quantity = line.Quantity; + item.TaxAmount = line.TaxAmount; + item.TaxType = line.TaxType; + item.TrackingName1 = line.TrackingName1; + item.TrackingName2 = line.TrackingName2; + item.TrackingOption1 = line.TrackingOption1; + item.TrackingOption2 = line.TrackingOption2; + item.UnitAmount = line.UnitAmount; + + dictionaryList[invoiceNumber].LineItems.Add(item); + } + + return dictionaryList; + } + } +} diff --git a/BealeEngineering/BealeEngineering.Core/Logic/Import/wipXeroInvoiceFlatFile.cs b/BealeEngineering/BealeEngineering.Core/Logic/Import/wipXeroInvoiceFlatFile.cs new file mode 100644 index 0000000..ded6d9d --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Logic/Import/wipXeroInvoiceFlatFile.cs @@ -0,0 +1,54 @@ +using CsvHelper; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.VisualBasic.FileIO; + +namespace BealeEngineering.Core.Logic.Import +{ + public class wipXeroInvoiceFlatFile + { + public wipXeroInvoiceFlatFile(string sqlConnectionString) + { + SqlConnectionString = sqlConnectionString; + } + private string SqlConnectionString { get; set; } + public void ByFilePath(string filePath) + { + // get model list + + + + + + + + + + + + + //// get db invoices + //var saleInvInst = new Data.Database.Sale.InvoiceGet(SqlConnectionString); + //saleInvInst.InvoiceNumber = invDictionary.Keys.ToList(); + //var dataInvList = saleInvInst.GetByFilters(); + + // compare + + + // update modified records + + // insert new records <--------- only insert approved invoices + + // delete records (is this possible??) <------- i think so, include deleted and voided in flatfile?? + + + + + } + } +} diff --git a/BealeEngineering/BealeEngineering.Core/Logic/Utilities/CSVGetRFC4180Compliant.cs b/BealeEngineering/BealeEngineering.Core/Logic/Utilities/CSVGetRFC4180Compliant.cs new file mode 100644 index 0000000..743d395 --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Logic/Utilities/CSVGetRFC4180Compliant.cs @@ -0,0 +1,178 @@ +using CsvHelper; +using Microsoft.VisualBasic.FileIO; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BealeEngineering.Core.Logic.Utilities +{ + public class CSVGetRFC4180Compliant + { + public string OutputFilepath { get; set; } + public bool OutputFilepathIsSet + { + get + { + if (string.IsNullOrEmpty(OutputFilepath)) { return false; } + else { return true; } + } + } + public string OutputString { get; private set; } + public bool OutputStringIsSet + { + get + { + if (string.IsNullOrWhiteSpace(OutputString)) { return false; } + else { return true; } + } + } + private List> MsVbTextParserResult { get; set; } + public void ByFilePath(string filePath) + { + if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath)) + { throw new Exception("File or filepath error."); } + + string inputString = File.ReadAllText(filePath); + ByString(ref inputString); + inputString = ""; + inputString = null; + GC.Collect(); + } + public void ByString(ref string inputCsvString) + { + if (string.IsNullOrWhiteSpace(inputCsvString)) + { throw new Exception("Invalid CSV string"); } + + MsVbTextFieldParser(ref inputCsvString); + CreateCompliantCsv(); + //CreateCompliantCsvMyVer(); + } + private void MsVbTextFieldParser(ref string inputCsvString) + { + MsVbTextParserResult = new List>(); + + using (TextFieldParser parser = new TextFieldParser(new StringReader(inputCsvString))) + { + parser.SetDelimiters(new string[] { "," }); + parser.HasFieldsEnclosedInQuotes = true; + + // Skip over header line. + //parser.ReadLine(); + + int lineNumber = 0; + int columnCountFound = 0; + while (!parser.EndOfData) + { + lineNumber = lineNumber + 1; + + var values = new List(); + + var readFields = parser.ReadFields(); + + + if (readFields != null) + { + values.AddRange(readFields); + MsVbTextParserResult.Add(values); + if (lineNumber == 1) + { + columnCountFound = values.Count(); + } + else + { + if (values.Count != columnCountFound) + { + throw new Exception("Error parsing file, incorrect columns count."); + } + } + } + + } + } + } + private void CreateCompliantCsv() + { + var outputStringBuilder = new StringBuilder(); + + var csvConfig = new CsvHelper.Configuration.CsvConfiguration(CultureInfo.CurrentUICulture); + + // using (var writer = new StreamWriter(IntermediateCsv)) + using (var writer = new StringWriter(outputStringBuilder)) + using (var csv = new CsvWriter(writer, csvConfig)) + { + foreach (var record in MsVbTextParserResult) + { + foreach (var field in record) + { + csv.WriteField(field); + } + csv.NextRecord(); + } + writer.Flush(); + } + + // output + OutputString = outputStringBuilder.ToString(); + WriteToFile(); + + // clean up + outputStringBuilder.Clear(); + outputStringBuilder = null; + MsVbTextParserResult.Clear(); + MsVbTextParserResult = null; + GC.Collect(); + } + // redundant once class is tested + public void CreateCompliantCsvMyVer() + { + // get column count + int columnCount = MsVbTextParserResult[0].Count(); + + // create proper deliminatd string from result + var outputStringBuilder = new StringBuilder(""); + foreach (var line in MsVbTextParserResult) + { + int i = 0; + foreach (var field in line) + { + i = i + 1; + // check for double quotes within field, if found preceed/escape with double quote + string value = field.Replace("\"", "\"\""); + + if (i == 1) + { + outputStringBuilder.Append("\"" + value); + } + else if (i < columnCount) + { + outputStringBuilder.Append("\",\"" + value); + } + else + { + outputStringBuilder.AppendLine("\",\"" + value + "\""); + } + } + } + // output + OutputString = outputStringBuilder.ToString(); + WriteToFile(); + + // clean up + outputStringBuilder.Clear(); + outputStringBuilder = null; + MsVbTextParserResult.Clear(); + MsVbTextParserResult = null; + GC.Collect(); + } + + public void WriteToFile() + { + if (OutputFilepathIsSet && OutputStringIsSet) + { System.IO.File.WriteAllText(OutputFilepath, OutputString); } + } + } +} diff --git a/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFile.cs b/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFile.cs new file mode 100644 index 0000000..5509ed6 --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFile.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; + +namespace BealeEngineering.Core.Model.Import +{ + public class XeroInvoiceFlatFile + { + public string ContactName { get; set; } + public string EmailAddress { get; set; } + public string POAddressLine1 { get; set; } + public string POAddressLine2 { get; set; } + public string POAddressLine3 { get; set; } + public string POAddressLine4 { get; set; } + public string POCity { get; set; } + public string PORegion { get; set; } + public string POPostalCode { get; set; } + public string POCountry { get; set; } + public string SAAddressLine1 { get; set; } + public string SAAddressLine2 { get; set; } + public string SAAddressLine3 { get; set; } + public string SAAddressLine4 { get; set; } + public string SACity { get; set; } + public string SARegion { get; set; } + public string SAPostalCode { get; set; } + public string SACountry { get; set; } + public string InvoiceNumber { get; set; } + public string Reference { get; set; } + public DateTime InvoiceDate { get; set; } + public DateTime? DueDate { get; set; } + public DateTime? PlannedDate { get; set; } + public decimal Total { get; set; } + public decimal TaxTotal { get; set; } + public decimal InvoiceAmountPaid { get; set; } + public decimal InvoiceAmountDue { get; set; } + public List LineItems { get; set; } = new List(); + public class LineItem + { + public string InventoryItemCode { get; set; } + public string Description { get; set; } + public decimal Quantity { get; set; } + public decimal UnitAmount { get; set; } + public int? Discount { get; set; } + public decimal LineAmount { get; set; } + public string AccountCode { get; set; } + public string TaxType { get; set; } + public decimal TaxAmount { get; set; } + public string TrackingName1 { get; set; } + public string TrackingOption1 { get; set; } + public string TrackingName2 { get; set; } + public string TrackingOption2 { get; set; } + } + public string Currency { get; set; } + public string Type { get; set; } + public string Sent { get; set; } + public string Status { get; set; } + } +} \ No newline at end of file diff --git a/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFileDTO.cs b/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFileDTO.cs new file mode 100644 index 0000000..fed50e8 --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFileDTO.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; + +namespace BealeEngineering.Core.Model.Import +{ + public class XeroInvoiceFlatFileDTO + { + public string ContactName { get; set; } + public string EmailAddress { get; set; } + public string POAddressLine1 { get; set; } + public string POAddressLine2 { get; set; } + public string POAddressLine3 { get; set; } + public string POAddressLine4 { get; set; } + public string POCity { get; set; } + public string PORegion { get; set; } + public string POPostalCode { get; set; } + public string POCountry { get; set; } + public string SAAddressLine1 { get; set; } + public string SAAddressLine2 { get; set; } + public string SAAddressLine3 { get; set; } + public string SAAddressLine4 { get; set; } + public string SACity { get; set; } + public string SARegion { get; set; } + public string SAPostalCode { get; set; } + public string SACountry { get; set; } + public string InvoiceNumber { get; set; } + public string Reference { get; set; } + public DateTime InvoiceDate { get; set; } + public DateTime? DueDate { get; set; } + public DateTime? PlannedDate { get; set; } + public decimal Total { get; set; } + public decimal TaxTotal { get; set; } + public decimal InvoiceAmountPaid { get; set; } + public decimal InvoiceAmountDue { get; set; } + public string InventoryItemCode { get; set; } + public string Description { get; set; } + public decimal Quantity { get; set; } + public decimal UnitAmount { get; set; } + public int? Discount { get; set; } + public decimal LineAmount { get; set; } + public string AccountCode { get; set; } + public string TaxType { get; set; } + public decimal TaxAmount { get; set; } + public string TrackingName1 { get; set; } + public string TrackingOption1 { get; set; } + public string TrackingName2 { get; set; } + public string TrackingOption2 { get; set; } + public string Currency { get; set; } + public string Type { get; set; } + public string Sent { get; set; } + public string Status { get; set; } + } +} \ No newline at end of file diff --git a/BealeEngineering/BealeEngineering.Core/Test/AUtoexec.cs b/BealeEngineering/BealeEngineering.Core/Test/AUtoexec.cs index 26004e1..de61163 100644 --- a/BealeEngineering/BealeEngineering.Core/Test/AUtoexec.cs +++ b/BealeEngineering/BealeEngineering.Core/Test/AUtoexec.cs @@ -21,11 +21,11 @@ namespace BealeEngineering.Core.Test //var inst = new Core.Test.Sales.Invoice(SqlConnectionString); //inst.GetInvoice(); - var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString); - inst2.AllocateInvoicesToPurchaseOrders(); - - + //var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString); + //inst2.AllocateInvoicesToPurchaseOrders(); + var inst3 = new Test.Import.ImportFlatfile(); + inst3.Go(); } } } diff --git a/BealeEngineering/BealeEngineering.Core/Test/Import/ImportFlatfile.cs b/BealeEngineering/BealeEngineering.Core/Test/Import/ImportFlatfile.cs new file mode 100644 index 0000000..b682b49 --- /dev/null +++ b/BealeEngineering/BealeEngineering.Core/Test/Import/ImportFlatfile.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BealeEngineering.Core.Test.Import +{ + public class ImportFlatfile + { + public void Go() + { + string fileInputPath = + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + + @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv"; + + var inst = new Data.Xero.FlatFile.ImportInvoice(); + var lkdsjflsd = inst.ByFilePath(fileInputPath); + } + } +} diff --git a/BealeEngineering/BealeEngineering.Core/packages.config b/BealeEngineering/BealeEngineering.Core/packages.config index e38523d..c62bd0d 100644 --- a/BealeEngineering/BealeEngineering.Core/packages.config +++ b/BealeEngineering/BealeEngineering.Core/packages.config @@ -1,4 +1,10 @@  + + + + + + \ No newline at end of file