Added parsing of Xero invoice export file

BealeEngineering/BealeEngineering.Core/BealeEngineering.Core.csproj

@@ -31,11 +31,24 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="CsvHelper, Version=13.0.0.0, Culture=neutral, PublicKeyToken=8c4959082be5c823, processorArchitecture=MSIL">
+      <HintPath>..\packages\CsvHelper.13.0.0\lib\net47\CsvHelper.dll</HintPath>
+    </Reference>
     <Reference Include="Dapper, Version=2.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\Dapper.2.0.30\lib\net461\Dapper.dll</HintPath>
     </Reference>
+    <Reference Include="Microsoft.Bcl.AsyncInterfaces, Version=1.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Bcl.AsyncInterfaces.1.1.0\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.VisualBasic" />
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Threading.Tasks.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
+      <HintPath>..\packages\System.Threading.Tasks.Extensions.4.5.2\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll</HintPath>
+    </Reference>
     <Reference Include="System.Transactions" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -50,26 +63,32 @@
     <Compile Include="Data\Database\Client\PurchaseOrderAllocationGet.cs" />
     <Compile Include="Data\Database\Sale\InvoiceGet.cs" />
     <Compile Include="Data\Database\Sale\InvoiceHeaderGet.cs" />
+    <Compile Include="Data\Xero\SaleInvoiceGet.cs" />
     <Compile Include="Logic\Client\PurchaseOrderAutoAllocate.cs" />
     <Compile Include="Data\Database\Connection.cs" />
     <Compile Include="Data\Database\Contact\ContactHeaderGet.cs" />
     <Compile Include="Data\Database\Client\PurchaseOrderHeaderGet.cs" />
+    <Compile Include="Logic\Import\wipXeroInvoiceFlatFile.cs" />
+    <Compile Include="Logic\Utilities\CSVGetRFC4180Compliant.cs" />
     <Compile Include="Logic\Utilities\Reflection.cs" />
     <Compile Include="Logic\Utilities\StringCheck.cs" />
     <Compile Include="Model\Client\PurchaseOrder.cs" />
     <Compile Include="Model\Client\PurchaseOrderHeader.cs" />
     <Compile Include="Model\Client\PurchaseOrderLine.cs" />
     <Compile Include="Model\Contact\ContactHeader.cs" />
+    <Compile Include="Model\Import\XeroInvoiceFlatFile.cs" />
+    <Compile Include="Model\Import\XeroInvoiceFlatFileDTO.cs" />
     <Compile Include="Model\Sale\Invoice.cs" />
     <Compile Include="Model\Sale\InvoiceHeader.cs" />
     <Compile Include="Model\Sale\InvoiceLine.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Test\Autoexec.cs" />
     <Compile Include="Test\Client\PurchaseOrder.cs" />
+    <Compile Include="Test\Import\ImportFlatfile.cs" />
     <Compile Include="Test\Sales\Invoice.cs" />
   </ItemGroup>
   <ItemGroup>
-    <Folder Include="Data\XeroAPI\" />
+    <Folder Include="Data\Xero\FlatFile\" />
     <Folder Include="Service\" />
     <Folder Include="UI\" />
   </ItemGroup>

BealeEngineering/BealeEngineering.Core/Data/Xero/SaleInvoiceGet.cs

@@ -0,0 +1,158 @@
+using CsvHelper;
+using Microsoft.VisualBasic.FileIO;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BealeEngineering.Core.Data.Xero.FlatFile
+{
+    public class ImportInvoice
+    {
+        public ImportInvoice()
+        {
+            FileInputPath =
+                Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+                + @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv";
+            FileOutputPath =
+                Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+                + @"\Downloads\MyNewTextFile.txt";
+        }
+        private StringBuilder IntermediateCsv { get; set; }
+        public string FileInputPath { get; set; }
+        public string FileOutputPath { get; set; }
+        private List<List<string>> MsVbTextParserResult { get; set; }
+        private StringBuilder CsvContent { get; set; }
+        /// <summary>
+        /// Imports Xero invoice flat-file into model class.
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <returns>Dictionary, Invoice numbers against data.</returns>
+        public Dictionary<string, Model.Import.XeroInvoiceFlatFile> ByFilePath(string filePath)
+        {
+            /* So here's the rub. Any field in a CSV doc that has a double quote wihtin, must be enclosed by double quotes and 
+             * the double quote within must be 'escaped' by a double quote.
+             * However, in that situation, Xero flat file doesn't enclose the field or escape the double quote.
+             * CsvHelper cannot handle this situation.
+             * However, the MS VB TextFieldParser can.
+             * Therefore, parse with TextFieldParser, from this create file in the correct CSV format and the 
+             * feed that into CsvHelper to map to a class.
+             * 
+             * Long winded but easier than reinventing the wheel.
+             * 
+             * Keep an eye on 
+             * https://github.com/JoshClose/CsvHelper/issues/989
+             * CsvHelper may nativily support when this feature is complete.
+             */
+
+            // first off, get a RFC4180 compliant csv string
+            var csvRFC = new Logic.Utilities.CSVGetRFC4180Compliant();
+            csvRFC.ByFilePath(filePath);
+            if (!csvRFC.OutputStringIsSet)
+            { throw new Exception("CSV Error."); }
+
+            // parse intermediate csv into data class
+            var dto = new List<Model.Import.XeroInvoiceFlatFileDTO>();
+            using (TextReader reader = new StringReader(csvRFC.OutputString))
+            using (var csv = new CsvReader(reader, CultureInfo.CurrentUICulture))
+            {
+                csv.Configuration.DetectColumnCountChanges = true;
+
+                dto = csv.GetRecords<Model.Import.XeroInvoiceFlatFileDTO>().ToList();
+            }
+
+            return ConvertFlatDTO(ref dto);
+        }
+
+        private Dictionary<string, Model.Import.XeroInvoiceFlatFile> ConvertFlatDTO(ref List<Model.Import.XeroInvoiceFlatFileDTO> flatData)
+        {
+            // ensure flat data is in invoice number order
+            var invDictionary = new Dictionary<string, int>();
+            string lastNumber = null;
+            foreach (var line in flatData)
+            {
+                if (line.InvoiceNumber != lastNumber)
+                {
+                    lastNumber = line.InvoiceNumber;
+                    if (invDictionary.ContainsKey(lastNumber))
+                    {
+                        throw new Exception("Invoices are not grouped in CSV flatfile.");
+                    }
+                    else
+                    {
+                        invDictionary.Add(lastNumber, 0);
+                    }
+                }
+            }
+
+            // convert to one to many class data
+            var dictionaryList = new Dictionary<string, Model.Import.XeroInvoiceFlatFile>();
+            foreach (var line in flatData)
+            {
+                string invoiceNumber = line.InvoiceNumber;
+
+                if (!dictionaryList.ContainsKey(invoiceNumber))
+                {
+                    var invoice = new Model.Import.XeroInvoiceFlatFile();
+
+                    invoice.ContactName = line.ContactName;
+                    invoice.Currency = line.Currency;
+                    invoice.DueDate = line.DueDate;
+                    invoice.EmailAddress = line.EmailAddress;
+                    invoice.InvoiceAmountDue = line.InvoiceAmountDue;
+                    invoice.InvoiceAmountPaid = line.InvoiceAmountPaid;
+                    invoice.InvoiceDate = line.InvoiceDate;
+                    invoice.InvoiceNumber = line.InvoiceNumber;
+                    invoice.PlannedDate = line.PlannedDate;
+                    invoice.POAddressLine1 = line.POAddressLine1;
+                    invoice.POAddressLine2 = line.POAddressLine2;
+                    invoice.POAddressLine3 = line.POAddressLine3;
+                    invoice.POAddressLine4 = line.POAddressLine4;
+                    invoice.POCity = line.POCity;
+                    invoice.POCountry = line.POCountry;
+                    invoice.POPostalCode = line.POPostalCode;
+                    invoice.PORegion = line.PORegion;
+                    invoice.Reference = line.Reference;
+                    invoice.SAAddressLine1 = line.SAAddressLine1;
+                    invoice.SAAddressLine2 = line.SAAddressLine2;
+                    invoice.SAAddressLine3 = line.SAAddressLine3;
+                    invoice.SAAddressLine4 = line.SAAddressLine4;
+                    invoice.SACity = line.SACity;
+                    invoice.SACountry = line.SACountry;
+                    invoice.SAPostalCode = line.SAPostalCode;
+                    invoice.SARegion = line.SARegion;
+                    invoice.Sent = line.Sent;
+                    invoice.Status = line.Status;
+                    invoice.TaxTotal = line.TaxTotal;
+                    invoice.Total = line.Total;
+                    invoice.Type = line.Type;
+
+                    dictionaryList.Add(invoice.InvoiceNumber, invoice);
+                }
+
+                var item = new Model.Import.XeroInvoiceFlatFile.LineItem();
+
+                item.AccountCode = line.AccountCode;
+                item.Description = line.Description;
+                item.Discount = line.Discount;
+                item.InventoryItemCode = line.InventoryItemCode;
+                item.LineAmount = line.LineAmount;
+                item.Quantity = line.Quantity;
+                item.TaxAmount = line.TaxAmount;
+                item.TaxType = line.TaxType;
+                item.TrackingName1 = line.TrackingName1;
+                item.TrackingName2 = line.TrackingName2;
+                item.TrackingOption1 = line.TrackingOption1;
+                item.TrackingOption2 = line.TrackingOption2;
+                item.UnitAmount = line.UnitAmount;
+
+                dictionaryList[invoiceNumber].LineItems.Add(item);
+            }
+
+            return dictionaryList;
+        }
+    }
+}

BealeEngineering/BealeEngineering.Core/Logic/Import/wipXeroInvoiceFlatFile.cs

@@ -0,0 +1,54 @@
+using CsvHelper;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.VisualBasic.FileIO;
+
+namespace BealeEngineering.Core.Logic.Import
+{
+    public class wipXeroInvoiceFlatFile
+    {
+        public wipXeroInvoiceFlatFile(string sqlConnectionString)
+        {
+            SqlConnectionString = sqlConnectionString;
+        }
+        private string SqlConnectionString { get; set; }
+        public void ByFilePath(string filePath)
+        {
+            // get model list
+            
+
+
+
+
+
+
+
+
+
+
+
+            //// get db invoices
+            //var saleInvInst = new Data.Database.Sale.InvoiceGet(SqlConnectionString);
+            //saleInvInst.InvoiceNumber = invDictionary.Keys.ToList();
+            //var dataInvList = saleInvInst.GetByFilters();
+
+            // compare
+
+
+            // update modified records
+
+            // insert new records <--------- only insert approved invoices
+
+            // delete records (is this possible??) <------- i think so, include deleted and voided in flatfile??
+
+
+
+
+        }
+    }
+}

BealeEngineering/BealeEngineering.Core/Logic/Utilities/CSVGetRFC4180Compliant.cs

@@ -0,0 +1,178 @@
+using CsvHelper;
+using Microsoft.VisualBasic.FileIO;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BealeEngineering.Core.Logic.Utilities
+{
+    public class CSVGetRFC4180Compliant
+    {
+        public string OutputFilepath { get; set; }
+        public bool OutputFilepathIsSet
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(OutputFilepath)) { return false; }
+                else { return true; }
+            }
+        }
+        public string OutputString { get; private set; }
+        public bool OutputStringIsSet
+        {
+            get
+            {
+                if (string.IsNullOrWhiteSpace(OutputString)) { return false; }
+                else { return true; }
+            }
+        }
+        private List<List<string>> MsVbTextParserResult { get; set; }
+        public void ByFilePath(string filePath)
+        {
+            if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath))
+            { throw new Exception("File or filepath error."); }
+            
+            string inputString = File.ReadAllText(filePath);
+            ByString(ref inputString);
+            inputString = "";
+            inputString = null;
+            GC.Collect();
+        }
+        public void ByString(ref string inputCsvString)
+        {
+            if (string.IsNullOrWhiteSpace(inputCsvString))
+            { throw new Exception("Invalid CSV string"); }
+
+            MsVbTextFieldParser(ref inputCsvString);
+            CreateCompliantCsv();
+            //CreateCompliantCsvMyVer();
+        }
+        private void MsVbTextFieldParser(ref string inputCsvString)
+        {
+            MsVbTextParserResult = new List<List<string>>();
+
+            using (TextFieldParser parser = new TextFieldParser(new StringReader(inputCsvString)))
+            {
+                parser.SetDelimiters(new string[] { "," });
+                parser.HasFieldsEnclosedInQuotes = true;
+
+                // Skip over header line.
+                //parser.ReadLine();
+
+                int lineNumber = 0;
+                int columnCountFound = 0;
+                while (!parser.EndOfData)
+                {
+                    lineNumber = lineNumber + 1;
+
+                    var values = new List<string>();
+                    
+                    var readFields = parser.ReadFields();
+
+
+                    if (readFields != null)
+                    {
+                        values.AddRange(readFields);
+                        MsVbTextParserResult.Add(values);
+                        if (lineNumber == 1)
+                        {
+                            columnCountFound = values.Count();
+                        }
+                        else
+                        {
+                            if (values.Count != columnCountFound)
+                            {
+                                throw new Exception("Error parsing file, incorrect columns count.");
+                            }
+                        }
+                    }
+
+                }
+            }
+        }
+        private void CreateCompliantCsv()
+        {
+            var outputStringBuilder = new StringBuilder();
+
+            var csvConfig = new CsvHelper.Configuration.CsvConfiguration(CultureInfo.CurrentUICulture);
+
+            // using (var writer = new StreamWriter(IntermediateCsv))
+            using (var writer = new StringWriter(outputStringBuilder))
+            using (var csv = new CsvWriter(writer, csvConfig))
+            {
+                foreach (var record in MsVbTextParserResult)
+                {
+                    foreach (var field in record)
+                    {
+                        csv.WriteField(field);
+                    }
+                    csv.NextRecord();
+                }
+                writer.Flush();
+            }
+
+            // output
+            OutputString = outputStringBuilder.ToString();
+            WriteToFile();
+
+            // clean up
+            outputStringBuilder.Clear();
+            outputStringBuilder = null;
+            MsVbTextParserResult.Clear();
+            MsVbTextParserResult = null;
+            GC.Collect();
+        }
+        // redundant once class is tested
+        public void CreateCompliantCsvMyVer()
+        {
+            // get column count
+            int columnCount = MsVbTextParserResult[0].Count();
+
+            // create proper deliminatd string from result
+            var outputStringBuilder = new StringBuilder("");
+            foreach (var line in MsVbTextParserResult)
+            {
+                int i = 0;
+                foreach (var field in line)
+                {
+                    i = i + 1;
+                    // check for double quotes within field, if found preceed/escape with double quote
+                    string value = field.Replace("\"", "\"\"");
+
+                    if (i == 1)
+                    {
+                        outputStringBuilder.Append("\"" + value);
+                    }
+                    else if (i < columnCount)
+                    {
+                        outputStringBuilder.Append("\",\"" + value);
+                    }
+                    else
+                    {
+                        outputStringBuilder.AppendLine("\",\"" + value + "\"");
+                    }
+                }
+            }
+            // output
+            OutputString = outputStringBuilder.ToString();
+            WriteToFile();
+
+            // clean up
+            outputStringBuilder.Clear();
+            outputStringBuilder = null;
+            MsVbTextParserResult.Clear();
+            MsVbTextParserResult = null;
+            GC.Collect();
+        }
+
+        public void WriteToFile()
+        {
+            if (OutputFilepathIsSet && OutputStringIsSet) 
+            { System.IO.File.WriteAllText(OutputFilepath, OutputString); }
+        }
+    }
+}

BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFile.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+
+namespace BealeEngineering.Core.Model.Import
+{
+    public class XeroInvoiceFlatFile
+    {
+        public string ContactName { get; set; }
+        public string EmailAddress { get; set; }
+        public string POAddressLine1 { get; set; }
+        public string POAddressLine2 { get; set; }
+        public string POAddressLine3 { get; set; }
+        public string POAddressLine4 { get; set; }
+        public string POCity { get; set; }
+        public string PORegion { get; set; }
+        public string POPostalCode { get; set; }
+        public string POCountry { get; set; }
+        public string SAAddressLine1 { get; set; }
+        public string SAAddressLine2 { get; set; }
+        public string SAAddressLine3 { get; set; }
+        public string SAAddressLine4 { get; set; }
+        public string SACity { get; set; }
+        public string SARegion { get; set; }
+        public string SAPostalCode { get; set; }
+        public string SACountry { get; set; }
+        public string InvoiceNumber { get; set; }
+        public string Reference { get; set; }
+        public DateTime InvoiceDate { get; set; }
+        public DateTime? DueDate { get; set; }
+        public DateTime? PlannedDate { get; set; }
+        public decimal Total { get; set; }
+        public decimal TaxTotal { get; set; }
+        public decimal InvoiceAmountPaid { get; set; }
+        public decimal InvoiceAmountDue { get; set; }
+        public List<LineItem> LineItems { get; set; } = new List<LineItem>();
+        public class LineItem
+        {
+            public string InventoryItemCode { get; set; }
+            public string Description { get; set; }
+            public decimal Quantity { get; set; }
+            public decimal UnitAmount { get; set; }
+            public int? Discount { get; set; }
+            public decimal LineAmount { get; set; }
+            public string AccountCode { get; set; }
+            public string TaxType { get; set; }
+            public decimal TaxAmount { get; set; }
+            public string TrackingName1 { get; set; }
+            public string TrackingOption1 { get; set; }
+            public string TrackingName2 { get; set; }
+            public string TrackingOption2 { get; set; }
+        }
+        public string Currency { get; set; }
+        public string Type { get; set; }
+        public string Sent { get; set; }
+        public string Status { get; set; }
+    }
+}

BealeEngineering/BealeEngineering.Core/Model/Import/XeroInvoiceFlatFileDTO.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Collections.Generic;
+
+namespace BealeEngineering.Core.Model.Import
+{
+    public class XeroInvoiceFlatFileDTO
+    {
+        public string ContactName { get; set; }
+        public string EmailAddress { get; set; }
+        public string POAddressLine1 { get; set; }
+        public string POAddressLine2 { get; set; }
+        public string POAddressLine3 { get; set; }
+        public string POAddressLine4 { get; set; }
+        public string POCity { get; set; }
+        public string PORegion { get; set; }
+        public string POPostalCode { get; set; }
+        public string POCountry { get; set; }
+        public string SAAddressLine1 { get; set; }
+        public string SAAddressLine2 { get; set; }
+        public string SAAddressLine3 { get; set; }
+        public string SAAddressLine4 { get; set; }
+        public string SACity { get; set; }
+        public string SARegion { get; set; }
+        public string SAPostalCode { get; set; }
+        public string SACountry { get; set; }
+        public string InvoiceNumber { get; set; }
+        public string Reference { get; set; }
+        public DateTime InvoiceDate { get; set; }
+        public DateTime? DueDate { get; set; }
+        public DateTime? PlannedDate { get; set; }
+        public decimal Total { get; set; }
+        public decimal TaxTotal { get; set; }
+        public decimal InvoiceAmountPaid { get; set; }
+        public decimal InvoiceAmountDue { get; set; }
+        public string InventoryItemCode { get; set; }
+        public string Description { get; set; }
+        public decimal Quantity { get; set; }
+        public decimal UnitAmount { get; set; }
+        public int? Discount { get; set; }
+        public decimal LineAmount { get; set; }
+        public string AccountCode { get; set; }
+        public string TaxType { get; set; }
+        public decimal TaxAmount { get; set; }
+        public string TrackingName1 { get; set; }
+        public string TrackingOption1 { get; set; }
+        public string TrackingName2 { get; set; }
+        public string TrackingOption2 { get; set; }
+        public string Currency { get; set; }
+        public string Type { get; set; }
+        public string Sent { get; set; }
+        public string Status { get; set; }
+    }
+}

BealeEngineering/BealeEngineering.Core/Test/AUtoexec.cs

@@ -21,11 +21,11 @@ namespace BealeEngineering.Core.Test
             //var inst = new Core.Test.Sales.Invoice(SqlConnectionString);
             //inst.GetInvoice();
 
-            var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString);
+            //var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString);
-            inst2.AllocateInvoicesToPurchaseOrders();
+            //inst2.AllocateInvoicesToPurchaseOrders();
-
-
 
+            var inst3 = new Test.Import.ImportFlatfile();
+            inst3.Go();
         }
     }
 }

BealeEngineering/BealeEngineering.Core/Test/Import/ImportFlatfile.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BealeEngineering.Core.Test.Import
+{
+    public class ImportFlatfile
+    {
+        public void Go()
+        {
+            string fileInputPath =
+                Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+                + @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv";
+
+            var inst = new Data.Xero.FlatFile.ImportInvoice();
+            var lkdsjflsd = inst.ByFilePath(fileInputPath);
+        }
+    }
+}

BealeEngineering/BealeEngineering.Core/packages.config

@@ -1,4 +1,10 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="CsvHelper" version="13.0.0" targetFramework="net472" />
   <package id="Dapper" version="2.0.30" targetFramework="net472" />
+  <package id="Microsoft.Bcl.AsyncInterfaces" version="1.1.0" targetFramework="net472" />
+  <package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
+  <package id="Microsoft.VisualBasic" version="10.3.0" targetFramework="net472" />
+  <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
+  <package id="System.Threading.Tasks.Extensions" version="4.5.2" targetFramework="net472" />
 </packages>