Added parsing of Xero invoice export file

This commit is contained in:
Bobbie Hodgetts
2020-01-27 16:50:42 +00:00
committed by GitHub
parent 773ce4cee5
commit aea82da897
9 changed files with 551 additions and 5 deletions

View File

@@ -31,11 +31,24 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="CsvHelper, Version=13.0.0.0, Culture=neutral, PublicKeyToken=8c4959082be5c823, processorArchitecture=MSIL">
<HintPath>..\packages\CsvHelper.13.0.0\lib\net47\CsvHelper.dll</HintPath>
</Reference>
<Reference Include="Dapper, Version=2.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Dapper.2.0.30\lib\net461\Dapper.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Bcl.AsyncInterfaces, Version=1.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Bcl.AsyncInterfaces.1.1.0\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll</HintPath>
</Reference>
<Reference Include="Microsoft.VisualBasic" />
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Threading.Tasks.Extensions, Version=4.2.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Threading.Tasks.Extensions.4.5.2\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Transactions" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
@@ -50,26 +63,32 @@
<Compile Include="Data\Database\Client\PurchaseOrderAllocationGet.cs" />
<Compile Include="Data\Database\Sale\InvoiceGet.cs" />
<Compile Include="Data\Database\Sale\InvoiceHeaderGet.cs" />
<Compile Include="Data\Xero\SaleInvoiceGet.cs" />
<Compile Include="Logic\Client\PurchaseOrderAutoAllocate.cs" />
<Compile Include="Data\Database\Connection.cs" />
<Compile Include="Data\Database\Contact\ContactHeaderGet.cs" />
<Compile Include="Data\Database\Client\PurchaseOrderHeaderGet.cs" />
<Compile Include="Logic\Import\wipXeroInvoiceFlatFile.cs" />
<Compile Include="Logic\Utilities\CSVGetRFC4180Compliant.cs" />
<Compile Include="Logic\Utilities\Reflection.cs" />
<Compile Include="Logic\Utilities\StringCheck.cs" />
<Compile Include="Model\Client\PurchaseOrder.cs" />
<Compile Include="Model\Client\PurchaseOrderHeader.cs" />
<Compile Include="Model\Client\PurchaseOrderLine.cs" />
<Compile Include="Model\Contact\ContactHeader.cs" />
<Compile Include="Model\Import\XeroInvoiceFlatFile.cs" />
<Compile Include="Model\Import\XeroInvoiceFlatFileDTO.cs" />
<Compile Include="Model\Sale\Invoice.cs" />
<Compile Include="Model\Sale\InvoiceHeader.cs" />
<Compile Include="Model\Sale\InvoiceLine.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Test\Autoexec.cs" />
<Compile Include="Test\Client\PurchaseOrder.cs" />
<Compile Include="Test\Import\ImportFlatfile.cs" />
<Compile Include="Test\Sales\Invoice.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="Data\XeroAPI\" />
<Folder Include="Data\Xero\FlatFile\" />
<Folder Include="Service\" />
<Folder Include="UI\" />
</ItemGroup>

View File

@@ -0,0 +1,158 @@
using CsvHelper;
using Microsoft.VisualBasic.FileIO;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BealeEngineering.Core.Data.Xero.FlatFile
{
public class ImportInvoice
{
public ImportInvoice()
{
FileInputPath =
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+ @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv";
FileOutputPath =
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+ @"\Downloads\MyNewTextFile.txt";
}
private StringBuilder IntermediateCsv { get; set; }
public string FileInputPath { get; set; }
public string FileOutputPath { get; set; }
private List<List<string>> MsVbTextParserResult { get; set; }
private StringBuilder CsvContent { get; set; }
/// <summary>
/// Imports Xero invoice flat-file into model class.
/// </summary>
/// <param name="filePath"></param>
/// <returns>Dictionary, Invoice numbers against data.</returns>
public Dictionary<string, Model.Import.XeroInvoiceFlatFile> ByFilePath(string filePath)
{
/* So here's the rub. Any field in a CSV doc that has a double quote wihtin, must be enclosed by double quotes and
* the double quote within must be 'escaped' by a double quote.
* However, in that situation, Xero flat file doesn't enclose the field or escape the double quote.
* CsvHelper cannot handle this situation.
* However, the MS VB TextFieldParser can.
* Therefore, parse with TextFieldParser, from this create file in the correct CSV format and the
* feed that into CsvHelper to map to a class.
*
* Long winded but easier than reinventing the wheel.
*
* Keep an eye on
* https://github.com/JoshClose/CsvHelper/issues/989
* CsvHelper may nativily support when this feature is complete.
*/
// first off, get a RFC4180 compliant csv string
var csvRFC = new Logic.Utilities.CSVGetRFC4180Compliant();
csvRFC.ByFilePath(filePath);
if (!csvRFC.OutputStringIsSet)
{ throw new Exception("CSV Error."); }
// parse intermediate csv into data class
var dto = new List<Model.Import.XeroInvoiceFlatFileDTO>();
using (TextReader reader = new StringReader(csvRFC.OutputString))
using (var csv = new CsvReader(reader, CultureInfo.CurrentUICulture))
{
csv.Configuration.DetectColumnCountChanges = true;
dto = csv.GetRecords<Model.Import.XeroInvoiceFlatFileDTO>().ToList();
}
return ConvertFlatDTO(ref dto);
}
private Dictionary<string, Model.Import.XeroInvoiceFlatFile> ConvertFlatDTO(ref List<Model.Import.XeroInvoiceFlatFileDTO> flatData)
{
// ensure flat data is in invoice number order
var invDictionary = new Dictionary<string, int>();
string lastNumber = null;
foreach (var line in flatData)
{
if (line.InvoiceNumber != lastNumber)
{
lastNumber = line.InvoiceNumber;
if (invDictionary.ContainsKey(lastNumber))
{
throw new Exception("Invoices are not grouped in CSV flatfile.");
}
else
{
invDictionary.Add(lastNumber, 0);
}
}
}
// convert to one to many class data
var dictionaryList = new Dictionary<string, Model.Import.XeroInvoiceFlatFile>();
foreach (var line in flatData)
{
string invoiceNumber = line.InvoiceNumber;
if (!dictionaryList.ContainsKey(invoiceNumber))
{
var invoice = new Model.Import.XeroInvoiceFlatFile();
invoice.ContactName = line.ContactName;
invoice.Currency = line.Currency;
invoice.DueDate = line.DueDate;
invoice.EmailAddress = line.EmailAddress;
invoice.InvoiceAmountDue = line.InvoiceAmountDue;
invoice.InvoiceAmountPaid = line.InvoiceAmountPaid;
invoice.InvoiceDate = line.InvoiceDate;
invoice.InvoiceNumber = line.InvoiceNumber;
invoice.PlannedDate = line.PlannedDate;
invoice.POAddressLine1 = line.POAddressLine1;
invoice.POAddressLine2 = line.POAddressLine2;
invoice.POAddressLine3 = line.POAddressLine3;
invoice.POAddressLine4 = line.POAddressLine4;
invoice.POCity = line.POCity;
invoice.POCountry = line.POCountry;
invoice.POPostalCode = line.POPostalCode;
invoice.PORegion = line.PORegion;
invoice.Reference = line.Reference;
invoice.SAAddressLine1 = line.SAAddressLine1;
invoice.SAAddressLine2 = line.SAAddressLine2;
invoice.SAAddressLine3 = line.SAAddressLine3;
invoice.SAAddressLine4 = line.SAAddressLine4;
invoice.SACity = line.SACity;
invoice.SACountry = line.SACountry;
invoice.SAPostalCode = line.SAPostalCode;
invoice.SARegion = line.SARegion;
invoice.Sent = line.Sent;
invoice.Status = line.Status;
invoice.TaxTotal = line.TaxTotal;
invoice.Total = line.Total;
invoice.Type = line.Type;
dictionaryList.Add(invoice.InvoiceNumber, invoice);
}
var item = new Model.Import.XeroInvoiceFlatFile.LineItem();
item.AccountCode = line.AccountCode;
item.Description = line.Description;
item.Discount = line.Discount;
item.InventoryItemCode = line.InventoryItemCode;
item.LineAmount = line.LineAmount;
item.Quantity = line.Quantity;
item.TaxAmount = line.TaxAmount;
item.TaxType = line.TaxType;
item.TrackingName1 = line.TrackingName1;
item.TrackingName2 = line.TrackingName2;
item.TrackingOption1 = line.TrackingOption1;
item.TrackingOption2 = line.TrackingOption2;
item.UnitAmount = line.UnitAmount;
dictionaryList[invoiceNumber].LineItems.Add(item);
}
return dictionaryList;
}
}
}

View File

@@ -0,0 +1,54 @@
using CsvHelper;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualBasic.FileIO;
namespace BealeEngineering.Core.Logic.Import
{
public class wipXeroInvoiceFlatFile
{
public wipXeroInvoiceFlatFile(string sqlConnectionString)
{
SqlConnectionString = sqlConnectionString;
}
private string SqlConnectionString { get; set; }
public void ByFilePath(string filePath)
{
// get model list
//// get db invoices
//var saleInvInst = new Data.Database.Sale.InvoiceGet(SqlConnectionString);
//saleInvInst.InvoiceNumber = invDictionary.Keys.ToList();
//var dataInvList = saleInvInst.GetByFilters();
// compare
// update modified records
// insert new records <--------- only insert approved invoices
// delete records (is this possible??) <------- i think so, include deleted and voided in flatfile??
}
}
}

View File

@@ -0,0 +1,178 @@
using CsvHelper;
using Microsoft.VisualBasic.FileIO;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BealeEngineering.Core.Logic.Utilities
{
public class CSVGetRFC4180Compliant
{
public string OutputFilepath { get; set; }
public bool OutputFilepathIsSet
{
get
{
if (string.IsNullOrEmpty(OutputFilepath)) { return false; }
else { return true; }
}
}
public string OutputString { get; private set; }
public bool OutputStringIsSet
{
get
{
if (string.IsNullOrWhiteSpace(OutputString)) { return false; }
else { return true; }
}
}
private List<List<string>> MsVbTextParserResult { get; set; }
public void ByFilePath(string filePath)
{
if (string.IsNullOrWhiteSpace(filePath) || !File.Exists(filePath))
{ throw new Exception("File or filepath error."); }
string inputString = File.ReadAllText(filePath);
ByString(ref inputString);
inputString = "";
inputString = null;
GC.Collect();
}
public void ByString(ref string inputCsvString)
{
if (string.IsNullOrWhiteSpace(inputCsvString))
{ throw new Exception("Invalid CSV string"); }
MsVbTextFieldParser(ref inputCsvString);
CreateCompliantCsv();
//CreateCompliantCsvMyVer();
}
private void MsVbTextFieldParser(ref string inputCsvString)
{
MsVbTextParserResult = new List<List<string>>();
using (TextFieldParser parser = new TextFieldParser(new StringReader(inputCsvString)))
{
parser.SetDelimiters(new string[] { "," });
parser.HasFieldsEnclosedInQuotes = true;
// Skip over header line.
//parser.ReadLine();
int lineNumber = 0;
int columnCountFound = 0;
while (!parser.EndOfData)
{
lineNumber = lineNumber + 1;
var values = new List<string>();
var readFields = parser.ReadFields();
if (readFields != null)
{
values.AddRange(readFields);
MsVbTextParserResult.Add(values);
if (lineNumber == 1)
{
columnCountFound = values.Count();
}
else
{
if (values.Count != columnCountFound)
{
throw new Exception("Error parsing file, incorrect columns count.");
}
}
}
}
}
}
private void CreateCompliantCsv()
{
var outputStringBuilder = new StringBuilder();
var csvConfig = new CsvHelper.Configuration.CsvConfiguration(CultureInfo.CurrentUICulture);
// using (var writer = new StreamWriter(IntermediateCsv))
using (var writer = new StringWriter(outputStringBuilder))
using (var csv = new CsvWriter(writer, csvConfig))
{
foreach (var record in MsVbTextParserResult)
{
foreach (var field in record)
{
csv.WriteField(field);
}
csv.NextRecord();
}
writer.Flush();
}
// output
OutputString = outputStringBuilder.ToString();
WriteToFile();
// clean up
outputStringBuilder.Clear();
outputStringBuilder = null;
MsVbTextParserResult.Clear();
MsVbTextParserResult = null;
GC.Collect();
}
// redundant once class is tested
public void CreateCompliantCsvMyVer()
{
// get column count
int columnCount = MsVbTextParserResult[0].Count();
// create proper deliminatd string from result
var outputStringBuilder = new StringBuilder("");
foreach (var line in MsVbTextParserResult)
{
int i = 0;
foreach (var field in line)
{
i = i + 1;
// check for double quotes within field, if found preceed/escape with double quote
string value = field.Replace("\"", "\"\"");
if (i == 1)
{
outputStringBuilder.Append("\"" + value);
}
else if (i < columnCount)
{
outputStringBuilder.Append("\",\"" + value);
}
else
{
outputStringBuilder.AppendLine("\",\"" + value + "\"");
}
}
}
// output
OutputString = outputStringBuilder.ToString();
WriteToFile();
// clean up
outputStringBuilder.Clear();
outputStringBuilder = null;
MsVbTextParserResult.Clear();
MsVbTextParserResult = null;
GC.Collect();
}
public void WriteToFile()
{
if (OutputFilepathIsSet && OutputStringIsSet)
{ System.IO.File.WriteAllText(OutputFilepath, OutputString); }
}
}
}

View File

@@ -0,0 +1,57 @@
using System;
using System.Collections.Generic;
namespace BealeEngineering.Core.Model.Import
{
public class XeroInvoiceFlatFile
{
public string ContactName { get; set; }
public string EmailAddress { get; set; }
public string POAddressLine1 { get; set; }
public string POAddressLine2 { get; set; }
public string POAddressLine3 { get; set; }
public string POAddressLine4 { get; set; }
public string POCity { get; set; }
public string PORegion { get; set; }
public string POPostalCode { get; set; }
public string POCountry { get; set; }
public string SAAddressLine1 { get; set; }
public string SAAddressLine2 { get; set; }
public string SAAddressLine3 { get; set; }
public string SAAddressLine4 { get; set; }
public string SACity { get; set; }
public string SARegion { get; set; }
public string SAPostalCode { get; set; }
public string SACountry { get; set; }
public string InvoiceNumber { get; set; }
public string Reference { get; set; }
public DateTime InvoiceDate { get; set; }
public DateTime? DueDate { get; set; }
public DateTime? PlannedDate { get; set; }
public decimal Total { get; set; }
public decimal TaxTotal { get; set; }
public decimal InvoiceAmountPaid { get; set; }
public decimal InvoiceAmountDue { get; set; }
public List<LineItem> LineItems { get; set; } = new List<LineItem>();
public class LineItem
{
public string InventoryItemCode { get; set; }
public string Description { get; set; }
public decimal Quantity { get; set; }
public decimal UnitAmount { get; set; }
public int? Discount { get; set; }
public decimal LineAmount { get; set; }
public string AccountCode { get; set; }
public string TaxType { get; set; }
public decimal TaxAmount { get; set; }
public string TrackingName1 { get; set; }
public string TrackingOption1 { get; set; }
public string TrackingName2 { get; set; }
public string TrackingOption2 { get; set; }
}
public string Currency { get; set; }
public string Type { get; set; }
public string Sent { get; set; }
public string Status { get; set; }
}
}

View File

@@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
namespace BealeEngineering.Core.Model.Import
{
public class XeroInvoiceFlatFileDTO
{
public string ContactName { get; set; }
public string EmailAddress { get; set; }
public string POAddressLine1 { get; set; }
public string POAddressLine2 { get; set; }
public string POAddressLine3 { get; set; }
public string POAddressLine4 { get; set; }
public string POCity { get; set; }
public string PORegion { get; set; }
public string POPostalCode { get; set; }
public string POCountry { get; set; }
public string SAAddressLine1 { get; set; }
public string SAAddressLine2 { get; set; }
public string SAAddressLine3 { get; set; }
public string SAAddressLine4 { get; set; }
public string SACity { get; set; }
public string SARegion { get; set; }
public string SAPostalCode { get; set; }
public string SACountry { get; set; }
public string InvoiceNumber { get; set; }
public string Reference { get; set; }
public DateTime InvoiceDate { get; set; }
public DateTime? DueDate { get; set; }
public DateTime? PlannedDate { get; set; }
public decimal Total { get; set; }
public decimal TaxTotal { get; set; }
public decimal InvoiceAmountPaid { get; set; }
public decimal InvoiceAmountDue { get; set; }
public string InventoryItemCode { get; set; }
public string Description { get; set; }
public decimal Quantity { get; set; }
public decimal UnitAmount { get; set; }
public int? Discount { get; set; }
public decimal LineAmount { get; set; }
public string AccountCode { get; set; }
public string TaxType { get; set; }
public decimal TaxAmount { get; set; }
public string TrackingName1 { get; set; }
public string TrackingOption1 { get; set; }
public string TrackingName2 { get; set; }
public string TrackingOption2 { get; set; }
public string Currency { get; set; }
public string Type { get; set; }
public string Sent { get; set; }
public string Status { get; set; }
}
}

View File

@@ -21,11 +21,11 @@ namespace BealeEngineering.Core.Test
//var inst = new Core.Test.Sales.Invoice(SqlConnectionString);
//inst.GetInvoice();
var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString);
inst2.AllocateInvoicesToPurchaseOrders();
//var inst2 = new Test.Client.PurchaseOrder(SqlConnectionString);
//inst2.AllocateInvoicesToPurchaseOrders();
var inst3 = new Test.Import.ImportFlatfile();
inst3.Go();
}
}
}

View File

@@ -0,0 +1,21 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BealeEngineering.Core.Test.Import
{
public class ImportFlatfile
{
public void Go()
{
string fileInputPath =
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)
+ @"\Dropbox\Beale Engineering Services Ltd\BE Accounts\Xero-Export-Invoices.csv";
var inst = new Data.Xero.FlatFile.ImportInvoice();
var lkdsjflsd = inst.ByFilePath(fileInputPath);
}
}
}

View File

@@ -1,4 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="CsvHelper" version="13.0.0" targetFramework="net472" />
<package id="Dapper" version="2.0.30" targetFramework="net472" />
<package id="Microsoft.Bcl.AsyncInterfaces" version="1.1.0" targetFramework="net472" />
<package id="Microsoft.CSharp" version="4.5.0" targetFramework="net472" />
<package id="Microsoft.VisualBasic" version="10.3.0" targetFramework="net472" />
<package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net472" />
<package id="System.Threading.Tasks.Extensions" version="4.5.2" targetFramework="net472" />
</packages>