SFM-loan-Rest-Api/pay-in-pepper.go

package main

import (
	log "github.com/sirupsen/logrus"
	"regexp"
	"strings"
	"time"
)

type PepperRow struct {
	// Common
	InvoiceDate time.Time

	// Shared Fields for both upfront and Trailer
	LoanNumber        string
	Customer          string
	PrimaryIntroducer string
	NextGenNumber     string
	Settlement        time.Time
	NetAmount         float64
	GST               float64
	GrossAmount       float64

	//Trailer Only
	Period           time.Time
	Balance          float64
	SettlementAmount float64
	TrailerMargin    float64

	// Upfront only
	UpfrontRate float64

	// Derived
	IncomeType   string
	IncomeAmount float64
}

func (m *AiDecodeIncome) isPepperXls(raw string) bool {
	keyword := "ABN86092110079"
	lines := strings.Split(raw, "\n")
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove space
		lines[i] = strings.ReplaceAll(lines[i], ",", "") // remove ,
		lines[i] = strings.ReplaceAll(lines[i], ":", "") // remove :
	}
	return m.isKeywordExist(keyword, lines)
}

func (m *AiDecodeIncome) decodePepperXls(raw []byte) (e error) {
	m.Pepper = make([]PepperRow, 0, 30)

	if m.decodePepperIsUpfront(string(raw)) {
		e = m.decodePepperXlsUpfront(raw)
	} else {
		e = m.decodePepperXlsTrailer(raw)
	}
	return
}

func (m *AiDecodeIncome) decodePepperIsUpfront(raw string) bool {
	keyword := "upfrontcommissions"
	lines := strings.Split(raw, "\n")
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") //remove space
		lines[i] = strings.ReplaceAll(lines[i], ",", "") //remove ,
		lines[i] = strings.ToLower(lines[i])             //to lower case
	}
	return m.isKeywordExist(keyword, lines)
}

func (m *AiDecodeIncome) decodePepperXlsTrailer(raw []byte) (e error) {
	patternTrailer := `,([0-9]+),([ A-Za-z]+)[ ,]([ A-Za-z]+),(\d+),((?:(?:[12][0-9]|0[1-9])[/.-]02|(?:30|[12][0-9]|0[1-9])[/.-](?:0[469]|11)|(?:3[01]|[12][0-9]|0[1-9])[/.-](?:0[13578]|1[02]))[/.-][0-9]{4}),(\d+(?:\.\d{1,2})?),(\d+(?:\.\d{1,2})?),(\d+(?:\.\d{1,2})?),(\d+(?:\.\d{1,2})?),\d+\.?\d{1,2},\d+(?:\.\d{1,2})?,(\d+(?:\.\d{1,2})?),\d+(?:\.\d{1,2})?,(\d+(?:\.\d{1,2})?),\d+(?:\.\d{1,2})?,\d+(?:\.\d{1,2})?,\d+(?:\.\d{1,2})?`

	IncomeType := "Trail"
	InvoiceDate := m.decodePepperXlsGetInvoiceDate(raw)
	CurrentPeriod := m.decodePepperXlsGetTrailerPeriod(raw)

	validLine, e := regexp.Compile(patternTrailer) // error if regexp invalid
	if e != nil {
		return
	}

	matches := validLine.FindAllSubmatch(raw, -1)
	if matches == nil {
		log.Warn("pepper decode trailer found nothing no matches", IncomeType, string(raw))
		return
	}

	for _, v := range matches {
		pp := PepperRow{Period: CurrentPeriod, IncomeType: IncomeType}
		pp.LoanNumber = string(v[1])
		pp.Customer = string(v[2])
		pp.PrimaryIntroducer = string(v[3])
		pp.NextGenNumber = string(v[4])
		pp.Settlement, _ = time.Parse("02/01/2006", string(v[5]))
		pp.Balance = m.currencyToFloat64(string(v[6]))
		pp.SettlementAmount = m.currencyToFloat64(string(v[7]))
		pp.TrailerMargin = m.currencyToFloat64(string(v[8]))
		pp.NetAmount = m.currencyToFloat64(string(v[9]))
		pp.GST = m.currencyToFloat64(string(v[10]))
		pp.GrossAmount = m.currencyToFloat64(string(v[11]))
		pp.IncomeAmount = pp.NetAmount
		pp.InvoiceDate = InvoiceDate
		m.Pepper = append(m.Pepper, pp)
	}

	return
}
func (m *AiDecodeIncome) decodePepperXlsUpfront(raw []byte) (e error) {
	patternUpfront := `,(\d+),([ A-Za-z]+),([ A-Za-z]+),(\d+),((?:02[/.-](?:[12][0-9]|0[1-9])|(?:0[469]|11)[/.-](?:30|[12][0-9]|0[1-9])|(?:0[13578]|1[02])[/.-](?:3[01]|[12][0-9]|0[1-9]))[/.-][0-9]{4}),"(\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)",(\d+(?:\.\d{1,2})?),-?\d+(?:\.\d{1,2})?,(-?\d+(?:\.\d{1,2})?),-?\d+(?:\.\d{1,2})?,(-?\d+(?:\.\d{1,2})?),-?\d+(?:\.\d{1,2})?,(-?\d+(?:\.\d{1,2})?)`
	IncomeType := "Upfront"

	InvoiceDate := m.decodePepperXlsGetInvoiceDate(raw)

	validLine, e := regexp.Compile(patternUpfront) // error if regexp invalid
	if e != nil {
		return
	}

	matches := validLine.FindAllSubmatch(raw, -1)
	if matches == nil {
		log.Warn("pepper decode upfront found nothing no matches", IncomeType, string(raw))
		return
	}

	for _, v := range matches {
		pp := PepperRow{IncomeType: IncomeType}
		pp.LoanNumber = string(v[1])
		pp.Customer = string(v[2])
		pp.PrimaryIntroducer = string(v[3])
		pp.NextGenNumber = string(v[4])
		pp.Settlement, _ = time.Parse("01/02/2006", string(v[5]))
		pp.Balance = m.currencyToFloat64(string(v[6]))
		pp.UpfrontRate = m.currencyToFloat64(string(v[7]))
		pp.NetAmount = m.currencyToFloat64(string(v[8]))
		pp.GST = m.currencyToFloat64(string(v[9]))
		pp.GrossAmount = m.currencyToFloat64(string(v[10]))
		pp.IncomeAmount = pp.NetAmount
		pp.InvoiceDate = InvoiceDate
		m.Pepper = append(m.Pepper, pp)
	}
	return
}

func (m *AiDecodeIncome) decodePepperXlsGetInvoiceDate(raw []byte) (ret time.Time) {
	invoiceDatePattern := `(?i)Date *:+ *((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}|(?:(?:[12][0-9]|0?[1-9])-(?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)|(?:30|[12][0-9]|0?[1-9])-(?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)|(?:3[01]|[12][0-9]|0?[1-9])-(?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December))-[0-9]{4})`
	invoiceDateLine, e := regexp.Compile(invoiceDatePattern)
	if e != nil {
		return
	}
	matches := invoiceDateLine.FindSubmatch(raw)
	ret, _ = time.Parse("02-Jan-2006", string(matches[1]))
	return
}

func (m *AiDecodeIncome) decodePepperXlsGetTrailerPeriod(raw []byte) (ret time.Time) {
	period := `(?i)Trailer *Commissions[ ,]+Trail +((?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)[\t ]+[0-9]{4})`

	periodLine, e := regexp.Compile(period)
	if e != nil {
		return
	}
	periods := periodLine.FindSubmatch(raw)
	if periods == nil {
		log.Error("pepper Trailer cannot find Period", string(raw), m)
		return
	}
	ret, _ = time.Parse("January 2006", string(periods[1]))
	return
}

var sample_pepper_trailer = `
FORMAT XLONE REPORT,DefnSheetName=_defntmp_,,,,,,,,,,,,,,,,,,,,
SET,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,RECIPIENT CREATED TAX INVOICE,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,Date: 15-Jul-2020,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,Pepper Homeloans Pty Limited ,,,,,,,,,,,,,,,,
*,,,,,ABN:    86 092 110 079,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,To:,Super Finance Markets,,,,,,,,,,,,,,,
*,,,,,,ABN: 68 620 533 333,,,,,,,,,,,,,,,
*,,,,,,"International Tower One, Suite 2304, Level 23 100 Barangaroo Avenue, ",,,,,,,,,,,,,,,
*,,,,,,Barangaroo NSW 2000,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,Trailer Commissions,TRAIL JUNE 2020,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,Account Number,Email Address,Description,UploadDate,Loan Number,Customer,Primary Introducer,Franchisee \ Next Gen Number \ Notes,Settlement Date,Balance as at end of month,Settlement Amount,Trailer Comm Margin,Net Amount,Net Amount,GST,GST*,Gross Amount,Gross Amount,,,
h.*,,,,,,,,,,,,,,,,,,,,,
h.REPEAT,,,,,,,,,,,,,,,,,,,,,
h.COLUMNS,,,,,,,,,,,,,,,,,,,,,
h.REPEAT,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1239633,Benny Youbin He,LAW ANDREW,720478,25/07/2019,1497907.3,1500000,0.25,307.79,307.79,30.78,30.78,307.79,338.57,1497907.3,1500000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1244928,Yingjiao Zhang,LAW ANDREW,729657,05/02/2020,636177.21,739200,0.25,130.72,130.72,13.07,13.07,130.72,143.79,636177.21,739200,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1245619,Darren Julian Cantrill,LAW ANDREW,730570,03/03/2020,650690.27,656900,0.25,133.7,133.7,13.37,13.37,133.7,147.07,650690.27,656900,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1246270,Khanh Huyen Nguyen Cung,LAW ANDREW,731134,30/03/2020,344808.41,423000,0.25,70.85,70.85,7.09,7.09,70.85,77.94,344808.41,423000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247301,Phuong Moc Luu,LAW ANDREW,731481,07/05/2020,1198547.28,1200000,0.25,246.28,246.28,24.63,24.63,246.28,270.91,1198547.28,1200000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1246232,Monica Cao,LAW ANDREW,731538,25/03/2020,647250.56,650000,0.25,133,133,13.3,13.3,133,146.3,647250.56,650000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1245878,Teresa Thach Thao Le Nguyen,LAW ANDREW,731751,06/03/2020,256162.15,260000,0.25,52.64,52.64,5.26,5.26,52.64,57.9,256162.15,260000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1245983,Nuam Thiha,LAW ANDREW,731906,13/03/2020,597299.23,600000,0.25,122.73,122.73,12.27,12.27,122.73,135,597299.23,600000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247828,Diep Thi Ngoc Truong,LAW ANDREW,731951,16/06/2020,520000,520000,0.25,106.85,106.85,10.69,10.69,106.85,117.54,520000,520000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1246826,Thi Kim Lan Vo,CHEN LIN,732745,05/05/2020,449125.63,450000,0.25,92.29,92.29,9.23,9.23,92.29,101.52,449125.63,450000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247293,Vu Thuy Dung Phan,CHEN LIN,733940,04/05/2020,487267.77,488000,0.25,100.12,100.12,10.01,10.01,100.12,110.13,487267.77,488000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247314,Ngoc Dung Tran,CHEN LIN,734192,08/05/2020,612228.65,613000,0.25,125.8,125.8,12.58,12.58,125.8,138.38,612228.65,613000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247541,Trung Thuy Bui,CHEN LIN,734279,18/05/2020,361345.39,624000,0.25,74.25,74.25,7.43,7.43,74.25,81.68,361345.39,624000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247774,Kim Son Dang,CHEN LIN,734307,03/06/2020,364000,364000,0.25,74.79,74.79,7.48,7.48,74.79,82.27,364000,364000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247775,Kim Son Dang,CHEN LIN,734308,03/06/2020,616000,616000,0.25,126.57,126.57,12.66,12.66,126.57,139.23,616000,616000,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247677,Helen Nguyen,CHEN LIN,734315,26/05/2020,417034.11,417506.4,0.25,85.69,85.69,8.57,8.57,85.69,94.26,417034.11,417506.4,0.25
LIST,1002621011000000000000,,PHL             Trail Commissio LenLMOgGG       NA              Not Applicable                  Cost of Sales,07/15/2020,1247977,Trieu Hong Nhung Nguyen,CHEN LIN,734403,23/06/2020,416000,416000,0.25,85.48,85.48,8.55,8.55,85.48,94.03,416000,416000,0.25
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
h.REPEAT END,,,,,,,,,,,,,,,,,,,,,
h.*,,,,,,,,,,,,,,,,,,,,,
h.*,,,,,,,,,,,,,,,,,,,,,
h.REPEAT END,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,2069.55,2069.55,206.97,206.97,2069.55,2276.52,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,* The GST shown above is payable by the Supplier,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,Remittance Advice:,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,Payment has been directly credited to your bank account as follows:,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,Account Name:,Super Finance Markets,,,,,,,,,,,,,,,
*,,,,,Bank:,&COLUMNDEFN1.F1CHA_BANKNAME,,,,,,,,,,,,,,,
*,,,,,BSB:,082-401,,,,,,,,,,,,,,,
*,,,,,Account Number:,267602721,,,,,,,,,,,,,,,
*,,,,,Amount:,2276.52,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,,,,,
`

var sample_pepper_upfront = `
FORMAT XLONE REPORT,DefnSheetName=_defntmp_,,,,,,,,,,,,,,,,
SET,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,RECIPIENT CREATED TAX INVOICE
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,Date: 30-Jun-2020
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,Pepper Homeloans Pty Limited,,,,,,,,,,,,
*,,,,,ABN:    86 092 110 079,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,To:,Super Finance Markets,,,,,,,,,,,
*,,,,,,ABN: 68 620 533 333,,,,,,,,,,,
*,,,,,,"International Tower One, Suite 2304, Level 23 100 Barangaroo Avenue, ",,,,,,,,,,,
*,,,,,,Barangaroo NSW 2000,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,Upfront Commissions,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,Account Number,Email,Description,UploadDate,Loan Number,Customer,Primary Introducer,Notes\Next Gen Number,Settlement Date,Settlement Amount less Offset & Redraw,Upfront Rate,Net Amount,Net Amount,GST,GST*,Gross Amount,Gross Amount
*,,,,,,,,,,,,,,,,,
LIST,SUFMARCOM,andrew@supercredit.com.au,Super Finance Markets,06/30/2020,1247977,Trieu Hong Nhung Nguyen,CHEN LIN,734403,06/22/2020,"416,000.00",1.00,-4160,4160,-416,416,-4576,4576
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,-4160,4160,-416,416,-4576,4576
*,,,,,,,,,,,,,,,,,
*,,,,,* The GST shown above is payable by the Supplier,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,Remittance Advice:,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,Payment has been directly credited to your bank account as follows:,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,Account Name:,Super Finance Markets,,,,,,,,,,,
*,,,,,BSB:,082-401,,,,,,,,,,,
*,,,,,Account Number:,267602721,,,,,,,,,,,
*,,,,,Amount:,4576,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
*,,,,,,,,,,,,,,,,,
`