SFM-loan-Rest-Api/pay-in-resimac.go

package main

import (
	"fmt"
	log "github.com/sirupsen/logrus"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var sample_resimac = `
TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM
335,8779,1,A ,Huang        ,113423,3.79,3.36,0.43,02/08/2019,81865.89,29.73,0.6,60,0.18,-12.45,0,17.28,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,2,A ,Ding         ,321030,3.72,3.16,0.56,12/10/2018,311077.1,143.25,1,60,0.28,-71.62,0,71.63,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,4,A ,Song         ,640000,3.69,3.29,0.4,01/23/2019,607519.14,198.79,0.8,60,0.23,-114.3,0,84.49,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,4,B ,Song         ,208000,3.69,3.29,0.4,01/23/2019,208055,68.4,0.8,60,0.23,-39.33,0,29.07,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
337,8779,7,A ,Gill         ,404376,3.79,3.26,0.53,02/25/2019,393745.79,171.5,1,60,0.28,-90.6,0,80.9,0,0,0, ,F,Super Finance Markets Pty Ltd                               ,202006
507,8779,9,A ,Lu           ,980000,4.6,4.07,0.53,06/18/2019,980000,426.9,1,60,0.28,-225.53,0,201.37,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
335,8779,10,A ,Sideris      ,556000,3.32,2.79,0.53,04/23/2019,543091.68,236.7,1,60,0.28,-125.05,0,111.65,0,0,0, ,W,Super Finance Markets Pty Ltd                               ,202006
507,8779,13,A ,Lu           ,570000,4.6,4.07,0.53,06/18/2019,50000,21.78,1,60,0.28,-11.51,0,10.27,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
507,8779,14,A ,Lu           ,970000,4.15,3.62,0.53,06/18/2019,954336.82,416.06,1,60,0.28,-219.8,0,196.26,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
337,8779,18,A ,Chen         ,444000,3.82,3.29,0.53,06/18/2019,444000,193.41,1,60,0.28,-102.18,0,91.23,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
522,8779,19,A ,Horm         ,944000,4.7,4.17,0.53,07/24/2019,544000,295.06,1,60,0.28,-155.88,0,139.18,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
522,8779,21,A ,Kemp         ,479952,4.25,3.72,0.53,07/24/2019,470613.4,204.94,1,60,0.28,-108.27,0,96.67,0,0,0, ,W,Super Finance Markets Pty Ltd                               ,202006
522,8779,24,A ,Tiang        ,684481,4.7,4.17,0.53,08/23/2019,668000,290.99,1,60,0.28,-153.73,0,137.26,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,26,A ,Li           ,564000,4.25,3.72,0.53,09/26/2019,557190.2,243,1,60,0.28,-128.38,0,114.62,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,27,A ,Quach        ,406000,4.4,3.87,0.53,10/21/2019,398603.27,174.39,1,60,0.28,-92.13,0,82.26,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,27,B ,Quach        ,406000,4.4,3.87,0.53,10/21/2019,406000,176.86,1,60,0.28,-93.44,0,83.42,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
502,8779,28,A ,Zhu          ,1000000,4.55,4.02,0.53,09/18/2019,988561.45,430.95,1,60,0.28,-227.67,0,203.28,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
307,8779,30,A ,Huynh        ,410453,3.22,2.69,0.53,09/19/2019,394606.79,172.23,1,60,0.28,-90.99,0,81.24,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,31,A ,Xu           ,830000,4.15,3.62,0.53,01/30/2020,824329.48,359.53,1,60,0.28,-189.94,0,169.59,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
305,8779,32,A ,Tran         ,476000,3.52,2.99,0.53,10/03/2019,470274.11,204.88,1,60,0.28,-108.24,0,96.64,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,37,A ,Zulfaqari    ,552000,3.22,2.69,0.53,12/19/2019,543482.81,237.12,1,60,0.28,-125.27,0,111.85,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,38,A ,Pham         ,512000,3.22,2.69,0.53,01/10/2020,507925.37,221.37,1,60,0.28,-116.95,0,104.42,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
310,8779,42,A ,Nguyen       ,318542,3.52,2.99,0.53,12/06/2019,311782.42,135.71,1,60,0.28,-71.7,0,64.01,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,43,A ,Hopkins      ,427505,3.52,2.99,0.53,01/08/2020,415112.03,180.87,1,60,0.28,-95.55,0,85.32,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,43,B ,Hopkins      ,41567,3.52,2.99,0.53,01/08/2020,40011.29,17.43,1,60,0.28,-9.21,0,8.22,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
334,8779,43,C ,Hopkins      ,10292,3.52,2.99,0.53,01/08/2020,9991.49,4.35,1,60,0.28,-2.3,0,2.05,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,45,A ,Tran         ,408000,3.22,2.69,0.53,12/06/2019,403381.11,175.81,1,60,0.28,-92.88,0,82.93,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,47,A ,Lebon        ,100000,3.32,2.79,0.53,12/09/2019,44000,20.14,1,60,0.28,-10.64,0,9.5,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
337,8779,48,A ,Le           ,400000,3.22,2.69,0.53,12/03/2019,396171.84,172.6,1,60,0.28,-91.18,0,81.42,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,49,A ,Vu           ,509600,3.22,2.69,0.53,12/13/2019,504687.98,220.02,1,60,0.28,-116.24,0,103.78,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,50,A ,Wang         ,1500000,4.15,3.62,0.53,01/15/2020,1387663.97,604.28,1,60,0.28,-319.24,0,285.04,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,52,A ,Le           ,520000,3.32,2.79,0.53,01/20/2020,513359.23,224.03,1,60,0.28,-118.36,0,105.67,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,54,A ,Nguyen       ,268000,3.32,2.79,0.53,01/08/2020,265878.84,115.87,1,60,0.28,-61.21,0,54.66,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,56,A ,Hanna        ,250000,3.22,2.69,0.53,01/09/2020,245368.96,107,1,60,0.28,-56.53,0,50.47,0,0,0, ,W,Super Finance Markets Pty Ltd                               ,202006
308,8779,57,A ,Le           ,300000,3.22,2.69,0.53,01/08/2020,293231.8,127.97,1,60,0.28,-67.61,0,60.36,0,0,0, ,W,Super Finance Markets Pty Ltd                               ,202006
309,8779,58,A ,Le           ,432000,3.22,2.69,0.53,12/17/2019,427824.14,186.53,1,60,0.28,-98.54,0,87.99,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,63,A ,Nguyen       ,449175,3.22,2.69,0.53,12/23/2019,444814.79,194,1,60,0.28,-102.49,0,91.51,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,64,A ,Leav         ,444000,3.42,2.89,0.53,01/07/2020,443334.6,193.23,1,60,0.28,-102.08,0,91.15,0,0,0, ,F,Super Finance Markets Pty Ltd                               ,202006
308,8779,66,A ,Nguyen       ,516750,3.22,2.62,0.6,01/21/2020,511192.69,252.48,1,60,0.28,-117.82,0,134.66,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,69,A ,Leab         ,435000,4.14,3.61,0.53,02/10/2020,432580.51,188.52,1,60,0.28,-99.59,0,88.93,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,71,A ,Yang         ,511000,3.22,2.62,0.6,02/07/2020,506610.13,249.98,1,60,0.28,-116.66,0,133.32,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,76,A ,Huynh        ,442060,3.22,2.62,0.6,02/28/2020,438854.39,216.79,1,60,0.28,-101.17,0,115.62,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
308,8779,76,B ,Huynh        ,77940,3.22,2.62,0.6,02/28/2020,77374.82,38.22,1,60,0.28,-17.84,0,20.38,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
500,8779,82,A ,Pham         ,788000,4.7,4.17,0.53,05/12/2020,791145.52,344.13,1,60,0.28,-181.81,0,162.32,0,0,0, ,M,Super Finance Markets Pty Ltd                               ,202006
`

type ResimacRow struct {
	TRSTCD int
	ORGNTR int
	LOANNO int
	PORTN  string
	BRNAMX string  //Brower
	LNAMT  float64 //Loan amount
	INTRTE float64
	DELRTE float64
	MARGIN float64
	SETLDX time.Time //settled date
	LNBAL  float64
	MANFEE float64
	RSINCP float64
	RSSACT float64
	RSSACP float64
	SACAMT float64
	MOFEE  float64
	MANTOT float64
	NEWLON float64
	REFADJ float64
	NETNEW float64
	FIXED  string
	LNFORT string
	ORGNAM string
	LNEOM  time.Time

	//Derived attributes
	LoanNumber   string
	Borrower     string
	LoanAmount   float64
	Balance      float64
	IncomeAmount float64
}

type ResimacPdf struct {
	Period        time.Time
	LoanNumber    string
	Borrower      string
	LoanAmount    float64
	InterestRate  float64
	DelvRate      float64
	Margin        float64
	Settlement    time.Time
	Balance       float64
	OffsetBalance float64
	GrossManFee   float64
	IncentiveP    float64
	Months        int
	SacrificeP    float64
	Fee           float64
	MoFee         float64
	NetManFee     float64
	IncomeAmount  float64
}

func (m *AiDecodeIncome) isResimacXls(raw string) bool {
	keyword := "TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM"
	lines := strings.Split(raw, "\n")
	//remove all spaces
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
	}
	return m.isKeywordExist(keyword, lines)
}

func (m *AiDecodeIncome) isResimacPdf(raw string) bool {
	keyword := "SuperFinanceMarketsPtyLtd-8779"
	lines := strings.Split(raw, "\n")
	//remove all spaces
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
	}
	return m.isKeywordExist(keyword, lines)
}

func (m *AiDecodeIncome) decodeResimacXls(raw []byte) (e error) {
	m.ResimacXls = make([]ResimacRow, 0, 100)
	lines := strings.Split(string(raw), "\n")
	foundHeader := false
	var idx map[string]int
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
		el := strings.Split(lines[i], ",")
		if len(el) < 25 {
			continue //bypass
		}

		if !foundHeader {
			foundHeader, idx = m.decodeResimacXlsHeader(lines[i])
			continue
		}
		//we have already found header
		r := ResimacRow{}

		r.TRSTCD, _ = strconv.Atoi(el[idx["TRSTCD"]])
		r.ORGNTR, _ = strconv.Atoi(el[1])
		r.LOANNO, _ = strconv.Atoi(el[2])
		r.PORTN = el[3]
		r.LoanNumber = fmt.Sprintf("%d-%06d-%06d%s", r.TRSTCD, r.ORGNTR, r.LOANNO, r.PORTN)
		r.BRNAMX = el[4]
		r.Borrower = r.BRNAMX
		r.LNAMT = m.currencyToFloat64(el[5])
		r.LoanAmount = r.LNAMT
		r.INTRTE = m.currencyToFloat64(el[6])
		r.DELRTE = m.currencyToFloat64(el[7])
		r.MARGIN = m.currencyToFloat64(el[8])
		r.SETLDX, _ = time.Parse("1/02/2006", el[9])
		r.LNBAL = m.currencyToFloat64(el[10])
		r.Balance = r.LNBAL
		r.MANFEE = m.currencyToFloat64(el[11])
		r.RSINCP = m.currencyToFloat64(el[12])
		r.RSSACT = m.currencyToFloat64(el[13])
		r.RSSACP = m.currencyToFloat64(el[14])
		r.SACAMT = m.currencyToFloat64(el[15])
		r.MOFEE = m.currencyToFloat64(el[16])
		r.MANTOT = m.currencyToFloat64(el[17])
		r.IncomeAmount = r.MANTOT
		r.NEWLON = m.currencyToFloat64(el[18])
		r.REFADJ = m.currencyToFloat64(el[19])
		r.NETNEW = m.currencyToFloat64(el[20])
		r.FIXED = el[21]
		r.LNFORT = el[22]
		r.ORGNAM = strings.TrimSpace(el[23])
		r.LNEOM, _ = time.Parse("200601", el[24])
		m.ResimacXls = append(m.ResimacXls, r)
	}
	return
}

func (m *AiDecodeIncome) decodeResimacXlsHeader(line string) (isHeaderLine bool, idx map[string]int) {
	keys := []string{
		"TRSTCD", "ORGNTR", "LOANNO", "PORTNO", "BRNAMX", "LNAMT", "INTRTE", "DELRTE",
		"MARGIN", "SETLDX", "LNBAL", "MANFEE", "RSINCP", "RSSACT", "RSSACP", "SACAMT",
		"MOFEE", "MANTOT", "NEWLON", "REFADJ", "NETNEW", "FIXED", "LNFORT", "ORGNAM",
		"LNEOM"}

	idx = make(map[string]int)
	l := strings.ReplaceAll(line, " ", "") // remove space
	el := strings.Split(l, ",")
	found := 0
	for i := 0; i < len(el); i++ {
		a := strings.TrimSpace(el[i])
		a = strings.ToUpper(a)
		for _, k := range keys {
			if a == k {
				idx[k] = i
				found++
				break
			}
		}
	}
	isHeaderLine = found > 20 // if we found more than 20 headers
	return
}

func (m *AiDecodeIncome) decodeResimacPdf(raw []byte) (e error) {
	m.ResimacPdf = make([]ResimacPdf, 0, 30)
	period := `(?i)Trailer +Fees Report[ :]+((?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)[\t ]+[0-9]{4})`
	pattern := `(?i)(\d+-0*8779-[\d A-Za-z-]+[A-Za-z]) +([A-Za-z]+) +(\$\d{1,3}(?:,\d{3})*(?:[\d.]\d{2})?) +(\d{1,3}(?:\.\d{1,2})?) +([\d ]{1,3}(?:[ .]\d{1,2})?) +(\d{1,3}(?:[\d.]\d{1,2})?) +((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}) +(-?\$\d{1,3}(?:,\d{3,5})*(?:\.\d{1,2})?) +(-?[\d$]\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{2,3})*(?:\.\d{1,2})?) +(\d{1,3}(?:\.\d{1,2})?) +(\d+) +(\d{1,3}\.?\d{1,2}) +(-?\$\d{1,3}(?:,,)*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)`

	periodLine, e := regexp.Compile(period)
	if e != nil {
		return
	}
	validLine, e := regexp.Compile(pattern) // error if regexp invalid
	if e != nil {
		return
	}

	periods := periodLine.FindSubmatch(raw)
	CurrentPeriod, e1 := time.Parse("January 2006", string(periods[1]))
	if e1 != nil {
		log.Warn("failed to parse period of resimac PDF", e1)
	}

	matches := validLine.FindAllSubmatch(raw, -1)
	if matches == nil {
		log.Warn("Resimac PDF decode found nothing no matches", raw)
		return
	}

	for _, v := range matches {
		rp := ResimacPdf{Period: CurrentPeriod}
		rp.LoanNumber = string(v[1])
		rp.Borrower = string(v[2])
		rp.LoanAmount = m.currencyToFloat64(string(v[3]))
		rp.InterestRate = m.currencyToFloat64(string(v[4]))
		rp.DelvRate = m.currencyToFloat64(string(v[5]))
		rp.Margin = m.currencyToFloat64(string(v[6]))
		rp.Settlement, _ = time.Parse("02/01/2006", string(v[7]))
		rp.Balance = m.currencyToFloat64(string(v[8]))
		rp.OffsetBalance = m.currencyToFloat64(string(v[9]))
		rp.GrossManFee = m.currencyToFloat64(string(v[10]))
		rp.IncentiveP = m.currencyToFloat64(string(v[11]))
		rp.Months, _ = strconv.Atoi(string(v[12]))
		rp.SacrificeP = m.currencyToFloat64(string(v[13]))
		rp.Fee = m.currencyToFloat64(string(v[14]))
		rp.MoFee = m.currencyToFloat64(string(v[15]))
		rp.NetManFee = m.currencyToFloat64(string(v[16]))
		rp.IncomeAmount = rp.NetManFee
		m.ResimacPdf = append(m.ResimacPdf, rp)
	}
	return
}