|
|
|
@@ -0,0 +1,244 @@ |
|
|
|
package main |
|
|
|
|
|
|
|
import ( |
|
|
|
"fmt" |
|
|
|
log "github.com/sirupsen/logrus" |
|
|
|
"regexp" |
|
|
|
"strconv" |
|
|
|
"strings" |
|
|
|
"time" |
|
|
|
) |
|
|
|
|
|
|
|
var sample_resimac = ` |
|
|
|
TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM |
|
|
|
335,8779,1,A ,Huang ,113423,3.79,3.36,0.43,02/08/2019,81865.89,29.73,0.6,60,0.18,-12.45,0,17.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,2,A ,Ding ,321030,3.72,3.16,0.56,12/10/2018,311077.1,143.25,1,60,0.28,-71.62,0,71.63,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,4,A ,Song ,640000,3.69,3.29,0.4,01/23/2019,607519.14,198.79,0.8,60,0.23,-114.3,0,84.49,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,4,B ,Song ,208000,3.69,3.29,0.4,01/23/2019,208055,68.4,0.8,60,0.23,-39.33,0,29.07,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
337,8779,7,A ,Gill ,404376,3.79,3.26,0.53,02/25/2019,393745.79,171.5,1,60,0.28,-90.6,0,80.9,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006 |
|
|
|
507,8779,9,A ,Lu ,980000,4.6,4.07,0.53,06/18/2019,980000,426.9,1,60,0.28,-225.53,0,201.37,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
335,8779,10,A ,Sideris ,556000,3.32,2.79,0.53,04/23/2019,543091.68,236.7,1,60,0.28,-125.05,0,111.65,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 |
|
|
|
507,8779,13,A ,Lu ,570000,4.6,4.07,0.53,06/18/2019,50000,21.78,1,60,0.28,-11.51,0,10.27,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
507,8779,14,A ,Lu ,970000,4.15,3.62,0.53,06/18/2019,954336.82,416.06,1,60,0.28,-219.8,0,196.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
337,8779,18,A ,Chen ,444000,3.82,3.29,0.53,06/18/2019,444000,193.41,1,60,0.28,-102.18,0,91.23,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
522,8779,19,A ,Horm ,944000,4.7,4.17,0.53,07/24/2019,544000,295.06,1,60,0.28,-155.88,0,139.18,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
522,8779,21,A ,Kemp ,479952,4.25,3.72,0.53,07/24/2019,470613.4,204.94,1,60,0.28,-108.27,0,96.67,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 |
|
|
|
522,8779,24,A ,Tiang ,684481,4.7,4.17,0.53,08/23/2019,668000,290.99,1,60,0.28,-153.73,0,137.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,26,A ,Li ,564000,4.25,3.72,0.53,09/26/2019,557190.2,243,1,60,0.28,-128.38,0,114.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,27,A ,Quach ,406000,4.4,3.87,0.53,10/21/2019,398603.27,174.39,1,60,0.28,-92.13,0,82.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,27,B ,Quach ,406000,4.4,3.87,0.53,10/21/2019,406000,176.86,1,60,0.28,-93.44,0,83.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
502,8779,28,A ,Zhu ,1000000,4.55,4.02,0.53,09/18/2019,988561.45,430.95,1,60,0.28,-227.67,0,203.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
307,8779,30,A ,Huynh ,410453,3.22,2.69,0.53,09/19/2019,394606.79,172.23,1,60,0.28,-90.99,0,81.24,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,31,A ,Xu ,830000,4.15,3.62,0.53,01/30/2020,824329.48,359.53,1,60,0.28,-189.94,0,169.59,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
305,8779,32,A ,Tran ,476000,3.52,2.99,0.53,10/03/2019,470274.11,204.88,1,60,0.28,-108.24,0,96.64,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,37,A ,Zulfaqari ,552000,3.22,2.69,0.53,12/19/2019,543482.81,237.12,1,60,0.28,-125.27,0,111.85,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,38,A ,Pham ,512000,3.22,2.69,0.53,01/10/2020,507925.37,221.37,1,60,0.28,-116.95,0,104.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
310,8779,42,A ,Nguyen ,318542,3.52,2.99,0.53,12/06/2019,311782.42,135.71,1,60,0.28,-71.7,0,64.01,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,43,A ,Hopkins ,427505,3.52,2.99,0.53,01/08/2020,415112.03,180.87,1,60,0.28,-95.55,0,85.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,43,B ,Hopkins ,41567,3.52,2.99,0.53,01/08/2020,40011.29,17.43,1,60,0.28,-9.21,0,8.22,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
334,8779,43,C ,Hopkins ,10292,3.52,2.99,0.53,01/08/2020,9991.49,4.35,1,60,0.28,-2.3,0,2.05,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,45,A ,Tran ,408000,3.22,2.69,0.53,12/06/2019,403381.11,175.81,1,60,0.28,-92.88,0,82.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,47,A ,Lebon ,100000,3.32,2.79,0.53,12/09/2019,44000,20.14,1,60,0.28,-10.64,0,9.5,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
337,8779,48,A ,Le ,400000,3.22,2.69,0.53,12/03/2019,396171.84,172.6,1,60,0.28,-91.18,0,81.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,49,A ,Vu ,509600,3.22,2.69,0.53,12/13/2019,504687.98,220.02,1,60,0.28,-116.24,0,103.78,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,50,A ,Wang ,1500000,4.15,3.62,0.53,01/15/2020,1387663.97,604.28,1,60,0.28,-319.24,0,285.04,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,52,A ,Le ,520000,3.32,2.79,0.53,01/20/2020,513359.23,224.03,1,60,0.28,-118.36,0,105.67,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,54,A ,Nguyen ,268000,3.32,2.79,0.53,01/08/2020,265878.84,115.87,1,60,0.28,-61.21,0,54.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,56,A ,Hanna ,250000,3.22,2.69,0.53,01/09/2020,245368.96,107,1,60,0.28,-56.53,0,50.47,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,57,A ,Le ,300000,3.22,2.69,0.53,01/08/2020,293231.8,127.97,1,60,0.28,-67.61,0,60.36,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 |
|
|
|
309,8779,58,A ,Le ,432000,3.22,2.69,0.53,12/17/2019,427824.14,186.53,1,60,0.28,-98.54,0,87.99,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,63,A ,Nguyen ,449175,3.22,2.69,0.53,12/23/2019,444814.79,194,1,60,0.28,-102.49,0,91.51,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,64,A ,Leav ,444000,3.42,2.89,0.53,01/07/2020,443334.6,193.23,1,60,0.28,-102.08,0,91.15,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,66,A ,Nguyen ,516750,3.22,2.62,0.6,01/21/2020,511192.69,252.48,1,60,0.28,-117.82,0,134.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,69,A ,Leab ,435000,4.14,3.61,0.53,02/10/2020,432580.51,188.52,1,60,0.28,-99.59,0,88.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,71,A ,Yang ,511000,3.22,2.62,0.6,02/07/2020,506610.13,249.98,1,60,0.28,-116.66,0,133.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,76,A ,Huynh ,442060,3.22,2.62,0.6,02/28/2020,438854.39,216.79,1,60,0.28,-101.17,0,115.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
308,8779,76,B ,Huynh ,77940,3.22,2.62,0.6,02/28/2020,77374.82,38.22,1,60,0.28,-17.84,0,20.38,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
500,8779,82,A ,Pham ,788000,4.7,4.17,0.53,05/12/2020,791145.52,344.13,1,60,0.28,-181.81,0,162.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 |
|
|
|
` |
|
|
|
|
|
|
|
type ResimacRow struct { |
|
|
|
TRSTCD int |
|
|
|
ORGNTR int |
|
|
|
LOANNO int |
|
|
|
PORTN string |
|
|
|
BRNAMX string //Brower |
|
|
|
LNAMT float64 //Loan amount |
|
|
|
INTRTE float64 |
|
|
|
DELRTE float64 |
|
|
|
MARGIN float64 |
|
|
|
SETLDX time.Time //settled date |
|
|
|
LNBAL float64 |
|
|
|
MANFEE float64 |
|
|
|
RSINCP float64 |
|
|
|
RSSACT float64 |
|
|
|
RSSACP float64 |
|
|
|
SACAMT float64 |
|
|
|
MOFEE float64 |
|
|
|
MANTOT float64 |
|
|
|
NEWLON float64 |
|
|
|
REFADJ float64 |
|
|
|
NETNEW float64 |
|
|
|
FIXED string |
|
|
|
LNFORT string |
|
|
|
ORGNAM string |
|
|
|
LNEOM time.Time |
|
|
|
|
|
|
|
//Derived attributes |
|
|
|
LoanNumber string |
|
|
|
Borrower string |
|
|
|
LoanAmount float64 |
|
|
|
Balance float64 |
|
|
|
IncomeAmount float64 |
|
|
|
} |
|
|
|
|
|
|
|
type ResimacPdf struct { |
|
|
|
Period time.Time |
|
|
|
LoanNumber string |
|
|
|
Borrower string |
|
|
|
LoanAmount float64 |
|
|
|
InterestRate float64 |
|
|
|
DelvRate float64 |
|
|
|
Margin float64 |
|
|
|
Settlement time.Time |
|
|
|
Balance float64 |
|
|
|
OffsetBalance float64 |
|
|
|
GrossManFee float64 |
|
|
|
IncentiveP float64 |
|
|
|
Months int |
|
|
|
SacrificeP float64 |
|
|
|
Fee float64 |
|
|
|
MoFee float64 |
|
|
|
NetManFee float64 |
|
|
|
IncomeAmount float64 |
|
|
|
} |
|
|
|
|
|
|
|
func (m *AiDecodeIncome) decodeResimacXls(raw []byte) (e error) { |
|
|
|
m.ResimacXls = make([]ResimacRow, 0, 100) |
|
|
|
lines := strings.Split(string(raw), "\n") |
|
|
|
foundHeader := false |
|
|
|
var idx map[string]int |
|
|
|
for i := 0; i < len(lines); i++ { |
|
|
|
lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces |
|
|
|
el := strings.Split(lines[i], ",") |
|
|
|
if len(el) < 25 { |
|
|
|
continue //bypass |
|
|
|
} |
|
|
|
|
|
|
|
if !foundHeader { |
|
|
|
foundHeader, idx = m.decodeResimacXlsHeader(lines[i]) |
|
|
|
continue |
|
|
|
} |
|
|
|
//we have already found header |
|
|
|
r := ResimacRow{} |
|
|
|
|
|
|
|
r.TRSTCD, _ = strconv.Atoi(el[idx["TRSTCD"]]) |
|
|
|
r.ORGNTR, _ = strconv.Atoi(el[1]) |
|
|
|
r.LOANNO, _ = strconv.Atoi(el[2]) |
|
|
|
r.PORTN = el[3] |
|
|
|
r.LoanNumber = fmt.Sprintf("%d-%06d-%06d%s", r.TRSTCD, r.ORGNTR, r.LOANNO, r.PORTN) |
|
|
|
r.BRNAMX = el[4] |
|
|
|
r.Borrower = r.BRNAMX |
|
|
|
r.LNAMT = m.currencyToFloat64(el[5]) |
|
|
|
r.LoanAmount = r.LNAMT |
|
|
|
r.INTRTE = m.currencyToFloat64(el[6]) |
|
|
|
r.DELRTE = m.currencyToFloat64(el[7]) |
|
|
|
r.MARGIN = m.currencyToFloat64(el[8]) |
|
|
|
r.SETLDX, _ = time.Parse("1/02/2006", el[9]) |
|
|
|
r.LNBAL = m.currencyToFloat64(el[10]) |
|
|
|
r.Balance = r.LNBAL |
|
|
|
r.MANFEE = m.currencyToFloat64(el[11]) |
|
|
|
r.RSINCP = m.currencyToFloat64(el[12]) |
|
|
|
r.RSSACT = m.currencyToFloat64(el[13]) |
|
|
|
r.RSSACP = m.currencyToFloat64(el[14]) |
|
|
|
r.SACAMT = m.currencyToFloat64(el[15]) |
|
|
|
r.MOFEE = m.currencyToFloat64(el[16]) |
|
|
|
r.MANTOT = m.currencyToFloat64(el[17]) |
|
|
|
r.IncomeAmount = r.MANTOT |
|
|
|
r.NEWLON = m.currencyToFloat64(el[18]) |
|
|
|
r.REFADJ = m.currencyToFloat64(el[19]) |
|
|
|
r.NETNEW = m.currencyToFloat64(el[20]) |
|
|
|
r.FIXED = el[21] |
|
|
|
r.LNFORT = el[22] |
|
|
|
r.ORGNAM = strings.TrimSpace(el[23]) |
|
|
|
r.LNEOM, _ = time.Parse("200601", el[24]) |
|
|
|
m.ResimacXls = append(m.ResimacXls, r) |
|
|
|
} |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
func (m *AiDecodeIncome) decodeResimacXlsHeader(line string) (isHeaderLine bool, idx map[string]int) { |
|
|
|
keys := []string{ |
|
|
|
"TRSTCD", "ORGNTR", "LOANNO", "PORTNO", "BRNAMX", "LNAMT", "INTRTE", "DELRTE", |
|
|
|
"MARGIN", "SETLDX", "LNBAL", "MANFEE", "RSINCP", "RSSACT", "RSSACP", "SACAMT", |
|
|
|
"MOFEE", "MANTOT", "NEWLON", "REFADJ", "NETNEW", "FIXED", "LNFORT", "ORGNAM", |
|
|
|
"LNEOM"} |
|
|
|
|
|
|
|
idx = make(map[string]int) |
|
|
|
l := strings.ReplaceAll(line, " ", "") // remove space |
|
|
|
el := strings.Split(l, ",") |
|
|
|
found := 0 |
|
|
|
for i := 0; i < len(el); i++ { |
|
|
|
a := strings.TrimSpace(el[i]) |
|
|
|
a = strings.ToUpper(a) |
|
|
|
for _, k := range keys { |
|
|
|
if a == k { |
|
|
|
idx[k] = i |
|
|
|
found++ |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
isHeaderLine = found > 20 // if we found more than 20 headers |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
func (m *AiDecodeIncome) decodeResimacPdf(raw []byte) (e error) { |
|
|
|
m.ResimacPdf = make([]ResimacPdf, 0, 30) |
|
|
|
period := `(?i)Trailer +Fees Report[ :]+((?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)[\t ]+[0-9]{4})` |
|
|
|
pattern := `(?i)(\d+-0*8779-[\d A-Za-z-]+[A-Za-z]) +([A-Za-z]+) +(\$\d{1,3}(?:,\d{3})*(?:[\d.]\d{2})?) +(\d{1,3}(?:\.\d{1,2})?) +([\d ]{1,3}(?:[ .]\d{1,2})?) +(\d{1,3}(?:[\d.]\d{1,2})?) +((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}) +(-?\$\d{1,3}(?:,\d{3,5})*(?:\.\d{1,2})?) +(-?[\d$]\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{2,3})*(?:\.\d{1,2})?) +(\d{1,3}(?:\.\d{1,2})?) +(\d+) +(\d{1,3}\.?\d{1,2}) +(-?\$\d{1,3}(?:,,)*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)` |
|
|
|
|
|
|
|
periodLine, e := regexp.Compile(period) |
|
|
|
if e != nil { |
|
|
|
return |
|
|
|
} |
|
|
|
validLine, e := regexp.Compile(pattern) // error if regexp invalid |
|
|
|
if e != nil { |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
periods := periodLine.FindSubmatch(raw) |
|
|
|
CurrentPeriod, e1 := time.Parse("January 2006", string(periods[1])) |
|
|
|
if e1 != nil { |
|
|
|
log.Warn("failed to parse period of resimac PDF", e1) |
|
|
|
} |
|
|
|
|
|
|
|
matches := validLine.FindAllSubmatch(raw, -1) |
|
|
|
if matches == nil { |
|
|
|
log.Warn("Resimac PDF decode found nothing no matches", raw) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
for _, v := range matches { |
|
|
|
rp := ResimacPdf{Period: CurrentPeriod} |
|
|
|
rp.LoanNumber = string(v[1]) |
|
|
|
rp.Borrower = string(v[2]) |
|
|
|
rp.LoanAmount = m.currencyToFloat64(string(v[3])) |
|
|
|
rp.InterestRate = m.currencyToFloat64(string(v[4])) |
|
|
|
rp.DelvRate = m.currencyToFloat64(string(v[5])) |
|
|
|
rp.Margin = m.currencyToFloat64(string(v[6])) |
|
|
|
rp.Settlement, _ = time.Parse("02/01/2006", string(v[7])) |
|
|
|
rp.Balance = m.currencyToFloat64(string(v[8])) |
|
|
|
rp.OffsetBalance = m.currencyToFloat64(string(v[9])) |
|
|
|
rp.GrossManFee = m.currencyToFloat64(string(v[10])) |
|
|
|
rp.IncentiveP = m.currencyToFloat64(string(v[11])) |
|
|
|
rp.Months, _ = strconv.Atoi(string(v[12])) |
|
|
|
rp.SacrificeP = m.currencyToFloat64(string(v[13])) |
|
|
|
rp.Fee = m.currencyToFloat64(string(v[14])) |
|
|
|
rp.MoFee = m.currencyToFloat64(string(v[15])) |
|
|
|
rp.NetManFee = m.currencyToFloat64(string(v[16])) |
|
|
|
rp.IncomeAmount = rp.NetManFee |
|
|
|
m.ResimacPdf = append(m.ResimacPdf, rp) |
|
|
|
} |
|
|
|
return |
|
|
|
} |