package main import ( "fmt" log "github.com/sirupsen/logrus" "regexp" "strconv" "strings" "time" ) var sample_resimac = ` TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM 335,8779,1,A ,Huang ,113423,3.79,3.36,0.43,02/08/2019,81865.89,29.73,0.6,60,0.18,-12.45,0,17.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,2,A ,Ding ,321030,3.72,3.16,0.56,12/10/2018,311077.1,143.25,1,60,0.28,-71.62,0,71.63,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,4,A ,Song ,640000,3.69,3.29,0.4,01/23/2019,607519.14,198.79,0.8,60,0.23,-114.3,0,84.49,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,4,B ,Song ,208000,3.69,3.29,0.4,01/23/2019,208055,68.4,0.8,60,0.23,-39.33,0,29.07,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 337,8779,7,A ,Gill ,404376,3.79,3.26,0.53,02/25/2019,393745.79,171.5,1,60,0.28,-90.6,0,80.9,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006 507,8779,9,A ,Lu ,980000,4.6,4.07,0.53,06/18/2019,980000,426.9,1,60,0.28,-225.53,0,201.37,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 335,8779,10,A ,Sideris ,556000,3.32,2.79,0.53,04/23/2019,543091.68,236.7,1,60,0.28,-125.05,0,111.65,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 507,8779,13,A ,Lu ,570000,4.6,4.07,0.53,06/18/2019,50000,21.78,1,60,0.28,-11.51,0,10.27,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 507,8779,14,A ,Lu ,970000,4.15,3.62,0.53,06/18/2019,954336.82,416.06,1,60,0.28,-219.8,0,196.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 337,8779,18,A ,Chen ,444000,3.82,3.29,0.53,06/18/2019,444000,193.41,1,60,0.28,-102.18,0,91.23,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 522,8779,19,A ,Horm ,944000,4.7,4.17,0.53,07/24/2019,544000,295.06,1,60,0.28,-155.88,0,139.18,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 522,8779,21,A ,Kemp ,479952,4.25,3.72,0.53,07/24/2019,470613.4,204.94,1,60,0.28,-108.27,0,96.67,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 522,8779,24,A ,Tiang ,684481,4.7,4.17,0.53,08/23/2019,668000,290.99,1,60,0.28,-153.73,0,137.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,26,A ,Li ,564000,4.25,3.72,0.53,09/26/2019,557190.2,243,1,60,0.28,-128.38,0,114.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,27,A ,Quach ,406000,4.4,3.87,0.53,10/21/2019,398603.27,174.39,1,60,0.28,-92.13,0,82.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,27,B ,Quach ,406000,4.4,3.87,0.53,10/21/2019,406000,176.86,1,60,0.28,-93.44,0,83.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 502,8779,28,A ,Zhu ,1000000,4.55,4.02,0.53,09/18/2019,988561.45,430.95,1,60,0.28,-227.67,0,203.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 307,8779,30,A ,Huynh ,410453,3.22,2.69,0.53,09/19/2019,394606.79,172.23,1,60,0.28,-90.99,0,81.24,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,31,A ,Xu ,830000,4.15,3.62,0.53,01/30/2020,824329.48,359.53,1,60,0.28,-189.94,0,169.59,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 305,8779,32,A ,Tran ,476000,3.52,2.99,0.53,10/03/2019,470274.11,204.88,1,60,0.28,-108.24,0,96.64,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,37,A ,Zulfaqari ,552000,3.22,2.69,0.53,12/19/2019,543482.81,237.12,1,60,0.28,-125.27,0,111.85,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,38,A ,Pham ,512000,3.22,2.69,0.53,01/10/2020,507925.37,221.37,1,60,0.28,-116.95,0,104.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 310,8779,42,A ,Nguyen ,318542,3.52,2.99,0.53,12/06/2019,311782.42,135.71,1,60,0.28,-71.7,0,64.01,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,43,A ,Hopkins ,427505,3.52,2.99,0.53,01/08/2020,415112.03,180.87,1,60,0.28,-95.55,0,85.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,43,B ,Hopkins ,41567,3.52,2.99,0.53,01/08/2020,40011.29,17.43,1,60,0.28,-9.21,0,8.22,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 334,8779,43,C ,Hopkins ,10292,3.52,2.99,0.53,01/08/2020,9991.49,4.35,1,60,0.28,-2.3,0,2.05,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,45,A ,Tran ,408000,3.22,2.69,0.53,12/06/2019,403381.11,175.81,1,60,0.28,-92.88,0,82.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,47,A ,Lebon ,100000,3.32,2.79,0.53,12/09/2019,44000,20.14,1,60,0.28,-10.64,0,9.5,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 337,8779,48,A ,Le ,400000,3.22,2.69,0.53,12/03/2019,396171.84,172.6,1,60,0.28,-91.18,0,81.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,49,A ,Vu ,509600,3.22,2.69,0.53,12/13/2019,504687.98,220.02,1,60,0.28,-116.24,0,103.78,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,50,A ,Wang ,1500000,4.15,3.62,0.53,01/15/2020,1387663.97,604.28,1,60,0.28,-319.24,0,285.04,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,52,A ,Le ,520000,3.32,2.79,0.53,01/20/2020,513359.23,224.03,1,60,0.28,-118.36,0,105.67,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,54,A ,Nguyen ,268000,3.32,2.79,0.53,01/08/2020,265878.84,115.87,1,60,0.28,-61.21,0,54.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,56,A ,Hanna ,250000,3.22,2.69,0.53,01/09/2020,245368.96,107,1,60,0.28,-56.53,0,50.47,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 308,8779,57,A ,Le ,300000,3.22,2.69,0.53,01/08/2020,293231.8,127.97,1,60,0.28,-67.61,0,60.36,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006 309,8779,58,A ,Le ,432000,3.22,2.69,0.53,12/17/2019,427824.14,186.53,1,60,0.28,-98.54,0,87.99,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,63,A ,Nguyen ,449175,3.22,2.69,0.53,12/23/2019,444814.79,194,1,60,0.28,-102.49,0,91.51,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,64,A ,Leav ,444000,3.42,2.89,0.53,01/07/2020,443334.6,193.23,1,60,0.28,-102.08,0,91.15,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006 308,8779,66,A ,Nguyen ,516750,3.22,2.62,0.6,01/21/2020,511192.69,252.48,1,60,0.28,-117.82,0,134.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,69,A ,Leab ,435000,4.14,3.61,0.53,02/10/2020,432580.51,188.52,1,60,0.28,-99.59,0,88.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,71,A ,Yang ,511000,3.22,2.62,0.6,02/07/2020,506610.13,249.98,1,60,0.28,-116.66,0,133.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,76,A ,Huynh ,442060,3.22,2.62,0.6,02/28/2020,438854.39,216.79,1,60,0.28,-101.17,0,115.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 308,8779,76,B ,Huynh ,77940,3.22,2.62,0.6,02/28/2020,77374.82,38.22,1,60,0.28,-17.84,0,20.38,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 500,8779,82,A ,Pham ,788000,4.7,4.17,0.53,05/12/2020,791145.52,344.13,1,60,0.28,-181.81,0,162.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006 ` type ResimacRow struct { TRSTCD int ORGNTR int LOANNO int PORTN string BRNAMX string //Brower LNAMT float64 //Loan amount INTRTE float64 DELRTE float64 MARGIN float64 SETLDX time.Time //settled date LNBAL float64 MANFEE float64 RSINCP float64 RSSACT float64 RSSACP float64 SACAMT float64 MOFEE float64 MANTOT float64 NEWLON float64 REFADJ float64 NETNEW float64 FIXED string LNFORT string ORGNAM string LNEOM time.Time //Derived attributes LoanNumber string Borrower string LoanAmount float64 Balance float64 IncomeAmount float64 } type ResimacPdf struct { Period time.Time LoanNumber string Borrower string LoanAmount float64 InterestRate float64 DelvRate float64 Margin float64 Settlement time.Time Balance float64 OffsetBalance float64 GrossManFee float64 IncentiveP float64 Months int SacrificeP float64 Fee float64 MoFee float64 NetManFee float64 IncomeAmount float64 } func (m *AiDecodeIncome) isResimacXls(raw string) bool { keyword := "TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM" lines := strings.Split(raw, "\n") //remove all spaces for i := 0; i < len(lines); i++ { lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces } return m.isKeywordExist(keyword, lines) } func (m *AiDecodeIncome) isResimacPdf(raw string) bool { keyword := "SuperFinanceMarketsPtyLtd-8779" lines := strings.Split(raw, "\n") //remove all spaces for i := 0; i < len(lines); i++ { lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces } return m.isKeywordExist(keyword, lines) } func (m *AiDecodeIncome) decodeResimacXls(raw []byte) (e error) { m.ResimacXls = make([]ResimacRow, 0, 100) lines := strings.Split(string(raw), "\n") foundHeader := false var idx map[string]int for i := 0; i < len(lines); i++ { lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces el := strings.Split(lines[i], ",") if len(el) < 25 { continue //bypass } if !foundHeader { foundHeader, idx = m.decodeResimacXlsHeader(lines[i]) continue } //we have already found header r := ResimacRow{} r.TRSTCD, _ = strconv.Atoi(el[idx["TRSTCD"]]) r.ORGNTR, _ = strconv.Atoi(el[1]) r.LOANNO, _ = strconv.Atoi(el[2]) r.PORTN = el[3] r.LoanNumber = fmt.Sprintf("%d-%06d-%06d%s", r.TRSTCD, r.ORGNTR, r.LOANNO, r.PORTN) r.BRNAMX = el[4] r.Borrower = r.BRNAMX r.LNAMT = m.currencyToFloat64(el[5]) r.LoanAmount = r.LNAMT r.INTRTE = m.currencyToFloat64(el[6]) r.DELRTE = m.currencyToFloat64(el[7]) r.MARGIN = m.currencyToFloat64(el[8]) r.SETLDX, _ = time.Parse("1/02/2006", el[9]) r.LNBAL = m.currencyToFloat64(el[10]) r.Balance = r.LNBAL r.MANFEE = m.currencyToFloat64(el[11]) r.RSINCP = m.currencyToFloat64(el[12]) r.RSSACT = m.currencyToFloat64(el[13]) r.RSSACP = m.currencyToFloat64(el[14]) r.SACAMT = m.currencyToFloat64(el[15]) r.MOFEE = m.currencyToFloat64(el[16]) r.MANTOT = m.currencyToFloat64(el[17]) r.IncomeAmount = r.MANTOT r.NEWLON = m.currencyToFloat64(el[18]) r.REFADJ = m.currencyToFloat64(el[19]) r.NETNEW = m.currencyToFloat64(el[20]) r.FIXED = el[21] r.LNFORT = el[22] r.ORGNAM = strings.TrimSpace(el[23]) r.LNEOM, _ = time.Parse("200601", el[24]) m.ResimacXls = append(m.ResimacXls, r) } return } func (m *AiDecodeIncome) decodeResimacXlsHeader(line string) (isHeaderLine bool, idx map[string]int) { keys := []string{ "TRSTCD", "ORGNTR", "LOANNO", "PORTNO", "BRNAMX", "LNAMT", "INTRTE", "DELRTE", "MARGIN", "SETLDX", "LNBAL", "MANFEE", "RSINCP", "RSSACT", "RSSACP", "SACAMT", "MOFEE", "MANTOT", "NEWLON", "REFADJ", "NETNEW", "FIXED", "LNFORT", "ORGNAM", "LNEOM"} idx = make(map[string]int) l := strings.ReplaceAll(line, " ", "") // remove space el := strings.Split(l, ",") found := 0 for i := 0; i < len(el); i++ { a := strings.TrimSpace(el[i]) a = strings.ToUpper(a) for _, k := range keys { if a == k { idx[k] = i found++ break } } } isHeaderLine = found > 20 // if we found more than 20 headers return } func (m *AiDecodeIncome) decodeResimacPdf(raw []byte) (e error) { m.ResimacPdf = make([]ResimacPdf, 0, 30) period := `(?i)Trailer +Fees Report[ :]+((?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)[\t ]+[0-9]{4})` pattern := `(?i)(\d+-0*8779-[\d A-Za-z-]+[A-Za-z]) +([A-Za-z]+) +(\$\d{1,3}(?:,\d{3})*(?:[\d.]\d{2})?) +(\d{1,3}(?:\.\d{1,2})?) +([\d ]{1,3}(?:[ .]\d{1,2})?) +(\d{1,3}(?:[\d.]\d{1,2})?) +((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}) +(-?\$\d{1,3}(?:,\d{3,5})*(?:\.\d{1,2})?) +(-?[\d$]\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{2,3})*(?:\.\d{1,2})?) +(\d{1,3}(?:\.\d{1,2})?) +(\d+) +(\d{1,3}\.?\d{1,2}) +(-?\$\d{1,3}(?:,,)*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)` periodLine, e := regexp.Compile(period) if e != nil { return } validLine, e := regexp.Compile(pattern) // error if regexp invalid if e != nil { return } periods := periodLine.FindSubmatch(raw) CurrentPeriod, e1 := time.Parse("January 2006", string(periods[1])) if e1 != nil { log.Warn("failed to parse period of resimac PDF", e1) } matches := validLine.FindAllSubmatch(raw, -1) if matches == nil { log.Warn("Resimac PDF decode found nothing no matches", raw) return } for _, v := range matches { rp := ResimacPdf{Period: CurrentPeriod} rp.LoanNumber = string(v[1]) rp.Borrower = string(v[2]) rp.LoanAmount = m.currencyToFloat64(string(v[3])) rp.InterestRate = m.currencyToFloat64(string(v[4])) rp.DelvRate = m.currencyToFloat64(string(v[5])) rp.Margin = m.currencyToFloat64(string(v[6])) rp.Settlement, _ = time.Parse("02/01/2006", string(v[7])) rp.Balance = m.currencyToFloat64(string(v[8])) rp.OffsetBalance = m.currencyToFloat64(string(v[9])) rp.GrossManFee = m.currencyToFloat64(string(v[10])) rp.IncentiveP = m.currencyToFloat64(string(v[11])) rp.Months, _ = strconv.Atoi(string(v[12])) rp.SacrificeP = m.currencyToFloat64(string(v[13])) rp.Fee = m.currencyToFloat64(string(v[14])) rp.MoFee = m.currencyToFloat64(string(v[15])) rp.NetManFee = m.currencyToFloat64(string(v[16])) rp.IncomeAmount = rp.NetManFee m.ResimacPdf = append(m.ResimacPdf, rp) } return }