Bläddra i källkod

resimac analysis is done

master
sp 4 år sedan
förälder
incheckning
6e17e48adb
3 ändrade filer med 304 tillägg och 8 borttagningar
  1. +11
    -3
      UploadsOnDisk.go
  2. +49
    -5
      pay-in-decode.go
  3. +244
    -0
      pay-in-resimac.go

+ 11
- 3
UploadsOnDisk.go Visa fil

@@ -18,6 +18,7 @@ import (

type uploadsOnDisk struct {
Upload loan.Uploads
tmp []byte //temporary buffer for file conversion such as csv
}

func (m *uploadsOnDisk) convertUploadsToPDF() (e error) {
@@ -124,7 +125,7 @@ func (m *uploadsOnDisk) convertExcelToJpg() (e error) {

var libreOfficeMutex sync.Mutex // make sure we only have one libreoffice running at a time
func (m *uploadsOnDisk) convertExcelTo(format string) (e error) {
if format != "pdf" && format != "jpg" {
if format != "pdf" && format != "jpg" && format != "csv" {
e = errors.New("convert excel to unsupported format " + format)
return
}
@@ -144,7 +145,7 @@ func (m *uploadsOnDisk) convertExcelTo(format string) (e error) {
libreOfficeMutex.Lock() //ensure only one libreoffice is running

// Create a new context and add a timeout to it
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel() // The cancel should be deferred so resources are cleaned up

// Create the command with our context
@@ -203,8 +204,15 @@ func (m *uploadsOnDisk) convertExcelTo(format string) (e error) {
_, name := filepath.Split(m.filePath())

src := dir + string(os.PathSeparator) + fileNameWithoutExtTrimSuffix(name) + "." + format
e = os.Rename(src, dst) // there should be only one jpg

switch format {
case "pdf", "jpg":
e = os.Rename(src, dst) // there should be only one jpg
break
case "csv":
m.tmp, e = ioutil.ReadFile(src)
break
}
return
}


+ 49
- 5
pay-in-decode.go Visa fil

@@ -24,6 +24,8 @@ type DecodeIncomeDetails struct {
Lender loan.LenderType
AAA []PayInAAARow
Connective []ConnectiveRow
ResimacXls []ResimacRow
ResimacPdf []ResimacPdf
}

type AiDecodeJson struct {
@@ -141,6 +143,10 @@ func (m *AiDecodeIncome) decodePdf() (e error) {
m.Lender = loan.Lender_Connective
e = m.decodeConnectivePdf(out)
break
case loan.Lender_Resimac:
m.Lender = loan.Lender_Resimac
e = m.decodeResimacPdf(out)
break
case loan.Lender_Unknown:
e = errors.New(loan.Lender_Unknown)
break // not able to detect Funder
@@ -149,7 +155,20 @@ func (m *AiDecodeIncome) decodePdf() (e error) {
}

func (m *AiDecodeIncome) decodeXls() (e error) {
e = errors.New("not implemented yet")
e = m.ul.convertExcelTo("csv")
if e != nil {
log.Error("cannot convert xls to csv ", e.Error())
}
raw := string(m.ul.tmp)
switch m.detectFunder(raw) {
case loan.Lender_Resimac:
m.Lender = loan.Lender_Resimac
e = m.decodeResimacXls(m.ul.tmp)
break
case loan.Lender_Unknown:
e = errors.New(loan.Lender_Unknown)
break // not able to detect Funder
}
return
}

@@ -160,24 +179,49 @@ func (m *AiDecodeIncome) detectFunder(raw string) loan.LenderType {
if m.isConnective(raw) {
return loan.Lender_Connective
}
if m.isResimacXls(raw) {
return loan.Lender_Resimac
}
if m.isResimacPdf(raw) {
return loan.Lender_Resimac
}
return loan.Lender_Unknown
}

func (m *AiDecodeIncome) isAAA(raw string) bool {
keyword := "AAA Financial Trail Report"
lines := strings.Split(raw, "\n")
return m.checkFunderKeyword(keyword, lines, 0, 3)
return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isConnective(raw string) bool {
keyword := "connective.com.au"
lines := strings.Split(raw, "\n")
return m.checkFunderKeyword(keyword, lines, 0, 3)
return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isResimacXls(raw string) bool {
keyword := "TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM"
lines := strings.Split(raw, "\n")
//remove all spaces
for i := 0; i < len(lines); i++ {
lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
}
return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isResimacPdf(raw string) bool {
keyword := "SuperFinanceMarketsPtyLtd-8779"
lines := strings.Split(raw, "\n")
//remove all spaces
for i := 0; i < len(lines); i++ {
lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
}
return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool {
func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string) bool {
for _, line := range lines {
// as long as it has it
if strings.Contains(line, keyword) {
return true
}

+ 244
- 0
pay-in-resimac.go Visa fil

@@ -0,0 +1,244 @@
package main

import (
"fmt"
log "github.com/sirupsen/logrus"
"regexp"
"strconv"
"strings"
"time"
)

var sample_resimac = `
TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM
335,8779,1,A ,Huang ,113423,3.79,3.36,0.43,02/08/2019,81865.89,29.73,0.6,60,0.18,-12.45,0,17.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,2,A ,Ding ,321030,3.72,3.16,0.56,12/10/2018,311077.1,143.25,1,60,0.28,-71.62,0,71.63,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,4,A ,Song ,640000,3.69,3.29,0.4,01/23/2019,607519.14,198.79,0.8,60,0.23,-114.3,0,84.49,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,4,B ,Song ,208000,3.69,3.29,0.4,01/23/2019,208055,68.4,0.8,60,0.23,-39.33,0,29.07,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
337,8779,7,A ,Gill ,404376,3.79,3.26,0.53,02/25/2019,393745.79,171.5,1,60,0.28,-90.6,0,80.9,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006
507,8779,9,A ,Lu ,980000,4.6,4.07,0.53,06/18/2019,980000,426.9,1,60,0.28,-225.53,0,201.37,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
335,8779,10,A ,Sideris ,556000,3.32,2.79,0.53,04/23/2019,543091.68,236.7,1,60,0.28,-125.05,0,111.65,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006
507,8779,13,A ,Lu ,570000,4.6,4.07,0.53,06/18/2019,50000,21.78,1,60,0.28,-11.51,0,10.27,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
507,8779,14,A ,Lu ,970000,4.15,3.62,0.53,06/18/2019,954336.82,416.06,1,60,0.28,-219.8,0,196.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
337,8779,18,A ,Chen ,444000,3.82,3.29,0.53,06/18/2019,444000,193.41,1,60,0.28,-102.18,0,91.23,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
522,8779,19,A ,Horm ,944000,4.7,4.17,0.53,07/24/2019,544000,295.06,1,60,0.28,-155.88,0,139.18,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
522,8779,21,A ,Kemp ,479952,4.25,3.72,0.53,07/24/2019,470613.4,204.94,1,60,0.28,-108.27,0,96.67,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006
522,8779,24,A ,Tiang ,684481,4.7,4.17,0.53,08/23/2019,668000,290.99,1,60,0.28,-153.73,0,137.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,26,A ,Li ,564000,4.25,3.72,0.53,09/26/2019,557190.2,243,1,60,0.28,-128.38,0,114.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,27,A ,Quach ,406000,4.4,3.87,0.53,10/21/2019,398603.27,174.39,1,60,0.28,-92.13,0,82.26,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,27,B ,Quach ,406000,4.4,3.87,0.53,10/21/2019,406000,176.86,1,60,0.28,-93.44,0,83.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
502,8779,28,A ,Zhu ,1000000,4.55,4.02,0.53,09/18/2019,988561.45,430.95,1,60,0.28,-227.67,0,203.28,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
307,8779,30,A ,Huynh ,410453,3.22,2.69,0.53,09/19/2019,394606.79,172.23,1,60,0.28,-90.99,0,81.24,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,31,A ,Xu ,830000,4.15,3.62,0.53,01/30/2020,824329.48,359.53,1,60,0.28,-189.94,0,169.59,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
305,8779,32,A ,Tran ,476000,3.52,2.99,0.53,10/03/2019,470274.11,204.88,1,60,0.28,-108.24,0,96.64,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,37,A ,Zulfaqari ,552000,3.22,2.69,0.53,12/19/2019,543482.81,237.12,1,60,0.28,-125.27,0,111.85,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,38,A ,Pham ,512000,3.22,2.69,0.53,01/10/2020,507925.37,221.37,1,60,0.28,-116.95,0,104.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
310,8779,42,A ,Nguyen ,318542,3.52,2.99,0.53,12/06/2019,311782.42,135.71,1,60,0.28,-71.7,0,64.01,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,43,A ,Hopkins ,427505,3.52,2.99,0.53,01/08/2020,415112.03,180.87,1,60,0.28,-95.55,0,85.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,43,B ,Hopkins ,41567,3.52,2.99,0.53,01/08/2020,40011.29,17.43,1,60,0.28,-9.21,0,8.22,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
334,8779,43,C ,Hopkins ,10292,3.52,2.99,0.53,01/08/2020,9991.49,4.35,1,60,0.28,-2.3,0,2.05,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,45,A ,Tran ,408000,3.22,2.69,0.53,12/06/2019,403381.11,175.81,1,60,0.28,-92.88,0,82.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,47,A ,Lebon ,100000,3.32,2.79,0.53,12/09/2019,44000,20.14,1,60,0.28,-10.64,0,9.5,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
337,8779,48,A ,Le ,400000,3.22,2.69,0.53,12/03/2019,396171.84,172.6,1,60,0.28,-91.18,0,81.42,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,49,A ,Vu ,509600,3.22,2.69,0.53,12/13/2019,504687.98,220.02,1,60,0.28,-116.24,0,103.78,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,50,A ,Wang ,1500000,4.15,3.62,0.53,01/15/2020,1387663.97,604.28,1,60,0.28,-319.24,0,285.04,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,52,A ,Le ,520000,3.32,2.79,0.53,01/20/2020,513359.23,224.03,1,60,0.28,-118.36,0,105.67,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,54,A ,Nguyen ,268000,3.32,2.79,0.53,01/08/2020,265878.84,115.87,1,60,0.28,-61.21,0,54.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,56,A ,Hanna ,250000,3.22,2.69,0.53,01/09/2020,245368.96,107,1,60,0.28,-56.53,0,50.47,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006
308,8779,57,A ,Le ,300000,3.22,2.69,0.53,01/08/2020,293231.8,127.97,1,60,0.28,-67.61,0,60.36,0,0,0, ,W,Super Finance Markets Pty Ltd ,202006
309,8779,58,A ,Le ,432000,3.22,2.69,0.53,12/17/2019,427824.14,186.53,1,60,0.28,-98.54,0,87.99,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,63,A ,Nguyen ,449175,3.22,2.69,0.53,12/23/2019,444814.79,194,1,60,0.28,-102.49,0,91.51,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,64,A ,Leav ,444000,3.42,2.89,0.53,01/07/2020,443334.6,193.23,1,60,0.28,-102.08,0,91.15,0,0,0, ,F,Super Finance Markets Pty Ltd ,202006
308,8779,66,A ,Nguyen ,516750,3.22,2.62,0.6,01/21/2020,511192.69,252.48,1,60,0.28,-117.82,0,134.66,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,69,A ,Leab ,435000,4.14,3.61,0.53,02/10/2020,432580.51,188.52,1,60,0.28,-99.59,0,88.93,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,71,A ,Yang ,511000,3.22,2.62,0.6,02/07/2020,506610.13,249.98,1,60,0.28,-116.66,0,133.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,76,A ,Huynh ,442060,3.22,2.62,0.6,02/28/2020,438854.39,216.79,1,60,0.28,-101.17,0,115.62,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
308,8779,76,B ,Huynh ,77940,3.22,2.62,0.6,02/28/2020,77374.82,38.22,1,60,0.28,-17.84,0,20.38,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
500,8779,82,A ,Pham ,788000,4.7,4.17,0.53,05/12/2020,791145.52,344.13,1,60,0.28,-181.81,0,162.32,0,0,0, ,M,Super Finance Markets Pty Ltd ,202006
`

type ResimacRow struct {
TRSTCD int
ORGNTR int
LOANNO int
PORTN string
BRNAMX string //Brower
LNAMT float64 //Loan amount
INTRTE float64
DELRTE float64
MARGIN float64
SETLDX time.Time //settled date
LNBAL float64
MANFEE float64
RSINCP float64
RSSACT float64
RSSACP float64
SACAMT float64
MOFEE float64
MANTOT float64
NEWLON float64
REFADJ float64
NETNEW float64
FIXED string
LNFORT string
ORGNAM string
LNEOM time.Time

//Derived attributes
LoanNumber string
Borrower string
LoanAmount float64
Balance float64
IncomeAmount float64
}

type ResimacPdf struct {
Period time.Time
LoanNumber string
Borrower string
LoanAmount float64
InterestRate float64
DelvRate float64
Margin float64
Settlement time.Time
Balance float64
OffsetBalance float64
GrossManFee float64
IncentiveP float64
Months int
SacrificeP float64
Fee float64
MoFee float64
NetManFee float64
IncomeAmount float64
}

func (m *AiDecodeIncome) decodeResimacXls(raw []byte) (e error) {
m.ResimacXls = make([]ResimacRow, 0, 100)
lines := strings.Split(string(raw), "\n")
foundHeader := false
var idx map[string]int
for i := 0; i < len(lines); i++ {
lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
el := strings.Split(lines[i], ",")
if len(el) < 25 {
continue //bypass
}

if !foundHeader {
foundHeader, idx = m.decodeResimacXlsHeader(lines[i])
continue
}
//we have already found header
r := ResimacRow{}

r.TRSTCD, _ = strconv.Atoi(el[idx["TRSTCD"]])
r.ORGNTR, _ = strconv.Atoi(el[1])
r.LOANNO, _ = strconv.Atoi(el[2])
r.PORTN = el[3]
r.LoanNumber = fmt.Sprintf("%d-%06d-%06d%s", r.TRSTCD, r.ORGNTR, r.LOANNO, r.PORTN)
r.BRNAMX = el[4]
r.Borrower = r.BRNAMX
r.LNAMT = m.currencyToFloat64(el[5])
r.LoanAmount = r.LNAMT
r.INTRTE = m.currencyToFloat64(el[6])
r.DELRTE = m.currencyToFloat64(el[7])
r.MARGIN = m.currencyToFloat64(el[8])
r.SETLDX, _ = time.Parse("1/02/2006", el[9])
r.LNBAL = m.currencyToFloat64(el[10])
r.Balance = r.LNBAL
r.MANFEE = m.currencyToFloat64(el[11])
r.RSINCP = m.currencyToFloat64(el[12])
r.RSSACT = m.currencyToFloat64(el[13])
r.RSSACP = m.currencyToFloat64(el[14])
r.SACAMT = m.currencyToFloat64(el[15])
r.MOFEE = m.currencyToFloat64(el[16])
r.MANTOT = m.currencyToFloat64(el[17])
r.IncomeAmount = r.MANTOT
r.NEWLON = m.currencyToFloat64(el[18])
r.REFADJ = m.currencyToFloat64(el[19])
r.NETNEW = m.currencyToFloat64(el[20])
r.FIXED = el[21]
r.LNFORT = el[22]
r.ORGNAM = strings.TrimSpace(el[23])
r.LNEOM, _ = time.Parse("200601", el[24])
m.ResimacXls = append(m.ResimacXls, r)
}
return
}

func (m *AiDecodeIncome) decodeResimacXlsHeader(line string) (isHeaderLine bool, idx map[string]int) {
keys := []string{
"TRSTCD", "ORGNTR", "LOANNO", "PORTNO", "BRNAMX", "LNAMT", "INTRTE", "DELRTE",
"MARGIN", "SETLDX", "LNBAL", "MANFEE", "RSINCP", "RSSACT", "RSSACP", "SACAMT",
"MOFEE", "MANTOT", "NEWLON", "REFADJ", "NETNEW", "FIXED", "LNFORT", "ORGNAM",
"LNEOM"}

idx = make(map[string]int)
l := strings.ReplaceAll(line, " ", "") // remove space
el := strings.Split(l, ",")
found := 0
for i := 0; i < len(el); i++ {
a := strings.TrimSpace(el[i])
a = strings.ToUpper(a)
for _, k := range keys {
if a == k {
idx[k] = i
found++
break
}
}
}
isHeaderLine = found > 20 // if we found more than 20 headers
return
}

func (m *AiDecodeIncome) decodeResimacPdf(raw []byte) (e error) {
m.ResimacPdf = make([]ResimacPdf, 0, 30)
period := `(?i)Trailer +Fees Report[ :]+((?:Jan|January|Feb|February|Mar|March|Apr|April|May|May|Jun|June|Jul|July|Aug|August|Sep|September|Oct|October|Nov|November|Dec|December)[\t ]+[0-9]{4})`
pattern := `(?i)(\d+-0*8779-[\d A-Za-z-]+[A-Za-z]) +([A-Za-z]+) +(\$\d{1,3}(?:,\d{3})*(?:[\d.]\d{2})?) +(\d{1,3}(?:\.\d{1,2})?) +([\d ]{1,3}(?:[ .]\d{1,2})?) +(\d{1,3}(?:[\d.]\d{1,2})?) +((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}) +(-?\$\d{1,3}(?:,\d{3,5})*(?:\.\d{1,2})?) +(-?[\d$]\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{2,3})*(?:\.\d{1,2})?) +(\d{1,3}(?:\.\d{1,2})?) +(\d+) +(\d{1,3}\.?\d{1,2}) +(-?\$\d{1,3}(?:,,)*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{1,2})?)`

periodLine, e := regexp.Compile(period)
if e != nil {
return
}
validLine, e := regexp.Compile(pattern) // error if regexp invalid
if e != nil {
return
}

periods := periodLine.FindSubmatch(raw)
CurrentPeriod, e1 := time.Parse("January 2006", string(periods[1]))
if e1 != nil {
log.Warn("failed to parse period of resimac PDF", e1)
}

matches := validLine.FindAllSubmatch(raw, -1)
if matches == nil {
log.Warn("Resimac PDF decode found nothing no matches", raw)
return
}

for _, v := range matches {
rp := ResimacPdf{Period: CurrentPeriod}
rp.LoanNumber = string(v[1])
rp.Borrower = string(v[2])
rp.LoanAmount = m.currencyToFloat64(string(v[3]))
rp.InterestRate = m.currencyToFloat64(string(v[4]))
rp.DelvRate = m.currencyToFloat64(string(v[5]))
rp.Margin = m.currencyToFloat64(string(v[6]))
rp.Settlement, _ = time.Parse("02/01/2006", string(v[7]))
rp.Balance = m.currencyToFloat64(string(v[8]))
rp.OffsetBalance = m.currencyToFloat64(string(v[9]))
rp.GrossManFee = m.currencyToFloat64(string(v[10]))
rp.IncentiveP = m.currencyToFloat64(string(v[11]))
rp.Months, _ = strconv.Atoi(string(v[12]))
rp.SacrificeP = m.currencyToFloat64(string(v[13]))
rp.Fee = m.currencyToFloat64(string(v[14]))
rp.MoFee = m.currencyToFloat64(string(v[15]))
rp.NetManFee = m.currencyToFloat64(string(v[16]))
rp.IncomeAmount = rp.NetManFee
m.ResimacPdf = append(m.ResimacPdf, rp)
}
return
}

Laddar…
Avbryt
Spara