Biukop
/
SFM-loan-Rest-Api
Beobachten 1
Favorisieren 0
Fork 0
Code Issues 0 Pull-Requests 0 Releases 0 Wiki Aktivität
Quellcode durchsuchen

change Trail to income_amount to add income_type

master
sp vor 4 Jahren
Ursprung
c86fa2c4f2
Commit
c483206491
6 geänderte Dateien mit 325 neuen und 63 gelöschten Zeilen
Geteilte Ansicht
Diff-Optionen
Statistiken anzeigen Patch-Datei herunterladen Vergleichs-Datei herunterladen
  1. +3
    -3
      apiV1UploadAnalysis.go
  2. +14
    -3
      apiV1Uploads.go
  3. +8
    -0
      config.go
  4. +97
    -27
      pay-in-decode.go
  5. +24
    -30
      payIn-AAA.go
  6. +179
    -0
      payin-connective.go

+ 3
- 3
apiV1UploadAnalysis.go Datei anzeigen

@@ -29,10 +29,10 @@ func apiV1UploadAnalysis(w http.ResponseWriter, r *http.Request, ss *loan.Sessio

analysisMutex.Lock()
ai := AiDecodeIncome{}
e = ai.decodeUploadToPayIn(ul)
e = ai.decodeUploadToPayIn(ul, true)
analysisMutex.Unlock()
if e != nil {
log.Error("Invalid uploads Id cannot conver to integer", Id, e)
log.Error("Invalid uploads Id cannot convert to integer", Id, e)
apiV1Server500Error(w, r)
return
}
@@ -59,7 +59,7 @@ func apiV1UploadCreateAnalysis(w http.ResponseWriter, r *http.Request, ss *loan.
}

ai := AiDecodeIncome{}
e = ai.decodeUploadToPayIn(ul)
e = ai.decodeUploadToPayIn(ul, false)
if e != nil {
log.Error("Cannot decode upload", Id, e)
apiV1Server500Error(w, r)

+ 14
- 3
apiV1Uploads.go Datei anzeigen

@@ -263,11 +263,22 @@ func getRequestedUpload(strId string, w http.ResponseWriter, r *http.Request, ss
}

func forceHttpDownload(r *http.Request) bool {
keys, ok := r.URL.Query()["download"]
return httpQueryParamHas(r, "download", "force")
//keys, ok := r.URL.Query()["download"]
//
//if !ok || len(keys[0]) < 1 {
// return false
//}
//key := keys[0]
//return key == "force"
}

func httpQueryParamHas(r *http.Request, key string, expectedValue string) (has bool) {
keys, ok := r.URL.Query()[key]

if !ok || len(keys[0]) < 1 {
return false
}
key := keys[0]
return key == "force"
k := keys[0]
return k == expectedValue
}

+ 8
- 0
config.go Datei anzeigen

@@ -150,6 +150,14 @@ func (m *configuration) checkUploadDir() (valid bool) {
}
m.UploadsDir.PdfDir = p + string(os.PathSeparator) //change it to absolute dir

// convert to absolute path : JsonDir
p, e = filepath.Abs(m.UploadsDir.JsonDir)
if e != nil {
valid = false
log.Fatal("bad pdf file dir", m.UploadsDir.JsonDir, e)
}
m.UploadsDir.JsonDir = p + string(os.PathSeparator) //change it to absolute dir

// convert to absolute path : TmpDir
p, e = filepath.Abs(m.TempDir)
if e != nil {

+ 97
- 27
pay-in-decode.go Datei anzeigen

@@ -2,49 +2,104 @@ package main

import (
"biukop.com/sfm/loan"
"encoding/json"
"errors"
log "github.com/sirupsen/logrus"
"io/ioutil"
"os"
"os/exec"
"strings"
)

type FunderType string
type AiDecodeIncome struct {
Id int64
Input loan.Uploads
ul uploadsOnDisk // internal data
Mime string // mime actually detected.
PayIn []loan.PayIn // generated PayIn Info
DecodeIncomeDetails
}

const (
Funder_AAA FunderType = "AAA Financial"
Funder_Pepper = "Pepper"
Funder_Resimac = "Resimac"
Funder_Unknown = "cannot detect funder type"
)
type DecodeIncomeDetails struct {
Funders []loan.FunderType
AAA []PayInAAARow
Connective_ANZ []ConnectiveRow
Connective_BOC []ConnectiveRow
Connective_CHLAB []ConnectiveRow
Connective_HSL []ConnectiveRow
Connective_ING []ConnectiveRow
Connective_MEB []ConnectiveRow
Connective_SGB []ConnectiveRow
Connective_WPC []ConnectiveRow
}

type AiDecodeIncome struct {
Id int64
Input loan.Uploads
ul uploadsOnDisk // internal data
Mime string //mime actually detected.
PayIn []loan.PayIn
Funder FunderType
AAA []PayInAAAPeriod
type AiDecodeJson struct {
Version string
V1 DecodeIncomeDetails
}

func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) {
func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) {
m.Id = ulMeta.Id
m.Input = ulMeta
m.ul.Upload = ulMeta

m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated
m.Funders = make([]loan.FunderType, 0, 50) // array of valid funders

if allowCachedResult {
e = m.ReadJson()
if e == nil {
return // already decoded
} else {
log.Warn("trying to read existing json failed ", e.Error())
e = nil
}
}
m.PayIn = make([]loan.PayIn, 0, 10)
switch m.getFileType() {
case "pdf":
m.decodePdf()
e = m.decodePdf()
break
case "excel", "opensheet":
m.decodeXls()
e = m.decodeXls()
break
default:
e = errors.New("unknown format")
m.Funder = "" // mark unknown decoding
}

if e == nil {
eJson := m.WriteJson()
if eJson != nil {
log.Error("failed to write analysis to json", eJson.Error())
}
}
return
}

func (m *AiDecodeIncome) ReadJson() (e error) {
if !fileExists(m.ul.jsonPath()) {
return errors.New(m.ul.jsonPath() + " not found")
}
f, e := os.Open(m.ul.jsonPath())
if e != nil {
return
}

decoder := json.NewDecoder(f)
data := AiDecodeJson{}
e = decoder.Decode(&data)
if e != nil {
log.Error("failed load existing decode json", e.Error())
return
}
return
}

func (m *AiDecodeIncome) WriteJson() (e error) {
b, e := json.Marshal(m.DecodeIncomeDetails)
if e != nil {
return
}
ioutil.WriteFile(m.ul.jsonPath(), b, 0644)
return
}

@@ -71,28 +126,43 @@ func (m *AiDecodeIncome) decodePdf() (e error) {

raw := string(out)
switch m.detectFunder(raw) {
case Funder_AAA:
m.Funder = Funder_AAA
case loan.Funder_AAA:
m.Funders = append(m.Funders, loan.Funder_AAA)
e = m.decodeAAAPdf(raw)
for _, row := range m.AAA {
pi := loan.PayIn{}
pi.Id = 0
pi.Ts = row.Period
pi.Amount = row.LoanAmount
pi.Lender = loan.Funder_AAA
pi.Settlement = row.Settlement
pi.Balance = row.Balance
pi.OffsetBalance = -1
pi.IncomeAmount = row.InTrail
pi.IncomeType = "Trail"

m.PayIn = append(m.PayIn, pi)
}
log.Println("AAA final result", m.AAA)
break
case Funder_Unknown:
e = errors.New(Funder_Unknown)
case loan.Funder_Unknown:
e = errors.New(loan.Funder_Unknown)
break // not able to detect Funder
}
return
}

func (m *AiDecodeIncome) decodeXls() (e error) {
e = errors.New("not implemented yet")
return
}

func (m *AiDecodeIncome) detectFunder(raw string) FunderType {
func (m *AiDecodeIncome) detectFunder(raw string) loan.FunderType {
if m.isAAA(raw) {
return Funder_AAA
return loan.Funder_AAA
}

return Funder_Unknown
return loan.Funder_Unknown
}

func (m *AiDecodeIncome) isAAA(raw string) bool {

+ 24
- 30
payIn-AAA.go Datei anzeigen

@@ -26,6 +26,7 @@ Super Finance Markets Pty Ltd
*/

type PayInAAARow struct {
Period time.Time
LoanNumber string
Settlement time.Time
LoanAmount float64
@@ -33,49 +34,42 @@ type PayInAAARow struct {
InTrail float64
}

type PayInAAAPeriod struct {
Period time.Time
Rows []PayInAAARow
}

func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) {
m.AAA = make([]PayInAAAPeriod, 0, 10)
m.AAA = make([]PayInAAARow, 0, 10)
lines := strings.Split(raw, "\n")

var currentDecoder *PayInAAAPeriod = nil
var currentRow = PayInAAARow{}
var currentPeriod = time.Time{}
state := "start"
for _, l := range lines { // DFA, wow, finally it's used. after years of learning
switch state {
case "start":
state = currentDecoder.processStart(l)
state = currentRow.processStart(l)
if state == "LookingForPeriod" {
// determine column index, if their column is changing
}
break
case "LookingForPeriod":
state = currentDecoder.processPeriod(l)
state = currentRow.processPeriod(l)
if state == "LookingForRows" {
Period, e := currentDecoder.getPeriod(l)
currentPeriod, e = currentRow.getPeriod(l)
if e != nil {
log.Warn("cannot find period", l, e)
state = "LookingForPeriod"
} else {
m.AAA = append(m.AAA, PayInAAAPeriod{})
idx := len(m.AAA) - 1
currentDecoder = &(m.AAA[idx])
currentDecoder.Rows = make([]PayInAAARow, 0, 10)
currentDecoder.Period = Period
currentRow.Period = currentPeriod
}
}
break
case "LookingForRows", "LookingForRowsSkipCurrent":
nextState, row, valid := currentDecoder.processRow(l)
if valid && currentDecoder != nil {
currentDecoder.Rows = append(currentDecoder.Rows, row)
nextState, valid := currentRow.processRow(l)
if valid {
m.AAA = append(m.AAA, currentRow)
}
state = nextState
if nextState == "start" {
currentDecoder = nil //renew to a empty state
if nextState == "start" { // reset current state
currentRow = PayInAAARow{}
currentRow.Period = currentPeriod
}
break
}
@@ -83,7 +77,7 @@ func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) {
return
}

func (m *PayInAAAPeriod) processStart(line string) (nextState string) {
func (m *PayInAAARow) processStart(line string) (nextState string) {
nextState = "start"
if strings.Contains(line, "Loan Number") &&
strings.Contains(line, "SettDate") &&
@@ -94,7 +88,7 @@ func (m *PayInAAAPeriod) processStart(line string) (nextState string) {
return
}

func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) {
func (m *PayInAAARow) processPeriod(line string) (nextState string) {
nextState = "LookingForPeriod"
if strings.Contains(line, "Period Servicing:") {
nextState = "LookingForRows"
@@ -103,14 +97,14 @@ func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) {
}

// Period Servicing: Feb 2020
func (m *PayInAAAPeriod) getPeriod(line string) (p time.Time, e error) {
func (m *PayInAAARow) getPeriod(line string) (p time.Time, e error) {
idx := strings.Index(line, ":")
subStr := strings.TrimSpace(line[idx+1:])
p, e = time.Parse("Jan 2006", subStr)
return
}

func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAARow, valid bool) {
func (m *PayInAAARow) processRow(line string) (nextState string, valid bool) {
nextState = "LookingForRows"
valid = false
allParts := strings.Split(line, " ")
@@ -122,11 +116,11 @@ func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAA
}

if len(el) >= 5 {
row.LoanNumber = el[0]
row.Settlement, _ = time.Parse("02-Jan-06", el[1])
row.LoanAmount = m.currencyToFloat64(el[2])
row.Balance = m.currencyToFloat64(el[3])
row.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element
m.LoanNumber = el[0]
m.Settlement, _ = time.Parse("02-Jan-06", el[1])
m.LoanAmount = m.currencyToFloat64(el[2])
m.Balance = m.currencyToFloat64(el[3])
m.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element
valid = true
} else {
if strings.Contains(line, "Total:") {
@@ -138,7 +132,7 @@ func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAA
return
}

func (m *PayInAAAPeriod) currencyToFloat64(cur string) (ret float64) {
func (m *PayInAAARow) currencyToFloat64(cur string) (ret float64) {
cur = strings.ReplaceAll(cur, " ", "") //remove space
cur = strings.ReplaceAll(cur, "$", "") //remove $
cur = strings.ReplaceAll(cur, ",", "") //remove ,

+ 179
- 0
payin-connective.go Datei anzeigen

@@ -0,0 +1,179 @@
package main

//sample data
//9/25/2020 Commission Statement
//
//
//
//
//Commission Statement (Trail) - 01/08/2020 to 31/08/2020 - Super Finance Markets Pty Ltd
//
//
//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total
//Split
//
//
//
//ANZ
//687156246 DENG ASHLEY 25/08/2020 ANZ $200,000.00 $97,640.15 Bo Zhu $4.15 tc 100 % $4.15 $0.42 $4.57
//
//687156254 DENG ASHLEY 25/08/2020 ANZ $360,000.00 $360,000.00 Bo Zhu $10.35 tc 100 % $10.35 $1.04 $11.39
//
//687156262 DENG ASHLEY 25/08/2020 ANZ $500,000.00 $500,000.00 Bo Zhu $14.38 tc 100 % $14.38 $1.44 $15.82
//
//687109829 FU JINGYUN 18/08/2020 ANZ $170,000.00 $170,000.00 Bo Zhu $9.78 tc 100 % $9.78 $0.98 $10.76
//
//687109837 FU JINGYUN 18/08/2020 ANZ $245,000.00 $0.00 Bo Zhu $2.04 tc 100 % $2.04 $0.20 $2.24
//
//687145432 GU ZAC ZHEN 24/08/2020 ANZ $464,000.00 $464,000.00 Bo Zhu $15.25 tc 100 % $15.25 $1.53 $16.78
//
//686768902 HAO YANAN 01/07/2020 ANZ $470,000.00 $426,358.02 Bo Zhu $59.40 tc 100 % $59.40 $5.94 $65.34
//
//686766595 LAI JIAOLONG 01/07/2020 ANZ $188,000.00 $17,726.29 Bo Zhu $2.50 tc 100 % $2.50 $0.25 $2.75
//
//686766608 LAI JIAOLONG 01/07/2020 ANZ $180,000.00 $179,690.48 Bo Zhu $22.89 tc 100 % $22.89 $2.29 $25.18
//
//685061729 MENG SICHEN 02/09/2019 ANZ $105,000.00 $100,911.09 Bo Zhu $12.81 tc 100 % $12.81 $1.28 $14.09
//
//685061737 MENG SICHEN 02/09/2019 ANZ $399,000.00 $391,608.66 Bo Zhu $49.89 tc 100 % $49.89 $4.99 $54.88
//
//687008884 NERCESSIAN 05/08/2020 ANZ $158,031.59 $158,031.59 Bo Zhu $17.55 tc 100 % $17.55 $1.76 $19.31
//ALEX OSCAR
//
//685245413 WEN TAO 09/10/2019 ANZ $810,000.00 $463,149.89 Bo Zhu $59.72 tc 100 % $59.72 $5.97 $65.69
//
//686508297 XIA RENNAN 29/05/2020 ANZ $1,100,000.00 $736,349.83 Bo Zhu $95.95 tc 100 % $95.95 $9.60 $105.55
//
//686508318 XIA RENNAN 29/05/2020 ANZ $400,000.00 $398,130.00 Bo Zhu $50.80 tc 100 % $50.80 $5.08 $55.88
//Total (ANZ) $5,749,031.59 $4,463,596.00 $427.46 $427.46 $42.77 $470.23
//
//
//
//BOC
//100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.64 tc 100 % $42.64 $4.26 $46.90
//ZHANG
//
//100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.59 tc 100 % $42.59 $4.26 $46.85
//ZHANG
//
//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 1/3
//9/25/2020 Commission Statement
//
//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total
//Split
//
//
//Total (BOC) $1,500,000.00 $0.00 $85.23 $85.23 $8.52 $93.75
//
//
//
//CHLAB
//0041914813 YX01 CHRISTOPHER 12/12/2019 CHLAB $411,775.00 $315,875.03 Ian Lloyd $40.13 tc 100 % $40.13 $4.01 $44.15
//JOSEPH
//PISANI
//
//0041914813 YX02 CHRISTOPHER 12/12/2019 CHLAB $85,000.00 $85,000.00 Ian Lloyd $10.80 tc 100 % $10.80 $1.08 $11.88
//JOSEPH
//PISANI
//Total (CHLAB) $496,775.00 $400,875.03 $50.93 $50.93 $5.09 $56.03
//
//
//
//HSL
//341960587 Huang T 19/09/2019 HSL $400,000.00 $391,801.03 Ian Lloyd $49.91 tc 100 % $49.91 $4.99 $54.90
//
//851874400 Huang T 03/12/2019 HSL $287,000.00 $149,019.02 Ian Lloyd $18.98 tc 100 % $18.98 $1.90 $20.88
//Total (HSL) $687,000.00 $540,820.05 $68.89 $68.89 $6.89 $75.78
//
//
//
//
//ING
//28446283 MR Dylan 01/02/2019 ING $100,431.54 $99,228.57 Ian Lloyd $12.59 tc 100 % $12.59 $1.26 $13.85
//Waring
//
//28446294 MR Dylan 01/02/2019 ING $326,000.00 $323,513.96 Ian Lloyd $41.08 tc 100 % $41.08 $4.11 $45.19
//Waring
//
//28915359 MS Emily 03/02/2020 ING $412,000.00 $409,509.47 Ian Lloyd $52.02 tc 100 % $52.02 $5.20 $57.22
//Jeoung
//
//28924374 MS Rhiannon 10/02/2020 ING $320,000.00 $316,427.48 Ian Lloyd $40.28 tc 100 % $40.28 $4.03 $44.31
//Lloyd
//
//29044457 MS ZDENKA 13/05/2020 ING $308,000.00 $306,270.89 Ian Lloyd $38.98 tc 100 % $38.98 $3.90 $42.88
//MARY
//ASHFIELD
//
//29044468 MS ZDENKA 13/05/2020 ING $57,000.00 $56,755.78 Ian Lloyd $7.21 tc 100 % $7.21 $0.72 $7.93
//MARY
//ASHFIELD
//
//29084842 MS Sharon Joy 11/06/2020 ING $375,000.00 $374,993.96 Ian Lloyd $47.68 tc 100 % $47.68 $4.77 $52.45
//McClelland
//
//29084853 MS Sharon Joy 11/06/2020 ING $75,000.00 $72,057.85 Ian Lloyd $9.17 tc 100 % $9.17 $0.92 $10.09
//McClelland
//Total (ING) $1,973,431.54 $1,958,757.96 $249.01 $249.01 $24.90 $273.91
//
//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 2/3
//9/25/2020 Commission Statement
//
//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total
//Split
//
//
//
//
//MEB
//1676620 Ian Francis 03/02/2020 MEB $205,600.00 $203,280.20 Ian Lloyd $25.41 tc 100 % $25.41 $2.54 $27.95
//Lloyd
//
//823275 Reece Kenneth 24/07/2019 MEB $376,088.56 $368,253.20 Ian Lloyd $46.03 tc 100 % $46.03 $4.60 $50.63
//Godfrey and
//Sarah Anne
//Heyes
//Total (MEB) $581,688.56 $571,533.40 $71.44 $71.44 $7.14 $78.58
//
//
//
//
//SGB
//100212227685200 BILAL AKIL 21/02/2020 SGB $700,000.00 $692,515.82 Bo Zhu $86.56 TC 100 % $86.56 $8.66 $95.22
//JIAOYUE CAO
//Total (SGB) $700,000.00 $692,515.82 $86.56 $86.56 $8.66 $95.22
//
//
//
//WPC
//700183234270 Lin Feng 25/06/2020 WPC $370,000.00 $210,607.62 Bo Zhu $26.76 tc 100 % $26.76 $2.68 $29.44
//
//700183234297 Lin Feng 24/06/2020 WPC $250,000.00 -$19.98 Bo Zhu $0.00 tc 0% $0.00 $0.00 $0.00
//
//700183234238 Lin Feng 24/06/2020 WPC $280,000.00 $278,301.00 Bo Zhu $35.36 tc 100 % $35.36 $3.54 $38.90
//
//700183234254 Lin Feng 24/06/2020 WPC $600,593.00 $596,337.00 Bo Zhu $75.76 tc 100 % $75.76 $7.58 $83.34
//
//700183234131 Lin Feng 24/06/2020 WPC $370,000.00 $367,916.00 Bo Zhu $46.74 tc 100 % $46.74 $4.67 $51.41
//
//700183234174 Lin Feng 24/06/2020 WPC $260,000.00 $258,524.00 Bo Zhu $32.85 tc 100 % $32.85 $3.29 $36.14
//
//700183234094 Lin Feng 24/06/2020 WPC $393,666.00 $390,833.00 Bo Zhu $49.66 tc 100 % $49.66 $4.97 $54.63
//
//700183233972 Lin Feng 24/06/2020 WPC $410,000.00 $407,087.00 Bo Zhu $51.72 tc 100 % $51.72 $5.17 $56.89
//
//700183122084 Zhou Zeyu 01/04/2020 WPC $845,000.00 $309,335.07 Bo Zhu $39.30 tc 100 % $39.30 $3.93 $43.23
//Total (WPC) $3,779,259.00 $2,818,920.71 $358.15 $358.15 $35.83 $393.98
//
//
//
//Total (Overall) $15,467,185.69 $11,447,018.97 $1,397.67 $1,397.67 $139.81 $1,537.48
//
//
//
//
//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 3/3
//

type ConnectiveRow struct {
}

Verfassen Vorschau
Laden…
Abbrechen
Speichern