diff --git a/apiV1UploadAnalysis.go b/apiV1UploadAnalysis.go index d4ce607..c00f52c 100644 --- a/apiV1UploadAnalysis.go +++ b/apiV1UploadAnalysis.go @@ -29,10 +29,10 @@ func apiV1UploadAnalysis(w http.ResponseWriter, r *http.Request, ss *loan.Sessio analysisMutex.Lock() ai := AiDecodeIncome{} - e = ai.decodeUploadToPayIn(ul) + e = ai.decodeUploadToPayIn(ul, true) analysisMutex.Unlock() if e != nil { - log.Error("Invalid uploads Id cannot conver to integer", Id, e) + log.Error("Invalid uploads Id cannot convert to integer", Id, e) apiV1Server500Error(w, r) return } @@ -59,7 +59,7 @@ func apiV1UploadCreateAnalysis(w http.ResponseWriter, r *http.Request, ss *loan. } ai := AiDecodeIncome{} - e = ai.decodeUploadToPayIn(ul) + e = ai.decodeUploadToPayIn(ul, false) if e != nil { log.Error("Cannot decode upload", Id, e) apiV1Server500Error(w, r) diff --git a/apiV1Uploads.go b/apiV1Uploads.go index 209f33d..2e91483 100644 --- a/apiV1Uploads.go +++ b/apiV1Uploads.go @@ -263,11 +263,22 @@ func getRequestedUpload(strId string, w http.ResponseWriter, r *http.Request, ss } func forceHttpDownload(r *http.Request) bool { - keys, ok := r.URL.Query()["download"] + return httpQueryParamHas(r, "download", "force") + //keys, ok := r.URL.Query()["download"] + // + //if !ok || len(keys[0]) < 1 { + // return false + //} + //key := keys[0] + //return key == "force" +} + +func httpQueryParamHas(r *http.Request, key string, expectedValue string) (has bool) { + keys, ok := r.URL.Query()[key] if !ok || len(keys[0]) < 1 { return false } - key := keys[0] - return key == "force" + k := keys[0] + return k == expectedValue } diff --git a/config.go b/config.go index eb6b047..5c2ae56 100644 --- a/config.go +++ b/config.go @@ -150,6 +150,14 @@ func (m *configuration) checkUploadDir() (valid bool) { } m.UploadsDir.PdfDir = p + string(os.PathSeparator) //change it to absolute dir + // convert to absolute path : JsonDir + p, e = filepath.Abs(m.UploadsDir.JsonDir) + if e != nil { + valid = false + log.Fatal("bad pdf file dir", m.UploadsDir.JsonDir, e) + } + m.UploadsDir.JsonDir = p + string(os.PathSeparator) //change it to absolute dir + // convert to absolute path : TmpDir p, e = filepath.Abs(m.TempDir) if e != nil { diff --git a/pay-in-decode.go b/pay-in-decode.go index 3f1e7c8..54ba67a 100644 --- a/pay-in-decode.go +++ b/pay-in-decode.go @@ -2,49 +2,104 @@ package main import ( "biukop.com/sfm/loan" + "encoding/json" "errors" log "github.com/sirupsen/logrus" + "io/ioutil" + "os" "os/exec" "strings" ) -type FunderType string +type AiDecodeIncome struct { + Id int64 + Input loan.Uploads + ul uploadsOnDisk // internal data + Mime string // mime actually detected. + PayIn []loan.PayIn // generated PayIn Info + DecodeIncomeDetails +} -const ( - Funder_AAA FunderType = "AAA Financial" - Funder_Pepper = "Pepper" - Funder_Resimac = "Resimac" - Funder_Unknown = "cannot detect funder type" -) +type DecodeIncomeDetails struct { + Funders []loan.FunderType + AAA []PayInAAARow + Connective_ANZ []ConnectiveRow + Connective_BOC []ConnectiveRow + Connective_CHLAB []ConnectiveRow + Connective_HSL []ConnectiveRow + Connective_ING []ConnectiveRow + Connective_MEB []ConnectiveRow + Connective_SGB []ConnectiveRow + Connective_WPC []ConnectiveRow +} -type AiDecodeIncome struct { - Id int64 - Input loan.Uploads - ul uploadsOnDisk // internal data - Mime string //mime actually detected. - PayIn []loan.PayIn - Funder FunderType - AAA []PayInAAAPeriod +type AiDecodeJson struct { + Version string + V1 DecodeIncomeDetails } -func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) { +func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) { m.Id = ulMeta.Id m.Input = ulMeta m.ul.Upload = ulMeta - + m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated + m.Funders = make([]loan.FunderType, 0, 50) // array of valid funders + + if allowCachedResult { + e = m.ReadJson() + if e == nil { + return // already decoded + } else { + log.Warn("trying to read existing json failed ", e.Error()) + e = nil + } + } m.PayIn = make([]loan.PayIn, 0, 10) switch m.getFileType() { case "pdf": - m.decodePdf() + e = m.decodePdf() break case "excel", "opensheet": - m.decodeXls() + e = m.decodeXls() break default: e = errors.New("unknown format") - m.Funder = "" // mark unknown decoding } + if e == nil { + eJson := m.WriteJson() + if eJson != nil { + log.Error("failed to write analysis to json", eJson.Error()) + } + } + return +} + +func (m *AiDecodeIncome) ReadJson() (e error) { + if !fileExists(m.ul.jsonPath()) { + return errors.New(m.ul.jsonPath() + " not found") + } + f, e := os.Open(m.ul.jsonPath()) + if e != nil { + return + } + + decoder := json.NewDecoder(f) + data := AiDecodeJson{} + e = decoder.Decode(&data) + if e != nil { + log.Error("failed load existing decode json", e.Error()) + return + } + return +} + +func (m *AiDecodeIncome) WriteJson() (e error) { + b, e := json.Marshal(m.DecodeIncomeDetails) + if e != nil { + return + } + ioutil.WriteFile(m.ul.jsonPath(), b, 0644) return } @@ -71,28 +126,43 @@ func (m *AiDecodeIncome) decodePdf() (e error) { raw := string(out) switch m.detectFunder(raw) { - case Funder_AAA: - m.Funder = Funder_AAA + case loan.Funder_AAA: + m.Funders = append(m.Funders, loan.Funder_AAA) e = m.decodeAAAPdf(raw) + for _, row := range m.AAA { + pi := loan.PayIn{} + pi.Id = 0 + pi.Ts = row.Period + pi.Amount = row.LoanAmount + pi.Lender = loan.Funder_AAA + pi.Settlement = row.Settlement + pi.Balance = row.Balance + pi.OffsetBalance = -1 + pi.IncomeAmount = row.InTrail + pi.IncomeType = "Trail" + + m.PayIn = append(m.PayIn, pi) + } log.Println("AAA final result", m.AAA) break - case Funder_Unknown: - e = errors.New(Funder_Unknown) + case loan.Funder_Unknown: + e = errors.New(loan.Funder_Unknown) break // not able to detect Funder } return } func (m *AiDecodeIncome) decodeXls() (e error) { + e = errors.New("not implemented yet") return } -func (m *AiDecodeIncome) detectFunder(raw string) FunderType { +func (m *AiDecodeIncome) detectFunder(raw string) loan.FunderType { if m.isAAA(raw) { - return Funder_AAA + return loan.Funder_AAA } - return Funder_Unknown + return loan.Funder_Unknown } func (m *AiDecodeIncome) isAAA(raw string) bool { diff --git a/payIn-AAA.go b/payIn-AAA.go index 8cda041..23d3325 100644 --- a/payIn-AAA.go +++ b/payIn-AAA.go @@ -26,6 +26,7 @@ Super Finance Markets Pty Ltd */ type PayInAAARow struct { + Period time.Time LoanNumber string Settlement time.Time LoanAmount float64 @@ -33,49 +34,42 @@ type PayInAAARow struct { InTrail float64 } -type PayInAAAPeriod struct { - Period time.Time - Rows []PayInAAARow -} - func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) { - m.AAA = make([]PayInAAAPeriod, 0, 10) + m.AAA = make([]PayInAAARow, 0, 10) lines := strings.Split(raw, "\n") - var currentDecoder *PayInAAAPeriod = nil + var currentRow = PayInAAARow{} + var currentPeriod = time.Time{} state := "start" for _, l := range lines { // DFA, wow, finally it's used. after years of learning switch state { case "start": - state = currentDecoder.processStart(l) + state = currentRow.processStart(l) if state == "LookingForPeriod" { // determine column index, if their column is changing } break case "LookingForPeriod": - state = currentDecoder.processPeriod(l) + state = currentRow.processPeriod(l) if state == "LookingForRows" { - Period, e := currentDecoder.getPeriod(l) + currentPeriod, e = currentRow.getPeriod(l) if e != nil { log.Warn("cannot find period", l, e) state = "LookingForPeriod" } else { - m.AAA = append(m.AAA, PayInAAAPeriod{}) - idx := len(m.AAA) - 1 - currentDecoder = &(m.AAA[idx]) - currentDecoder.Rows = make([]PayInAAARow, 0, 10) - currentDecoder.Period = Period + currentRow.Period = currentPeriod } } break case "LookingForRows", "LookingForRowsSkipCurrent": - nextState, row, valid := currentDecoder.processRow(l) - if valid && currentDecoder != nil { - currentDecoder.Rows = append(currentDecoder.Rows, row) + nextState, valid := currentRow.processRow(l) + if valid { + m.AAA = append(m.AAA, currentRow) } state = nextState - if nextState == "start" { - currentDecoder = nil //renew to a empty state + if nextState == "start" { // reset current state + currentRow = PayInAAARow{} + currentRow.Period = currentPeriod } break } @@ -83,7 +77,7 @@ func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) { return } -func (m *PayInAAAPeriod) processStart(line string) (nextState string) { +func (m *PayInAAARow) processStart(line string) (nextState string) { nextState = "start" if strings.Contains(line, "Loan Number") && strings.Contains(line, "SettDate") && @@ -94,7 +88,7 @@ func (m *PayInAAAPeriod) processStart(line string) (nextState string) { return } -func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) { +func (m *PayInAAARow) processPeriod(line string) (nextState string) { nextState = "LookingForPeriod" if strings.Contains(line, "Period Servicing:") { nextState = "LookingForRows" @@ -103,14 +97,14 @@ func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) { } // Period Servicing: Feb 2020 -func (m *PayInAAAPeriod) getPeriod(line string) (p time.Time, e error) { +func (m *PayInAAARow) getPeriod(line string) (p time.Time, e error) { idx := strings.Index(line, ":") subStr := strings.TrimSpace(line[idx+1:]) p, e = time.Parse("Jan 2006", subStr) return } -func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAARow, valid bool) { +func (m *PayInAAARow) processRow(line string) (nextState string, valid bool) { nextState = "LookingForRows" valid = false allParts := strings.Split(line, " ") @@ -122,11 +116,11 @@ func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAA } if len(el) >= 5 { - row.LoanNumber = el[0] - row.Settlement, _ = time.Parse("02-Jan-06", el[1]) - row.LoanAmount = m.currencyToFloat64(el[2]) - row.Balance = m.currencyToFloat64(el[3]) - row.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element + m.LoanNumber = el[0] + m.Settlement, _ = time.Parse("02-Jan-06", el[1]) + m.LoanAmount = m.currencyToFloat64(el[2]) + m.Balance = m.currencyToFloat64(el[3]) + m.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element valid = true } else { if strings.Contains(line, "Total:") { @@ -138,7 +132,7 @@ func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAA return } -func (m *PayInAAAPeriod) currencyToFloat64(cur string) (ret float64) { +func (m *PayInAAARow) currencyToFloat64(cur string) (ret float64) { cur = strings.ReplaceAll(cur, " ", "") //remove space cur = strings.ReplaceAll(cur, "$", "") //remove $ cur = strings.ReplaceAll(cur, ",", "") //remove , diff --git a/payin-connective.go b/payin-connective.go new file mode 100644 index 0000000..cb53b6c --- /dev/null +++ b/payin-connective.go @@ -0,0 +1,179 @@ +package main + +//sample data +//9/25/2020 Commission Statement +// +// +// +// +//Commission Statement (Trail) - 01/08/2020 to 31/08/2020 - Super Finance Markets Pty Ltd +// +// +//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total +//Split +// +// +// +//ANZ +//687156246 DENG ASHLEY 25/08/2020 ANZ $200,000.00 $97,640.15 Bo Zhu $4.15 tc 100 % $4.15 $0.42 $4.57 +// +//687156254 DENG ASHLEY 25/08/2020 ANZ $360,000.00 $360,000.00 Bo Zhu $10.35 tc 100 % $10.35 $1.04 $11.39 +// +//687156262 DENG ASHLEY 25/08/2020 ANZ $500,000.00 $500,000.00 Bo Zhu $14.38 tc 100 % $14.38 $1.44 $15.82 +// +//687109829 FU JINGYUN 18/08/2020 ANZ $170,000.00 $170,000.00 Bo Zhu $9.78 tc 100 % $9.78 $0.98 $10.76 +// +//687109837 FU JINGYUN 18/08/2020 ANZ $245,000.00 $0.00 Bo Zhu $2.04 tc 100 % $2.04 $0.20 $2.24 +// +//687145432 GU ZAC ZHEN 24/08/2020 ANZ $464,000.00 $464,000.00 Bo Zhu $15.25 tc 100 % $15.25 $1.53 $16.78 +// +//686768902 HAO YANAN 01/07/2020 ANZ $470,000.00 $426,358.02 Bo Zhu $59.40 tc 100 % $59.40 $5.94 $65.34 +// +//686766595 LAI JIAOLONG 01/07/2020 ANZ $188,000.00 $17,726.29 Bo Zhu $2.50 tc 100 % $2.50 $0.25 $2.75 +// +//686766608 LAI JIAOLONG 01/07/2020 ANZ $180,000.00 $179,690.48 Bo Zhu $22.89 tc 100 % $22.89 $2.29 $25.18 +// +//685061729 MENG SICHEN 02/09/2019 ANZ $105,000.00 $100,911.09 Bo Zhu $12.81 tc 100 % $12.81 $1.28 $14.09 +// +//685061737 MENG SICHEN 02/09/2019 ANZ $399,000.00 $391,608.66 Bo Zhu $49.89 tc 100 % $49.89 $4.99 $54.88 +// +//687008884 NERCESSIAN 05/08/2020 ANZ $158,031.59 $158,031.59 Bo Zhu $17.55 tc 100 % $17.55 $1.76 $19.31 +//ALEX OSCAR +// +//685245413 WEN TAO 09/10/2019 ANZ $810,000.00 $463,149.89 Bo Zhu $59.72 tc 100 % $59.72 $5.97 $65.69 +// +//686508297 XIA RENNAN 29/05/2020 ANZ $1,100,000.00 $736,349.83 Bo Zhu $95.95 tc 100 % $95.95 $9.60 $105.55 +// +//686508318 XIA RENNAN 29/05/2020 ANZ $400,000.00 $398,130.00 Bo Zhu $50.80 tc 100 % $50.80 $5.08 $55.88 +//Total (ANZ) $5,749,031.59 $4,463,596.00 $427.46 $427.46 $42.77 $470.23 +// +// +// +//BOC +//100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.64 tc 100 % $42.64 $4.26 $46.90 +//ZHANG +// +//100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.59 tc 100 % $42.59 $4.26 $46.85 +//ZHANG +// +//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 1/3 +//9/25/2020 Commission Statement +// +//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total +//Split +// +// +//Total (BOC) $1,500,000.00 $0.00 $85.23 $85.23 $8.52 $93.75 +// +// +// +//CHLAB +//0041914813 YX01 CHRISTOPHER 12/12/2019 CHLAB $411,775.00 $315,875.03 Ian Lloyd $40.13 tc 100 % $40.13 $4.01 $44.15 +//JOSEPH +//PISANI +// +//0041914813 YX02 CHRISTOPHER 12/12/2019 CHLAB $85,000.00 $85,000.00 Ian Lloyd $10.80 tc 100 % $10.80 $1.08 $11.88 +//JOSEPH +//PISANI +//Total (CHLAB) $496,775.00 $400,875.03 $50.93 $50.93 $5.09 $56.03 +// +// +// +//HSL +//341960587 Huang T 19/09/2019 HSL $400,000.00 $391,801.03 Ian Lloyd $49.91 tc 100 % $49.91 $4.99 $54.90 +// +//851874400 Huang T 03/12/2019 HSL $287,000.00 $149,019.02 Ian Lloyd $18.98 tc 100 % $18.98 $1.90 $20.88 +//Total (HSL) $687,000.00 $540,820.05 $68.89 $68.89 $6.89 $75.78 +// +// +// +// +//ING +//28446283 MR Dylan 01/02/2019 ING $100,431.54 $99,228.57 Ian Lloyd $12.59 tc 100 % $12.59 $1.26 $13.85 +//Waring +// +//28446294 MR Dylan 01/02/2019 ING $326,000.00 $323,513.96 Ian Lloyd $41.08 tc 100 % $41.08 $4.11 $45.19 +//Waring +// +//28915359 MS Emily 03/02/2020 ING $412,000.00 $409,509.47 Ian Lloyd $52.02 tc 100 % $52.02 $5.20 $57.22 +//Jeoung +// +//28924374 MS Rhiannon 10/02/2020 ING $320,000.00 $316,427.48 Ian Lloyd $40.28 tc 100 % $40.28 $4.03 $44.31 +//Lloyd +// +//29044457 MS ZDENKA 13/05/2020 ING $308,000.00 $306,270.89 Ian Lloyd $38.98 tc 100 % $38.98 $3.90 $42.88 +//MARY +//ASHFIELD +// +//29044468 MS ZDENKA 13/05/2020 ING $57,000.00 $56,755.78 Ian Lloyd $7.21 tc 100 % $7.21 $0.72 $7.93 +//MARY +//ASHFIELD +// +//29084842 MS Sharon Joy 11/06/2020 ING $375,000.00 $374,993.96 Ian Lloyd $47.68 tc 100 % $47.68 $4.77 $52.45 +//McClelland +// +//29084853 MS Sharon Joy 11/06/2020 ING $75,000.00 $72,057.85 Ian Lloyd $9.17 tc 100 % $9.17 $0.92 $10.09 +//McClelland +//Total (ING) $1,973,431.54 $1,958,757.96 $249.01 $249.01 $24.90 $273.91 +// +//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 2/3 +//9/25/2020 Commission Statement +// +//Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total +//Split +// +// +// +// +//MEB +//1676620 Ian Francis 03/02/2020 MEB $205,600.00 $203,280.20 Ian Lloyd $25.41 tc 100 % $25.41 $2.54 $27.95 +//Lloyd +// +//823275 Reece Kenneth 24/07/2019 MEB $376,088.56 $368,253.20 Ian Lloyd $46.03 tc 100 % $46.03 $4.60 $50.63 +//Godfrey and +//Sarah Anne +//Heyes +//Total (MEB) $581,688.56 $571,533.40 $71.44 $71.44 $7.14 $78.58 +// +// +// +// +//SGB +//100212227685200 BILAL AKIL 21/02/2020 SGB $700,000.00 $692,515.82 Bo Zhu $86.56 TC 100 % $86.56 $8.66 $95.22 +//JIAOYUE CAO +//Total (SGB) $700,000.00 $692,515.82 $86.56 $86.56 $8.66 $95.22 +// +// +// +//WPC +//700183234270 Lin Feng 25/06/2020 WPC $370,000.00 $210,607.62 Bo Zhu $26.76 tc 100 % $26.76 $2.68 $29.44 +// +//700183234297 Lin Feng 24/06/2020 WPC $250,000.00 -$19.98 Bo Zhu $0.00 tc 0% $0.00 $0.00 $0.00 +// +//700183234238 Lin Feng 24/06/2020 WPC $280,000.00 $278,301.00 Bo Zhu $35.36 tc 100 % $35.36 $3.54 $38.90 +// +//700183234254 Lin Feng 24/06/2020 WPC $600,593.00 $596,337.00 Bo Zhu $75.76 tc 100 % $75.76 $7.58 $83.34 +// +//700183234131 Lin Feng 24/06/2020 WPC $370,000.00 $367,916.00 Bo Zhu $46.74 tc 100 % $46.74 $4.67 $51.41 +// +//700183234174 Lin Feng 24/06/2020 WPC $260,000.00 $258,524.00 Bo Zhu $32.85 tc 100 % $32.85 $3.29 $36.14 +// +//700183234094 Lin Feng 24/06/2020 WPC $393,666.00 $390,833.00 Bo Zhu $49.66 tc 100 % $49.66 $4.97 $54.63 +// +//700183233972 Lin Feng 24/06/2020 WPC $410,000.00 $407,087.00 Bo Zhu $51.72 tc 100 % $51.72 $5.17 $56.89 +// +//700183122084 Zhou Zeyu 01/04/2020 WPC $845,000.00 $309,335.07 Bo Zhu $39.30 tc 100 % $39.30 $3.93 $43.23 +//Total (WPC) $3,779,259.00 $2,818,920.71 $358.15 $358.15 $35.83 $393.98 +// +// +// +//Total (Overall) $15,467,185.69 $11,447,018.97 $1,397.67 $1,397.67 $139.81 $1,537.48 +// +// +// +// +//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 3/3 +// + +type ConnectiveRow struct { +}