change Trail to income_amount to add income_type

apiV1UploadAnalysis.go

 
 	analysisMutex.Lock()
 	ai := AiDecodeIncome{}
-	e = ai.decodeUploadToPayIn(ul)
+	e = ai.decodeUploadToPayIn(ul, true)
 	analysisMutex.Unlock()
 	if e != nil {
-		log.Error("Invalid uploads Id cannot conver to integer", Id, e)
+		log.Error("Invalid uploads Id cannot convert to integer", Id, e)
 		apiV1Server500Error(w, r)
 		return
 	}
 	}
 
 	ai := AiDecodeIncome{}
-	e = ai.decodeUploadToPayIn(ul)
+	e = ai.decodeUploadToPayIn(ul, false)
 	if e != nil {
 		log.Error("Cannot decode upload", Id, e)
 		apiV1Server500Error(w, r)

apiV1Uploads.go

 }
 
 func forceHttpDownload(r *http.Request) bool {
-	keys, ok := r.URL.Query()["download"]
+	return httpQueryParamHas(r, "download", "force")
+	//keys, ok := r.URL.Query()["download"]
+	//
+	//if !ok || len(keys[0]) < 1 {
+	//	return false
+	//}
+	//key := keys[0]
+	//return key == "force"
+}
+
+func httpQueryParamHas(r *http.Request, key string, expectedValue string) (has bool) {
+	keys, ok := r.URL.Query()[key]
 
 	if !ok || len(keys[0]) < 1 {
 		return false
 	}
-	key := keys[0]
-	return key == "force"
+	k := keys[0]
+	return k == expectedValue
 }

config.go

 	}
 	m.UploadsDir.PdfDir = p + string(os.PathSeparator) //change it to absolute dir
 
+	// convert to absolute path : JsonDir
+	p, e = filepath.Abs(m.UploadsDir.JsonDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad pdf file dir", m.UploadsDir.JsonDir, e)
+	}
+	m.UploadsDir.JsonDir = p + string(os.PathSeparator) //change it to absolute dir
+
 	// convert to absolute path : TmpDir
 	p, e = filepath.Abs(m.TempDir)
 	if e != nil {

pay-in-decode.go

 
 import (
 	"biukop.com/sfm/loan"
+	"encoding/json"
 	"errors"
 	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+	"os"
 	"os/exec"
 	"strings"
 )
 
-type FunderType string
+type AiDecodeIncome struct {
+	Id    int64
+	Input loan.Uploads
+	ul    uploadsOnDisk // internal data
+	Mime  string        // mime actually detected.
+	PayIn []loan.PayIn  // generated PayIn Info
+	DecodeIncomeDetails
+}
 
-const (
-	Funder_AAA     FunderType = "AAA Financial"
-	Funder_Pepper             = "Pepper"
-	Funder_Resimac            = "Resimac"
-	Funder_Unknown            = "cannot detect funder type"
-)
+type DecodeIncomeDetails struct {
+	Funders          []loan.FunderType
+	AAA              []PayInAAARow
+	Connective_ANZ   []ConnectiveRow
+	Connective_BOC   []ConnectiveRow
+	Connective_CHLAB []ConnectiveRow
+	Connective_HSL   []ConnectiveRow
+	Connective_ING   []ConnectiveRow
+	Connective_MEB   []ConnectiveRow
+	Connective_SGB   []ConnectiveRow
+	Connective_WPC   []ConnectiveRow
+}
 
-type AiDecodeIncome struct {
-	Id     int64
-	Input  loan.Uploads
-	ul     uploadsOnDisk // internal data
-	Mime   string        //mime actually detected.
-	PayIn  []loan.PayIn
-	Funder FunderType
-	AAA    []PayInAAAPeriod
+type AiDecodeJson struct {
+	Version string
+	V1      DecodeIncomeDetails
 }
 
-func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) {
+func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) {
 	m.Id = ulMeta.Id
 	m.Input = ulMeta
 	m.ul.Upload = ulMeta
-
+	m.PayIn = make([]loan.PayIn, 0, 100)       // finalized payIns, generated
+	m.Funders = make([]loan.FunderType, 0, 50) // array of valid funders
+
+	if allowCachedResult {
+		e = m.ReadJson()
+		if e == nil {
+			return // already decoded
+		} else {
+			log.Warn("trying to read existing json failed ", e.Error())
+			e = nil
+		}
+	}
 	m.PayIn = make([]loan.PayIn, 0, 10)
 	switch m.getFileType() {
 	case "pdf":
-		m.decodePdf()
+		e = m.decodePdf()
 		break
 	case "excel", "opensheet":
-		m.decodeXls()
+		e = m.decodeXls()
 		break
 	default:
 		e = errors.New("unknown format")
-		m.Funder = "" // mark unknown decoding
 	}
 
+	if e == nil {
+		eJson := m.WriteJson()
+		if eJson != nil {
+			log.Error("failed to write analysis to json", eJson.Error())
+		}
+	}
+	return
+}
+
+func (m *AiDecodeIncome) ReadJson() (e error) {
+	if !fileExists(m.ul.jsonPath()) {
+		return errors.New(m.ul.jsonPath() + " not found")
+	}
+	f, e := os.Open(m.ul.jsonPath())
+	if e != nil {
+		return
+	}
+
+	decoder := json.NewDecoder(f)
+	data := AiDecodeJson{}
+	e = decoder.Decode(&data)
+	if e != nil {
+		log.Error("failed load existing decode json", e.Error())
+		return
+	}
+	return
+}
+
+func (m *AiDecodeIncome) WriteJson() (e error) {
+	b, e := json.Marshal(m.DecodeIncomeDetails)
+	if e != nil {
+		return
+	}
+	ioutil.WriteFile(m.ul.jsonPath(), b, 0644)
 	return
 }
 
 
 	raw := string(out)
 	switch m.detectFunder(raw) {
-	case Funder_AAA:
-		m.Funder = Funder_AAA
+	case loan.Funder_AAA:
+		m.Funders = append(m.Funders, loan.Funder_AAA)
 		e = m.decodeAAAPdf(raw)
+		for _, row := range m.AAA {
+			pi := loan.PayIn{}
+			pi.Id = 0
+			pi.Ts = row.Period
+			pi.Amount = row.LoanAmount
+			pi.Lender = loan.Funder_AAA
+			pi.Settlement = row.Settlement
+			pi.Balance = row.Balance
+			pi.OffsetBalance = -1
+			pi.IncomeAmount = row.InTrail
+			pi.IncomeType = "Trail"
+
+			m.PayIn = append(m.PayIn, pi)
+		}
 		log.Println("AAA final result", m.AAA)
 		break
-	case Funder_Unknown:
-		e = errors.New(Funder_Unknown)
+	case loan.Funder_Unknown:
+		e = errors.New(loan.Funder_Unknown)
 		break // not able to detect Funder
 	}
 	return
 }
 
 func (m *AiDecodeIncome) decodeXls() (e error) {
+	e = errors.New("not implemented yet")
 	return
 }
 
-func (m *AiDecodeIncome) detectFunder(raw string) FunderType {
+func (m *AiDecodeIncome) detectFunder(raw string) loan.FunderType {
 	if m.isAAA(raw) {
-		return Funder_AAA
+		return loan.Funder_AAA
 	}
 
-	return Funder_Unknown
+	return loan.Funder_Unknown
 }
 
 func (m *AiDecodeIncome) isAAA(raw string) bool {

payIn-AAA.go

 */
 
 type PayInAAARow struct {
+	Period     time.Time
 	LoanNumber string
 	Settlement time.Time
 	LoanAmount float64
 	InTrail    float64
 }
 
-type PayInAAAPeriod struct {
-	Period time.Time
-	Rows   []PayInAAARow
-}
-
 func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) {
-	m.AAA = make([]PayInAAAPeriod, 0, 10)
+	m.AAA = make([]PayInAAARow, 0, 10)
 	lines := strings.Split(raw, "\n")
 
-	var currentDecoder *PayInAAAPeriod = nil
+	var currentRow = PayInAAARow{}
+	var currentPeriod = time.Time{}
 	state := "start"
 	for _, l := range lines { // DFA, wow, finally it's used. after years of learning
 		switch state {
 		case "start":
-			state = currentDecoder.processStart(l)
+			state = currentRow.processStart(l)
 			if state == "LookingForPeriod" {
 				// determine column index, if their column is changing
 			}
 			break
 		case "LookingForPeriod":
-			state = currentDecoder.processPeriod(l)
+			state = currentRow.processPeriod(l)
 			if state == "LookingForRows" {
-				Period, e := currentDecoder.getPeriod(l)
+				currentPeriod, e = currentRow.getPeriod(l)
 				if e != nil {
 					log.Warn("cannot find period", l, e)
 					state = "LookingForPeriod"
 				} else {
-					m.AAA = append(m.AAA, PayInAAAPeriod{})
-					idx := len(m.AAA) - 1
-					currentDecoder = &(m.AAA[idx])
-					currentDecoder.Rows = make([]PayInAAARow, 0, 10)
-					currentDecoder.Period = Period
+					currentRow.Period = currentPeriod
 				}
 			}
 			break
 		case "LookingForRows", "LookingForRowsSkipCurrent":
-			nextState, row, valid := currentDecoder.processRow(l)
-			if valid && currentDecoder != nil {
-				currentDecoder.Rows = append(currentDecoder.Rows, row)
+			nextState, valid := currentRow.processRow(l)
+			if valid {
+				m.AAA = append(m.AAA, currentRow)
 			}
 			state = nextState
-			if nextState == "start" {
-				currentDecoder = nil //renew to a empty state
+			if nextState == "start" { // reset current state
+				currentRow = PayInAAARow{}
+				currentRow.Period = currentPeriod
 			}
 			break
 		}
 	return
 }
 
-func (m *PayInAAAPeriod) processStart(line string) (nextState string) {
+func (m *PayInAAARow) processStart(line string) (nextState string) {
 	nextState = "start"
 	if strings.Contains(line, "Loan Number") &&
 		strings.Contains(line, "SettDate") &&
 	return
 }
 
-func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) {
+func (m *PayInAAARow) processPeriod(line string) (nextState string) {
 	nextState = "LookingForPeriod"
 	if strings.Contains(line, "Period Servicing:") {
 		nextState = "LookingForRows"
 }
 
 // Period Servicing: Feb 2020
-func (m *PayInAAAPeriod) getPeriod(line string) (p time.Time, e error) {
+func (m *PayInAAARow) getPeriod(line string) (p time.Time, e error) {
 	idx := strings.Index(line, ":")
 	subStr := strings.TrimSpace(line[idx+1:])
 	p, e = time.Parse("Jan 2006", subStr)
 	return
 }
 
-func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAARow, valid bool) {
+func (m *PayInAAARow) processRow(line string) (nextState string, valid bool) {
 	nextState = "LookingForRows"
 	valid = false
 	allParts := strings.Split(line, " ")
 	}
 
 	if len(el) >= 5 {
-		row.LoanNumber = el[0]
-		row.Settlement, _ = time.Parse("02-Jan-06", el[1])
-		row.LoanAmount = m.currencyToFloat64(el[2])
-		row.Balance = m.currencyToFloat64(el[3])
-		row.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element
+		m.LoanNumber = el[0]
+		m.Settlement, _ = time.Parse("02-Jan-06", el[1])
+		m.LoanAmount = m.currencyToFloat64(el[2])
+		m.Balance = m.currencyToFloat64(el[3])
+		m.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element
 		valid = true
 	} else {
 		if strings.Contains(line, "Total:") {
 	return
 }
 
-func (m *PayInAAAPeriod) currencyToFloat64(cur string) (ret float64) {
+func (m *PayInAAARow) currencyToFloat64(cur string) (ret float64) {
 	cur = strings.ReplaceAll(cur, " ", "") //remove space
 	cur = strings.ReplaceAll(cur, "$", "") //remove $
 	cur = strings.ReplaceAll(cur, ",", "") //remove ,

payin-connective.go

+package main
+
+//sample data
+//9/25/2020                                                                                       Commission Statement
+//
+//
+//
+//
+//Commission Statement (Trail) - 01/08/2020 to 31/08/2020 - Super Finance Markets Pty Ltd
+//
+//
+//Loan Account      Name             Settled     Lender   Loan Amount          Balance               Associate   Comm              Type   %       Paid       GST       Total
+//Split
+//
+//
+//
+//ANZ
+//687156246         DENG ASHLEY     25/08/2020   ANZ      $200,000.00          $97,640.15           Bo Zhu       $4.15             tc     100 %   $4.15      $0.42     $4.57
+//
+//687156254         DENG ASHLEY     25/08/2020   ANZ      $360,000.00          $360,000.00          Bo Zhu       $10.35            tc     100 %   $10.35     $1.04     $11.39
+//
+//687156262         DENG ASHLEY     25/08/2020   ANZ      $500,000.00          $500,000.00          Bo Zhu       $14.38            tc     100 %   $14.38     $1.44     $15.82
+//
+//687109829         FU JINGYUN      18/08/2020   ANZ      $170,000.00          $170,000.00          Bo Zhu       $9.78             tc     100 %   $9.78      $0.98     $10.76
+//
+//687109837         FU JINGYUN      18/08/2020   ANZ      $245,000.00          $0.00                Bo Zhu       $2.04             tc     100 %   $2.04      $0.20     $2.24
+//
+//687145432         GU ZAC ZHEN     24/08/2020   ANZ      $464,000.00          $464,000.00          Bo Zhu       $15.25            tc     100 %   $15.25     $1.53     $16.78
+//
+//686768902         HAO YANAN       01/07/2020   ANZ      $470,000.00          $426,358.02          Bo Zhu       $59.40            tc     100 %   $59.40     $5.94     $65.34
+//
+//686766595         LAI JIAOLONG    01/07/2020   ANZ      $188,000.00          $17,726.29           Bo Zhu       $2.50             tc     100 %   $2.50      $0.25     $2.75
+//
+//686766608         LAI JIAOLONG    01/07/2020   ANZ      $180,000.00          $179,690.48          Bo Zhu       $22.89            tc     100 %   $22.89     $2.29     $25.18
+//
+//685061729         MENG SICHEN     02/09/2019   ANZ      $105,000.00          $100,911.09          Bo Zhu       $12.81            tc     100 %   $12.81     $1.28     $14.09
+//
+//685061737         MENG SICHEN     02/09/2019   ANZ      $399,000.00          $391,608.66          Bo Zhu       $49.89            tc     100 %   $49.89     $4.99     $54.88
+//
+//687008884         NERCESSIAN      05/08/2020   ANZ      $158,031.59          $158,031.59          Bo Zhu       $17.55            tc     100 %   $17.55     $1.76     $19.31
+//ALEX OSCAR
+//
+//685245413         WEN TAO         09/10/2019   ANZ      $810,000.00          $463,149.89          Bo Zhu       $59.72            tc     100 %   $59.72     $5.97     $65.69
+//
+//686508297         XIA RENNAN      29/05/2020   ANZ      $1,100,000.00        $736,349.83          Bo Zhu       $95.95            tc     100 %   $95.95     $9.60     $105.55
+//
+//686508318         XIA RENNAN      29/05/2020   ANZ      $400,000.00          $398,130.00          Bo Zhu       $50.80            tc     100 %   $50.80     $5.08     $55.88
+//Total (ANZ)                                             $5,749,031.59        $4,463,596.00                       $427.46                        $427.46    $42.77    $470.23
+//
+//
+//
+//BOC
+//100001403165615   YINGJIAO        18/02/2020   BOC      $750,000.00          $0.00                Bo Zhu       $42.64            tc     100 %   $42.64     $4.26     $46.90
+//ZHANG
+//
+//100001403165615   YINGJIAO        18/02/2020   BOC      $750,000.00          $0.00                Bo Zhu       $42.59            tc     100 %   $42.59     $4.26     $46.85
+//ZHANG
+//
+//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId=                                                  1/3
+//9/25/2020                                                                                        Commission Statement
+//
+//Loan Account      Name             Settled     Lender   Loan Amount          Balance               Associate   Comm              Type   %       Paid       GST       Total
+//Split
+//
+//
+//Total (BOC)                                             $1,500,000.00        $0.00                                $85.23                        $85.23     $8.52     $93.75
+//
+//
+//
+//CHLAB
+//0041914813 YX01   CHRISTOPHER     12/12/2019   CHLAB    $411,775.00          $315,875.03           Ian Lloyd   $40.13            tc     100 %   $40.13     $4.01     $44.15
+//JOSEPH
+//PISANI
+//
+//0041914813 YX02   CHRISTOPHER     12/12/2019   CHLAB    $85,000.00           $85,000.00            Ian Lloyd   $10.80            tc     100 %   $10.80     $1.08     $11.88
+//JOSEPH
+//PISANI
+//Total (CHLAB)                                           $496,775.00          $400,875.03                          $50.93                        $50.93     $5.09     $56.03
+//
+//
+//
+//HSL
+//341960587         Huang T         19/09/2019   HSL      $400,000.00          $391,801.03           Ian Lloyd   $49.91            tc     100 %   $49.91     $4.99     $54.90
+//
+//851874400         Huang T         03/12/2019   HSL      $287,000.00          $149,019.02           Ian Lloyd   $18.98            tc     100 %   $18.98     $1.90     $20.88
+//Total (HSL)                                             $687,000.00          $540,820.05                          $68.89                        $68.89     $6.89     $75.78
+//
+//
+//
+//
+//ING
+//28446283          MR Dylan        01/02/2019   ING      $100,431.54          $99,228.57            Ian Lloyd   $12.59            tc     100 %   $12.59     $1.26     $13.85
+//Waring
+//
+//28446294          MR Dylan        01/02/2019   ING      $326,000.00          $323,513.96           Ian Lloyd   $41.08            tc     100 %   $41.08     $4.11     $45.19
+//Waring
+//
+//28915359          MS Emily        03/02/2020   ING      $412,000.00          $409,509.47           Ian Lloyd   $52.02            tc     100 %   $52.02     $5.20     $57.22
+//Jeoung
+//
+//28924374          MS Rhiannon     10/02/2020   ING      $320,000.00          $316,427.48           Ian Lloyd   $40.28            tc     100 %   $40.28     $4.03     $44.31
+//Lloyd
+//
+//29044457          MS ZDENKA       13/05/2020   ING      $308,000.00          $306,270.89           Ian Lloyd   $38.98            tc     100 %   $38.98     $3.90     $42.88
+//MARY
+//ASHFIELD
+//
+//29044468          MS ZDENKA       13/05/2020   ING      $57,000.00           $56,755.78            Ian Lloyd   $7.21             tc     100 %   $7.21      $0.72     $7.93
+//MARY
+//ASHFIELD
+//
+//29084842          MS Sharon Joy   11/06/2020   ING      $375,000.00          $374,993.96           Ian Lloyd   $47.68            tc     100 %   $47.68     $4.77     $52.45
+//McClelland
+//
+//29084853          MS Sharon Joy   11/06/2020   ING      $75,000.00           $72,057.85            Ian Lloyd   $9.17             tc     100 %   $9.17      $0.92     $10.09
+//McClelland
+//Total (ING)                                             $1,973,431.54        $1,958,757.96                        $249.01                       $249.01    $24.90    $273.91
+//
+//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId=                                                  2/3
+//9/25/2020                                                                                        Commission Statement
+//
+//Loan Account       Name            Settled      Lender   Loan Amount         Balance               Associate   Comm              Type   %       Paid         GST        Total
+//Split
+//
+//
+//
+//
+//MEB
+//1676620            Ian Francis     03/02/2020   MEB      $205,600.00         $203,280.20           Ian Lloyd   $25.41            tc     100 %   $25.41       $2.54      $27.95
+//Lloyd
+//
+//823275             Reece Kenneth   24/07/2019   MEB      $376,088.56         $368,253.20           Ian Lloyd   $46.03            tc     100 %   $46.03       $4.60      $50.63
+//Godfrey and
+//Sarah Anne
+//Heyes
+//Total (MEB)                                              $581,688.56         $571,533.40                          $71.44                        $71.44       $7.14      $78.58
+//
+//
+//
+//
+//SGB
+//100212227685200    BILAL AKIL      21/02/2020   SGB      $700,000.00         $692,515.82           Bo Zhu      $86.56            TC     100 %   $86.56       $8.66      $95.22
+//JIAOYUE CAO
+//Total (SGB)                                              $700,000.00         $692,515.82                          $86.56                        $86.56       $8.66      $95.22
+//
+//
+//
+//WPC
+//700183234270       Lin Feng        25/06/2020   WPC      $370,000.00         $210,607.62           Bo Zhu      $26.76            tc     100 %   $26.76       $2.68      $29.44
+//
+//700183234297       Lin Feng        24/06/2020   WPC      $250,000.00         -$19.98               Bo Zhu      $0.00             tc     0%      $0.00        $0.00      $0.00
+//
+//700183234238       Lin Feng        24/06/2020   WPC      $280,000.00         $278,301.00           Bo Zhu      $35.36            tc     100 %   $35.36       $3.54      $38.90
+//
+//700183234254       Lin Feng        24/06/2020   WPC      $600,593.00         $596,337.00           Bo Zhu      $75.76            tc     100 %   $75.76       $7.58      $83.34
+//
+//700183234131       Lin Feng        24/06/2020   WPC      $370,000.00         $367,916.00           Bo Zhu      $46.74            tc     100 %   $46.74       $4.67      $51.41
+//
+//700183234174       Lin Feng        24/06/2020   WPC      $260,000.00         $258,524.00           Bo Zhu      $32.85            tc     100 %   $32.85       $3.29      $36.14
+//
+//700183234094       Lin Feng        24/06/2020   WPC      $393,666.00         $390,833.00           Bo Zhu      $49.66            tc     100 %   $49.66       $4.97      $54.63
+//
+//700183233972       Lin Feng        24/06/2020   WPC      $410,000.00         $407,087.00           Bo Zhu      $51.72            tc     100 %   $51.72       $5.17      $56.89
+//
+//700183122084       Zhou Zeyu       01/04/2020   WPC      $845,000.00         $309,335.07           Bo Zhu      $39.30            tc     100 %   $39.30       $3.93      $43.23
+//Total (WPC)                                              $3,779,259.00       $2,818,920.71                        $358.15                       $358.15      $35.83     $393.98
+//
+//
+//
+//Total (Overall)                                          $15,467,185.69      $11,447,018.97                       $1,397.67                     $1,397.67    $139.81    $1,537.48
+//
+//
+//
+//
+//https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId=                                                       3/3
+//
+
+type ConnectiveRow struct {
+}