package main import ( log "github.com/sirupsen/logrus" "regexp" "strconv" "strings" "time" ) var sample_data_connective = ` 9/25/2020 Commission Statement Commission Statement (Trail) - 01/08/2020 to 31/08/2020 - Super Finance Markets Pty Ltd Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total Split ANZ 687156246 DENG ASHLEY 25/08/2020 ANZ $200,000.00 $97,640.15 Bo Zhu $4.15 tc 100 % $4.15 $0.42 $4.57 687156254 DENG ASHLEY 25/08/2020 ANZ $360,000.00 $360,000.00 Bo Zhu $10.35 tc 100 % $10.35 $1.04 $11.39 687156262 DENG ASHLEY 25/08/2020 ANZ $500,000.00 $500,000.00 Bo Zhu $14.38 tc 100 % $14.38 $1.44 $15.82 687109829 FU JINGYUN 18/08/2020 ANZ $170,000.00 $170,000.00 Bo Zhu $9.78 tc 100 % $9.78 $0.98 $10.76 687109837 FU JINGYUN 18/08/2020 ANZ $245,000.00 $0.00 Bo Zhu $2.04 tc 100 % $2.04 $0.20 $2.24 687145432 GU ZAC ZHEN 24/08/2020 ANZ $464,000.00 $464,000.00 Bo Zhu $15.25 tc 100 % $15.25 $1.53 $16.78 686768902 HAO YANAN 01/07/2020 ANZ $470,000.00 $426,358.02 Bo Zhu $59.40 tc 100 % $59.40 $5.94 $65.34 686766595 LAI JIAOLONG 01/07/2020 ANZ $188,000.00 $17,726.29 Bo Zhu $2.50 tc 100 % $2.50 $0.25 $2.75 686766608 LAI JIAOLONG 01/07/2020 ANZ $180,000.00 $179,690.48 Bo Zhu $22.89 tc 100 % $22.89 $2.29 $25.18 685061729 MENG SICHEN 02/09/2019 ANZ $105,000.00 $100,911.09 Bo Zhu $12.81 tc 100 % $12.81 $1.28 $14.09 685061737 MENG SICHEN 02/09/2019 ANZ $399,000.00 $391,608.66 Bo Zhu $49.89 tc 100 % $49.89 $4.99 $54.88 687008884 NERCESSIAN 05/08/2020 ANZ $158,031.59 $158,031.59 Bo Zhu $17.55 tc 100 % $17.55 $1.76 $19.31 ALEX OSCAR 685245413 WEN TAO 09/10/2019 ANZ $810,000.00 $463,149.89 Bo Zhu $59.72 tc 100 % $59.72 $5.97 $65.69 686508297 XIA RENNAN 29/05/2020 ANZ $1,100,000.00 $736,349.83 Bo Zhu $95.95 tc 100 % $95.95 $9.60 $105.55 686508318 XIA RENNAN 29/05/2020 ANZ $400,000.00 $398,130.00 Bo Zhu $50.80 tc 100 % $50.80 $5.08 $55.88 Total (ANZ) $5,749,031.59 $4,463,596.00 $427.46 $427.46 $42.77 $470.23 BOC 100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.64 tc 100 % $42.64 $4.26 $46.90 ZHANG 100001403165615 YINGJIAO 18/02/2020 BOC $750,000.00 $0.00 Bo Zhu $42.59 tc 100 % $42.59 $4.26 $46.85 ZHANG https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 1/3 9/25/2020 Commission Statement Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total Split Total (BOC) $1,500,000.00 $0.00 $85.23 $85.23 $8.52 $93.75 CHLAB 0041914813 YX01 CHRISTOPHER 12/12/2019 CHLAB $411,775.00 $315,875.03 Ian Lloyd $40.13 tc 100 % $40.13 $4.01 $44.15 JOSEPH PISANI 0041914813 YX02 CHRISTOPHER 12/12/2019 CHLAB $85,000.00 $85,000.00 Ian Lloyd $10.80 tc 100 % $10.80 $1.08 $11.88 JOSEPH PISANI Total (CHLAB) $496,775.00 $400,875.03 $50.93 $50.93 $5.09 $56.03 HSL 341960587 Huang T 19/09/2019 HSL $400,000.00 $391,801.03 Ian Lloyd $49.91 tc 100 % $49.91 $4.99 $54.90 851874400 Huang T 03/12/2019 HSL $287,000.00 $149,019.02 Ian Lloyd $18.98 tc 100 % $18.98 $1.90 $20.88 Total (HSL) $687,000.00 $540,820.05 $68.89 $68.89 $6.89 $75.78 ING 28446283 MR Dylan 01/02/2019 ING $100,431.54 $99,228.57 Ian Lloyd $12.59 tc 100 % $12.59 $1.26 $13.85 Waring 28446294 MR Dylan 01/02/2019 ING $326,000.00 $323,513.96 Ian Lloyd $41.08 tc 100 % $41.08 $4.11 $45.19 Waring 28915359 MS Emily 03/02/2020 ING $412,000.00 $409,509.47 Ian Lloyd $52.02 tc 100 % $52.02 $5.20 $57.22 Jeoung 28924374 MS Rhiannon 10/02/2020 ING $320,000.00 $316,427.48 Ian Lloyd $40.28 tc 100 % $40.28 $4.03 $44.31 Lloyd 29044457 MS ZDENKA 13/05/2020 ING $308,000.00 $306,270.89 Ian Lloyd $38.98 tc 100 % $38.98 $3.90 $42.88 MARY ASHFIELD 29044468 MS ZDENKA 13/05/2020 ING $57,000.00 $56,755.78 Ian Lloyd $7.21 tc 100 % $7.21 $0.72 $7.93 MARY ASHFIELD 29084842 MS Sharon Joy 11/06/2020 ING $375,000.00 $374,993.96 Ian Lloyd $47.68 tc 100 % $47.68 $4.77 $52.45 McClelland 29084853 MS Sharon Joy 11/06/2020 ING $75,000.00 $72,057.85 Ian Lloyd $9.17 tc 100 % $9.17 $0.92 $10.09 McClelland Total (ING) $1,973,431.54 $1,958,757.96 $249.01 $249.01 $24.90 $273.91 https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 2/3 9/25/2020 Commission Statement Loan Account Name Settled Lender Loan Amount Balance Associate Comm Type % Paid GST Total Split MEB 1676620 Ian Francis 03/02/2020 MEB $205,600.00 $203,280.20 Ian Lloyd $25.41 tc 100 % $25.41 $2.54 $27.95 Lloyd 823275 Reece Kenneth 24/07/2019 MEB $376,088.56 $368,253.20 Ian Lloyd $46.03 tc 100 % $46.03 $4.60 $50.63 Godfrey and Sarah Anne Heyes Total (MEB) $581,688.56 $571,533.40 $71.44 $71.44 $7.14 $78.58 SGB 100212227685200 BILAL AKIL 21/02/2020 SGB $700,000.00 $692,515.82 Bo Zhu $86.56 TC 100 % $86.56 $8.66 $95.22 JIAOYUE CAO Total (SGB) $700,000.00 $692,515.82 $86.56 $86.56 $8.66 $95.22 WPC 700183234270 Lin Feng 25/06/2020 WPC $370,000.00 $210,607.62 Bo Zhu $26.76 tc 100 % $26.76 $2.68 $29.44 700183234297 Lin Feng 24/06/2020 WPC $250,000.00 -$19.98 Bo Zhu $0.00 tc 0% $0.00 $0.00 $0.00 700183234238 Lin Feng 24/06/2020 WPC $280,000.00 $278,301.00 Bo Zhu $35.36 tc 100 % $35.36 $3.54 $38.90 700183234254 Lin Feng 24/06/2020 WPC $600,593.00 $596,337.00 Bo Zhu $75.76 tc 100 % $75.76 $7.58 $83.34 700183234131 Lin Feng 24/06/2020 WPC $370,000.00 $367,916.00 Bo Zhu $46.74 tc 100 % $46.74 $4.67 $51.41 700183234174 Lin Feng 24/06/2020 WPC $260,000.00 $258,524.00 Bo Zhu $32.85 tc 100 % $32.85 $3.29 $36.14 700183234094 Lin Feng 24/06/2020 WPC $393,666.00 $390,833.00 Bo Zhu $49.66 tc 100 % $49.66 $4.97 $54.63 700183233972 Lin Feng 24/06/2020 WPC $410,000.00 $407,087.00 Bo Zhu $51.72 tc 100 % $51.72 $5.17 $56.89 700183122084 Zhou Zeyu 01/04/2020 WPC $845,000.00 $309,335.07 Bo Zhu $39.30 tc 100 % $39.30 $3.93 $43.23 Total (WPC) $3,779,259.00 $2,818,920.71 $358.15 $358.15 $35.83 $393.98 Total (Overall) $15,467,185.69 $11,447,018.97 $1,397.67 $1,397.67 $139.81 $1,537.48 https://m5.connective.com.au/api/comms/showCommList.jsp?id=b472e1c4-fec4-11ea-94e6-005056b5e136&comm_type=tc&lenderId=&associateId= 3/3 ` type ConnectiveRow struct { Period time.Time LoanAccount string Name string Settled time.Time Lender string LoanAmount float64 Balance float64 Associate string Comm float64 Type string Percent float64 Paid float64 GST float64 Total float64 } func (m *AiDecodeIncome) isConnective(raw string) bool { keyword := "connective.com.au" lines := strings.Split(raw, "\n") return m.isKeywordExist(keyword, lines) } func (m *AiDecodeIncome) decodeConnectivePdf(raw []byte) (e error) { m.Connective = make([]ConnectiveRow, 0, 30) period := `(?i)Commission +Statement {0}[ ()a-z-]*((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4}) +to +((?:(?:[12][0-9]|0?[1-9])[/.-]0?2|(?:30|[12][0-9]|0?[1-9])[/.-](?:0?[469]|11)|(?:3[01]|[12][0-9]|0?[1-9])[/.-](?:0?[13578]|1[02]))[/.-][0-9]{4})[ -]+Super Finance Markets Pty Ltd` pattern := `((?:\d{5,15} ?[\dA-Z]+){1,16}) +([A-Za-z]+(?: [ A-Za-z]){0,5}[A-Za-z]+) +((?:3[01]|[12][0-9]|0?[1-9])[/.-](?:1[0-2]|0?[1-9])[/.-][0-9]{4}) +([A-Z]+) +(\$[\d,.]+) +([ $][\d,.]+) +([ A-Za-z]+ ?[A-Za-z]+ ?[A-Za-z]+) +([ $][\d ]{1,3}(?:[ ,][\d ]{3})*(?:[ .][\d ]{2})?) +([A-Za-z]+) +(\d+)[ %]+ +(\$[\d ]{1,3}(?:,\d{1,3})*(?:\.\d{2})?) +(\$\d{1,3}(?:,\d{3})*(?:\.\d{2})?) +([ $][\d ]{1,3}(?:,\d{3})?(?:\.\d{2})?)` periodLine, e := regexp.Compile(period) if e != nil { return } validLine, e := regexp.Compile(pattern) // error if regexp invalid if e != nil { return } periods := periodLine.FindSubmatch(raw) CurrentPeriod, _ := time.Parse("02/01/2006", string(periods[1])) matches := validLine.FindAllSubmatch(raw, -1) if matches == nil { log.Warn("connective decode found nothing no matches", string(raw)) return } for _, v := range matches { cr := ConnectiveRow{Period: CurrentPeriod} cr.LoanAccount = string(v[1]) cr.Name = string(v[2]) cr.Settled, _ = time.Parse("02/01/2006", string(v[3])) cr.Lender = string(v[4]) cr.LoanAmount = m.currencyToFloat64(string(v[5])) cr.Balance = m.currencyToFloat64(string(v[6])) cr.Associate = string(v[7]) cr.Comm = m.currencyToFloat64(string(v[8])) cr.Type = string(v[9]) percentInt, _ := strconv.Atoi(string(v[10])) cr.Percent = float64(percentInt) / 100 cr.Paid = m.currencyToFloat64(string(v[11])) cr.GST = m.currencyToFloat64(string(v[12])) cr.Total = m.currencyToFloat64(string(v[13])) m.Connective = append(m.Connective, cr) } return } func (m *AiDecodeIncome) currencyToFloat64(cur string) (ret float64) { cur = strings.ReplaceAll(cur, " ", "") //remove space cur = strings.ReplaceAll(cur, "$", "") //remove $ cur = strings.ReplaceAll(cur, ",", "") //remove , ret, _ = strconv.ParseFloat(cur, 64) return ret }