SFM-loan-Rest-Api/pay-in-decode.go

package main

import (
	"biukop.com/sfm/loan"
	"encoding/json"
	"errors"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type AiDecodeIncome struct {
	Id    int64
	Input loan.Uploads
	ul    uploadsOnDisk // internal data
	Mime  string        // mime actually detected.
	PayIn []loan.PayIn  // generated PayIn Info
	DecodeIncomeDetails
}

type DecodeIncomeDetails struct {
	Lender     loan.LenderType
	AAA        []PayInAAARow
	Connective []ConnectiveRow
	ResimacXls []ResimacRow
	ResimacPdf []ResimacPdf
}

type AiDecodeJson struct {
	Version string
	V1      DecodeIncomeDetails
}

func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) {
	m.Id = ulMeta.Id
	m.Input = ulMeta
	m.ul.Upload = ulMeta
	m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated

	if allowCachedResult {
		e = m.ReadJson()
		if e == nil {
			return // already decoded
		} else {
			log.Warn("trying to read existing json failed ", e.Error())
			e = nil
		}
	}
	m.PayIn = make([]loan.PayIn, 0, 10)
	switch m.getFileType() {
	case "pdf":
		e = m.decodePdf()
		break
	case "excel", "opensheet":
		e = m.decodeXls()
		break
	default:
		e = errors.New("unknown format")
	}

	if e == nil {
		eJson := m.WriteJson()
		if eJson != nil {
			log.Error("failed to write analysis to json", eJson.Error())
		}
	}
	return
}

func (m *AiDecodeIncome) ReadJson() (e error) {
	if !fileExists(m.ul.jsonPath()) {
		return errors.New(m.ul.jsonPath() + " not found")
	}
	f, e := os.Open(m.ul.jsonPath())
	if e != nil {
		return
	}

	decoder := json.NewDecoder(f)
	data := AiDecodeJson{}
	e = decoder.Decode(&data)
	if e != nil {
		log.Error("failed load existing decode json", e.Error())
		return
	}
	return
}

func (m *AiDecodeIncome) WriteJson() (e error) {
	b, e := json.Marshal(m.DecodeIncomeDetails)
	if e != nil {
		return
	}
	ioutil.WriteFile(m.ul.jsonPath(), b, 0644)
	return
}

func (m *AiDecodeIncome) getFileType() (ret string) {
	strMime, e := GetFileContentType(m.ul.filePath())
	if e != nil {
		return
	}
	m.Mime = strMime

	ret, e = m.ul.GetFileType()
	if e != nil {
		ret = ""
	}
	return
}

func (m *AiDecodeIncome) decodePdf() (e error) {
	cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-")
	out, e := cmd.Output()
	if e != nil {
		log.Error("cannot convert pdf to text ", e)
	}

	raw := string(out)
	switch m.detectFunder(raw) {
	case loan.Lender_AAA:
		m.Lender = loan.Lender_AAA
		e = m.decodeAAAPdf(raw)
		// regardless of error, we pump in all available row successed so far
		for _, row := range m.AAA {
			pi := loan.PayIn{}
			pi.Id = 0
			pi.Ts = row.Period
			pi.Amount = row.LoanFacility
			pi.Lender = loan.Lender_AAA
			pi.Settlement = row.Settlement
			pi.Balance = row.Balance
			pi.OffsetBalance = -1
			pi.IncomeAmount = row.InTrail
			pi.IncomeType = "Trail"
			m.PayIn = append(m.PayIn, pi)
		}
		// log.Println("AAA final result", m.AAA)
		break
	case loan.Lender_Connective:
		m.Lender = loan.Lender_Connective
		e = m.decodeConnectivePdf(out)
		break
	case loan.Lender_Resimac:
		m.Lender = loan.Lender_Resimac
		e = m.decodeResimacPdf(out)
		break
	case loan.Lender_Unknown:
		e = errors.New(loan.Lender_Unknown)
		break // not able to detect Funder
	}
	return
}

func (m *AiDecodeIncome) decodeXls() (e error) {
	e = m.ul.convertExcelTo("csv")
	if e != nil {
		log.Error("cannot convert xls to csv ", e.Error())
	}
	raw := string(m.ul.tmp)
	switch m.detectFunder(raw) {
	case loan.Lender_Resimac:
		m.Lender = loan.Lender_Resimac
		e = m.decodeResimacXls(m.ul.tmp)
		break
	case loan.Lender_Unknown:
		e = errors.New(loan.Lender_Unknown)
		break // not able to detect Funder
	}
	return
}

func (m *AiDecodeIncome) detectFunder(raw string) loan.LenderType {
	if m.isAAA(raw) {
		return loan.Lender_AAA
	}
	if m.isConnective(raw) {
		return loan.Lender_Connective
	}
	if m.isResimacXls(raw) {
		return loan.Lender_Resimac
	}
	if m.isResimacPdf(raw) {
		return loan.Lender_Resimac
	}
	return loan.Lender_Unknown
}

func (m *AiDecodeIncome) isAAA(raw string) bool {
	keyword := "AAA Financial Trail Report"
	lines := strings.Split(raw, "\n")
	return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isConnective(raw string) bool {
	keyword := "connective.com.au"
	lines := strings.Split(raw, "\n")
	return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isResimacXls(raw string) bool {
	keyword := "TRSTCD,ORGNTR,LOANNO,PORTNO,BRNAMX,LNAMT,INTRTE,DELRTE,MARGIN,SETLDX,LNBAL,MANFEE,RSINCP,RSSACT,RSSACP,SACAMT,MOFEE,MANTOT,NEWLON,REFADJ,NETNEW,FIXED,LNFORT,ORGNAM,LNEOM"
	lines := strings.Split(raw, "\n")
	//remove all spaces
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
	}
	return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) isResimacPdf(raw string) bool {
	keyword := "SuperFinanceMarketsPtyLtd-8779"
	lines := strings.Split(raw, "\n")
	//remove all spaces
	for i := 0; i < len(lines); i++ {
		lines[i] = strings.ReplaceAll(lines[i], " ", "") // remove all spaces
	}
	return m.checkFunderKeyword(keyword, lines)
}

func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string) bool {
	for _, line := range lines {
		if strings.Contains(line, keyword) {
			return true
		}
	}
	return false
}