SFM-loan-Rest-Api/pay-in-decode.go

package main

import (
	"biukop.com/sfm/loan"
	"encoding/json"
	"errors"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type AiDecodeIncome struct {
	Id    int64
	Input loan.Uploads
	ul    uploadsOnDisk // internal data
	Mime  string        // mime actually detected.
	PayIn []loan.PayIn  // generated PayIn Info
	DecodeIncomeDetails
}

type DecodeIncomeDetails struct {
	Funders          []loan.FunderType
	AAA              []PayInAAARow
	Connective_ANZ   []ConnectiveRow
	Connective_BOC   []ConnectiveRow
	Connective_CHLAB []ConnectiveRow
	Connective_HSL   []ConnectiveRow
	Connective_ING   []ConnectiveRow
	Connective_MEB   []ConnectiveRow
	Connective_SGB   []ConnectiveRow
	Connective_WPC   []ConnectiveRow
}

type AiDecodeJson struct {
	Version string
	V1      DecodeIncomeDetails
}

func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) {
	m.Id = ulMeta.Id
	m.Input = ulMeta
	m.ul.Upload = ulMeta
	m.PayIn = make([]loan.PayIn, 0, 100)       // finalized payIns, generated
	m.Funders = make([]loan.FunderType, 0, 50) // array of valid funders

	if allowCachedResult {
		e = m.ReadJson()
		if e == nil {
			return // already decoded
		} else {
			log.Warn("trying to read existing json failed ", e.Error())
			e = nil
		}
	}
	m.PayIn = make([]loan.PayIn, 0, 10)
	switch m.getFileType() {
	case "pdf":
		e = m.decodePdf()
		break
	case "excel", "opensheet":
		e = m.decodeXls()
		break
	default:
		e = errors.New("unknown format")
	}

	if e == nil {
		eJson := m.WriteJson()
		if eJson != nil {
			log.Error("failed to write analysis to json", eJson.Error())
		}
	}
	return
}

func (m *AiDecodeIncome) ReadJson() (e error) {
	if !fileExists(m.ul.jsonPath()) {
		return errors.New(m.ul.jsonPath() + " not found")
	}
	f, e := os.Open(m.ul.jsonPath())
	if e != nil {
		return
	}

	decoder := json.NewDecoder(f)
	data := AiDecodeJson{}
	e = decoder.Decode(&data)
	if e != nil {
		log.Error("failed load existing decode json", e.Error())
		return
	}
	return
}

func (m *AiDecodeIncome) WriteJson() (e error) {
	b, e := json.Marshal(m.DecodeIncomeDetails)
	if e != nil {
		return
	}
	ioutil.WriteFile(m.ul.jsonPath(), b, 0644)
	return
}

func (m *AiDecodeIncome) getFileType() (ret string) {
	strMime, e := GetFileContentType(m.ul.filePath())
	if e != nil {
		return
	}
	m.Mime = strMime

	ret, e = m.ul.GetFileType()
	if e != nil {
		ret = ""
	}
	return
}

func (m *AiDecodeIncome) decodePdf() (e error) {
	cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-")
	out, e := cmd.Output()
	if e != nil {
		log.Error("cannot convert pdf to text ", e)
	}

	raw := string(out)
	switch m.detectFunder(raw) {
	case loan.Funder_AAA:
		m.Funders = append(m.Funders, loan.Funder_AAA)
		e = m.decodeAAAPdf(raw)
		// regardless of error, we pump in all available row successed so far
		for _, row := range m.AAA {
			pi := loan.PayIn{}
			pi.Id = 0
			pi.Ts = row.Period
			pi.Amount = row.LoanAmount
			pi.Lender = loan.Funder_AAA
			pi.Settlement = row.Settlement
			pi.Balance = row.Balance
			pi.OffsetBalance = -1
			pi.IncomeAmount = row.InTrail
			pi.IncomeType = "Trail"
			m.PayIn = append(m.PayIn, pi)
		}
		log.Println("AAA final result", m.AAA)
		break
	case loan.Funder_Unknown:
		e = errors.New(loan.Funder_Unknown)
		break // not able to detect Funder
	}
	return
}

func (m *AiDecodeIncome) decodeXls() (e error) {
	e = errors.New("not implemented yet")
	return
}

func (m *AiDecodeIncome) detectFunder(raw string) loan.FunderType {
	if m.isAAA(raw) {
		return loan.Funder_AAA
	}

	return loan.Funder_Unknown
}

func (m *AiDecodeIncome) isAAA(raw string) bool {
	keyword := "AAA Financial Trail Report"
	lines := strings.Split(raw, "\n")
	return m.checkFunderKeyword(keyword, lines, 0, 3)
}

func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool {
	for idx, line := range lines {
		// first 10 lines has Key word
		if strings.Contains(line, keyword) && idx >= start && idx <= 10 {
			return true
		}
	}
	return false
}