package main import ( "biukop.com/sfm/loan" "encoding/json" "errors" log "github.com/sirupsen/logrus" "io/ioutil" "os" "os/exec" "strings" ) type AiDecodeIncome struct { Id int64 Input loan.Uploads ul uploadsOnDisk // internal data Mime string // mime actually detected. PayIn []loan.PayIn // generated PayIn Info DecodeIncomeDetails } type DecodeIncomeDetails struct { Lender loan.LenderType AAA []PayInAAARow Connective []ConnectiveRow } type AiDecodeJson struct { Version string V1 DecodeIncomeDetails } func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) { m.Id = ulMeta.Id m.Input = ulMeta m.ul.Upload = ulMeta m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated if allowCachedResult { e = m.ReadJson() if e == nil { return // already decoded } else { log.Warn("trying to read existing json failed ", e.Error()) e = nil } } m.PayIn = make([]loan.PayIn, 0, 10) switch m.getFileType() { case "pdf": e = m.decodePdf() break case "excel", "opensheet": e = m.decodeXls() break default: e = errors.New("unknown format") } if e == nil { eJson := m.WriteJson() if eJson != nil { log.Error("failed to write analysis to json", eJson.Error()) } } return } func (m *AiDecodeIncome) ReadJson() (e error) { if !fileExists(m.ul.jsonPath()) { return errors.New(m.ul.jsonPath() + " not found") } f, e := os.Open(m.ul.jsonPath()) if e != nil { return } decoder := json.NewDecoder(f) data := AiDecodeJson{} e = decoder.Decode(&data) if e != nil { log.Error("failed load existing decode json", e.Error()) return } return } func (m *AiDecodeIncome) WriteJson() (e error) { b, e := json.Marshal(m.DecodeIncomeDetails) if e != nil { return } ioutil.WriteFile(m.ul.jsonPath(), b, 0644) return } func (m *AiDecodeIncome) getFileType() (ret string) { strMime, e := GetFileContentType(m.ul.filePath()) if e != nil { return } m.Mime = strMime ret, e = m.ul.GetFileType() if e != nil { ret = "" } return } func (m *AiDecodeIncome) decodePdf() (e error) { cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-") out, e := cmd.Output() if e != nil { log.Error("cannot convert pdf to text ", e) } raw := string(out) switch m.detectFunder(raw) { case loan.Lender_AAA: m.Lender = loan.Lender_AAA e = m.decodeAAAPdf(raw) // regardless of error, we pump in all available row successed so far for _, row := range m.AAA { pi := loan.PayIn{} pi.Id = 0 pi.Ts = row.Period pi.Amount = row.LoanFacility pi.Lender = loan.Lender_AAA pi.Settlement = row.Settlement pi.Balance = row.Balance pi.OffsetBalance = -1 pi.IncomeAmount = row.InTrail pi.IncomeType = "Trail" m.PayIn = append(m.PayIn, pi) } // log.Println("AAA final result", m.AAA) break case loan.Lender_Connective: m.Lender = loan.Lender_Connective e = m.decodeConnectivePdf(out) break case loan.Lender_Unknown: e = errors.New(loan.Lender_Unknown) break // not able to detect Funder } return } func (m *AiDecodeIncome) decodeXls() (e error) { e = errors.New("not implemented yet") return } func (m *AiDecodeIncome) detectFunder(raw string) loan.LenderType { if m.isAAA(raw) { return loan.Lender_AAA } if m.isConnective(raw) { return loan.Lender_Connective } return loan.Lender_Unknown } func (m *AiDecodeIncome) isAAA(raw string) bool { keyword := "AAA Financial Trail Report" lines := strings.Split(raw, "\n") return m.checkFunderKeyword(keyword, lines, 0, 3) } func (m *AiDecodeIncome) isConnective(raw string) bool { keyword := "connective.com.au" lines := strings.Split(raw, "\n") return m.checkFunderKeyword(keyword, lines, 0, 3) } func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool { for _, line := range lines { // as long as it has it if strings.Contains(line, keyword) { return true } } return false }