package main import ( "biukop.com/sfm/loan" "encoding/json" "errors" log "github.com/sirupsen/logrus" "io/ioutil" "os" "os/exec" "strings" ) type AiDecodeIncome struct { Id int64 Input loan.Uploads ul uploadsOnDisk // internal data Mime string // mime actually detected. PayIn []loan.PayIn // generated PayIn Info DecodeIncomeDetails } type DecodeIncomeDetails struct { Lender loan.LenderType AAA []PayInAAARow Connective []ConnectiveRow ResimacXls []ResimacRow ResimacPdf []ResimacPdf Pepper []PepperRow } type AiDecodeJson struct { Version string V1 DecodeIncomeDetails } func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) { m.Id = ulMeta.Id m.Input = ulMeta m.ul.Upload = ulMeta m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated if allowCachedResult { e = m.ReadJson() if e == nil { return // already decoded } else { log.Warn("trying to read existing json failed ", e.Error()) e = nil } } m.PayIn = make([]loan.PayIn, 0, 10) switch m.getFileType() { case "pdf": e = m.decodePdf() break case "excel", "opensheet": e = m.decodeXls() break default: e = errors.New("unknown format") } if e == nil { eJson := m.WriteJson() if eJson != nil { log.Error("failed to write analysis to json", eJson.Error()) } } return } func (m *AiDecodeIncome) ReadJson() (e error) { if !fileExists(m.ul.jsonPath()) { return errors.New(m.ul.jsonPath() + " not found") } f, e := os.Open(m.ul.jsonPath()) if e != nil { return } decoder := json.NewDecoder(f) data := AiDecodeJson{} e = decoder.Decode(&data) if e != nil { log.Error("failed load existing decode json", e.Error()) return } return } func (m *AiDecodeIncome) WriteJson() (e error) { b, e := json.Marshal(m.DecodeIncomeDetails) if e != nil { return } ioutil.WriteFile(m.ul.jsonPath(), b, 0644) return } func (m *AiDecodeIncome) getFileType() (ret string) { strMime, e := GetFileContentType(m.ul.filePath()) if e != nil { return } m.Mime = strMime ret, e = m.ul.GetFileType() if e != nil { ret = "" } return } func (m *AiDecodeIncome) decodePdf() (e error) { cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-") out, e := cmd.Output() if e != nil { log.Error("cannot convert pdf to text ", e) } raw := string(out) switch m.detectFunder(raw) { case loan.Lender_AAA: m.Lender = loan.Lender_AAA e = m.decodeAAAPdf(raw) // regardless of error, we pump in all available row successed so far for _, row := range m.AAA { pi := loan.PayIn{} pi.Id = 0 pi.Ts = row.Period pi.Amount = row.LoanFacility pi.Lender = loan.Lender_AAA pi.Settlement = row.Settlement pi.Balance = row.Balance pi.OffsetBalance = -1 pi.IncomeAmount = row.InTrail pi.IncomeType = "Trail" m.PayIn = append(m.PayIn, pi) } // log.Println("AAA final result", m.AAA) break case loan.Lender_Connective: m.Lender = loan.Lender_Connective e = m.decodeConnectivePdf(out) break case loan.Lender_Resimac: m.Lender = loan.Lender_Resimac e = m.decodeResimacPdf(out) break case loan.Lender_Unknown: e = errors.New(loan.Lender_Unknown) break // not able to detect Funder } return } func (m *AiDecodeIncome) decodeXls() (e error) { e = m.ul.convertExcelTo("csv") if e != nil { log.Error("cannot convert xls to csv ", e.Error()) } raw := string(m.ul.tmp) switch m.detectFunder(raw) { case loan.Lender_Resimac: m.Lender = loan.Lender_Resimac e = m.decodeResimacXls(m.ul.tmp) break case loan.Lender_Pepper: m.Lender = loan.Lender_Pepper e = m.decodePepperXls(m.ul.tmp) break case loan.Lender_Unknown: e = errors.New(loan.Lender_Unknown) break // not able to detect Funder } return } func (m *AiDecodeIncome) detectFunder(raw string) loan.LenderType { if m.isAAA(raw) { return loan.Lender_AAA } if m.isConnective(raw) { return loan.Lender_Connective } if m.isResimacXls(raw) { return loan.Lender_Resimac } if m.isResimacPdf(raw) { return loan.Lender_Resimac } if m.isPepperXls(raw) { return loan.Lender_Pepper } return loan.Lender_Unknown } func (m *AiDecodeIncome) isKeywordExist(keyword string, lines []string) bool { for _, line := range lines { if strings.Contains(line, keyword) { return true } } return false }