package main import ( "biukop.com/sfm/loan" "encoding/json" "errors" log "github.com/sirupsen/logrus" "io/ioutil" "os" "os/exec" "strings" ) type AiDecodeIncome struct { Id int64 Input loan.Uploads ul uploadsOnDisk // internal data Mime string // mime actually detected. PayIn []loan.PayIn // generated PayIn Info DecodeIncomeDetails } type DecodeIncomeDetails struct { Funders []loan.FunderType AAA []PayInAAARow Connective_ANZ []ConnectiveRow Connective_BOC []ConnectiveRow Connective_CHLAB []ConnectiveRow Connective_HSL []ConnectiveRow Connective_ING []ConnectiveRow Connective_MEB []ConnectiveRow Connective_SGB []ConnectiveRow Connective_WPC []ConnectiveRow } type AiDecodeJson struct { Version string V1 DecodeIncomeDetails } func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads, allowCachedResult bool) (e error) { m.Id = ulMeta.Id m.Input = ulMeta m.ul.Upload = ulMeta m.PayIn = make([]loan.PayIn, 0, 100) // finalized payIns, generated m.Funders = make([]loan.FunderType, 0, 50) // array of valid funders if allowCachedResult { e = m.ReadJson() if e == nil { return // already decoded } else { log.Warn("trying to read existing json failed ", e.Error()) e = nil } } m.PayIn = make([]loan.PayIn, 0, 10) switch m.getFileType() { case "pdf": e = m.decodePdf() break case "excel", "opensheet": e = m.decodeXls() break default: e = errors.New("unknown format") } if e == nil { eJson := m.WriteJson() if eJson != nil { log.Error("failed to write analysis to json", eJson.Error()) } } return } func (m *AiDecodeIncome) ReadJson() (e error) { if !fileExists(m.ul.jsonPath()) { return errors.New(m.ul.jsonPath() + " not found") } f, e := os.Open(m.ul.jsonPath()) if e != nil { return } decoder := json.NewDecoder(f) data := AiDecodeJson{} e = decoder.Decode(&data) if e != nil { log.Error("failed load existing decode json", e.Error()) return } return } func (m *AiDecodeIncome) WriteJson() (e error) { b, e := json.Marshal(m.DecodeIncomeDetails) if e != nil { return } ioutil.WriteFile(m.ul.jsonPath(), b, 0644) return } func (m *AiDecodeIncome) getFileType() (ret string) { strMime, e := GetFileContentType(m.ul.filePath()) if e != nil { return } m.Mime = strMime ret, e = m.ul.GetFileType() if e != nil { ret = "" } return } func (m *AiDecodeIncome) decodePdf() (e error) { cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-") out, e := cmd.Output() if e != nil { log.Error("cannot convert pdf to text ", e) } raw := string(out) switch m.detectFunder(raw) { case loan.Funder_AAA: m.Funders = append(m.Funders, loan.Funder_AAA) e = m.decodeAAAPdf(raw) for _, row := range m.AAA { pi := loan.PayIn{} pi.Id = 0 pi.Ts = row.Period pi.Amount = row.LoanAmount pi.Lender = loan.Funder_AAA pi.Settlement = row.Settlement pi.Balance = row.Balance pi.OffsetBalance = -1 pi.IncomeAmount = row.InTrail pi.IncomeType = "Trail" m.PayIn = append(m.PayIn, pi) } log.Println("AAA final result", m.AAA) break case loan.Funder_Unknown: e = errors.New(loan.Funder_Unknown) break // not able to detect Funder } return } func (m *AiDecodeIncome) decodeXls() (e error) { e = errors.New("not implemented yet") return } func (m *AiDecodeIncome) detectFunder(raw string) loan.FunderType { if m.isAAA(raw) { return loan.Funder_AAA } return loan.Funder_Unknown } func (m *AiDecodeIncome) isAAA(raw string) bool { keyword := "AAA Financial Trail Report" lines := strings.Split(raw, "\n") return m.checkFunderKeyword(keyword, lines, 0, 3) } func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool { for idx, line := range lines { // first 10 lines has Key word if strings.Contains(line, keyword) && idx >= start && idx <= 10 { return true } } return false }