From ed619dcbae506ceb5bb966f2e6ac4d1a4767e52b Mon Sep 17 00:00:00 2001 From: sp Date: Sat, 3 Apr 2021 22:09:06 +1100 Subject: [PATCH] AAA decoding works. --- apiV1Response.go | 6 ++ apiV1Uploads.go | 186 ++++++++++++++++++++++++++++++++++++++++++ apiv1.go | 21 ++++- debug.go | 6 +- pay-in-decode.go | 112 +++++++++++++++++++++++++ pay-in-decode_test.go | 13 +++ payIn-AAA.go | 150 ++++++++++++++++++++++++++++++++++ 7 files changed, 491 insertions(+), 3 deletions(-) create mode 100644 apiV1Uploads.go create mode 100644 pay-in-decode.go create mode 100644 pay-in-decode_test.go create mode 100644 payIn-AAA.go diff --git a/apiV1Response.go b/apiV1Response.go index 23d2300..2429450 100644 --- a/apiV1Response.go +++ b/apiV1Response.go @@ -54,6 +54,9 @@ func (m *apiV1Response) toJson() (ret []byte, e error) { } func (m *apiV1Response) sendJson(w http.ResponseWriter) (ret []byte, e error) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + tempMap := m.tmp m.tmp = nil if config.Debug { @@ -67,6 +70,9 @@ func (m *apiV1Response) sendJson(w http.ResponseWriter) (ret []byte, e error) { } func apiV1SendJson(result interface{}, w http.ResponseWriter, r *http.Request, ss *loan.Session) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + out, e := json.Marshal(result) if e != nil { log.Warn("Cannot convert result to json ", result) diff --git a/apiV1Uploads.go b/apiV1Uploads.go new file mode 100644 index 0000000..a8224f2 --- /dev/null +++ b/apiV1Uploads.go @@ -0,0 +1,186 @@ +package main + +import ( + "biukop.com/sfm/loan" + "crypto/sha256" + "errors" + "fmt" + log "github.com/sirupsen/logrus" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strconv" + "time" +) + +func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix + intId, e := strconv.Atoi(id) + if e != nil { + log.Println("invalid id for upload get", id, e) + apiV1Client403Error(w, r, ss) // bad request + return + } + + ul := loan.Uploads{} + e = ul.Read(int64(intId)) + if e != nil { + log.Println("no file uploaded", intId, e) + apiV1Client404Error(w, r, ss) // bad request + return + } + + //check local file first + path := config.Uploads + strconv.FormatInt(ul.Id, 10) + ".uploads" + if fileExists(path) { + http.ServeFile(w, r, path) + return + } + + log.Error("Upload file not found on disk", ul) + apiV1Server500Error(w, r) // bad request +} + +func apiV1UploadsPost(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix + + filename, e := saveUploadToFile(r) + if e != nil { + log.Println("no file uploaded", filename, e) + apiV1Client404Error(w, r, ss) // bad request + return + } + + intId, e := strconv.Atoi(id) + if id != "" { + if e != nil { + log.Println("Error Getting File", e) + apiV1Client404Error(w, r, ss) // bad request + return + } + updateUploads(int64(intId), filename, w, r, ss) + } else { + createUploads(filename, w, r, ss) + } +} + +func sha256File(input io.Reader) string { + hash := sha256.New() + if _, err := io.Copy(hash, input); err != nil { + log.Fatal(err) + } + sum := hash.Sum(nil) + + return fmt.Sprintf("%x", sum) +} + +func updateUploads(id int64, fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul := loan.Uploads{} + e := ul.Read(int64(id)) + if e != nil { + log.Println("bad upload id is given ", id, e) + apiV1Client404Error(w, r, ss) // bad request + return + } + + ul1, _, e := saveUploadsToDB(id, fileName, r, ss) + if e != nil { + os.Remove(config.Uploads + ul.FileName) + ul1.Delete() + log.Println("cannot save file info to db ", e) + apiV1Server500Error(w, r) // bad request + return + } + + apiV1SendJson(ul1, w, r, ss) +} + +func createUploads(fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul, _, e := saveUploadsToDB(0, fileName, r, ss) + if e != nil { + log.Println("cannot save file info to db ", e) + e = ul.Delete() // delete the newly created, if failed, db will clean it + if e != nil { + log.Error("failed to remove unused uploads", ul) + } + + e = os.Remove(config.Uploads + fileName) + if e != nil { + log.Error("failed to remove unused temp file", fileName) + } + + apiV1Server500Error(w, r) // bad request + return + } + apiV1SendJson(ul, w, r, ss) +} + +func saveUploadsToDB(id int64, fileName string, + r *http.Request, ss *loan.Session) (ul loan.Uploads, duplicate bool, e error) { + duplicate = false + e = r.ParseMultipartForm(10 << 20) // we should have ready parsed this, just in case + if e != nil { + return + } + file, header, e := r.FormFile("files") + + file.Seek(0, 0) //seek to beginning + checksum := sha256File(file) + ul.Id = id + ul.Ts = time.Now() + ul.FileName = header.Filename + file.Seek(0, 0) //seek to beginning + ul.Format = header.Header.Get("Content-type") + ul.Size = header.Size // necessary to prevent duplicate + ul.LastModified = 0 + ul.Sha256 = checksum // necessary to prevent duplicate + ul.By = ss.User + e = ul.Write() // this Id will have the real Id if there is duplicates + + if e != nil { + log.Error("Fail to update db ", ul, e) + } else { + if id > 0 && ul.Id != id { + duplicate = true + } + target := fmt.Sprintf("%d.uploads", ul.Id) + e = os.Rename(config.Uploads+fileName, config.Uploads+target) + if e != nil { + ul.FileName = fileName // some how failed to rename + } + } + return +} + +func saveUploadToFile(r *http.Request) (filename string, e error) { + e = r.ParseMultipartForm(10 << 20) + if e != nil { + return + } + file, header, e := r.FormFile("files") + if e != nil { + log.Println("Error Getting File", e) + return + } + + out, pathError := ioutil.TempFile(config.Uploads, "can-del-upload-*.tmp") + if pathError != nil { + log.Println("Error Creating a file for writing", pathError) + return + } + + out.Seek(0, 0) //seek to beginning + size, e := io.Copy(out, file) + if e != nil { + os.Remove(out.Name()) //remove on failure + log.Println("Error copying", e) + return + } + + if size != header.Size { + e = errors.New("written file with incorrect size") + } + return filepath.Base(out.Name()), e +} diff --git a/apiv1.go b/apiv1.go index d5e030e..19627cd 100644 --- a/apiv1.go +++ b/apiv1.go @@ -70,6 +70,10 @@ func setupApiV1Handler() []apiV1HandlerMap { {"DELETE", "payIn/", apiV1PayInDelete}, {"GET", "user-reward/", apiV1UserReward}, {"GET", "login-available/", apiV1LoginAvailable}, + + {"POST", "lender-upload/", apiV1UploadsPost}, + {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "login", apiV1DumpRequest}, } } else { //production @@ -117,6 +121,10 @@ func setupApiV1Handler() []apiV1HandlerMap { {"DELETE", "payIn/", apiV1PayInDelete}, {"GET", "user-reward/", apiV1UserReward}, {"GET", "login-available/", apiV1LoginAvailable}, + + {"POST", "lender-upload/", apiV1UploadsPost}, + {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "login", apiV1EmptyResponse}, } } @@ -126,8 +134,6 @@ func setupApiV1Handler() []apiV1HandlerMap { //apiV1Main version 1 main entry for all REST API // func apiV1Main(w http.ResponseWriter, r *http.Request) { - //general setup - w.Header().Set("Content-Type", "application/json;charset=UTF-8") //CORS setup setupCrossOriginResponse(&w, r) @@ -293,6 +299,8 @@ func apiV1ErrorCheck(e error) { } func apiV1Server500Error(w http.ResponseWriter, r *http.Request) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") w.WriteHeader(500) apiV1AddTrackingCookie(w, r, nil) //always the last one to set cookies @@ -305,6 +313,9 @@ func apiV1Server500Error(w http.ResponseWriter, r *http.Request) { } func apiV1Client403Error(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + w.WriteHeader(403) type struct403 struct { Error int @@ -324,6 +335,9 @@ func apiV1Client403Error(w http.ResponseWriter, r *http.Request, ss *loan.Sessio } func apiV1Client404Error(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + w.WriteHeader(404) type struct404 struct { Error int @@ -361,6 +375,9 @@ func apiV1DumpRequest(w http.ResponseWriter, r *http.Request, ss *loan.Session) } func apiV1EmptyResponse(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + //general setup + w.Header().Set("Content-Type", "application/json;charset=UTF-8") + apiV1AddTrackingCookie(w, r, ss) //always the last one to set cookies fmt.Fprintf(w, "") } diff --git a/debug.go b/debug.go index 13a1b5e..99f28f7 100644 --- a/debug.go +++ b/debug.go @@ -7,7 +7,11 @@ import ( func logRequestDebug(data []byte, err error) (ret string) { if err == nil { - ret = fmt.Sprintf("%s\n\n", string(data)) + output := data + if len(data) > 4096 { + output = data[0:4095] + } + ret = fmt.Sprintf("%s\n\n", string(output)) fmt.Println(ret) } else { log.Fatalf("%s\n\n", err) diff --git a/pay-in-decode.go b/pay-in-decode.go new file mode 100644 index 0000000..bca3ea1 --- /dev/null +++ b/pay-in-decode.go @@ -0,0 +1,112 @@ +package main + +import ( + "biukop.com/sfm/loan" + "errors" + log "github.com/sirupsen/logrus" + "net/http" + "os" + "os/exec" + "strings" +) + +type FunderType string + +const ( + Funder_AAA FunderType = "AAA Financial" + Funder_Pepper = "Pepper" + Funder_Resimac = "Resimac" + Funder_Unknown = "cannot detect funder type" +) + +type AiDecodeIncome struct { + Input struct { + Uploads loan.Uploads + FileName string //a local file on disk + InMime string //may not be correct, just some suggestion only. + } + Mime string //mime actually detected. + PayIn []loan.PayIn + Funder FunderType + AAA PayInAAAData +} + +func decodePayInMain(filename string, format string) (ai AiDecodeIncome, e error) { + ai.Input.FileName = filename + ai.Input.InMime = format + ai.PayIn = make([]loan.PayIn, 0, 10) + ai.Mime, e = GetFileContentType(filename) + if e != nil { + return + } + + switch ai.Mime { + case "application/pdf": + ai.decodePayInPdf(filename, format) + } + + return ai, e +} + +// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime. +func GetFileContentType(filename string) (contentType string, e error) { + contentType = "" + input, e := os.OpenFile(filename, os.O_RDONLY, 0755) + // Only the first 512 bytes are used to sniff the content type. + buffer := make([]byte, 512) + + _, e = input.Read(buffer) + if e != nil { + return + } + + // Use the net/http package's handy DectectContentType function. Always returns a valid + // content-type by returning "application/octet-stream" if no others seemed to match. + contentType = http.DetectContentType(buffer) + return +} + +func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []loan.PayIn, e error) { + cmd := exec.Command("pdftotext", "-layout", filename, "-") + //log.Println(cmd.String()) + out, e := cmd.Output() + if e != nil { + log.Fatal(e) + } + + raw := string(out) + switch m.detectFunder(raw) { + case Funder_AAA: + e = m.AAA.decodeAAAPdf(raw) + log.Println("AAA final result", m.AAA) + break + case Funder_Unknown: + e = errors.New(Funder_Unknown) + break // not able to detect Funder + } + return +} + +func (m *AiDecodeIncome) detectFunder(raw string) FunderType { + if m.isAAA(raw) { + return Funder_AAA + } + + return Funder_Unknown +} + +func (m *AiDecodeIncome) isAAA(raw string) bool { + keyword := "AAA Financial Trail Report" + lines := strings.Split(raw, "\n") + return m.checkFunderKeyword(keyword, lines, 0, 3) +} + +func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool { + for idx, line := range lines { + // first 10 lines has Key word + if strings.Contains(line, keyword) && idx >= start && idx <= 10 { + return true + } + } + return false +} diff --git a/pay-in-decode_test.go b/pay-in-decode_test.go new file mode 100644 index 0000000..6740d6b --- /dev/null +++ b/pay-in-decode_test.go @@ -0,0 +1,13 @@ +package main + +import ( + log "github.com/sirupsen/logrus" + "testing" +) + +func TestDecodePayInMain(t *testing.T) { + fileName := "./uploads/30.uploads" + fileName = "/home/sp/go/src/SFM_Loan_RestApi/uploads/30.uploads" + data, _ := decodePayInMain(fileName, "application/vnd.ms-excel") + log.Println(data) +} diff --git a/payIn-AAA.go b/payIn-AAA.go new file mode 100644 index 0000000..76d02d4 --- /dev/null +++ b/payIn-AAA.go @@ -0,0 +1,150 @@ +package main + +import ( + log "github.com/sirupsen/logrus" + "strconv" + "strings" + "time" +) + +/* Sample Text + +AAA Financial Trail Report +Super Finance Markets Pty Ltd + Loan Number SettDate Loan Balance Arrears DisDate IntTrail$ Comments + Facility + Columbus + Period Servicing: Feb 2020 +400053440 02-Sep-19 $552,463 552,579.52 $32.19 +400063271 19-Feb-20 $832,000 832,000.00 $0.00 + Columbus Total: $32.19 + + Grand Total: $32.19 + +Super Finance Markets Pty Ltd + +*/ + +type PayInAAARow struct { + LoanNUmber string + Settlement time.Time + LoanAmount float64 + Balance float64 + InTrail float64 +} + +type PayInAAAPeriod struct { + Period time.Time + Rows []PayInAAARow +} + +type PayInAAAData struct { + Periods []PayInAAAPeriod +} + +func (m *PayInAAAData) decodeAAAPdf(raw string) (e error) { + m.Periods = make([]PayInAAAPeriod, 0, 10) + lines := strings.Split(raw, "\n") + + var tableHeader []string + var tableHeaderLine int + currentPeriod := -1 + state := "start" + for idx, l := range lines { // DFA, wow, finally it's used. after years of learning + switch state { + case "start": + state = m.processStart(l) + if state == "LookingForPeriod" { + tableHeaderLine = idx + tableHeader = strings.Split(l, " ") + log.Println("Find table header", tableHeader, l, tableHeaderLine) + } + break + case "LookingForPeriod": + state = m.processPeriod(l) + if state == "LookingForRows" { + batch := PayInAAAPeriod{} + m.Periods = append(m.Periods, batch) + currentPeriod++ //move index to next , or 0 for a start + m.Periods[currentPeriod].Period, e = m.getPeriod(l) + m.Periods[currentPeriod].Rows = make([]PayInAAARow, 0, 10) + if e != nil { + log.Warn("cannot find period", l, e) + state = "LookingForPeriod" + } + } + break + case "LookingForRows", "LookingForRowsSkipCurrent": + nextState, row, valid := m.processRow(l) + if valid { + m.Periods[currentPeriod].Rows = append(m.Periods[currentPeriod].Rows, row) + } + state = nextState + break + } + } + return +} + +func (m *PayInAAAData) processStart(line string) (nextState string) { + nextState = "start" + if strings.Contains(line, "Loan Number") && + strings.Contains(line, "SettDate") && + strings.Contains(line, "Balance") && + strings.Contains(line, "IntTrail$") { + nextState = "LookingForPeriod" + } + return +} + +func (m *PayInAAAData) processPeriod(line string) (nextState string) { + nextState = "LookingForPeriod" + if strings.Contains(line, "Period Servicing:") { + nextState = "LookingForRows" + } + return +} + +// Period Servicing: Feb 2020 +func (m *PayInAAAData) getPeriod(line string) (p time.Time, e error) { + idx := strings.Index(line, ":") + subStr := strings.TrimSpace(line[idx+1:]) + p, e = time.Parse("Jan 2006", subStr) + return +} + +func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARow, valid bool) { + nextState = "LookingForRows" + valid = false + allParts := strings.Split(line, " ") + el := make([]string, 0, 10) + for _, item := range allParts { + if len(item) > 0 { + el = append(el, item) + } + } + + if len(el) >= 5 { + row.LoanNUmber = el[0] + row.Settlement, _ = time.Parse("02-Jan-06", el[1]) + row.LoanAmount = m.currencyToFloat64(el[2]) + row.Balance = m.currencyToFloat64(el[3]) + row.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element + valid = true + } else { + if strings.Contains(line, "Total:") { + nextState = "start" + } else { + nextState = "LookingForRowsSkipCurrent" + } + } + return +} + +func (m *PayInAAAData) currencyToFloat64(cur string) (ret float64) { + cur = strings.ReplaceAll(cur, " ", "") //remove space + cur = strings.ReplaceAll(cur, "$", "") //remove $ + cur = strings.ReplaceAll(cur, ",", "") //remove , + ret, _ = strconv.ParseFloat(cur, 64) + return ret +}