diff --git a/UploadsOnDisk.go b/UploadsOnDisk.go index 8a11942..46d9d89 100644 --- a/UploadsOnDisk.go +++ b/UploadsOnDisk.go @@ -217,7 +217,8 @@ func (m *uploadsOnDisk) GetFileType() (ret string, e error) { return "pdf", nil } - if strings.ToLower(strMime) == "application/vnd.ms-excel" { + if strings.ToLower(strMime) == "application/vnd.ms-excel" || strings.ToLower(m.Upload.Format) == + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" { return "excel", nil } @@ -227,6 +228,20 @@ func (m *uploadsOnDisk) GetFileType() (ret string, e error) { return "opensheet", nil } + // check suffix which is not reliable + ext := filepath.Ext(m.Upload.FileName) + ext = strings.ToLower(ext) + if ext[0] == '.' { + ext = ext[1:] // remove the first char 'dot' . + } + switch ext { + case "xls", "xlsx": + return "excel", nil + case "pdf": + return "pdf", nil + default: + log.Warn("unhandled uploads type", ext) + } return "", nil } diff --git a/UploadsOnDisk_test.go b/UploadsOnDisk_test.go index c4b28c6..418d505 100644 --- a/UploadsOnDisk_test.go +++ b/UploadsOnDisk_test.go @@ -41,4 +41,8 @@ func TestConvertImageToThumbnail(t *testing.T) { _, name := filepath.Split("/home/sp/uploads/abc.uploads") test := fileNameWithoutExtTrimSuffix(name) + ".jpg" log.Println(test) + test1 := filepath.Ext("abc.uploads.pdf") + log.Println(test1) + ext := "." + log.Println(ext[1:]) } diff --git a/apiV1Uploads.go b/apiV1Uploads.go index 7f0073b..24fae6a 100644 --- a/apiV1Uploads.go +++ b/apiV1Uploads.go @@ -15,8 +15,27 @@ import ( "time" ) -func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { - id := r.URL.Path[len(apiV1Prefix+"upload/"):] //remove prefix +func apiV1UploadMetaGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + id := r.URL.Path[len(apiV1Prefix+"upload-meta/"):] //remove prefix + intId, e := strconv.Atoi(id) + if e != nil { + log.Println("invalid id for upload get", id, e) + apiV1Client403Error(w, r, ss) // bad request + return + } + + ulmeta := loan.Uploads{} + e = ulmeta.Read(int64(intId)) + if e != nil { + log.Println("upload not found", id, e) + apiV1Client404Error(w, r, ss) // bad request + return + } + apiV1SendJson(ulmeta, w, r, ss) +} + +func apiV1UploadOriginalFileGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + id := r.URL.Path[len(apiV1Prefix+"upload-original/"):] //remove prefix intId, e := strconv.Atoi(id) if e != nil { log.Println("invalid id for upload get", id, e) @@ -118,11 +137,7 @@ func createUploads(filePath string, w http.ResponseWriter, r *http.Request, ss * apiV1Server500Error(w, r) // bad request return } - - ai := AiDecodeIncome{} - ai.decodeUploadToPayIn(ul.Upload) - - apiV1SendJson(ai, w, r, ss) + apiV1SendJson(ulMeta, w, r, ss) } func saveUploadsMetaToDB(id int64, filePath string, @@ -195,8 +210,8 @@ func saveUploadToFile(r *http.Request) (filename string, e error) { return out.Name(), e } -func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) { - strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix +func getRequestedUpload(strId string, w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) { + Id, e := strconv.Atoi(strId) if e != nil { log.Error("Invalid uploads Id cannot convert to integer", Id, e) @@ -217,7 +232,13 @@ func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session } func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session) { - ul, e := getRequestedUpload(w, r, ss) + strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix + if strId == "default" { + http.ServeFile(w, r, config.UploadsDir.JpgDefault) + return + } + + ul, e := getRequestedUpload(strId, w, r, ss) if e != nil { return } @@ -229,7 +250,7 @@ func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session defer f.Close() fi, e := f.Stat() if e == nil { - w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) + //w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f) return } @@ -259,7 +280,12 @@ func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session } func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Session) { - ul, e := getRequestedUpload(w, r, ss) + strId := r.URL.Path[len(apiV1Prefix+"upload-as-thumbnail/"):] //remove prefix + if strId == "default" { + http.ServeFile(w, r, config.UploadsDir.ThumbDefault) + return + } + ul, e := getRequestedUpload(strId, w, r, ss) if e != nil { return } @@ -282,8 +308,13 @@ func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Ses } } -func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) { - ul, e := getRequestedUpload(w, r, ss) +func apiV1UploadAsPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + strId := r.URL.Path[len(apiV1Prefix+"upload-as-pdf/"):] //remove prefix + if strId == "default" { + http.ServeFile(w, r, config.UploadsDir.PdfDefault) + return + } + ul, e := getRequestedUpload(strId, w, r, ss) if e != nil { return } @@ -302,7 +333,9 @@ func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) { defer f.Close() fi, e := f.Stat() if e == nil { - w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) + if forceHttpDownload(r) { + w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) + } http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f) return } @@ -330,3 +363,13 @@ func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) { apiV1Client404Error(w, r, ss) } } + +func forceHttpDownload(r *http.Request) bool { + keys, ok := r.URL.Query()["download"] + + if !ok || len(keys[0]) < 1 { + return false + } + key := keys[0] + return key == "force" +} diff --git a/apiv1.go b/apiv1.go index 69c0752..cb21af1 100644 --- a/apiv1.go +++ b/apiv1.go @@ -72,9 +72,14 @@ func setupApiV1Handler() []apiV1HandlerMap { {"GET", "login-available/", apiV1LoginAvailable}, {"POST", "lender-upload/", apiV1UploadsPost}, - {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "lender-upload/", apiV1UploadOriginalFileGet}, + + {"GET", "upload-analysis/", apiV1UploadAnalysis}, + {"GET", "upload-as-image/", apiV1UploadAsImage}, {"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail}, - {"GET", "upload-as-pdf/", apiV1UploadAPDF}, + {"GET", "upload-as-pdf/", apiV1UploadAsPDF}, + {"GET", "upload-original/", apiV1UploadOriginalFileGet}, + {"GET", "upload/", apiV1UploadMetaGet}, {"GET", "login", apiV1DumpRequest}, } @@ -125,13 +130,14 @@ func setupApiV1Handler() []apiV1HandlerMap { {"GET", "login-available/", apiV1LoginAvailable}, {"POST", "lender-upload/", apiV1UploadsPost}, - {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "lender-upload/", apiV1UploadOriginalFileGet}, {"GET", "upload-analysis/", apiV1UploadAnalysis}, {"GET", "upload-as-image/", apiV1UploadAsImage}, {"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail}, - {"GET", "upload-as-pdf/", apiV1UploadAPDF}, - {"GET", "upload/", apiV1UploadsGet}, + {"GET", "upload-as-pdf/", apiV1UploadAsPDF}, + {"GET", "upload-original/", apiV1UploadOriginalFileGet}, + {"GET", "upload-meta/", apiV1UploadMetaGet}, {"GET", "login", apiV1EmptyResponse}, } diff --git a/pay-in-decode.go b/pay-in-decode.go index 5ba3ffc..752e054 100644 --- a/pay-in-decode.go +++ b/pay-in-decode.go @@ -18,20 +18,21 @@ const ( ) type AiDecodeIncome struct { + Id int64 Input loan.Uploads ul uploadsOnDisk // internal data Mime string //mime actually detected. PayIn []loan.PayIn Funder FunderType - AAA PayInAAAData + AAA []PayInAAAPeriod } func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) { + m.Id = ulMeta.Id m.Input = ulMeta m.ul.Upload = ulMeta m.PayIn = make([]loan.PayIn, 0, 10) - switch m.getFileType() { case "pdf": m.decodePdf() @@ -72,7 +73,7 @@ func (m *AiDecodeIncome) decodePdf() (e error) { switch m.detectFunder(raw) { case Funder_AAA: m.Funder = Funder_AAA - e = m.AAA.decodeAAAPdf(raw) + e = m.decodeAAAPdf(raw) log.Println("AAA final result", m.AAA) break case Funder_Unknown: diff --git a/payIn-AAA.go b/payIn-AAA.go index 220fdcb..c2ce446 100644 --- a/payIn-AAA.go +++ b/payIn-AAA.go @@ -38,51 +38,50 @@ type PayInAAAPeriod struct { Rows []PayInAAARow } -type PayInAAAData struct { - Periods []PayInAAAPeriod -} - -func (m *PayInAAAData) decodeAAAPdf(raw string) (e error) { - m.Periods = make([]PayInAAAPeriod, 0, 10) +func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) { + m.AAA = make([]PayInAAAPeriod, 0, 10) lines := strings.Split(raw, "\n") - currentPeriod := -1 + currentDecoder := PayInAAAPeriod{} state := "start" for _, l := range lines { // DFA, wow, finally it's used. after years of learning switch state { case "start": - state = m.processStart(l) + state = currentDecoder.processStart(l) if state == "LookingForPeriod" { // determine column index, if their column is changing } break case "LookingForPeriod": - state = m.processPeriod(l) + state = currentDecoder.processPeriod(l) if state == "LookingForRows" { - batch := PayInAAAPeriod{} - m.Periods = append(m.Periods, batch) - currentPeriod++ //move index to next , or 0 for a start - m.Periods[currentPeriod].Period, e = m.getPeriod(l) - m.Periods[currentPeriod].Rows = make([]PayInAAARow, 0, 10) + currentDecoder.Period, e = currentDecoder.getPeriod(l) + currentDecoder.Rows = make([]PayInAAARow, 0, 10) if e != nil { log.Warn("cannot find period", l, e) state = "LookingForPeriod" + } else { + m.AAA = append(m.AAA, currentDecoder) + } } break case "LookingForRows", "LookingForRowsSkipCurrent": - nextState, row, valid := m.processRow(l) + nextState, row, valid := currentDecoder.processRow(l) if valid { - m.Periods[currentPeriod].Rows = append(m.Periods[currentPeriod].Rows, row) + currentDecoder.Rows = append(currentDecoder.Rows, row) } state = nextState + if nextState == "start" { + currentDecoder = PayInAAAPeriod{} //renew to a empty state + } break } } return } -func (m *PayInAAAData) processStart(line string) (nextState string) { +func (m *PayInAAAPeriod) processStart(line string) (nextState string) { nextState = "start" if strings.Contains(line, "Loan Number") && strings.Contains(line, "SettDate") && @@ -93,7 +92,7 @@ func (m *PayInAAAData) processStart(line string) (nextState string) { return } -func (m *PayInAAAData) processPeriod(line string) (nextState string) { +func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) { nextState = "LookingForPeriod" if strings.Contains(line, "Period Servicing:") { nextState = "LookingForRows" @@ -102,14 +101,14 @@ func (m *PayInAAAData) processPeriod(line string) (nextState string) { } // Period Servicing: Feb 2020 -func (m *PayInAAAData) getPeriod(line string) (p time.Time, e error) { +func (m *PayInAAAPeriod) getPeriod(line string) (p time.Time, e error) { idx := strings.Index(line, ":") subStr := strings.TrimSpace(line[idx+1:]) p, e = time.Parse("Jan 2006", subStr) return } -func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARow, valid bool) { +func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAARow, valid bool) { nextState = "LookingForRows" valid = false allParts := strings.Split(line, " ") @@ -137,7 +136,7 @@ func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARo return } -func (m *PayInAAAData) currencyToFloat64(cur string) (ret float64) { +func (m *PayInAAAPeriod) currencyToFloat64(cur string) (ret float64) { cur = strings.ReplaceAll(cur, " ", "") //remove space cur = strings.ReplaceAll(cur, "$", "") //remove $ cur = strings.ReplaceAll(cur, ",", "") //remove ,