uploads polished.

UploadsOnDisk.go

@@ -217,7 +217,8 @@ func (m *uploadsOnDisk) GetFileType() (ret string, e error) {
 		return "pdf", nil
 	}
 
-	if strings.ToLower(strMime) == "application/vnd.ms-excel" {
+	if strings.ToLower(strMime) == "application/vnd.ms-excel" || strings.ToLower(m.Upload.Format) ==
+		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
 		return "excel", nil
 	}
 
@@ -227,6 +228,20 @@ func (m *uploadsOnDisk) GetFileType() (ret string, e error) {
 		return "opensheet", nil
 	}
 
+	// check suffix which is not reliable
+	ext := filepath.Ext(m.Upload.FileName)
+	ext = strings.ToLower(ext)
+	if ext[0] == '.' {
+		ext = ext[1:] // remove the first char 'dot' .
+	}
+	switch ext {
+	case "xls", "xlsx":
+		return "excel", nil
+	case "pdf":
+		return "pdf", nil
+	default:
+		log.Warn("unhandled uploads type", ext)
+	}
 	return "", nil
 }
 

UploadsOnDisk_test.go

@@ -41,4 +41,8 @@ func TestConvertImageToThumbnail(t *testing.T) {
 	_, name := filepath.Split("/home/sp/uploads/abc.uploads")
 	test := fileNameWithoutExtTrimSuffix(name) + ".jpg"
 	log.Println(test)
+	test1 := filepath.Ext("abc.uploads.pdf")
+	log.Println(test1)
+	ext := "."
+	log.Println(ext[1:])
 }

apiV1Uploads.go

@@ -15,8 +15,27 @@ import (
 	"time"
 )
 
-func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	id := r.URL.Path[len(apiV1Prefix+"upload/"):] //remove prefix
+func apiV1UploadMetaGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	id := r.URL.Path[len(apiV1Prefix+"upload-meta/"):] //remove prefix
+	intId, e := strconv.Atoi(id)
+	if e != nil {
+		log.Println("invalid id for upload get", id, e)
+		apiV1Client403Error(w, r, ss) // bad request
+		return
+	}
+
+	ulmeta := loan.Uploads{}
+	e = ulmeta.Read(int64(intId))
+	if e != nil {
+		log.Println("upload not found", id, e)
+		apiV1Client404Error(w, r, ss) // bad request
+		return
+	}
+	apiV1SendJson(ulmeta, w, r, ss)
+}
+
+func apiV1UploadOriginalFileGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	id := r.URL.Path[len(apiV1Prefix+"upload-original/"):] //remove prefix
 	intId, e := strconv.Atoi(id)
 	if e != nil {
 		log.Println("invalid id for upload get", id, e)
@@ -118,11 +137,7 @@ func createUploads(filePath string, w http.ResponseWriter, r *http.Request, ss *
 		apiV1Server500Error(w, r) // bad request
 		return
 	}
-
-	ai := AiDecodeIncome{}
-	ai.decodeUploadToPayIn(ul.Upload)
-
-	apiV1SendJson(ai, w, r, ss)
+	apiV1SendJson(ulMeta, w, r, ss)
 }
 
 func saveUploadsMetaToDB(id int64, filePath string,
@@ -195,8 +210,8 @@ func saveUploadToFile(r *http.Request) (filename string, e error) {
 	return out.Name(), e
 }
 
-func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) {
-	strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix
+func getRequestedUpload(strId string, w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) {
+
 	Id, e := strconv.Atoi(strId)
 	if e != nil {
 		log.Error("Invalid uploads Id cannot convert to integer", Id, e)
@@ -217,7 +232,13 @@ func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session
 }
 
 func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	ul, e := getRequestedUpload(w, r, ss)
+	strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix
+	if strId == "default" {
+		http.ServeFile(w, r, config.UploadsDir.JpgDefault)
+		return
+	}
+
+	ul, e := getRequestedUpload(strId, w, r, ss)
 	if e != nil {
 		return
 	}
@@ -229,7 +250,7 @@ func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session
 			defer f.Close()
 			fi, e := f.Stat()
 			if e == nil {
-				w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
+				//w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
 				http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f)
 				return
 			}
@@ -259,7 +280,12 @@ func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session
 
 }
 func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	ul, e := getRequestedUpload(w, r, ss)
+	strId := r.URL.Path[len(apiV1Prefix+"upload-as-thumbnail/"):] //remove prefix
+	if strId == "default" {
+		http.ServeFile(w, r, config.UploadsDir.ThumbDefault)
+		return
+	}
+	ul, e := getRequestedUpload(strId, w, r, ss)
 	if e != nil {
 		return
 	}
@@ -282,8 +308,13 @@ func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Ses
 	}
 }
 
-func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	ul, e := getRequestedUpload(w, r, ss)
+func apiV1UploadAsPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	strId := r.URL.Path[len(apiV1Prefix+"upload-as-pdf/"):] //remove prefix
+	if strId == "default" {
+		http.ServeFile(w, r, config.UploadsDir.PdfDefault)
+		return
+	}
+	ul, e := getRequestedUpload(strId, w, r, ss)
 	if e != nil {
 		return
 	}
@@ -302,7 +333,9 @@ func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
 			defer f.Close()
 			fi, e := f.Stat()
 			if e == nil {
-				w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
+				if forceHttpDownload(r) {
+					w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
+				}
 				http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f)
 				return
 			}
@@ -330,3 +363,13 @@ func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
 		apiV1Client404Error(w, r, ss)
 	}
 }
+
+func forceHttpDownload(r *http.Request) bool {
+	keys, ok := r.URL.Query()["download"]
+
+	if !ok || len(keys[0]) < 1 {
+		return false
+	}
+	key := keys[0]
+	return key == "force"
+}

apiv1.go

@@ -72,9 +72,14 @@ func setupApiV1Handler() []apiV1HandlerMap {
 			{"GET", "login-available/", apiV1LoginAvailable},
 
 			{"POST", "lender-upload/", apiV1UploadsPost},
-			{"GET", "lender-upload/", apiV1UploadsGet},
+			{"GET", "lender-upload/", apiV1UploadOriginalFileGet},
+
+			{"GET", "upload-analysis/", apiV1UploadAnalysis},
+			{"GET", "upload-as-image/", apiV1UploadAsImage},
 			{"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail},
-			{"GET", "upload-as-pdf/", apiV1UploadAPDF},
+			{"GET", "upload-as-pdf/", apiV1UploadAsPDF},
+			{"GET", "upload-original/", apiV1UploadOriginalFileGet},
+			{"GET", "upload/", apiV1UploadMetaGet},
 
 			{"GET", "login", apiV1DumpRequest},
 		}
@@ -125,13 +130,14 @@ func setupApiV1Handler() []apiV1HandlerMap {
 			{"GET", "login-available/", apiV1LoginAvailable},
 
 			{"POST", "lender-upload/", apiV1UploadsPost},
-			{"GET", "lender-upload/", apiV1UploadsGet},
+			{"GET", "lender-upload/", apiV1UploadOriginalFileGet},
 
 			{"GET", "upload-analysis/", apiV1UploadAnalysis},
 			{"GET", "upload-as-image/", apiV1UploadAsImage},
 			{"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail},
-			{"GET", "upload-as-pdf/", apiV1UploadAPDF},
-			{"GET", "upload/", apiV1UploadsGet},
+			{"GET", "upload-as-pdf/", apiV1UploadAsPDF},
+			{"GET", "upload-original/", apiV1UploadOriginalFileGet},
+			{"GET", "upload-meta/", apiV1UploadMetaGet},
 
 			{"GET", "login", apiV1EmptyResponse},
 		}

pay-in-decode.go

@@ -18,20 +18,21 @@ const (
 )
 
 type AiDecodeIncome struct {
+	Id     int64
 	Input  loan.Uploads
 	ul     uploadsOnDisk // internal data
 	Mime   string        //mime actually detected.
 	PayIn  []loan.PayIn
 	Funder FunderType
-	AAA    PayInAAAData
+	AAA    []PayInAAAPeriod
 }
 
 func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) {
+	m.Id = ulMeta.Id
 	m.Input = ulMeta
 	m.ul.Upload = ulMeta
 
 	m.PayIn = make([]loan.PayIn, 0, 10)
-
 	switch m.getFileType() {
 	case "pdf":
 		m.decodePdf()
@@ -72,7 +73,7 @@ func (m *AiDecodeIncome) decodePdf() (e error) {
 	switch m.detectFunder(raw) {
 	case Funder_AAA:
 		m.Funder = Funder_AAA
-		e = m.AAA.decodeAAAPdf(raw)
+		e = m.decodeAAAPdf(raw)
 		log.Println("AAA final result", m.AAA)
 		break
 	case Funder_Unknown:

payIn-AAA.go

@@ -38,51 +38,50 @@ type PayInAAAPeriod struct {
 	Rows   []PayInAAARow
 }
 
-type PayInAAAData struct {
-	Periods []PayInAAAPeriod
-}
-
-func (m *PayInAAAData) decodeAAAPdf(raw string) (e error) {
-	m.Periods = make([]PayInAAAPeriod, 0, 10)
+func (m *AiDecodeIncome) decodeAAAPdf(raw string) (e error) {
+	m.AAA = make([]PayInAAAPeriod, 0, 10)
 	lines := strings.Split(raw, "\n")
 
-	currentPeriod := -1
+	currentDecoder := PayInAAAPeriod{}
 	state := "start"
 	for _, l := range lines { // DFA, wow, finally it's used. after years of learning
 		switch state {
 		case "start":
-			state = m.processStart(l)
+			state = currentDecoder.processStart(l)
 			if state == "LookingForPeriod" {
 				// determine column index, if their column is changing
 			}
 			break
 		case "LookingForPeriod":
-			state = m.processPeriod(l)
+			state = currentDecoder.processPeriod(l)
 			if state == "LookingForRows" {
-				batch := PayInAAAPeriod{}
-				m.Periods = append(m.Periods, batch)
-				currentPeriod++ //move index to next , or 0 for a start
-				m.Periods[currentPeriod].Period, e = m.getPeriod(l)
-				m.Periods[currentPeriod].Rows = make([]PayInAAARow, 0, 10)
+				currentDecoder.Period, e = currentDecoder.getPeriod(l)
+				currentDecoder.Rows = make([]PayInAAARow, 0, 10)
 				if e != nil {
 					log.Warn("cannot find period", l, e)
 					state = "LookingForPeriod"
+				} else {
+					m.AAA = append(m.AAA, currentDecoder)
+
 				}
 			}
 			break
 		case "LookingForRows", "LookingForRowsSkipCurrent":
-			nextState, row, valid := m.processRow(l)
+			nextState, row, valid := currentDecoder.processRow(l)
 			if valid {
-				m.Periods[currentPeriod].Rows = append(m.Periods[currentPeriod].Rows, row)
+				currentDecoder.Rows = append(currentDecoder.Rows, row)
 			}
 			state = nextState
+			if nextState == "start" {
+				currentDecoder = PayInAAAPeriod{} //renew to a empty state
+			}
 			break
 		}
 	}
 	return
 }
 
-func (m *PayInAAAData) processStart(line string) (nextState string) {
+func (m *PayInAAAPeriod) processStart(line string) (nextState string) {
 	nextState = "start"
 	if strings.Contains(line, "Loan Number") &&
 		strings.Contains(line, "SettDate") &&
@@ -93,7 +92,7 @@ func (m *PayInAAAData) processStart(line string) (nextState string) {
 	return
 }
 
-func (m *PayInAAAData) processPeriod(line string) (nextState string) {
+func (m *PayInAAAPeriod) processPeriod(line string) (nextState string) {
 	nextState = "LookingForPeriod"
 	if strings.Contains(line, "Period Servicing:") {
 		nextState = "LookingForRows"
@@ -102,14 +101,14 @@ func (m *PayInAAAData) processPeriod(line string) (nextState string) {
 }
 
 // Period Servicing: Feb 2020
-func (m *PayInAAAData) getPeriod(line string) (p time.Time, e error) {
+func (m *PayInAAAPeriod) getPeriod(line string) (p time.Time, e error) {
 	idx := strings.Index(line, ":")
 	subStr := strings.TrimSpace(line[idx+1:])
 	p, e = time.Parse("Jan 2006", subStr)
 	return
 }
 
-func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARow, valid bool) {
+func (m *PayInAAAPeriod) processRow(line string) (nextState string, row PayInAAARow, valid bool) {
 	nextState = "LookingForRows"
 	valid = false
 	allParts := strings.Split(line, " ")
@@ -137,7 +136,7 @@ func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARo
 	return
 }
 
-func (m *PayInAAAData) currencyToFloat64(cur string) (ret float64) {
+func (m *PayInAAAPeriod) currencyToFloat64(cur string) (ret float64) {
 	cur = strings.ReplaceAll(cur, " ", "") //remove space
 	cur = strings.ReplaceAll(cur, "$", "") //remove $
 	cur = strings.ReplaceAll(cur, ",", "") //remove ,