AAA decoding works.

apiV1Response.go

@@ -54,6 +54,9 @@ func (m *apiV1Response) toJson() (ret []byte, e error) {
 }
 
 func (m *apiV1Response) sendJson(w http.ResponseWriter) (ret []byte, e error) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
+
 	tempMap := m.tmp
 	m.tmp = nil
 	if config.Debug {
@@ -67,6 +70,9 @@ func (m *apiV1Response) sendJson(w http.ResponseWriter) (ret []byte, e error) {
 }
 
 func apiV1SendJson(result interface{}, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
+
 	out, e := json.Marshal(result)
 	if e != nil {
 		log.Warn("Cannot convert result to json ", result)

apiV1Uploads.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"biukop.com/sfm/loan"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+)
+
+func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix
+	intId, e := strconv.Atoi(id)
+	if e != nil {
+		log.Println("invalid id for upload get", id, e)
+		apiV1Client403Error(w, r, ss) // bad request
+		return
+	}
+
+	ul := loan.Uploads{}
+	e = ul.Read(int64(intId))
+	if e != nil {
+		log.Println("no file uploaded", intId, e)
+		apiV1Client404Error(w, r, ss) // bad request
+		return
+	}
+
+	//check local file first
+	path := config.Uploads + strconv.FormatInt(ul.Id, 10) + ".uploads"
+	if fileExists(path) {
+		http.ServeFile(w, r, path)
+		return
+	}
+
+	log.Error("Upload file not found on disk", ul)
+	apiV1Server500Error(w, r) // bad request
+}
+
+func apiV1UploadsPost(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix
+
+	filename, e := saveUploadToFile(r)
+	if e != nil {
+		log.Println("no file uploaded", filename, e)
+		apiV1Client404Error(w, r, ss) // bad request
+		return
+	}
+
+	intId, e := strconv.Atoi(id)
+	if id != "" {
+		if e != nil {
+			log.Println("Error Getting File", e)
+			apiV1Client404Error(w, r, ss) // bad request
+			return
+		}
+		updateUploads(int64(intId), filename, w, r, ss)
+	} else {
+		createUploads(filename, w, r, ss)
+	}
+}
+
+func sha256File(input io.Reader) string {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, input); err != nil {
+		log.Fatal(err)
+	}
+	sum := hash.Sum(nil)
+
+	return fmt.Sprintf("%x", sum)
+}
+
+func updateUploads(id int64, fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul := loan.Uploads{}
+	e := ul.Read(int64(id))
+	if e != nil {
+		log.Println("bad upload id is given ", id, e)
+		apiV1Client404Error(w, r, ss) // bad request
+		return
+	}
+
+	ul1, _, e := saveUploadsToDB(id, fileName, r, ss)
+	if e != nil {
+		os.Remove(config.Uploads + ul.FileName)
+		ul1.Delete()
+		log.Println("cannot save file info to db ", e)
+		apiV1Server500Error(w, r) // bad request
+		return
+	}
+
+	apiV1SendJson(ul1, w, r, ss)
+}
+
+func createUploads(fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul, _, e := saveUploadsToDB(0, fileName, r, ss)
+	if e != nil {
+		log.Println("cannot save file info to db ", e)
+		e = ul.Delete() // delete the newly created, if failed, db will clean it
+		if e != nil {
+			log.Error("failed to remove unused uploads", ul)
+		}
+
+		e = os.Remove(config.Uploads + fileName)
+		if e != nil {
+			log.Error("failed to remove unused temp file", fileName)
+		}
+
+		apiV1Server500Error(w, r) // bad request
+		return
+	}
+	apiV1SendJson(ul, w, r, ss)
+}
+
+func saveUploadsToDB(id int64, fileName string,
+	r *http.Request, ss *loan.Session) (ul loan.Uploads, duplicate bool, e error) {
+	duplicate = false
+	e = r.ParseMultipartForm(10 << 20) // we should have ready parsed this, just in case
+	if e != nil {
+		return
+	}
+	file, header, e := r.FormFile("files")
+
+	file.Seek(0, 0) //seek to beginning
+	checksum := sha256File(file)
+	ul.Id = id
+	ul.Ts = time.Now()
+	ul.FileName = header.Filename
+	file.Seek(0, 0) //seek to beginning
+	ul.Format = header.Header.Get("Content-type")
+	ul.Size = header.Size // necessary to prevent duplicate
+	ul.LastModified = 0
+	ul.Sha256 = checksum // necessary to prevent duplicate
+	ul.By = ss.User
+	e = ul.Write() // this Id will have the real Id if there is duplicates
+
+	if e != nil {
+		log.Error("Fail to update db ", ul, e)
+	} else {
+		if id > 0 && ul.Id != id {
+			duplicate = true
+		}
+		target := fmt.Sprintf("%d.uploads", ul.Id)
+		e = os.Rename(config.Uploads+fileName, config.Uploads+target)
+		if e != nil {
+			ul.FileName = fileName // some how failed to rename
+		}
+	}
+	return
+}
+
+func saveUploadToFile(r *http.Request) (filename string, e error) {
+	e = r.ParseMultipartForm(10 << 20)
+	if e != nil {
+		return
+	}
+	file, header, e := r.FormFile("files")
+	if e != nil {
+		log.Println("Error Getting File", e)
+		return
+	}
+
+	out, pathError := ioutil.TempFile(config.Uploads, "can-del-upload-*.tmp")
+	if pathError != nil {
+		log.Println("Error Creating a file for writing", pathError)
+		return
+	}
+
+	out.Seek(0, 0) //seek to beginning
+	size, e := io.Copy(out, file)
+	if e != nil {
+		os.Remove(out.Name()) //remove on failure
+		log.Println("Error copying", e)
+		return
+	}
+
+	if size != header.Size {
+		e = errors.New("written file with incorrect size")
+	}
+	return filepath.Base(out.Name()), e
+}

apiv1.go

@@ -70,6 +70,10 @@ func setupApiV1Handler() []apiV1HandlerMap {
 			{"DELETE", "payIn/", apiV1PayInDelete},
 			{"GET", "user-reward/", apiV1UserReward},
 			{"GET", "login-available/", apiV1LoginAvailable},
+
+			{"POST", "lender-upload/", apiV1UploadsPost},
+			{"GET", "lender-upload/", apiV1UploadsGet},
+
 			{"GET", "login", apiV1DumpRequest},
 		}
 	} else { //production
@@ -117,6 +121,10 @@ func setupApiV1Handler() []apiV1HandlerMap {
 			{"DELETE", "payIn/", apiV1PayInDelete},
 			{"GET", "user-reward/", apiV1UserReward},
 			{"GET", "login-available/", apiV1LoginAvailable},
+
+			{"POST", "lender-upload/", apiV1UploadsPost},
+			{"GET", "lender-upload/", apiV1UploadsGet},
+
 			{"GET", "login", apiV1EmptyResponse},
 		}
 	}
@@ -126,8 +134,6 @@ func setupApiV1Handler() []apiV1HandlerMap {
 //apiV1Main  version 1 main entry for all REST API
 //
 func apiV1Main(w http.ResponseWriter, r *http.Request) {
-	//general setup
-	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
 
 	//CORS setup
 	setupCrossOriginResponse(&w, r)
@@ -293,6 +299,8 @@ func apiV1ErrorCheck(e error) {
 }
 
 func apiV1Server500Error(w http.ResponseWriter, r *http.Request) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
 
 	w.WriteHeader(500)
 	apiV1AddTrackingCookie(w, r, nil) //always the last one to set cookies
@@ -305,6 +313,9 @@ func apiV1Server500Error(w http.ResponseWriter, r *http.Request) {
 }
 
 func apiV1Client403Error(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
+
 	w.WriteHeader(403)
 	type struct403 struct {
 		Error    int
@@ -324,6 +335,9 @@ func apiV1Client403Error(w http.ResponseWriter, r *http.Request, ss *loan.Sessio
 }
 
 func apiV1Client404Error(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
+
 	w.WriteHeader(404)
 	type struct404 struct {
 		Error    int
@@ -361,6 +375,9 @@ func apiV1DumpRequest(w http.ResponseWriter, r *http.Request, ss *loan.Session)
 }
 
 func apiV1EmptyResponse(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	//general setup
+	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
+
 	apiV1AddTrackingCookie(w, r, ss) //always the last one to set cookies
 	fmt.Fprintf(w, "")
 }

debug.go

@@ -7,7 +7,11 @@ import (
 
 func logRequestDebug(data []byte, err error) (ret string) {
 	if err == nil {
-		ret = fmt.Sprintf("%s\n\n", string(data))
+		output := data
+		if len(data) > 4096 {
+			output = data[0:4095]
+		}
+		ret = fmt.Sprintf("%s\n\n", string(output))
 		fmt.Println(ret)
 	} else {
 		log.Fatalf("%s\n\n", err)

pay-in-decode.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"biukop.com/sfm/loan"
+	"errors"
+	log "github.com/sirupsen/logrus"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+type FunderType string
+
+const (
+	Funder_AAA     FunderType = "AAA Financial"
+	Funder_Pepper             = "Pepper"
+	Funder_Resimac            = "Resimac"
+	Funder_Unknown            = "cannot detect funder type"
+)
+
+type AiDecodeIncome struct {
+	Input struct {
+		Uploads  loan.Uploads
+		FileName string //a local file on disk
+		InMime   string //may not be correct, just some suggestion only.
+	}
+	Mime   string //mime actually detected.
+	PayIn  []loan.PayIn
+	Funder FunderType
+	AAA    PayInAAAData
+}
+
+func decodePayInMain(filename string, format string) (ai AiDecodeIncome, e error) {
+	ai.Input.FileName = filename
+	ai.Input.InMime = format
+	ai.PayIn = make([]loan.PayIn, 0, 10)
+	ai.Mime, e = GetFileContentType(filename)
+	if e != nil {
+		return
+	}
+
+	switch ai.Mime {
+	case "application/pdf":
+		ai.decodePayInPdf(filename, format)
+	}
+
+	return ai, e
+}
+
+// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime.
+func GetFileContentType(filename string) (contentType string, e error) {
+	contentType = ""
+	input, e := os.OpenFile(filename, os.O_RDONLY, 0755)
+	// Only the first 512 bytes are used to sniff the content type.
+	buffer := make([]byte, 512)
+
+	_, e = input.Read(buffer)
+	if e != nil {
+		return
+	}
+
+	// Use the net/http package's handy DectectContentType function. Always returns a valid
+	// content-type by returning "application/octet-stream" if no others seemed to match.
+	contentType = http.DetectContentType(buffer)
+	return
+}
+
+func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []loan.PayIn, e error) {
+	cmd := exec.Command("pdftotext", "-layout", filename, "-")
+	//log.Println(cmd.String())
+	out, e := cmd.Output()
+	if e != nil {
+		log.Fatal(e)
+	}
+
+	raw := string(out)
+	switch m.detectFunder(raw) {
+	case Funder_AAA:
+		e = m.AAA.decodeAAAPdf(raw)
+		log.Println("AAA final result", m.AAA)
+		break
+	case Funder_Unknown:
+		e = errors.New(Funder_Unknown)
+		break // not able to detect Funder
+	}
+	return
+}
+
+func (m *AiDecodeIncome) detectFunder(raw string) FunderType {
+	if m.isAAA(raw) {
+		return Funder_AAA
+	}
+
+	return Funder_Unknown
+}
+
+func (m *AiDecodeIncome) isAAA(raw string) bool {
+	keyword := "AAA Financial Trail Report"
+	lines := strings.Split(raw, "\n")
+	return m.checkFunderKeyword(keyword, lines, 0, 3)
+}
+
+func (m *AiDecodeIncome) checkFunderKeyword(keyword string, lines []string, start int, end int) bool {
+	for idx, line := range lines {
+		// first 10 lines has Key word
+		if strings.Contains(line, keyword) && idx >= start && idx <= 10 {
+			return true
+		}
+	}
+	return false
+}

pay-in-decode_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	log "github.com/sirupsen/logrus"
+	"testing"
+)
+
+func TestDecodePayInMain(t *testing.T) {
+	fileName := "./uploads/30.uploads"
+	fileName = "/home/sp/go/src/SFM_Loan_RestApi/uploads/30.uploads"
+	data, _ := decodePayInMain(fileName, "application/vnd.ms-excel")
+	log.Println(data)
+}

payIn-AAA.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	log "github.com/sirupsen/logrus"
+	"strconv"
+	"strings"
+	"time"
+)
+
+/* Sample Text
+
+AAA Financial Trail Report
+Super Finance Markets Pty Ltd
+   Loan Number                  SettDate      Loan          Balance    Arrears   DisDate   IntTrail$    Comments
+                                             Facility
+  Columbus
+  Period Servicing: Feb 2020
+400053440                       02-Sep-19        $552,463      552,579.52                     $32.19
+400063271                        19-Feb-20       $832,000      832,000.00                      $0.00
+                                                                       Columbus Total:         $32.19
+
+                                                                            Grand Total:       $32.19
+
+Super Finance Markets Pty Ltd
+
+*/
+
+type PayInAAARow struct {
+	LoanNUmber string
+	Settlement time.Time
+	LoanAmount float64
+	Balance    float64
+	InTrail    float64
+}
+
+type PayInAAAPeriod struct {
+	Period time.Time
+	Rows   []PayInAAARow
+}
+
+type PayInAAAData struct {
+	Periods []PayInAAAPeriod
+}
+
+func (m *PayInAAAData) decodeAAAPdf(raw string) (e error) {
+	m.Periods = make([]PayInAAAPeriod, 0, 10)
+	lines := strings.Split(raw, "\n")
+
+	var tableHeader []string
+	var tableHeaderLine int
+	currentPeriod := -1
+	state := "start"
+	for idx, l := range lines { // DFA, wow, finally it's used. after years of learning
+		switch state {
+		case "start":
+			state = m.processStart(l)
+			if state == "LookingForPeriod" {
+				tableHeaderLine = idx
+				tableHeader = strings.Split(l, " ")
+				log.Println("Find table header", tableHeader, l, tableHeaderLine)
+			}
+			break
+		case "LookingForPeriod":
+			state = m.processPeriod(l)
+			if state == "LookingForRows" {
+				batch := PayInAAAPeriod{}
+				m.Periods = append(m.Periods, batch)
+				currentPeriod++ //move index to next , or 0 for a start
+				m.Periods[currentPeriod].Period, e = m.getPeriod(l)
+				m.Periods[currentPeriod].Rows = make([]PayInAAARow, 0, 10)
+				if e != nil {
+					log.Warn("cannot find period", l, e)
+					state = "LookingForPeriod"
+				}
+			}
+			break
+		case "LookingForRows", "LookingForRowsSkipCurrent":
+			nextState, row, valid := m.processRow(l)
+			if valid {
+				m.Periods[currentPeriod].Rows = append(m.Periods[currentPeriod].Rows, row)
+			}
+			state = nextState
+			break
+		}
+	}
+	return
+}
+
+func (m *PayInAAAData) processStart(line string) (nextState string) {
+	nextState = "start"
+	if strings.Contains(line, "Loan Number") &&
+		strings.Contains(line, "SettDate") &&
+		strings.Contains(line, "Balance") &&
+		strings.Contains(line, "IntTrail$") {
+		nextState = "LookingForPeriod"
+	}
+	return
+}
+
+func (m *PayInAAAData) processPeriod(line string) (nextState string) {
+	nextState = "LookingForPeriod"
+	if strings.Contains(line, "Period Servicing:") {
+		nextState = "LookingForRows"
+	}
+	return
+}
+
+// Period Servicing: Feb 2020
+func (m *PayInAAAData) getPeriod(line string) (p time.Time, e error) {
+	idx := strings.Index(line, ":")
+	subStr := strings.TrimSpace(line[idx+1:])
+	p, e = time.Parse("Jan 2006", subStr)
+	return
+}
+
+func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARow, valid bool) {
+	nextState = "LookingForRows"
+	valid = false
+	allParts := strings.Split(line, " ")
+	el := make([]string, 0, 10)
+	for _, item := range allParts {
+		if len(item) > 0 {
+			el = append(el, item)
+		}
+	}
+
+	if len(el) >= 5 {
+		row.LoanNUmber = el[0]
+		row.Settlement, _ = time.Parse("02-Jan-06", el[1])
+		row.LoanAmount = m.currencyToFloat64(el[2])
+		row.Balance = m.currencyToFloat64(el[3])
+		row.InTrail = m.currencyToFloat64(el[len(el)-1]) //last element
+		valid = true
+	} else {
+		if strings.Contains(line, "Total:") {
+			nextState = "start"
+		} else {
+			nextState = "LookingForRowsSkipCurrent"
+		}
+	}
+	return
+}
+
+func (m *PayInAAAData) currencyToFloat64(cur string) (ret float64) {
+	cur = strings.ReplaceAll(cur, " ", "") //remove space
+	cur = strings.ReplaceAll(cur, "$", "") //remove $
+	cur = strings.ReplaceAll(cur, ",", "") //remove ,
+	ret, _ = strconv.ParseFloat(cur, 64)
+	return ret
+}