uploads with pdf/image/format

.gitignore

 /apiv1
 /uploads/
+/tmp/

UploadsOnDisk.go

+package main
+
+import (
+	"biukop.com/sfm/loan"
+	"errors"
+	log "github.com/sirupsen/logrus"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+type uploadsOnDisk struct {
+	Upload loan.Uploads
+}
+
+func (m *uploadsOnDisk) convertUploadsToPDF() (e error) {
+	return convertUploadsToPDF(m.Upload)
+}
+func convertUploadsToPDF(ul loan.Uploads) (e error) {
+	m := uploadsOnDisk{}
+	m.Upload = ul
+
+	if strings.Contains(strings.ToLower(ul.Format), "excel") ||
+		strings.Contains(strings.ToLower(ul.Format), "spreadsheet") {
+		e = m.convertExcelToPDF()
+		return // excel is converted
+	}
+
+	if strings.Contains(strings.ToLower(ul.Format), "/pdf") {
+		return // no need to convert
+	}
+
+	e = errors.New("don't know how to convert file to PDF")
+	log.Error("don't know how to convert file to PDF", ul)
+	return
+}
+
+func (m *uploadsOnDisk) convertUploadsToJpg() (e error) {
+	return convertUploadsToJpg(m.Upload)
+}
+
+func convertUploadsToJpg(ul loan.Uploads) (e error) {
+	m := uploadsOnDisk{}
+	m.Upload = ul
+
+	if strings.Contains(strings.ToLower(ul.Format), "excel") ||
+		strings.Contains(strings.ToLower(ul.Format), "spreadsheet") {
+		e = m.convertExcelToJpg()
+		return // excel is converted
+	}
+	if strings.Contains(strings.ToLower(ul.Format), "/pdf") {
+		e = m.convertPDFToJpg()
+		return // excel is converted
+	}
+
+	e = errors.New("don't know how to convert file to image")
+	log.Error("don't know how to convert file to image", ul)
+	return
+}
+
+func (m *uploadsOnDisk) convertUploadsToThumb() (e error) {
+	return convertUploadsToThumb(m.Upload)
+}
+func convertUploadsToThumb(ul loan.Uploads) (e error) {
+	m := uploadsOnDisk{}
+	m.Upload = ul
+	if !fileExists(m.jpgPath()) {
+		e = m.convertUploadsToJpg()
+		if e != nil {
+			return
+		}
+	}
+
+	e = ConvertImageToThumbnail(m.jpgPath(), m.thumbPath(), 256)
+	if e != nil {
+		log.Error("cannot create thumbnail for uploads", m.Upload, e)
+		return
+	}
+	return
+}
+
+func (m *uploadsOnDisk) filePath() string {
+	return config.UploadsDir.FileDir + strconv.Itoa(int(m.Upload.Id)) + ".uploads"
+}
+
+func (m *uploadsOnDisk) jpgPath() string {
+	return config.UploadsDir.JpgDir + strconv.Itoa(int(m.Upload.Id)) + ".jpg"
+}
+
+func (m *uploadsOnDisk) thumbPath() string {
+	return config.UploadsDir.ThumbDir + strconv.Itoa(int(m.Upload.Id)) + ".webp"
+}
+
+func (m *uploadsOnDisk) pdfPath() string {
+	return config.UploadsDir.PdfDir + strconv.Itoa(int(m.Upload.Id)) + ".pdf"
+}
+func (m *uploadsOnDisk) convertExcelToPDF() (e error) {
+	if fileExists(m.pdfPath()) {
+		log.Info("Skip conversion excel to PDF , already exists", m)
+		return
+	}
+	return m.convertExcelTo("pdf")
+}
+
+func (m *uploadsOnDisk) convertExcelToJpg() (e error) {
+	if fileExists(m.jpgPath()) {
+		log.Info("Skip conversion excel to Jpg , already exists", m)
+		return
+	}
+	return m.convertExcelTo("jpg")
+}
+
+func (m *uploadsOnDisk) convertExcelTo(format string) (e error) {
+	if format != "pdf" && format != "jpg" {
+		e = errors.New("unsupported format")
+		return
+	}
+
+	dst := m.jpgPath()
+	if format == "pdf" {
+		dst = m.pdfPath()
+	}
+
+	dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-xls-to-"+format+"-")
+	if e != nil {
+		log.Error("cannot create tmp dir for converting image", m.Upload, e)
+		return
+	}
+	defer os.RemoveAll(dir)
+
+	cmd := exec.Command("libreoffice", "--convert-to", format, "--outdir", dir, m.filePath())
+	strCmd := cmd.String()
+	log.Debug("command is ", strCmd)
+	out, e := cmd.Output()
+	if e != nil {
+		log.Error("cannot converting Excel to "+format+":", m.Upload, e)
+		return
+	} else { // success
+		log.Info("convert to "+format, m.Upload, " output: ", string(out))
+	}
+
+	_, name := filepath.Split(m.filePath())
+
+	src := dir + string(os.PathSeparator) + fileNameWithoutExtTrimSuffix(name) + "." + format
+	e = os.Rename(src, dst) // there should be only one jpg
+
+	return
+}
+
+// first page to thumbnail
+// all page to single jpg
+func (m *uploadsOnDisk) convertPDFToJpg() (e error) {
+
+	if fileExists(m.jpgPath()) {
+		// no need to reconvert it again
+		log.Info("PDF to JPG skipped it already exists ", m)
+		return
+	}
+
+	dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-pdf-to-jpg-")
+	if e != nil {
+		log.Error("cannot create tmp dir for converting image", m.Upload, e)
+		return
+	}
+	defer os.RemoveAll(dir)
+
+	// convert -density 3000  abc.pdf  path/tmp/result.png
+	// could be path/tmp/result-0, result-1, result-2, ... png
+	target := dir + string(os.PathSeparator) + "result.jpg" //.jpg suffix is important
+	cmd := exec.Command("convert", "-density", "300", m.filePath(), target)
+	strCmd := cmd.String()
+	log.Debug("command is ", strCmd)
+	out, e := cmd.Output()
+	if e != nil {
+		log.Error("cannot create png file for PDF", m.Upload, e)
+		return
+	} else {
+		log.Info("convert ", m.Upload, " output: ", string(out))
+	}
+
+	// montage -mode concatenate -tile 1x 30*png 30.jpg
+	if fileExists(target) { // single file,
+		e = os.Rename(target, m.jpgPath()) // there should be only one jpg
+		_ = ConvertImageToThumbnail(target, m.thumbPath(), 256)
+	} else { // multi-page, we have -0 -1 -2 -3 -4  files
+		firstPage := dir + string(os.PathSeparator) + "result-0.jpg"
+		_ = ConvertImageToThumbnail(firstPage, m.thumbPath(), 256)
+
+		batch := dir + string(os.PathSeparator) + "result*jpg" // result* is important
+		target = dir + string(os.PathSeparator) + "final.jpg"  // .jpg suffix is important
+		cmd = exec.Command("montage", "-mode", "concatenate", "-tile", "1x", batch, target)
+
+		strCmd = cmd.String()
+		log.Debug("command is ", strCmd)
+		out, e = cmd.Output()
+		if e != nil {
+			return
+		} else {
+			log.Info("montage ", m, " output: ", string(out))
+		}
+		e = os.Rename(target, m.jpgPath()) // give combined file to target
+	}
+	return
+}
+
+func (m *uploadsOnDisk) GetFileType() (ret string, e error) {
+	strMime, e := GetFileContentType(m.filePath())
+	if e != nil {
+		return
+	}
+
+	if strings.ToLower(strMime) == "application/pdf" {
+		return "pdf", nil
+	}
+
+	if strings.ToLower(strMime) == "application/vnd.ms-excel" {
+		return "excel", nil
+	}
+
+	if strings.ToLower(strMime) == "application/zip" &&
+		strings.ToLower(m.Upload.Format) ==
+			"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
+		return "opensheet", nil
+	}
+
+	return "", nil
+}
+
+// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime.
+func GetFileContentType(filename string) (contentType string, e error) {
+	contentType = ""
+	input, e := os.OpenFile(filename, os.O_RDONLY, 0755)
+	// Only the first 512 bytes are used to sniff the content type.
+	buffer := make([]byte, 512)
+
+	_, e = input.Read(buffer)
+	if e != nil {
+		return
+	}
+
+	// Use the net/http package's handy DectectContentType function. Always returns a valid
+	// content-type by returning "application/octet-stream" if no others seemed to match.
+	contentType = http.DetectContentType(buffer)
+	return
+}
+
+func ConvertImageToThumbnail(srcPath string, dstPath string, size int) (e error) {
+	if fileExists(dstPath) {
+		log.Info("skip converting thumbnail it exists", dstPath)
+		return
+	}
+	if size <= 0 {
+		size = 256
+	}
+	// convert -thumbnail 200 abc.png thumb.abc.png
+	cmd := exec.Command("convert", "-thumbnail", strconv.Itoa(size), srcPath, dstPath)
+	strCmd := cmd.String()
+	log.Debug("create thumbnail: ", strCmd)
+
+	out, e := cmd.Output()
+	if e != nil {
+		return
+	} else { // success
+		log.Info("success output: \n: ", string(out))
+	}
+	return
+}
+
+func fileNameWithoutExtTrimSuffix(fileName string) string {
+	return strings.TrimSuffix(fileName, filepath.Ext(fileName))
+}

UploadsOnDisk_test.go

+package main
+
+import (
+	"biukop.com/sfm/loan"
+	log "github.com/sirupsen/logrus"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetConvertUploadsToImage(t *testing.T) {
+	for id := 30; id <= 44; id++ {
+		ul := loan.Uploads{}
+		e := ul.Read(int64(id))
+		if e != nil {
+			log.Error(e)
+			return
+		}
+
+		e = convertUploadsToPDF(ul)
+		if e != nil {
+			log.Error(e)
+			return
+		}
+
+		e = convertUploadsToJpg(ul)
+		if e != nil {
+			log.Error(e)
+			return
+		}
+
+		e = convertUploadsToThumb(ul)
+		if e != nil {
+			log.Error(e)
+			return
+		}
+	}
+
+}
+
+func TestConvertImageToThumbnail(t *testing.T) {
+	_, name := filepath.Split("/home/sp/uploads/abc.uploads")
+	test := fileNameWithoutExtTrimSuffix(name) + ".jpg"
+	log.Println(test)
+}

apiV1UploadAnalysis.go

+package main
+
+import (
+	"biukop.com/sfm/loan"
+	log "github.com/sirupsen/logrus"
+	"net/http"
+	"strconv"
+)
+
+func apiV1UploadAnalysis(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	strId := r.URL.Path[len(apiV1Prefix+"upload-analysis/"):] //remove prefix
+	Id, e := strconv.Atoi(strId)
+	if e != nil {
+		log.Error("Invalid uploads Id cannot convert to integer", Id, e)
+		apiV1Client403Error(w, r, ss)
+		return
+	}
+
+	ul := loan.Uploads{}
+	e = ul.Read(int64(Id))
+	if e != nil {
+		log.Error("Upload not found or read error from db", Id, e)
+		apiV1Client404Error(w, r, ss)
+		return
+	}
+
+	ai := AiDecodeIncome{}
+	e = ai.decodeUploadToPayIn(ul)
+	if e != nil {
+		log.Error("Invalid uploads Id cannot conver to integer", Id, e)
+		apiV1Server500Error(w, r)
+		return
+	}
+	apiV1SendJson(ai, w, r, ss)
+}

apiV1Uploads.go

 	"io/ioutil"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 )
 
 func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix
+	id := r.URL.Path[len(apiV1Prefix+"upload/"):] //remove prefix
 	intId, e := strconv.Atoi(id)
 	if e != nil {
 		log.Println("invalid id for upload get", id, e)
 		return
 	}
 
-	ul := loan.Uploads{}
-	e = ul.Read(int64(intId))
+	ul := uploadsOnDisk{}
+	e = ul.Upload.Read(int64(intId))
 	if e != nil {
 		log.Println("no file uploaded", intId, e)
 		apiV1Client404Error(w, r, ss) // bad request
 	}
 
 	//check local file first
-	path := config.Uploads + strconv.FormatInt(ul.Id, 10) + ".uploads"
+	path := ul.filePath()
 	if fileExists(path) {
 		http.ServeFile(w, r, path)
 		return
 func apiV1UploadsPost(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
 	id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix
 
-	filename, e := saveUploadToFile(r)
+	filepath, e := saveUploadToFile(r)
 	if e != nil {
-		log.Println("no file uploaded", filename, e)
+		log.Println("no file uploaded", filepath, e)
 		apiV1Client404Error(w, r, ss) // bad request
 		return
 	}
 			apiV1Client404Error(w, r, ss) // bad request
 			return
 		}
-		updateUploads(int64(intId), filename, w, r, ss)
+		updateUploads(int64(intId), filepath, w, r, ss)
 	} else {
-		createUploads(filename, w, r, ss)
+		createUploads(filepath, w, r, ss)
 	}
 }
 
 	return fmt.Sprintf("%x", sum)
 }
 
-func updateUploads(id int64, fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	ul := loan.Uploads{}
-	e := ul.Read(int64(id))
+func updateUploads(id int64, filePath string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul := uploadsOnDisk{}
+	e := ul.Upload.Read(int64(id))
 	if e != nil {
 		log.Println("bad upload id is given ", id, e)
 		apiV1Client404Error(w, r, ss) // bad request
 		return
 	}
 
-	ul1, _, e := saveUploadsToDB(id, fileName, r, ss)
+	ul1, isDuplicate, e := saveUploadsMetaToDB(id, filePath, r, ss)
+	ul.Upload.IsDuplicate = isDuplicate
 	if e != nil {
-		os.Remove(config.Uploads + ul.FileName)
+		os.Remove(ul.filePath())
 		ul1.Delete()
 		log.Println("cannot save file info to db ", e)
 		apiV1Server500Error(w, r) // bad request
 	apiV1SendJson(ul1, w, r, ss)
 }
 
-func createUploads(fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
-	ul, _, e := saveUploadsToDB(0, fileName, r, ss)
+func createUploads(filePath string, w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul := uploadsOnDisk{}
+	ulMeta, isDuplicate, e := saveUploadsMetaToDB(0, filePath, r, ss)
+	ul.Upload = ulMeta
+	ul.Upload.IsDuplicate = isDuplicate
 	if e != nil {
 		log.Println("cannot save file info to db ", e)
-		e = ul.Delete() // delete the newly created, if failed, db will clean it
+		e = ulMeta.Delete() // delete the newly created, if failed, db will clean it
 		if e != nil {
 			log.Error("failed to remove unused uploads", ul)
 		}
 
-		e = os.Remove(config.Uploads + fileName)
+		e = os.Remove(ul.filePath())
 		if e != nil {
-			log.Error("failed to remove unused temp file", fileName)
+			log.Error("failed to remove unused temp file", filePath)
 		}
 
 		apiV1Server500Error(w, r) // bad request
 		return
 	}
-	apiV1SendJson(ul, w, r, ss)
+
+	ai := AiDecodeIncome{}
+	ai.decodeUploadToPayIn(ul.Upload)
+
+	apiV1SendJson(ai, w, r, ss)
 }
 
-func saveUploadsToDB(id int64, fileName string,
-	r *http.Request, ss *loan.Session) (ul loan.Uploads, duplicate bool, e error) {
+func saveUploadsMetaToDB(id int64, filePath string,
+	r *http.Request, ss *loan.Session) (ulMeta loan.Uploads, duplicate bool, e error) {
 	duplicate = false
 	e = r.ParseMultipartForm(10 << 20) // we should have ready parsed this, just in case
 	if e != nil {
 
 	file.Seek(0, 0) //seek to beginning
 	checksum := sha256File(file)
-	ul.Id = id
-	ul.Ts = time.Now()
-	ul.FileName = header.Filename
+	ulMeta.Id = id
+	ulMeta.Ts = time.Now()
+	ulMeta.FileName = header.Filename
 	file.Seek(0, 0) //seek to beginning
-	ul.Format = header.Header.Get("Content-type")
-	ul.Size = header.Size // necessary to prevent duplicate
-	ul.LastModified = 0
-	ul.Sha256 = checksum // necessary to prevent duplicate
-	ul.By = ss.User
-	e = ul.Write() // this Id will have the real Id if there is duplicates
+	ulMeta.Format = header.Header.Get("Content-type")
+	ulMeta.Size = header.Size // necessary to prevent duplicate
+	ulMeta.LastModified = 0
+	ulMeta.Sha256 = checksum // necessary to prevent duplicate
+	ulMeta.By = ss.User
+	e = ulMeta.Write() // this Id will have the real Id if there is duplicates
 
 	if e != nil {
-		log.Error("Fail to update db ", ul, e)
+		log.Error("Fail to update db ", ulMeta, e)
 	} else {
-		if id > 0 && ul.Id != id {
+		if (id > 0 && ulMeta.Id != id) || (id == 0 && ulMeta.IsDuplicate) {
 			duplicate = true
 		}
-		target := fmt.Sprintf("%d.uploads", ul.Id)
-		e = os.Rename(config.Uploads+fileName, config.Uploads+target)
+		ul := uploadsOnDisk{}
+		ul.Upload = ulMeta
+		e = os.Rename(filePath, ul.filePath())
 		if e != nil {
-			ul.FileName = fileName // some how failed to rename
+			os.Remove(filePath)
+			log.Error("fail to move file from ", filePath, "to", ul.filePath())
 		}
 	}
 	return
 		return
 	}
 
-	out, pathError := ioutil.TempFile(config.Uploads, "can-del-upload-*.tmp")
+	out, pathError := ioutil.TempFile(config.UploadsDir.FileDir, "can-del-upload-*.tmp")
 	if pathError != nil {
 		log.Println("Error Creating a file for writing", pathError)
 		return
 	if size != header.Size {
 		e = errors.New("written file with incorrect size")
 	}
-	return filepath.Base(out.Name()), e
+	return out.Name(), e
+}
+
+func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) {
+	strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix
+	Id, e := strconv.Atoi(strId)
+	if e != nil {
+		log.Error("Invalid uploads Id cannot convert to integer", Id, e)
+		apiV1Client403Error(w, r, ss)
+		return
+	}
+
+	ul := loan.Uploads{}
+	e = ul.Read(int64(Id))
+	if e != nil {
+		log.Error("Upload not found or read error from db", Id, e)
+		apiV1Client404Error(w, r, ss)
+		return
+	}
+
+	ret.Upload = ul
+	return
+}
+
+func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul, e := getRequestedUpload(w, r, ss)
+	if e != nil {
+		return
+	}
+
+	// if this is image itself, serve it directly
+	if strings.Contains(strings.ToLower(ul.Upload.Format), "image") {
+		f, e := os.Open(ul.filePath())
+		if e == nil {
+			defer f.Close()
+			fi, e := f.Stat()
+			if e == nil {
+				w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
+				http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f)
+				return
+			}
+		}
+		// if we reach here, some err has happened
+		log.Error("failed to serve image file ", ul, e)
+		apiV1Server500Error(w, r)
+		return
+	}
+
+	// see if a converted image exist, if not convert it and then send
+	if !fileExists(ul.jpgPath()) {
+		e = ul.convertUploadsToJpg()
+		if e != nil {
+			// serve a default no preview is available
+			http.ServeFile(w, r, config.UploadsDir.JpgDefault)
+			log.Error("error creating preview image", ul, e)
+			return
+		}
+	}
+	if fileExists(ul.jpgPath()) {
+		http.ServeFile(w, r, ul.jpgPath())
+		return
+	} else {
+		apiV1Client404Error(w, r, ss)
+	}
+
+}
+func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul, e := getRequestedUpload(w, r, ss)
+	if e != nil {
+		return
+	}
+
+	// see if a thumbnail is available already
+	if !fileExists(ul.thumbPath()) {
+		e = ul.convertUploadsToThumb()
+		if e != nil {
+			// serve a default no preview is available
+			http.ServeFile(w, r, config.UploadsDir.ThumbDefault)
+			log.Error("error creating preview image", ul, e)
+			return
+		}
+	}
+	if fileExists(ul.thumbPath()) {
+		http.ServeFile(w, r, ul.thumbPath())
+		return
+	} else {
+		apiV1Client404Error(w, r, ss)
+	}
+}
+
+func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) {
+	ul, e := getRequestedUpload(w, r, ss)
+	if e != nil {
+		return
+	}
+
+	//get file type
+	fileType, e := ul.GetFileType()
+	if e != nil {
+		apiV1Client403Error(w, r, ss)
+		return
+	}
+
+	// if its ready pdf, no need to convert
+	if fileType == "pdf" {
+		f, e := os.Open(ul.filePath())
+		if e == nil {
+			defer f.Close()
+			fi, e := f.Stat()
+			if e == nil {
+				w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName)
+				http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f)
+				return
+			}
+		}
+		// if we reach here, some err has happened
+		log.Error("failed to serve pdf file ", ul, e)
+		apiV1Server500Error(w, r)
+		return
+	}
+
+	// see if a converted pdf exist, if not convert it and then send
+	if !fileExists(ul.pdfPath()) {
+		e = ul.convertUploadsToPDF()
+		if e != nil {
+			// serve a default no preview is available
+			http.ServeFile(w, r, config.UploadsDir.PdfDefault)
+			log.Error("error creating preview image", ul, e)
+			return
+		}
+	}
+	if fileExists(ul.pdfPath()) {
+		http.ServeFile(w, r, ul.pdfPath())
+		return
+	} else {
+		apiV1Client404Error(w, r, ss)
+	}
 }

apiv1.go

 
 			{"POST", "lender-upload/", apiV1UploadsPost},
 			{"GET", "lender-upload/", apiV1UploadsGet},
+			{"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail},
+			{"GET", "upload-as-pdf/", apiV1UploadAPDF},
 
 			{"GET", "login", apiV1DumpRequest},
 		}
 			{"POST", "lender-upload/", apiV1UploadsPost},
 			{"GET", "lender-upload/", apiV1UploadsGet},
 
+			{"GET", "upload-analysis/", apiV1UploadAnalysis},
+			{"GET", "upload-as-image/", apiV1UploadAsImage},
+			{"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail},
+			{"GET", "upload-as-pdf/", apiV1UploadAPDF},
+			{"GET", "upload/", apiV1UploadsGet},
+
 			{"GET", "login", apiV1EmptyResponse},
 		}
 	}

config.go

 	"encoding/json"
 	log "github.com/sirupsen/logrus"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
 }
 
 type configuration struct {
-	Host    string
-	Port    string
-	DSN     string
-	TlsCert string
-	TlsKey  string
-	Static  []configStaticHtml
-	Debug   bool
-	Uploads string
+	Host       string
+	Port       string
+	DSN        string
+	TlsCert    string
+	TlsKey     string
+	Static     []configStaticHtml
+	Debug      bool
+	UploadsDir struct {
+		FileDir      string
+		FileDefault  string
+		JpgDir       string
+		JpgDefault   string
+		ThumbDir     string
+		ThumbDefault string
+		PdfDir       string
+		PdfDefault   string
+	}
+	TempDir string
 	Session struct { //TODO: figure what is this intended for
 		Guest bool
 		Year  int //how many years
 	}
 	e = json.Unmarshal(body, m)
 
+	// Check upload dir and defaults
+	if !config.checkUploadDir() {
+		log.Fatal("bad config file", configFile)
+		return
+	}
+
+	if config.Debug {
+		log.Println(config)
+	}
+
 	//TODO: check config before proceed further
 	return
 }
 
+func (m *configuration) checkUploadDir() (valid bool) {
+	valid = true
+	if !fileExists(m.UploadsDir.ThumbDefault) {
+		valid = false
+		log.Fatal("default thumbnail is missing ", m.UploadsDir.ThumbDefault)
+	}
+
+	if !fileExists(m.UploadsDir.FileDefault) {
+		valid = false
+		log.Fatal("default file for upload  is missing ", m.UploadsDir.FileDefault)
+	}
+	if !fileExists(m.UploadsDir.JpgDefault) {
+		valid = false
+		log.Fatal("default jpg for upload is missing ", m.UploadsDir.JpgDefault)
+	}
+	if !fileExists(m.UploadsDir.PdfDefault) {
+		valid = false
+		log.Fatal("default pdf for upload is missing ", &m.UploadsDir.PdfDefault)
+	}
+
+	//check dir
+	if !fileExists(m.UploadsDir.FileDir) {
+		valid = false
+		log.Fatal("UploadsDir.FileDir is missing ", &m.UploadsDir.PdfDefault)
+	}
+
+	if !fileExists(m.UploadsDir.JpgDir) {
+		valid = false
+		log.Fatal("UploadsDir.JpgDir is missing ", &m.UploadsDir.PdfDefault)
+	}
+	if !fileExists(m.UploadsDir.ThumbDir) {
+		valid = false
+		log.Fatal("UploadsDir.ThumbDir is missing ", &m.UploadsDir.PdfDefault)
+	}
+	if !fileExists(m.UploadsDir.PdfDir) {
+		valid = false
+		log.Fatal("UploadsDir.PdfDir is missing ", &m.UploadsDir.PdfDefault)
+	}
+
+	if !fileExists(m.TempDir) {
+		valid = false
+		log.Fatal("temp Dir is missing ", &m.UploadsDir.PdfDefault)
+	}
+
+	// convert to absolute path : fileDir
+	p, e := filepath.Abs(m.UploadsDir.FileDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad upload file dir", m.UploadsDir.FileDir, e)
+	}
+	m.UploadsDir.FileDir = p + string(os.PathSeparator) //change it to absolute dir
+
+	// convert to absolute path : jpgDir
+	p, e = filepath.Abs(m.UploadsDir.JpgDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad jpg file dir", m.UploadsDir.JpgDir, e)
+	}
+	m.UploadsDir.JpgDir = p + string(os.PathSeparator) //change it to absolute dir
+
+	// convert to absolute path : thumbDir
+	p, e = filepath.Abs(m.UploadsDir.ThumbDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad thumbnail dir", m.UploadsDir.ThumbDir, e)
+	}
+	m.UploadsDir.ThumbDir = p + string(os.PathSeparator) //change it to absolute dir
+
+	// convert to absolute path : PdfDir
+	p, e = filepath.Abs(m.UploadsDir.PdfDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad pdf file dir", m.UploadsDir.PdfDir, e)
+	}
+	m.UploadsDir.PdfDir = p + string(os.PathSeparator) //change it to absolute dir
+
+	// convert to absolute path : TmpDir
+	p, e = filepath.Abs(m.TempDir)
+	if e != nil {
+		valid = false
+		log.Fatal("bad TempDir dir", m.TempDir, e)
+	}
+	m.TempDir = p + string(os.PathSeparator) //change it to absolute dir
+	return
+}
+
 func (m *configuration) getAvatarPath() (ret string) {
 	for _, v := range m.Static {
 		if strings.ToLower(v.Dir) == "avatar" {

config.json

   "TlsCert": "/home/sp/go/src/fullchain.pem",
   "TlsKey": "/home/sp/go/src/privkey.pem",
   "Debug": true,
-  "Uploads": "./uploads/",
+  "UploadsDir": {
+    "FileDir": "./uploads/file",
+    "FileDefault": "./assets/no_preview.jpg",
+    "JpgDir": "./uploads/jpg",
+    "JpgDefault":  "./assets/no_preview.jpg",
+    "ThumbDir": "./uploads/thumb",
+    "ThumbDefault": "./assets/thumb_file_icon.webp",
+    "PdfDir": "./uploads/pdf",
+    "PdfDefault": "./assets/no_preview.pdf"
+  },
+  "TempDir": "./tmp/",
   "Static": [
     {
       "Dir": "./html/",

main_test.go

 import (
 	"biukop.com/sfm/loan"
 	"encoding/gob"
+	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"os"
 	"testing"
 )
 
 	AB   []byte
 }
 
+func TestMain(m *testing.M) {
+	err := config.readConfig() //wechat API config
+	if err != nil {
+		log.Println(err)
+		log.Fatalf("unable to read %s, program quit\n", configFile)
+		return
+	}
+
+	loan.SetDSN(config.DSN)
+	runTests := m.Run()
+	os.Exit(runTests)
+}
+
 func TestSession_SaveOtherType(t *testing.T) {
 	gob.Register(ABC{})
 

pay-in-decode.go

 	"biukop.com/sfm/loan"
 	"errors"
 	log "github.com/sirupsen/logrus"
-	"net/http"
-	"os"
 	"os/exec"
 	"strings"
 )
 )
 
 type AiDecodeIncome struct {
-	Input struct {
-		Uploads  loan.Uploads
-		FileName string //a local file on disk
-		InMime   string //may not be correct, just some suggestion only.
-	}
-	Mime   string //mime actually detected.
+	Input  loan.Uploads
+	ul     uploadsOnDisk // internal data
+	Mime   string        //mime actually detected.
 	PayIn  []loan.PayIn
 	Funder FunderType
 	AAA    PayInAAAData
 }
 
-func decodePayInMain(filename string, format string) (ai AiDecodeIncome, e error) {
-	ai.Input.FileName = filename
-	ai.Input.InMime = format
-	ai.PayIn = make([]loan.PayIn, 0, 10)
-	ai.Mime, e = GetFileContentType(filename)
-	if e != nil {
-		return
-	}
+func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) {
+	m.Input = ulMeta
+	m.ul.Upload = ulMeta
 
-	switch ai.Mime {
-	case "application/pdf":
-		ai.decodePayInPdf(filename, format)
+	m.PayIn = make([]loan.PayIn, 0, 10)
+
+	switch m.getFileType() {
+	case "pdf":
+		m.decodePdf()
+		break
+	case "excel", "opensheet":
+		m.decodeXls()
+		break
+	default:
+		e = errors.New("unknown format")
+		m.Funder = "" // mark unknown decoding
 	}
 
-	return ai, e
+	return
 }
 
-// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime.
-func GetFileContentType(filename string) (contentType string, e error) {
-	contentType = ""
-	input, e := os.OpenFile(filename, os.O_RDONLY, 0755)
-	// Only the first 512 bytes are used to sniff the content type.
-	buffer := make([]byte, 512)
-
-	_, e = input.Read(buffer)
+func (m *AiDecodeIncome) getFileType() (ret string) {
+	strMime, e := GetFileContentType(m.ul.filePath())
 	if e != nil {
 		return
 	}
+	m.Mime = strMime
 
-	// Use the net/http package's handy DectectContentType function. Always returns a valid
-	// content-type by returning "application/octet-stream" if no others seemed to match.
-	contentType = http.DetectContentType(buffer)
+	ret, e = m.ul.GetFileType()
+	if e != nil {
+		ret = ""
+	}
 	return
 }
 
-func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []loan.PayIn, e error) {
-	cmd := exec.Command("pdftotext", "-layout", filename, "-")
-	//log.Println(cmd.String())
+func (m *AiDecodeIncome) decodePdf() (e error) {
+	cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-")
 	out, e := cmd.Output()
 	if e != nil {
 		log.Fatal(e)
 	raw := string(out)
 	switch m.detectFunder(raw) {
 	case Funder_AAA:
+		m.Funder = Funder_AAA
 		e = m.AAA.decodeAAAPdf(raw)
 		log.Println("AAA final result", m.AAA)
 		break
 	return
 }
 
+func (m *AiDecodeIncome) decodeXls() (e error) {
+	return
+}
+
 func (m *AiDecodeIncome) detectFunder(raw string) FunderType {
 	if m.isAAA(raw) {
 		return Funder_AAA

pay-in-decode_test.go

 package main
 
 import (
+	"biukop.com/sfm/loan"
 	log "github.com/sirupsen/logrus"
 	"testing"
 )
 
 func TestDecodePayInMain(t *testing.T) {
-	fileName := "./uploads/30.uploads"
-	fileName = "/home/sp/go/src/SFM_Loan_RestApi/uploads/30.uploads"
-	data, _ := decodePayInMain(fileName, "application/vnd.ms-excel")
-	log.Println(data)
+	ai := AiDecodeIncome{}
+	ul := loan.Uploads{}
+	ul.Read(30)
+	_ = ai.decodeUploadToPayIn(ul)
+	log.Println(ai)
 }

payIn-AAA.go

 */
 
 type PayInAAARow struct {
-	LoanNUmber string
+	LoanNumber string
 	Settlement time.Time
 	LoanAmount float64
 	Balance    float64
 	m.Periods = make([]PayInAAAPeriod, 0, 10)
 	lines := strings.Split(raw, "\n")
 
-	var tableHeader []string
-	var tableHeaderLine int
 	currentPeriod := -1
 	state := "start"
-	for idx, l := range lines { // DFA, wow, finally it's used. after years of learning
+	for _, l := range lines { // DFA, wow, finally it's used. after years of learning
 		switch state {
 		case "start":
 			state = m.processStart(l)
 			if state == "LookingForPeriod" {
-				tableHeaderLine = idx
-				tableHeader = strings.Split(l, " ")
-				log.Println("Find table header", tableHeader, l, tableHeaderLine)
+				// determine column index, if their column is changing
 			}
 			break
 		case "LookingForPeriod":
 	}
 
 	if len(el) >= 5 {
-		row.LoanNUmber = el[0]
+		row.LoanNumber = el[0]
 		row.Settlement, _ = time.Parse("02-Jan-06", el[1])
 		row.LoanAmount = m.currencyToFloat64(el[2])
 		row.Balance = m.currencyToFloat64(el[3])