diff --git a/.gitignore b/.gitignore index 5c5050a..7b322d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /apiv1 /uploads/ +/tmp/ diff --git a/UploadsOnDisk.go b/UploadsOnDisk.go new file mode 100644 index 0000000..8a11942 --- /dev/null +++ b/UploadsOnDisk.go @@ -0,0 +1,275 @@ +package main + +import ( + "biukop.com/sfm/loan" + "errors" + log "github.com/sirupsen/logrus" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +type uploadsOnDisk struct { + Upload loan.Uploads +} + +func (m *uploadsOnDisk) convertUploadsToPDF() (e error) { + return convertUploadsToPDF(m.Upload) +} +func convertUploadsToPDF(ul loan.Uploads) (e error) { + m := uploadsOnDisk{} + m.Upload = ul + + if strings.Contains(strings.ToLower(ul.Format), "excel") || + strings.Contains(strings.ToLower(ul.Format), "spreadsheet") { + e = m.convertExcelToPDF() + return // excel is converted + } + + if strings.Contains(strings.ToLower(ul.Format), "/pdf") { + return // no need to convert + } + + e = errors.New("don't know how to convert file to PDF") + log.Error("don't know how to convert file to PDF", ul) + return +} + +func (m *uploadsOnDisk) convertUploadsToJpg() (e error) { + return convertUploadsToJpg(m.Upload) +} + +func convertUploadsToJpg(ul loan.Uploads) (e error) { + m := uploadsOnDisk{} + m.Upload = ul + + if strings.Contains(strings.ToLower(ul.Format), "excel") || + strings.Contains(strings.ToLower(ul.Format), "spreadsheet") { + e = m.convertExcelToJpg() + return // excel is converted + } + if strings.Contains(strings.ToLower(ul.Format), "/pdf") { + e = m.convertPDFToJpg() + return // excel is converted + } + + e = errors.New("don't know how to convert file to image") + log.Error("don't know how to convert file to image", ul) + return +} + +func (m *uploadsOnDisk) convertUploadsToThumb() (e error) { + return convertUploadsToThumb(m.Upload) +} +func convertUploadsToThumb(ul loan.Uploads) (e error) { + m := uploadsOnDisk{} + m.Upload = ul + if !fileExists(m.jpgPath()) { + e = m.convertUploadsToJpg() + if e != nil { + return + } + } + + e = ConvertImageToThumbnail(m.jpgPath(), m.thumbPath(), 256) + if e != nil { + log.Error("cannot create thumbnail for uploads", m.Upload, e) + return + } + return +} + +func (m *uploadsOnDisk) filePath() string { + return config.UploadsDir.FileDir + strconv.Itoa(int(m.Upload.Id)) + ".uploads" +} + +func (m *uploadsOnDisk) jpgPath() string { + return config.UploadsDir.JpgDir + strconv.Itoa(int(m.Upload.Id)) + ".jpg" +} + +func (m *uploadsOnDisk) thumbPath() string { + return config.UploadsDir.ThumbDir + strconv.Itoa(int(m.Upload.Id)) + ".webp" +} + +func (m *uploadsOnDisk) pdfPath() string { + return config.UploadsDir.PdfDir + strconv.Itoa(int(m.Upload.Id)) + ".pdf" +} +func (m *uploadsOnDisk) convertExcelToPDF() (e error) { + if fileExists(m.pdfPath()) { + log.Info("Skip conversion excel to PDF , already exists", m) + return + } + return m.convertExcelTo("pdf") +} + +func (m *uploadsOnDisk) convertExcelToJpg() (e error) { + if fileExists(m.jpgPath()) { + log.Info("Skip conversion excel to Jpg , already exists", m) + return + } + return m.convertExcelTo("jpg") +} + +func (m *uploadsOnDisk) convertExcelTo(format string) (e error) { + if format != "pdf" && format != "jpg" { + e = errors.New("unsupported format") + return + } + + dst := m.jpgPath() + if format == "pdf" { + dst = m.pdfPath() + } + + dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-xls-to-"+format+"-") + if e != nil { + log.Error("cannot create tmp dir for converting image", m.Upload, e) + return + } + defer os.RemoveAll(dir) + + cmd := exec.Command("libreoffice", "--convert-to", format, "--outdir", dir, m.filePath()) + strCmd := cmd.String() + log.Debug("command is ", strCmd) + out, e := cmd.Output() + if e != nil { + log.Error("cannot converting Excel to "+format+":", m.Upload, e) + return + } else { // success + log.Info("convert to "+format, m.Upload, " output: ", string(out)) + } + + _, name := filepath.Split(m.filePath()) + + src := dir + string(os.PathSeparator) + fileNameWithoutExtTrimSuffix(name) + "." + format + e = os.Rename(src, dst) // there should be only one jpg + + return +} + +// first page to thumbnail +// all page to single jpg +func (m *uploadsOnDisk) convertPDFToJpg() (e error) { + + if fileExists(m.jpgPath()) { + // no need to reconvert it again + log.Info("PDF to JPG skipped it already exists ", m) + return + } + + dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-pdf-to-jpg-") + if e != nil { + log.Error("cannot create tmp dir for converting image", m.Upload, e) + return + } + defer os.RemoveAll(dir) + + // convert -density 3000 abc.pdf path/tmp/result.png + // could be path/tmp/result-0, result-1, result-2, ... png + target := dir + string(os.PathSeparator) + "result.jpg" //.jpg suffix is important + cmd := exec.Command("convert", "-density", "300", m.filePath(), target) + strCmd := cmd.String() + log.Debug("command is ", strCmd) + out, e := cmd.Output() + if e != nil { + log.Error("cannot create png file for PDF", m.Upload, e) + return + } else { + log.Info("convert ", m.Upload, " output: ", string(out)) + } + + // montage -mode concatenate -tile 1x 30*png 30.jpg + if fileExists(target) { // single file, + e = os.Rename(target, m.jpgPath()) // there should be only one jpg + _ = ConvertImageToThumbnail(target, m.thumbPath(), 256) + } else { // multi-page, we have -0 -1 -2 -3 -4 files + firstPage := dir + string(os.PathSeparator) + "result-0.jpg" + _ = ConvertImageToThumbnail(firstPage, m.thumbPath(), 256) + + batch := dir + string(os.PathSeparator) + "result*jpg" // result* is important + target = dir + string(os.PathSeparator) + "final.jpg" // .jpg suffix is important + cmd = exec.Command("montage", "-mode", "concatenate", "-tile", "1x", batch, target) + + strCmd = cmd.String() + log.Debug("command is ", strCmd) + out, e = cmd.Output() + if e != nil { + return + } else { + log.Info("montage ", m, " output: ", string(out)) + } + e = os.Rename(target, m.jpgPath()) // give combined file to target + } + return +} + +func (m *uploadsOnDisk) GetFileType() (ret string, e error) { + strMime, e := GetFileContentType(m.filePath()) + if e != nil { + return + } + + if strings.ToLower(strMime) == "application/pdf" { + return "pdf", nil + } + + if strings.ToLower(strMime) == "application/vnd.ms-excel" { + return "excel", nil + } + + if strings.ToLower(strMime) == "application/zip" && + strings.ToLower(m.Upload.Format) == + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" { + return "opensheet", nil + } + + return "", nil +} + +// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime. +func GetFileContentType(filename string) (contentType string, e error) { + contentType = "" + input, e := os.OpenFile(filename, os.O_RDONLY, 0755) + // Only the first 512 bytes are used to sniff the content type. + buffer := make([]byte, 512) + + _, e = input.Read(buffer) + if e != nil { + return + } + + // Use the net/http package's handy DectectContentType function. Always returns a valid + // content-type by returning "application/octet-stream" if no others seemed to match. + contentType = http.DetectContentType(buffer) + return +} + +func ConvertImageToThumbnail(srcPath string, dstPath string, size int) (e error) { + if fileExists(dstPath) { + log.Info("skip converting thumbnail it exists", dstPath) + return + } + if size <= 0 { + size = 256 + } + // convert -thumbnail 200 abc.png thumb.abc.png + cmd := exec.Command("convert", "-thumbnail", strconv.Itoa(size), srcPath, dstPath) + strCmd := cmd.String() + log.Debug("create thumbnail: ", strCmd) + + out, e := cmd.Output() + if e != nil { + return + } else { // success + log.Info("success output: \n: ", string(out)) + } + return +} + +func fileNameWithoutExtTrimSuffix(fileName string) string { + return strings.TrimSuffix(fileName, filepath.Ext(fileName)) +} diff --git a/UploadsOnDisk_test.go b/UploadsOnDisk_test.go new file mode 100644 index 0000000..c4b28c6 --- /dev/null +++ b/UploadsOnDisk_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "biukop.com/sfm/loan" + log "github.com/sirupsen/logrus" + "path/filepath" + "testing" +) + +func TestGetConvertUploadsToImage(t *testing.T) { + for id := 30; id <= 44; id++ { + ul := loan.Uploads{} + e := ul.Read(int64(id)) + if e != nil { + log.Error(e) + return + } + + e = convertUploadsToPDF(ul) + if e != nil { + log.Error(e) + return + } + + e = convertUploadsToJpg(ul) + if e != nil { + log.Error(e) + return + } + + e = convertUploadsToThumb(ul) + if e != nil { + log.Error(e) + return + } + } + +} + +func TestConvertImageToThumbnail(t *testing.T) { + _, name := filepath.Split("/home/sp/uploads/abc.uploads") + test := fileNameWithoutExtTrimSuffix(name) + ".jpg" + log.Println(test) +} diff --git a/apiV1UploadAnalysis.go b/apiV1UploadAnalysis.go new file mode 100644 index 0000000..1e8e9d7 --- /dev/null +++ b/apiV1UploadAnalysis.go @@ -0,0 +1,35 @@ +package main + +import ( + "biukop.com/sfm/loan" + log "github.com/sirupsen/logrus" + "net/http" + "strconv" +) + +func apiV1UploadAnalysis(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + strId := r.URL.Path[len(apiV1Prefix+"upload-analysis/"):] //remove prefix + Id, e := strconv.Atoi(strId) + if e != nil { + log.Error("Invalid uploads Id cannot convert to integer", Id, e) + apiV1Client403Error(w, r, ss) + return + } + + ul := loan.Uploads{} + e = ul.Read(int64(Id)) + if e != nil { + log.Error("Upload not found or read error from db", Id, e) + apiV1Client404Error(w, r, ss) + return + } + + ai := AiDecodeIncome{} + e = ai.decodeUploadToPayIn(ul) + if e != nil { + log.Error("Invalid uploads Id cannot conver to integer", Id, e) + apiV1Server500Error(w, r) + return + } + apiV1SendJson(ai, w, r, ss) +} diff --git a/apiV1Uploads.go b/apiV1Uploads.go index a8224f2..7f0073b 100644 --- a/apiV1Uploads.go +++ b/apiV1Uploads.go @@ -10,13 +10,13 @@ import ( "io/ioutil" "net/http" "os" - "path/filepath" "strconv" + "strings" "time" ) func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { - id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix + id := r.URL.Path[len(apiV1Prefix+"upload/"):] //remove prefix intId, e := strconv.Atoi(id) if e != nil { log.Println("invalid id for upload get", id, e) @@ -24,8 +24,8 @@ func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { return } - ul := loan.Uploads{} - e = ul.Read(int64(intId)) + ul := uploadsOnDisk{} + e = ul.Upload.Read(int64(intId)) if e != nil { log.Println("no file uploaded", intId, e) apiV1Client404Error(w, r, ss) // bad request @@ -33,7 +33,7 @@ func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { } //check local file first - path := config.Uploads + strconv.FormatInt(ul.Id, 10) + ".uploads" + path := ul.filePath() if fileExists(path) { http.ServeFile(w, r, path) return @@ -46,9 +46,9 @@ func apiV1UploadsGet(w http.ResponseWriter, r *http.Request, ss *loan.Session) { func apiV1UploadsPost(w http.ResponseWriter, r *http.Request, ss *loan.Session) { id := r.URL.Path[len(apiV1Prefix+"lender-upload/"):] //remove prefix - filename, e := saveUploadToFile(r) + filepath, e := saveUploadToFile(r) if e != nil { - log.Println("no file uploaded", filename, e) + log.Println("no file uploaded", filepath, e) apiV1Client404Error(w, r, ss) // bad request return } @@ -60,9 +60,9 @@ func apiV1UploadsPost(w http.ResponseWriter, r *http.Request, ss *loan.Session) apiV1Client404Error(w, r, ss) // bad request return } - updateUploads(int64(intId), filename, w, r, ss) + updateUploads(int64(intId), filepath, w, r, ss) } else { - createUploads(filename, w, r, ss) + createUploads(filepath, w, r, ss) } } @@ -76,18 +76,19 @@ func sha256File(input io.Reader) string { return fmt.Sprintf("%x", sum) } -func updateUploads(id int64, fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { - ul := loan.Uploads{} - e := ul.Read(int64(id)) +func updateUploads(id int64, filePath string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul := uploadsOnDisk{} + e := ul.Upload.Read(int64(id)) if e != nil { log.Println("bad upload id is given ", id, e) apiV1Client404Error(w, r, ss) // bad request return } - ul1, _, e := saveUploadsToDB(id, fileName, r, ss) + ul1, isDuplicate, e := saveUploadsMetaToDB(id, filePath, r, ss) + ul.Upload.IsDuplicate = isDuplicate if e != nil { - os.Remove(config.Uploads + ul.FileName) + os.Remove(ul.filePath()) ul1.Delete() log.Println("cannot save file info to db ", e) apiV1Server500Error(w, r) // bad request @@ -97,28 +98,35 @@ func updateUploads(id int64, fileName string, w http.ResponseWriter, r *http.Req apiV1SendJson(ul1, w, r, ss) } -func createUploads(fileName string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { - ul, _, e := saveUploadsToDB(0, fileName, r, ss) +func createUploads(filePath string, w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul := uploadsOnDisk{} + ulMeta, isDuplicate, e := saveUploadsMetaToDB(0, filePath, r, ss) + ul.Upload = ulMeta + ul.Upload.IsDuplicate = isDuplicate if e != nil { log.Println("cannot save file info to db ", e) - e = ul.Delete() // delete the newly created, if failed, db will clean it + e = ulMeta.Delete() // delete the newly created, if failed, db will clean it if e != nil { log.Error("failed to remove unused uploads", ul) } - e = os.Remove(config.Uploads + fileName) + e = os.Remove(ul.filePath()) if e != nil { - log.Error("failed to remove unused temp file", fileName) + log.Error("failed to remove unused temp file", filePath) } apiV1Server500Error(w, r) // bad request return } - apiV1SendJson(ul, w, r, ss) + + ai := AiDecodeIncome{} + ai.decodeUploadToPayIn(ul.Upload) + + apiV1SendJson(ai, w, r, ss) } -func saveUploadsToDB(id int64, fileName string, - r *http.Request, ss *loan.Session) (ul loan.Uploads, duplicate bool, e error) { +func saveUploadsMetaToDB(id int64, filePath string, + r *http.Request, ss *loan.Session) (ulMeta loan.Uploads, duplicate bool, e error) { duplicate = false e = r.ParseMultipartForm(10 << 20) // we should have ready parsed this, just in case if e != nil { @@ -128,27 +136,29 @@ func saveUploadsToDB(id int64, fileName string, file.Seek(0, 0) //seek to beginning checksum := sha256File(file) - ul.Id = id - ul.Ts = time.Now() - ul.FileName = header.Filename + ulMeta.Id = id + ulMeta.Ts = time.Now() + ulMeta.FileName = header.Filename file.Seek(0, 0) //seek to beginning - ul.Format = header.Header.Get("Content-type") - ul.Size = header.Size // necessary to prevent duplicate - ul.LastModified = 0 - ul.Sha256 = checksum // necessary to prevent duplicate - ul.By = ss.User - e = ul.Write() // this Id will have the real Id if there is duplicates + ulMeta.Format = header.Header.Get("Content-type") + ulMeta.Size = header.Size // necessary to prevent duplicate + ulMeta.LastModified = 0 + ulMeta.Sha256 = checksum // necessary to prevent duplicate + ulMeta.By = ss.User + e = ulMeta.Write() // this Id will have the real Id if there is duplicates if e != nil { - log.Error("Fail to update db ", ul, e) + log.Error("Fail to update db ", ulMeta, e) } else { - if id > 0 && ul.Id != id { + if (id > 0 && ulMeta.Id != id) || (id == 0 && ulMeta.IsDuplicate) { duplicate = true } - target := fmt.Sprintf("%d.uploads", ul.Id) - e = os.Rename(config.Uploads+fileName, config.Uploads+target) + ul := uploadsOnDisk{} + ul.Upload = ulMeta + e = os.Rename(filePath, ul.filePath()) if e != nil { - ul.FileName = fileName // some how failed to rename + os.Remove(filePath) + log.Error("fail to move file from ", filePath, "to", ul.filePath()) } } return @@ -165,7 +175,7 @@ func saveUploadToFile(r *http.Request) (filename string, e error) { return } - out, pathError := ioutil.TempFile(config.Uploads, "can-del-upload-*.tmp") + out, pathError := ioutil.TempFile(config.UploadsDir.FileDir, "can-del-upload-*.tmp") if pathError != nil { log.Println("Error Creating a file for writing", pathError) return @@ -182,5 +192,141 @@ func saveUploadToFile(r *http.Request) (filename string, e error) { if size != header.Size { e = errors.New("written file with incorrect size") } - return filepath.Base(out.Name()), e + return out.Name(), e +} + +func getRequestedUpload(w http.ResponseWriter, r *http.Request, ss *loan.Session) (ret uploadsOnDisk, e error) { + strId := r.URL.Path[len(apiV1Prefix+"upload-as-image/"):] //remove prefix + Id, e := strconv.Atoi(strId) + if e != nil { + log.Error("Invalid uploads Id cannot convert to integer", Id, e) + apiV1Client403Error(w, r, ss) + return + } + + ul := loan.Uploads{} + e = ul.Read(int64(Id)) + if e != nil { + log.Error("Upload not found or read error from db", Id, e) + apiV1Client404Error(w, r, ss) + return + } + + ret.Upload = ul + return +} + +func apiV1UploadAsImage(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul, e := getRequestedUpload(w, r, ss) + if e != nil { + return + } + + // if this is image itself, serve it directly + if strings.Contains(strings.ToLower(ul.Upload.Format), "image") { + f, e := os.Open(ul.filePath()) + if e == nil { + defer f.Close() + fi, e := f.Stat() + if e == nil { + w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) + http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f) + return + } + } + // if we reach here, some err has happened + log.Error("failed to serve image file ", ul, e) + apiV1Server500Error(w, r) + return + } + + // see if a converted image exist, if not convert it and then send + if !fileExists(ul.jpgPath()) { + e = ul.convertUploadsToJpg() + if e != nil { + // serve a default no preview is available + http.ServeFile(w, r, config.UploadsDir.JpgDefault) + log.Error("error creating preview image", ul, e) + return + } + } + if fileExists(ul.jpgPath()) { + http.ServeFile(w, r, ul.jpgPath()) + return + } else { + apiV1Client404Error(w, r, ss) + } + +} +func apiV1UploadAsThumbnail(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul, e := getRequestedUpload(w, r, ss) + if e != nil { + return + } + + // see if a thumbnail is available already + if !fileExists(ul.thumbPath()) { + e = ul.convertUploadsToThumb() + if e != nil { + // serve a default no preview is available + http.ServeFile(w, r, config.UploadsDir.ThumbDefault) + log.Error("error creating preview image", ul, e) + return + } + } + if fileExists(ul.thumbPath()) { + http.ServeFile(w, r, ul.thumbPath()) + return + } else { + apiV1Client404Error(w, r, ss) + } +} + +func apiV1UploadAPDF(w http.ResponseWriter, r *http.Request, ss *loan.Session) { + ul, e := getRequestedUpload(w, r, ss) + if e != nil { + return + } + + //get file type + fileType, e := ul.GetFileType() + if e != nil { + apiV1Client403Error(w, r, ss) + return + } + + // if its ready pdf, no need to convert + if fileType == "pdf" { + f, e := os.Open(ul.filePath()) + if e == nil { + defer f.Close() + fi, e := f.Stat() + if e == nil { + w.Header().Set("Content-Disposition", "attachment; filename="+ul.Upload.FileName) + http.ServeContent(w, r, ul.filePath(), fi.ModTime(), f) + return + } + } + // if we reach here, some err has happened + log.Error("failed to serve pdf file ", ul, e) + apiV1Server500Error(w, r) + return + } + + // see if a converted pdf exist, if not convert it and then send + if !fileExists(ul.pdfPath()) { + e = ul.convertUploadsToPDF() + if e != nil { + // serve a default no preview is available + http.ServeFile(w, r, config.UploadsDir.PdfDefault) + log.Error("error creating preview image", ul, e) + return + } + } + if fileExists(ul.pdfPath()) { + http.ServeFile(w, r, ul.pdfPath()) + return + } else { + apiV1Client404Error(w, r, ss) + } } diff --git a/apiv1.go b/apiv1.go index 19627cd..69c0752 100644 --- a/apiv1.go +++ b/apiv1.go @@ -73,6 +73,8 @@ func setupApiV1Handler() []apiV1HandlerMap { {"POST", "lender-upload/", apiV1UploadsPost}, {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail}, + {"GET", "upload-as-pdf/", apiV1UploadAPDF}, {"GET", "login", apiV1DumpRequest}, } @@ -125,6 +127,12 @@ func setupApiV1Handler() []apiV1HandlerMap { {"POST", "lender-upload/", apiV1UploadsPost}, {"GET", "lender-upload/", apiV1UploadsGet}, + {"GET", "upload-analysis/", apiV1UploadAnalysis}, + {"GET", "upload-as-image/", apiV1UploadAsImage}, + {"GET", "upload-as-thumbnail/", apiV1UploadAsThumbnail}, + {"GET", "upload-as-pdf/", apiV1UploadAPDF}, + {"GET", "upload/", apiV1UploadsGet}, + {"GET", "login", apiV1EmptyResponse}, } } diff --git a/assets/no_preview.jpg b/assets/no_preview.jpg new file mode 100644 index 0000000..23ea855 Binary files /dev/null and b/assets/no_preview.jpg differ diff --git a/assets/no_preview.pdf b/assets/no_preview.pdf new file mode 100644 index 0000000..f3634c6 Binary files /dev/null and b/assets/no_preview.pdf differ diff --git a/assets/thumb_file_icon.webp b/assets/thumb_file_icon.webp new file mode 100644 index 0000000..550d62f Binary files /dev/null and b/assets/thumb_file_icon.webp differ diff --git a/config.go b/config.go index 802c86a..c83aa50 100644 --- a/config.go +++ b/config.go @@ -4,6 +4,8 @@ import ( "encoding/json" log "github.com/sirupsen/logrus" "io/ioutil" + "os" + "path/filepath" "strings" ) @@ -14,14 +16,24 @@ type configStaticHtml struct { } type configuration struct { - Host string - Port string - DSN string - TlsCert string - TlsKey string - Static []configStaticHtml - Debug bool - Uploads string + Host string + Port string + DSN string + TlsCert string + TlsKey string + Static []configStaticHtml + Debug bool + UploadsDir struct { + FileDir string + FileDefault string + JpgDir string + JpgDefault string + ThumbDir string + ThumbDefault string + PdfDir string + PdfDefault string + } + TempDir string Session struct { //TODO: figure what is this intended for Guest bool Year int //how many years @@ -42,10 +54,106 @@ func (m *configuration) readConfig() (e error) { } e = json.Unmarshal(body, m) + // Check upload dir and defaults + if !config.checkUploadDir() { + log.Fatal("bad config file", configFile) + return + } + + if config.Debug { + log.Println(config) + } + //TODO: check config before proceed further return } +func (m *configuration) checkUploadDir() (valid bool) { + valid = true + if !fileExists(m.UploadsDir.ThumbDefault) { + valid = false + log.Fatal("default thumbnail is missing ", m.UploadsDir.ThumbDefault) + } + + if !fileExists(m.UploadsDir.FileDefault) { + valid = false + log.Fatal("default file for upload is missing ", m.UploadsDir.FileDefault) + } + if !fileExists(m.UploadsDir.JpgDefault) { + valid = false + log.Fatal("default jpg for upload is missing ", m.UploadsDir.JpgDefault) + } + if !fileExists(m.UploadsDir.PdfDefault) { + valid = false + log.Fatal("default pdf for upload is missing ", &m.UploadsDir.PdfDefault) + } + + //check dir + if !fileExists(m.UploadsDir.FileDir) { + valid = false + log.Fatal("UploadsDir.FileDir is missing ", &m.UploadsDir.PdfDefault) + } + + if !fileExists(m.UploadsDir.JpgDir) { + valid = false + log.Fatal("UploadsDir.JpgDir is missing ", &m.UploadsDir.PdfDefault) + } + if !fileExists(m.UploadsDir.ThumbDir) { + valid = false + log.Fatal("UploadsDir.ThumbDir is missing ", &m.UploadsDir.PdfDefault) + } + if !fileExists(m.UploadsDir.PdfDir) { + valid = false + log.Fatal("UploadsDir.PdfDir is missing ", &m.UploadsDir.PdfDefault) + } + + if !fileExists(m.TempDir) { + valid = false + log.Fatal("temp Dir is missing ", &m.UploadsDir.PdfDefault) + } + + // convert to absolute path : fileDir + p, e := filepath.Abs(m.UploadsDir.FileDir) + if e != nil { + valid = false + log.Fatal("bad upload file dir", m.UploadsDir.FileDir, e) + } + m.UploadsDir.FileDir = p + string(os.PathSeparator) //change it to absolute dir + + // convert to absolute path : jpgDir + p, e = filepath.Abs(m.UploadsDir.JpgDir) + if e != nil { + valid = false + log.Fatal("bad jpg file dir", m.UploadsDir.JpgDir, e) + } + m.UploadsDir.JpgDir = p + string(os.PathSeparator) //change it to absolute dir + + // convert to absolute path : thumbDir + p, e = filepath.Abs(m.UploadsDir.ThumbDir) + if e != nil { + valid = false + log.Fatal("bad thumbnail dir", m.UploadsDir.ThumbDir, e) + } + m.UploadsDir.ThumbDir = p + string(os.PathSeparator) //change it to absolute dir + + // convert to absolute path : PdfDir + p, e = filepath.Abs(m.UploadsDir.PdfDir) + if e != nil { + valid = false + log.Fatal("bad pdf file dir", m.UploadsDir.PdfDir, e) + } + m.UploadsDir.PdfDir = p + string(os.PathSeparator) //change it to absolute dir + + // convert to absolute path : TmpDir + p, e = filepath.Abs(m.TempDir) + if e != nil { + valid = false + log.Fatal("bad TempDir dir", m.TempDir, e) + } + m.TempDir = p + string(os.PathSeparator) //change it to absolute dir + return +} + func (m *configuration) getAvatarPath() (ret string) { for _, v := range m.Static { if strings.ToLower(v.Dir) == "avatar" { diff --git a/config.json b/config.json index 80797ba..040a093 100644 --- a/config.json +++ b/config.json @@ -5,7 +5,17 @@ "TlsCert": "/home/sp/go/src/fullchain.pem", "TlsKey": "/home/sp/go/src/privkey.pem", "Debug": true, - "Uploads": "./uploads/", + "UploadsDir": { + "FileDir": "./uploads/file", + "FileDefault": "./assets/no_preview.jpg", + "JpgDir": "./uploads/jpg", + "JpgDefault": "./assets/no_preview.jpg", + "ThumbDir": "./uploads/thumb", + "ThumbDefault": "./assets/thumb_file_icon.webp", + "PdfDir": "./uploads/pdf", + "PdfDefault": "./assets/no_preview.pdf" + }, + "TempDir": "./tmp/", "Static": [ { "Dir": "./html/", diff --git a/main_test.go b/main_test.go index fecb902..c8bfecc 100644 --- a/main_test.go +++ b/main_test.go @@ -3,7 +3,9 @@ package main import ( "biukop.com/sfm/loan" "encoding/gob" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "os" "testing" ) @@ -13,6 +15,19 @@ type ABC struct { AB []byte } +func TestMain(m *testing.M) { + err := config.readConfig() //wechat API config + if err != nil { + log.Println(err) + log.Fatalf("unable to read %s, program quit\n", configFile) + return + } + + loan.SetDSN(config.DSN) + runTests := m.Run() + os.Exit(runTests) +} + func TestSession_SaveOtherType(t *testing.T) { gob.Register(ABC{}) diff --git a/pay-in-decode.go b/pay-in-decode.go index bca3ea1..5ba3ffc 100644 --- a/pay-in-decode.go +++ b/pay-in-decode.go @@ -4,8 +4,6 @@ import ( "biukop.com/sfm/loan" "errors" log "github.com/sirupsen/logrus" - "net/http" - "os" "os/exec" "strings" ) @@ -20,55 +18,51 @@ const ( ) type AiDecodeIncome struct { - Input struct { - Uploads loan.Uploads - FileName string //a local file on disk - InMime string //may not be correct, just some suggestion only. - } - Mime string //mime actually detected. + Input loan.Uploads + ul uploadsOnDisk // internal data + Mime string //mime actually detected. PayIn []loan.PayIn Funder FunderType AAA PayInAAAData } -func decodePayInMain(filename string, format string) (ai AiDecodeIncome, e error) { - ai.Input.FileName = filename - ai.Input.InMime = format - ai.PayIn = make([]loan.PayIn, 0, 10) - ai.Mime, e = GetFileContentType(filename) - if e != nil { - return - } +func (m *AiDecodeIncome) decodeUploadToPayIn(ulMeta loan.Uploads) (e error) { + m.Input = ulMeta + m.ul.Upload = ulMeta - switch ai.Mime { - case "application/pdf": - ai.decodePayInPdf(filename, format) + m.PayIn = make([]loan.PayIn, 0, 10) + + switch m.getFileType() { + case "pdf": + m.decodePdf() + break + case "excel", "opensheet": + m.decodeXls() + break + default: + e = errors.New("unknown format") + m.Funder = "" // mark unknown decoding } - return ai, e + return } -// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime. -func GetFileContentType(filename string) (contentType string, e error) { - contentType = "" - input, e := os.OpenFile(filename, os.O_RDONLY, 0755) - // Only the first 512 bytes are used to sniff the content type. - buffer := make([]byte, 512) - - _, e = input.Read(buffer) +func (m *AiDecodeIncome) getFileType() (ret string) { + strMime, e := GetFileContentType(m.ul.filePath()) if e != nil { return } + m.Mime = strMime - // Use the net/http package's handy DectectContentType function. Always returns a valid - // content-type by returning "application/octet-stream" if no others seemed to match. - contentType = http.DetectContentType(buffer) + ret, e = m.ul.GetFileType() + if e != nil { + ret = "" + } return } -func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []loan.PayIn, e error) { - cmd := exec.Command("pdftotext", "-layout", filename, "-") - //log.Println(cmd.String()) +func (m *AiDecodeIncome) decodePdf() (e error) { + cmd := exec.Command("pdftotext", "-layout", m.ul.filePath(), "-") out, e := cmd.Output() if e != nil { log.Fatal(e) @@ -77,6 +71,7 @@ func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []l raw := string(out) switch m.detectFunder(raw) { case Funder_AAA: + m.Funder = Funder_AAA e = m.AAA.decodeAAAPdf(raw) log.Println("AAA final result", m.AAA) break @@ -87,6 +82,10 @@ func (m *AiDecodeIncome) decodePayInPdf(filename string, format string) (ret []l return } +func (m *AiDecodeIncome) decodeXls() (e error) { + return +} + func (m *AiDecodeIncome) detectFunder(raw string) FunderType { if m.isAAA(raw) { return Funder_AAA diff --git a/pay-in-decode_test.go b/pay-in-decode_test.go index 6740d6b..81d5377 100644 --- a/pay-in-decode_test.go +++ b/pay-in-decode_test.go @@ -1,13 +1,15 @@ package main import ( + "biukop.com/sfm/loan" log "github.com/sirupsen/logrus" "testing" ) func TestDecodePayInMain(t *testing.T) { - fileName := "./uploads/30.uploads" - fileName = "/home/sp/go/src/SFM_Loan_RestApi/uploads/30.uploads" - data, _ := decodePayInMain(fileName, "application/vnd.ms-excel") - log.Println(data) + ai := AiDecodeIncome{} + ul := loan.Uploads{} + ul.Read(30) + _ = ai.decodeUploadToPayIn(ul) + log.Println(ai) } diff --git a/payIn-AAA.go b/payIn-AAA.go index 76d02d4..220fdcb 100644 --- a/payIn-AAA.go +++ b/payIn-AAA.go @@ -26,7 +26,7 @@ Super Finance Markets Pty Ltd */ type PayInAAARow struct { - LoanNUmber string + LoanNumber string Settlement time.Time LoanAmount float64 Balance float64 @@ -46,18 +46,14 @@ func (m *PayInAAAData) decodeAAAPdf(raw string) (e error) { m.Periods = make([]PayInAAAPeriod, 0, 10) lines := strings.Split(raw, "\n") - var tableHeader []string - var tableHeaderLine int currentPeriod := -1 state := "start" - for idx, l := range lines { // DFA, wow, finally it's used. after years of learning + for _, l := range lines { // DFA, wow, finally it's used. after years of learning switch state { case "start": state = m.processStart(l) if state == "LookingForPeriod" { - tableHeaderLine = idx - tableHeader = strings.Split(l, " ") - log.Println("Find table header", tableHeader, l, tableHeaderLine) + // determine column index, if their column is changing } break case "LookingForPeriod": @@ -125,7 +121,7 @@ func (m *PayInAAAData) processRow(line string) (nextState string, row PayInAAARo } if len(el) >= 5 { - row.LoanNUmber = el[0] + row.LoanNumber = el[0] row.Settlement, _ = time.Parse("02-Jan-06", el[1]) row.LoanAmount = m.currencyToFloat64(el[2]) row.Balance = m.currencyToFloat64(el[3])