package main import ( "biukop.com/sfm/loan" "context" "errors" log "github.com/sirupsen/logrus" "io/ioutil" "net/http" "os" "os/exec" "path/filepath" "strconv" "strings" "sync" "time" ) type uploadsOnDisk struct { Upload loan.Uploads } func (m *uploadsOnDisk) convertUploadsToPDF() (e error) { return convertUploadsToPDF(m.Upload) } func convertUploadsToPDF(ul loan.Uploads) (e error) { m := uploadsOnDisk{} m.Upload = ul if strings.Contains(strings.ToLower(ul.Format), "excel") || strings.Contains(strings.ToLower(ul.Format), "spreadsheet") { e = m.convertExcelToPDF() return // excel is converted } if strings.Contains(strings.ToLower(ul.Format), "/pdf") { return // no need to convert } e = errors.New("don't know how to convert file to PDF") log.Error("don't know how to convert file to PDF", ul) return } func (m *uploadsOnDisk) convertUploadsToJpg() (e error) { return convertUploadsToJpg(m.Upload) } func convertUploadsToJpg(ul loan.Uploads) (e error) { m := uploadsOnDisk{} m.Upload = ul if strings.Contains(strings.ToLower(ul.Format), "excel") || strings.Contains(strings.ToLower(ul.Format), "spreadsheet") { e = m.convertExcelToJpg() return // excel is converted } if strings.Contains(strings.ToLower(ul.Format), "/pdf") { e = m.convertPDFToJpg() return // excel is converted } e = errors.New("don't know how to convert file to image") log.Error("don't know how to convert file to image", ul) return } func (m *uploadsOnDisk) convertUploadsToThumb() (e error) { return convertUploadsToThumb(m.Upload) } func convertUploadsToThumb(ul loan.Uploads) (e error) { m := uploadsOnDisk{} m.Upload = ul if !fileExists(m.jpgPath()) { e = m.convertUploadsToJpg() if e != nil { return } } e = ConvertImageToThumbnail(m.jpgPath(), m.thumbPath(), 256) if e != nil { log.Error("cannot create thumbnail for uploads", m.Upload, e) return } return } func (m *uploadsOnDisk) filePath() string { return config.UploadsDir.FileDir + strconv.Itoa(int(m.Upload.Id)) + ".uploads" } func (m *uploadsOnDisk) jpgPath() string { return config.UploadsDir.JpgDir + strconv.Itoa(int(m.Upload.Id)) + ".jpg" } func (m *uploadsOnDisk) thumbPath() string { return config.UploadsDir.ThumbDir + strconv.Itoa(int(m.Upload.Id)) + ".webp" } func (m *uploadsOnDisk) pdfPath() string { return config.UploadsDir.PdfDir + strconv.Itoa(int(m.Upload.Id)) + ".pdf" } func (m *uploadsOnDisk) convertExcelToPDF() (e error) { if fileExists(m.pdfPath()) { log.Info("Skip conversion excel to PDF , already exists", m) return } return m.convertExcelTo("pdf") } func (m *uploadsOnDisk) convertExcelToJpg() (e error) { if fileExists(m.jpgPath()) { log.Info("Skip conversion excel to Jpg , already exists", m) return } return m.convertExcelTo("jpg") } var libreOfficeMutex sync.Mutex // make sure we only have one libreoffice running at a time func (m *uploadsOnDisk) convertExcelTo(format string) (e error) { if format != "pdf" && format != "jpg" { e = errors.New("convert excel to unsupported format " + format) return } dst := m.jpgPath() if format == "pdf" { dst = m.pdfPath() } dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-xls-to-"+format+"-") if e != nil { log.Error("cannot create tmp dir for converting image", m.Upload, e) return } defer os.RemoveAll(dir) // Create a new context and add a timeout to it ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // The cancel should be deferred so resources are cleaned up // Create the command with our context cmd := exec.CommandContext(ctx, "libreoffice", "--convert-to", format, "--outdir", dir, m.filePath()) libreOfficeMutex.Lock() //ensure only one libreoffice is running // This time we can simply use Output() to get the result. out, e := cmd.Output() libreOfficeMutex.Unlock() // We want to check the context error to see if the timeout was executed. // The error returned by cmd.Output() will be OS specific based on what // happens when a process is killed. if ctx.Err() == context.DeadlineExceeded { log.Error(cmd.String(), " timed out") switch format { case "pdf": _, e = copyFile(config.UploadsDir.PdfDefault, m.pdfPath()) break case "jpg": _, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath()) break } return } // If there's no context error, we know the command completed (or errored). //fmt.Println("Output:", string(out)) //if err != nil { // fmt.Println("Non-zero exit code:", err) //} // ---------- for cases not using ctx // for some unknown reason, libreoffice may just hung for ever //cmd := exec.Command("libreoffice", "--convert-to", format, "--outdir", dir, m.filePath()) //strCmd := cmd.String() //log.Debug("command is ", strCmd) //out, e := cmd.Output() // ------------ end of without ctx if e != nil { log.Error("cannot converting Excel to "+format+":", m.Upload, e) switch format { case "pdf": _, e = copyFile(config.UploadsDir.PdfDefault, m.pdfPath()) break case "jpg": _, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath()) break } return } else { // success log.Info("convert to "+format, m.Upload, " output: ", string(out)) } _, name := filepath.Split(m.filePath()) src := dir + string(os.PathSeparator) + fileNameWithoutExtTrimSuffix(name) + "." + format e = os.Rename(src, dst) // there should be only one jpg return } // first page to thumbnail // all page to single jpg func (m *uploadsOnDisk) convertPDFToJpg() (e error) { if fileExists(m.jpgPath()) { // no need to reconvert it again log.Info("PDF to JPG skipped it already exists ", m) return } dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-pdf-to-jpg-") if e != nil { log.Error("cannot create tmp dir for converting image", m.Upload, e) return } defer os.RemoveAll(dir) // convert -density 3000 abc.pdf path/tmp/result.png // could be path/tmp/result-0, result-1, result-2, ... png target := dir + string(os.PathSeparator) + "result.jpg" //.jpg suffix is important cmd := exec.Command("convert", "-density", "300", m.filePath(), target) strCmd := cmd.String() log.Debug("command is ", strCmd) out, e := cmd.Output() if e != nil { log.Error("cannot create png file for PDF", m.Upload, e) _, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath()) return } else { log.Info("convert ", m.Upload, " output: ", string(out)) } // montage -mode concatenate -tile 1x 30*png 30.jpg if fileExists(target) { // single file, _ = ConvertImageToThumbnail(target, m.thumbPath(), 256) e = os.Rename(target, m.jpgPath()) // there should be only one jpg } else { // multi-page, we have -0 -1 -2 -3 -4 files firstPage := dir + string(os.PathSeparator) + "result-0.jpg" _ = ConvertImageToThumbnail(firstPage, m.thumbPath(), 256) batch := dir + string(os.PathSeparator) + "result*jpg" // result* is important target = dir + string(os.PathSeparator) + "final.jpg" // .jpg suffix is important cmd = exec.Command("montage", "-mode", "concatenate", "-tile", "1x", batch, target) strCmd = cmd.String() log.Debug("command is ", strCmd) out, e = cmd.Output() if e != nil { return } else { log.Info("montage ", m, " output: ", string(out)) } e = os.Rename(target, m.jpgPath()) // give combined file to target } return } func (m *uploadsOnDisk) GetFileType() (ret string, e error) { strMime, e := GetFileContentType(m.filePath()) if e != nil { return } if strings.ToLower(strMime) == "application/pdf" { return "pdf", nil } if strings.ToLower(strMime) == "application/vnd.ms-excel" || strings.ToLower(m.Upload.Format) == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" { return "excel", nil } if strings.ToLower(strMime) == "application/zip" && strings.ToLower(m.Upload.Format) == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" { return "opensheet", nil } // check suffix which is not reliable ext := filepath.Ext(m.Upload.FileName) ext = strings.ToLower(ext) if ext[0] == '.' { ext = ext[1:] // remove the first char 'dot' . } switch ext { case "xls", "xlsx": return "excel", nil case "pdf": return "pdf", nil default: log.Warn("unhandled uploads type", ext) } return "", nil } func (m *uploadsOnDisk) DeleteAll() (e error) { eJpg := os.Remove(m.jpgPath()) eFile := os.Remove(m.filePath()) ePdf := os.Remove(m.pdfPath()) eThumb := os.Remove(m.thumbPath()) eMeta := m.Upload.Delete() strId := strconv.Itoa(int(m.Upload.Id)) errMsg := "" if eJpg != nil { errMsg += " jpg " } if eFile != nil { errMsg += " original " } if ePdf != nil { errMsg += " pdf " } if eThumb != nil { errMsg += " thumb " } if eMeta != nil { errMsg += " Meta in DB " e = errors.New(errMsg + " cannot be deleted " + strId) } if errMsg != "" { log.Error(errMsg, "files on disk cannot be deleted need disk cleaning ", m.Upload) } return } // GetFileContentType // tested, not accurate with xls, xlsx, it becomes zip and octstream sometime. func GetFileContentType(filename string) (contentType string, e error) { contentType = "" input, e := os.OpenFile(filename, os.O_RDONLY, 0755) // Only the first 512 bytes are used to sniff the content type. buffer := make([]byte, 512) _, e = input.Read(buffer) if e != nil { return } // Use the net/http package's handy Detect ContentType function. Always returns a valid // content-type by returning "application/octet-stream" if no others seemed to match. contentType = http.DetectContentType(buffer) return } func ConvertImageToThumbnail(srcPath string, dstPath string, size int) (e error) { if fileExists(dstPath) { log.Info("skip converting thumbnail it exists", dstPath) return } if size <= 0 { //thumb nail width size = 256 } // convert -thumbnail 200 abc.png thumb.abc.png cmd := exec.Command("convert", "-thumbnail", strconv.Itoa(size), srcPath, dstPath) strCmd := cmd.String() log.Debug("create thumbnail: ", strCmd) out, e := cmd.Output() if e != nil { log.Error("Failed to convert thumbnail", e) _, e = copyFile(config.UploadsDir.ThumbDefault, dstPath) return } else { // success log.Info("success output: \n: ", string(out)) } return }