SFM-loan-Rest-Api/UploadsOnDisk.go

package main

import (
	"biukop.com/sfm/loan"
	"context"
	"errors"
	log "github.com/sirupsen/logrus"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

type uploadsOnDisk struct {
	Upload loan.Uploads
	tmp    []byte //temporary buffer for file conversion such as csv
}

func (m *uploadsOnDisk) convertUploadsToPDF() (e error) {
	return convertUploadsToPDF(m.Upload)
}
func convertUploadsToPDF(ul loan.Uploads) (e error) {
	m := uploadsOnDisk{}
	m.Upload = ul

	if strings.Contains(strings.ToLower(ul.Format), "excel") ||
		strings.Contains(strings.ToLower(ul.Format), "spreadsheet") {
		e = m.convertExcelToPDF()
		return // excel is converted
	}

	if strings.Contains(strings.ToLower(ul.Format), "/pdf") {
		return // no need to convert
	}

	e = errors.New("don't know how to convert file to PDF")
	log.Error("don't know how to convert file to PDF", ul)
	return
}

func (m *uploadsOnDisk) convertUploadsToJpg() (e error) {
	return convertUploadsToJpg(m.Upload)
}

func convertUploadsToJpg(ul loan.Uploads) (e error) {
	m := uploadsOnDisk{}
	m.Upload = ul

	if strings.Contains(strings.ToLower(ul.Format), "excel") ||
		strings.Contains(strings.ToLower(ul.Format), "spreadsheet") {
		e = m.convertExcelToJpg()
		return // excel is converted
	}
	if strings.Contains(strings.ToLower(ul.Format), "/pdf") {
		e = m.convertPDFToJpg()
		return // excel is converted
	}

	e = errors.New("don't know how to convert file to image")
	log.Error("don't know how to convert file to image", ul)
	return
}

func (m *uploadsOnDisk) convertUploadsToThumb() (e error) {
	return convertUploadsToThumb(m.Upload)
}
func convertUploadsToThumb(ul loan.Uploads) (e error) {
	m := uploadsOnDisk{}
	m.Upload = ul
	if !fileExists(m.jpgPath()) {
		e = m.convertUploadsToJpg()
		if e != nil {
			return
		}
	}

	e = ConvertImageToThumbnail(m.jpgPath(), m.thumbPath(), 256)
	if e != nil {
		log.Error("cannot create thumbnail for uploads", m.Upload, e)
		return
	}
	return
}

func (m *uploadsOnDisk) filePath() string {
	return config.UploadsDir.FileDir + strconv.Itoa(int(m.Upload.Id)) + ".uploads"
}

func (m *uploadsOnDisk) jpgPath() string {
	return config.UploadsDir.JpgDir + strconv.Itoa(int(m.Upload.Id)) + ".jpg"
}

func (m *uploadsOnDisk) thumbPath() string {
	return config.UploadsDir.ThumbDir + strconv.Itoa(int(m.Upload.Id)) + ".webp"
}

func (m *uploadsOnDisk) pdfPath() string {
	return config.UploadsDir.PdfDir + strconv.Itoa(int(m.Upload.Id)) + ".pdf"
}

func (m *uploadsOnDisk) jsonPath() string {
	return config.UploadsDir.JsonDir + strconv.Itoa(int(m.Upload.Id)) + ".json"
}

func (m *uploadsOnDisk) convertExcelToPDF() (e error) {
	if fileExists(m.pdfPath()) {
		log.Info("Skip conversion excel to PDF , already exists", m)
		return
	}
	return m.convertExcelTo("pdf")
}

func (m *uploadsOnDisk) convertExcelToJpg() (e error) {
	if fileExists(m.jpgPath()) {
		log.Info("Skip conversion excel to Jpg , already exists", m)
		return
	}
	return m.convertExcelTo("jpg")
}

var libreOfficeMutex sync.Mutex // make sure we only have one libreoffice running at a time
func (m *uploadsOnDisk) convertExcelTo(format string) (e error) {
	if format != "pdf" && format != "jpg" && format != "csv" {
		e = errors.New("convert excel to unsupported format " + format)
		return
	}

	dst := m.jpgPath()
	if format == "pdf" {
		dst = m.pdfPath()
	}

	dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-xls-to-"+format+"-")
	if e != nil {
		log.Error("cannot create tmp dir for converting image", m.Upload, e)
		return
	}
	defer os.RemoveAll(dir)

	libreOfficeMutex.Lock() //ensure only one libreoffice is running

	// Create a new context and add a timeout to it
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel() // The cancel should be deferred so resources are cleaned up

	// Create the command with our context
	cmd := exec.CommandContext(ctx, "libreoffice", "--convert-to", format, "--outdir", dir, m.filePath())

	// This time we can simply use Output() to get the result.
	out, e := cmd.Output()
	libreOfficeMutex.Unlock()

	// We want to check the context error to see if the timeout was executed.
	// The error returned by cmd.Output() will be OS specific based on what
	// happens when a process is killed.
	if ctx.Err() == context.DeadlineExceeded {
		log.Error(cmd.String(), " timed out")

		switch format {
		case "pdf":
			_, e = copyFile(config.UploadsDir.PdfDefault, m.pdfPath())
			break
		case "jpg":
			_, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath())
			break
		}
		return
	}

	// If there's no context error, we know the command completed (or errored).
	//fmt.Println("Output:", string(out))
	//if err != nil {
	//	fmt.Println("Non-zero exit code:", err)
	//}

	// ---------- for cases not using ctx
	// for some unknown reason, libreoffice may just hung for ever
	//cmd := exec.Command("libreoffice", "--convert-to", format, "--outdir", dir, m.filePath())
	//strCmd := cmd.String()
	//log.Debug("command is ", strCmd)
	//out, e := cmd.Output()
	// ------------ end of without ctx

	if e != nil {
		log.Error("cannot converting Excel to "+format+":", m.Upload, e)
		switch format {
		case "pdf":
			_, e = copyFile(config.UploadsDir.PdfDefault, m.pdfPath())
			break
		case "jpg":
			_, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath())
			break
		}
		return
	} else { // success
		log.Info("convert to "+format, m.Upload, " output: ", string(out))
	}

	_, name := filepath.Split(m.filePath())

	src := dir + string(os.PathSeparator) + fileNameWithoutExtTrimSuffix(name) + "." + format

	switch format {
	case "pdf", "jpg":
		e = os.Rename(src, dst) // there should be only one jpg
		break
	case "csv":
		m.tmp, e = ioutil.ReadFile(src)
		break
	}
	return
}

// first page to thumbnail
// all page to single jpg
var convertPDFMutex sync.Mutex // make sure we only have one convert running at a time
func (m *uploadsOnDisk) convertPDFToJpg() (e error) {

	if fileExists(m.jpgPath()) {
		// no need to reconvert it again
		log.Info("PDF to JPG skipped it already exists ", m)
		return
	}

	dir, e := ioutil.TempDir(config.TempDir, "tmp-convert-pdf-to-jpg-")
	if e != nil {
		log.Error("cannot create tmp dir for converting image", m.Upload, e)
		return
	}
	defer os.RemoveAll(dir)

	convertPDFMutex.Lock()
	// convert -density 3000  abc.pdf  path/tmp/result.png
	// could be path/tmp/result-0, result-1, result-2, ... png
	target := dir + string(os.PathSeparator) + "result.jpg" //.jpg suffix is important
	cmd := exec.Command("convert", "-density", "300", m.filePath(), target)
	strCmd := cmd.String()
	log.Debug("command is ", strCmd)
	out, e := cmd.Output()
	convertPDFMutex.Unlock()

	if e != nil {
		log.Error("cannot create png file for PDF", m.Upload, e)
		_, e = copyFile(config.UploadsDir.JpgDefault, m.jpgPath())
		return
	} else {
		log.Info("convert ", m.Upload, " output: ", string(out))
	}

	// montage -mode concatenate -tile 1x 30*png 30.jpg
	if fileExists(target) { // single file,
		_ = ConvertImageToThumbnail(target, m.thumbPath(), 256)
		e = os.Rename(target, m.jpgPath()) // there should be only one jpg
	} else { // multi-page, we have -0 -1 -2 -3 -4  files
		firstPage := dir + string(os.PathSeparator) + "result-0.jpg"
		_ = ConvertImageToThumbnail(firstPage, m.thumbPath(), 256)

		batch := dir + string(os.PathSeparator) + "result*jpg" // result* is important
		target = dir + string(os.PathSeparator) + "final.jpg"  // .jpg suffix is important
		cmd = exec.Command("montage", "-mode", "concatenate", "-tile", "1x", batch, target)

		strCmd = cmd.String()
		log.Debug("command is ", strCmd)
		out, e = cmd.Output()
		if e != nil {
			return
		} else {
			log.Info("montage ", m, " output: ", string(out))
		}
		e = os.Rename(target, m.jpgPath()) // give combined file to target
	}
	return
}

func (m *uploadsOnDisk) GetFileType() (ret string, e error) {
	strMime, e := GetFileContentType(m.filePath())
	if e != nil {
		return
	}

	if strings.ToLower(strMime) == "application/pdf" {
		return "pdf", nil
	}

	if strings.ToLower(strMime) == "application/vnd.ms-excel" || strings.ToLower(m.Upload.Format) ==
		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
		return "excel", nil
	}

	if strings.ToLower(strMime) == "application/zip" &&
		strings.ToLower(m.Upload.Format) ==
			"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
		return "opensheet", nil
	}

	// check suffix which is not reliable
	ext := filepath.Ext(m.Upload.FileName)
	ext = strings.ToLower(ext)
	if ext[0] == '.' {
		ext = ext[1:] // remove the first char 'dot' .
	}
	switch ext {
	case "xls", "xlsx":
		return "excel", nil
	case "pdf":
		return "pdf", nil
	default:
		log.Warn("unhandled uploads type", ext)
	}
	return "", nil
}

func (m *uploadsOnDisk) DeleteAll() (e error) {
	eJpg := os.Remove(m.jpgPath())
	eFile := os.Remove(m.filePath())
	ePdf := os.Remove(m.pdfPath())
	eThumb := os.Remove(m.thumbPath())
	eJson := os.Remove(m.jsonPath())
	eMeta := m.Upload.Delete()

	strId := strconv.Itoa(int(m.Upload.Id))

	errMsg := ""
	if eJpg != nil {
		errMsg += " jpg "
	}

	if eFile != nil {
		errMsg += " original "
	}

	if ePdf != nil {
		errMsg += " pdf "
	}

	if eThumb != nil {
		errMsg += " thumb  "
	}

	if eJson != nil {
		errMsg += " json "
	}

	if eMeta != nil {
		errMsg += " Meta in DB "
		e = errors.New(errMsg + " cannot be deleted " + strId)
	}

	if errMsg != "" {
		log.Error(errMsg, "files on disk cannot be deleted need disk cleaning ", m.Upload)
	}
	return
}

// GetFileContentType
// tested, not accurate with xls, xlsx, it becomes zip and octstream sometime.
func GetFileContentType(filename string) (contentType string, e error) {
	contentType = ""
	input, e := os.OpenFile(filename, os.O_RDONLY, 0755)
	// Only the first 512 bytes are used to sniff the content type.
	buffer := make([]byte, 512)

	_, e = input.Read(buffer)
	if e != nil {
		return
	}

	// Use the net/http package's handy Detect ContentType function. Always returns a valid
	// content-type by returning "application/octet-stream" if no others seemed to match.
	contentType = http.DetectContentType(buffer)
	return
}

func ConvertImageToThumbnail(srcPath string, dstPath string, size int) (e error) {
	if fileExists(dstPath) {
		log.Info("skip converting thumbnail it exists", dstPath)
		return
	}
	if size <= 0 { //thumb nail width
		size = 256
	}
	// convert -thumbnail 200 abc.png thumb.abc.png
	cmd := exec.Command("convert", "-thumbnail", strconv.Itoa(size), srcPath, dstPath)
	strCmd := cmd.String()
	log.Debug("create thumbnail: ", strCmd)

	out, e := cmd.Output()
	if e != nil {
		log.Error("Failed to convert thumbnail", e)
		_, e = copyFile(config.UploadsDir.ThumbDefault, dstPath)
		return
	} else { // success
		log.Info("success output: \n: ", string(out))
	}
	return
}