v2,v3增加缓存功能, 大幅提升xlsx格式导出速度

Author: davyxu

doc/Manual_V2.md

@@ -351,6 +351,17 @@
 
 * 范例: 语言名支持en_us(默认),  zh_cn(简体中文)
 
+## 并发导出
+
+* 格式: --para=true
+
+* 功能: 使用多线程技术并发导出, 提升导出速度
+
+## 使用缓冲
+
+* 格式: --usecache=true
+
+* 功能: 导出时, 使用缓冲, 大幅提升复杂xlsx格式导出速度, 导出目录使用--cachedir设置, 默认目录为/.tabtoycache
 
 ## 多文件合并
 

entry_v2.go

@@ -33,6 +33,8 @@ func V2Entry() {
 	}
 
 	g.ParaMode = *paramPara
+	g.CacheDir = *paramCacheDir
+	g.UseCache = *paramUseCache
 	g.CombineStructName = *paramCombineStructName
 	g.ProtoVersion = *paramProtoVersion
 	g.LuaEnumIntValue = *paramLuaEnumIntValue

entry_v2tov3.go

@@ -16,7 +16,7 @@ func V2ToV3Entry() {
 
 	globals := model.NewGlobals()
 
-	globals.TableGetter = helper.NewFileLoader(true)
+	globals.TableGetter = helper.NewFileLoader(true, "")
 
 	globals.SourceFileList = flag.Args()
 	globals.OutputDir = *paramUpgradeOut

entry_v3.go

@@ -86,6 +86,10 @@ func V3Entry() {
 	globals := model.NewGlobals()
 	globals.Version = Version
 	globals.ParaLoading = *paramPara
+	if *paramUseCache {
+		globals.CacheDir = *paramCacheDir
+		os.Mkdir(globals.CacheDir, 0666)
+	}
 	globals.IndexFile = *paramIndexFile
 	globals.PackageName = *paramPackageName
 	globals.CombineStructName = *paramCombineStructName
@@ -96,7 +100,7 @@ func V3Entry() {
 		report.Log.Infof("MatchTag: %s", globals.MatchTag)
 	}
 
-	idxloader := helper.NewFileLoader(true)
+	idxloader := helper.NewFileLoader(true, globals.CacheDir)
 	idxloader.UseGBKCSV = *paramUseGBKCSV
 	globals.IndexGetter = idxloader
 	globals.UseGBKCSV = *paramUseGBKCSV

flag.go

@@ -13,6 +13,10 @@ var (
 	// 并发导出,提高导出速度, 输出日志会混乱
 	paramPara = flag.Bool("para", false, "parallel export by your cpu count")
 
+	// 并发导出,提高导出速度, 输出日志会混乱
+	paramCacheDir = flag.String("cachedir", "./.tabtoycache", "cache file output dir")
+	paramUseCache = flag.Bool("usecache", false, "use cache file enhanced exporting speed")
+
 	paramLanguage = flag.String("lan", "en_us", "set output language")
 )
 

main.go

@@ -11,7 +11,7 @@ import (
 var log = golog.New("main")
 
 const (
-	Version = "2.9.3"
+	Version = "2.9.4"
 )
 
 var enableProfile bool = false

util/cache.go

@@ -0,0 +1,179 @@
+package util
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"fmt"
+	"github.com/tealeg/xlsx"
+	"os"
+)
+
+type XlsxHashFile struct {
+	CRC32Map map[string]uint32
+}
+
+type XlsxFileCache struct {
+	Name   string
+	Sheets []*XlsxSheet
+}
+
+type XlsxSheet struct {
+	Name  string
+	Cells [][]string
+}
+
+type TableCache struct {
+	z        *zip.ReadCloser
+	name     string
+	cacheDir string
+
+	originFile *xlsx.File
+}
+
+func NewTableCache(name, cachedir string) *TableCache {
+	return &TableCache{
+		name:     name,
+		cacheDir: cachedir,
+	}
+}
+
+func (self *TableCache) UseCache() bool {
+	return self.originFile == nil
+}
+
+func (self *TableCache) cacheFileName() string {
+	return fmt.Sprintf("%s/%s.cache", self.cacheDir, self.name)
+}
+
+func (self *TableCache) hashFileName() string {
+	return fmt.Sprintf("%s/%s.hash", self.cacheDir, self.name)
+}
+
+func (self *TableCache) Open() error {
+
+	z, err := zip.OpenReader(self.name)
+	if err != nil {
+		return err
+	}
+
+	self.z = z
+
+	return nil
+}
+
+func readJsonFile(filename string, m interface{}) error {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil
+	}
+
+	return json.NewDecoder(f).Decode(m)
+}
+
+func writeJsonFile(filename string, m interface{}) error {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	return json.NewEncoder(f).Encode(m)
+}
+
+func (self *TableCache) readCache() (xf *xlsx.File, err error) {
+
+	var hashFile XlsxHashFile
+	err = readJsonFile(self.hashFileName(), &hashFile)
+	if err != nil {
+		return nil, nil
+	}
+
+	if len(self.z.File) != len(hashFile.CRC32Map) {
+		return nil, nil
+	}
+
+	for _, f := range self.z.File {
+		if hashFile.CRC32Map[f.Name] != f.CRC32 {
+			return nil, nil
+		}
+	}
+
+	var cacheFile XlsxFileCache
+	err = readJsonFile(self.cacheFileName(), &cacheFile)
+
+	if err != nil {
+		return nil, nil
+	}
+
+	xf = xlsx.NewFile()
+
+	for _, s := range cacheFile.Sheets {
+		sheet, err := xf.AddSheet(s.Name)
+		if err != nil {
+			return nil, err
+		}
+		for _, srcRow := range s.Cells {
+			tgtRow := sheet.AddRow()
+
+			for _, c := range srcRow {
+				cell := tgtRow.AddCell()
+				cell.Value = c
+			}
+
+		}
+	}
+
+	return
+}
+
+func (self *TableCache) Load() (xf *xlsx.File, err error) {
+
+	cfile, err := self.readCache()
+
+	// cache未击中, 从原文件读取
+	if err == nil && cfile == nil {
+		xf, err = xlsx.ReadZipWithRowLimit(self.z, xlsx.NoRowLimit)
+		self.originFile = xf
+		return
+	}
+
+	return cfile, err
+}
+
+func (self *TableCache) Save() error {
+
+	var hashFile XlsxHashFile
+	hashFile.CRC32Map = make(map[string]uint32)
+
+	for _, f := range self.z.File {
+		hashFile.CRC32Map[f.Name] = f.CRC32
+	}
+
+	writeJsonFile(self.hashFileName(), &hashFile)
+
+	var newFile XlsxFileCache
+	newFile.Name = self.name
+
+	for _, sheet := range self.originFile.Sheets {
+
+		var newSheet XlsxSheet
+		newSheet.Name = sheet.Name
+
+		newSheet.Cells = make([][]string, 0, len(sheet.Rows))
+
+		for _, row := range sheet.Rows {
+
+			var rowData = make([]string, 0, len(row.Cells))
+			for _, c := range row.Cells {
+				rowData = append(rowData, c.Value)
+			}
+
+			newSheet.Cells = append(newSheet.Cells, rowData)
+		}
+
+		newFile.Sheets = append(newFile.Sheets, &newSheet)
+	}
+
+	writeJsonFile(self.cacheFileName(), &newFile)
+
+	return nil
+}

v2/file.go

@@ -1,6 +1,7 @@
 package v2
 
 import (
+	"github.com/davyxu/tabtoy/util"
 	"strings"
 
 	"github.com/davyxu/tabtoy/v2/i18n"
@@ -155,23 +156,43 @@ func isTypeSheet(name string) bool {
 	return strings.TrimSpace(name) == model.TypeSheetName
 }
 
-func NewFile(filename string) *File {
+func NewFile(filename string, cacheDir string) (f *File, fromCache bool) {
 
 	self := &File{
 		valueRepByKey: make(map[valueRepeatData]bool),
 		LocalFD:       model.NewFileDescriptor(),
 		FileName:      filename,
 	}
 
+	if cacheDir != "" {
+		cache := util.NewTableCache(filename, cacheDir)
+
+		if err := cache.Open(); err != nil {
+			log.Errorf("%s, %v", i18n.String(i18n.System_OpenReadXlsxFailed), err.Error())
+			return nil, false
+		}
+
+		if cfile, err := cache.Load(); err != nil {
+			log.Errorln(err.Error())
+			log.Errorf("%s, %v", i18n.String(i18n.System_OpenReadXlsxFailed), err.Error())
+			return nil, false
+		} else {
+			self.coreFile = cfile
+
+			if !cache.UseCache() {
+				cache.Save()
+			}
+
+			return self, cache.UseCache()
+		}
+	}
+
 	var err error
 	self.coreFile, err = xlsx.OpenFile(filename)
-
 	if err != nil {
-		log.Errorln(err.Error())
 		log.Errorf("%s, %v", i18n.String(i18n.System_OpenReadXlsxFailed), err.Error())
-
-		return nil
+		return nil, false
 	}
 
-	return self
+	return self, false
 }

v2/filecache.go

@@ -3,6 +3,7 @@ package v2
 import (
 	"github.com/davyxu/tabtoy/v2/i18n"
 	"github.com/davyxu/tabtoy/v2/printer"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -33,20 +34,36 @@ func cacheFile(g *printer.Globals) (fileObjByName map[string]*File) {
 
 	filelist := getFileList(g)
 
+	var cachedir string
+	if g.UseCache {
+		cachedir = g.CacheDir
+
+		os.Mkdir(g.CacheDir, 0666)
+	}
+
+	writeOK := func(xlsxFileName string) {
+		file, fromCache := NewFile(xlsxFileName, cachedir)
+
+		fileObjByNameGuard.Lock()
+		fileObjByName[xlsxFileName] = file
+		fileObjByNameGuard.Unlock()
+		if fromCache {
+			log.Infof("%s [Cache]", filepath.Base(xlsxFileName))
+		} else {
+			log.Infof("%s", filepath.Base(xlsxFileName))
+		}
+
+	}
+
+	// 这里加速效果良好, 默认开启
 	var task sync.WaitGroup
 	task.Add(len(filelist))
 
 	for _, filename := range filelist {
 
 		go func(xlsxFileName string) {
 
-			log.Infoln(filepath.Base(xlsxFileName))
-			file := NewFile(xlsxFileName)
-
-			fileObjByNameGuard.Lock()
-			fileObjByName[xlsxFileName] = file
-			fileObjByNameGuard.Unlock()
-
+			writeOK(xlsxFileName)
 			task.Done()
 
 		}(filename)
@@ -55,5 +72,11 @@ func cacheFile(g *printer.Globals) (fileObjByName map[string]*File) {
 
 	task.Wait()
 
+	// 调试用
+	//for _, filename := range filelist {
+	//
+	//	writeOK(filename)
+	//}
+
 	return
 }

v2/printer/globals.go

@@ -15,6 +15,8 @@ type Globals struct {
 	Version            string
 	InputFileList      []interface{}
 	ParaMode           bool
+	CacheDir           string
+	UseCache           bool
 	ProtoVersion       int
 	LuaEnumIntValue    bool
 	LuaTabHeader       string

v2tov3/model/globals.go

@@ -25,7 +25,7 @@ type Globals struct {
 
 func (self *Globals) AddTableByFile(tableFileName, tableName string, inputFile *xlsx.File) {
 
-	file := helper.NewXlsxFile()
+	file := helper.NewXlsxFile("")
 
 	file.(interface {
 		FromXFile(file *xlsx.File)