Skip to content

Commit

Permalink
Stream to Excel table (qax-os#530)
Browse files Browse the repository at this point in the history
* Support all datatypes for StreamWriter

* Support setting styles with StreamWriter

**NOTE:** This is a breaking change. Values are now explicitly
passed as a []interface{} for simplicity. We also let styles to be
set at the same time.

* Create function to write stream into a table

* Write rows directly to buffer

Avoiding the xml.Encoder makes the streamer faster and use less
memory.

Using the included benchmark, the results went from:

> BenchmarkStreamWriter-4   514  2576155 ns/op  454918 B/op  6592 allocs/op

down to:

> BenchmarkStreamWriter-4  1614   777480 ns/op  147608 B/op  5570 allocs/op

* Use AddTable instead of SetTable

This requires reading the cells after they have been written,
which requires additional structure for the temp file.

As a bonus, we now efficiently allocate only one buffer when
reading the file back into memory, using the same approach
as ioutil.ReadFile.

* Use an exported Cell type to handle inline styles for StreamWriter
  • Loading branch information
chowey authored and xuri committed Dec 29, 2019
1 parent 8b960ee commit 5c87eff
Show file tree
Hide file tree
Showing 4 changed files with 540 additions and 160 deletions.
23 changes: 20 additions & 3 deletions adjust.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,14 @@ func (f *File) adjustAutoFilterHelper(dir adjustDirection, coordinates []int, nu
// areaRefToCoordinates provides a function to convert area reference to a
// pair of coordinates.
func (f *File) areaRefToCoordinates(ref string) ([]int, error) {
coordinates := make([]int, 4)
rng := strings.Split(ref, ":")
firstCell := rng[0]
lastCell := rng[1]
return areaRangeToCoordinates(rng[0], rng[1])
}

// areaRangeToCoordinates provides a function to convert cell range to a
// pair of coordinates.
func areaRangeToCoordinates(firstCell, lastCell string) ([]int, error) {
coordinates := make([]int, 4)
var err error
coordinates[0], coordinates[1], err = CellNameToCoordinates(firstCell)
if err != nil {
Expand All @@ -209,6 +213,19 @@ func (f *File) areaRefToCoordinates(ref string) ([]int, error) {
return coordinates, err
}

func sortCoordinates(coordinates []int) error {
if len(coordinates) != 4 {
return errors.New("coordinates length must be 4")
}
if coordinates[2] < coordinates[0] {
coordinates[2], coordinates[0] = coordinates[0], coordinates[2]
}
if coordinates[3] < coordinates[1] {
coordinates[3], coordinates[1] = coordinates[1], coordinates[3]
}
return nil
}

// coordinatesToAreaRef provides a function to convert a pair of coordinates
// to area reference.
func (f *File) coordinatesToAreaRef(coordinates []int) (string, error) {
Expand Down
100 changes: 72 additions & 28 deletions cell.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ func (f *File) SetCellValue(sheet, axis string, value interface{}) error {
case []byte:
err = f.SetCellStr(sheet, axis, string(v))
case time.Duration:
err = f.SetCellDefault(sheet, axis, strconv.FormatFloat(v.Seconds()/86400.0, 'f', -1, 32))
_, d := setCellDuration(v)
err = f.SetCellDefault(sheet, axis, d)
if err != nil {
return err
}
Expand Down Expand Up @@ -131,28 +132,50 @@ func (f *File) setCellIntFunc(sheet, axis string, value interface{}) error {
// setCellTimeFunc provides a method to process time type of value for
// SetCellValue.
func (f *File) setCellTimeFunc(sheet, axis string, value time.Time) error {
excelTime, err := timeToExcelTime(value)
xlsx, err := f.workSheetReader(sheet)
if err != nil {
return err
}
if excelTime > 0 {
err = f.SetCellDefault(sheet, axis, strconv.FormatFloat(excelTime, 'f', -1, 64))
if err != nil {
return err
}
cellData, col, _, err := f.prepareCell(xlsx, sheet, axis)
if err != nil {
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)

var isNum bool
cellData.T, cellData.V, isNum, err = setCellTime(value)
if err != nil {
return err
}
if isNum {
err = f.setDefaultTimeStyle(sheet, axis, 22)
if err != nil {
return err
}
} else {
err = f.SetCellStr(sheet, axis, value.Format(time.RFC3339Nano))
if err != nil {
return err
}
}
return err
}

func setCellTime(value time.Time) (t string, b string, isNum bool, err error) {
var excelTime float64
excelTime, err = timeToExcelTime(value)
if err != nil {
return
}
isNum = excelTime > 0
if isNum {
t, b = setCellDefault(strconv.FormatFloat(excelTime, 'f', -1, 64))
} else {
t, b = setCellDefault(value.Format(time.RFC3339Nano))
}
return
}

func setCellDuration(value time.Duration) (t string, v string) {
v = strconv.FormatFloat(value.Seconds()/86400.0, 'f', -1, 32)
return
}

// SetCellInt provides a function to set int type value of a cell by given
// worksheet name, cell coordinates and cell value.
func (f *File) SetCellInt(sheet, axis string, value int) error {
Expand All @@ -165,11 +188,15 @@ func (f *File) SetCellInt(sheet, axis string, value int) error {
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T = ""
cellData.V = strconv.Itoa(value)
cellData.T, cellData.V = setCellInt(value)
return err
}

func setCellInt(value int) (t string, v string) {
v = strconv.Itoa(value)
return
}

// SetCellBool provides a function to set bool type value of a cell by given
// worksheet name, cell name and cell value.
func (f *File) SetCellBool(sheet, axis string, value bool) error {
Expand All @@ -182,13 +209,18 @@ func (f *File) SetCellBool(sheet, axis string, value bool) error {
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T = "b"
cellData.T, cellData.V = setCellBool(value)
return err
}

func setCellBool(value bool) (t string, v string) {
t = "b"
if value {
cellData.V = "1"
v = "1"
} else {
cellData.V = "0"
v = "0"
}
return err
return
}

// SetCellFloat sets a floating point value into a cell. The prec parameter
Expand All @@ -210,11 +242,15 @@ func (f *File) SetCellFloat(sheet, axis string, value float64, prec, bitSize int
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T = ""
cellData.V = strconv.FormatFloat(value, 'f', prec, bitSize)
cellData.T, cellData.V = setCellFloat(value, prec, bitSize)
return err
}

func setCellFloat(value float64, prec, bitSize int) (t string, v string) {
v = strconv.FormatFloat(value, 'f', prec, bitSize)
return
}

// SetCellStr provides a function to set string type value of a cell. Total
// number of characters that a cell can contain 32767 characters.
func (f *File) SetCellStr(sheet, axis, value string) error {
Expand All @@ -226,21 +262,25 @@ func (f *File) SetCellStr(sheet, axis, value string) error {
if err != nil {
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T, cellData.V, cellData.XMLSpace = setCellStr(value)
return err
}

func setCellStr(value string) (t string, v string, ns xml.Attr) {
if len(value) > 32767 {
value = value[0:32767]
}
// Leading and ending space(s) character detection.
if len(value) > 0 && (value[0] == 32 || value[len(value)-1] == 32) {
cellData.XMLSpace = xml.Attr{
ns = xml.Attr{
Name: xml.Name{Space: NameSpaceXML, Local: "space"},
Value: "preserve",
}
}

cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T = "str"
cellData.V = value
return err
t = "str"
v = value
return
}

// SetCellDefault provides a function to set string type value of a cell as
Expand All @@ -255,11 +295,15 @@ func (f *File) SetCellDefault(sheet, axis, value string) error {
return err
}
cellData.S = f.prepareCellStyle(xlsx, col, cellData.S)
cellData.T = ""
cellData.V = value
cellData.T, cellData.V = setCellDefault(value)
return err
}

func setCellDefault(value string) (t string, v string) {
v = value
return
}

// GetCellFormula provides a function to get formula from cell by given
// worksheet name and axis in XLSX file.
func (f *File) GetCellFormula(sheet, axis string) (string, error) {
Expand Down
Loading

0 comments on commit 5c87eff

Please sign in to comment.