Refactor renders (#15175)

* Refactor renders

* Some performance optimization

* Fix comment

* Transform reader

* Fix csv test

* Fix test

* Fix tests

* Improve optimaziation

* Fix test

* Fix test

* Detect file encoding with reader

* Improve optimaziation

* reduce memory usage

* improve code

* fix build

* Fix test

* Fix for go1.15

* Fix render

* Fix comment

* Fix lint

* Fix test

* Don't use NormalEOF when unnecessary

* revert change on util.go

* Apply suggestions from code review

Co-authored-by: zeripath <art27@cantab.net>

* rename function

* Take NormalEOF back

Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
Lunny Xiao 2021-04-20 06:25:08 +08:00 committed by GitHub
parent c9cc6698d2
commit 9d99f6ab19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1027 additions and 627 deletions

View file

@ -7,7 +7,9 @@ package csv
import (
"bytes"
"encoding/csv"
stdcsv "encoding/csv"
"errors"
"io"
"regexp"
"strings"
@ -18,17 +20,31 @@ import (
var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
// CreateReader creates a csv.Reader with the given delimiter.
func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
rd := csv.NewReader(bytes.NewReader(rawBytes))
func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
rd := stdcsv.NewReader(input)
rd.Comma = delimiter
rd.TrimLeadingSpace = true
return rd
}
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
delimiter := guessDelimiter(rawBytes)
return CreateReader(rawBytes, delimiter)
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
var data = make([]byte, 1e4)
size, err := rd.Read(data)
if err != nil {
return nil, err
}
delimiter := guessDelimiter(data[:size])
var newInput io.Reader
if size < 1e4 {
newInput = bytes.NewReader(data[:size])
} else {
newInput = io.MultiReader(bytes.NewReader(data), rd)
}
return CreateReader(newInput, delimiter), nil
}
// guessDelimiter scores the input CSV data against delimiters, and returns the best match.