forked from NYANDEV/forgejo
Refactor renders (#15175)
* Refactor renders * Some performance optimization * Fix comment * Transform reader * Fix csv test * Fix test * Fix tests * Improve optimaziation * Fix test * Fix test * Detect file encoding with reader * Improve optimaziation * reduce memory usage * improve code * fix build * Fix test * Fix for go1.15 * Fix render * Fix comment * Fix lint * Fix test * Don't use NormalEOF when unnecessary * revert change on util.go * Apply suggestions from code review Co-authored-by: zeripath <art27@cantab.net> * rename function * Take NormalEOF back Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
parent
c9cc6698d2
commit
9d99f6ab19
41 changed files with 1027 additions and 627 deletions
|
@ -7,7 +7,9 @@ package csv
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
stdcsv "encoding/csv"
|
||||
"errors"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
|
@ -18,17 +20,31 @@ import (
|
|||
var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
|
||||
|
||||
// CreateReader creates a csv.Reader with the given delimiter.
|
||||
func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
|
||||
rd := csv.NewReader(bytes.NewReader(rawBytes))
|
||||
func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
|
||||
rd := stdcsv.NewReader(input)
|
||||
rd.Comma = delimiter
|
||||
rd.TrimLeadingSpace = true
|
||||
return rd
|
||||
}
|
||||
|
||||
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
|
||||
func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
|
||||
delimiter := guessDelimiter(rawBytes)
|
||||
return CreateReader(rawBytes, delimiter)
|
||||
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
|
||||
var data = make([]byte, 1e4)
|
||||
size, err := rd.Read(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
delimiter := guessDelimiter(data[:size])
|
||||
|
||||
var newInput io.Reader
|
||||
if size < 1e4 {
|
||||
newInput = bytes.NewReader(data[:size])
|
||||
} else {
|
||||
newInput = io.MultiReader(bytes.NewReader(data), rd)
|
||||
}
|
||||
|
||||
return CreateReader(newInput, delimiter), nil
|
||||
}
|
||||
|
||||
// guessDelimiter scores the input CSV data against delimiters, and returns the best match.
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue