Integrate public as bindata optionally (#293)

* Dropped unused codekit config

* Integrated dynamic and static bindata for public

* Ignore public bindata

* Add a general generate make task

* Integrated flexible public assets into web command

* Updated vendoring, added all missiong govendor deps

* Made the linter happy with the bindata and dynamic code

* Moved public bindata definition to modules directory

* Ignoring the new bindata path now

* Updated to the new public modules import path

* Updated public bindata command and drop the new prefix
This commit is contained in:
Thomas Boerger 2016-11-29 17:26:36 +01:00 committed by Lunny Xiao
parent 4680c349dd
commit b6a95a8cb3
691 changed files with 305318 additions and 1272 deletions

142
vendor/github.com/pingcap/tidb/kv/btree_buffer.go generated vendored Normal file
View file

@ -0,0 +1,142 @@
// Copyright 2015 PingCAP, Inc.
//
// Copyright 2015 Wenbin Xiao
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"io"
"github.com/juju/errors"
"github.com/pingcap/tidb/kv/memkv"
"github.com/pingcap/tidb/terror"
"github.com/pingcap/tidb/util/types"
)
type btreeBuffer struct {
tree *memkv.Tree
}
// NewBTreeBuffer returns a breeBuffer.
func NewBTreeBuffer() MemBuffer {
return &btreeBuffer{
tree: memkv.NewTree(types.Collators[true]),
}
}
// Get returns the value associated with the key; ErrNotExist error if the key does not exist.
func (b *btreeBuffer) Get(k Key) ([]byte, error) {
v, ok := b.tree.Get(toIfaces(k))
if !ok {
return nil, ErrNotExist
}
return fromIfaces(v), nil
}
// Set associates the key with the value.
func (b *btreeBuffer) Set(k Key, v []byte) error {
if len(v) == 0 {
return errors.Trace(ErrCannotSetNilValue)
}
b.tree.Set(toIfaces(k), toIfaces(v))
return nil
}
// Delete removes the entry from buffer with provided key.
func (b *btreeBuffer) Delete(k Key) error {
b.tree.Set(toIfaces(k), nil)
return nil
}
// Release clear the whole buffer.
func (b *btreeBuffer) Release() {
b.tree.Clear()
}
type btreeIter struct {
e *memkv.Enumerator
k Key
v []byte
ok bool
}
// Seek creates a new Iterator based on the provided key.
func (b *btreeBuffer) Seek(k Key) (Iterator, error) {
var e *memkv.Enumerator
var err error
if k == nil {
e, err = b.tree.SeekFirst()
if err != nil {
if terror.ErrorEqual(err, io.EOF) {
return &btreeIter{ok: false}, nil
}
return &btreeIter{ok: false}, errors.Trace(err)
}
} else {
key := toIfaces([]byte(k))
e, _ = b.tree.Seek(key)
}
iter := &btreeIter{e: e}
// the initial push...
err = iter.Next()
if err != nil {
return &btreeIter{ok: false}, errors.Trace(err)
}
return iter, nil
}
// Close implements Iterator Close.
func (i *btreeIter) Close() {
//noop
}
// Key implements Iterator Key.
func (i *btreeIter) Key() Key {
return i.k
}
// Value implements Iterator Value.
func (i *btreeIter) Value() []byte {
return i.v
}
// Next implements Iterator Next.
func (i *btreeIter) Next() error {
k, v, err := i.e.Next()
if err != nil {
i.ok = false
if terror.ErrorEqual(err, io.EOF) {
return nil
}
return errors.Trace(err)
}
i.k, i.v, i.ok = fromIfaces(k), fromIfaces(v), true
return nil
}
// Valid implements Iterator Valid.
func (i *btreeIter) Valid() bool {
return i.ok
}
func toIfaces(v []byte) []interface{} {
return []interface{}{v}
}
func fromIfaces(v []interface{}) []byte {
if v == nil {
return nil
}
return v[0].([]byte)
}

93
vendor/github.com/pingcap/tidb/kv/buffer_store.go generated vendored Normal file
View file

@ -0,0 +1,93 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"github.com/juju/errors"
)
// BufferStore wraps a Retriever for read and a MemBuffer for buffered write.
// Common usage pattern:
// bs := NewBufferStore(r) // use BufferStore to wrap a Retriever
// defer bs.Release() // make sure it will be released
// // ...
// // read/write on bs
// // ...
// bs.SaveTo(m) // save above operations to a Mutator
type BufferStore struct {
MemBuffer
r Retriever
}
// NewBufferStore creates a BufferStore using r for read.
func NewBufferStore(r Retriever) *BufferStore {
return &BufferStore{
r: r,
MemBuffer: &lazyMemBuffer{},
}
}
// Get implements the Retriever interface.
func (s *BufferStore) Get(k Key) ([]byte, error) {
val, err := s.MemBuffer.Get(k)
if IsErrNotFound(err) {
val, err = s.r.Get(k)
}
if err != nil {
return nil, errors.Trace(err)
}
if len(val) == 0 {
return nil, errors.Trace(ErrNotExist)
}
return val, nil
}
// Seek implements the Retriever interface.
func (s *BufferStore) Seek(k Key) (Iterator, error) {
bufferIt, err := s.MemBuffer.Seek(k)
if err != nil {
return nil, errors.Trace(err)
}
retrieverIt, err := s.r.Seek(k)
if err != nil {
return nil, errors.Trace(err)
}
return newUnionIter(bufferIt, retrieverIt), nil
}
// WalkBuffer iterates all buffered kv pairs.
func (s *BufferStore) WalkBuffer(f func(k Key, v []byte) error) error {
iter, err := s.MemBuffer.Seek(nil)
if err != nil {
return errors.Trace(err)
}
defer iter.Close()
for ; iter.Valid(); iter.Next() {
if err := f(iter.Key(), iter.Value()); err != nil {
return errors.Trace(err)
}
}
return nil
}
// SaveTo saves all buffered kv pairs into a Mutator.
func (s *BufferStore) SaveTo(m Mutator) error {
err := s.WalkBuffer(func(k Key, v []byte) error {
if len(v) == 0 {
return errors.Trace(m.Delete(k))
}
return errors.Trace(m.Set(k, v))
})
return errors.Trace(err)
}

58
vendor/github.com/pingcap/tidb/kv/bufpool.go generated vendored Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"sync"
"github.com/ngaut/log"
)
// A cache holds a set of reusable objects.
// The slice is a stack (LIFO).
// If more are needed, the cache creates them by calling new.
type cache struct {
mu sync.Mutex
name string
saved []MemBuffer
// factory
fact func() MemBuffer
}
func (c *cache) put(x MemBuffer) {
c.mu.Lock()
if len(c.saved) < cap(c.saved) {
c.saved = append(c.saved, x)
} else {
log.Warnf("%s is full, size: %d, you may need to increase pool size", c.name, len(c.saved))
}
c.mu.Unlock()
}
func (c *cache) get() MemBuffer {
c.mu.Lock()
n := len(c.saved)
if n == 0 {
c.mu.Unlock()
return c.fact()
}
x := c.saved[n-1]
c.saved = c.saved[0 : n-1]
c.mu.Unlock()
return x
}
func newCache(name string, cap int, fact func() MemBuffer) *cache {
return &cache{name: name, saved: make([]MemBuffer, 0, cap), fact: fact}
}

91
vendor/github.com/pingcap/tidb/kv/error.go generated vendored Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"errors"
"strings"
"github.com/pingcap/go-themis"
"github.com/pingcap/tidb/mysql"
"github.com/pingcap/tidb/terror"
)
// KV error codes.
const (
CodeIncompatibleDBFormat terror.ErrCode = 1
CodeNoDataForHandle terror.ErrCode = 2
CodeKeyExists terror.ErrCode = 3
)
var (
// ErrClosed is used when close an already closed txn.
ErrClosed = errors.New("Error: Transaction already closed")
// ErrNotExist is used when try to get an entry with an unexist key from KV store.
ErrNotExist = errors.New("Error: key not exist")
// ErrConditionNotMatch is used when condition is not met.
ErrConditionNotMatch = errors.New("Error: Condition not match")
// ErrLockConflict is used when try to lock an already locked key.
ErrLockConflict = errors.New("Error: Lock conflict")
// ErrLazyConditionPairsNotMatch is used when value in store differs from expect pairs.
ErrLazyConditionPairsNotMatch = errors.New("Error: Lazy condition pairs not match")
// ErrRetryable is used when KV store occurs RPC error or some other
// errors which SQL layer can safely retry.
ErrRetryable = errors.New("Error: KV error safe to retry")
// ErrCannotSetNilValue is the error when sets an empty value.
ErrCannotSetNilValue = errors.New("can not set nil value")
// ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction.
ErrInvalidTxn = errors.New("invalid transaction")
// ErrNotCommitted is the error returned by CommitVersion when this
// transaction is not committed.
ErrNotCommitted = errors.New("this transaction has not committed")
// ErrKeyExists returns when key is already exist.
ErrKeyExists = terror.ClassKV.New(CodeKeyExists, "key already exist")
)
func init() {
kvMySQLErrCodes := map[terror.ErrCode]uint16{
CodeKeyExists: mysql.ErrDupEntry,
}
terror.ErrClassToMySQLCodes[terror.ClassKV] = kvMySQLErrCodes
}
// IsRetryableError checks if the err is a fatal error and the under going operation is worth to retry.
func IsRetryableError(err error) bool {
if err == nil {
return false
}
if terror.ErrorEqual(err, ErrRetryable) ||
terror.ErrorEqual(err, ErrLockConflict) ||
terror.ErrorEqual(err, ErrConditionNotMatch) ||
terror.ErrorEqual(err, themis.ErrRetryable) ||
// HBase exception message will tell you if you should retry or not
strings.Contains(err.Error(), "try again later") {
return true
}
return false
}
// IsErrNotFound checks if err is a kind of NotFound error.
func IsErrNotFound(err error) bool {
if terror.ErrorEqual(err, ErrNotExist) {
return true
}
return false
}

290
vendor/github.com/pingcap/tidb/kv/index_iter.go generated vendored Normal file
View file

@ -0,0 +1,290 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"bytes"
"encoding/binary"
"io"
"github.com/juju/errors"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/types"
)
var (
_ Index = (*kvIndex)(nil)
_ IndexIterator = (*indexIter)(nil)
)
// IndexIterator is the interface for iterator of index data on KV store.
type IndexIterator interface {
Next() (k []types.Datum, h int64, err error)
Close()
}
// Index is the interface for index data on KV store.
type Index interface {
// Create supports insert into statement.
Create(rm RetrieverMutator, indexedValues []types.Datum, h int64) error
// Delete supports delete from statement.
Delete(m Mutator, indexedValues []types.Datum, h int64) error
// Drop supports drop table, drop index statements.
Drop(rm RetrieverMutator) error
// Exist supports check index exists or not.
Exist(rm RetrieverMutator, indexedValues []types.Datum, h int64) (bool, int64, error)
// GenIndexKey generates an index key.
GenIndexKey(indexedValues []types.Datum, h int64) (key []byte, distinct bool, err error)
// Seek supports where clause.
Seek(r Retriever, indexedValues []types.Datum) (iter IndexIterator, hit bool, err error)
// SeekFirst supports aggregate min and ascend order by.
SeekFirst(r Retriever) (iter IndexIterator, err error)
}
func encodeHandle(h int64) []byte {
buf := &bytes.Buffer{}
err := binary.Write(buf, binary.BigEndian, h)
if err != nil {
panic(err)
}
return buf.Bytes()
}
func decodeHandle(data []byte) (int64, error) {
var h int64
buf := bytes.NewBuffer(data)
err := binary.Read(buf, binary.BigEndian, &h)
return h, errors.Trace(err)
}
// indexIter is for KV store index iterator.
type indexIter struct {
it Iterator
idx *kvIndex
prefix Key
}
// Close does the clean up works when KV store index iterator is closed.
func (c *indexIter) Close() {
if c.it != nil {
c.it.Close()
c.it = nil
}
}
// Next returns current key and moves iterator to the next step.
func (c *indexIter) Next() (val []types.Datum, h int64, err error) {
if !c.it.Valid() {
return nil, 0, errors.Trace(io.EOF)
}
if !c.it.Key().HasPrefix(c.prefix) {
return nil, 0, errors.Trace(io.EOF)
}
// get indexedValues
buf := c.it.Key()[len(c.prefix):]
vv, err := codec.Decode(buf)
if err != nil {
return nil, 0, errors.Trace(err)
}
// if index is *not* unique, the handle is in keybuf
if !c.idx.unique {
h = vv[len(vv)-1].GetInt64()
val = vv[0 : len(vv)-1]
} else {
// otherwise handle is value
h, err = decodeHandle(c.it.Value())
if err != nil {
return nil, 0, errors.Trace(err)
}
val = vv
}
// update new iter to next
err = c.it.Next()
if err != nil {
return nil, 0, errors.Trace(err)
}
return
}
// kvIndex is the data structure for index data in the KV store.
type kvIndex struct {
indexName string
indexID int64
unique bool
prefix Key
}
// GenIndexPrefix generates the index prefix.
func GenIndexPrefix(indexPrefix Key, indexID int64) Key {
buf := make([]byte, 0, len(indexPrefix)+8)
buf = append(buf, indexPrefix...)
buf = codec.EncodeInt(buf, indexID)
return buf
}
// NewKVIndex builds a new kvIndex object.
func NewKVIndex(indexPrefix Key, indexName string, indexID int64, unique bool) Index {
index := &kvIndex{
indexName: indexName,
indexID: indexID,
unique: unique,
prefix: GenIndexPrefix(indexPrefix, indexID),
}
return index
}
// GenIndexKey generates storage key for index values. Returned distinct indicates whether the
// indexed values should be distinct in storage (i.e. whether handle is encoded in the key).
func (c *kvIndex) GenIndexKey(indexedValues []types.Datum, h int64) (key []byte, distinct bool, err error) {
if c.unique {
// See: https://dev.mysql.com/doc/refman/5.7/en/create-index.html
// A UNIQUE index creates a constraint such that all values in the index must be distinct.
// An error occurs if you try to add a new row with a key value that matches an existing row.
// For all engines, a UNIQUE index permits multiple NULL values for columns that can contain NULL.
distinct = true
for _, cv := range indexedValues {
if cv.Kind() == types.KindNull {
distinct = false
break
}
}
}
key = append(key, c.prefix...)
if distinct {
key, err = codec.EncodeKey(key, indexedValues...)
} else {
key, err = codec.EncodeKey(key, append(indexedValues, types.NewDatum(h))...)
}
if err != nil {
return nil, false, errors.Trace(err)
}
return
}
// Create creates a new entry in the kvIndex data.
// If the index is unique and there is an existing entry with the same key, Create will return ErrKeyExists.
func (c *kvIndex) Create(rm RetrieverMutator, indexedValues []types.Datum, h int64) error {
key, distinct, err := c.GenIndexKey(indexedValues, h)
if err != nil {
return errors.Trace(err)
}
if !distinct {
// TODO: reconsider value
err = rm.Set(key, []byte("timestamp?"))
return errors.Trace(err)
}
_, err = rm.Get(key)
if IsErrNotFound(err) {
err = rm.Set(key, encodeHandle(h))
return errors.Trace(err)
}
return errors.Trace(ErrKeyExists)
}
// Delete removes the entry for handle h and indexdValues from KV index.
func (c *kvIndex) Delete(m Mutator, indexedValues []types.Datum, h int64) error {
key, _, err := c.GenIndexKey(indexedValues, h)
if err != nil {
return errors.Trace(err)
}
err = m.Delete(key)
return errors.Trace(err)
}
// Drop removes the KV index from store.
func (c *kvIndex) Drop(rm RetrieverMutator) error {
it, err := rm.Seek(c.prefix)
if err != nil {
return errors.Trace(err)
}
defer it.Close()
// remove all indices
for it.Valid() {
if !it.Key().HasPrefix(c.prefix) {
break
}
err := rm.Delete(it.Key())
if err != nil {
return errors.Trace(err)
}
err = it.Next()
if err != nil {
return errors.Trace(err)
}
}
return nil
}
// Seek searches KV index for the entry with indexedValues.
func (c *kvIndex) Seek(r Retriever, indexedValues []types.Datum) (iter IndexIterator, hit bool, err error) {
key, _, err := c.GenIndexKey(indexedValues, 0)
if err != nil {
return nil, false, errors.Trace(err)
}
it, err := r.Seek(key)
if err != nil {
return nil, false, errors.Trace(err)
}
// check if hit
hit = false
if it.Valid() && it.Key().Cmp(key) == 0 {
hit = true
}
return &indexIter{it: it, idx: c, prefix: c.prefix}, hit, nil
}
// SeekFirst returns an iterator which points to the first entry of the KV index.
func (c *kvIndex) SeekFirst(r Retriever) (iter IndexIterator, err error) {
it, err := r.Seek(c.prefix)
if err != nil {
return nil, errors.Trace(err)
}
return &indexIter{it: it, idx: c, prefix: c.prefix}, nil
}
func (c *kvIndex) Exist(rm RetrieverMutator, indexedValues []types.Datum, h int64) (bool, int64, error) {
key, distinct, err := c.GenIndexKey(indexedValues, h)
if err != nil {
return false, 0, errors.Trace(err)
}
value, err := rm.Get(key)
if IsErrNotFound(err) {
return false, 0, nil
}
if err != nil {
return false, 0, errors.Trace(err)
}
// For distinct index, the value of key is handle.
if distinct {
handle, err := decodeHandle(value)
if err != nil {
return false, 0, errors.Trace(err)
}
if handle != h {
return true, handle, errors.Trace(ErrKeyExists)
}
return true, handle, nil
}
return true, h, nil
}

29
vendor/github.com/pingcap/tidb/kv/iter.go generated vendored Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import "github.com/juju/errors"
// NextUntil applies FnKeyCmp to each entry of the iterator until meets some condition.
// It will stop when fn returns true, or iterator is invalid or an error occurs.
func NextUntil(it Iterator, fn FnKeyCmp) error {
var err error
for it.Valid() && !fn(it.Key()) {
err = it.Next()
if err != nil {
return errors.Trace(err)
}
}
return nil
}

57
vendor/github.com/pingcap/tidb/kv/key.go generated vendored Normal file
View file

@ -0,0 +1,57 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import "bytes"
// Key represents high-level Key type.
type Key []byte
// Next returns the next key in byte-order.
func (k Key) Next() Key {
// add 0x0 to the end of key
buf := make([]byte, len([]byte(k))+1)
copy(buf, []byte(k))
return buf
}
// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k Key) Cmp(another Key) int {
return bytes.Compare(k, another)
}
// HasPrefix tests whether the Key begins with prefix.
func (k Key) HasPrefix(prefix Key) bool {
return bytes.HasPrefix(k, prefix)
}
// Clone returns a copy of the Key.
func (k Key) Clone() Key {
return append([]byte(nil), k...)
}
// EncodedKey represents encoded key in low-level storage engine.
type EncodedKey []byte
// Cmp returns the comparison result of two key.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (k EncodedKey) Cmp(another EncodedKey) int {
return bytes.Compare(k, another)
}
// Next returns the next key in byte-order.
func (k EncodedKey) Next() EncodedKey {
return EncodedKey(bytes.Join([][]byte{k, Key{0}}, nil))
}

172
vendor/github.com/pingcap/tidb/kv/kv.go generated vendored Normal file
View file

@ -0,0 +1,172 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import "io"
const (
// PresumeKeyNotExists directives that when dealing with a Get operation but failing to read data from cache,
// we presume that the key does not exist in Store. The actual existence will be checked before the
// transaction's commit.
// This option is an optimization for frequent checks during a transaction, e.g. batch inserts.
PresumeKeyNotExists Option = iota + 1
// PresumeKeyNotExistsError is the option key for error.
// When PresumeKeyNotExists is set and condition is not match, should throw the error.
PresumeKeyNotExistsError
)
// Retriever is the interface wraps the basic Get and Seek methods.
type Retriever interface {
// Get gets the value for key k from kv store.
// If corresponding kv pair does not exist, it returns nil and ErrNotExist.
Get(k Key) ([]byte, error)
// Seek creates an Iterator positioned on the first entry that k <= entry's key.
// If such entry is not found, it returns an invalid Iterator with no error.
// The Iterator must be Closed after use.
Seek(k Key) (Iterator, error)
}
// Mutator is the interface wraps the basic Set and Delete methods.
type Mutator interface {
// Set sets the value for key k as v into kv store.
// v must NOT be nil or empty, otherwise it returns ErrCannotSetNilValue.
Set(k Key, v []byte) error
// Delete removes the entry for key k from kv store.
Delete(k Key) error
}
// RetrieverMutator is the interface that groups Retriever and Mutator interfaces.
type RetrieverMutator interface {
Retriever
Mutator
}
// MemBuffer is an in-memory kv collection. It should be released after use.
type MemBuffer interface {
RetrieverMutator
// Release releases the buffer.
Release()
}
// Transaction defines the interface for operations inside a Transaction.
// This is not thread safe.
type Transaction interface {
RetrieverMutator
// Commit commits the transaction operations to KV store.
Commit() error
// Rollback undoes the transaction operations to KV store.
Rollback() error
// String implements fmt.Stringer interface.
String() string
// LockKeys tries to lock the entries with the keys in KV store.
LockKeys(keys ...Key) error
// SetOption sets an option with a value, when val is nil, uses the default
// value of this option.
SetOption(opt Option, val interface{})
// DelOption deletes an option.
DelOption(opt Option)
// IsReadOnly checks if the transaction has only performed read operations.
IsReadOnly() bool
// GetClient gets a client instance.
GetClient() Client
// StartTS returns the transaction start timestamp.
StartTS() int64
}
// Client is used to send request to KV layer.
type Client interface {
// Send sends request to KV layer, returns a Response.
Send(req *Request) Response
// SupportRequestType checks if reqType and subType is supported.
SupportRequestType(reqType, subType int64) bool
}
// ReqTypes.
const (
ReqTypeSelect = 101
ReqTypeIndex = 102
)
// KeyRange represents a range where StartKey <= key < EndKey.
type KeyRange struct {
StartKey Key
EndKey Key
}
// Request represents a kv request.
type Request struct {
// The request type.
Tp int64
Data []byte
// Key Ranges
KeyRanges []KeyRange
// If desc is true, the request is sent in descending order.
Desc bool
// If concurrency is 1, it only sends the request to a single storage unit when
// ResponseIterator.Next is called. If concurrency is greater than 1, the request will be
// sent to multiple storage units concurrently.
Concurrency int
}
// Response represents the response returned from KV layer.
type Response interface {
// Next returns a resultSubset from a single storage unit.
// When full result set is returned, nil is returned.
Next() (resultSubset io.ReadCloser, err error)
}
// Snapshot defines the interface for the snapshot fetched from KV store.
type Snapshot interface {
Retriever
// BatchGet gets a batch of values from snapshot.
BatchGet(keys []Key) (map[string][]byte, error)
// Release releases the snapshot to store.
Release()
}
// Driver is the interface that must be implemented by a KV storage.
type Driver interface {
// Open returns a new Storage.
// The path is the string for storage specific format.
Open(path string) (Storage, error)
}
// Storage defines the interface for storage.
// Isolation should be at least SI(SNAPSHOT ISOLATION)
type Storage interface {
// Begin transaction
Begin() (Transaction, error)
// GetSnapshot gets a snapshot that is able to read any data which data is <= ver.
// if ver is MaxVersion or > current max committed version, we will use current version for this snapshot.
GetSnapshot(ver Version) (Snapshot, error)
// Close store
Close() error
// Storage's unique ID
UUID() string
// CurrentVersion returns current max committed version.
CurrentVersion() (Version, error)
}
// FnKeyCmp is the function for iterator the keys
type FnKeyCmp func(key Key) bool
// Iterator is the interface for a iterator on KV store.
type Iterator interface {
Valid() bool
Key() Key
Value() []byte
Next() error
Close()
}

106
vendor/github.com/pingcap/tidb/kv/memdb_buffer.go generated vendored Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2015 PingCAP, Inc.
//
// Copyright 2015 Wenbin Xiao
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"github.com/juju/errors"
"github.com/pingcap/tidb/terror"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/comparer"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/memdb"
"github.com/syndtr/goleveldb/leveldb/util"
)
type memDbBuffer struct {
db *memdb.DB
}
type memDbIter struct {
iter iterator.Iterator
}
// NewMemDbBuffer creates a new memDbBuffer.
func NewMemDbBuffer() MemBuffer {
return &memDbBuffer{db: memdb.New(comparer.DefaultComparer, 4*1024)}
}
// Seek creates an Iterator.
func (m *memDbBuffer) Seek(k Key) (Iterator, error) {
var i Iterator
if k == nil {
i = &memDbIter{iter: m.db.NewIterator(&util.Range{})}
} else {
i = &memDbIter{iter: m.db.NewIterator(&util.Range{Start: []byte(k)})}
}
i.Next()
return i, nil
}
// Get returns the value associated with key.
func (m *memDbBuffer) Get(k Key) ([]byte, error) {
v, err := m.db.Get(k)
if terror.ErrorEqual(err, leveldb.ErrNotFound) {
return nil, ErrNotExist
}
return v, nil
}
// Set associates key with value.
func (m *memDbBuffer) Set(k Key, v []byte) error {
if len(v) == 0 {
return errors.Trace(ErrCannotSetNilValue)
}
err := m.db.Put(k, v)
return errors.Trace(err)
}
// Delete removes the entry from buffer with provided key.
func (m *memDbBuffer) Delete(k Key) error {
err := m.db.Put(k, nil)
return errors.Trace(err)
}
// Release reset the buffer.
func (m *memDbBuffer) Release() {
m.db.Reset()
}
// Next implements the Iterator Next.
func (i *memDbIter) Next() error {
i.iter.Next()
return nil
}
// Valid implements the Iterator Valid.
func (i *memDbIter) Valid() bool {
return i.iter.Valid()
}
// Key implements the Iterator Key.
func (i *memDbIter) Key() Key {
return i.iter.Key()
}
// Value implements the Iterator Value.
func (i *memDbIter) Value() []byte {
return i.iter.Value()
}
// Close Implements the Iterator Close.
func (i *memDbIter) Close() {
i.iter.Release()
}

739
vendor/github.com/pingcap/tidb/kv/memkv/btree.go generated vendored Normal file
View file

@ -0,0 +1,739 @@
// Copyright 2013 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package memkv
import (
"io"
)
const (
kx = 128 //DONE benchmark tune this number if using custom key/value type(s).
kd = 64 //DONE benchmark tune this number if using custom key/value type(s).
)
type (
// cmp compares a and b. Return value is:
//
// < 0 if a < b
// 0 if a == b
// > 0 if a > b
//
cmp func(a, b []interface{}) int
d struct { // data page
c int
d [2*kd + 1]de
n *d
p *d
}
de struct { // d element
k []interface{}
v []interface{}
}
// Enumerator is the iterator for btree
Enumerator struct {
err error
hit bool
i int
k []interface{}
q *d
t *Tree
ver int64
}
// Tree is a B+tree.
Tree struct {
c int
cmp cmp
first *d
last *d
r interface{}
ver int64
}
xe struct { // x element
ch interface{}
sep *d
}
x struct { // index page
c int
x [2*kx + 2]xe
}
)
var ( // R/O zero values
zd d
zde de
zx x
zxe xe
)
func clr(q interface{}) {
switch z := q.(type) {
case *x:
for i := 0; i <= z.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn
clr(z.x[i].ch)
}
*z = zx // GC
case *d:
*z = zd // GC
}
}
// -------------------------------------------------------------------------- x
func newX(ch0 interface{}) *x {
r := &x{}
r.x[0].ch = ch0
return r
}
func (q *x) extract(i int) {
q.c--
if i < q.c {
copy(q.x[i:], q.x[i+1:q.c+1])
q.x[q.c].ch = q.x[q.c+1].ch
q.x[q.c].sep = nil // GC
q.x[q.c+1] = zxe // GC
}
}
func (q *x) insert(i int, d *d, ch interface{}) *x {
c := q.c
if i < c {
q.x[c+1].ch = q.x[c].ch
copy(q.x[i+2:], q.x[i+1:c])
q.x[i+1].sep = q.x[i].sep
}
c++
q.c = c
q.x[i].sep = d
q.x[i+1].ch = ch
return q
}
func (q *x) siblings(i int) (l, r *d) {
if i >= 0 {
if i > 0 {
l = q.x[i-1].ch.(*d)
}
if i < q.c {
r = q.x[i+1].ch.(*d)
}
}
return
}
// -------------------------------------------------------------------------- d
func (l *d) mvL(r *d, c int) {
copy(l.d[l.c:], r.d[:c])
copy(r.d[:], r.d[c:r.c])
l.c += c
r.c -= c
}
func (l *d) mvR(r *d, c int) {
copy(r.d[c:], r.d[:r.c])
copy(r.d[:c], l.d[l.c-c:])
r.c += c
l.c -= c
}
// ----------------------------------------------------------------------- tree
// NewTree returns a newly created, empty tree. The compare function is used
// for key collation.
func NewTree(cmp cmp) *Tree {
return &Tree{cmp: cmp}
}
// Clear removes all K/V pairs from the tree.
func (t *Tree) Clear() {
if t.r == nil {
return
}
clr(t.r)
t.c, t.first, t.last, t.r = 0, nil, nil, nil
t.ver++
}
func (t *Tree) cat(p *x, q, r *d, pi int) {
t.ver++
q.mvL(r, r.c)
if r.n != nil {
r.n.p = q
} else {
t.last = q
}
q.n = r.n
if p.c > 1 {
p.extract(pi)
p.x[pi].ch = q
} else {
t.r = q
}
}
func (t *Tree) catX(p, q, r *x, pi int) {
t.ver++
q.x[q.c].sep = p.x[pi].sep
copy(q.x[q.c+1:], r.x[:r.c])
q.c += r.c + 1
q.x[q.c].ch = r.x[r.c].ch
if p.c > 1 {
p.c--
pc := p.c
if pi < pc {
p.x[pi].sep = p.x[pi+1].sep
copy(p.x[pi+1:], p.x[pi+2:pc+1])
p.x[pc].ch = p.x[pc+1].ch
p.x[pc].sep = nil // GC
p.x[pc+1].ch = nil // GC
}
return
}
t.r = q
}
//Delete removes the k's KV pair, if it exists, in which case Delete returns
//true.
func (t *Tree) Delete(k []interface{}) (ok bool) {
pi := -1
var p *x
q := t.r
if q == nil {
return
}
for {
var i int
i, ok = t.find(q, k)
if ok {
switch z := q.(type) {
case *x:
dp := z.x[i].sep
switch {
case dp.c > kd:
t.extract(dp, 0)
default:
if z.c < kx && q != t.r {
t.underflowX(p, &z, pi, &i)
}
pi = i + 1
p = z
q = z.x[pi].ch
ok = false
continue
}
case *d:
t.extract(z, i)
if z.c >= kd {
return
}
if q != t.r {
t.underflow(p, z, pi)
} else if t.c == 0 {
t.Clear()
}
}
return
}
switch z := q.(type) {
case *x:
if z.c < kx && q != t.r {
t.underflowX(p, &z, pi, &i)
}
pi = i
p = z
q = z.x[i].ch
case *d:
return
}
}
}
func (t *Tree) extract(q *d, i int) { // (r []interface{}) {
t.ver++
//r = q.d[i].v // prepared for Extract
q.c--
if i < q.c {
copy(q.d[i:], q.d[i+1:q.c+1])
}
q.d[q.c] = zde // GC
t.c--
return
}
func (t *Tree) find(q interface{}, k []interface{}) (i int, ok bool) {
var mk []interface{}
l := 0
switch z := q.(type) {
case *x:
h := z.c - 1
for l <= h {
m := (l + h) >> 1
mk = z.x[m].sep.d[0].k
switch cmp := t.cmp(k, mk); {
case cmp > 0:
l = m + 1
case cmp == 0:
return m, true
default:
h = m - 1
}
}
case *d:
h := z.c - 1
for l <= h {
m := (l + h) >> 1
mk = z.d[m].k
switch cmp := t.cmp(k, mk); {
case cmp > 0:
l = m + 1
case cmp == 0:
return m, true
default:
h = m - 1
}
}
}
return l, false
}
// First returns the first item of the tree in the key collating order, or
// (nil, nil) if the tree is empty.
func (t *Tree) First() (k []interface{}, v []interface{}) {
if q := t.first; q != nil {
q := &q.d[0]
k, v = q.k, q.v
}
return
}
// Get returns the value associated with k and true if it exists. Otherwise Get
// returns (nil, false).
func (t *Tree) Get(k []interface{}) (v []interface{}, ok bool) {
q := t.r
if q == nil {
return
}
for {
var i int
if i, ok = t.find(q, k); ok {
switch z := q.(type) {
case *x:
return z.x[i].sep.d[0].v, true
case *d:
return z.d[i].v, true
}
}
switch z := q.(type) {
case *x:
q = z.x[i].ch
default:
return
}
}
}
func (t *Tree) insert(q *d, i int, k []interface{}, v []interface{}) *d {
t.ver++
c := q.c
if i < c {
copy(q.d[i+1:], q.d[i:c])
}
c++
q.c = c
q.d[i].k, q.d[i].v = k, v
t.c++
return q
}
// Last returns the last item of the tree in the key collating order, or (nil,
// nil) if the tree is empty.
func (t *Tree) Last() (k []interface{}, v []interface{}) {
if q := t.last; q != nil {
q := &q.d[q.c-1]
k, v = q.k, q.v
}
return
}
// Len returns the number of items in the tree.
func (t *Tree) Len() int {
return t.c
}
func (t *Tree) overflow(p *x, q *d, pi, i int, k []interface{}, v []interface{}) {
t.ver++
l, r := p.siblings(pi)
if l != nil && l.c < 2*kd {
l.mvL(q, 1)
t.insert(q, i-1, k, v)
return
}
if r != nil && r.c < 2*kd {
if i < 2*kd {
q.mvR(r, 1)
t.insert(q, i, k, v)
} else {
t.insert(r, 0, k, v)
}
return
}
t.split(p, q, pi, i, k, v)
}
// Seek returns an Enumerator positioned on a an item such that k >= item's
// key. ok reports if k == item.key The Enumerator's position is possibly
// after the last item in the tree.
func (t *Tree) Seek(k []interface{}) (e *Enumerator, ok bool) {
q := t.r
if q == nil {
e = &Enumerator{nil, false, 0, k, nil, t, t.ver}
return
}
for {
var i int
if i, ok = t.find(q, k); ok {
switch z := q.(type) {
case *x:
e = &Enumerator{nil, ok, 0, k, z.x[i].sep, t, t.ver}
return
case *d:
e = &Enumerator{nil, ok, i, k, z, t, t.ver}
return
}
}
switch z := q.(type) {
case *x:
q = z.x[i].ch
case *d:
e = &Enumerator{nil, ok, i, k, z, t, t.ver}
return
}
}
}
// SeekFirst returns an Enumerator positioned on the first KV pair in the tree,
// if any. For an empty tree, err == io.EOF is returned and e will be nil.
func (t *Tree) SeekFirst() (e *Enumerator, err error) {
q := t.first
if q == nil {
return nil, io.EOF
}
return &Enumerator{nil, true, 0, q.d[0].k, q, t, t.ver}, nil
}
// SeekLast returns an Enumerator positioned on the last KV pair in the tree,
// if any. For an empty tree, err == io.EOF is returned and e will be nil.
func (t *Tree) SeekLast() (e *Enumerator, err error) {
q := t.last
if q == nil {
return nil, io.EOF
}
return &Enumerator{nil, true, q.c - 1, q.d[q.c-1].k, q, t, t.ver}, nil
}
// Set sets the value associated with k.
func (t *Tree) Set(k []interface{}, v []interface{}) {
pi := -1
var p *x
q := t.r
if q != nil {
for {
i, ok := t.find(q, k)
if ok {
switch z := q.(type) {
case *x:
z.x[i].sep.d[0].v = v
case *d:
z.d[i].v = v
}
return
}
switch z := q.(type) {
case *x:
if z.c > 2*kx {
t.splitX(p, &z, pi, &i)
}
pi = i
p = z
q = z.x[i].ch
case *d:
switch {
case z.c < 2*kd:
t.insert(z, i, k, v)
default:
t.overflow(p, z, pi, i, k, v)
}
return
}
}
}
z := t.insert(&d{}, 0, k, v)
t.r, t.first, t.last = z, z, z
return
}
func (t *Tree) split(p *x, q *d, pi, i int, k []interface{}, v []interface{}) {
t.ver++
r := &d{}
if q.n != nil {
r.n = q.n
r.n.p = r
} else {
t.last = r
}
q.n = r
r.p = q
copy(r.d[:], q.d[kd:2*kd])
for i := range q.d[kd:] {
q.d[kd+i] = zde
}
q.c = kd
r.c = kd
if pi >= 0 {
p.insert(pi, r, r)
} else {
t.r = newX(q).insert(0, r, r)
}
if i > kd {
t.insert(r, i-kd, k, v)
return
}
t.insert(q, i, k, v)
}
func (t *Tree) splitX(p *x, pp **x, pi int, i *int) {
t.ver++
q := *pp
r := &x{}
copy(r.x[:], q.x[kx+1:])
q.c = kx
r.c = kx
if pi >= 0 {
p.insert(pi, q.x[kx].sep, r)
} else {
t.r = newX(q).insert(0, q.x[kx].sep, r)
}
q.x[kx].sep = nil
for i := range q.x[kx+1:] {
q.x[kx+i+1] = zxe
}
if *i > kx {
*pp = r
*i -= kx + 1
}
}
func (t *Tree) underflow(p *x, q *d, pi int) {
t.ver++
l, r := p.siblings(pi)
if l != nil && l.c+q.c >= 2*kd {
l.mvR(q, 1)
} else if r != nil && q.c+r.c >= 2*kd {
q.mvL(r, 1)
r.d[r.c] = zde // GC
} else if l != nil {
t.cat(p, l, q, pi-1)
} else {
t.cat(p, q, r, pi)
}
}
func (t *Tree) underflowX(p *x, pp **x, pi int, i *int) {
t.ver++
var l, r *x
q := *pp
if pi >= 0 {
if pi > 0 {
l = p.x[pi-1].ch.(*x)
}
if pi < p.c {
r = p.x[pi+1].ch.(*x)
}
}
if l != nil && l.c > kx {
q.x[q.c+1].ch = q.x[q.c].ch
copy(q.x[1:], q.x[:q.c])
q.x[0].ch = l.x[l.c].ch
q.x[0].sep = p.x[pi-1].sep
q.c++
*i++
l.c--
p.x[pi-1].sep = l.x[l.c].sep
return
}
if r != nil && r.c > kx {
q.x[q.c].sep = p.x[pi].sep
q.c++
q.x[q.c].ch = r.x[0].ch
p.x[pi].sep = r.x[0].sep
copy(r.x[:], r.x[1:r.c])
r.c--
rc := r.c
r.x[rc].ch = r.x[rc+1].ch
r.x[rc].sep = nil
r.x[rc+1].ch = nil
return
}
if l != nil {
*i += l.c + 1
t.catX(p, l, q, pi-1)
*pp = l
return
}
t.catX(p, q, r, pi)
}
// ----------------------------------------------------------------- Enumerator
// Next returns the currently enumerated item, if it exists and moves to the
// next item in the key collation order. If there is no item to return, err ==
// io.EOF is returned.
func (e *Enumerator) Next() (k []interface{}, v []interface{}, err error) {
if err = e.err; err != nil {
return
}
if e.ver != e.t.ver {
f, hit := e.t.Seek(e.k)
if !e.hit && hit {
if err = f.next(); err != nil {
return
}
}
*e = *f
}
if e.q == nil {
e.err, err = io.EOF, io.EOF
return
}
if e.i >= e.q.c {
if err = e.next(); err != nil {
return
}
}
i := e.q.d[e.i]
k, v = i.k, i.v
e.k, e.hit = k, false
e.next()
return
}
func (e *Enumerator) next() error {
if e.q == nil {
e.err = io.EOF
return io.EOF
}
switch {
case e.i < e.q.c-1:
e.i++
default:
if e.q, e.i = e.q.n, 0; e.q == nil {
e.err = io.EOF
}
}
return e.err
}
// Prev returns the currently enumerated item, if it exists and moves to the
// previous item in the key collation order. If there is no item to return, err
// == io.EOF is returned.
func (e *Enumerator) Prev() (k []interface{}, v []interface{}, err error) {
if err = e.err; err != nil {
return
}
if e.ver != e.t.ver {
f, hit := e.t.Seek(e.k)
if !e.hit && hit {
if err = f.prev(); err != nil {
return
}
}
*e = *f
}
if e.q == nil {
e.err, err = io.EOF, io.EOF
return
}
if e.i >= e.q.c {
if err = e.next(); err != nil {
return
}
}
i := e.q.d[e.i]
k, v = i.k, i.v
e.k, e.hit = k, false
e.prev()
return
}
func (e *Enumerator) prev() error {
if e.q == nil {
e.err = io.EOF
return io.EOF
}
switch {
case e.i > 0:
e.i--
default:
if e.q = e.q.p; e.q == nil {
e.err = io.EOF
break
}
e.i = e.q.c - 1
}
return e.err
}

72
vendor/github.com/pingcap/tidb/kv/memkv/temp.go generated vendored Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2013 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSES/QL-LICENSE file.
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package memkv
import (
"github.com/pingcap/tidb/util/types"
)
type btreeIterator interface {
Next() (k, v []interface{}, err error)
}
// Temp is the interface of a memory kv storage
type Temp interface {
Drop() (err error)
Get(k []interface{}) (v []interface{}, err error)
SeekFirst() (e btreeIterator, err error)
Set(k, v []interface{}) (err error)
}
// memtemp for join/groupby or any aggregation operation
type memTemp struct {
// memory btree
tree *Tree
}
// CreateTemp returns a new empty memory kv
func CreateTemp(asc bool) (_ Temp, err error) {
return &memTemp{
tree: NewTree(types.Collators[asc]),
}, nil
}
func (t *memTemp) Get(k []interface{}) (v []interface{}, err error) {
v, _ = t.tree.Get(k)
return
}
func (t *memTemp) Drop() (err error) { return }
func (t *memTemp) Set(k, v []interface{}) (err error) {
vv, err := types.Clone(v)
if err != nil {
return err
}
t.tree.Set(append([]interface{}(nil), k...), vv.([]interface{}))
return
}
func (t *memTemp) SeekFirst() (e btreeIterator, err error) {
it, err := t.tree.SeekFirst()
if err != nil {
return
}
return it, nil
}

78
vendor/github.com/pingcap/tidb/kv/txn.go generated vendored Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"math"
"math/rand"
"time"
"github.com/juju/errors"
"github.com/ngaut/log"
)
// RunInNewTxn will run the f in a new transaction environment.
func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error {
for i := 0; i < maxRetryCnt; i++ {
txn, err := store.Begin()
if err != nil {
log.Errorf("[kv] RunInNewTxn error - %v", err)
return errors.Trace(err)
}
err = f(txn)
if retryable && IsRetryableError(err) {
log.Warnf("[kv] Retry txn %v", txn)
txn.Rollback()
continue
}
if err != nil {
txn.Rollback()
return errors.Trace(err)
}
err = txn.Commit()
if retryable && IsRetryableError(err) {
log.Warnf("[kv] Retry txn %v", txn)
txn.Rollback()
BackOff(i)
continue
}
if err != nil {
return errors.Trace(err)
}
break
}
return nil
}
var (
// Max retry count in RunInNewTxn
maxRetryCnt = 100
// retryBackOffBase is the initial duration, in microsecond, a failed transaction stays dormancy before it retries
retryBackOffBase = 1
// retryBackOffCap is the max amount of duration, in microsecond, a failed transaction stays dormancy before it retries
retryBackOffCap = 100
)
// BackOff Implements exponential backoff with full jitter.
// Returns real back off time in microsecond.
// See: http://www.awsarchitectureblog.com/2015/03/backoff.html.
func BackOff(attempts int) int {
upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts))))
sleep := time.Duration(rand.Intn(upper)) * time.Millisecond
time.Sleep(sleep)
return int(sleep)
}

156
vendor/github.com/pingcap/tidb/kv/union_iter.go generated vendored Normal file
View file

@ -0,0 +1,156 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import "github.com/ngaut/log"
// UnionIter is the iterator on an UnionStore.
type UnionIter struct {
dirtyIt Iterator
snapshotIt Iterator
dirtyValid bool
snapshotValid bool
curIsDirty bool
isValid bool
}
func newUnionIter(dirtyIt Iterator, snapshotIt Iterator) *UnionIter {
it := &UnionIter{
dirtyIt: dirtyIt,
snapshotIt: snapshotIt,
dirtyValid: dirtyIt.Valid(),
snapshotValid: snapshotIt.Valid(),
}
it.updateCur()
return it
}
// Go next and update valid status.
func (iter *UnionIter) dirtyNext() {
iter.dirtyIt.Next()
iter.dirtyValid = iter.dirtyIt.Valid()
}
// Go next and update valid status.
func (iter *UnionIter) snapshotNext() {
iter.snapshotIt.Next()
iter.snapshotValid = iter.snapshotIt.Valid()
}
func (iter *UnionIter) updateCur() {
iter.isValid = true
for {
if !iter.dirtyValid && !iter.snapshotValid {
iter.isValid = false
return
}
if !iter.dirtyValid {
iter.curIsDirty = false
return
}
if !iter.snapshotValid {
iter.curIsDirty = true
// if delete it
if len(iter.dirtyIt.Value()) == 0 {
iter.dirtyNext()
continue
}
break
}
// both valid
if iter.snapshotValid && iter.dirtyValid {
snapshotKey := iter.snapshotIt.Key()
dirtyKey := iter.dirtyIt.Key()
cmp := dirtyKey.Cmp(snapshotKey)
// if equal, means both have value
if cmp == 0 {
if len(iter.dirtyIt.Value()) == 0 {
// snapshot has a record, but txn says we have deleted it
// just go next
iter.dirtyNext()
iter.snapshotNext()
continue
}
// both go next
iter.snapshotNext()
iter.curIsDirty = true
break
} else if cmp > 0 {
// record from snapshot comes first
iter.curIsDirty = false
break
} else {
// record from dirty comes first
if len(iter.dirtyIt.Value()) == 0 {
log.Warnf("[kv] delete a record not exists? k = %q", iter.dirtyIt.Key())
// jump over this deletion
iter.dirtyNext()
continue
}
iter.curIsDirty = true
break
}
}
}
}
// Next implements the Iterator Next interface.
func (iter *UnionIter) Next() error {
if !iter.curIsDirty {
iter.snapshotNext()
} else {
iter.dirtyNext()
}
iter.updateCur()
return nil
}
// Value implements the Iterator Value interface.
// Multi columns
func (iter *UnionIter) Value() []byte {
if !iter.curIsDirty {
return iter.snapshotIt.Value()
}
return iter.dirtyIt.Value()
}
// Key implements the Iterator Key interface.
func (iter *UnionIter) Key() Key {
if !iter.curIsDirty {
return iter.snapshotIt.Key()
}
return iter.dirtyIt.Key()
}
// Valid implements the Iterator Valid interface.
func (iter *UnionIter) Valid() bool {
return iter.isValid
}
// Close implements the Iterator Close interface.
func (iter *UnionIter) Close() {
if iter.snapshotIt != nil {
iter.snapshotIt.Close()
iter.snapshotIt = nil
}
if iter.dirtyIt != nil {
iter.dirtyIt.Close()
iter.dirtyIt = nil
}
}

212
vendor/github.com/pingcap/tidb/kv/union_store.go generated vendored Normal file
View file

@ -0,0 +1,212 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"bytes"
"github.com/juju/errors"
)
// UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write.
// Also, it provides some transaction related utilities.
type UnionStore interface {
MemBuffer
// CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched.
// Lazy condition pairs should be checked before transaction commit.
CheckLazyConditionPairs() error
// WalkBuffer iterates all buffered kv pairs.
WalkBuffer(f func(k Key, v []byte) error) error
// SetOption sets an option with a value, when val is nil, uses the default
// value of this option.
SetOption(opt Option, val interface{})
// DelOption deletes an option.
DelOption(opt Option)
}
// Option is used for customizing kv store's behaviors during a transaction.
type Option int
// Options is an interface of a set of options. Each option is associated with a value.
type Options interface {
// Get gets an option value.
Get(opt Option) (v interface{}, ok bool)
}
var (
p = newCache("memdb pool", 100, func() MemBuffer {
return NewMemDbBuffer()
})
)
// conditionPair is used to store lazy check condition.
// If condition not match (value is not equal as expected one), returns err.
type conditionPair struct {
key Key
value []byte
err error
}
// UnionStore is an in-memory Store which contains a buffer for write and a
// snapshot for read.
type unionStore struct {
*BufferStore
snapshot Snapshot // for read
lazyConditionPairs map[string](*conditionPair) // for delay check
opts options
}
// NewUnionStore builds a new UnionStore.
func NewUnionStore(snapshot Snapshot) UnionStore {
return &unionStore{
BufferStore: NewBufferStore(snapshot),
snapshot: snapshot,
lazyConditionPairs: make(map[string](*conditionPair)),
opts: make(map[Option]interface{}),
}
}
type lazyMemBuffer struct {
mb MemBuffer
}
func (lmb *lazyMemBuffer) Get(k Key) ([]byte, error) {
if lmb.mb == nil {
return nil, ErrNotExist
}
return lmb.mb.Get(k)
}
func (lmb *lazyMemBuffer) Set(key Key, value []byte) error {
if lmb.mb == nil {
lmb.mb = p.get()
}
return lmb.mb.Set(key, value)
}
func (lmb *lazyMemBuffer) Delete(k Key) error {
if lmb.mb == nil {
lmb.mb = p.get()
}
return lmb.mb.Delete(k)
}
func (lmb *lazyMemBuffer) Seek(k Key) (Iterator, error) {
if lmb.mb == nil {
lmb.mb = p.get()
}
return lmb.mb.Seek(k)
}
func (lmb *lazyMemBuffer) Release() {
if lmb.mb == nil {
return
}
lmb.mb.Release()
p.put(lmb.mb)
lmb.mb = nil
}
// Get implements the Retriever interface.
func (us *unionStore) Get(k Key) ([]byte, error) {
v, err := us.MemBuffer.Get(k)
if IsErrNotFound(err) {
if _, ok := us.opts.Get(PresumeKeyNotExists); ok {
e, ok := us.opts.Get(PresumeKeyNotExistsError)
if ok && e != nil {
us.markLazyConditionPair(k, nil, e.(error))
} else {
us.markLazyConditionPair(k, nil, ErrKeyExists)
}
return nil, errors.Trace(ErrNotExist)
}
}
if IsErrNotFound(err) {
v, err = us.BufferStore.r.Get(k)
}
if err != nil {
return v, errors.Trace(err)
}
if len(v) == 0 {
return nil, errors.Trace(ErrNotExist)
}
return v, nil
}
// markLazyConditionPair marks a kv pair for later check.
// If condition not match, should return e as error.
func (us *unionStore) markLazyConditionPair(k Key, v []byte, e error) {
us.lazyConditionPairs[string(k)] = &conditionPair{
key: k.Clone(),
value: v,
err: e,
}
}
// CheckLazyConditionPairs implements the UnionStore interface.
func (us *unionStore) CheckLazyConditionPairs() error {
if len(us.lazyConditionPairs) == 0 {
return nil
}
keys := make([]Key, 0, len(us.lazyConditionPairs))
for _, v := range us.lazyConditionPairs {
keys = append(keys, v.key)
}
values, err := us.snapshot.BatchGet(keys)
if err != nil {
return errors.Trace(err)
}
for k, v := range us.lazyConditionPairs {
if len(v.value) == 0 {
if _, exist := values[k]; exist {
return errors.Trace(v.err)
}
} else {
if bytes.Compare(values[k], v.value) != 0 {
return errors.Trace(ErrLazyConditionPairsNotMatch)
}
}
}
return nil
}
// SetOption implements the UnionStore SetOption interface.
func (us *unionStore) SetOption(opt Option, val interface{}) {
us.opts[opt] = val
}
// DelOption implements the UnionStore DelOption interface.
func (us *unionStore) DelOption(opt Option) {
delete(us.opts, opt)
}
// Release implements the UnionStore Release interface.
func (us *unionStore) Release() {
us.snapshot.Release()
us.BufferStore.Release()
}
type options map[Option]interface{}
func (opts options) Get(opt Option) (interface{}, bool) {
v, ok := opts[opt]
return v, ok
}

60
vendor/github.com/pingcap/tidb/kv/utils.go generated vendored Normal file
View file

@ -0,0 +1,60 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import (
"strconv"
"github.com/juju/errors"
)
// IncInt64 increases the value for key k in kv store by step.
func IncInt64(rm RetrieverMutator, k Key, step int64) (int64, error) {
val, err := rm.Get(k)
if IsErrNotFound(err) {
err = rm.Set(k, []byte(strconv.FormatInt(step, 10)))
if err != nil {
return 0, errors.Trace(err)
}
return step, nil
}
if err != nil {
return 0, errors.Trace(err)
}
intVal, err := strconv.ParseInt(string(val), 10, 0)
if err != nil {
return 0, errors.Trace(err)
}
intVal += step
err = rm.Set(k, []byte(strconv.FormatInt(intVal, 10)))
if err != nil {
return 0, errors.Trace(err)
}
return intVal, nil
}
// GetInt64 get int64 value which created by IncInt64 method.
func GetInt64(r Retriever, k Key) (int64, error) {
val, err := r.Get(k)
if IsErrNotFound(err) {
return 0, nil
}
if err != nil {
return 0, errors.Trace(err)
}
intVal, err := strconv.ParseInt(string(val), 10, 0)
return intVal, errors.Trace(err)
}

51
vendor/github.com/pingcap/tidb/kv/version.go generated vendored Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package kv
import "math"
// VersionProvider provides increasing IDs.
type VersionProvider interface {
CurrentVersion() (Version, error)
}
// Version is the wrapper of KV's version.
type Version struct {
Ver uint64
}
var (
// MaxVersion is the maximum version, notice that it's not a valid version.
MaxVersion = Version{Ver: math.MaxUint64}
// MinVersion is the minimum version, it's not a valid version, too.
MinVersion = Version{Ver: 0}
)
// NewVersion creates a new Version struct.
func NewVersion(v uint64) Version {
return Version{
Ver: v,
}
}
// Cmp returns the comparison result of two versions.
// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
func (v Version) Cmp(another Version) int {
if v.Ver > another.Ver {
return 1
} else if v.Ver < another.Ver {
return -1
}
return 0
}