diff --git a/broker/service/fdb_store.go b/broker/service/fdb_store.go index 19c799223..5ff2ea6dc 100755 --- a/broker/service/fdb_store.go +++ b/broker/service/fdb_store.go @@ -58,7 +58,7 @@ const ( /*------------------------------Storage interface implemented------------------------*/ func (f *FdbStore) Init() { - fdb.MustAPIVersion(510) + fdb.MustAPIVersion(600) f.dbs = make([]*database, f.bk.conf.Store.FDB.Threads) f.pubchs = make([](chan []*proto.PubMsg), f.bk.conf.Store.FDB.Threads) @@ -378,7 +378,7 @@ func put(d *database, msgs []*proto.PubMsg) { } func (f *FdbStore) process(i int) { - fdb.MustAPIVersion(510) + fdb.MustAPIVersion(600) db := fdb.MustOpenDefault() dir, err := directory.CreateOrOpen(db, []string{f.bk.conf.Store.FDB.Namespace}, nil) if err != nil { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/README.md b/vendor/github.com/apple/foundationdb/bindings/go/README.md index 15beef47e..5c412da3e 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/README.md +++ b/vendor/github.com/apple/foundationdb/bindings/go/README.md @@ -1,14 +1,15 @@ fdb-go ====== -[Go language](http://golang.org) bindings for [FoundationDB](https://www.foundationdb.org/documentation/), a distributed key-value store with ACID transactions. +[Go language](http://golang.org) bindings for [FoundationDB](https://apple.github.io/foundationdb/index.html#documentation), a distributed key-value store with ACID transactions. This package requires: - Go 1.1+ with CGO enabled -- FoundationDB C API 2.0.x, 3.0.x, or 4.x.y (part of the [FoundationDB clients package](https://www.foundationdb.org/downloads/fdb-c/)) +- [Mono](http://www.mono-project.com/) (macOS or Windows) or [Visual Studio](https://www.visualstudio.com/) (Windows) (build-time only) +- FoundationDB C API 2.0.x-6.0.x (part of the [FoundationDB clients package](https://apple.github.io/foundationdb/downloads.html#c)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-510. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-600. To build this package, in the top level of this repository run: @@ -28,4 +29,4 @@ Documentation ------------- * [API documentation](https://godoc.org/github.com/apple/foundationdb/bindings/go/src/fdb) -* [Tutorial](https://www.foundationdb.org/documentation/class-scheduling-go.html) +* [Tutorial](https://apple.github.io/foundationdb/class-scheduling.html) diff --git a/vendor/github.com/apple/foundationdb/bindings/go/fdb-go-install.sh b/vendor/github.com/apple/foundationdb/bindings/go/fdb-go-install.sh index a18da3248..56591dce8 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/fdb-go-install.sh +++ b/vendor/github.com/apple/foundationdb/bindings/go/fdb-go-install.sh @@ -23,8 +23,32 @@ if [[ "${platform}" == "Darwin" ]] ; then FDBLIBDIR="${FDBLIBDIR:-/usr/local/lib}" libfdbc="libfdb_c.dylib" elif [[ "${platform}" == "Linux" ]] ; then - FDBLIBDIR="${FDBLIBDIR:-/usr/lib}" libfdbc="libfdb_c.so" + custom_libdir="${FDBLIBDIR:-}" + FDBLIBDIR="" + + if [[ -z "${custom_libdir}" ]]; then + search_libdirs=( '/usr/lib' '/usr/lib64' ) + else + search_libdirs=( "${custom_libdir}" ) + fi + + for libdir in "${search_libdirs[@]}" ; do + if [[ -e "${libdir}/${libfdbc}" ]]; then + FDBLIBDIR="${libdir}" + break + fi + done + + if [[ -z "${FDBLIBDIR}" ]]; then + echo "The FoundationDB C library could not be found in any of:" + for libdir in "${search_libdirs[@]}" ; do + echo " ${libdir}" + done + echo "Your installation may be incomplete, or you need to set a custom FDBLIBDIR." + let status="${status} + 1" + fi + else echo "Unsupported platform ${platform}". echo "At the moment, only macOS and Linux are supported by this script." @@ -167,7 +191,7 @@ else if [[ "${status}" -eq 0 ]] ; then destdir=$( cd "${destdir}" && pwd ) # Get absolute path of destination dir. - fdbdir="${destdir}/foundation" + fdbdir="${destdir}/foundationdb" if [[ ! -d "${destdir}" ]] ; then cmd=("mkdir" "-p" "${destdir}") @@ -197,7 +221,7 @@ else fi else echo "Downloading foundation repository into ${destdir}:" - cmd=( 'git' '-C' "${destdir}" 'clone' '--branch' "release-${FDBVER}" "git@${REMOTE}:${FDBREPO}.git" ) + cmd=( 'git' '-C' "${destdir}" 'clone' '--branch' "release-${FDBVER}" "https://${REMOTE}/${FDBREPO}.git" ) echo "${cmd[*]}" if ! "${cmd[@]}" ; then @@ -272,17 +296,12 @@ else # Do not install if only downloading : elif [[ "${status}" -eq 0 ]] ; then - cgo_cflags="-g -O2 -I${linkpath}/bindings/c" - cgo_ldflags="-g -O2 -L${FDBLIBDIR}" + cgo_cppflags="-I${linkpath}/bindings/c" + cgo_cflags="-g -O2" + cgo_ldflags="-L${FDBLIBDIR}" fdb_go_path="${REMOTE}/${FDBREPO}/bindings/go/src" - if [[ ! -e "${FDBLIBDIR}/${libfdbc}" ]] ; then - # Just a warning. Don't fail script. - echo - echo "WARNING: The FoundationDB C library was not found within ${FDBLIBDIR}." - echo "Your installation may be incomplete." - echo - elif ! CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" go install "${fdb_go_path}/fdb" "${fdb_go_path}/fdb/tuple" "${fdb_go_path}/fdb/subspace" "${fdb_go_path}/fdb/directory" ; then + if ! CGO_CPPFLAGS="${cgo_cppflags}" CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" go install "${fdb_go_path}/fdb" "${fdb_go_path}/fdb/tuple" "${fdb_go_path}/fdb/subspace" "${fdb_go_path}/fdb/directory" ; then let status="${status} + 1" echo "Could not build FoundationDB go libraries." fi @@ -295,6 +314,7 @@ else echo "The FoundationDB go bindings were successfully installed." echo "To build packages which use the go bindings, you will need to" echo "set the following environment variables:" + echo " CGO_CPPFLAGS=\"${cgo_cppflags}\"" echo " CGO_CFLAGS=\"${cgo_cflags}\"" echo " CGO_LDFLAGS=\"${cgo_ldflags}\"" fi diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/_stacktester/stacktester.go b/vendor/github.com/apple/foundationdb/bindings/go/src/_stacktester/stacktester.go index f1c17c0f5..aff3059c2 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/_stacktester/stacktester.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/_stacktester/stacktester.go @@ -28,6 +28,7 @@ import ( "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" "log" + "math/big" "os" "reflect" "runtime" @@ -103,7 +104,7 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { switch el := ret.item.(type) { case []byte: ret.item = el - case int64, string, bool, tuple.UUID, float32, float64, tuple.Tuple: + case int64, uint64, *big.Int, string, bool, tuple.UUID, float32, float64, tuple.Tuple: ret.item = el case fdb.Key: ret.item = []byte(el) @@ -174,8 +175,10 @@ func tupleToString(t tuple.Tuple) string { buffer.WriteString(", ") } switch el := el.(type) { - case int64: + case int64, uint64: buffer.WriteString(fmt.Sprintf("%d", el)) + case *big.Int: + buffer.WriteString(fmt.Sprintf("%s", el)) case []byte: buffer.WriteString(fmt.Sprintf("%+q", string(el))) case string: @@ -184,9 +187,7 @@ func tupleToString(t tuple.Tuple) string { buffer.WriteString(fmt.Sprintf("%t", el)) case tuple.UUID: buffer.WriteString(hex.EncodeToString(el[:])) - case float32: - buffer.WriteString(fmt.Sprintf("%f", el)) - case float64: + case float32, float64: buffer.WriteString(fmt.Sprintf("%f", el)) case nil: buffer.WriteString("nil") @@ -205,8 +206,10 @@ func (sm *StackMachine) dumpStack() { fmt.Printf(" %d.", sm.stack[i].idx) el := sm.stack[i].item switch el := el.(type) { - case int64: + case int64, uint64: fmt.Printf(" %d", el) + case *big.Int: + fmt.Printf(" %s", el) case fdb.FutureNil: fmt.Printf(" FutureNil") case fdb.FutureByteSlice: @@ -225,9 +228,7 @@ func (sm *StackMachine) dumpStack() { fmt.Printf(" %s", tupleToString(el)) case tuple.UUID: fmt.Printf(" %s", hex.EncodeToString(el[:])) - case float32: - fmt.Printf(" %f", el) - case float64: + case float32, float64: fmt.Printf(" %f", el) case nil: fmt.Printf(" nil") @@ -490,7 +491,27 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { case op == "POP": sm.stack = sm.stack[:len(sm.stack)-1] case op == "SUB": - sm.store(idx, sm.waitAndPop().item.(int64)-sm.waitAndPop().item.(int64)) + var x, y *big.Int + switch x1 := sm.waitAndPop().item.(type) { + case *big.Int: + x = x1 + case int64: + x = big.NewInt(x1) + case uint64: + x = new(big.Int) + x.SetUint64(x1) + } + switch y1 := sm.waitAndPop().item.(type) { + case *big.Int: + y = y1 + case int64: + y = big.NewInt(y1) + case uint64: + y = new(big.Int) + y.SetUint64(y1) + } + + sm.store(idx, x.Sub(x, y)) case op == "CONCAT": str1 := sm.waitAndPop().item str2 := sm.waitAndPop().item @@ -752,16 +773,41 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { db.Options().SetLocationCacheSize(100001) db.Options().SetMaxWatches(10001) + if !fdb.IsAPIVersionSelected() { + log.Fatal("API version should be selected") + } + apiVersion := fdb.MustGetAPIVersion() + if apiVersion == 0 { + log.Fatal("API version is 0") + } + e1 := fdb.APIVersion(apiVersion + 1) + if e1 != nil { + fdbE := e1.(fdb.Error) + if fdbE.Code != 2201 { + panic(e1) + } + } else { + log.Fatal("Was not stopped from selecting two API versions") + } + e2 := fdb.APIVersion(apiVersion - 1) + if e2 != nil { + fdbE := e2.(fdb.Error) + if fdbE.Code != 2201 { + panic(e2) + } + } else { + log.Fatal("Was not stopped from selecting two API versions") + } + fdb.MustAPIVersion(apiVersion) + _, e := db.Transact(func(tr fdb.Transaction) (interface{}, error) { tr.Options().SetPrioritySystemImmediate() tr.Options().SetPriorityBatch() tr.Options().SetCausalReadRisky() tr.Options().SetCausalWriteRisky() tr.Options().SetReadYourWritesDisable() - tr.Options().SetReadAheadDisable() tr.Options().SetReadSystemKeys() tr.Options().SetAccessSystemKeys() - tr.Options().SetDurabilityDevNullIsWebScale() tr.Options().SetTimeout(60 * 1000) tr.Options().SetRetryLimit(50) tr.Options().SetMaxRetryDelay(100) @@ -835,10 +881,17 @@ func main() { log.Fatal(e) } + if fdb.IsAPIVersionSelected() { + log.Fatal("API version already selected") + } + e = fdb.APIVersion(apiVersion) if e != nil { log.Fatal(e) } + if fdb.MustGetAPIVersion() != apiVersion { + log.Fatal("API version not equal to value selected") + } db, e = fdb.Open(clusterFile, []byte("DB")) if e != nil { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/_util/translate_fdb_options.go b/vendor/github.com/apple/foundationdb/bindings/go/src/_util/translate_fdb_options.go index 2f96d5e49..37d64af6c 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/_util/translate_fdb_options.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/_util/translate_fdb_options.go @@ -30,8 +30,6 @@ import ( "log" "os" "strings" - "unicode" - "unicode/utf8" ) type Option struct { @@ -114,23 +112,14 @@ func translateName(old string) string { return strings.Replace(strings.Title(strings.Replace(old, "_", " ", -1)), " ", "", -1) } -func lowerFirst(s string) string { - if s == "" { - return "" - } - r, n := utf8.DecodeRuneInString(s) - return string(unicode.ToLower(r)) + s[n:] -} - func writeMutation(opt Option) { - desc := lowerFirst(opt.Description) tname := translateName(opt.Name) fmt.Printf(` -// %s %s +// %s func (t Transaction) %s(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, %d) } -`, tname, desc, tname, opt.Code) +`, opt.Description, tname, opt.Code) } func writeEnum(scope Scope, opt Option, delta int) { @@ -207,7 +196,7 @@ func int64ToBytes(i int64) ([]byte, error) { receiver := scope.Name + "s" for _, opt := range scope.Option { - if opt.Description != "Deprecated" && !opt.Hidden { // Eww + if !opt.Hidden { writeOpt(receiver, opt) } } @@ -216,7 +205,7 @@ func int64ToBytes(i int64) ([]byte, error) { if scope.Name == "MutationType" { for _, opt := range scope.Option { - if opt.Description != "Deprecated" && !opt.Hidden { // Eww + if !opt.Hidden { writeMutation(opt) } } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/cluster.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/cluster.go index b9adb3f1a..a900f6569 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/cluster.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/cluster.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include */ import "C" diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/database.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/database.go index b8a37bda5..84aa6ebb8 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/database.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/database.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include */ import "C" @@ -111,7 +111,7 @@ func retryable(wrapped func() (interface{}, error), onError func(Error) FutureNi // retried or, if fatal, return the error to the caller. // // When working with Future objects in a transactional function, you may either -// explicity check and return error values using Get, or call MustGet. Transact +// explicitly check and return error values using Get, or call MustGet. Transact // will recover a panicked Error and either retry the transaction or return the // error. // @@ -151,7 +151,7 @@ func (d Database) Transact(f func(Transaction) (interface{}, error)) (interface{ // retried or, if fatal, return the error to the caller. // // When working with Future objects in a read-only transactional function, you -// may either explicity check and return error values using Get, or call +// may either explicitly check and return error values using Get, or call // MustGet. ReadTransact will recover a panicked Error and either retry the // transaction or return the error. // diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/allocator.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/allocator.go index 6dffad84e..4d47128f1 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/allocator.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/allocator.go @@ -65,7 +65,10 @@ func windowSize(start int64) int64 { func (hca highContentionAllocator) allocate(tr fdb.Transaction, s subspace.Subspace) (subspace.Subspace, error) { for { rr := tr.Snapshot().GetRange(hca.counters, fdb.RangeOptions{Limit: 1, Reverse: true}) - kvs := rr.GetSliceOrPanic() + kvs, e := rr.GetSliceWithError() + if e != nil { + return nil, e + } var start int64 var window int64 @@ -135,7 +138,10 @@ func (hca highContentionAllocator) allocate(tr fdb.Transaction, s subspace.Subsp allocatorMutex.Unlock() - kvs = latestCounter.GetSliceOrPanic() + kvs, e = latestCounter.GetSliceWithError() + if e != nil { + return nil, e + } if len(kvs) > 0 { t, e := hca.counters.Unpack(kvs[0].Key) if e != nil { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directory.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directory.go index 21f576d1f..792621b38 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directory.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directory.go @@ -26,7 +26,7 @@ // // For general guidance on directory usage, see the Directories section of the // Developer Guide -// (https://www.foundationdb.org/documentation/developer-guide.html#developer-guide-directories). +// (https://apple.github.io/foundationdb/developer-guide.html#directories). // // Directories are identified by hierarchical paths analogous to the paths in a // Unix-like file system. A path is represented as a slice of strings. Each diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryLayer.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryLayer.go index 14064b53a..e6abf3df2 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryLayer.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryLayer.go @@ -27,6 +27,7 @@ import ( "encoding/binary" "errors" "fmt" + "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/subspace" "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" @@ -78,9 +79,8 @@ func (dl directoryLayer) createOrOpen(rtr fdb.ReadTransaction, tr *fdb.Transacti if prefix != nil && !dl.allowManualPrefixes { if len(dl.path) == 0 { return nil, errors.New("cannot specify a prefix unless manual prefixes are enabled") - } else { - return nil, errors.New("cannot specify a prefix in a partition") } + return nil, errors.New("cannot specify a prefix in a partition") } if len(path) == 0 { @@ -102,8 +102,10 @@ func (dl directoryLayer) createOrOpen(rtr fdb.ReadTransaction, tr *fdb.Transacti return nil, errors.New("the directory already exists") } - if layer != nil && bytes.Compare(existingNode._layer.MustGet(), layer) != 0 { - return nil, errors.New("the directory was created with an incompatible layer") + if layer != nil { + if l, e := existingNode._layer.Get(); e != nil || bytes.Compare(l, layer) != 0 { + return nil, errors.New("the directory was created with an incompatible layer") + } } return existingNode.getContents(dl, nil) @@ -325,7 +327,11 @@ func (dl directoryLayer) Move(t fdb.Transactor, oldPath []string, newPath []stri dl.removeFromParent(tr, oldPath) - return dl.contentsOfNode(oldNode.subspace, newPath, oldNode._layer.MustGet()) + l, e := oldNode._layer.Get() + if e != nil { + return nil, e + } + return dl.contentsOfNode(oldNode.subspace, newPath, l) }) if e != nil { return nil, e @@ -415,7 +421,10 @@ func (dl directoryLayer) subdirNames(rtr fdb.ReadTransaction, node subspace.Subs var ret []string for ri.Advance() { - kv := ri.MustGet() + kv, e := ri.Get() + if e != nil { + return nil, e + } p, e := sd.Unpack(kv.Key) if e != nil { @@ -453,7 +462,10 @@ func (dl directoryLayer) nodeContainingKey(rtr fdb.ReadTransaction, key []byte) bk, _ := dl.nodeSS.FDBRangeKeys() kr := fdb.KeyRange{bk, fdb.Key(append(dl.nodeSS.Pack(tuple.Tuple{key}), 0x00))} - kvs := rtr.GetRange(kr, fdb.RangeOptions{Reverse: true, Limit: 1}).GetSliceOrPanic() + kvs, e := rtr.GetRange(kr, fdb.RangeOptions{Reverse: true, Limit: 1}).GetSliceWithError() + if e != nil { + return nil, e + } if len(kvs) == 1 { pp, e := dl.nodeSS.Unpack(kvs[0].Key) if e != nil { @@ -495,7 +507,10 @@ func (dl directoryLayer) isPrefixFree(rtr fdb.ReadTransaction, prefix []byte) (b } func (dl directoryLayer) checkVersion(rtr fdb.ReadTransaction, tr *fdb.Transaction) error { - version := rtr.Get(dl.rootNode.Sub([]byte("version"))).MustGet() + version, err := rtr.Get(dl.rootNode.Sub([]byte("version"))).Get() + if err != nil { + return err + } if version == nil { if tr != nil { @@ -562,9 +577,8 @@ func (dl directoryLayer) contentsOfNode(node subspace.Subspace, path []string, l ndl := NewDirectoryLayer(subspace.FromBytes(nssb), ss, false).(directoryLayer) ndl.path = newPath return directoryPartition{ndl, dl}, nil - } else { - return directorySubspace{ss, dl, newPath, layer}, nil } + return directorySubspace{ss, dl, newPath, layer}, nil } func (dl directoryLayer) nodeWithPrefix(prefix []byte) subspace.Subspace { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryPartition.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryPartition.go index 0a16870e7..d6e0275f0 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryPartition.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/directoryPartition.go @@ -72,9 +72,8 @@ func (dp directoryPartition) GetLayer() []byte { func (dp directoryPartition) getLayerForPath(path []string) directoryLayer { if len(path) == 0 { return dp.parentDirectoryLayer - } else { - return dp.directoryLayer } + return dp.directoryLayer } func (dp directoryPartition) MoveTo(t fdb.Transactor, newAbsolutePath []string) (DirectorySubspace, error) { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/node.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/node.go index 3280a50c4..eb5c6857a 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/node.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/directory/node.go @@ -67,5 +67,9 @@ func (n *node) getPartitionSubpath() []string { } func (n *node) getContents(dl directoryLayer, tr *fdb.Transaction) (DirectorySubspace, error) { - return dl.contentsOfNode(n.subspace, n.path, n._layer.MustGet()) + l, err := n._layer.Get() + if err != nil { + return nil, err + } + return dl.contentsOfNode(n.subspace, n.path, l) } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/doc.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/doc.go index fba7f6432..a7308887a 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/doc.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/doc.go @@ -25,12 +25,12 @@ Package fdb provides an interface to FoundationDB databases (version 2.0 or high To build and run programs using this package, you must have an installed copy of the FoundationDB client libraries (version 2.0.0 or later), available for Linux, -Windows and OS X at https://www.foundationdb.org/downloads/fdb-c/. +Windows and OS X at https://www.foundationdb.org/download/. This documentation specifically applies to the FoundationDB Go binding. For more extensive guidance to programming with FoundationDB, as well as API documentation for the other FoundationDB interfaces, please see -https://www.foundationdb.org/documentation/index.html. +https://apple.github.io/foundationdb/index.html. Basic Usage @@ -46,7 +46,7 @@ A basic interaction with the FoundationDB API is demonstrated below: func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(200) + fdb.MustAPIVersion(600) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -198,7 +198,7 @@ operations perform different transformations. Like other database operations, an atomic operation is used within a transaction. For more information on atomic operations in FoundationDB, please see -https://www.foundationdb.org/documentation/developer-guide.html#atomic-operations. The +https://apple.github.io/foundationdb/developer-guide.html#atomic-operations. The operands to atomic operations in this API must be provided as appropriately encoded byte slices. To convert a Go type to a byte slice, see the binary package. diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/errors.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/errors.go index 7317799fa..15bd8081c 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/errors.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/errors.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include */ import "C" @@ -37,7 +37,7 @@ import ( // as a panic from any FoundationDB API function whose name ends with OrPanic. // // You may compare the Code field of an Error against the list of FoundationDB -// error codes at https://www.foundationdb.org/documentation/api-error-codes.html, +// error codes at https://apple.github.io/foundationdb/api-error-codes.html, // but generally an Error should be passed to (Transaction).OnError. When using // (Database).Transact, non-fatal errors will be retried automatically. type Error struct { diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/fdb.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/fdb.go index 23f3a68e5..76cf5ac67 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/fdb.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/fdb.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include #include */ @@ -109,7 +109,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // library, an error will be returned. APIVersion must be called prior to any // other functions in the fdb package. // -// Currently, this package supports API versions 200 through 510. +// Currently, this package supports API versions 200 through 600. // // Warning: When using the multi-version client API, setting an API version that // is not supported by a particular client library will prevent that client from @@ -117,7 +117,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // the API version of your application after upgrading your client until the // cluster has also been upgraded. func APIVersion(version int) error { - headerVersion := 510 + headerVersion := 600 networkMutex.Lock() defer networkMutex.Unlock() @@ -129,7 +129,7 @@ func APIVersion(version int) error { return errAPIVersionAlreadySet } - if version < 200 || version > 510 { + if version < 200 || version > 600 { return errAPIVersionNotSupported } @@ -139,9 +139,8 @@ func APIVersion(version int) error { maxSupportedVersion := C.fdb_get_max_api_version() if headerVersion > int(maxSupportedVersion) { return fmt.Errorf("This version of the FoundationDB Go binding is not supported by the installed FoundationDB C library. The binding requires a library that supports API version %d, but the installed library supports a maximum version of %d.", version, maxSupportedVersion) - } else { - return fmt.Errorf("API version %d is not supported by the installed FoundationDB C library.", version) } + return fmt.Errorf("API version %d is not supported by the installed FoundationDB C library.", version) } return Error{int(e)} } @@ -152,6 +151,24 @@ func APIVersion(version int) error { return nil } +// Determines if an API version has already been selected, i.e., if +// APIVersion or MustAPIVersion have already been called. +func IsAPIVersionSelected() bool { + return apiVersion != 0 +} + +// Returns the API version that has been selected through APIVersion +// or MustAPIVersion. If the version has already been selected, then +// the first value returned is the API version and the error is +// nil. If the API version has not yet been set, then the error +// will be non-nil. +func GetAPIVersion() (int, error) { + if IsAPIVersionSelected() { + return apiVersion, nil + } + return 0, errAPIVersionUnset +} + // MustAPIVersion is like APIVersion but panics if the API version is not // supported. func MustAPIVersion(version int) { @@ -161,16 +178,31 @@ func MustAPIVersion(version int) { } } +// MustGetAPIVersion is like GetAPIVersion but panics if the API version +// has not yet been set. +func MustGetAPIVersion() int { + apiVersion, err := GetAPIVersion() + if err != nil { + panic(err) + } + return apiVersion +} + var apiVersion int var networkStarted bool var networkMutex sync.Mutex +type DatabaseId struct { + clusterFile string + dbName string +} + var openClusters map[string]Cluster -var openDatabases map[string]Database +var openDatabases map[DatabaseId]Database func init() { openClusters = make(map[string]Cluster) - openDatabases = make(map[string]Database) + openDatabases = make(map[DatabaseId]Database) } func startNetwork() error { @@ -260,13 +292,13 @@ func Open(clusterFile string, dbName []byte) (Database, error) { openClusters[clusterFile] = cluster } - db, ok := openDatabases[string(dbName)] + db, ok := openDatabases[DatabaseId{clusterFile, string(dbName)}] if !ok { db, e = cluster.OpenDatabase(dbName) if e != nil { return Database{}, e } - openDatabases[string(dbName)] = db + openDatabases[DatabaseId{clusterFile, string(dbName)}] = db } return db, nil @@ -326,9 +358,8 @@ func CreateCluster(clusterFile string) (Cluster, error) { func byteSliceToPtr(b []byte) *C.uint8_t { if len(b) > 0 { return (*C.uint8_t)(unsafe.Pointer(&b[0])) - } else { - return nil } + return nil } // A KeyConvertible can be converted to a FoundationDB Key. All functions in the diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/futures.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/futures.go index f0da9fe21..f3541f068 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/futures.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/futures.go @@ -24,7 +24,7 @@ package fdb /* #cgo LDFLAGS: -lfdb_c -lm - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include #include @@ -87,6 +87,13 @@ func fdb_future_block_until_ready(f *C.FDBFuture) { return } + // The mutex here is used as a signal that the callback is complete. + // We first lock it, then pass it to the callback, and then lock it + // again. The second call to lock won't return until the callback has + // fired. + // + // See https://groups.google.com/forum/#!topic/golang-nuts/SPjQEcsdORA + // for the history of why this pattern came to be used. m := &sync.Mutex{} m.Lock() C.go_set_callback(unsafe.Pointer(f), unsafe.Pointer(m)) diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/generated.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/generated.go index a3d842d62..c42757d67 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/generated.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/generated.go @@ -42,6 +42,20 @@ func int64ToBytes(i int64) ([]byte, error) { return buf.Bytes(), nil } +// Deprecated +// +// Parameter: IP:PORT +func (o NetworkOptions) SetLocalAddress(param string) error { + return o.setOpt(10, []byte(param)) +} + +// Deprecated +// +// Parameter: path to cluster file +func (o NetworkOptions) SetClusterFile(param string) error { + return o.setOpt(20, []byte(param)) +} + // Enables trace output to a file in a directory of the clients choosing // // Parameter: path to output directory (or NULL for current working directory) @@ -71,9 +85,9 @@ func (o NetworkOptions) SetTraceMaxLogsSize(param int64) error { return o.setOpt(32, b) } -// Sets the 'logGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'. +// Sets the 'LogGroup' attribute with the specified value for all events in the trace output files. The default log group is 'default'. // -// Parameter: value of the logGroup attribute +// Parameter: value of the LogGroup attribute func (o NetworkOptions) SetTraceLogGroup(param string) error { return o.setOpt(33, []byte(param)) } @@ -85,7 +99,7 @@ func (o NetworkOptions) SetKnob(param string) error { return o.setOpt(40, []byte(param)) } -// Set the TLS plugin to load. This option, if used, must be set before any other TLS options +// Deprecated // // Parameter: file path or linker-resolved name func (o NetworkOptions) SetTLSPlugin(param string) error { @@ -159,6 +173,27 @@ func (o NetworkOptions) SetBuggifySectionFiredProbability(param int64) error { return o.setOpt(51, b) } +// Set the ca bundle +// +// Parameter: ca bundle +func (o NetworkOptions) SetTLSCaBytes(param []byte) error { + return o.setOpt(52, param) +} + +// Set the file from which to load the certificate authority bundle +// +// Parameter: file path +func (o NetworkOptions) SetTLSCaPath(param string) error { + return o.setOpt(53, []byte(param)) +} + +// Set the passphrase for encrypted private key. Password should be set before setting the key for the password to be used. +// +// Parameter: key passphrase +func (o NetworkOptions) SetTLSPassword(param string) error { + return o.setOpt(54, []byte(param)) +} + // Disables the multi-version client API and instead uses the local client directly. Must be set before setting up the network. func (o NetworkOptions) SetDisableMultiVersionClientApi() error { return o.setOpt(60, nil) @@ -254,22 +289,12 @@ func (o TransactionOptions) SetNextWriteNoWriteConflictRange() error { return o.setOpt(30, nil) } -// Committing this transaction will bypass the normal load balancing across proxies and go directly to the specifically nominated 'first proxy'. -func (o TransactionOptions) SetCommitOnFirstProxy() error { - return o.setOpt(40, nil) -} - -// Not yet implemented. -func (o TransactionOptions) SetCheckWritesEnable() error { - return o.setOpt(50, nil) -} - // Reads performed by a transaction will not see any prior mutations that occured in that transaction, instead seeing the value which was in the database at the transaction's read version. This option may provide a small performance benefit for the client, but also disables a number of client-side optimizations which are beneficial for transactions which tend to read and write the same keys within a single transaction. func (o TransactionOptions) SetReadYourWritesDisable() error { return o.setOpt(51, nil) } -// Disables read-ahead caching for range reads. Under normal operation, a transaction will read extra rows from the database into cache if range reads are used to page through a series of data one row at a time (i.e. if a range read with a one row limit is followed by another one row range read starting immediately after the result of the first). +// Deprecated func (o TransactionOptions) SetReadAheadDisable() error { return o.setOpt(52, nil) } @@ -284,7 +309,7 @@ func (o TransactionOptions) SetDurabilityRisky() error { return o.setOpt(120, nil) } -// Not yet implemented. +// Deprecated func (o TransactionOptions) SetDurabilityDevNullIsWebScale() error { return o.setOpt(130, nil) } @@ -314,11 +339,6 @@ func (o TransactionOptions) SetReadSystemKeys() error { return o.setOpt(302, nil) } -// Not yet implemented. -func (o TransactionOptions) SetDebugDump() error { - return o.setOpt(400, nil) -} - // Not yet implemented. func (o TransactionOptions) SetDebugRetryLogging(param string) error { return o.setOpt(401, []byte(param)) @@ -435,52 +455,72 @@ const ( StreamingModeSerial StreamingMode = 5 ) -// Add performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation. They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow. +// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation. They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow. func (t Transaction) Add(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 2) } -// BitAnd performs a bitwise ``and`` operation. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Deprecated +func (t Transaction) And(key KeyConvertible, param []byte) { + t.atomicOp(key.FDBKey(), param, 6) +} + +// Performs a bitwise ``and`` operation. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. func (t Transaction) BitAnd(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 6) } -// BitOr performs a bitwise ``or`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Deprecated +func (t Transaction) Or(key KeyConvertible, param []byte) { + t.atomicOp(key.FDBKey(), param, 7) +} + +// Performs a bitwise ``or`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. func (t Transaction) BitOr(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 7) } -// BitXor performs a bitwise ``xor`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Deprecated +func (t Transaction) Xor(key KeyConvertible, param []byte) { + t.atomicOp(key.FDBKey(), param, 8) +} + +// Performs a bitwise ``xor`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. func (t Transaction) BitXor(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 8) } -// Max performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database. +// Appends ``param`` to the end of the existing value already in the database at the given key (or creates the key and sets the value to ``param`` if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size. +func (t Transaction) AppendIfFits(key KeyConvertible, param []byte) { + t.atomicOp(key.FDBKey(), param, 9) +} + +// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database. func (t Transaction) Max(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 12) } -// Min performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database. +// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database. func (t Transaction) Min(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 13) } -// SetVersionstampedKey transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages. +// Transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes. func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 14) } -// SetVersionstampedValue transforms ``param`` using a versionstamp for the transaction. Sets ``key`` in the database to the transformed parameter. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time versionstamped values are not compatible with the Tuple layer. +// Transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java and Python bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset. func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 15) } -// ByteMin performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database. +// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database. func (t Transaction) ByteMin(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 16) } -// ByteMax performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database. +// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database. func (t Transaction) ByteMax(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 17) } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/keyselector.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/keyselector.go index 538a626a1..7d4f45ffe 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/keyselector.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/keyselector.go @@ -34,7 +34,7 @@ type Selectable interface { // // The most common key selectors are constructed with the functions documented // below. For details of how KeySelectors are specified and resolved, see -// https://www.foundationdb.org/documentation/developer-guide.html#key-selectors. +// https://apple.github.io/foundationdb/developer-guide.html#key-selectors. type KeySelector struct { Key KeyConvertible OrEqual bool diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/range.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/range.go index 4155128c8..06a8b9471 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/range.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/range.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include */ import "C" @@ -247,7 +247,7 @@ func (ri *RangeIterator) fetchNextBatch() { ri.sr.Begin = FirstGreaterThan(ri.kvs[ri.index-1].Key) } - ri.iteration += 1 + ri.iteration++ f := ri.t.doGetRange(ri.sr, ri.options, ri.snapshot, ri.iteration) ri.f = &f @@ -265,7 +265,7 @@ func (ri *RangeIterator) Get() (kv KeyValue, e error) { kv = ri.kvs[ri.index] - ri.index += 1 + ri.index++ if ri.index == len(ri.kvs) { ri.fetchNextBatch() @@ -286,12 +286,14 @@ func (ri *RangeIterator) MustGet() KeyValue { return kv } +// Strinc returns the first key that would sort outside the range prefixed by +// prefix, or an error if prefix is empty or contains only 0xFF bytes. func Strinc(prefix []byte) ([]byte, error) { for i := len(prefix) - 1; i >= 0; i-- { if prefix[i] != 0xFF { ret := make([]byte, i+1) copy(ret, prefix[:i+1]) - ret[i] += 1 + ret[i]++ return ret, nil } } @@ -311,7 +313,7 @@ func PrefixRange(prefix []byte) (KeyRange, error) { copy(begin, prefix) end, e := Strinc(begin) if e != nil { - return KeyRange{}, nil + return KeyRange{}, e } return KeyRange{Key(begin), Key(end)}, nil } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/snapshot.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/snapshot.go index 699344bc6..18c77d79b 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/snapshot.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/snapshot.go @@ -28,7 +28,7 @@ package fdb // transaction conflicts but making it harder to reason about concurrency. // // For more information on snapshot reads, see -// https://www.foundationdb.org/documentation/developer-guide.html#snapshot-reads. +// https://apple.github.io/foundationdb/developer-guide.html#snapshot-reads. type Snapshot struct { *transaction } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/subspace/subspace.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/subspace/subspace.go index f2523a81e..b779d5a9f 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/subspace/subspace.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/subspace/subspace.go @@ -29,7 +29,7 @@ // As a best practice, API clients should use at least one subspace for // application data. For general guidance on subspace usage, see the Subspaces // section of the Developer Guide -// (https://www.foundationdb.org/documentation/developer-guide.html#developer-guide-sub-keyspaces). +// (https://apple.github.io/foundationdb/developer-guide.html#subspaces). package subspace import ( diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/transaction.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/transaction.go index 87ff29ae8..549558406 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/transaction.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/transaction.go @@ -23,7 +23,7 @@ package fdb /* - #define FDB_API_VERSION 510 + #define FDB_API_VERSION 600 #include */ import "C" @@ -171,7 +171,7 @@ func (t Transaction) SetReadVersion(version int64) { // but making it harder to reason about concurrency. // // For more information on snapshot reads, see -// https://www.foundationdb.org/documentation/developer-guide.html#using-snapshot-reads. +// https://apple.github.io/foundationdb/developer-guide.html#snapshot-reads. func (t Transaction) Snapshot() Snapshot { return Snapshot{t.transaction} } @@ -196,7 +196,7 @@ func (t Transaction) OnError(e Error) FutureNil { // As with other client/server databases, in some failure scenarios a client may // be unable to determine whether a transaction succeeded. For more information, // see -// https://www.foundationdb.org/documentation/developer-guide.html#developer-guide-unknown-results. +// https://apple.github.io/foundationdb/developer-guide.html#transactions-with-unknown-results. func (t Transaction) Commit() FutureNil { return &futureNil{newFuture(C.fdb_transaction_commit(t.ptr))} } @@ -352,9 +352,8 @@ func (t Transaction) Reset() { func boolToInt(b bool) int { if b { return 1 - } else { - return 0 } + return 0 } func (t *transaction) getKey(sel KeySelector, snapshot int) FutureKey { @@ -396,7 +395,7 @@ func addConflictRange(t *transaction, er ExactRange, crtype conflictRangeType) e // conflict. // // For more information on conflict ranges, see -// https://www.foundationdb.org/documentation/developer-guide.html#conflict-ranges. +// https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges. func (t Transaction) AddReadConflictRange(er ExactRange) error { return addConflictRange(t.transaction, er, conflictRangeTypeRead) } @@ -413,7 +412,7 @@ func copyAndAppend(orig []byte, b byte) []byte { // this key could cause the transaction to fail with a conflict. // // For more information on conflict ranges, see -// https://www.foundationdb.org/documentation/developer-guide.html#conflict-ranges. +// https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges. func (t Transaction) AddReadConflictKey(key KeyConvertible) error { return addConflictRange(t.transaction, KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))}, conflictRangeTypeRead) } @@ -424,7 +423,7 @@ func (t Transaction) AddReadConflictKey(key KeyConvertible) error { // conflict. // // For more information on conflict ranges, see -// https://www.foundationdb.org/documentation/developer-guide.html#conflict-ranges. +// https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges. func (t Transaction) AddWriteConflictRange(er ExactRange) error { return addConflictRange(t.transaction, er, conflictRangeTypeWrite) } @@ -434,7 +433,7 @@ func (t Transaction) AddWriteConflictRange(er ExactRange) error { // read this key could fail with a conflict. // // For more information on conflict ranges, see -// https://www.foundationdb.org/documentation/developer-guide.html#conflict-ranges. +// https://apple.github.io/foundationdb/developer-guide.html#conflict-ranges. func (t Transaction) AddWriteConflictKey(key KeyConvertible) error { return addConflictRange(t.transaction, KeyRange{key, Key(copyAndAppend(key.FDBKey(), 0x00))}, conflictRangeTypeWrite) } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/testdata/tuples.golden b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/testdata/tuples.golden new file mode 100755 index 000000000..9f1c3f11d Binary files /dev/null and b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/testdata/tuples.golden differ diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple.go index d00a594ab..afd959420 100755 --- a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple.go +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple.go @@ -27,17 +27,22 @@ // of higher-level data models. // // For general guidance on tuple usage, see the Tuple section of Data Modeling -// (https://www.foundationdb.org/documentation/data-modeling.html#data-modeling-tuples). +// (https://apple.github.io/foundationdb/data-modeling.html#tuples). // -// FoundationDB tuples can currently encode byte and unicode strings, integers -// and NULL values. In Go these are represented as []byte, string, int64 and -// nil. +// FoundationDB tuples can currently encode byte and unicode strings, integers, +// large integers, floats, doubles, booleans, UUIDs, tuples, and NULL values. +// In Go these are represented as []byte (or fdb.KeyConvertible), string, int64 +// (or int, uint, uint64), *big.Int (or big.Int), float32, float64, bool, +// UUID, Tuple, and nil. package tuple import ( "bytes" "encoding/binary" "fmt" + "math" + "math/big" + "github.com/apple/foundationdb/bindings/go/src/fdb" ) @@ -47,7 +52,8 @@ import ( // result in a runtime panic). // // The valid types for TupleElement are []byte (or fdb.KeyConvertible), string, -// int64 (or int), float, double, bool, UUID, Tuple, and nil. +// int64 (or int, uint, uint64), *big.Int (or big.Int), float, double, bool, +// UUID, Tuple, and nil. type TupleElement interface{} // Tuple is a slice of objects that can be encoded as FoundationDB tuples. If @@ -56,7 +62,7 @@ type TupleElement interface{} // // Given a Tuple T containing objects only of these types, then T will be // identical to the Tuple returned by unpacking the byte slice obtained by -// packing T (modulo type normalization to []byte and int64). +// packing T (modulo type normalization to []byte, uint64, and int64). type Tuple []TupleElement // UUID wraps a basic byte array as a UUID. We do not provide any special @@ -73,8 +79,8 @@ const bytesCode = 0x01 const stringCode = 0x02 const nestedCode = 0x05 const intZeroCode = 0x14 -const posIntEnd = 0x1c -const negIntStart = 0x0c +const posIntEnd = 0x1d +const negIntStart = 0x0b const floatCode = 0x20 const doubleCode = 0x21 const falseCode = 0x26 @@ -93,6 +99,16 @@ var sizeLimits = []uint64{ 1<<(8*8) - 1, } +var minInt64BigInt = big.NewInt(math.MinInt64) + +func bisectLeft(u uint64) int { + var n int + for sizeLimits[n] < u { + n++ + } + return n +} + func adjustFloatBytes(b []byte, encode bool) { if (encode && b[0]&0x80 != 0x00) || (!encode && b[0]&0x80 == 0x00) { // Negative numbers: flip all of the bytes. @@ -105,124 +121,202 @@ func adjustFloatBytes(b []byte, encode bool) { } } -func encodeBytes(buf *bytes.Buffer, code byte, b []byte) { - buf.WriteByte(code) - buf.Write(bytes.Replace(b, []byte{0x00}, []byte{0x00, 0xFF}, -1)) - buf.WriteByte(0x00) +type packer struct { + buf []byte } -func bisectLeft(u uint64) int { - var n int - for sizeLimits[n] < u { - n += 1 +func (p *packer) putByte(b byte) { + p.buf = append(p.buf, b) +} + +func (p *packer) putBytes(b []byte) { + p.buf = append(p.buf, b...) +} + +func (p *packer) putBytesNil(b []byte, i int) { + for i >= 0 { + p.putBytes(b[:i+1]) + p.putByte(0xFF) + b = b[i+1:] + i = bytes.IndexByte(b, 0x00) } - return n + p.putBytes(b) } -func encodeInt(buf *bytes.Buffer, i int64) { +func (p *packer) encodeBytes(code byte, b []byte) { + p.putByte(code) + if i := bytes.IndexByte(b, 0x00); i >= 0 { + p.putBytesNil(b, i) + } else { + p.putBytes(b) + } + p.putByte(0x00) +} + +func (p *packer) encodeUint(i uint64) { if i == 0 { - buf.WriteByte(0x14) + p.putByte(intZeroCode) return } - var n int - var ibuf bytes.Buffer - - switch { - case i > 0: - n = bisectLeft(uint64(i)) - buf.WriteByte(byte(intZeroCode + n)) - binary.Write(&ibuf, binary.BigEndian, i) - case i < 0: - n = bisectLeft(uint64(-i)) - buf.WriteByte(byte(0x14 - n)) - binary.Write(&ibuf, binary.BigEndian, int64(sizeLimits[n])+i) + n := bisectLeft(i) + var scratch [8]byte + + p.putByte(byte(intZeroCode + n)) + binary.BigEndian.PutUint64(scratch[:], i) + + p.putBytes(scratch[8-n:]) +} + +func (p *packer) encodeInt(i int64) { + if i >= 0 { + p.encodeUint(uint64(i)) + return } - buf.Write(ibuf.Bytes()[8-n:]) + n := bisectLeft(uint64(-i)) + var scratch [8]byte + + p.putByte(byte(intZeroCode - n)) + offsetEncoded := int64(sizeLimits[n]) + i + binary.BigEndian.PutUint64(scratch[:], uint64(offsetEncoded)) + + p.putBytes(scratch[8-n:]) } -func encodeFloat(buf *bytes.Buffer, f float32) { - var ibuf bytes.Buffer - binary.Write(&ibuf, binary.BigEndian, f) - buf.WriteByte(floatCode) - out := ibuf.Bytes() - adjustFloatBytes(out, true) - buf.Write(out) +func (p *packer) encodeBigInt(i *big.Int) { + length := len(i.Bytes()) + if length > 0xff { + panic(fmt.Sprintf("Integer magnitude is too large (more than 255 bytes)")) + } + + if i.Sign() >= 0 { + intBytes := i.Bytes() + if length > 8 { + p.putByte(byte(posIntEnd)) + p.putByte(byte(len(intBytes))) + } else { + p.putByte(byte(intZeroCode + length)) + } + + p.putBytes(intBytes) + } else { + add := new(big.Int).Lsh(big.NewInt(1), uint(length*8)) + add.Sub(add, big.NewInt(1)) + transformed := new(big.Int) + transformed.Add(i, add) + + intBytes := transformed.Bytes() + if length > 8 { + p.putByte(byte(negIntStart)) + p.putByte(byte(length ^ 0xff)) + } else { + p.putByte(byte(intZeroCode - length)) + } + + // For large negative numbers whose absolute value begins with 0xff bytes, + // the transformed bytes may begin with 0x00 bytes. However, intBytes + // will only contain the non-zero suffix, so this loop is needed to make + // the value written be the correct length. + for i := len(intBytes); i < length; i++ { + p.putByte(0x00) + } + + p.putBytes(intBytes) + } +} + +func (p *packer) encodeFloat(f float32) { + var scratch [4]byte + binary.BigEndian.PutUint32(scratch[:], math.Float32bits(f)) + adjustFloatBytes(scratch[:], true) + + p.putByte(floatCode) + p.putBytes(scratch[:]) } -func encodeDouble(buf *bytes.Buffer, d float64) { - var ibuf bytes.Buffer - binary.Write(&ibuf, binary.BigEndian, d) - buf.WriteByte(doubleCode) - out := ibuf.Bytes() - adjustFloatBytes(out, true) - buf.Write(out) +func (p *packer) encodeDouble(d float64) { + var scratch [8]byte + binary.BigEndian.PutUint64(scratch[:], math.Float64bits(d)) + adjustFloatBytes(scratch[:], true) + + p.putByte(doubleCode) + p.putBytes(scratch[:]) } -func encodeUUID(buf *bytes.Buffer, u UUID) { - buf.WriteByte(uuidCode) - buf.Write(u[:]) +func (p *packer) encodeUUID(u UUID) { + p.putByte(uuidCode) + p.putBytes(u[:]) } -func encodeTuple(buf *bytes.Buffer, t Tuple, nested bool) { +func (p *packer) encodeTuple(t Tuple, nested bool) { if nested { - buf.WriteByte(nestedCode) + p.putByte(nestedCode) } for i, e := range t { switch e := e.(type) { case Tuple: - encodeTuple(buf, e, true) + p.encodeTuple(e, true) case nil: - buf.WriteByte(nilCode) + p.putByte(nilCode) if nested { - buf.WriteByte(0xff) + p.putByte(0xff) } - case int64: - encodeInt(buf, e) case int: - encodeInt(buf, int64(e)) + p.encodeInt(int64(e)) + case int64: + p.encodeInt(e) + case uint: + p.encodeUint(uint64(e)) + case uint64: + p.encodeUint(e) + case *big.Int: + p.encodeBigInt(e) + case big.Int: + p.encodeBigInt(&e) case []byte: - encodeBytes(buf, bytesCode, e) + p.encodeBytes(bytesCode, e) case fdb.KeyConvertible: - encodeBytes(buf, bytesCode, []byte(e.FDBKey())) + p.encodeBytes(bytesCode, []byte(e.FDBKey())) case string: - encodeBytes(buf, stringCode, []byte(e)) + p.encodeBytes(stringCode, []byte(e)) case float32: - encodeFloat(buf, e) + p.encodeFloat(e) case float64: - encodeDouble(buf, e) + p.encodeDouble(e) case bool: if e { - buf.WriteByte(trueCode) + p.putByte(trueCode) } else { - buf.WriteByte(falseCode) + p.putByte(falseCode) } case UUID: - encodeUUID(buf, e) + p.encodeUUID(e) default: panic(fmt.Sprintf("unencodable element at index %d (%v, type %T)", i, t[i], t[i])) } } if nested { - buf.WriteByte(0x00) + p.putByte(0x00) } } // Pack returns a new byte slice encoding the provided tuple. Pack will panic if // the tuple contains an element of any type other than []byte, -// fdb.KeyConvertible, string, int64, int, float32, float64, bool, tuple.UUID, -// nil, or a Tuple with elements of valid types. +// fdb.KeyConvertible, string, int64, int, uint64, uint, *big.Int, big.Int, float32, +// float64, bool, tuple.UUID, nil, or a Tuple with elements of valid types. It will +// also panic if an integer is specified with a value outside the range +// [-2**2040+1, 2**2040-1] // // Tuple satisfies the fdb.KeyConvertible interface, so it is not necessary to // call Pack when using a Tuple with a FoundationDB API function that requires a // key. func (t Tuple) Pack() []byte { - buf := new(bytes.Buffer) - encodeTuple(buf, t, false) - return buf.Bytes() + p := packer{buf: make([]byte, 0, 64)} + p.encodeTuple(t, false) + return p.buf } func findTerminator(b []byte) int { @@ -252,9 +346,9 @@ func decodeString(b []byte) (string, int) { return string(bp), idx } -func decodeInt(b []byte) (int64, int) { +func decodeInt(b []byte) (interface{}, int) { if b[0] == intZeroCode { - return 0, 1 + return int64(0), 1 } var neg bool @@ -269,14 +363,55 @@ func decodeInt(b []byte) (int64, int) { copy(bp[8-n:], b[1:n+1]) var ret int64 - binary.Read(bytes.NewBuffer(bp), binary.BigEndian, &ret) if neg { - ret -= int64(sizeLimits[n]) + return ret - int64(sizeLimits[n]), n + 1 + } + + if ret > 0 { + return ret, n + 1 + } + + // The encoded value claimed to be positive yet when put in an int64 + // produced a negative value. This means that the number must be a positive + // 64-bit value that uses the most significant bit. This can be fit in a + // uint64, so return that. Note that this is the *only* time we return + // a uint64. + return uint64(ret), n + 1 +} + +func decodeBigInt(b []byte) (interface{}, int) { + val := new(big.Int) + offset := 1 + var length int + + if b[0] == negIntStart || b[0] == posIntEnd { + length = int(b[1]) + if b[0] == negIntStart { + length ^= 0xff + } + + offset += 1 + } else { + // Must be a negative 8 byte integer + length = 8 } - return ret, n + 1 + val.SetBytes(b[offset : length+offset]) + + if b[0] < intZeroCode { + sub := new(big.Int).Lsh(big.NewInt(1), uint(length)*8) + sub.Sub(sub, big.NewInt(1)) + val.Sub(val, sub) + } + + // This is the only value that fits in an int64 or uint64 that is decoded with this function + if val.Cmp(minInt64BigInt) == 0 { + return val.Int64(), length + offset + } + + return val, length + offset } func decodeFloat(b []byte) (float32, int) { @@ -327,8 +462,12 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { el, off = decodeBytes(b[i:]) case b[i] == stringCode: el, off = decodeString(b[i:]) - case negIntStart <= b[i] && b[i] <= posIntEnd: + case negIntStart+1 < b[i] && b[i] < posIntEnd: el, off = decodeInt(b[i:]) + case negIntStart+1 == b[i] && (b[i+1]&0x80 != 0): + el, off = decodeInt(b[i:]) + case negIntStart <= b[i] && b[i] <= posIntEnd: + el, off = decodeBigInt(b[i:]) case b[i] == floatCode: if i+5 > len(b) { return nil, i, fmt.Errorf("insufficient bytes to decode float starting at position %d of byte array for tuple", i) @@ -356,7 +495,7 @@ func decodeTuple(b []byte, nested bool) (Tuple, int, error) { if err != nil { return nil, i, err } - off += 1 + off++ default: return nil, i, fmt.Errorf("unable to decode tuple element with unknown typecode %02x", b[i]) } diff --git a/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple_test.go b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple_test.go new file mode 100755 index 000000000..59c37bc7b --- /dev/null +++ b/vendor/github.com/apple/foundationdb/bindings/go/src/fdb/tuple/tuple_test.go @@ -0,0 +1,120 @@ +package tuple + +import ( + "bytes" + "encoding/gob" + "flag" + "math/rand" + "os" + "testing" +) + +var update = flag.Bool("update", false, "update .golden files") + +func loadGolden(t *testing.T) (golden map[string][]byte) { + f, err := os.Open("testdata/tuples.golden") + if err != nil { + t.Fatalf("failed to open golden file: %s", err) + } + defer f.Close() + + err = gob.NewDecoder(f).Decode(&golden) + if err != nil { + t.Fatalf("failed to decode golden file: %s", err) + } + return +} + +func writeGolden(t *testing.T, golden map[string][]byte) { + f, err := os.Create("testdata/tuples.golden") + if err != nil { + t.Fatalf("failed to open golden file: %s", err) + } + defer f.Close() + + err = gob.NewEncoder(f).Encode(golden) + if err != nil { + t.Fatalf("failed to encode golden file: %s", err) + } +} + +var testUUID = UUID{ + 0x11, 0x00, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, + 0x11, 0x00, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, +} + +func genBytes() interface{} { return []byte("namespace") } +func genBytesNil() interface{} { return []byte{0xFF, 0x00, 0xFF} } +func genString() interface{} { return "namespace" } +func genStringNil() interface{} { return "nam\x00es\xFFpace" } +func genInt() interface{} { return rand.Int63() } +func genFloat() interface{} { return float32(rand.NormFloat64()) } +func genDouble() interface{} { return rand.NormFloat64() } + +func mktuple(gen func() interface{}, count int) Tuple { + tt := make(Tuple, count) + for i := 0; i < count; i++ { + tt[i] = gen() + } + return tt +} + +var testCases = []struct { + name string + tuple Tuple +}{ + {"Simple", Tuple{testUUID, "foobarbaz", 1234, nil}}, + {"Namespaces", Tuple{testUUID, "github", "com", "apple", "foundationdb", "tree"}}, + {"ManyStrings", mktuple(genString, 8)}, + {"ManyStringsNil", mktuple(genStringNil, 8)}, + {"ManyBytes", mktuple(genBytes, 20)}, + {"ManyBytesNil", mktuple(genBytesNil, 20)}, + {"LargeBytes", Tuple{testUUID, bytes.Repeat([]byte("abcd"), 20)}}, + {"LargeBytesNil", Tuple{testUUID, bytes.Repeat([]byte{0xFF, 0x0, 0xFF}, 20)}}, + {"Integers", mktuple(genInt, 20)}, + {"Floats", mktuple(genFloat, 20)}, + {"Doubles", mktuple(genDouble, 20)}, + {"UUIDs", Tuple{testUUID, true, testUUID, false, testUUID, true, testUUID, false, testUUID, true}}, + {"NilCases", Tuple{"\x00", "\x00\xFF", "\x00\x00\x00", "\xFF\x00", ""}}, + {"Nested", Tuple{testUUID, mktuple(genInt, 4), nil, mktuple(genBytes, 4), nil, mktuple(genDouble, 4), nil}}, +} + +func TestTuplePacking(t *testing.T) { + var golden map[string][]byte + + if *update { + golden = make(map[string][]byte) + } else { + golden = loadGolden(t) + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + result := tt.tuple.Pack() + + if *update { + golden[tt.name] = result + return + } + + if !bytes.Equal(result, golden[tt.name]) { + t.Errorf("packing mismatch: expected %v, got %v", golden[tt.name], result) + } + }) + } + + if *update { + writeGolden(t, golden) + } +} + +func BenchmarkTuplePacking(b *testing.B) { + for _, bm := range testCases { + b.Run(bm.name, func(b *testing.B) { + tuple := bm.tuple + for i := 0; i < b.N; i++ { + _ = tuple.Pack() + } + }) + } +}