2014-01-08 15:06:17 +00:00
|
|
|
package bolt
|
|
|
|
|
|
|
|
import (
|
2014-01-31 18:18:51 +00:00
|
|
|
"fmt"
|
|
|
|
"os"
|
2015-06-13 01:32:55 +00:00
|
|
|
"sort"
|
2014-01-08 15:06:17 +00:00
|
|
|
"unsafe"
|
|
|
|
)
|
|
|
|
|
2014-01-27 15:11:54 +00:00
|
|
|
const pageHeaderSize = int(unsafe.Offsetof(((*page)(nil)).ptr))
|
2014-01-10 14:32:12 +00:00
|
|
|
|
2014-01-27 15:11:54 +00:00
|
|
|
const minKeysPerPage = 2
|
2014-01-08 15:06:17 +00:00
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
const branchPageElementSize = int(unsafe.Sizeof(branchPageElement{}))
|
|
|
|
const leafPageElementSize = int(unsafe.Sizeof(leafPageElement{}))
|
|
|
|
|
2014-01-08 15:06:17 +00:00
|
|
|
const (
|
2014-02-12 21:57:27 +00:00
|
|
|
branchPageFlag = 0x01
|
|
|
|
leafPageFlag = 0x02
|
|
|
|
metaPageFlag = 0x04
|
|
|
|
freelistPageFlag = 0x10
|
2014-01-13 17:35:04 +00:00
|
|
|
)
|
2014-01-09 16:07:10 +00:00
|
|
|
|
2014-04-07 22:24:51 +00:00
|
|
|
const (
|
|
|
|
bucketLeafFlag = 0x01
|
|
|
|
)
|
|
|
|
|
2014-01-24 23:32:18 +00:00
|
|
|
type pgid uint64
|
2014-01-09 16:07:10 +00:00
|
|
|
|
2014-01-08 15:06:17 +00:00
|
|
|
type page struct {
|
2014-01-27 15:11:54 +00:00
|
|
|
id pgid
|
|
|
|
flags uint16
|
|
|
|
count uint16
|
|
|
|
overflow uint32
|
|
|
|
ptr uintptr
|
2014-01-09 16:07:10 +00:00
|
|
|
}
|
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// typ returns a human readable page type string used for debugging.
|
|
|
|
func (p *page) typ() string {
|
2014-02-12 21:57:27 +00:00
|
|
|
if (p.flags & branchPageFlag) != 0 {
|
2014-01-31 18:18:51 +00:00
|
|
|
return "branch"
|
2014-02-12 21:57:27 +00:00
|
|
|
} else if (p.flags & leafPageFlag) != 0 {
|
2014-01-31 18:18:51 +00:00
|
|
|
return "leaf"
|
2014-02-12 21:57:27 +00:00
|
|
|
} else if (p.flags & metaPageFlag) != 0 {
|
2014-01-31 18:18:51 +00:00
|
|
|
return "meta"
|
2014-02-12 21:57:27 +00:00
|
|
|
} else if (p.flags & freelistPageFlag) != 0 {
|
2014-01-31 18:18:51 +00:00
|
|
|
return "freelist"
|
|
|
|
}
|
|
|
|
return fmt.Sprintf("unknown<%02x>", p.flags)
|
|
|
|
}
|
|
|
|
|
2014-01-10 14:32:12 +00:00
|
|
|
// meta returns a pointer to the metadata section of the page.
|
2014-01-30 03:35:58 +00:00
|
|
|
func (p *page) meta() *meta {
|
|
|
|
return (*meta)(unsafe.Pointer(&p.ptr))
|
2014-01-27 15:11:54 +00:00
|
|
|
}
|
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// leafPageElement retrieves the leaf node by index
|
|
|
|
func (p *page) leafPageElement(index uint16) *leafPageElement {
|
2014-06-18 17:53:01 +00:00
|
|
|
n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index]
|
2014-01-31 18:18:51 +00:00
|
|
|
return n
|
2014-01-17 22:23:39 +00:00
|
|
|
}
|
2014-01-30 03:35:58 +00:00
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// leafPageElements retrieves a list of leaf nodes.
|
|
|
|
func (p *page) leafPageElements() []leafPageElement {
|
2016-08-18 14:44:57 +00:00
|
|
|
if p.count == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
2014-06-18 17:53:01 +00:00
|
|
|
return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:]
|
2014-01-30 05:11:46 +00:00
|
|
|
}
|
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// branchPageElement retrieves the branch node by index
|
|
|
|
func (p *page) branchPageElement(index uint16) *branchPageElement {
|
2014-06-18 17:53:01 +00:00
|
|
|
return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index]
|
2014-01-30 03:35:58 +00:00
|
|
|
}
|
2014-01-30 03:50:29 +00:00
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// branchPageElements retrieves a list of branch nodes.
|
|
|
|
func (p *page) branchPageElements() []branchPageElement {
|
2016-08-18 14:44:57 +00:00
|
|
|
if p.count == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
2014-06-18 17:53:01 +00:00
|
|
|
return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:]
|
2014-01-30 05:11:46 +00:00
|
|
|
}
|
|
|
|
|
2014-01-31 18:18:51 +00:00
|
|
|
// dump writes n bytes of the page to STDERR as hex output.
|
|
|
|
func (p *page) hexdump(n int) {
|
|
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:n]
|
|
|
|
fmt.Fprintf(os.Stderr, "%x\n", buf)
|
|
|
|
}
|
|
|
|
|
2014-01-30 23:22:02 +00:00
|
|
|
type pages []*page
|
|
|
|
|
|
|
|
func (s pages) Len() int { return len(s) }
|
|
|
|
func (s pages) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
|
|
|
func (s pages) Less(i, j int) bool { return s[i].id < s[j].id }
|
2014-01-31 18:18:51 +00:00
|
|
|
|
|
|
|
// branchPageElement represents a node on a branch page.
|
|
|
|
type branchPageElement struct {
|
|
|
|
pos uint32
|
|
|
|
ksize uint32
|
|
|
|
pgid pgid
|
|
|
|
}
|
|
|
|
|
|
|
|
// key returns a byte slice of the node key.
|
|
|
|
func (n *branchPageElement) key() []byte {
|
|
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
fix `slice bounds out of range`/maxAllocSize bugs
when accessing the node data we used to use cast to
*[maxAllocSize]byte, which breaks if we try to go across maxAllocSize boundary.
This leads to occasional panics.
Sample stacktrace:
```
panic: runtime error: slice bounds out of range
goroutine 1 [running]:
github.com/boltdb/bolt.(*node).write(0xc208010f50, 0xc27452a000)
$GOPATH/src/github.com/boltdb/bolt/node.go:228 +0x5a5
github.com/boltdb/bolt.(*node).spill(0xc208010f50, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:364 +0x506
github.com/boltdb/bolt.(*node).spill(0xc208010700, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d
github.com/boltdb/bolt.(*node).spill(0xc208010620, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d
github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae880, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:535 +0x1c4
github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae840, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2
github.com/boltdb/bolt.(*Bucket).spill(0xc22f4e2018, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2
github.com/boltdb/bolt.(*Tx).Commit(0xc22f4e2000, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/tx.go:150 +0x1ee
github.com/boltdb/bolt.(*DB).Update(0xc2080e4000, 0xc24d077508, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/db.go:483 +0x169
```
It usually happens when working with large (50M/100M) values.
One way to reproduce it is to change maxAllocSize in bolt_amd64.go to 70000 and run the tests.
TestBucket_Put_Large crashes.
2015-03-26 23:47:24 +00:00
|
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
|
2014-01-31 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// leafPageElement represents a node on a leaf page.
|
|
|
|
type leafPageElement struct {
|
|
|
|
flags uint32
|
|
|
|
pos uint32
|
|
|
|
ksize uint32
|
|
|
|
vsize uint32
|
|
|
|
}
|
|
|
|
|
|
|
|
// key returns a byte slice of the node key.
|
|
|
|
func (n *leafPageElement) key() []byte {
|
|
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
2016-03-25 14:58:56 +00:00
|
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize]
|
2014-01-31 18:18:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// value returns a byte slice of the node value.
|
|
|
|
func (n *leafPageElement) value() []byte {
|
|
|
|
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
2016-03-25 14:58:56 +00:00
|
|
|
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize]
|
2014-01-31 18:18:51 +00:00
|
|
|
}
|
2014-03-22 04:34:54 +00:00
|
|
|
|
|
|
|
// PageInfo represents human readable information about a page.
|
|
|
|
type PageInfo struct {
|
|
|
|
ID int
|
|
|
|
Type string
|
|
|
|
Count int
|
|
|
|
OverflowCount int
|
|
|
|
}
|
2014-05-19 18:08:33 +00:00
|
|
|
|
|
|
|
type pgids []pgid
|
|
|
|
|
|
|
|
func (s pgids) Len() int { return len(s) }
|
|
|
|
func (s pgids) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
2014-05-19 20:11:32 +00:00
|
|
|
func (s pgids) Less(i, j int) bool { return s[i] < s[j] }
|
2015-06-13 01:32:55 +00:00
|
|
|
|
|
|
|
// merge returns the sorted union of a and b.
|
|
|
|
func (a pgids) merge(b pgids) pgids {
|
|
|
|
// Return the opposite slice if one is nil.
|
|
|
|
if len(a) == 0 {
|
|
|
|
return b
|
2016-12-20 22:04:46 +00:00
|
|
|
}
|
|
|
|
if len(b) == 0 {
|
2015-06-13 01:32:55 +00:00
|
|
|
return a
|
|
|
|
}
|
2016-12-20 22:04:46 +00:00
|
|
|
merged := make(pgids, len(a)+len(b))
|
|
|
|
mergepgids(merged, a, b)
|
|
|
|
return merged
|
|
|
|
}
|
|
|
|
|
2016-12-23 01:05:52 +00:00
|
|
|
// mergepgids copies the sorted union of a and b into dst.
|
2016-12-20 22:04:46 +00:00
|
|
|
// If dst is too small, it panics.
|
|
|
|
func mergepgids(dst, a, b pgids) {
|
|
|
|
if len(dst) < len(a)+len(b) {
|
|
|
|
panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b)))
|
|
|
|
}
|
|
|
|
// Copy in the opposite slice if one is nil.
|
|
|
|
if len(a) == 0 {
|
|
|
|
copy(dst, b)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if len(b) == 0 {
|
|
|
|
copy(dst, a)
|
|
|
|
return
|
|
|
|
}
|
2015-06-13 01:32:55 +00:00
|
|
|
|
2016-12-20 22:04:46 +00:00
|
|
|
// Merged will hold all elements from both lists.
|
|
|
|
merged := dst[:0]
|
2015-06-13 01:32:55 +00:00
|
|
|
|
|
|
|
// Assign lead to the slice with a lower starting value, follow to the higher value.
|
|
|
|
lead, follow := a, b
|
|
|
|
if b[0] < a[0] {
|
|
|
|
lead, follow = b, a
|
|
|
|
}
|
|
|
|
|
|
|
|
// Continue while there are elements in the lead.
|
|
|
|
for len(lead) > 0 {
|
|
|
|
// Merge largest prefix of lead that is ahead of follow[0].
|
|
|
|
n := sort.Search(len(lead), func(i int) bool { return lead[i] > follow[0] })
|
|
|
|
merged = append(merged, lead[:n]...)
|
|
|
|
if n >= len(lead) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// Swap lead and follow.
|
|
|
|
lead, follow = follow, lead[n:]
|
|
|
|
}
|
|
|
|
|
|
|
|
// Append what's left in follow.
|
2016-12-20 22:04:46 +00:00
|
|
|
_ = append(merged, follow...)
|
2015-06-13 01:32:55 +00:00
|
|
|
}
|