mirror of https://github.com/hak5/bolt.git
fix `slice bounds out of range`/maxAllocSize bugs
when accessing the node data we used to use cast to *[maxAllocSize]byte, which breaks if we try to go across maxAllocSize boundary. This leads to occasional panics. Sample stacktrace: ``` panic: runtime error: slice bounds out of range goroutine 1 [running]: github.com/boltdb/bolt.(*node).write(0xc208010f50, 0xc27452a000) $GOPATH/src/github.com/boltdb/bolt/node.go:228 +0x5a5 github.com/boltdb/bolt.(*node).spill(0xc208010f50, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/node.go:364 +0x506 github.com/boltdb/bolt.(*node).spill(0xc208010700, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d github.com/boltdb/bolt.(*node).spill(0xc208010620, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae880, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/bucket.go:535 +0x1c4 github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae840, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2 github.com/boltdb/bolt.(*Bucket).spill(0xc22f4e2018, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2 github.com/boltdb/bolt.(*Tx).Commit(0xc22f4e2000, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/tx.go:150 +0x1ee github.com/boltdb/bolt.(*DB).Update(0xc2080e4000, 0xc24d077508, 0x0, 0x0) $GOPATH/src/github.com/boltdb/bolt/db.go:483 +0x169 ``` It usually happens when working with large (50M/100M) values. One way to reproduce it is to change maxAllocSize in bolt_amd64.go to 70000 and run the tests. TestBucket_Put_Large crashes.master
parent
3b449559cf
commit
bdc109bdc7
|
@ -5,3 +5,7 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
|||
|
||||
// maxAllocSize is the size used when creating array pointers.
|
||||
const maxAllocSize = 0x7FFFFFFF
|
||||
|
||||
// Setting
|
||||
// const maxAllocSize = 70000
|
||||
// reveals the index out of bound bug(s)
|
||||
|
|
12
node.go
12
node.go
|
@ -220,12 +220,16 @@ func (n *node) write(p *page) {
|
|||
elem.pgid = item.pgid
|
||||
_assert(elem.pgid != p.id, "write: circular dependency occurred")
|
||||
}
|
||||
lk, lv := len(item.key), len(item.value)
|
||||
if len(b) < lk+lv {
|
||||
b = (*[maxAllocSize]byte)(unsafe.Pointer(&b[0]))[:]
|
||||
}
|
||||
|
||||
// Write data for the element to the end of the page.
|
||||
copy(b[0:], item.key)
|
||||
b = b[len(item.key):]
|
||||
b = b[lk:]
|
||||
copy(b[0:], item.value)
|
||||
b = b[len(item.value):]
|
||||
b = b[lv:]
|
||||
}
|
||||
|
||||
// DEBUG ONLY: n.dump()
|
||||
|
@ -351,7 +355,9 @@ func (n *node) spill() error {
|
|||
}
|
||||
|
||||
// Allocate contiguous space for the node.
|
||||
p, err := tx.allocate((node.size() / tx.db.pageSize) + 1)
|
||||
// sz := node.size() + n.pageElementSize()*len(n.inodes)
|
||||
sz := node.size()
|
||||
p, err := tx.allocate((sz / tx.db.pageSize) + 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
6
page.go
6
page.go
|
@ -96,7 +96,7 @@ type branchPageElement struct {
|
|||
// key returns a byte slice of the node key.
|
||||
func (n *branchPageElement) key() []byte {
|
||||
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
||||
return buf[n.pos : n.pos+n.ksize]
|
||||
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
|
||||
}
|
||||
|
||||
// leafPageElement represents a node on a leaf page.
|
||||
|
@ -110,13 +110,13 @@ type leafPageElement struct {
|
|||
// key returns a byte slice of the node key.
|
||||
func (n *leafPageElement) key() []byte {
|
||||
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
||||
return buf[n.pos : n.pos+n.ksize]
|
||||
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
|
||||
}
|
||||
|
||||
// value returns a byte slice of the node value.
|
||||
func (n *leafPageElement) value() []byte {
|
||||
buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
|
||||
return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize]
|
||||
return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize]
|
||||
}
|
||||
|
||||
// PageInfo represents human readable information about a page.
|
||||
|
|
24
tx.go
24
tx.go
|
@ -421,14 +421,26 @@ func (tx *Tx) write() error {
|
|||
// Write pages to disk in order.
|
||||
for _, p := range pages {
|
||||
size := (int(p.overflow) + 1) * tx.db.pageSize
|
||||
buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:size]
|
||||
offset := int64(p.id) * int64(tx.db.pageSize)
|
||||
if _, err := tx.db.ops.writeAt(buf, offset); err != nil {
|
||||
return err
|
||||
ptr := (*[maxAllocSize]byte)(unsafe.Pointer(p))
|
||||
for {
|
||||
sz := size
|
||||
if sz > maxAllocSize-1 {
|
||||
sz = maxAllocSize - 1
|
||||
}
|
||||
buf := ptr[:sz]
|
||||
if _, err := tx.db.ops.writeAt(buf, offset); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update statistics.
|
||||
tx.stats.Write++
|
||||
size -= sz
|
||||
if size == 0 {
|
||||
break
|
||||
}
|
||||
offset += int64(sz)
|
||||
ptr = (*[maxAllocSize]byte)(unsafe.Pointer(&ptr[sz]))
|
||||
}
|
||||
|
||||
// Update statistics.
|
||||
tx.stats.Write++
|
||||
}
|
||||
if !tx.db.NoSync || IgnoreNoSync {
|
||||
if err := fdatasync(tx.db); err != nil {
|
||||
|
|
Loading…
Reference in New Issue