From 0cae98efc5fd76f0f5159b6615f29ccf8bf97aa8 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 3 Feb 2014 14:33:51 -0700 Subject: [PATCH 1/2] Add RWTransaction.Delete(). --- db_test.go | 14 ++++++++++++++ node.go | 14 ++++++++++++++ rwtransaction.go | 20 +++++++++++++++----- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/db_test.go b/db_test.go index 50ec3c3..224cfc7 100644 --- a/db_test.go +++ b/db_test.go @@ -191,6 +191,20 @@ func TestDBPutRandom(t *testing.T) { } } +// Ensure that a bucket can delete an existing key. +func TestDBDelete(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Put("widgets", []byte("foo"), []byte("bar")) + err := db.Delete("widgets", []byte("foo")) + assert.NoError(t, err) + value, err := db.Get("widgets", []byte("foo")) + if assert.NoError(t, err) { + assert.Nil(t, value) + } + }) +} + // withDB executes a function with a database reference. func withDB(fn func(*DB, string)) { f, _ := ioutil.TempFile("", "bolt-") diff --git a/node.go b/node.go index 8d03681..2b3fded 100644 --- a/node.go +++ b/node.go @@ -61,6 +61,20 @@ func (n *node) put(oldKey, newKey, value []byte, pgid pgid) { inode.pgid = pgid } +// del removes a key from the node. +func (n *node) del(key []byte) { + // Find index of key. + index := sort.Search(len(n.inodes), func(i int) bool { return bytes.Compare(n.inodes[i].key, key) != -1 }) + + // Exit if the key isn't found. + if !bytes.Equal(n.inodes[index].key, key) { + return + } + + // Delete inode from the node. + n.inodes = append(n.inodes[:index], n.inodes[index+1:]...) +} + // read initializes the node from a page. func (n *node) read(p *page) { n.pgid = p.id diff --git a/rwtransaction.go b/rwtransaction.go index 2485c62..7911188 100644 --- a/rwtransaction.go +++ b/rwtransaction.go @@ -70,19 +70,29 @@ func (t *RWTransaction) Put(name string, key []byte, value []byte) error { return &Error{"data too large", nil} } - // Insert a new node. + // Move cursor to correct position. c := b.cursor() c.Get(key) + + // Insert the key/value. t.node(c.stack).put(key, key, value, 0) return nil } func (t *RWTransaction) Delete(name string, key []byte) error { - // TODO: Traverse to the correct node. - // TODO: If missing, exit. - // TODO: Remove node from page. - // TODO: If page is empty then add it to the freelist. + b := t.Bucket(name) + if b == nil { + return &Error{"bucket not found", nil} + } + + // Move cursor to correct position. + c := b.cursor() + c.Get(key) + + // Delete the node if we have a matching key. + t.node(c.stack).del(key) + return nil } From 8b3b81ef47d8eaa1f95e152943fd10b03b782034 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 4 Feb 2014 15:30:05 -0700 Subject: [PATCH 2/2] Fix quick tests. --- cursor.go | 1 - db_test.go | 38 --------------------- meta.go | 2 +- node.go | 12 ++++--- node_test.go | 2 +- quick_test.go | 81 +++++++++++++++++++++++++++++++++------------ rwtransaction.go | 5 ++- sys.go | 10 ------ transaction_test.go | 30 +++++++++++++++++ 9 files changed, 102 insertions(+), 79 deletions(-) create mode 100644 transaction_test.go diff --git a/cursor.go b/cursor.go index d56119b..3df325a 100644 --- a/cursor.go +++ b/cursor.go @@ -41,7 +41,6 @@ func (c *Cursor) Get(key []byte) []byte { } // If our target node isn't the same key as what's passed in then return nil. - // c.page().hexdump(512) if !bytes.Equal(key, c.node().key()) { return nil } diff --git a/db_test.go b/db_test.go index 224cfc7..565adb8 100644 --- a/db_test.go +++ b/db_test.go @@ -1,14 +1,11 @@ package bolt import ( - "bytes" - "fmt" "io" "io/ioutil" "os" "syscall" "testing" - "testing/quick" "time" "unsafe" @@ -156,41 +153,6 @@ func TestDBPut(t *testing.T) { }) } -// Ensure that a bucket can write random keys and values across multiple txns. -func TestDBPutRandom(t *testing.T) { - f := func(items testKeyValuePairs) bool { - withOpenDB(func(db *DB, path string) { - db.CreateBucket("widgets") - for _, item := range items { - if len(item.Key) == 0 { - continue - } - if err := db.Put("widgets", item.Key, item.Value); err != nil { - panic("put error: " + err.Error()) - } - } - for _, item := range items { - if len(item.Key) == 0 { - continue - } - value, err := db.Get("widgets", item.Key) - if err != nil { - panic("get error: " + err.Error()) - } - if !bytes.Equal(value, []byte(item.Value)) { - // db.CopyFile("/tmp/bolt.random.db") - t.Fatalf("value mismatch:\n%x\n%x", item.Value, value) - } - } - fmt.Fprint(os.Stderr, ".") - }) - return true - } - if err := quick.Check(f, qc()); err != nil { - t.Error(err) - } -} - // Ensure that a bucket can delete an existing key. func TestDBDelete(t *testing.T) { withOpenDB(func(db *DB, path string) { diff --git a/meta.go b/meta.go index 871d092..33f45d4 100644 --- a/meta.go +++ b/meta.go @@ -1,6 +1,6 @@ package bolt -const magic uint32 = 0xDEADC0DE +const magic uint32 = 0xED0CDAED type meta struct { magic uint32 diff --git a/node.go b/node.go index 2b3fded..a6cc334 100644 --- a/node.go +++ b/node.go @@ -148,14 +148,16 @@ func (n *node) split(pageSize int) []*node { threshold := pageSize / 2 // Group into smaller pages and target a given fill size. - size := 0 - current := &node{isLeaf: n.isLeaf} - nodes := make([]*node, 0) + size := pageHeaderSize + inodes := n.inodes + current := n + current.inodes = nil + var nodes []*node - for i, inode := range n.inodes { + for i, inode := range inodes { elemSize := n.pageElementSize() + len(inode.key) + len(inode.value) - if len(current.inodes) >= minKeysPerPage && i < len(n.inodes)-minKeysPerPage && size+elemSize > threshold { + if len(current.inodes) >= minKeysPerPage && i < len(inodes)-minKeysPerPage && size+elemSize > threshold { size = pageHeaderSize nodes = append(nodes, current) current = &node{isLeaf: n.isLeaf} diff --git a/node_test.go b/node_test.go index 15d6498..6334fbe 100644 --- a/node_test.go +++ b/node_test.go @@ -91,7 +91,7 @@ func TestNodeSplit(t *testing.T) { n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0) n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0) - // Split between 3 & 4. + // Split between 2 & 3. nodes := n.split(100) assert.Equal(t, len(nodes), 2) diff --git a/quick_test.go b/quick_test.go index d85249d..e7fb22a 100644 --- a/quick_test.go +++ b/quick_test.go @@ -1,9 +1,13 @@ package bolt import ( + "bytes" "flag" + "fmt" "math/rand" + "os" "reflect" + "testing" "testing/quick" "time" ) @@ -18,45 +22,78 @@ import ( // -quick.maxvsize The maximum size of a value. // -var seed, testMaxItemCount, testMaxKeySize, testMaxValueSize int +var qseed, qmaxitems, qmaxksize, qmaxvsize int func init() { - flag.IntVar(&seed, "quick.seed", int(time.Now().UnixNano())%100000, "") - flag.IntVar(&testMaxItemCount, "quick.maxitems", 1024, "") - flag.IntVar(&testMaxKeySize, "quick.maxksize", 1024, "") - flag.IntVar(&testMaxValueSize, "quick.maxvsize", 1024, "") - warn("seed:", seed) + flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") + flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") + flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") + flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") + flag.Parse() + warn("seed:", qseed) } -// qc creates a testing/quick configuration. -func qc() *quick.Config { - return &quick.Config{Rand: rand.New(rand.NewSource(int64(seed)))} +// Ensure that a bucket can write random keys and values across multiple txns. +func TestQuickPut(t *testing.T) { + index := 0 + f := func(items testdata) bool { + withOpenDB(func(db *DB, path string) { + m := make(map[string][]byte) + + db.CreateBucket("widgets") + + for _, item := range items { + if err := db.Put("widgets", item.Key, item.Value); err != nil { + panic("put error: " + err.Error()) + } + m[string(item.Key)] = item.Value + + // Verify all key/values so far. + i := 0 + for k, v := range m { + value, err := db.Get("widgets", []byte(k)) + if err != nil { + panic("get error: " + err.Error()) + } + if !bytes.Equal(value, v) { + db.CopyFile("/tmp/bolt.random.db") + t.Fatalf("value mismatch [run %d] (%d of %d):\nkey: %x\ngot: %x\nexp: %x", index, i, len(m), []byte(k), v, value) + } + i++ + } + } + + fmt.Fprint(os.Stderr, ".") + }) + index++ + return true + } + if err := quick.Check(f, &quick.Config{Rand: rand.New(rand.NewSource(int64(qseed)))}); err != nil { + t.Error(err) + } + fmt.Fprint(os.Stderr, "\n") } -type testKeyValuePairs []testKeyValuePair +type testdata []testdataitem -func (t testKeyValuePairs) Generate(rand *rand.Rand, size int) reflect.Value { - n := rand.Intn(testMaxItemCount-1) + 1 - items := make(testKeyValuePairs, n) +func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { + n := rand.Intn(qmaxitems-1) + 1 + items := make(testdata, n) for i := 0; i < n; i++ { - items[i].Generate(rand, size) + item := &items[i] + item.Key = randByteSlice(rand, 1, qmaxksize) + item.Value = randByteSlice(rand, 0, qmaxvsize) } return reflect.ValueOf(items) } -type testKeyValuePair struct { +type testdataitem struct { Key []byte Value []byte } -func (t testKeyValuePair) Generate(rand *rand.Rand, size int) reflect.Value { - t.Key = randByteSlice(rand, 1, testMaxKeySize) - t.Value = randByteSlice(rand, 0, testMaxValueSize) - return reflect.ValueOf(t) -} - func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { - n := rand.Intn(maxSize - minSize) + minSize + n := rand.Intn(maxSize-minSize) + minSize b := make([]byte, n) for i := 0; i < n; i++ { b[i] = byte(rand.Intn(255)) diff --git a/rwtransaction.go b/rwtransaction.go index 7911188..93e544b 100644 --- a/rwtransaction.go +++ b/rwtransaction.go @@ -247,7 +247,10 @@ func (t *RWTransaction) write() error { for _, p := range pages { size := (int(p.overflow) + 1) * t.db.pageSize buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:size] - t.db.file.WriteAt(buf, int64(p.id)*int64(t.db.pageSize)) + offset := int64(p.id) * int64(t.db.pageSize) + if _, err := t.db.file.WriteAt(buf, offset); err != nil { + return err + } } return nil diff --git a/sys.go b/sys.go index cf15413..ec1b858 100644 --- a/sys.go +++ b/sys.go @@ -25,16 +25,6 @@ func (s *sys) get(key string) *bucket { return s.buckets[key] } -// getByRoot retrieves a bucket by root page id. -func (s *sys) getByRoot(pgid pgid) *bucket { - for _, b := range s.buckets { - if b.root == pgid { - return b - } - } - panic("root not found") -} - // put sets a new value for a bucket. func (s *sys) put(key string, b *bucket) { s.buckets[key] = b diff --git a/transaction_test.go b/transaction_test.go new file mode 100644 index 0000000..55e8bde --- /dev/null +++ b/transaction_test.go @@ -0,0 +1,30 @@ +package bolt + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ensure that a Transaction can retrieve a bucket. +func TestTransactionBucketMissing(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + b, err := db.Bucket("widgets") + assert.NoError(t, err) + if assert.NotNil(t, b) { + assert.Equal(t, "widgets", b.Name()) + } + }) +} + +// Ensure that a Transaction retrieving a non-existent key returns nil. +func TestTransactionGetMising(t *testing.T) { + withOpenDB(func(db *DB, path string) { + db.CreateBucket("widgets") + db.Put("widgets", []byte("foo"), []byte("bar")) + value, err := db.Get("widgets", []byte("no_such_key")) + assert.NoError(t, err) + assert.Nil(t, value) + }) +}