From e7f5c931e2e0471a72d88fda6fff738626dbb7d9 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 16 Feb 2015 15:22:12 -0700 Subject: [PATCH] Fix large mmap resize. This commit fixes an issue where large databases were being resized to larger sizes on every open. --- db.go | 4 ++-- db_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/db.go b/db.go index 773620d..4775850 100644 --- a/db.go +++ b/db.go @@ -231,9 +231,9 @@ func (db *DB) mmapSize(size int) (int, error) { } // If larger than 1GB then grow by 1GB at a time. - sz := int64(size) + int64(maxMmapStep) + sz := int64(size) if remainder := sz % int64(maxMmapStep); remainder > 0 { - sz -= remainder + sz += int64(maxMmapStep) - remainder } // Ensure that the mmap size is a multiple of the page size. diff --git a/db_test.go b/db_test.go index 4f05ad9..ecff7d8 100644 --- a/db_test.go +++ b/db_test.go @@ -1,6 +1,7 @@ package bolt_test import ( + "encoding/binary" "errors" "flag" "fmt" @@ -125,6 +126,56 @@ func TestOpen_Size(t *testing.T) { } } +// Ensure that opening a database beyond the max step size does not increase its size. +// https://github.com/boltdb/bolt/issues/303 +func TestOpen_Size_Large(t *testing.T) { + if testing.Short() { + t.Skip("short mode") + } + + // Open a data file. + db := NewTestDB() + path := db.Path() + defer db.Close() + + // Insert until we get above the minimum 4MB size. + var index uint64 + for i := 0; i < 10000; i++ { + ok(t, db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists([]byte("data")) + for j := 0; j < 1000; j++ { + ok(t, b.Put(u64tob(index), make([]byte, 50))) + index++ + } + return nil + })) + } + + // Close database and grab the size. + db.DB.Close() + sz := fileSize(path) + if sz == 0 { + t.Fatalf("unexpected new file size: %d", sz) + } else if sz < (1 << 30) { + t.Fatalf("expected larger initial size: %d", sz) + } + + // Reopen database, update, and check size again. + db0, err := bolt.Open(path, 0666, nil) + ok(t, err) + ok(t, db0.Update(func(tx *bolt.Tx) error { return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) })) + ok(t, db0.Close()) + newSz := fileSize(path) + if newSz == 0 { + t.Fatalf("unexpected new file size: %d", newSz) + } + + // Compare the original size with the new size. + if sz != newSz { + t.Fatalf("unexpected file growth: %d => %d", sz, newSz) + } +} + // Ensure that a re-opened database is consistent. func TestOpen_Check(t *testing.T) { path := tempfile() @@ -699,3 +750,13 @@ func fileSize(path string) int64 { func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } + +// u64tob converts a uint64 into an 8-byte slice. +func u64tob(v uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, v) + return b +} + +// btou64 converts an 8-byte slice into an uint64. +func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }