From e67705ed6348675b7bae405ebeb37bb69b53a96d Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Wed, 4 Nov 2015 15:12:18 -0800 Subject: [PATCH] do not grow dbsize agressively Only grow the database size when the high watermark increases. We also grows the database size a little bit aggressively to save a few ftruncates. I have tested this on various environments. The performance impact is ignorable with 16MB over allocation. Without over allocation, the performance might decrease 100% when each Tx.Commit needs a new page on a very slow disk (seek time dominates the total write). --- bolt_unix.go | 11 ----------- bolt_unix_solaris.go | 13 +------------ db.go | 32 ++++++++++++++++++++++++++++++++ db_test.go | 10 ++++++++-- tx.go | 6 ++++++ 5 files changed, 47 insertions(+), 25 deletions(-) diff --git a/bolt_unix.go b/bolt_unix.go index 6eef6b2..5a199c7 100644 --- a/bolt_unix.go +++ b/bolt_unix.go @@ -46,17 +46,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go index f480ee7..13c800b 100644 --- a/bolt_unix_solaris.go +++ b/bolt_unix_solaris.go @@ -1,4 +1,3 @@ - package bolt import ( @@ -7,6 +6,7 @@ import ( "syscall" "time" "unsafe" + "golang.org/x/sys/unix" ) @@ -56,17 +56,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { diff --git a/db.go b/db.go index d39c4aa..fd920a0 100644 --- a/db.go +++ b/db.go @@ -84,6 +84,7 @@ type DB struct { dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int + filesz int // current on disk file size meta0 *meta meta1 *meta pageSize int @@ -655,6 +656,37 @@ func (db *DB) allocate(count int) (*page, error) { return p, nil } +// growSize grows the size of the database to the given sz. +func (db *DB) growSize(sz int) error { + if sz <= db.filesz { + return nil + } + + // over allocate 16MB to avoid calling Truncate aggressively + // for efficiency + overAllocation := 16 * 1024 * 1024 + sz = sz + overAllocation + + // do not over allocate + if sz > db.datasz { + sz = db.datasz + } + + // Truncate and fsync to ensure file size metadata is flushed. + // https://github.com/boltdb/bolt/issues/284 + if !db.NoGrowSync && !db.readOnly { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } + if err := db.file.Sync(); err != nil { + return fmt.Errorf("file sync error: %s", err) + } + } + + db.filesz = sz + return nil +} + func (db *DB) IsReadOnly() bool { return db.readOnly } diff --git a/db_test.go b/db_test.go index ae21938..0f89c33 100644 --- a/db_test.go +++ b/db_test.go @@ -100,6 +100,8 @@ func TestOpen_Size(t *testing.T) { path := db.Path() defer db.Close() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. ok(t, db.Update(func(tx *bolt.Tx) error { b, _ := tx.CreateBucketIfNotExists([]byte("data")) @@ -127,7 +129,8 @@ func TestOpen_Size(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } @@ -144,6 +147,8 @@ func TestOpen_Size_Large(t *testing.T) { path := db.Path() defer db.Close() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. var index uint64 for i := 0; i < 10000; i++ { @@ -177,7 +182,8 @@ func TestOpen_Size_Large(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } diff --git a/tx.go b/tx.go index cb60149..80d9799 100644 --- a/tx.go +++ b/tx.go @@ -157,6 +157,8 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root + opgid := tx.meta.pgid + // Free the freelist and allocate new pages for it. This will overestimate // the size of the freelist but not underestimate the size (which would be bad). tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) @@ -171,6 +173,10 @@ func (tx *Tx) Commit() error { } tx.meta.freelist = p.id + if tx.meta.pgid > opgid { + tx.db.growSize(int(tx.meta.pgid+1) * tx.db.pageSize) + } + // Write dirty pages to disk. startTime = time.Now() if err := tx.write(); err != nil {