Merge branch 'grow' of https://github.com/xiang90/bolt into xiang90-grow

bolt_unix.go

@@ -46,17 +46,6 @@ func funlock(f *os.File) error {
 
 // mmap memory maps a DB's data file.
 func mmap(db *DB, sz int) error {
-	// Truncate and fsync to ensure file size metadata is flushed.
-	// https://github.com/boltdb/bolt/issues/284
-	if !db.NoGrowSync && !db.readOnly {
-		if err := db.file.Truncate(int64(sz)); err != nil {
-			return fmt.Errorf("file resize error: %s", err)
-		}
-		if err := db.file.Sync(); err != nil {
-			return fmt.Errorf("file sync error: %s", err)
-		}
-	}
-
 	// Map the data file to memory.
 	b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags)
 	if err != nil {

bolt_unix_solaris.go

@@ -56,17 +56,6 @@ func funlock(f *os.File) error {
 
 // mmap memory maps a DB's data file.
 func mmap(db *DB, sz int) error {
-	// Truncate and fsync to ensure file size metadata is flushed.
-	// https://github.com/boltdb/bolt/issues/284
-	if !db.NoGrowSync && !db.readOnly {
-		if err := db.file.Truncate(int64(sz)); err != nil {
-			return fmt.Errorf("file resize error: %s", err)
-		}
-		if err := db.file.Sync(); err != nil {
-			return fmt.Errorf("file sync error: %s", err)
-		}
-	}
-
 	// Map the data file to memory.
 	b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags)
 	if err != nil {

db.go

@@ -90,6 +90,7 @@ type DB struct {
 	dataref  []byte // mmap'ed readonly, write throws SEGV
 	data     *[maxMapSize]byte
 	datasz   int
+	filesz   int // current on disk file size
 	meta0    *meta
 	meta1    *meta
 	pageSize int
@@ -798,6 +799,37 @@ func (db *DB) allocate(count int) (*page, error) {
 	return p, nil
 }
 
+// growSize grows the size of the database to the given sz.
+func (db *DB) growSize(sz int) error {
+	if sz <= db.filesz {
+		return nil
+	}
+
+	// over allocate 16MB to avoid calling Truncate aggressively
+	// for efficiency
+	overAllocation := 16 * 1024 * 1024
+	sz = sz + overAllocation
+
+	// do not over allocate
+	if sz > db.datasz {
+		sz = db.datasz
+	}
+
+	// Truncate and fsync to ensure file size metadata is flushed.
+	// https://github.com/boltdb/bolt/issues/284
+	if !db.NoGrowSync && !db.readOnly {
+		if err := db.file.Truncate(int64(sz)); err != nil {
+			return fmt.Errorf("file resize error: %s", err)
+		}
+		if err := db.file.Sync(); err != nil {
+			return fmt.Errorf("file sync error: %s", err)
+		}
+	}
+
+	db.filesz = sz
+	return nil
+}
+
 func (db *DB) IsReadOnly() bool {
 	return db.readOnly
 }

db_test.go

@@ -229,6 +229,8 @@ func TestOpen_Size(t *testing.T) {
 	path := db.Path()
 	defer db.MustClose()
 
+	pagesize := db.Info().PageSize
+
 	// Insert until we get above the minimum 4MB size.
 	if err := db.Update(func(tx *bolt.Tx) error {
 		b, _ := tx.CreateBucketIfNotExists([]byte("data"))
@@ -273,7 +275,8 @@ func TestOpen_Size(t *testing.T) {
 	}
 
 	// Compare the original size with the new size.
-	if sz != newSz {
+	// db size might increase by a few page sizes due to the new small update.
+	if sz < newSz-5*int64(pagesize) {
 		t.Fatalf("unexpected file growth: %d => %d", sz, newSz)
 	}
 }
@@ -290,6 +293,8 @@ func TestOpen_Size_Large(t *testing.T) {
 	path := db.Path()
 	defer db.MustClose()
 
+	pagesize := db.Info().PageSize
+
 	// Insert until we get above the minimum 4MB size.
 	var index uint64
 	for i := 0; i < 10000; i++ {
@@ -338,7 +343,8 @@ func TestOpen_Size_Large(t *testing.T) {
 	}
 
 	// Compare the original size with the new size.
-	if sz != newSz {
+	// db size might increase by a few page sizes due to the new small update.
+	if sz < newSz-5*int64(pagesize) {
 		t.Fatalf("unexpected file growth: %d => %d", sz, newSz)
 	}
 }

tx.go

@@ -168,6 +168,8 @@ func (tx *Tx) Commit() error {
 	// Free the old root bucket.
 	tx.meta.root.root = tx.root.root
 
+	opgid := tx.meta.pgid
+
 	// Free the freelist and allocate new pages for it. This will overestimate
 	// the size of the freelist but not underestimate the size (which would be bad).
 	tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
@@ -182,6 +184,10 @@ func (tx *Tx) Commit() error {
 	}
 	tx.meta.freelist = p.id
 
+	if tx.meta.pgid > opgid {
+		tx.db.growSize(int(tx.meta.pgid+1) * tx.db.pageSize)
+	}
+
 	// Write dirty pages to disk.
 	startTime = time.Now()
 	if err := tx.write(); err != nil {