hak5
/
bolt
mirror of https://github.com/hak5/bolt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor bolt CLI.

master
Ben Johnson 2015-04-14 16:32:20 -06:00
parent 3b449559cf
commit d0e8a99e30
15 changed files with 913 additions and 1172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

425
cmd/bolt/bench.go
View File

@ -1,425 +0,0 @@
package main
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"os"
"runtime"
"runtime/pprof"
"time"
"github.com/boltdb/bolt"
)
// File handlers for the various profiles.
var cpuprofile, memprofile, blockprofile *os.File
var benchBucketName = []byte("bench")
// Bench executes a customizable, synthetic benchmark against Bolt.
func Bench(options *BenchOptions) {
var results BenchResults
// Validate options.
if options.BatchSize == 0 {
options.BatchSize = options.Iterations
} else if options.Iterations%options.BatchSize != 0 {
fatal("number of iterations must be divisible by the batch size")
}
// Generate temp path if one is not passed in.
path := options.Path
if path == "" {
path = tempfile()
}
if options.Clean {
defer os.Remove(path)
} else {
println("work:", path)
}
// Create database.
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
db.NoSync = options.NoSync
defer db.Close()
// Enable streaming stats.
if options.StatsInterval > 0 {
go printStats(db, options.StatsInterval)
}
// Start profiling for writes.
if options.ProfileMode == "rw" || options.ProfileMode == "w" {
benchStartProfiling(options)
}
// Write to the database.
if err := benchWrite(db, options, &results); err != nil {
fatal("bench: write: ", err)
}
// Stop profiling for writes only.
if options.ProfileMode == "w" {
benchStopProfiling()
}
// Start profiling for reads.
if options.ProfileMode == "r" {
benchStartProfiling(options)
}
// Read from the database.
if err := benchRead(db, options, &results); err != nil {
fatal("bench: read: ", err)
}
// Stop profiling for writes only.
if options.ProfileMode == "rw" || options.ProfileMode == "r" {
benchStopProfiling()
}
// Print results.
fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond())
fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond())
fmt.Fprintln(os.Stderr, "")
}
// Writes to the database.
func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var err error
var t = time.Now()
switch options.WriteMode {
case "seq":
err = benchWriteSequential(db, options, results)
case "rnd":
err = benchWriteRandom(db, options, results)
case "seq-nest":
err = benchWriteSequentialNested(db, options, results)
case "rnd-nest":
err = benchWriteRandomNested(db, options, results)
default:
return fmt.Errorf("invalid write mode: %s", options.WriteMode)
}
results.WriteDuration = time.Since(t)
return err
}
func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var i = uint32(0)
return benchWriteWithSource(db, options, results, func() uint32 { i++; return i })
}
func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() })
}
func benchWriteSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var i = uint32(0)
return benchWriteNestedWithSource(db, options, results, func() uint32 { i++; return i })
}
func benchWriteRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return benchWriteNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
}
func benchWriteWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
b.FillPercent = options.FillPercent
for j := 0; j < options.BatchSize; j++ {
var key = make([]byte, options.KeySize)
var value = make([]byte, options.ValueSize)
binary.BigEndian.PutUint32(key, keySource())
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
}
return nil
}
func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
err := db.Update(func(tx *bolt.Tx) error {
top, _ := tx.CreateBucketIfNotExists(benchBucketName)
top.FillPercent = options.FillPercent
var name = make([]byte, options.KeySize)
binary.BigEndian.PutUint32(name, keySource())
b, _ := top.CreateBucketIfNotExists(name)
b.FillPercent = options.FillPercent
for j := 0; j < options.BatchSize; j++ {
var key = make([]byte, options.KeySize)
var value = make([]byte, options.ValueSize)
binary.BigEndian.PutUint32(key, keySource())
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
})
if err != nil {
return err
}
}
return nil
}
// Reads from the database.
func benchRead(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var err error
var t = time.Now()
switch options.ReadMode {
case "seq":
if options.WriteMode == "seq-nest" || options.WriteMode == "rnd-nest" {
err = benchReadSequentialNested(db, options, results)
} else {
err = benchReadSequential(db, options, results)
}
default:
return fmt.Errorf("invalid read mode: %s", options.ReadMode)
}
results.ReadDuration = time.Since(t)
return err
}
func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
var t = time.Now()
for {
c := tx.Bucket(benchBucketName).Cursor()
var count int
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return errors.New("invalid value")
}
count++
}
if options.WriteMode == "seq" && count != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
func benchReadSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
var t = time.Now()
for {
var count int
var top = tx.Bucket(benchBucketName)
top.ForEach(func(name, _ []byte) error {
c := top.Bucket(name).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return errors.New("invalid value")
}
count++
}
return nil
})
if options.WriteMode == "seq-nest" && count != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
// Starts all profiles set on the options.
func benchStartProfiling(options *BenchOptions) {
var err error
// Start CPU profiling.
if options.CPUProfile != "" {
cpuprofile, err = os.Create(options.CPUProfile)
if err != nil {
fatalf("bench: could not create cpu profile %q: %v", options.CPUProfile, err)
}
pprof.StartCPUProfile(cpuprofile)
}
// Start memory profiling.
if options.MemProfile != "" {
memprofile, err = os.Create(options.MemProfile)
if err != nil {
fatalf("bench: could not create memory profile %q: %v", options.MemProfile, err)
}
runtime.MemProfileRate = 4096
}
// Start fatal profiling.
if options.BlockProfile != "" {
blockprofile, err = os.Create(options.BlockProfile)
if err != nil {
fatalf("bench: could not create block profile %q: %v", options.BlockProfile, err)
}
runtime.SetBlockProfileRate(1)
}
}
// Stops all profiles.
func benchStopProfiling() {
if cpuprofile != nil {
pprof.StopCPUProfile()
cpuprofile.Close()
cpuprofile = nil
}
if memprofile != nil {
pprof.Lookup("heap").WriteTo(memprofile, 0)
memprofile.Close()
memprofile = nil
}
if blockprofile != nil {
pprof.Lookup("block").WriteTo(blockprofile, 0)
blockprofile.Close()
blockprofile = nil
runtime.SetBlockProfileRate(0)
}
}
// Continuously prints stats on the database at given intervals.
func printStats(db *bolt.DB, interval time.Duration) {
var prevStats = db.Stats()
var encoder = json.NewEncoder(os.Stdout)
for {
// Wait for the stats interval.
time.Sleep(interval)
// Retrieve new stats and find difference from previous iteration.
var stats = db.Stats()
var diff = stats.Sub(&prevStats)
// Print as JSON to STDOUT.
if err := encoder.Encode(diff); err != nil {
fatal(err)
}
// Save stats for next iteration.
prevStats = stats
}
}
// BenchOptions represents the set of options that can be passed to Bench().
type BenchOptions struct {
ProfileMode string
WriteMode string
ReadMode string
Iterations int
BatchSize int
KeySize int
ValueSize int
CPUProfile string
MemProfile string
BlockProfile string
StatsInterval time.Duration
FillPercent float64
NoSync bool
Clean bool
Path string
}
// BenchResults represents the performance results of the benchmark.
type BenchResults struct {
WriteOps int
WriteDuration time.Duration
ReadOps int
ReadDuration time.Duration
}
// Returns the duration for a single write operation.
func (r *BenchResults) WriteOpDuration() time.Duration {
if r.WriteOps == 0 {
return 0
}
return r.WriteDuration / time.Duration(r.WriteOps)
}
// Returns average number of write operations that can be performed per second.
func (r *BenchResults) WriteOpsPerSecond() int {
var op = r.WriteOpDuration()
if op == 0 {
return 0
}
return int(time.Second) / int(op)
}
// Returns the duration for a single read operation.
func (r *BenchResults) ReadOpDuration() time.Duration {
if r.ReadOps == 0 {
return 0
}
return r.ReadDuration / time.Duration(r.ReadOps)
}
// Returns average number of read operations that can be performed per second.
func (r *BenchResults) ReadOpsPerSecond() int {
var op = r.ReadOpDuration()
if op == 0 {
return 0
}
return int(time.Second) / int(op)
}
// tempfile returns a temporary file path.
func tempfile() string {
f, _ := ioutil.TempFile("", "bolt-bench-")
f.Close()
os.Remove(f.Name())
return f.Name()
}

33
cmd/bolt/buckets.go
View File

@ -1,33 +0,0 @@
package main
import (
"os"
"github.com/boltdb/bolt"
)
// Buckets prints a list of all buckets.
func Buckets(path string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
err = db.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
println(string(name))
return nil
})
})
if err != nil {
fatal(err)
return
}
}

31
cmd/bolt/buckets_test.go
View File

@ -1,31 +0,0 @@
package main_test
import (
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
)
// Ensure that a list of buckets can be retrieved.
func TestBuckets(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
tx.CreateBucket([]byte("woojits"))
tx.CreateBucket([]byte("widgets"))
tx.CreateBucket([]byte("whatchits"))
return nil
})
db.Close()
output := run("buckets", path)
equals(t, "whatchits\nwidgets\nwoojits", output)
})
}
// Ensure that an error is reported if the database is not found.
func TestBucketsDBNotFound(t *testing.T) {
SetTestMode(true)
output := run("buckets", "no/such/db")
equals(t, "stat no/such/db: no such file or directory", output)
}

47
cmd/bolt/check.go
View File

@ -1,47 +0,0 @@
package main
import (
"os"
"github.com/boltdb/bolt"
)
// Check performs a consistency check on the database and prints any errors found.
func Check(path string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
// Perform consistency check.
_ = db.View(func(tx *bolt.Tx) error {
var count int
ch := tx.Check()
loop:
for {
select {
case err, ok := <-ch:
if !ok {
break loop
}
println(err)
count++
}
}
// Print summary of errors.
if count > 0 {
fatalf("%d errors found", count)
} else {
println("OK")
}
return nil
})
}

45
cmd/bolt/get.go
View File

@ -1,45 +0,0 @@
package main
import (
"os"
"github.com/boltdb/bolt"
)
// Get retrieves the value for a given bucket/key.
func Get(path, name, key string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
err = db.View(func(tx *bolt.Tx) error {
// Find bucket.
b := tx.Bucket([]byte(name))
if b == nil {
fatalf("bucket not found: %s", name)
return nil
}
// Find value for a given key.
value := b.Get([]byte(key))
if value == nil {
fatalf("key not found: %s", key)
return nil
}
println(string(value))
return nil
})
if err != nil {
fatal(err)
return
}
}

54
cmd/bolt/get_test.go
View File

@ -1,54 +0,0 @@
package main_test
import (
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
)
// Ensure that a value can be retrieved from the CLI.
func TestGet(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
tx.CreateBucket([]byte("widgets"))
tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar"))
return nil
})
db.Close()
output := run("get", path, "widgets", "foo")
equals(t, "bar", output)
})
}
// Ensure that an error is reported if the database is not found.
func TestGetDBNotFound(t *testing.T) {
SetTestMode(true)
output := run("get", "no/such/db", "widgets", "foo")
equals(t, "stat no/such/db: no such file or directory", output)
}
// Ensure that an error is reported if the bucket is not found.
func TestGetBucketNotFound(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Close()
output := run("get", path, "widgets", "foo")
equals(t, "bucket not found: widgets", output)
})
}
// Ensure that an error is reported if the key is not found.
func TestGetKeyNotFound(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("widgets"))
return err
})
db.Close()
output := run("get", path, "widgets", "foo")
equals(t, "key not found: foo", output)
})
}

26
cmd/bolt/info.go
View File

@ -1,26 +0,0 @@
package main
import (
"os"
"github.com/boltdb/bolt"
)
// Info prints basic information about a database.
func Info(path string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
// Print basic database info.
var info = db.Info()
printf("Page Size: %d\n", info.PageSize)
}

31
cmd/bolt/info_test.go
View File

@ -1,31 +0,0 @@
package main_test
import (
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
)
// Ensure that a database info can be printed.
func TestInfo(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
tx.CreateBucket([]byte("widgets"))
b := tx.Bucket([]byte("widgets"))
b.Put([]byte("foo"), []byte("0000"))
return nil
})
db.Close()
output := run("info", path)
equals(t, `Page Size: 4096`, output)
})
}
// Ensure that an error is reported if the database is not found.
func TestInfo_NotFound(t *testing.T) {
SetTestMode(true)
output := run("info", "no/such/db")
equals(t, "stat no/such/db: no such file or directory", output)
}

41
cmd/bolt/keys.go
View File

@ -1,41 +0,0 @@
package main
import (
"os"
"github.com/boltdb/bolt"
)
// Keys retrieves a list of keys for a given bucket.
func Keys(path, name string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
err = db.View(func(tx *bolt.Tx) error {
// Find bucket.
b := tx.Bucket([]byte(name))
if b == nil {
fatalf("bucket not found: %s", name)
return nil
}
// Iterate over each key.
return b.ForEach(func(key, _ []byte) error {
println(string(key))
return nil
})
})
if err != nil {
fatal(err)
return
}
}

42
cmd/bolt/keys_test.go
View File

@ -1,42 +0,0 @@
package main_test
import (
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
)
// Ensure that a list of keys can be retrieved for a given bucket.
func TestKeys(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
tx.CreateBucket([]byte("widgets"))
tx.Bucket([]byte("widgets")).Put([]byte("0002"), []byte(""))
tx.Bucket([]byte("widgets")).Put([]byte("0001"), []byte(""))
tx.Bucket([]byte("widgets")).Put([]byte("0003"), []byte(""))
return nil
})
db.Close()
output := run("keys", path, "widgets")
equals(t, "0001\n0002\n0003", output)
})
}
// Ensure that an error is reported if the database is not found.
func TestKeysDBNotFound(t *testing.T) {
SetTestMode(true)
output := run("keys", "no/such/db", "widgets")
equals(t, "stat no/such/db: no such file or directory", output)
}
// Ensure that an error is reported if the bucket is not found.
func TestKeysBucketNotFound(t *testing.T) {
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Close()
output := run("keys", path, "widgets")
equals(t, "bucket not found: widgets", output)
})
}

951
cmd/bolt/main.go
View File

@ -2,199 +2,834 @@ package main
import (
"bytes"
"encoding/binary"
"errors"
"flag"
"fmt"
"log"
"io"
"io/ioutil"
"math/rand"
"os"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"time"
"github.com/boltdb/bolt"
"github.com/codegangsta/cli"
)
var branch, commit string
var (
// ErrCommandRequired is returned when a CLI command is not specified.
ErrCommandRequired = errors.New("command required")
// ErrUnknownCommand is returned when a CLI command is not specified.
ErrUnknownCommand = errors.New("unknown command")
// ErrPathRequired is returned when the path to a Bolt database is not specified.
ErrPathRequired = errors.New("path required")
// ErrFileNotFound is returned when a Bolt database does not exist.
ErrFileNotFound = errors.New("file not found")
// ErrInvalidValue is returned when a benchmark reads an unexpected value.
ErrInvalidValue = errors.New("invalid value")
// ErrCorrupt is returned when a checking a data file finds errors.
ErrCorrupt = errors.New("invalid value")
// ErrNonDivisibleBatchSize is returned when the batch size can't be evenly
// divided by the iteration count.
ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size")
)
func main() {
log.SetFlags(0)
NewApp().Run(os.Args)
m := NewMain()
if err := m.Run(os.Args[1:]...); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}
// NewApp creates an Application instance.
func NewApp() *cli.App {
app := cli.NewApp()
app.Name = "bolt"
app.Usage = "BoltDB toolkit"
app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit)
app.Commands = []cli.Command{
{
Name: "info",
Usage: "Print basic information about a database",
Action: func(c *cli.Context) {
path := c.Args().Get(0)
Info(path)
},
},
{
Name: "get",
Usage: "Retrieve a value for given key in a bucket",
Action: func(c *cli.Context) {
path, name, key := c.Args().Get(0), c.Args().Get(1), c.Args().Get(2)
Get(path, name, key)
},
},
{
Name: "keys",
Usage: "Retrieve a list of all keys in a bucket",
Action: func(c *cli.Context) {
path, name := c.Args().Get(0), c.Args().Get(1)
Keys(path, name)
},
},
{
Name: "buckets",
Usage: "Retrieves a list of all buckets",
Action: func(c *cli.Context) {
path := c.Args().Get(0)
Buckets(path)
},
},
{
Name: "pages",
Usage: "Dumps page information for a database",
Action: func(c *cli.Context) {
path := c.Args().Get(0)
Pages(path)
},
},
{
Name: "check",
Usage: "Performs a consistency check on the database",
Action: func(c *cli.Context) {
path := c.Args().Get(0)
Check(path)
},
},
{
Name: "stats",
Usage: "Aggregate statistics for all buckets matching specified prefix",
Action: func(c *cli.Context) {
path, name := c.Args().Get(0), c.Args().Get(1)
Stats(path, name)
},
},
{
Name: "bench",
Usage: "Performs a synthetic benchmark",
Flags: []cli.Flag{
&cli.StringFlag{Name: "profile-mode", Value: "rw", Usage: "Profile mode"},
&cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"},
&cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"},
&cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"},
&cli.IntFlag{Name: "batch-size", Usage: "Write batch size"},
&cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"},
&cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"},
&cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"},
&cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"},
&cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"},
&cli.StringFlag{Name: "stats-interval", Value: "0s", Usage: "Continuous stats interval"},
&cli.Float64Flag{Name: "fill-percent", Value: bolt.DefaultFillPercent, Usage: "Fill percentage"},
&cli.BoolFlag{Name: "no-sync", Usage: "Skip fsync on every commit"},
&cli.BoolFlag{Name: "work", Usage: "Print the temp db and do not delete on exit"},
&cli.StringFlag{Name: "path", Usage: "Path to database to use"},
},
Action: func(c *cli.Context) {
statsInterval, err := time.ParseDuration(c.String("stats-interval"))
if err != nil {
fatal(err)
// Main represents the main program execution.
type Main struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewMain returns a new instance of Main connect to the standard input/output.
func NewMain() *Main {
return &Main{
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run executes the program.
func (m *Main) Run(args ...string) error {
// Require a command at the beginning.
if len(args) == 0 || strings.HasPrefix(args[0], "-") {
return ErrCommandRequired
}
// Execute command.
switch args[0] {
case "bench":
return newBenchCommand(m).Run(args[1:]...)
case "check":
return newCheckCommand(m).Run(args[1:]...)
case "info":
return newInfoCommand(m).Run(args[1:]...)
case "pages":
return newPagesCommand(m).Run(args[1:]...)
case "stats":
return newStatsCommand(m).Run(args[1:]...)
default:
return ErrUnknownCommand
}
}
// CheckCommand represents the "check" command execution.
type CheckCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewCheckCommand returns a CheckCommand.
func newCheckCommand(m *Main) *CheckCommand {
return &CheckCommand{
Stdin: m.Stdin,
Stdout: m.Stdout,
Stderr: m.Stderr,
}
}
// Run executes the command.
func (cmd *CheckCommand) Run(args ...string) error {
// Parse flags.
fs := flag.NewFlagSet("", flag.ContinueOnError)
if err := fs.Parse(args); err != nil {
return err
}
// Require database path.
path := fs.Arg(0)
if path == "" {
return ErrPathRequired
} else if _, err := os.Stat(path); os.IsNotExist(err) {
return ErrFileNotFound
}
// Open database.
db, err := bolt.Open(path, 0666, nil)
if err != nil {
return err
}
defer db.Close()
// Perform consistency check.
return db.View(func(tx *bolt.Tx) error {
var count int
ch := tx.Check()
loop:
for {
select {
case err, ok := <-ch:
if !ok {
break loop
}
fmt.Fprintln(cmd.Stdout, err)
count++
}
}
Bench(&BenchOptions{
ProfileMode: c.String("profile-mode"),
WriteMode: c.String("write-mode"),
ReadMode: c.String("read-mode"),
Iterations: c.Int("count"),
BatchSize: c.Int("batch-size"),
KeySize: c.Int("key-size"),
ValueSize: c.Int("value-size"),
CPUProfile: c.String("cpuprofile"),
MemProfile: c.String("memprofile"),
BlockProfile: c.String("blockprofile"),
StatsInterval: statsInterval,
FillPercent: c.Float64("fill-percent"),
NoSync: c.Bool("no-sync"),
Clean: !c.Bool("work"),
Path: c.String("path"),
})
},
}}
return app
// Print summary of errors.
if count > 0 {
fmt.Fprintf(cmd.Stdout, "%d errors found\n", count)
return ErrCorrupt
}
// Notify user that database is valid.
fmt.Fprintln(cmd.Stdout, "OK")
return nil
})
}
var logger = log.New(os.Stderr, "", 0)
var logBuffer *bytes.Buffer
// InfoCommand represents the "info" command execution.
type InfoCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
func print(v ...interface{}) {
if testMode {
logger.Print(v...)
// NewInfoCommand returns a InfoCommand.
func newInfoCommand(m *Main) *InfoCommand {
return &InfoCommand{
Stdin: m.Stdin,
Stdout: m.Stdout,
Stderr: m.Stderr,
}
}
// Run executes the command.
func (cmd *InfoCommand) Run(args ...string) error {
// Parse flags.
fs := flag.NewFlagSet("", flag.ContinueOnError)
if err := fs.Parse(args); err != nil {
return err
}
// Require database path.
path := fs.Arg(0)
if path == "" {
return ErrPathRequired
} else if _, err := os.Stat(path); os.IsNotExist(err) {
return ErrFileNotFound
}
// Open the database.
db, err := bolt.Open(path, 0666, nil)
if err != nil {
return err
}
defer db.Close()
// Print basic database info.
info := db.Info()
fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize)
return nil
}
// PagesCommand represents the "pages" command execution.
type PagesCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewPagesCommand returns a PagesCommand.
func newPagesCommand(m *Main) *PagesCommand {
return &PagesCommand{
Stdin: m.Stdin,
Stdout: m.Stdout,
Stderr: m.Stderr,
}
}
// Run executes the command.
func (cmd *PagesCommand) Run(args ...string) error {
// Parse flags.
fs := flag.NewFlagSet("", flag.ContinueOnError)
if err := fs.Parse(args); err != nil {
return err
}
// Require database path.
path := fs.Arg(0)
if path == "" {
return ErrPathRequired
} else if _, err := os.Stat(path); os.IsNotExist(err) {
return ErrFileNotFound
}
// Open database.
db, err := bolt.Open(path, 0666, nil)
if err != nil {
return err
}
defer func() { _ = db.Close() }()
// Write header.
fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW")
fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======")
return db.Update(func(tx *bolt.Tx) error {
var id int
for {
p, err := tx.Page(id)
if err != nil {
return &PageError{ID: id, Err: err}
} else if p == nil {
break
}
// Only display count and overflow if this is a non-free page.
var count, overflow string
if p.Type != "free" {
count = strconv.Itoa(p.Count)
if p.OverflowCount > 0 {
overflow = strconv.Itoa(p.OverflowCount)
}
}
// Print table row.
fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow)
// Move to the next non-overflow page.
id += 1
if p.Type != "free" {
id += p.OverflowCount
}
}
return nil
})
}
// StatsCommand represents the "stats" command execution.
type StatsCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewStatsCommand returns a StatsCommand.
func newStatsCommand(m *Main) *StatsCommand {
return &StatsCommand{
Stdin: m.Stdin,
Stdout: m.Stdout,
Stderr: m.Stderr,
}
}
// Run executes the command.
func (cmd *StatsCommand) Run(args ...string) error {
// Parse flags.
fs := flag.NewFlagSet("", flag.ContinueOnError)
if err := fs.Parse(args); err != nil {
return err
}
// Require database path.
path, prefix := fs.Arg(0), fs.Arg(1)
if path == "" {
return ErrPathRequired
} else if _, err := os.Stat(path); os.IsNotExist(err) {
return ErrFileNotFound
}
// Open database.
db, err := bolt.Open(path, 0666, nil)
if err != nil {
return err
}
defer db.Close()
return db.View(func(tx *bolt.Tx) error {
var s bolt.BucketStats
var count int
if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
if bytes.HasPrefix(name, []byte(prefix)) {
s.Add(b.Stats())
count += 1
}
return nil
}); err != nil {
return err
}
fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count)
fmt.Fprintln(cmd.Stdout, "Page count statistics")
fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN)
fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN)
fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN)
fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN)
fmt.Fprintln(cmd.Stdout, "Tree statistics")
fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN)
fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth)
fmt.Fprintln(cmd.Stdout, "Page size utilization")
fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc)
var percentage int
if s.BranchAlloc != 0 {
percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc))
}
fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage)
fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc)
percentage = 0
if s.LeafAlloc != 0 {
percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc))
}
fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage)
fmt.Fprintln(cmd.Stdout, "Bucket statistics")
fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN)
percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN))
fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage)
percentage = 0
if s.LeafInuse != 0 {
percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse))
}
fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage)
return nil
})
}
var benchBucketName = []byte("bench")
// BenchCommand represents the "bench" command execution.
type BenchCommand struct {
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewBenchCommand returns a BenchCommand using the
func newBenchCommand(m *Main) *BenchCommand {
return &BenchCommand{
Stdin: m.Stdin,
Stdout: m.Stdout,
Stderr: m.Stderr,
}
}
// Run executes the "bench" command.
func (cmd *BenchCommand) Run(args ...string) error {
// Parse CLI arguments.
options, err := cmd.ParseFlags(args)
if err != nil {
return err
}
// Remove path if "-work" is not set. Otherwise keep path.
if options.Work {
fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path)
} else {
fmt.Print(v...)
defer os.Remove(options.Path)
}
// Create database.
db, err := bolt.Open(options.Path, 0666, nil)
if err != nil {
return err
}
db.NoSync = options.NoSync
defer db.Close()
// Write to the database.
var results BenchResults
if err := cmd.runWrites(db, options, &results); err != nil {
return fmt.Errorf("write: ", err)
}
// Read from the database.
if err := cmd.runReads(db, options, &results); err != nil {
return fmt.Errorf("bench: read: %s", err)
}
// Print results.
fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond())
fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond())
fmt.Fprintln(os.Stderr, "")
return nil
}
// ParseFlags parses the command line flags.
func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) {
var options BenchOptions
// Parse flagset.
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "")
fs.StringVar(&options.WriteMode, "write-mode", "seq", "")
fs.StringVar(&options.ReadMode, "read-mode", "seq", "")
fs.IntVar(&options.Iterations, "count", 1000, "")
fs.IntVar(&options.BatchSize, "batch-size", 0, "")
fs.IntVar(&options.KeySize, "key-size", 8, "")
fs.IntVar(&options.ValueSize, "value-size", 32, "")
fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
fs.StringVar(&options.MemProfile, "memprofile", "", "")
fs.StringVar(&options.BlockProfile, "blockprofile", "", "")
fs.StringVar(&options.BlockProfile, "blockprofile", "", "")
fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "")
fs.BoolVar(&options.NoSync, "no-sync", false, "")
fs.BoolVar(&options.Work, "work", false, "")
fs.StringVar(&options.Path, "path", "", "")
fs.SetOutput(cmd.Stderr)
if err := fs.Parse(args); err != nil {
return nil, err
}
// Set batch size to iteration size if not set.
// Require that batch size can be evenly divided by the iteration count.
if options.BatchSize == 0 {
options.BatchSize = options.Iterations
} else if options.Iterations%options.BatchSize != 0 {
return nil, ErrNonDivisibleBatchSize
}
// Generate temp path if one is not passed in.
if options.Path == "" {
f, err := ioutil.TempFile("", "bolt-bench-")
if err != nil {
return nil, fmt.Errorf("temp file: %s", err)
}
f.Close()
os.Remove(f.Name())
options.Path = f.Name()
}
return &options, nil
}
// Writes to the database.
func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
// Start profiling for writes.
if options.ProfileMode == "rw" || options.ProfileMode == "w" {
cmd.startProfiling(options)
}
t := time.Now()
var err error
switch options.WriteMode {
case "seq":
err = cmd.runWritesSequential(db, options, results)
case "rnd":
err = cmd.runWritesRandom(db, options, results)
case "seq-nest":
err = cmd.runWritesSequentialNested(db, options, results)
case "rnd-nest":
err = cmd.runWritesRandomNested(db, options, results)
default:
return fmt.Errorf("invalid write mode: %s", options.WriteMode)
}
// Save time to write.
results.WriteDuration = time.Since(t)
// Stop profiling for writes only.
if options.ProfileMode == "w" {
cmd.stopProfiling()
}
return err
}
func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var i = uint32(0)
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
}
func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
}
func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
var i = uint32(0)
return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
}
func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
}
func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
b, _ := tx.CreateBucketIfNotExists(benchBucketName)
b.FillPercent = options.FillPercent
for j := 0; j < options.BatchSize; j++ {
key := make([]byte, options.KeySize)
value := make([]byte, options.ValueSize)
// Write key as uint32.
binary.BigEndian.PutUint32(key, keySource())
// Insert key/value.
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
}
return nil
}
func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
results.WriteOps = options.Iterations
for i := 0; i < options.Iterations; i += options.BatchSize {
if err := db.Update(func(tx *bolt.Tx) error {
top, err := tx.CreateBucketIfNotExists(benchBucketName)
if err != nil {
return err
}
top.FillPercent = options.FillPercent
// Create bucket key.
name := make([]byte, options.KeySize)
binary.BigEndian.PutUint32(name, keySource())
// Create bucket.
b, err := top.CreateBucketIfNotExists(name)
if err != nil {
return err
}
b.FillPercent = options.FillPercent
for j := 0; j < options.BatchSize; j++ {
var key = make([]byte, options.KeySize)
var value = make([]byte, options.ValueSize)
// Generate key as uint32.
binary.BigEndian.PutUint32(key, keySource())
// Insert value into subbucket.
if err := b.Put(key, value); err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
}
return nil
}
// Reads from the database.
func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
// Start profiling for reads.
if options.ProfileMode == "r" {
cmd.startProfiling(options)
}
t := time.Now()
var err error
switch options.ReadMode {
case "seq":
switch options.WriteMode {
case "seq-nest", "rnd-nest":
err = cmd.runReadsSequentialNested(db, options, results)
default:
err = cmd.runReadsSequential(db, options, results)
}
default:
return fmt.Errorf("invalid read mode: %s", options.ReadMode)
}
// Save read time.
results.ReadDuration = time.Since(t)
// Stop profiling for reads.
if options.ProfileMode == "rw" || options.ProfileMode == "r" {
cmd.stopProfiling()
}
return err
}
func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
for {
var count int
c := tx.Bucket(benchBucketName).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return errors.New("invalid value")
}
count++
}
if options.WriteMode == "seq" && count != options.Iterations {
return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
return db.View(func(tx *bolt.Tx) error {
t := time.Now()
for {
var count int
var top = tx.Bucket(benchBucketName)
if err := top.ForEach(func(name, _ []byte) error {
c := top.Bucket(name).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if v == nil {
return ErrInvalidValue
}
count++
}
return nil
}); err != nil {
return err
}
if options.WriteMode == "seq-nest" && count != options.Iterations {
return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
}
results.ReadOps += count
// Make sure we do this for at least a second.
if time.Since(t) >= time.Second {
break
}
}
return nil
})
}
// File handlers for the various profiles.
var cpuprofile, memprofile, blockprofile *os.File
// Starts all profiles set on the options.
func (cmd *BenchCommand) startProfiling(options *BenchOptions) {
var err error
// Start CPU profiling.
if options.CPUProfile != "" {
cpuprofile, err = os.Create(options.CPUProfile)
if err != nil {
fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err)
os.Exit(1)
}
pprof.StartCPUProfile(cpuprofile)
}
// Start memory profiling.
if options.MemProfile != "" {
memprofile, err = os.Create(options.MemProfile)
if err != nil {
fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err)
os.Exit(1)
}
runtime.MemProfileRate = 4096
}
// Start fatal profiling.
if options.BlockProfile != "" {
blockprofile, err = os.Create(options.BlockProfile)
if err != nil {
fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err)
os.Exit(1)
}
runtime.SetBlockProfileRate(1)
}
}
func printf(format string, v ...interface{}) {
if testMode {
logger.Printf(format, v...)
} else {
fmt.Printf(format, v...)
// Stops all profiles.
func (cmd *BenchCommand) stopProfiling() {
if cpuprofile != nil {
pprof.StopCPUProfile()
cpuprofile.Close()
cpuprofile = nil
}
if memprofile != nil {
pprof.Lookup("heap").WriteTo(memprofile, 0)
memprofile.Close()
memprofile = nil
}
if blockprofile != nil {
pprof.Lookup("block").WriteTo(blockprofile, 0)
blockprofile.Close()
blockprofile = nil
runtime.SetBlockProfileRate(0)
}
}
func println(v ...interface{}) {
if testMode {
logger.Println(v...)
} else {
fmt.Println(v...)
}
// BenchOptions represents the set of options that can be passed to "bolt bench".
type BenchOptions struct {
ProfileMode string
WriteMode string
ReadMode string
Iterations int
BatchSize int
KeySize int
ValueSize int
CPUProfile string
MemProfile string
BlockProfile string
StatsInterval time.Duration
FillPercent float64
NoSync bool
Work bool
Path string
}
func fatal(v ...interface{}) {
logger.Print(v...)
if !testMode {
os.Exit(1)
}
// BenchResults represents the performance results of the benchmark.
type BenchResults struct {
WriteOps int
WriteDuration time.Duration
ReadOps int
ReadDuration time.Duration
}
func fatalf(format string, v ...interface{}) {
logger.Printf(format, v...)
if !testMode {
os.Exit(1)
// Returns the duration for a single write operation.
func (r *BenchResults) WriteOpDuration() time.Duration {
if r.WriteOps == 0 {
return 0
}
return r.WriteDuration / time.Duration(r.WriteOps)
}
func fatalln(v ...interface{}) {
logger.Println(v...)
if !testMode {
os.Exit(1)
// Returns average number of write operations that can be performed per second.
func (r *BenchResults) WriteOpsPerSecond() int {
var op = r.WriteOpDuration()
if op == 0 {
return 0
}
return int(time.Second) / int(op)
}
// LogBuffer returns the contents of the log.
// This only works while the CLI is in test mode.
func LogBuffer() string {
if logBuffer != nil {
return logBuffer.String()
// Returns the duration for a single read operation.
func (r *BenchResults) ReadOpDuration() time.Duration {
if r.ReadOps == 0 {
return 0
}
return ""
return r.ReadDuration / time.Duration(r.ReadOps)
}
var testMode bool
// SetTestMode sets whether the CLI is running in test mode and resets the logger.
func SetTestMode(value bool) {
testMode = value
if testMode {
logBuffer = bytes.NewBuffer(nil)
logger = log.New(logBuffer, "", 0)
} else {
logger = log.New(os.Stderr, "", 0)
// Returns average number of read operations that can be performed per second.
func (r *BenchResults) ReadOpsPerSecond() int {
var op = r.ReadOpDuration()
if op == 0 {
return 0
}
return int(time.Second) / int(op)
}
type PageError struct {
ID int
Err error
}
func (e *PageError) Error() string {
return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err)
}

164
cmd/bolt/main_test.go
View File

@ -1,69 +1,145 @@
package main_test
import (
"fmt"
"bytes"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"strconv"
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
"github.com/boltdb/bolt/cmd/bolt"
)
// open creates and opens a Bolt database in the temp directory.
func open(fn func(*bolt.DB, string)) {
path := tempfile()
defer os.RemoveAll(path)
// Ensure the "info" command can print information about a database.
func TestInfoCommand_Run(t *testing.T) {
db := MustOpen(0666, nil)
db.DB.Close()
defer db.Close()
db, err := bolt.Open(path, 0600, nil)
if err != nil {
panic("db open error: " + err.Error())
// Run the info command.
m := NewMain()
if err := m.Run("info", db.Path); err != nil {
t.Fatal(err)
}
fn(db, path)
}
// run executes a command against the CLI and returns the output.
func run(args ...string) string {
args = append([]string{"bolt"}, args...)
NewApp().Run(args)
return strings.TrimSpace(LogBuffer())
// Ensure the "stats" command can execute correctly.
func TestStatsCommand_Run(t *testing.T) {
// Ignore
if os.Getpagesize() != 4096 {
t.Skip("system does not use 4KB page size")
}
db := MustOpen(0666, nil)
defer db.Close()
if err := db.Update(func(tx *bolt.Tx) error {
// Create "foo" bucket.
b, err := tx.CreateBucket([]byte("foo"))
if err != nil {
return err
}
for i := 0; i < 10; i++ {
if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil {
return err
}
}
// Create "bar" bucket.
b, err = tx.CreateBucket([]byte("bar"))
if err != nil {
return err
}
for i := 0; i < 100; i++ {
if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil {
return err
}
}
// Create "baz" bucket.
b, err = tx.CreateBucket([]byte("baz"))
if err != nil {
return err
}
if err := b.Put([]byte("key"), []byte("value")); err != nil {
return err
}
return nil
}); err != nil {
t.Fatal(err)
}
db.DB.Close()
// Generate expected result.
exp := "Aggregate statistics for 3 buckets\n\n" +
"Page count statistics\n" +
"\tNumber of logical branch pages: 0\n" +
"\tNumber of physical branch overflow pages: 0\n" +
"\tNumber of logical leaf pages: 1\n" +
"\tNumber of physical leaf overflow pages: 0\n" +
"Tree statistics\n" +
"\tNumber of keys/value pairs: 111\n" +
"\tNumber of levels in B+tree: 1\n" +
"Page size utilization\n" +
"\tBytes allocated for physical branch pages: 0\n" +
"\tBytes actually used for branch data: 0 (0%)\n" +
"\tBytes allocated for physical leaf pages: 4096\n" +
"\tBytes actually used for leaf data: 1996 (48%)\n" +
"Bucket statistics\n" +
"\tTotal number of buckets: 3\n" +
"\tTotal number on inlined buckets: 2 (66%)\n" +
"\tBytes used for inlined buckets: 236 (11%)\n"
// Run the command.
m := NewMain()
if err := m.Run("stats", db.Path); err != nil {
t.Fatal(err)
} else if m.Stdout.String() != exp {
t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String())
}
}
// tempfile returns a temporary file path.
func tempfile() string {
// Main represents a test wrapper for main.Main that records output.
type Main struct {
*main.Main
Stdin bytes.Buffer
Stdout bytes.Buffer
Stderr bytes.Buffer
}
// NewMain returns a new instance of Main.
func NewMain() *Main {
m := &Main{Main: main.NewMain()}
m.Main.Stdin = &m.Stdin
m.Main.Stdout = &m.Stdout
m.Main.Stderr = &m.Stderr
return m
}
// MustOpen creates a Bolt database in a temporary location.
func MustOpen(mode os.FileMode, options *bolt.Options) *DB {
// Create temporary path.
f, _ := ioutil.TempFile("", "bolt-")
f.Close()
os.Remove(f.Name())
return f.Name()
}
// assert fails the test if the condition is false.
func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
if !condition {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
tb.FailNow()
}
}
// ok fails the test if an err is not nil.
func ok(tb testing.TB, err error) {
db, err := bolt.Open(f.Name(), mode, options)
if err != nil {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
tb.FailNow()
panic(err.Error())
}
return &DB{DB: db, Path: f.Name()}
}
// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
if !reflect.DeepEqual(exp, act) {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
tb.FailNow()
}
// DB is a test wrapper for bolt.DB.
type DB struct {
*bolt.DB
Path string
}
// Close closes and removes the database.
func (db *DB) Close() error {
defer os.Remove(db.Path)
return db.DB.Close()
}

57
cmd/bolt/pages.go
View File

@ -1,57 +0,0 @@
package main
import (
"os"
"strconv"
"github.com/boltdb/bolt"
)
// Pages prints a list of all pages in a database.
func Pages(path string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
println("ID TYPE ITEMS OVRFLW")
println("======== ========== ====== ======")
db.Update(func(tx *bolt.Tx) error {
var id int
for {
p, err := tx.Page(id)
if err != nil {
fatalf("page error: %d: %s", id, err)
} else if p == nil {
break
}
// Only display count and overflow if this is a non-free page.
var count, overflow string
if p.Type != "free" {
count = strconv.Itoa(p.Count)
if p.OverflowCount > 0 {
overflow = strconv.Itoa(p.OverflowCount)
}
}
// Print table row.
printf("%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow)
// Move to the next non-overflow page.
id += 1
if p.Type != "free" {
id += p.OverflowCount
}
}
return nil
})
}

77
cmd/bolt/stats.go
View File

@ -1,77 +0,0 @@
package main
import (
"bytes"
"os"
"github.com/boltdb/bolt"
)
// Collect stats for all top level buckets matching the prefix.
func Stats(path, prefix string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
fatal(err)
return
}
db, err := bolt.Open(path, 0600, nil)
if err != nil {
fatal(err)
return
}
defer db.Close()
err = db.View(func(tx *bolt.Tx) error {
var s bolt.BucketStats
var count int
var prefix = []byte(prefix)
tx.ForEach(func(name []byte, b *bolt.Bucket) error {
if bytes.HasPrefix(name, prefix) {
s.Add(b.Stats())
count += 1
}
return nil
})
printf("Aggregate statistics for %d buckets\n\n", count)
println("Page count statistics")
printf("\tNumber of logical branch pages: %d\n", s.BranchPageN)
printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN)
printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN)
printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN)
println("Tree statistics")
printf("\tNumber of keys/value pairs: %d\n", s.KeyN)
printf("\tNumber of levels in B+tree: %d\n", s.Depth)
println("Page size utilization")
printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc)
var percentage int
if s.BranchAlloc != 0 {
percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc))
}
printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage)
printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc)
percentage = 0
if s.LeafAlloc != 0 {
percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc))
}
printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage)
println("Bucket statistics")
printf("\tTotal number of buckets: %d\n", s.BucketN)
percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN))
printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage)
percentage = 0
if s.LeafInuse != 0 {
percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse))
}
printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage)
return nil
})
if err != nil {
fatal(err)
return
}
}

61
cmd/bolt/stats_test.go
View File

@ -1,61 +0,0 @@
package main_test
import (
"os"
"strconv"
"testing"
"github.com/boltdb/bolt"
. "github.com/boltdb/bolt/cmd/bolt"
)
func TestStats(t *testing.T) {
if os.Getpagesize() != 4096 {
t.Skip()
}
SetTestMode(true)
open(func(db *bolt.DB, path string) {
db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucket([]byte("foo"))
if err != nil {
return err
}
for i := 0; i < 10; i++ {
b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i)))
}
b, err = tx.CreateBucket([]byte("bar"))
if err != nil {
return err
}
for i := 0; i < 100; i++ {
b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i)))
}
b, err = tx.CreateBucket([]byte("baz"))
if err != nil {
return err
}
b.Put([]byte("key"), []byte("value"))
return nil
})
db.Close()
output := run("stats", path, "b")
equals(t, "Aggregate statistics for 2 buckets\n\n"+
"Page count statistics\n"+
"\tNumber of logical branch pages: 0\n"+
"\tNumber of physical branch overflow pages: 0\n"+
"\tNumber of logical leaf pages: 1\n"+
"\tNumber of physical leaf overflow pages: 0\n"+
"Tree statistics\n"+
"\tNumber of keys/value pairs: 101\n"+
"\tNumber of levels in B+tree: 1\n"+
"Page size utilization\n"+
"\tBytes allocated for physical branch pages: 0\n"+
"\tBytes actually used for branch data: 0 (0%)\n"+
"\tBytes allocated for physical leaf pages: 4096\n"+
"\tBytes actually used for leaf data: 1996 (48%)\n"+
"Bucket statistics\n"+
"\tTotal number of buckets: 2\n"+
"\tTotal number on inlined buckets: 1 (50%)\n"+
"\tBytes used for inlined buckets: 40 (2%)", output)
})
}