diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go deleted file mode 100644 index 3ade8b8..0000000 --- a/cmd/bolt/bench.go +++ /dev/null @@ -1,425 +0,0 @@ -package main - -import ( - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "math/rand" - "os" - "runtime" - "runtime/pprof" - "time" - - "github.com/boltdb/bolt" -) - -// File handlers for the various profiles. -var cpuprofile, memprofile, blockprofile *os.File - -var benchBucketName = []byte("bench") - -// Bench executes a customizable, synthetic benchmark against Bolt. -func Bench(options *BenchOptions) { - var results BenchResults - - // Validate options. - if options.BatchSize == 0 { - options.BatchSize = options.Iterations - } else if options.Iterations%options.BatchSize != 0 { - fatal("number of iterations must be divisible by the batch size") - } - - // Generate temp path if one is not passed in. - path := options.Path - if path == "" { - path = tempfile() - } - - if options.Clean { - defer os.Remove(path) - } else { - println("work:", path) - } - - // Create database. - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - db.NoSync = options.NoSync - defer db.Close() - - // Enable streaming stats. - if options.StatsInterval > 0 { - go printStats(db, options.StatsInterval) - } - - // Start profiling for writes. - if options.ProfileMode == "rw" || options.ProfileMode == "w" { - benchStartProfiling(options) - } - - // Write to the database. - if err := benchWrite(db, options, &results); err != nil { - fatal("bench: write: ", err) - } - - // Stop profiling for writes only. - if options.ProfileMode == "w" { - benchStopProfiling() - } - - // Start profiling for reads. - if options.ProfileMode == "r" { - benchStartProfiling(options) - } - - // Read from the database. - if err := benchRead(db, options, &results); err != nil { - fatal("bench: read: ", err) - } - - // Stop profiling for writes only. - if options.ProfileMode == "rw" || options.ProfileMode == "r" { - benchStopProfiling() - } - - // Print results. - fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) - fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) - fmt.Fprintln(os.Stderr, "") -} - -// Writes to the database. -func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var err error - var t = time.Now() - - switch options.WriteMode { - case "seq": - err = benchWriteSequential(db, options, results) - case "rnd": - err = benchWriteRandom(db, options, results) - case "seq-nest": - err = benchWriteSequentialNested(db, options, results) - case "rnd-nest": - err = benchWriteRandomNested(db, options, results) - default: - return fmt.Errorf("invalid write mode: %s", options.WriteMode) - } - - results.WriteDuration = time.Since(t) - - return err -} - -func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return benchWriteWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func benchWriteSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return benchWriteNestedWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func benchWriteRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return benchWriteNestedWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func benchWriteWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, keySource()) - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err - } - } - return nil -} - -func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - err := db.Update(func(tx *bolt.Tx) error { - top, _ := tx.CreateBucketIfNotExists(benchBucketName) - top.FillPercent = options.FillPercent - - var name = make([]byte, options.KeySize) - binary.BigEndian.PutUint32(name, keySource()) - b, _ := top.CreateBucketIfNotExists(name) - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, keySource()) - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err - } - } - return nil -} - -// Reads from the database. -func benchRead(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var err error - var t = time.Now() - - switch options.ReadMode { - case "seq": - if options.WriteMode == "seq-nest" || options.WriteMode == "rnd-nest" { - err = benchReadSequentialNested(db, options, results) - } else { - err = benchReadSequential(db, options, results) - } - default: - return fmt.Errorf("invalid read mode: %s", options.ReadMode) - } - - results.ReadDuration = time.Since(t) - - return err -} - -func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - var t = time.Now() - - for { - c := tx.Bucket(benchBucketName).Cursor() - var count int - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return errors.New("invalid value") - } - count++ - } - - if options.WriteMode == "seq" && count != options.Iterations { - return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -func benchReadSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - var t = time.Now() - - for { - var count int - var top = tx.Bucket(benchBucketName) - top.ForEach(func(name, _ []byte) error { - c := top.Bucket(name).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return errors.New("invalid value") - } - count++ - } - return nil - }) - - if options.WriteMode == "seq-nest" && count != options.Iterations { - return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -// Starts all profiles set on the options. -func benchStartProfiling(options *BenchOptions) { - var err error - - // Start CPU profiling. - if options.CPUProfile != "" { - cpuprofile, err = os.Create(options.CPUProfile) - if err != nil { - fatalf("bench: could not create cpu profile %q: %v", options.CPUProfile, err) - } - pprof.StartCPUProfile(cpuprofile) - } - - // Start memory profiling. - if options.MemProfile != "" { - memprofile, err = os.Create(options.MemProfile) - if err != nil { - fatalf("bench: could not create memory profile %q: %v", options.MemProfile, err) - } - runtime.MemProfileRate = 4096 - } - - // Start fatal profiling. - if options.BlockProfile != "" { - blockprofile, err = os.Create(options.BlockProfile) - if err != nil { - fatalf("bench: could not create block profile %q: %v", options.BlockProfile, err) - } - runtime.SetBlockProfileRate(1) - } -} - -// Stops all profiles. -func benchStopProfiling() { - if cpuprofile != nil { - pprof.StopCPUProfile() - cpuprofile.Close() - cpuprofile = nil - } - - if memprofile != nil { - pprof.Lookup("heap").WriteTo(memprofile, 0) - memprofile.Close() - memprofile = nil - } - - if blockprofile != nil { - pprof.Lookup("block").WriteTo(blockprofile, 0) - blockprofile.Close() - blockprofile = nil - runtime.SetBlockProfileRate(0) - } -} - -// Continuously prints stats on the database at given intervals. -func printStats(db *bolt.DB, interval time.Duration) { - var prevStats = db.Stats() - var encoder = json.NewEncoder(os.Stdout) - - for { - // Wait for the stats interval. - time.Sleep(interval) - - // Retrieve new stats and find difference from previous iteration. - var stats = db.Stats() - var diff = stats.Sub(&prevStats) - - // Print as JSON to STDOUT. - if err := encoder.Encode(diff); err != nil { - fatal(err) - } - - // Save stats for next iteration. - prevStats = stats - } -} - -// BenchOptions represents the set of options that can be passed to Bench(). -type BenchOptions struct { - ProfileMode string - WriteMode string - ReadMode string - Iterations int - BatchSize int - KeySize int - ValueSize int - CPUProfile string - MemProfile string - BlockProfile string - StatsInterval time.Duration - FillPercent float64 - NoSync bool - Clean bool - Path string -} - -// BenchResults represents the performance results of the benchmark. -type BenchResults struct { - WriteOps int - WriteDuration time.Duration - ReadOps int - ReadDuration time.Duration -} - -// Returns the duration for a single write operation. -func (r *BenchResults) WriteOpDuration() time.Duration { - if r.WriteOps == 0 { - return 0 - } - return r.WriteDuration / time.Duration(r.WriteOps) -} - -// Returns average number of write operations that can be performed per second. -func (r *BenchResults) WriteOpsPerSecond() int { - var op = r.WriteOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -// Returns the duration for a single read operation. -func (r *BenchResults) ReadOpDuration() time.Duration { - if r.ReadOps == 0 { - return 0 - } - return r.ReadDuration / time.Duration(r.ReadOps) -} - -// Returns average number of read operations that can be performed per second. -func (r *BenchResults) ReadOpsPerSecond() int { - var op = r.ReadOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -// tempfile returns a temporary file path. -func tempfile() string { - f, _ := ioutil.TempFile("", "bolt-bench-") - f.Close() - os.Remove(f.Name()) - return f.Name() -} diff --git a/cmd/bolt/buckets.go b/cmd/bolt/buckets.go deleted file mode 100644 index 68e7dde..0000000 --- a/cmd/bolt/buckets.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Buckets prints a list of all buckets. -func Buckets(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - return tx.ForEach(func(name []byte, _ *bolt.Bucket) error { - println(string(name)) - return nil - }) - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/buckets_test.go b/cmd/bolt/buckets_test.go deleted file mode 100644 index d5050fd..0000000 --- a/cmd/bolt/buckets_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a list of buckets can be retrieved. -func TestBuckets(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("woojits")) - tx.CreateBucket([]byte("widgets")) - tx.CreateBucket([]byte("whatchits")) - return nil - }) - db.Close() - output := run("buckets", path) - equals(t, "whatchits\nwidgets\nwoojits", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestBucketsDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("buckets", "no/such/db") - equals(t, "stat no/such/db: no such file or directory", output) -} diff --git a/cmd/bolt/check.go b/cmd/bolt/check.go deleted file mode 100644 index 125f2b8..0000000 --- a/cmd/bolt/check.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Check performs a consistency check on the database and prints any errors found. -func Check(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - // Perform consistency check. - _ = db.View(func(tx *bolt.Tx) error { - var count int - ch := tx.Check() - loop: - for { - select { - case err, ok := <-ch: - if !ok { - break loop - } - println(err) - count++ - } - } - - // Print summary of errors. - if count > 0 { - fatalf("%d errors found", count) - } else { - println("OK") - } - return nil - }) -} diff --git a/cmd/bolt/get.go b/cmd/bolt/get.go deleted file mode 100644 index 90e0c1d..0000000 --- a/cmd/bolt/get.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Get retrieves the value for a given bucket/key. -func Get(path, name, key string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) - return nil - } - - // Find value for a given key. - value := b.Get([]byte(key)) - if value == nil { - fatalf("key not found: %s", key) - return nil - } - - println(string(value)) - return nil - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/get_test.go b/cmd/bolt/get_test.go deleted file mode 100644 index 8acd0f4..0000000 --- a/cmd/bolt/get_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a value can be retrieved from the CLI. -func TestGet(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")) - return nil - }) - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "bar", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestGetDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("get", "no/such/db", "widgets", "foo") - equals(t, "stat no/such/db: no such file or directory", output) -} - -// Ensure that an error is reported if the bucket is not found. -func TestGetBucketNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "bucket not found: widgets", output) - }) -} - -// Ensure that an error is reported if the key is not found. -func TestGetKeyNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }) - db.Close() - output := run("get", path, "widgets", "foo") - equals(t, "key not found: foo", output) - }) -} diff --git a/cmd/bolt/info.go b/cmd/bolt/info.go deleted file mode 100644 index cb01e38..0000000 --- a/cmd/bolt/info.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Info prints basic information about a database. -func Info(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - // Print basic database info. - var info = db.Info() - printf("Page Size: %d\n", info.PageSize) -} diff --git a/cmd/bolt/info_test.go b/cmd/bolt/info_test.go deleted file mode 100644 index dab74f6..0000000 --- a/cmd/bolt/info_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a database info can be printed. -func TestInfo(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - b := tx.Bucket([]byte("widgets")) - b.Put([]byte("foo"), []byte("0000")) - return nil - }) - db.Close() - output := run("info", path) - equals(t, `Page Size: 4096`, output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestInfo_NotFound(t *testing.T) { - SetTestMode(true) - output := run("info", "no/such/db") - equals(t, "stat no/such/db: no such file or directory", output) -} diff --git a/cmd/bolt/keys.go b/cmd/bolt/keys.go deleted file mode 100644 index d4bb3c3..0000000 --- a/cmd/bolt/keys.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "os" - - "github.com/boltdb/bolt" -) - -// Keys retrieves a list of keys for a given bucket. -func Keys(path, name string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - // Find bucket. - b := tx.Bucket([]byte(name)) - if b == nil { - fatalf("bucket not found: %s", name) - return nil - } - - // Iterate over each key. - return b.ForEach(func(key, _ []byte) error { - println(string(key)) - return nil - }) - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/keys_test.go b/cmd/bolt/keys_test.go deleted file mode 100644 index 0cc4e0c..0000000 --- a/cmd/bolt/keys_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure that a list of keys can be retrieved for a given bucket. -func TestKeys(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucket([]byte("widgets")) - tx.Bucket([]byte("widgets")).Put([]byte("0002"), []byte("")) - tx.Bucket([]byte("widgets")).Put([]byte("0001"), []byte("")) - tx.Bucket([]byte("widgets")).Put([]byte("0003"), []byte("")) - return nil - }) - db.Close() - output := run("keys", path, "widgets") - equals(t, "0001\n0002\n0003", output) - }) -} - -// Ensure that an error is reported if the database is not found. -func TestKeysDBNotFound(t *testing.T) { - SetTestMode(true) - output := run("keys", "no/such/db", "widgets") - equals(t, "stat no/such/db: no such file or directory", output) -} - -// Ensure that an error is reported if the bucket is not found. -func TestKeysBucketNotFound(t *testing.T) { - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Close() - output := run("keys", path, "widgets") - equals(t, "bucket not found: widgets", output) - }) -} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 0372e19..7fcf530 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -2,199 +2,834 @@ package main import ( "bytes" + "encoding/binary" + "errors" + "flag" "fmt" - "log" + "io" + "io/ioutil" + "math/rand" "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" "time" "github.com/boltdb/bolt" - "github.com/codegangsta/cli" ) -var branch, commit string +var ( + // ErrCommandRequired is returned when a CLI command is not specified. + ErrCommandRequired = errors.New("command required") + + // ErrUnknownCommand is returned when a CLI command is not specified. + ErrUnknownCommand = errors.New("unknown command") + + // ErrPathRequired is returned when the path to a Bolt database is not specified. + ErrPathRequired = errors.New("path required") + + // ErrFileNotFound is returned when a Bolt database does not exist. + ErrFileNotFound = errors.New("file not found") + + // ErrInvalidValue is returned when a benchmark reads an unexpected value. + ErrInvalidValue = errors.New("invalid value") + + // ErrCorrupt is returned when a checking a data file finds errors. + ErrCorrupt = errors.New("invalid value") + + // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly + // divided by the iteration count. + ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size") +) func main() { - log.SetFlags(0) - NewApp().Run(os.Args) + m := NewMain() + if err := m.Run(os.Args[1:]...); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } } -// NewApp creates an Application instance. -func NewApp() *cli.App { - app := cli.NewApp() - app.Name = "bolt" - app.Usage = "BoltDB toolkit" - app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit) - app.Commands = []cli.Command{ - { - Name: "info", - Usage: "Print basic information about a database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Info(path) - }, - }, - { - Name: "get", - Usage: "Retrieve a value for given key in a bucket", - Action: func(c *cli.Context) { - path, name, key := c.Args().Get(0), c.Args().Get(1), c.Args().Get(2) - Get(path, name, key) - }, - }, - { - Name: "keys", - Usage: "Retrieve a list of all keys in a bucket", - Action: func(c *cli.Context) { - path, name := c.Args().Get(0), c.Args().Get(1) - Keys(path, name) - }, - }, - { - Name: "buckets", - Usage: "Retrieves a list of all buckets", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Buckets(path) - }, - }, - { - Name: "pages", - Usage: "Dumps page information for a database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Pages(path) - }, - }, - { - Name: "check", - Usage: "Performs a consistency check on the database", - Action: func(c *cli.Context) { - path := c.Args().Get(0) - Check(path) - }, - }, - { - Name: "stats", - Usage: "Aggregate statistics for all buckets matching specified prefix", - Action: func(c *cli.Context) { - path, name := c.Args().Get(0), c.Args().Get(1) - Stats(path, name) - }, - }, - { - Name: "bench", - Usage: "Performs a synthetic benchmark", - Flags: []cli.Flag{ - &cli.StringFlag{Name: "profile-mode", Value: "rw", Usage: "Profile mode"}, - &cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"}, - &cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"}, - &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"}, - &cli.IntFlag{Name: "batch-size", Usage: "Write batch size"}, - &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"}, - &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"}, - &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, - &cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"}, - &cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"}, - &cli.StringFlag{Name: "stats-interval", Value: "0s", Usage: "Continuous stats interval"}, - &cli.Float64Flag{Name: "fill-percent", Value: bolt.DefaultFillPercent, Usage: "Fill percentage"}, - &cli.BoolFlag{Name: "no-sync", Usage: "Skip fsync on every commit"}, - &cli.BoolFlag{Name: "work", Usage: "Print the temp db and do not delete on exit"}, - &cli.StringFlag{Name: "path", Usage: "Path to database to use"}, - }, - Action: func(c *cli.Context) { - statsInterval, err := time.ParseDuration(c.String("stats-interval")) - if err != nil { - fatal(err) +// Main represents the main program execution. +type Main struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewMain returns a new instance of Main connect to the standard input/output. +func NewMain() *Main { + return &Main{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } +} + +// Run executes the program. +func (m *Main) Run(args ...string) error { + // Require a command at the beginning. + if len(args) == 0 || strings.HasPrefix(args[0], "-") { + return ErrCommandRequired + } + + // Execute command. + switch args[0] { + case "bench": + return newBenchCommand(m).Run(args[1:]...) + case "check": + return newCheckCommand(m).Run(args[1:]...) + case "info": + return newInfoCommand(m).Run(args[1:]...) + case "pages": + return newPagesCommand(m).Run(args[1:]...) + case "stats": + return newStatsCommand(m).Run(args[1:]...) + default: + return ErrUnknownCommand + } +} + +// CheckCommand represents the "check" command execution. +type CheckCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewCheckCommand returns a CheckCommand. +func newCheckCommand(m *Main) *CheckCommand { + return &CheckCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *CheckCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Perform consistency check. + return db.View(func(tx *bolt.Tx) error { + var count int + ch := tx.Check() + loop: + for { + select { + case err, ok := <-ch: + if !ok { + break loop } + fmt.Fprintln(cmd.Stdout, err) + count++ + } + } - Bench(&BenchOptions{ - ProfileMode: c.String("profile-mode"), - WriteMode: c.String("write-mode"), - ReadMode: c.String("read-mode"), - Iterations: c.Int("count"), - BatchSize: c.Int("batch-size"), - KeySize: c.Int("key-size"), - ValueSize: c.Int("value-size"), - CPUProfile: c.String("cpuprofile"), - MemProfile: c.String("memprofile"), - BlockProfile: c.String("blockprofile"), - StatsInterval: statsInterval, - FillPercent: c.Float64("fill-percent"), - NoSync: c.Bool("no-sync"), - Clean: !c.Bool("work"), - Path: c.String("path"), - }) - }, - }} - return app + // Print summary of errors. + if count > 0 { + fmt.Fprintf(cmd.Stdout, "%d errors found\n", count) + return ErrCorrupt + } + + // Notify user that database is valid. + fmt.Fprintln(cmd.Stdout, "OK") + return nil + }) } -var logger = log.New(os.Stderr, "", 0) -var logBuffer *bytes.Buffer +// InfoCommand represents the "info" command execution. +type InfoCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} -func print(v ...interface{}) { - if testMode { - logger.Print(v...) +// NewInfoCommand returns a InfoCommand. +func newInfoCommand(m *Main) *InfoCommand { + return &InfoCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *InfoCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open the database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + // Print basic database info. + info := db.Info() + fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize) + + return nil +} + +// PagesCommand represents the "pages" command execution. +type PagesCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewPagesCommand returns a PagesCommand. +func newPagesCommand(m *Main) *PagesCommand { + return &PagesCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *PagesCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path := fs.Arg(0) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + + // Write header. + fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW") + fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======") + + return db.Update(func(tx *bolt.Tx) error { + var id int + for { + p, err := tx.Page(id) + if err != nil { + return &PageError{ID: id, Err: err} + } else if p == nil { + break + } + + // Only display count and overflow if this is a non-free page. + var count, overflow string + if p.Type != "free" { + count = strconv.Itoa(p.Count) + if p.OverflowCount > 0 { + overflow = strconv.Itoa(p.OverflowCount) + } + } + + // Print table row. + fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) + + // Move to the next non-overflow page. + id += 1 + if p.Type != "free" { + id += p.OverflowCount + } + } + return nil + }) +} + +// StatsCommand represents the "stats" command execution. +type StatsCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewStatsCommand returns a StatsCommand. +func newStatsCommand(m *Main) *StatsCommand { + return &StatsCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *StatsCommand) Run(args ...string) error { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + + // Require database path. + path, prefix := fs.Arg(0), fs.Arg(1) + if path == "" { + return ErrPathRequired + } else if _, err := os.Stat(path); os.IsNotExist(err) { + return ErrFileNotFound + } + + // Open database. + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return err + } + defer db.Close() + + return db.View(func(tx *bolt.Tx) error { + var s bolt.BucketStats + var count int + if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { + if bytes.HasPrefix(name, []byte(prefix)) { + s.Add(b.Stats()) + count += 1 + } + return nil + }); err != nil { + return err + } + + fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count) + + fmt.Fprintln(cmd.Stdout, "Page count statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) + fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN) + fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) + + fmt.Fprintln(cmd.Stdout, "Tree statistics") + fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN) + fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth) + + fmt.Fprintln(cmd.Stdout, "Page size utilization") + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) + var percentage int + if s.BranchAlloc != 0 { + percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) + fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) + percentage = 0 + if s.LeafAlloc != 0 { + percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) + + fmt.Fprintln(cmd.Stdout, "Bucket statistics") + fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN) + percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) + fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) + percentage = 0 + if s.LeafInuse != 0 { + percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) + } + fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) + + return nil + }) +} + +var benchBucketName = []byte("bench") + +// BenchCommand represents the "bench" command execution. +type BenchCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// NewBenchCommand returns a BenchCommand using the +func newBenchCommand(m *Main) *BenchCommand { + return &BenchCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the "bench" command. +func (cmd *BenchCommand) Run(args ...string) error { + // Parse CLI arguments. + options, err := cmd.ParseFlags(args) + if err != nil { + return err + } + + // Remove path if "-work" is not set. Otherwise keep path. + if options.Work { + fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path) } else { - fmt.Print(v...) + defer os.Remove(options.Path) + } + + // Create database. + db, err := bolt.Open(options.Path, 0666, nil) + if err != nil { + return err + } + db.NoSync = options.NoSync + defer db.Close() + + // Write to the database. + var results BenchResults + if err := cmd.runWrites(db, options, &results); err != nil { + return fmt.Errorf("write: ", err) + } + + // Read from the database. + if err := cmd.runReads(db, options, &results); err != nil { + return fmt.Errorf("bench: read: %s", err) + } + + // Print results. + fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) + fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) + fmt.Fprintln(os.Stderr, "") + return nil +} + +// ParseFlags parses the command line flags. +func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) { + var options BenchOptions + + // Parse flagset. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "") + fs.StringVar(&options.WriteMode, "write-mode", "seq", "") + fs.StringVar(&options.ReadMode, "read-mode", "seq", "") + fs.IntVar(&options.Iterations, "count", 1000, "") + fs.IntVar(&options.BatchSize, "batch-size", 0, "") + fs.IntVar(&options.KeySize, "key-size", 8, "") + fs.IntVar(&options.ValueSize, "value-size", 32, "") + fs.StringVar(&options.CPUProfile, "cpuprofile", "", "") + fs.StringVar(&options.MemProfile, "memprofile", "", "") + fs.StringVar(&options.BlockProfile, "blockprofile", "", "") + fs.StringVar(&options.BlockProfile, "blockprofile", "", "") + fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "") + fs.BoolVar(&options.NoSync, "no-sync", false, "") + fs.BoolVar(&options.Work, "work", false, "") + fs.StringVar(&options.Path, "path", "", "") + fs.SetOutput(cmd.Stderr) + if err := fs.Parse(args); err != nil { + return nil, err + } + + // Set batch size to iteration size if not set. + // Require that batch size can be evenly divided by the iteration count. + if options.BatchSize == 0 { + options.BatchSize = options.Iterations + } else if options.Iterations%options.BatchSize != 0 { + return nil, ErrNonDivisibleBatchSize + } + + // Generate temp path if one is not passed in. + if options.Path == "" { + f, err := ioutil.TempFile("", "bolt-bench-") + if err != nil { + return nil, fmt.Errorf("temp file: %s", err) + } + f.Close() + os.Remove(f.Name()) + options.Path = f.Name() + } + + return &options, nil +} + +// Writes to the database. +func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for writes. + if options.ProfileMode == "rw" || options.ProfileMode == "w" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.WriteMode { + case "seq": + err = cmd.runWritesSequential(db, options, results) + case "rnd": + err = cmd.runWritesRandom(db, options, results) + case "seq-nest": + err = cmd.runWritesSequentialNested(db, options, results) + case "rnd-nest": + err = cmd.runWritesRandomNested(db, options, results) + default: + return fmt.Errorf("invalid write mode: %s", options.WriteMode) + } + + // Save time to write. + results.WriteDuration = time.Since(t) + + // Stop profiling for writes only. + if options.ProfileMode == "w" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + key := make([]byte, options.KeySize) + value := make([]byte, options.ValueSize) + + // Write key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert key/value. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { + results.WriteOps = options.Iterations + + for i := 0; i < options.Iterations; i += options.BatchSize { + if err := db.Update(func(tx *bolt.Tx) error { + top, err := tx.CreateBucketIfNotExists(benchBucketName) + if err != nil { + return err + } + top.FillPercent = options.FillPercent + + // Create bucket key. + name := make([]byte, options.KeySize) + binary.BigEndian.PutUint32(name, keySource()) + + // Create bucket. + b, err := top.CreateBucketIfNotExists(name) + if err != nil { + return err + } + b.FillPercent = options.FillPercent + + for j := 0; j < options.BatchSize; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + + // Generate key as uint32. + binary.BigEndian.PutUint32(key, keySource()) + + // Insert value into subbucket. + if err := b.Put(key, value); err != nil { + return err + } + } + + return nil + }); err != nil { + return err + } + } + return nil +} + +// Reads from the database. +func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + // Start profiling for reads. + if options.ProfileMode == "r" { + cmd.startProfiling(options) + } + + t := time.Now() + + var err error + switch options.ReadMode { + case "seq": + switch options.WriteMode { + case "seq-nest", "rnd-nest": + err = cmd.runReadsSequentialNested(db, options, results) + default: + err = cmd.runReadsSequential(db, options, results) + } + default: + return fmt.Errorf("invalid read mode: %s", options.ReadMode) + } + + // Save read time. + results.ReadDuration = time.Since(t) + + // Stop profiling for reads. + if options.ProfileMode == "rw" || options.ProfileMode == "r" { + cmd.stopProfiling() + } + + return err +} + +func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + + c := tx.Bucket(benchBucketName).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return errors.New("invalid value") + } + count++ + } + + if options.WriteMode == "seq" && count != options.Iterations { + return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + return db.View(func(tx *bolt.Tx) error { + t := time.Now() + + for { + var count int + var top = tx.Bucket(benchBucketName) + if err := top.ForEach(func(name, _ []byte) error { + c := top.Bucket(name).Cursor() + for k, v := c.First(); k != nil; k, v = c.Next() { + if v == nil { + return ErrInvalidValue + } + count++ + } + return nil + }); err != nil { + return err + } + + if options.WriteMode == "seq-nest" && count != options.Iterations { + return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) + } + + results.ReadOps += count + + // Make sure we do this for at least a second. + if time.Since(t) >= time.Second { + break + } + } + + return nil + }) +} + +// File handlers for the various profiles. +var cpuprofile, memprofile, blockprofile *os.File + +// Starts all profiles set on the options. +func (cmd *BenchCommand) startProfiling(options *BenchOptions) { + var err error + + // Start CPU profiling. + if options.CPUProfile != "" { + cpuprofile, err = os.Create(options.CPUProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err) + os.Exit(1) + } + pprof.StartCPUProfile(cpuprofile) + } + + // Start memory profiling. + if options.MemProfile != "" { + memprofile, err = os.Create(options.MemProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err) + os.Exit(1) + } + runtime.MemProfileRate = 4096 + } + + // Start fatal profiling. + if options.BlockProfile != "" { + blockprofile, err = os.Create(options.BlockProfile) + if err != nil { + fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err) + os.Exit(1) + } + runtime.SetBlockProfileRate(1) } } -func printf(format string, v ...interface{}) { - if testMode { - logger.Printf(format, v...) - } else { - fmt.Printf(format, v...) +// Stops all profiles. +func (cmd *BenchCommand) stopProfiling() { + if cpuprofile != nil { + pprof.StopCPUProfile() + cpuprofile.Close() + cpuprofile = nil + } + + if memprofile != nil { + pprof.Lookup("heap").WriteTo(memprofile, 0) + memprofile.Close() + memprofile = nil + } + + if blockprofile != nil { + pprof.Lookup("block").WriteTo(blockprofile, 0) + blockprofile.Close() + blockprofile = nil + runtime.SetBlockProfileRate(0) } } -func println(v ...interface{}) { - if testMode { - logger.Println(v...) - } else { - fmt.Println(v...) - } +// BenchOptions represents the set of options that can be passed to "bolt bench". +type BenchOptions struct { + ProfileMode string + WriteMode string + ReadMode string + Iterations int + BatchSize int + KeySize int + ValueSize int + CPUProfile string + MemProfile string + BlockProfile string + StatsInterval time.Duration + FillPercent float64 + NoSync bool + Work bool + Path string } -func fatal(v ...interface{}) { - logger.Print(v...) - if !testMode { - os.Exit(1) - } +// BenchResults represents the performance results of the benchmark. +type BenchResults struct { + WriteOps int + WriteDuration time.Duration + ReadOps int + ReadDuration time.Duration } -func fatalf(format string, v ...interface{}) { - logger.Printf(format, v...) - if !testMode { - os.Exit(1) +// Returns the duration for a single write operation. +func (r *BenchResults) WriteOpDuration() time.Duration { + if r.WriteOps == 0 { + return 0 } + return r.WriteDuration / time.Duration(r.WriteOps) } -func fatalln(v ...interface{}) { - logger.Println(v...) - if !testMode { - os.Exit(1) +// Returns average number of write operations that can be performed per second. +func (r *BenchResults) WriteOpsPerSecond() int { + var op = r.WriteOpDuration() + if op == 0 { + return 0 } + return int(time.Second) / int(op) } -// LogBuffer returns the contents of the log. -// This only works while the CLI is in test mode. -func LogBuffer() string { - if logBuffer != nil { - return logBuffer.String() +// Returns the duration for a single read operation. +func (r *BenchResults) ReadOpDuration() time.Duration { + if r.ReadOps == 0 { + return 0 } - return "" + return r.ReadDuration / time.Duration(r.ReadOps) } -var testMode bool - -// SetTestMode sets whether the CLI is running in test mode and resets the logger. -func SetTestMode(value bool) { - testMode = value - if testMode { - logBuffer = bytes.NewBuffer(nil) - logger = log.New(logBuffer, "", 0) - } else { - logger = log.New(os.Stderr, "", 0) +// Returns average number of read operations that can be performed per second. +func (r *BenchResults) ReadOpsPerSecond() int { + var op = r.ReadOpDuration() + if op == 0 { + return 0 } + return int(time.Second) / int(op) +} + +type PageError struct { + ID int + Err error +} + +func (e *PageError) Error() string { + return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err) } diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go index 4448d6e..b9e8c67 100644 --- a/cmd/bolt/main_test.go +++ b/cmd/bolt/main_test.go @@ -1,69 +1,145 @@ package main_test import ( - "fmt" + "bytes" "io/ioutil" "os" - "path/filepath" - "reflect" - "runtime" - "strings" + "strconv" "testing" "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" + "github.com/boltdb/bolt/cmd/bolt" ) -// open creates and opens a Bolt database in the temp directory. -func open(fn func(*bolt.DB, string)) { - path := tempfile() - defer os.RemoveAll(path) +// Ensure the "info" command can print information about a database. +func TestInfoCommand_Run(t *testing.T) { + db := MustOpen(0666, nil) + db.DB.Close() + defer db.Close() - db, err := bolt.Open(path, 0600, nil) - if err != nil { - panic("db open error: " + err.Error()) + // Run the info command. + m := NewMain() + if err := m.Run("info", db.Path); err != nil { + t.Fatal(err) } - fn(db, path) } -// run executes a command against the CLI and returns the output. -func run(args ...string) string { - args = append([]string{"bolt"}, args...) - NewApp().Run(args) - return strings.TrimSpace(LogBuffer()) +// Ensure the "stats" command can execute correctly. +func TestStatsCommand_Run(t *testing.T) { + // Ignore + if os.Getpagesize() != 4096 { + t.Skip("system does not use 4KB page size") + } + + db := MustOpen(0666, nil) + defer db.Close() + + if err := db.Update(func(tx *bolt.Tx) error { + // Create "foo" bucket. + b, err := tx.CreateBucket([]byte("foo")) + if err != nil { + return err + } + for i := 0; i < 10; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "bar" bucket. + b, err = tx.CreateBucket([]byte("bar")) + if err != nil { + return err + } + for i := 0; i < 100; i++ { + if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { + return err + } + } + + // Create "baz" bucket. + b, err = tx.CreateBucket([]byte("baz")) + if err != nil { + return err + } + if err := b.Put([]byte("key"), []byte("value")); err != nil { + return err + } + + return nil + }); err != nil { + t.Fatal(err) + } + db.DB.Close() + + // Generate expected result. + exp := "Aggregate statistics for 3 buckets\n\n" + + "Page count statistics\n" + + "\tNumber of logical branch pages: 0\n" + + "\tNumber of physical branch overflow pages: 0\n" + + "\tNumber of logical leaf pages: 1\n" + + "\tNumber of physical leaf overflow pages: 0\n" + + "Tree statistics\n" + + "\tNumber of keys/value pairs: 111\n" + + "\tNumber of levels in B+tree: 1\n" + + "Page size utilization\n" + + "\tBytes allocated for physical branch pages: 0\n" + + "\tBytes actually used for branch data: 0 (0%)\n" + + "\tBytes allocated for physical leaf pages: 4096\n" + + "\tBytes actually used for leaf data: 1996 (48%)\n" + + "Bucket statistics\n" + + "\tTotal number of buckets: 3\n" + + "\tTotal number on inlined buckets: 2 (66%)\n" + + "\tBytes used for inlined buckets: 236 (11%)\n" + + // Run the command. + m := NewMain() + if err := m.Run("stats", db.Path); err != nil { + t.Fatal(err) + } else if m.Stdout.String() != exp { + t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) + } } -// tempfile returns a temporary file path. -func tempfile() string { +// Main represents a test wrapper for main.Main that records output. +type Main struct { + *main.Main + Stdin bytes.Buffer + Stdout bytes.Buffer + Stderr bytes.Buffer +} + +// NewMain returns a new instance of Main. +func NewMain() *Main { + m := &Main{Main: main.NewMain()} + m.Main.Stdin = &m.Stdin + m.Main.Stdout = &m.Stdout + m.Main.Stderr = &m.Stderr + return m +} + +// MustOpen creates a Bolt database in a temporary location. +func MustOpen(mode os.FileMode, options *bolt.Options) *DB { + // Create temporary path. f, _ := ioutil.TempFile("", "bolt-") f.Close() os.Remove(f.Name()) - return f.Name() -} -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) - tb.FailNow() - } -} - -// ok fails the test if an err is not nil. -func ok(tb testing.TB, err error) { + db, err := bolt.Open(f.Name(), mode, options) if err != nil { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) - tb.FailNow() + panic(err.Error()) } + return &DB{DB: db, Path: f.Name()} } -// equals fails the test if exp is not equal to act. -func equals(tb testing.TB, exp, act interface{}) { - if !reflect.DeepEqual(exp, act) { - _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) - tb.FailNow() - } +// DB is a test wrapper for bolt.DB. +type DB struct { + *bolt.DB + Path string +} + +// Close closes and removes the database. +func (db *DB) Close() error { + defer os.Remove(db.Path) + return db.DB.Close() } diff --git a/cmd/bolt/pages.go b/cmd/bolt/pages.go deleted file mode 100644 index ec1c4b4..0000000 --- a/cmd/bolt/pages.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "os" - "strconv" - - "github.com/boltdb/bolt" -) - -// Pages prints a list of all pages in a database. -func Pages(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - println("ID TYPE ITEMS OVRFLW") - println("======== ========== ====== ======") - - db.Update(func(tx *bolt.Tx) error { - var id int - for { - p, err := tx.Page(id) - if err != nil { - fatalf("page error: %d: %s", id, err) - } else if p == nil { - break - } - - // Only display count and overflow if this is a non-free page. - var count, overflow string - if p.Type != "free" { - count = strconv.Itoa(p.Count) - if p.OverflowCount > 0 { - overflow = strconv.Itoa(p.OverflowCount) - } - } - - // Print table row. - printf("%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) - - // Move to the next non-overflow page. - id += 1 - if p.Type != "free" { - id += p.OverflowCount - } - } - return nil - }) -} diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go deleted file mode 100644 index b5d0083..0000000 --- a/cmd/bolt/stats.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "bytes" - "os" - - "github.com/boltdb/bolt" -) - -// Collect stats for all top level buckets matching the prefix. -func Stats(path, prefix string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - fatal(err) - return - } - - db, err := bolt.Open(path, 0600, nil) - if err != nil { - fatal(err) - return - } - defer db.Close() - - err = db.View(func(tx *bolt.Tx) error { - var s bolt.BucketStats - var count int - var prefix = []byte(prefix) - tx.ForEach(func(name []byte, b *bolt.Bucket) error { - if bytes.HasPrefix(name, prefix) { - s.Add(b.Stats()) - count += 1 - } - return nil - }) - printf("Aggregate statistics for %d buckets\n\n", count) - - println("Page count statistics") - printf("\tNumber of logical branch pages: %d\n", s.BranchPageN) - printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) - printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN) - printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) - - println("Tree statistics") - printf("\tNumber of keys/value pairs: %d\n", s.KeyN) - printf("\tNumber of levels in B+tree: %d\n", s.Depth) - - println("Page size utilization") - printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - var percentage int - if s.BranchAlloc != 0 { - percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) - } - printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) - printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - percentage = 0 - if s.LeafAlloc != 0 { - percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) - } - printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) - - println("Bucket statistics") - printf("\tTotal number of buckets: %d\n", s.BucketN) - percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) - printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) - percentage = 0 - if s.LeafInuse != 0 { - percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) - } - printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) - - return nil - }) - if err != nil { - fatal(err) - return - } -} diff --git a/cmd/bolt/stats_test.go b/cmd/bolt/stats_test.go deleted file mode 100644 index 44ed434..0000000 --- a/cmd/bolt/stats_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package main_test - -import ( - "os" - "strconv" - "testing" - - "github.com/boltdb/bolt" - . "github.com/boltdb/bolt/cmd/bolt" -) - -func TestStats(t *testing.T) { - if os.Getpagesize() != 4096 { - t.Skip() - } - SetTestMode(true) - open(func(db *bolt.DB, path string) { - db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("foo")) - if err != nil { - return err - } - for i := 0; i < 10; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - } - b, err = tx.CreateBucket([]byte("bar")) - if err != nil { - return err - } - for i := 0; i < 100; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) - } - b, err = tx.CreateBucket([]byte("baz")) - if err != nil { - return err - } - b.Put([]byte("key"), []byte("value")) - return nil - }) - db.Close() - output := run("stats", path, "b") - equals(t, "Aggregate statistics for 2 buckets\n\n"+ - "Page count statistics\n"+ - "\tNumber of logical branch pages: 0\n"+ - "\tNumber of physical branch overflow pages: 0\n"+ - "\tNumber of logical leaf pages: 1\n"+ - "\tNumber of physical leaf overflow pages: 0\n"+ - "Tree statistics\n"+ - "\tNumber of keys/value pairs: 101\n"+ - "\tNumber of levels in B+tree: 1\n"+ - "Page size utilization\n"+ - "\tBytes allocated for physical branch pages: 0\n"+ - "\tBytes actually used for branch data: 0 (0%)\n"+ - "\tBytes allocated for physical leaf pages: 4096\n"+ - "\tBytes actually used for leaf data: 1996 (48%)\n"+ - "Bucket statistics\n"+ - "\tTotal number of buckets: 2\n"+ - "\tTotal number on inlined buckets: 1 (50%)\n"+ - "\tBytes used for inlined buckets: 40 (2%)", output) - }) -}