From d9e7e0257b4855bd2a4072f56597485a341841f0 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Thu, 24 Apr 2014 16:30:39 -0400 Subject: [PATCH 1/3] add -stats and -batch-size support to bench cmd --- cmd/bolt/bench.go | 67 ++++++++++++++++++++++++++++++++++++++--------- cmd/bolt/main.go | 8 ++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go index 72144b8..e80f51a 100644 --- a/cmd/bolt/bench.go +++ b/cmd/bolt/bench.go @@ -5,7 +5,9 @@ import ( "errors" "fmt" "io/ioutil" + "math/rand" "os" + "reflect" "runtime" "runtime/pprof" "time" @@ -68,6 +70,19 @@ func Bench(options *BenchOptions) { fmt.Printf("# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) fmt.Printf("# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) fmt.Println("") + + if options.Stats { + fmt.Println("Transaction Stats") + printStruct(db.Stats().TxStats) + fmt.Println("") + db.View(func(tx *bolt.Tx) error { + b := tx.Bucket(benchBucketName) + fmt.Println("Storage Stats") + printStruct(b.Stats()) + fmt.Println("") + return nil + }) + } } // Writes to the database. @@ -78,6 +93,8 @@ func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error switch options.WriteMode { case "seq": err = benchWriteSequential(db, options, results) + case "rnd": + err = benchWriteRandom(db, options, results) default: return fmt.Errorf("invalid write mode: %s", options.WriteMode) } @@ -88,22 +105,38 @@ func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error } func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + var i = uint32(0) + return benchWriteWithSource(db, options, results, func() uint32 { i++; return i }) +} + +func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { + r := rand.New(rand.NewSource(42)) + return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() }) +} + +func benchWriteWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { results.WriteOps = options.Iterations - return db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) + for i := 0; i < options.Iterations; i += options.BatchSize { + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) - for i := 0; i < options.Iterations; i++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, uint32(i)) - if err := b.Put(key, value); err != nil { - return err + for j := 0; j < options.BatchSize; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + binary.BigEndian.PutUint32(key, keySource()) + if err := b.Put(key, value); err != nil { + return err + } } - } - return nil - }) + return nil + }) + if err != nil { + return err + } + } + return nil } // Reads from the database. @@ -137,7 +170,7 @@ func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResul count++ } - if count != options.Iterations { + if options.WriteMode == "seq" && count != options.Iterations { return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) } @@ -215,6 +248,8 @@ type BenchOptions struct { Iterations int KeySize int ValueSize int + BatchSize int + Stats bool CPUProfile string MemProfile string BlockProfile string @@ -269,3 +304,11 @@ func tempfile() string { os.Remove(f.Name()) return f.Name() } + +func printStruct(s interface{}) { + v := reflect.ValueOf(s) + t := reflect.TypeOf(s) + for i := 0; i < v.NumField(); i++ { + fmt.Printf(" %s: %v\n", t.Field(i).Name, v.Field(i).Interface()) + } +} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 719bf00..cab4e0e 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -100,11 +100,17 @@ func NewApp() *cli.App { &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"}, &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"}, &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"}, + &cli.IntFlag{Name: "batch-size", Value: 0, Usage: "Write batch size"}, &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, &cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"}, &cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"}, + &cli.BoolFlag{Name: "stats", Usage: "Output storage and transaction stats"}, }, Action: func(c *cli.Context) { + bs := c.Int("batch-size") + if bs == 0 { + bs = c.Int("count") + } Bench(&BenchOptions{ ProfileMode: c.String("profile-mode"), WriteMode: c.String("write-mode"), @@ -112,9 +118,11 @@ func NewApp() *cli.App { Iterations: c.Int("count"), KeySize: c.Int("key-size"), ValueSize: c.Int("value-size"), + BatchSize: bs, CPUProfile: c.String("cpuprofile"), MemProfile: c.String("memprofile"), BlockProfile: c.String("blockprofile"), + Stats: c.Bool("stats"), }) }, }} From cabb44e01f70840e5a7057a1d778f5da968ac77e Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 29 Apr 2014 12:27:38 -0600 Subject: [PATCH 2/3] Add --batch-size to 'bolt bench'. This commit adds a --batch-size CLI argument to the 'bolt bench' tool. This argument will insert into Bolt in smaller batches which is a more typical use case. /cc @snormore --- cmd/bolt/bench.go | 44 ++++++++++++++++++++++++++++++++------------ cmd/bolt/main.go | 2 ++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go index 72144b8..193687b 100644 --- a/cmd/bolt/bench.go +++ b/cmd/bolt/bench.go @@ -88,22 +88,41 @@ func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error } func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - results.WriteOps = options.Iterations + // Default batch size to iteration count, if not specified. + var batchSize, iterations = options.BatchSize, options.Iterations + if batchSize == 0 { + batchSize = iterations + } - return db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) + // Insert in batches. + var count int + for i := 0; i < (iterations/batchSize)+1; i++ { + err := db.Update(func(tx *bolt.Tx) error { + b, _ := tx.CreateBucketIfNotExists(benchBucketName) - for i := 0; i < options.Iterations; i++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - binary.BigEndian.PutUint32(key, uint32(i)) - if err := b.Put(key, value); err != nil { - return err + for j := 0; j < batchSize && count < iterations; j++ { + var key = make([]byte, options.KeySize) + var value = make([]byte, options.ValueSize) + binary.BigEndian.PutUint32(key, uint32(count)) + + if err := b.Put(key, value); err != nil { + return err + } + + count++ } - } - return nil - }) + return nil + }) + if err != nil { + return err + } + } + + // Update the write op count. + results.WriteOps = count + + return nil } // Reads from the database. @@ -213,6 +232,7 @@ type BenchOptions struct { WriteMode string ReadMode string Iterations int + BatchSize int KeySize int ValueSize int CPUProfile string diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index 719bf00..54e1796 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -98,6 +98,7 @@ func NewApp() *cli.App { &cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"}, &cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"}, &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"}, + &cli.IntFlag{Name: "batch-size", Usage: "Insert batch size"}, &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"}, &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"}, &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, @@ -110,6 +111,7 @@ func NewApp() *cli.App { WriteMode: c.String("write-mode"), ReadMode: c.String("read-mode"), Iterations: c.Int("count"), + BatchSize: c.Int("batch-size"), KeySize: c.Int("key-size"), ValueSize: c.Int("value-size"), CPUProfile: c.String("cpuprofile"), From 6854ca415f358279c93066bf1115e91af1108493 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 30 Apr 2014 11:55:08 -0600 Subject: [PATCH 3/3] Remove bolt bench -stat. --- cmd/bolt/bench.go | 25 +------------------------ cmd/bolt/main.go | 2 -- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go index e80f51a..e1858df 100644 --- a/cmd/bolt/bench.go +++ b/cmd/bolt/bench.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "math/rand" "os" - "reflect" "runtime" "runtime/pprof" "time" @@ -70,19 +69,6 @@ func Bench(options *BenchOptions) { fmt.Printf("# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) fmt.Printf("# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) fmt.Println("") - - if options.Stats { - fmt.Println("Transaction Stats") - printStruct(db.Stats().TxStats) - fmt.Println("") - db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(benchBucketName) - fmt.Println("Storage Stats") - printStruct(b.Stats()) - fmt.Println("") - return nil - }) - } } // Writes to the database. @@ -110,7 +96,7 @@ func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResu } func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(42)) + r := rand.New(rand.NewSource(time.Now().UnixNano())) return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() }) } @@ -249,7 +235,6 @@ type BenchOptions struct { KeySize int ValueSize int BatchSize int - Stats bool CPUProfile string MemProfile string BlockProfile string @@ -304,11 +289,3 @@ func tempfile() string { os.Remove(f.Name()) return f.Name() } - -func printStruct(s interface{}) { - v := reflect.ValueOf(s) - t := reflect.TypeOf(s) - for i := 0; i < v.NumField(); i++ { - fmt.Printf(" %s: %v\n", t.Field(i).Name, v.Field(i).Interface()) - } -} diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go index cab4e0e..041f4bb 100644 --- a/cmd/bolt/main.go +++ b/cmd/bolt/main.go @@ -104,7 +104,6 @@ func NewApp() *cli.App { &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"}, &cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"}, &cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"}, - &cli.BoolFlag{Name: "stats", Usage: "Output storage and transaction stats"}, }, Action: func(c *cli.Context) { bs := c.Int("batch-size") @@ -122,7 +121,6 @@ func NewApp() *cli.App { CPUProfile: c.String("cpuprofile"), MemProfile: c.String("memprofile"), BlockProfile: c.String("blockprofile"), - Stats: c.Bool("stats"), }) }, }}