diff --git a/Makefile b/Makefile index 3fec2c7..cfbed51 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ build: get @mkdir -p bin @go build -ldflags=$(GOLDFLAGS) -a -o bin/bolt ./cmd/bolt -test: fmt errcheck +test: fmt @go get github.com/stretchr/testify/assert @echo "=== TESTS ===" @go test -v -cover -test.run=$(TEST) diff --git a/bolt.go b/bolt.go deleted file mode 100644 index a3f1eac..0000000 --- a/bolt.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !linux - -package bolt - -import ( - "os" -) - -var odirect int - -func fdatasync(f *os.File) error { - return f.Sync() -} diff --git a/bolt_386.go b/bolt_386.go new file mode 100644 index 0000000..856f401 --- /dev/null +++ b/bolt_386.go @@ -0,0 +1,4 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFF // 256MB diff --git a/bolt_amd64.go b/bolt_amd64.go new file mode 100644 index 0000000..4262932 --- /dev/null +++ b/bolt_amd64.go @@ -0,0 +1,4 @@ +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/bolt_darwin.go b/bolt_darwin.go new file mode 100644 index 0000000..2bffad3 --- /dev/null +++ b/bolt_darwin.go @@ -0,0 +1,53 @@ +package bolt + +import ( + "os" + "syscall" + "unsafe" +) + +var odirect int + +// fdatasync flushes written data to a file descriptor. +func fdatasync(f *os.File) error { + return f.Sync() +} + +// flock acquires an advisory lock on a file descriptor. +func flock(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_EX) +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := syscall.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} diff --git a/bolt_linux.go b/bolt_linux.go index 4351db5..761a83e 100644 --- a/bolt_linux.go +++ b/bolt_linux.go @@ -3,10 +3,51 @@ package bolt import ( "os" "syscall" + "unsafe" ) var odirect = syscall.O_DIRECT +// fdatasync flushes written data to a file descriptor. func fdatasync(f *os.File) error { return syscall.Fdatasync(int(f.Fd())) } + +// flock acquires an advisory lock on a file descriptor. +func flock(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_EX) +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(f *os.File) error { + return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := syscall.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} diff --git a/bolt_windows.go b/bolt_windows.go new file mode 100644 index 0000000..8300d40 --- /dev/null +++ b/bolt_windows.go @@ -0,0 +1,73 @@ +package bolt + +import ( + "fmt" + "os" + "syscall" + "unsafe" +) + +var odirect int + +// fdatasync flushes written data to a file descriptor. +func fdatasync(f *os.File) error { + return f.Sync() +} + +// flock acquires an advisory lock on a file descriptor. +func flock(f *os.File) error { + return nil +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(f *os.File) error { + return nil +} + +// mmap memory maps a DB's data file. +// Based on: https://github.com/edsrzf/mmap-go +func mmap(db *DB, sz int) error { + // Truncate the database to the size of the mmap. + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("truncate: %s", err) + } + + // Open a file mapping handle. + sizelo := uint32(sz >> 32) + sizehi := uint32(sz & 0xffffffff) + h, errno := syscall.CreateFileMapping(syscall.Handle(db.file.Fd()), nil, syscall.PAGE_READONLY, sizelo, sizehi, nil) + if h == 0 { + return os.NewSyscallError("CreateFileMapping", errno) + } + + // Create the memory map. + addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(sz)) + if addr == 0 { + return os.NewSyscallError("MapViewOfFile", errno) + } + + // Close mapping handle. + if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { + return os.NewSyscallError("CloseHandle", err) + } + + // Convert to a byte array. + db.data = ((*[maxMapSize]byte)(unsafe.Pointer(addr))) + db.datasz = sz + + return nil +} + +// munmap unmaps a pointer from a file. +// Based on: https://github.com/edsrzf/mmap-go +func munmap(db *DB) error { + if db.data == nil { + return nil + } + + addr := (uintptr)(unsafe.Pointer(&db.data[0])) + if err := syscall.UnmapViewOfFile(addr); err != nil { + return os.NewSyscallError("UnmapViewOfFile", err) + } + return nil +} diff --git a/db.go b/db.go index d759b1b..8f80515 100644 --- a/db.go +++ b/db.go @@ -6,7 +6,6 @@ import ( "hash/fnv" "os" "sync" - "syscall" "unsafe" ) @@ -68,7 +67,9 @@ type DB struct { path string file *os.File - data []byte + dataref []byte + data *[maxMapSize]byte + datasz int meta0 *meta meta1 *meta pageSize int @@ -120,7 +121,7 @@ func Open(path string, mode os.FileMode) (*DB, error) { // Lock file so that other processes using Bolt cannot use the database // at the same time. This would cause corruption since the two processes // would write meta pages and free pages separately. - if err := syscall.Flock(int(db.file.Fd()), syscall.LOCK_EX); err != nil { + if err := flock(db.file); err != nil { _ = db.close() return nil, err } @@ -193,7 +194,7 @@ func (db *DB) mmap(minsz int) error { size = db.mmapSize(size) // Memory-map the data file as a byte slice. - if db.data, err = syscall.Mmap(int(db.file.Fd()), 0, size, syscall.PROT_READ, syscall.MAP_SHARED); err != nil { + if err := mmap(db, size); err != nil { return err } @@ -214,11 +215,8 @@ func (db *DB) mmap(minsz int) error { // munmap unmaps the data file from memory. func (db *DB) munmap() error { - if db.data != nil { - if err := syscall.Munmap(db.data); err != nil { - return fmt.Errorf("unmap error: " + err.Error()) - } - db.data = nil + if err := munmap(db); err != nil { + return fmt.Errorf("unmap error: " + err.Error()) } return nil } @@ -314,7 +312,7 @@ func (db *DB) close() error { // Close file handles. if db.file != nil { // Unlock the file. - _ = syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN) + _ = funlock(db.file) // Close the file descriptor. if err := db.file.Close(); err != nil { @@ -503,7 +501,7 @@ func (db *DB) Stats() Stats { // This is for internal access to the raw data bytes from the C cursor, use // carefully, or not at all. func (db *DB) Info() *Info { - return &Info{db.data, db.pageSize} + return &Info{uintptr(unsafe.Pointer(&db.data[0])), db.pageSize} } // page retrieves a page reference from the mmap based on the current page size. @@ -540,7 +538,7 @@ func (db *DB) allocate(count int) (*page, error) { // Resize mmap() if we're at the end. p.id = db.rwtx.meta.pgid var minsz = int((p.id+pgid(count))+1) * db.pageSize - if minsz >= len(db.data) { + if minsz >= db.datasz { if err := db.mmap(minsz); err != nil { return nil, fmt.Errorf("mmap allocate error: %s", err) } @@ -575,7 +573,7 @@ func (s *Stats) add(other *Stats) { } type Info struct { - Data []byte + Data uintptr PageSize int } diff --git a/doc.go b/doc.go index caf66e9..b127f59 100644 --- a/doc.go +++ b/doc.go @@ -12,6 +12,9 @@ rolled back in the event of a crash. The design of Bolt is based on Howard Chu's LMDB database project. +Bolt currently works on Windows, Mac OS X, and Linux. + + Basics There are only a few types in Bolt: DB, Bucket, Tx, and Cursor. The DB is @@ -33,7 +36,5 @@ values returned from Bolt cannot be changed. Writing to a read-only byte slice will cause Go to panic. If you need to work with data returned from a Get() you need to first copy it to a new byte slice. -Bolt currently works on Mac OS and Linux. Windows support is coming soon. - */ package bolt