Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions bolt_aix.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ func mmap(db *DB, sz int) error {
return nil
}

// madviseWillNeed advises the kernel that the given part of the mmap is
// going to be accessed soon. It is only a hint, so a failure is ignored.
func madviseWillNeed(b []byte) {
_ = unix.Madvise(b, syscall.MADV_WILLNEED)
}

// munmap unmaps a DB's data file from memory.
func munmap(db *DB) error {
// Ignore the unmap if we have no mapped data.
Expand Down
6 changes: 6 additions & 0 deletions bolt_android.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ func mmap(db *DB, sz int) error {
return nil
}

// madviseWillNeed advises the kernel that the given part of the mmap is
// going to be accessed soon. It is only a hint, so a failure is ignored.
func madviseWillNeed(b []byte) {
_ = unix.Madvise(b, syscall.MADV_WILLNEED)
}

// munmap unmaps a DB's data file from memory.
func munmap(db *DB) error {
// Ignore the unmap if we have no mapped data.
Expand Down
6 changes: 6 additions & 0 deletions bolt_solaris.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ func mmap(db *DB, sz int) error {
return nil
}

// madviseWillNeed advises the kernel that the given part of the mmap is
// going to be accessed soon. It is only a hint, so a failure is ignored.
func madviseWillNeed(b []byte) {
_ = unix.Madvise(b, syscall.MADV_WILLNEED)
}

// munmap unmaps a DB's data file from memory.
func munmap(db *DB) error {
// Ignore the unmap if we have no mapped data.
Expand Down
6 changes: 6 additions & 0 deletions bolt_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ func mmap(db *DB, sz int) error {
return nil
}

// madviseWillNeed advises the kernel that the given part of the mmap is
// going to be accessed soon. It is only a hint, so a failure is ignored.
func madviseWillNeed(b []byte) {
_ = unix.Madvise(b, syscall.MADV_WILLNEED)
}

// munmap unmaps a DB's data file from memory.
func munmap(db *DB) error {
// Ignore the unmap if we have no mapped data.
Expand Down
4 changes: 4 additions & 0 deletions bolt_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ func mmap(db *DB, sz int) error {
return nil
}

// madviseWillNeed is a no-op on Windows.
func madviseWillNeed(_ []byte) {
}

// munmap unmaps a pointer from a file.
// Based on: https://github.com/edsrzf/mmap-go
func munmap(db *DB) error {
Expand Down
130 changes: 129 additions & 1 deletion db.go
Original file line number Diff line number Diff line change
Expand Up @@ -1132,6 +1132,134 @@ func (db *DB) page(id common.Pgid) *common.Page {
return (*common.Page)(unsafe.Pointer(&db.data[pos]))
}

// prefetchChildren hints to the kernel that the children of the given branch
// page are about to be read, so that it can start reading all of them from
// disk right away rather than one after the other as they are accessed. That
// includes the overflow pages of the children that are read. It does nothing
// for any other type of page.
func (db *DB) prefetchChildren(p *common.Page) {
if !p.IsBranchPage() || p.Count() == 0 {
return
}

// Children that are next to each other in the file are requested with

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about overflow pages?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, they weren't covered and it mattered more than I thought. Only the first page of each child was hinted, so for a leaf with big values the later keys (which sit after the values, in the overflow pages) were still faulted in one at a time.

Tried it on a 325 MB db with 6 KB values, same slow disk setup as in the description:

cold open
main 104.2s
this PR before 66s
this PR now 2.2s

What it does now: once the children of a branch page are requested, it looks at each child's header and also requests the overflow pages the scan will actually read. For a branch child that's all of them, for a leaf only the ones holding a key or a bucket, so we don't pull in value bytes nobody looks at. The db from the description is unchanged (5.3s).

// a single call.
first := p.BranchPageElement(0).Pgid()
last := first
for i := uint16(1); i < p.Count(); i++ {
id := p.BranchPageElement(i).Pgid()
if id == last+1 {
last = id
continue
}

db.prefetchPages(first, last)
first, last = id, id
}
db.prefetchPages(first, last)

// Whether a child has overflow pages is only known from its header,
// which is being read by now. Request these before any of the
// children is visited, so that they are read at the same time too.
limit := common.Pgid(db.datasz / db.pageSize)
for i := uint16(0); i < p.Count(); i++ {
id := p.BranchPageElement(i).Pgid()
if id >= limit {
continue
}
forEachOverflowRead(db.page(id), db.pageSize, db.prefetchPages)
}
}

// forEachOverflowRead calls fn with every range of overflow pages of the
// given page that is read when the freelist is reconstructed. For a branch
// page that is all of them. For a leaf page it is only those that hold a key
// or a bucket, the rest of the values is not looked at.
func forEachOverflowRead(p *common.Page, pageSize int, fn func(first, last common.Pgid)) {
overflow := common.Pgid(p.Overflow())
if overflow == 0 || pageSize <= 0 {
return
}

if p.IsBranchPage() {
fn(p.Id()+1, p.Id()+overflow)
return
}
if !p.IsLeafPage() {
return
}

// The elements are stored in order, so remembering the last page that
// was reported is enough to not report a page twice.
var reported common.Pgid
base := uintptr(unsafe.Pointer(p))
for i := uint16(0); i < p.Count(); i++ {
elem := p.LeafPageElement(i)
size := uintptr(elem.Ksize())
if elem.IsBucketEntry() {
size += uintptr(elem.Vsize())
}
if size == 0 {
continue
}

start := uintptr(unsafe.Pointer(elem)) - base + uintptr(elem.Pos())
first := common.Pgid(start / uintptr(pageSize))
last := common.Pgid((start + size - 1) / uintptr(pageSize))
if first <= reported {
first = reported + 1
}
if last > overflow {
last = overflow
}
if first > last {
continue
}

fn(p.Id()+first, p.Id()+last)
reported = last
}
}

// prefetchPages hints to the kernel that the given range of pages is about to
// be read.
func (db *DB) prefetchPages(first, last common.Pgid) {
start, end, ok := prefetchRange(first, last, db.pageSize, os.Getpagesize(), db.datasz)
if !ok {
return
}

madviseWillNeed(db.data[start:end:end])
}

// prefetchRange returns the range of bytes within a mmap of the given size
// that holds the given range of pages. The range is widened to start and end
// at boundaries of OS pages, which can be larger than the pages of a database.
func prefetchRange(first, last common.Pgid, pageSize, osPageSize, mmapSize int) (int, int, bool) {
if first > last || pageSize <= 0 || osPageSize <= 0 {
return 0, 0, false
}

// Don't touch anything beyond the mmap. That also takes care of page
// ids that are too large for the offsets below to be computed.
if uint64(last) >= uint64(mmapSize)/uint64(pageSize) {
return 0, 0, false
}

start := int(first) * pageSize
end := (int(last) + 1) * pageSize

start -= start % osPageSize
if rem := end % osPageSize; rem != 0 {
end += osPageSize - rem
}
if end > mmapSize {
end = mmapSize
}

return start, end, true
}

// pageInBuffer retrieves a page reference from a given byte array based on the current page size.
func (db *DB) pageInBuffer(b []byte, id common.Pgid) *common.Page {
return (*common.Page)(unsafe.Pointer(&b[id*common.Pgid(db.pageSize)]))
Expand Down Expand Up @@ -1292,7 +1420,7 @@ func (db *DB) freepages() []common.Pgid {

go func() {
defer close(ech)
tx.recursivelyCheckBucket(tx.root, reachable, nofreed, HexKVStringer(), ech)
tx.recursivelyCheckBucketPages(tx.root.RootPage(), reachable, nofreed, HexKVStringer(), ech)
}()
// following for loop will exit once channel is closed in the above goroutine.
// we don't need to wait explictly with a waitgroup
Expand Down
170 changes: 170 additions & 0 deletions db_whitebox_test.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package bbolt

import (
"math"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"go.etcd.io/bbolt/errors"
"go.etcd.io/bbolt/internal/common"
)

func TestOpenWithPreLoadFreelist(t *testing.T) {
Expand Down Expand Up @@ -124,3 +126,171 @@ func prepareData(t *testing.T) (string, error) {

return fileName, nil
}

func TestPrefetchRange(t *testing.T) {
const mmapSize = 1 << 20

testCases := []struct {
name string
first, last common.Pgid
pageSize int
osPageSize int

expectedStart, expectedEnd int
expectedOK bool
}{
{
name: "single page",
first: 3, last: 3, pageSize: 4096, osPageSize: 4096,
expectedStart: 3 * 4096, expectedEnd: 4 * 4096, expectedOK: true,
},
{
name: "multiple pages",
first: 3, last: 7, pageSize: 4096, osPageSize: 4096,
expectedStart: 3 * 4096, expectedEnd: 8 * 4096, expectedOK: true,
},
{
name: "pages smaller than OS pages",
first: 5, last: 6, pageSize: 1024, osPageSize: 4096,
expectedStart: 4096, expectedEnd: 8192, expectedOK: true,
},
{
name: "pages smaller than OS pages across a boundary",
first: 7, last: 8, pageSize: 1024, osPageSize: 4096,
expectedStart: 4096, expectedEnd: 3 * 4096, expectedOK: true,
},
{
name: "pages larger than OS pages",
first: 1, last: 1, pageSize: 16384, osPageSize: 4096,
expectedStart: 16384, expectedEnd: 2 * 16384, expectedOK: true,
},
{
name: "last page of the mmap",
first: 255, last: 255, pageSize: 4096, osPageSize: 16384,
expectedStart: 252 * 4096, expectedEnd: mmapSize, expectedOK: true,
},
{
name: "beyond the mmap",
first: 255, last: 256, pageSize: 4096, osPageSize: 4096,
},
{
name: "page id overflowing the offset",
first: 1, last: common.Pgid(math.MaxUint64), pageSize: 4096, osPageSize: 4096,
},
{
name: "empty range",
first: 4, last: 3, pageSize: 4096, osPageSize: 4096,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
start, end, ok := prefetchRange(tc.first, tc.last, tc.pageSize, tc.osPageSize, mmapSize)
require.Equal(t, tc.expectedOK, ok)
require.Equal(t, tc.expectedStart, start)
require.Equal(t, tc.expectedEnd, end)
})
}
}

func TestForEachOverflowRead(t *testing.T) {
const pageSize = 4096

type pageRange struct{ first, last common.Pgid }

bucket := make([]byte, common.BucketHeaderSize)

testCases := []struct {
name string
flags uint16
overflow uint32
inodes []struct {
flags uint32
key, value []byte
}
expected []pageRange
}{
{
name: "leaf page without overflow",
flags: common.LeafPageFlag,
inodes: []struct {
flags uint32
key, value []byte
}{
{key: []byte("a"), value: make([]byte, 100)},
{key: []byte("b"), value: make([]byte, 100)},
},
},
{
// The second key starts on the second page and the third
// one on the third page. The last two pages only hold the
// value of the third key.
name: "leaf page with overflow",
flags: common.LeafPageFlag,
overflow: 4,
inodes: []struct {
flags uint32
key, value []byte
}{
{key: []byte("a"), value: make([]byte, 6000)},
{key: []byte("b"), value: make([]byte, 5000)},
{key: []byte("c"), value: make([]byte, 9000)},
},
expected: []pageRange{{11, 11}, {12, 12}},
},
{
// Both the second key and the bucket are on the second
// page, which must be reported only once.
name: "leaf page with keys sharing an overflow page",
flags: common.LeafPageFlag,
overflow: 1,
inodes: []struct {
flags uint32
key, value []byte
}{
{key: []byte("a"), value: make([]byte, 6000)},
{key: []byte("b"), value: make([]byte, 100)},
{flags: common.BucketLeafFlag, key: []byte("c"), value: bucket},
},
expected: []pageRange{{11, 11}},
},
{
name: "branch page with overflow",
flags: common.BranchPageFlag,
overflow: 2,
inodes: []struct {
flags uint32
key, value []byte
}{
{key: make([]byte, 5000)},
{key: make([]byte, 5000)},
},
expected: []pageRange{{11, 12}},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
buf := make([]byte, (int(tc.overflow)+1)*pageSize)
p := common.LoadPage(buf)
p.SetId(10)
p.SetFlags(tc.flags)
p.SetCount(uint16(len(tc.inodes)))
p.SetOverflow(tc.overflow)

inodes := make(common.Inodes, len(tc.inodes))
for i, in := range tc.inodes {
inodes[i].SetFlags(in.flags)
inodes[i].SetKey(in.key)
inodes[i].SetValue(in.value)
}
common.WriteInodeToPage(inodes, p)

var got []pageRange
forEachOverflowRead(p, pageSize, func(first, last common.Pgid) {
got = append(got, pageRange{first, last})
})
require.Equal(t, tc.expected, got)
})
}
}
Loading