Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 42 additions & 5 deletions db.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@ type DB struct {
rwtx *Tx
stats *Stats

freelist fl.Interface
freelistLoad sync.Once
freelist fl.Interface
freelistLoad sync.Once
freelistLoadErr error

pagePool sync.Pool

Expand Down Expand Up @@ -301,7 +302,11 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
}

if db.PreLoadFreelist {
db.loadFreelist()
if err := db.loadFreelist(); err != nil {
_ = db.close()
lg.Errorf("failed to load freelist (%s): %v", path, err)
return nil, err
}
}

if db.readOnly {
Expand Down Expand Up @@ -419,7 +424,11 @@ func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
// loadFreelist reads the freelist if it is synced, or reconstructs it
// by scanning the DB if it is not synced. It assumes there are no
// concurrent accesses being made to the freelist.
func (db *DB) loadFreelist() {
//
// A freelist that violates the invariants the writer maintains (see
// validateFreelist) is reported as an error instead of being accepted
// and panicking in the middle of a later commit.
func (db *DB) loadFreelist() error {
db.freelistLoad.Do(func() {
db.freelist = newFreelist(db.FreelistType)
if !db.hasSyncedFreelist() {
Expand All @@ -429,10 +438,38 @@ func (db *DB) loadFreelist() {
// Read free list from freelist page.
db.freelist.Read(db.page(db.meta().Freelist()))
}
if db.stats != nil {
db.freelistLoadErr = db.validateFreelist()
if db.freelistLoadErr == nil && db.stats != nil {
db.stats.FreePageN = db.freelist.FreeCount()
}
})
return db.freelistLoadErr
}

// validateFreelist checks the cheap structural invariants of the loaded
// freelist that the writer always maintains: every entry must be a real
// page id (never a meta page), must lie below the high-water mark, and
// must appear at most once. Without this, a corrupt freelist is accepted
// at open and only dies later, mid-commit, with a panic such as
// "invalid page allocation: 0" or "page N already freed" (see #1319).
func (db *DB) validateFreelist() error {
highWaterMark := db.meta().Pgid()
ids := make([]common.Pgid, db.freelist.Count())
db.freelist.Copyall(ids)
seen := make(map[common.Pgid]struct{}, len(ids))
for _, id := range ids {
if id <= 1 {
return fmt.Errorf("freelist contains invalid page id %d", id)
}
if id >= highWaterMark {
return fmt.Errorf("freelist page id %d is at or above the high-water mark %d", id, highWaterMark)
}
if _, ok := seen[id]; ok {
return fmt.Errorf("freelist contains duplicate page id %d", id)
}
seen[id] = struct{}{}
}
return nil
}

func (db *DB) hasSyncedFreelist() bool {
Expand Down
54 changes: 54 additions & 0 deletions freelist_validate_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package bbolt

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
"go.etcd.io/bbolt/internal/common"
)

// TestLoadFreelistRejectsInvalidEntries is a regression test for
// https://github.com/etcd-io/bbolt/issues/1319: a corrupt freelist was
// accepted at open and only died later, mid-commit, with a panic
// ("invalid page allocation: 0", "page N already freed", or
// "freelist pgid (N) above high water mark"). Open now fails with a
// clear error instead. Fixtures are the reproducer databases attached
// to the issue.
func TestLoadFreelistRejectsInvalidEntries(t *testing.T) {
for _, tc := range []struct {
fixture string
want string
}{
{"bbolt_freelist.go_117.db", "freelist contains invalid page id 0"},
{"bbolt_freelist.go_175.db", "freelist contains invalid page id 0"},
{"bbolt_node.go_375.db", "freelist page id 8960 is at or above the high-water mark 4"},
} {
t.Run(tc.fixture, func(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "issue1319", tc.fixture))
require.NoError(t, err)
path := filepath.Join(t.TempDir(), "db")
require.NoError(t, os.WriteFile(path, data, 0o600))
_, err = Open(path, 0o600, &Options{PreLoadFreelist: true})
require.Error(t, err)
require.Contains(t, err.Error(), tc.want)
})
}
}

// TestValidateFreelistRejectsDuplicateEntries covers the duplicate-id
// branch of validateFreelist, which the issue fixtures do not reach:
// their sorted ids trip the invalid-id check first.
func TestValidateFreelistRejectsDuplicateEntries(t *testing.T) {
path := filepath.Join(t.TempDir(), "db")
db, err := Open(path, 0o600, nil)
require.NoError(t, err)
defer db.Close()

// A fresh database has meta pages 0/1, freelist page 2 and root
// page 3, so 2 is a plausible free page id below the high-water
// mark; listing it twice is the corruption under test.
db.freelist.Init(common.Pgids{2, 2})
require.EqualError(t, db.validateFreelist(), "freelist contains duplicate page id 2")
}
Binary file added testdata/issue1319/bbolt_freelist.go_117.db
Binary file not shown.
Binary file added testdata/issue1319/bbolt_freelist.go_175.db
Binary file not shown.
Binary file added testdata/issue1319/bbolt_node.go_375.db
Binary file not shown.
5 changes: 4 additions & 1 deletion tx_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ func (tx *Tx) check(cfg checkConfig, ch chan error) {
}
}()
// Force loading free list if opened in ReadOnly mode.
tx.db.loadFreelist()
if err := tx.db.loadFreelist(); err != nil {
ch <- err
return
}

// Check if any pages are double freed.
freed := make(map[common.Pgid]bool)
Expand Down