From 7d7267b062d68e6dac256dee6071caa1d7d13638 Mon Sep 17 00:00:00 2001 From: Mustafa Elbehery Date: Mon, 1 Jan 2024 17:33:35 +0100 Subject: [PATCH] add test check page Signed-off-by: Mustafa Elbehery --- db_whitebox_test.go | 232 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/db_whitebox_test.go b/db_whitebox_test.go index 130c3e349..a2a261863 100644 --- a/db_whitebox_test.go +++ b/db_whitebox_test.go @@ -1,13 +1,21 @@ package bbolt import ( + "bytes" + crand "crypto/rand" + "encoding/binary" + "fmt" + "math/rand" "path/filepath" "testing" + "unsafe" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/bbolt/errors" + "go.etcd.io/bbolt/internal/common" + "go.etcd.io/bbolt/internal/guts_cli" ) func TestOpenWithPreLoadFreelist(t *testing.T) { @@ -112,6 +120,230 @@ func TestMethodPage(t *testing.T) { } } +func TestTx_Check_CorruptPage_ViolateBtreeInvariant(t *testing.T) { + bucketKey := "testBucket" + pageSize := 4096 + + t.Log("Creating db file.") + db, err := Open(filepath.Join(t.TempDir(), "db"), 0600, &Options{PageSize: pageSize}) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + + uErr := db.Update(func(tx *Tx) error { + t.Logf("Creating bucket '%v'.", bucketKey) + b, bErr := tx.CreateBucketIfNotExists([]byte(bucketKey)) + require.NoError(t, bErr) + t.Logf("Generating random data in bucket '%v'.", bucketKey) + generateSampleDataInBucket(t, b, pageSize, 3) + return nil + }) + require.NoError(t, uErr) + + t.Logf("Corrupting random leaf page in bucket '%v'.", bucketKey) + victimPageId, validPageIds := corruptLeafPage(t, db, pageSize, false) + + t.Log("Running consistency check.") + vErr := db.View(func(tx *Tx) error { + chkConfig := checkConfig{ + kvStringer: HexKVStringer(), + } + + t.Log("Check corrupted page.") + ch := make(chan error) + chkConfig.pageId = uint(victimPageId) + go func() { + defer close(ch) + tx.check(chkConfig, ch) + }() + + var cErrs []error + for cErr := range ch { + cErrs = append(cErrs, cErr) + } + require.Greater(t, len(cErrs), 0) + + t.Log("Check valid pages.") + cErrs = cErrs[:0] + for _, pgId := range validPageIds { + ch = make(chan error) + chkConfig.pageId = uint(pgId) + go func() { + defer close(ch) + tx.check(chkConfig, ch) + }() + + for cErr := range ch { + cErrs = append(cErrs, cErr) + } + require.Equal(t, 0, len(cErrs)) + } + return nil + }) + require.NoError(t, vErr) +} + +func TestTx_Check_CorruptPage_DumpRandomBytes(t *testing.T) { + bucketKey := "testBucket" + pageSize := 4096 + + t.Log("Creating db file.") + db, err := Open(filepath.Join(t.TempDir(), "db"), 0600, &Options{PageSize: pageSize}) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Close()) + }() + + uErr := db.Update(func(tx *Tx) error { + t.Logf("Creating bucket '%v'.", bucketKey) + b, bErr := tx.CreateBucketIfNotExists([]byte(bucketKey)) + require.NoError(t, bErr) + t.Logf("Generating random data in bucket '%v'.", bucketKey) + generateSampleDataInBucket(t, b, pageSize, 3) + return nil + }) + require.NoError(t, uErr) + + t.Logf("Corrupting random leaf page in bucket '%v'.", bucketKey) + victimPageId, validPageIds := corruptLeafPage(t, db, pageSize, true) + + t.Log("Running consistency check.") + vErr := db.View(func(tx *Tx) error { + chkConfig := checkConfig{ + kvStringer: HexKVStringer(), + } + + t.Log("Check valid pages.") + var cErrs []error + for _, pgId := range validPageIds { + ch := make(chan error) + chkConfig.pageId = uint(pgId) + go func() { + defer close(ch) + tx.check(chkConfig, ch) + }() + + for cErr := range ch { + cErrs = append(cErrs, cErr) + } + require.Equal(t, 0, len(cErrs)) + } + + t.Log("Check corrupted page.") + ch := make(chan error) + defer close(ch) + + defer func() { + r := recover() + require.NotNil(t, r) + }() + chkConfig.pageId = uint(victimPageId) + tx.check(chkConfig, ch) + + return nil + }) + require.NoError(t, vErr) +} + +// corruptLeafPage write an invalid leafPageElement into the victim page. +func corruptLeafPage(t testing.TB, db *DB, pageSize int, expectPanic bool) (victimPageId common.Pgid, validPageIds []common.Pgid) { + t.Helper() + + victimPageId, validPageIds = findVictimPageId(t, db) + + victimPage, victimBuf, err := guts_cli.ReadPage(db.Path(), uint64(victimPageId)) + require.NoError(t, err) + require.True(t, victimPage.IsLeafPage()) + require.True(t, victimPage.Count() > 0) + + // Dumping random bytes in victim page for corruption. + copy(victimBuf[32:], generateCorruptionBytes(t, pageSize, expectPanic)) + // Write the corrupt page to db file. + err = guts_cli.WritePage(db.Path(), victimBuf) + require.NoError(t, err) + + return victimPageId, validPageIds +} + +// findVictimPageId finds all the leaf pages of a bucket and picks a random leaf page as a victim to be corrupted. +func findVictimPageId(t testing.TB, db *DB) (victimPageId common.Pgid, validPageIds []common.Pgid) { + t.Helper() + // Read DB's RootPage. + rootPageId, _, err := guts_cli.GetRootPage(db.Path()) + require.NoError(t, err) + rootPage, _, err := guts_cli.ReadPage(db.Path(), uint64(rootPageId)) + require.NoError(t, err) + require.True(t, rootPage.IsLeafPage()) + require.Equal(t, 1, len(rootPage.LeafPageElements())) + // Find Bucket's RootPage. + lpe := rootPage.LeafPageElement(uint16(0)) + require.Equal(t, uint32(common.BranchPageFlag), lpe.Flags()) + k := lpe.Key() + require.Equal(t, "testBucket", string(k)) + bucketRootPageId := lpe.Bucket().RootPage() + // Read Bucket's RootPage. + bucketRootPage, _, err := guts_cli.ReadPage(db.Path(), uint64(bucketRootPageId)) + require.NoError(t, err) + require.Equal(t, uint16(common.BranchPageFlag), bucketRootPage.Flags()) + // Retrieve Bucket's PageIds + var bucketPageIds []common.Pgid + for _, bpe := range bucketRootPage.BranchPageElements() { + bucketPageIds = append(bucketPageIds, bpe.Pgid()) + } + + randomIdx := rand.Intn(len(bucketPageIds)) + victimPageId = bucketPageIds[randomIdx] + validPageIds = append(bucketPageIds[:randomIdx], bucketPageIds[randomIdx+1:]...) + return victimPageId, validPageIds +} + +// generateSampleDataInBucket fill in sample data into given bucket to create the given +// number of leafPages. To control the number of leafPages, sample data are generated in order. +func generateSampleDataInBucket(t testing.TB, bk *Bucket, pageSize int, lPages int) { + t.Helper() + + maxBytesInPage := int(DefaultFillPercent * float32(pageSize)) + + currentKey := 1 + currentVal := 100 + for i := 0; i < lPages; i++ { + currentSize := common.PageHeaderSize + for { + err := bk.Put([]byte(fmt.Sprintf("key_%d", currentKey)), []byte(fmt.Sprintf("val_%d", currentVal))) + require.NoError(t, err) + currentSize += common.LeafPageElementSize + unsafe.Sizeof(currentKey) + unsafe.Sizeof(currentVal) + if int(currentSize) >= maxBytesInPage { + break + } + currentKey++ + currentVal++ + } + } +} + +// generateCorruptionBytes returns random bytes to corrupt a page. +// It inserts a page element which violates the btree key order if no panic is expected. +// Otherwise, it dumps random bytes into a page which causes a panic. +func generateCorruptionBytes(t testing.TB, pageSize int, expectPanic bool) []byte { + if expectPanic { + // Generated data size is between pageHeader and pageSize. + maxLen := pageSize - int(common.PageHeaderSize) + minLen := 16 + corruptDataLength := rand.Intn(maxLen-minLen) + minLen + corruptData := make([]byte, corruptDataLength) + _, err := crand.Read(corruptData) + require.NoError(t, err) + return corruptData + } + // Insert LeafPageElement which violates the BTree range. + invalidLPE := common.NewLeafPageElement(0, 0, 0, 0) + var buf bytes.Buffer + err := binary.Write(&buf, binary.BigEndian, invalidLPE) + require.NoError(t, err) + return buf.Bytes() +} + func prepareData(t *testing.T) (string, error) { fileName := filepath.Join(t.TempDir(), "db") db, err := Open(fileName, 0666, nil)