Skip to content

optimize get blobs #15902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 17 commits into
base: develop
Choose a base branch
from
Draft

optimize get blobs #15902

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4470204
init
james-prysm Oct 20, 2025
42e7adf
reverting some functions
james-prysm Oct 20, 2025
05c535b
rolling back a change and fixing linting
james-prysm Oct 20, 2025
2c918aa
wip
james-prysm Oct 22, 2025
3000a56
wip
james-prysm Oct 22, 2025
04e8e9f
fixing test
james-prysm Oct 22, 2025
edbe105
Merge branch 'develop' into optimize-get-blobs
james-prysm Oct 22, 2025
dd2a69b
breaking up proofs and cells for cleaner code
james-prysm Oct 22, 2025
7a6f90e
fixing test and type
james-prysm Oct 23, 2025
8cad4dd
Merge branch 'develop' into optimize-get-blobs
james-prysm Oct 23, 2025
e17882c
fixing safe conversion
james-prysm Oct 23, 2025
ee3c9ef
fixing test
james-prysm Oct 23, 2025
4f827a5
fixing more tests
james-prysm Oct 23, 2025
3a0028c
fixing even more tests
james-prysm Oct 23, 2025
07e6edc
fix the 0 indices option
james-prysm Oct 23, 2025
5acab70
adding a test for coverage
james-prysm Oct 24, 2025
da61cee
small test update
james-prysm Oct 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions beacon-chain/core/peerdas/reconstruction.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,10 @@ func ReconstructDataColumnSidecars(verifiedRoSidecars []blocks.VerifiedRODataCol
// - All `dataColumnSidecars` has to be committed to the same block, and
// - `dataColumnSidecars` must be sorted by index and should not contain duplicates.
// - `dataColumnSidecars` must contain either all sidecars corresponding to (non-extended) blobs,
// or either enough sidecars to reconstruct the blobs.
func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.VerifiedRODataColumn, indices []int) ([]*blocks.VerifiedROBlob, error) {
// - either enough sidecars to reconstruct the blobs.git a
//
// The skipKzgProofs parameter indicates whether to skip expensive KZG proof computations (post-Fulu optimization).
func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.VerifiedRODataColumn, indices []int, skipKzgProofs bool) ([]*blocks.VerifiedROBlob, error) {
Copy link
Contributor

@nalepae nalepae Oct 21, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should not return a verified blob sidecar with a missing blob proof.
It breaks the promise of what is a verified blob sidecar.

// Return early if no blobs are requested.
if len(indices) == 0 {
return nil, nil
Expand Down Expand Up @@ -172,7 +174,7 @@ func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.
// If all column sidecars corresponding to (non-extended) blobs are present, no need to reconstruct.
if verifiedDataColumnSidecars[cellsPerBlob-1].Index == uint64(cellsPerBlob-1) {
// Convert verified data column sidecars to verified blob sidecars.
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, verifiedDataColumnSidecars, indices)
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, verifiedDataColumnSidecars, indices, skipKzgProofs)
if err != nil {
return nil, errors.Wrap(err, "blob sidecars from data column sidecars")
}
Expand All @@ -187,7 +189,7 @@ func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.
}

// Convert verified data column sidecars to verified blob sidecars.
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, reconstructedDataColumnSidecars, indices)
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, reconstructedDataColumnSidecars, indices, skipKzgProofs)
if err != nil {
return nil, errors.Wrap(err, "blob sidecars from data column sidecars")
}
Expand Down Expand Up @@ -279,7 +281,8 @@ func ComputeCellsAndProofsFromStructured(blobsAndProofs []*pb.BlobAndProofV2) ([
}

// blobSidecarsFromDataColumnSidecars converts verified data column sidecars to verified blob sidecars.
func blobSidecarsFromDataColumnSidecars(roBlock blocks.ROBlock, dataColumnSidecars []blocks.VerifiedRODataColumn, indices []int) ([]*blocks.VerifiedROBlob, error) {
// The skipKzgProofs parameter indicates whether to skip expensive KZG proof computations (post-Fulu optimization).
func blobSidecarsFromDataColumnSidecars(roBlock blocks.ROBlock, dataColumnSidecars []blocks.VerifiedRODataColumn, indices []int, skipKzgProofs bool) ([]*blocks.VerifiedROBlob, error) {
referenceSidecar := dataColumnSidecars[0]

kzgCommitments := referenceSidecar.KzgCommitments
Expand All @@ -304,11 +307,19 @@ func blobSidecarsFromDataColumnSidecars(roBlock blocks.ROBlock, dataColumnSideca
return nil, errors.New("wrong KZG commitment size - should never happen")
}

// Compute the blob KZG proof.
blobKzgProof, err := kzg.ComputeBlobKZGProof(&blob, kzgCommitment)
if err != nil {
return nil, errors.Wrap(err, "compute blob KZG proof")
// Conditionally compute the blob KZG proof based on caller's needs.
// For GetBlobs endpoint (which only returns blob data), we skip the expensive computation.
// For GetBlobSidecars endpoint (which returns full sidecars), we compute the proof.
var blobKzgProof kzg.Proof
if !skipKzgProofs {
// Compute the blob KZG proof only when needed (e.g., for GetBlobSidecars).
computedProof, err := kzg.ComputeBlobKZGProof(&blob, kzgCommitment)
if err != nil {
return nil, errors.Wrap(err, "compute blob KZG proof")
}
blobKzgProof = computedProof
}
// If skipKzgProofs is true, blobKzgProof remains zero (default value)

// Build the inclusion proof for the blob.
var kzgBlob kzg.Blob
Expand Down
20 changes: 10 additions & 10 deletions beacon-chain/core/peerdas/reconstruction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,13 @@ func TestReconstructBlobs(t *testing.T) {
fs := util.SlotAtEpoch(t, params.BeaconConfig().FuluForkEpoch)

t.Run("no index", func(t *testing.T) {
actual, err := peerdas.ReconstructBlobs(emptyBlock, nil, nil)
actual, err := peerdas.ReconstructBlobs(emptyBlock, nil, nil, false)
require.NoError(t, err)
require.IsNil(t, actual)
})

t.Run("empty input", func(t *testing.T) {
_, err := peerdas.ReconstructBlobs(emptyBlock, nil, []int{0})
_, err := peerdas.ReconstructBlobs(emptyBlock, nil, []int{0}, false)
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})

Expand All @@ -149,7 +149,7 @@ func TestReconstructBlobs(t *testing.T) {
// Arbitrarily change the order of the sidecars.
verifiedRoSidecars[3], verifiedRoSidecars[2] = verifiedRoSidecars[2], verifiedRoSidecars[3]

_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0}, false)
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})

Expand All @@ -159,7 +159,7 @@ func TestReconstructBlobs(t *testing.T) {
// [0, 1, 1, 3, 4, ...]
verifiedRoSidecars[2] = verifiedRoSidecars[1]

_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0}, false)
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})

Expand All @@ -169,15 +169,15 @@ func TestReconstructBlobs(t *testing.T) {
// [0, 1, 2, 1, 4, ...]
verifiedRoSidecars[3] = verifiedRoSidecars[1]

_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0}, false)
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})

t.Run("not enough columns", func(t *testing.T) {
_, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, 3)

inputSidecars := verifiedRoSidecars[:fieldparams.CellsPerBlob-1]
_, err := peerdas.ReconstructBlobs(emptyBlock, inputSidecars, []int{0})
_, err := peerdas.ReconstructBlobs(emptyBlock, inputSidecars, []int{0}, false)
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})

Expand All @@ -186,15 +186,15 @@ func TestReconstructBlobs(t *testing.T) {

roBlock, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, blobCount)

_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{1, blobCount})
_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{1, blobCount}, false)
require.ErrorIs(t, err, peerdas.ErrBlobIndexTooHigh)
})

t.Run("not committed to the same block", func(t *testing.T) {
_, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, 3, util.WithParentRoot([fieldparams.RootLength]byte{1}), util.WithSlot(fs))
roBlock, _, _ := util.GenerateTestFuluBlockWithSidecars(t, 3, util.WithParentRoot([fieldparams.RootLength]byte{2}), util.WithSlot(fs))

_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{0}, false)
require.ErrorContains(t, peerdas.ErrRootMismatch.Error(), err)
})

Expand Down Expand Up @@ -260,7 +260,7 @@ func TestReconstructBlobs(t *testing.T) {

t.Run("no reconstruction needed", func(t *testing.T) {
// Reconstruct blobs.
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, indices)
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, indices, false)
require.NoError(t, err)

// Compare blobs.
Expand All @@ -280,7 +280,7 @@ func TestReconstructBlobs(t *testing.T) {
}

// Reconstruct blobs.
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, filteredSidecars, indices)
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, filteredSidecars, indices, false)
require.NoError(t, err)

// Compare blobs.
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/rpc/eth/blob/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func (s *Server) GetBlobs(w http.ResponseWriter, r *http.Request) {
versionedHashes[i] = hash
}
}
verifiedBlobs, rpcErr = s.Blocker.Blobs(ctx, blockId, options.WithVersionedHashes(versionedHashes))
verifiedBlobs, rpcErr = s.Blocker.Blobs(ctx, blockId, options.WithVersionedHashes(versionedHashes), options.WithSkipKzgProofs())
if rpcErr != nil {
code := core.ErrorReasonToHTTP(rpcErr.Reason)
switch code {
Expand Down
7 changes: 4 additions & 3 deletions beacon-chain/rpc/lookup/blocker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.
return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to create roBlock with root %#x", root), Reason: core.Internal}
}

return p.blobsFromStoredDataColumns(roBlock, indices)
return p.blobsFromStoredDataColumns(roBlock, indices, cfg.SkipKzgProofs)
}

return p.blobsFromStoredBlobs(commitments, root, indices)
Expand Down Expand Up @@ -399,7 +399,8 @@ func (p *BeaconDbBlocker) blobsFromStoredBlobs(commitments [][]byte, root [field
// This function expects data column sidecars to be stored (aka. no blob sidecars).
// If not enough data column sidecars are available to convert blobs from them
// (either directly or after reconstruction), an error is returned.
func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError) {
// The skipKzgProofs parameter indicates whether to skip expensive KZG proof computations (post-Fulu optimization).
func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indices []int, skipKzgProofs bool) ([]*blocks.VerifiedROBlob, *core.RpcError) {
root := block.Root()

// Use all indices if none are provided.
Expand Down Expand Up @@ -440,7 +441,7 @@ func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indic
}

// Reconstruct blob sidecars from data column sidecars.
verifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(block, verifiedRoDataColumnSidecars, indices)
verifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(block, verifiedRoDataColumnSidecars, indices, skipKzgProofs)
if err != nil {
return nil, &core.RpcError{
Err: errors.Wrap(err, "blobs from data columns"),
Expand Down
9 changes: 9 additions & 0 deletions beacon-chain/rpc/options/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ type BlobsOption func(*BlobsConfig)
type BlobsConfig struct {
Indices []int
VersionedHashes [][]byte
SkipKzgProofs bool // Post-Fulu optimization: skip expensive KZG proof computations when proofs aren't needed
}

// WithIndices specifies blob indices to retrieve
Expand All @@ -22,3 +23,11 @@ func WithVersionedHashes(hashes [][]byte) BlobsOption {
c.VersionedHashes = hashes
}
}

// WithSkipKzgProofs indicates that KZG proofs should not be computed (post-Fulu optimization)
// Use this when the caller only needs blob data and not the full sidecar with proofs
func WithSkipKzgProofs() BlobsOption {
return func(c *BlobsConfig) {
c.SkipKzgProofs = true
}
}
Loading