SSZ-QL: calculate generalized indices for elements #15873
+967
−35
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ### Added | ||
|
|
||
| - Added GeneralizedIndicesFromPath function to calculate the GIs for a given sszInfo object and a PathElement |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,306 @@ | ||
| package query | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/OffchainLabs/prysm/v6/encoding/ssz" | ||
| ) | ||
|
|
||
| const listBaseIndex = 2 | ||
|
|
||
| // GetGeneralizedIndexFromPath calculates the generalized index for a given path. | ||
| // To calculate the generalized index, two inputs are needed: | ||
| // 1. The sszInfo of the root object, to be able to navigate the SSZ structure | ||
| // 2. The path to the field (e.g., "field_a.field_b[3].field_c") | ||
| // It walks the path step by step, updating the generalized index at each step. | ||
| func GetGeneralizedIndexFromPath(info *SszInfo, path []PathElement) (uint64, error) { | ||
| if info == nil { | ||
| return 0, errors.New("SszInfo is nil") | ||
| } | ||
|
|
||
| // If path is empty, no generalized index can be computed. | ||
| if len(path) == 0 { | ||
| return 0, errors.New("cannot compute generalized index for an empty path") | ||
| } | ||
|
|
||
| // Starting from the root generalized index | ||
| root := uint64(1) | ||
| currentInfo := info | ||
|
|
||
| for _, pathElement := range path { | ||
| element := pathElement | ||
|
|
||
| // Check that we are in a container to access fields | ||
| if currentInfo.sszType != Container { | ||
fernantho marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return 0, fmt.Errorf("indexing requires a container field step first, got %s", currentInfo.sszType) | ||
| } | ||
|
|
||
| // Retrieve the field position and SSZInfo for the field in the current container | ||
| fieldPos, fieldSsz, err := getContainerFieldByName(currentInfo, element.Name) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("container field %s not found: %w", element.Name, err) | ||
| } | ||
|
|
||
| // Get the chunk count for the current container | ||
| chunkCount, err := getChunkCount(currentInfo) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("chunk count error: %w", err) | ||
| } | ||
|
|
||
| // Update the generalized index to point to the specified field | ||
| root = root*nextPowerOfTwo(chunkCount) + fieldPos | ||
| currentInfo = fieldSsz | ||
|
|
||
| // Check if a path element is a length field | ||
| if element.Length { | ||
| currentInfo, root, err = calculateLengthGeneralizedIndex(fieldSsz, element, root) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("length calculation error: %w", err) | ||
| } | ||
| continue | ||
| } | ||
|
|
||
| if element.Index == nil { | ||
| continue | ||
fernantho marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| switch fieldSsz.sszType { | ||
| case List: | ||
| currentInfo, root, err = calculateListGeneralizedIndex(fieldSsz, element, root) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("list calculation error: %w", err) | ||
| } | ||
|
|
||
| case Vector: | ||
| currentInfo, root, err = calculateVectorGeneralizedIndex(fieldSsz, element, root) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("vector calculation error: %w", err) | ||
| } | ||
|
|
||
| case Bitlist: | ||
| currentInfo, root, err = calculateBitlistGeneralizedIndex(fieldSsz, element, root) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("bitlist calculation error: %w", err) | ||
| } | ||
|
|
||
| case Bitvector: | ||
| currentInfo, root, err = calculateBitvectorGeneralizedIndex(fieldSsz, element, root) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("bitvector calculation error: %w", err) | ||
| } | ||
|
|
||
| default: | ||
| return 0, fmt.Errorf("indexing not supported for type %s", fieldSsz.sszType) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| return root, nil | ||
| } | ||
|
|
||
| // getContainerFieldByName finds a container field by its name | ||
| // and returns its index and SSZInfo. | ||
| func getContainerFieldByName(info *SszInfo, fieldName string) (uint64, *SszInfo, error) { | ||
| containerInfo, err := info.ContainerInfo() | ||
| if err != nil { | ||
| return 0, nil, err | ||
| } | ||
|
|
||
| for index, name := range containerInfo.order { | ||
| if name == fieldName { | ||
| fieldInfo := containerInfo.fields[name] | ||
| if fieldInfo == nil || fieldInfo.sszInfo == nil { | ||
| return 0, nil, fmt.Errorf("field %s has no ssz info", name) | ||
| } | ||
| return uint64(index), fieldInfo.sszInfo, nil | ||
| } | ||
| } | ||
|
|
||
| return 0, nil, fmt.Errorf("field %s not found", fieldName) | ||
| } | ||
|
|
||
| // Helpers for Generalized Index calculation per type | ||
|
|
||
| // calculateLengthGeneralizedIndex calculates the generalized index for a length field. | ||
| // note: length fields are only valid for List and Bitlist types. Multi-dimensional arrays are not supported. | ||
|
There was a problem hiding this comment. Shouldn't this be supported for There was a problem hiding this comment. In relation to this, I also followed the spec algo: for p in path:
# If we descend to a basic type, the path cannot continue further
assert not issubclass(typ, BasicValue)
if p == "__len__":
+ assert issubclass(typ, (List, ByteList))
typ = uint64
root = GeneralizedIndex(root * 2 + 1)To my understanding, there is no length field for |
||
| func calculateLengthGeneralizedIndex(fieldSsz *SszInfo, element PathElement, root uint64) (*SszInfo, uint64, error) { | ||
|
||
| if element.Index != nil { | ||
| return nil, 0, fmt.Errorf("len() is not supported for indexed elements (multi-dimensional arrays)") | ||
fernantho marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } | ||
| // Length field is only valid for List and Bitlist types | ||
| if fieldSsz.sszType != List && fieldSsz.sszType != Bitlist { | ||
| return nil, 0, fmt.Errorf("len() is only supported for List and Bitlist types, got %s", fieldSsz.sszType) | ||
| } | ||
| // Length is a uint64 per SSZ spec | ||
| currentInfo := &SszInfo{sszType: Uint64} | ||
| lengthRoot := root*2 + 1 | ||
| return currentInfo, lengthRoot, nil | ||
| } | ||
|
|
||
| // calculateListGeneralizedIndex calculates the generalized index for a list element. | ||
| func calculateListGeneralizedIndex(fieldSsz *SszInfo, element PathElement, root uint64) (*SszInfo, uint64, error) { | ||
| li, err := fieldSsz.ListInfo() | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("list info error: %w", err) | ||
| } | ||
| elem, err := li.Element() | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("list element error: %w", err) | ||
| } | ||
| if *element.Index >= li.Limit() { | ||
| return nil, 0, fmt.Errorf("index %d out of bounds for list with limit %d", *element.Index, li.Limit()) | ||
| } | ||
| // Compute chunk position for the element | ||
| var chunkPos uint64 | ||
| if elem.sszType.isBasic() { | ||
| start := *element.Index * itemLength(elem) | ||
| chunkPos = start / ssz.BytesPerChunk | ||
| } else { | ||
| chunkPos = *element.Index | ||
| } | ||
| innerChunkCount, err := getChunkCount(fieldSsz) | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("chunk count error: %w", err) | ||
| } | ||
| // root = root * base_index * pow2ceil(chunk_count(container)) + fieldPos | ||
| listRoot := root*listBaseIndex*nextPowerOfTwo(innerChunkCount) + chunkPos | ||
| currentInfo := elem | ||
|
|
||
| return currentInfo, listRoot, nil | ||
| } | ||
|
|
||
| // calculateVectorGeneralizedIndex calculates the generalized index for a vector element. | ||
| func calculateVectorGeneralizedIndex(fieldSsz *SszInfo, element PathElement, root uint64) (*SszInfo, uint64, error) { | ||
| vi, err := fieldSsz.VectorInfo() | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("vector info error: %w", err) | ||
| } | ||
| elem, err := vi.Element() | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("vector element error: %w", err) | ||
| } | ||
| if *element.Index >= vi.Length() { | ||
| return nil, 0, fmt.Errorf("index %d out of bounds for vector with length %d", *element.Index, vi.Length()) | ||
| } | ||
| var chunkPos uint64 | ||
| if elem.sszType.isBasic() { | ||
| start := *element.Index * itemLength(elem) | ||
| chunkPos = start / ssz.BytesPerChunk | ||
| } else { | ||
| chunkPos = *element.Index | ||
| } | ||
| innerChunkCount, err := getChunkCount(fieldSsz) | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("chunk count error: %w", err) | ||
| } | ||
| vectorRoot := root*nextPowerOfTwo(innerChunkCount) + chunkPos | ||
|
|
||
| currentInfo := elem | ||
| return currentInfo, vectorRoot, nil | ||
| } | ||
|
|
||
| // calculateBitlistGeneralizedIndex calculates the generalized index for a bitlist element. | ||
| func calculateBitlistGeneralizedIndex(fieldSsz *SszInfo, element PathElement, root uint64) (*SszInfo, uint64, error) { | ||
| // Bits packed into 256-bit chunks; select the chunk containing the bit | ||
| chunkPos := *element.Index / ssz.BitsPerChunk | ||
| innerChunkCount, err := getChunkCount(fieldSsz) | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("chunk count error: %w", err) | ||
| } | ||
| bitlistRoot := root*listBaseIndex*nextPowerOfTwo(innerChunkCount) + chunkPos | ||
|
|
||
| // Bits element is not further descendable; set to basic to guard further steps | ||
| currentInfo := &SszInfo{sszType: Boolean} | ||
| return currentInfo, bitlistRoot, nil | ||
| } | ||
|
|
||
| // calculateBitvectorGeneralizedIndex calculates the generalized index for a bitvector element. | ||
| func calculateBitvectorGeneralizedIndex(fieldSsz *SszInfo, element PathElement, root uint64) (*SszInfo, uint64, error) { | ||
| chunkPos := *element.Index / ssz.BitsPerChunk | ||
| innerChunkCount, err := getChunkCount(fieldSsz) | ||
| if err != nil { | ||
| return nil, 0, fmt.Errorf("chunk count error: %w", err) | ||
| } | ||
| bitvectorRoot := root*nextPowerOfTwo(innerChunkCount) + chunkPos | ||
|
|
||
| // Bits element is not further descendable; set to basic to guard further steps | ||
| currentInfo := &SszInfo{sszType: Boolean} | ||
| return currentInfo, bitvectorRoot, nil | ||
| } | ||
|
|
||
| // Helper functions from SSZ spec | ||
|
|
||
| // itemLength calculates the byte length of an SSZ item based on its type information. | ||
| // For basic SSZ types (uint8, uint16, uint32, uint64, bool, etc.), it returns the actual | ||
| // size of the type in bytes. For compound types (containers, lists, vectors), it returns | ||
| // BytesPerChunk which represents the standard SSZ chunk size (32 bytes) used for | ||
| // Merkle tree operations in the SSZ serialization format. | ||
| func itemLength(info *SszInfo) uint64 { | ||
| if info.sszType.isBasic() { | ||
| return info.Size() | ||
| } | ||
| return ssz.BytesPerChunk | ||
| } | ||
|
|
||
| // nextPowerOfTwo computes the next power of two greater than or equal to v. | ||
| func nextPowerOfTwo(v uint64) uint64 { | ||
| v-- | ||
| v |= v >> 1 | ||
| v |= v >> 2 | ||
| v |= v >> 4 | ||
| v |= v >> 8 | ||
| v |= v >> 16 | ||
| v++ | ||
| return uint64(v) | ||
| } | ||
|
|
||
| // getChunkCount returns the number of chunks for the given SSZInfo (equivalent to chunk_count in the spec) | ||
| func getChunkCount(info *SszInfo) (uint64, error) { | ||
| switch info.sszType { | ||
| case Uint8, Uint16, Uint32, Uint64, Boolean: | ||
| return 1, nil | ||
| case Container: | ||
| containerInfo, err := info.ContainerInfo() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| return uint64(len(containerInfo.fields)), nil | ||
| case List: | ||
| listInfo, err := info.ListInfo() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| elementInfo, err := listInfo.Element() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| elemLength := itemLength(elementInfo) | ||
| return (listInfo.Limit()*elemLength + 31) / ssz.BytesPerChunk, nil | ||
| case Vector: | ||
| vectorInfo, err := info.VectorInfo() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| elementInfo, err := vectorInfo.Element() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| elemLength := itemLength(elementInfo) | ||
| return (vectorInfo.Length()*elemLength + 31) / ssz.BytesPerChunk, nil | ||
| case Bitlist: | ||
| bitlistInfo, err := info.BitlistInfo() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| return (bitlistInfo.Limit() + 255) / ssz.BitsPerChunk, nil // Bits are packed into 256-bit chunks | ||
| case Bitvector: | ||
| bitvectorInfo, err := info.BitvectorInfo() | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| return (bitvectorInfo.Length() + 255) / ssz.BitsPerChunk, nil // Bits are packed into 256-bit chunks | ||
| default: | ||
| return 0, errors.New("unsupported SSZ type for chunk count calculation") | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you rename it to something like
currentIndex? It's a bit odd to call itrootsince it's not a root of anythingThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sure!
I got the inspiration for the
rootname from spec:But I do not like it because I do not associate it to an index.