diff --git a/src/Paprika.Tests/Chain/PrefetchingTests.cs b/src/Paprika.Tests/Chain/PrefetchingTests.cs index 6a24aa33..f5f9983e 100644 --- a/src/Paprika.Tests/Chain/PrefetchingTests.cs +++ b/src/Paprika.Tests/Chain/PrefetchingTests.cs @@ -71,67 +71,6 @@ public async Task Prefetches_properly_on_not_changed_structure() } } - [Test] - public async Task Makes_all_decompression_on_prefetch() - { - using var db = PagedDb.NativeMemoryDb(8 * 1024 * 1024, 2); - var merkle = new ComputeMerkleBehavior(ComputeMerkleBehavior.ParallelismNone); - await using var blockchain = new Blockchain(db, merkle); - - // Create one block with some values, commit it and finalize - var hash = Keccak.EmptyTreeHash; - - hash = BuildBlock(blockchain, hash, 1); - blockchain.Finalize(hash); - await blockchain.WaitTillFlush(hash); - - hash = BuildBlock(blockchain, hash, 2); - - return; - - static Keccak BuildBlock(Blockchain blockchain, Keccak parent, uint number) - { - var isFirst = number == 1; - - byte[] value = isFirst ? [17] : [23]; - - const int seed = 13; - const int contracts = 10; - const int slots = 10; - - using var block = blockchain.StartNew(parent); - var random = new Random(seed); - - // Open prefetcher on blocks beyond first - var prefetcher = isFirst == false ? block.OpenPrefetcher() : null; - - for (var i = 0; i < contracts; i++) - { - var contract = random.NextKeccak(); - prefetcher?.PrefetchAccount(contract); - - if (isFirst) - { - block.SetAccount(contract, new Account(1, 1, Keccak.Zero, Keccak.Zero)); - } - - for (var j = 0; j < slots; j++) - { - var storage = random.NextKeccak(); - prefetcher?.PrefetchStorage(contract, storage); - block.SetStorage(contract, storage, value); - } - } - - prefetcher?.SpinTillPrefetchDone(); - - using (RlpMemo.NoDecompression()) - { - return block.Commit(number); - } - } - } - private static void Set(Keccak[] accounts, uint account, IWorldState start, UInt256 bigNonce) { ref var k = ref accounts[account]; diff --git a/src/Paprika.Tests/Merkle/RlpMemoTests.cs b/src/Paprika.Tests/Merkle/RlpMemoTests.cs index 567f5985..857b8f7c 100644 --- a/src/Paprika.Tests/Merkle/RlpMemoTests.cs +++ b/src/Paprika.Tests/Merkle/RlpMemoTests.cs @@ -1,139 +1,502 @@ -using FluentAssertions; +using FluentAssertions; using Paprika.Crypto; -using Paprika.Data; using Paprika.Merkle; +using Paprika.Data; +using Paprika.Chain; namespace Paprika.Tests.Merkle; public class RlpMemoTests { - public const bool OddKey = true; - public const bool EvenKey = false; + // All the write operations on RlpMemo + private enum RlpMemoOperation + { + Set, + Clear, + Delete, + Insert + } [Test] - public void All_children_set_with_all_Keccaks_empty() + public void Insert_get_operation() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span workingMemory = new byte[RlpMemo.MaxSize]; + NibbleSet.Readonly children = new NibbleSet(0xA, 0xB, 0xC); + var memo = new RlpMemo([]); + + InsertRandomKeccak(ref memo, children, out var data, workingMemory); + + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + memo.Exists(i).Should().BeTrue(); + memo.TryGetKeccak(i, out var k).Should().BeTrue(); + k.SequenceEqual(data[i].Span).Should().BeTrue(); + } + } - Run(raw, 0, NibbleSet.Readonly.All, OddKey); + CompareMemoAndDict(memo, data); } [Test] - public void All_children_set_with_all_Keccaks_set() + public void Set_get_operation() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span workingMemory = new byte[RlpMemo.MaxSize]; + NibbleSet.Readonly children = new NibbleSet(0xA, 0xB, 0xC); + var memo = new RlpMemo([]); - for (var i = 0; i < RlpMemo.Size; i++) + InsertRandomKeccak(ref memo, children, out var data, workingMemory); + + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); + + var random = new Random(13); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) { - raw[i] = (byte)(i & 0xFF); + if (children[i]) + { + data[i] = random.NextKeccak(); + memo.Set(data[i].Span, i); + CompareMemoAndDict(memo, data); + } + } + + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + memo.Exists(i).Should().BeTrue(); + memo.TryGetKeccak(i, out var k).Should().BeTrue(); + k.SequenceEqual(data[i].Span).Should().BeTrue(); + } } - Run(raw, RlpMemo.Size, NibbleSet.Readonly.All, OddKey); + CompareMemoAndDict(memo, data); } - [TestCase(0)] - [TestCase(1)] - [TestCase(3)] - [TestCase(11)] - [TestCase(NibbleSet.NibbleCount - 1)] - public void All_children_set_with_one_zero(int zero) + [Test] + public void Clear_get_operation() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span workingMemory = new byte[RlpMemo.MaxSize]; + NibbleSet.Readonly children = new NibbleSet(0xA, 0xB, 0xC); + var memo = new RlpMemo([]); + + InsertRandomKeccak(ref memo, children, out var data, workingMemory); + + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); - for (var i = 0; i < RlpMemo.Size; i++) + for (byte i = 0; i < NibbleSet.NibbleCount; i++) { - raw[i] = (byte)(i & 0xFF); + if (children[i]) + { + data[i] = Keccak.Zero; + memo.Clear(i); + CompareMemoAndDict(memo, data); + } } - // zero one - raw.Slice(zero * Keccak.Size, Keccak.Size).Clear(); + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); - Run(raw, RlpMemo.Size - Keccak.Size + NibbleSet.MaxByteSize, NibbleSet.Readonly.All, OddKey); + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + memo.Exists(i).Should().BeTrue(); + memo.TryGetKeccak(i, out var k).Should().BeFalse(); + k.IsEmpty.Should().BeTrue(); + } + } + + CompareMemoAndDict(memo, data); } - [TestCase(0, NibbleSet.NibbleCount - 1)] - [TestCase(1, 11)] - public void All_children_set_with_two_zeros(int zero0, int zero1) + [Test] + public void Delete_get_operation() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span workingMemory = new byte[RlpMemo.MaxSize]; + NibbleSet.Readonly children = new NibbleSet(0xA, 0xB, 0xC); + var memo = new RlpMemo([]); + + InsertRandomKeccak(ref memo, children, out var data, workingMemory); - for (var i = 0; i < RlpMemo.Size; i++) + memo.Length.Should().Be(GetExpectedSize(children.SetCount)); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + data.Remove(i); + memo = RlpMemo.Delete(memo, i, workingMemory); + CompareMemoAndDict(memo, data); + } + } + + memo.Length.Should().Be(0); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + memo.Exists(i).Should().BeFalse(); + memo.TryGetKeccak(i, out var k).Should().BeFalse(); + k.IsEmpty.Should().BeTrue(); + } + } + + CompareMemoAndDict(memo, data); + } + + [Test] + public void Random_delete() + { + Span raw = stackalloc byte[RlpMemo.MaxSize]; + var children = new NibbleSet(); + + for (var i = 0; i < RlpMemo.MaxSize - NibbleSet.MaxByteSize; i++) { raw[i] = (byte)(i & 0xFF); } - // clear zeroes + // Set all the index bits at the end. + for (var i = RlpMemo.MaxSize - 1; i >= RlpMemo.MaxSize - NibbleSet.MaxByteSize; i--) + { + raw[i] = 0xFF; + } + + for (var i = 0; i < NibbleSet.NibbleCount; i++) + { + children[(byte)i] = true; + } + var memo = new RlpMemo(raw); - memo.Clear((byte)zero0); - memo.Clear((byte)zero1); + var rand = new Random(13); + + for (var i = 0; i < NibbleSet.NibbleCount; i++) + { + var child = (byte)rand.Next(NibbleSet.NibbleCount); + + while (children[child] == false) + { + child = (byte)rand.Next(NibbleSet.NibbleCount); + } + + children[child] = false; + memo = RlpMemo.Delete(memo, child, raw); + + memo.Length.Should().Be(GetExpectedSize(NibbleSet.NibbleCount - i - 1)); + memo.Exists(child).Should().BeFalse(); + memo.TryGetKeccak(child, out var keccak).Should().BeFalse(); + keccak.IsEmpty.Should().BeTrue(); + } + + memo.Length.Should().Be(0); + } + + [Test] + public void Random_insert() + { + Span raw = []; + Span workingMemory = new byte[RlpMemo.MaxSize]; + var children = new NibbleSet(); + + Span keccak = new byte[Keccak.Size]; + keccak.Fill(0xFF); + + for (var i = 0; i < NibbleSet.NibbleCount; i++) + { + children[(byte)i] = false; + } + + var memo = new RlpMemo(raw); + var rand = new Random(13); + + for (var i = 0; i < NibbleSet.NibbleCount; i++) + { + var child = (byte)rand.Next(NibbleSet.NibbleCount); + + while (children[child]) + { + child = (byte)rand.Next(NibbleSet.NibbleCount); + } + + children[child] = true; + memo = RlpMemo.Insert(memo, child, keccak, workingMemory); + + memo.Length.Should().Be(GetExpectedSize(i + 1)); + memo.Exists(child).Should().BeTrue(); + memo.TryGetKeccak(child, out var k).Should().BeTrue(); + k.SequenceEqual(keccak).Should().BeTrue(); + } - Run(raw, RlpMemo.Size - 2 * Keccak.Size + NibbleSet.MaxByteSize, NibbleSet.Readonly.All, OddKey); + memo.Length.Should().Be(RlpMemo.MaxSize); } - [TestCase(0, NibbleSet.NibbleCount - 1)] - [TestCase(1, 11)] - public void Two_children_set_with_Keccak(int child0, int child1) + [Test] + public void Grow_shrink() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span raw = new byte[RlpMemo.MaxSize]; + var memo = new RlpMemo([]); + var random = new Random(13); + var data = new Dictionary(); - // fill set - raw.Slice(child0 * Keccak.Size, Keccak.Size).Fill(1); - raw.Slice(child1 * Keccak.Size, Keccak.Size).Fill(17); + // Grow the RLPMemo + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + data[i] = random.NextKeccak(); + memo = RlpMemo.Insert(memo, i, data[i].Span, raw[..GetExpectedSize(i + 1)]); + CompareMemoAndDict(memo, data); + } - var children = new NibbleSet((byte)child0, (byte)child1); + memo.Length.Should().Be(RlpMemo.MaxSize); - // none of the children set is empty, keccaks encoded without the empty map - const int size = 2 * Keccak.Size; - Run(raw, size, children, OddKey); + // Shrink the RLPMemo + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + memo = RlpMemo.Delete(memo, i, raw[..GetExpectedSize(NibbleSet.NibbleCount - i - 1)]); + data.Remove(i); + CompareMemoAndDict(memo, data); + } + + memo.Length.Should().Be(0); } - [TestCase(1, 11)] - public void Two_children_on_even_level_have_no_memo(int child0, int child1) + [Test] + public void Shrink_grow() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span raw = stackalloc byte[RlpMemo.MaxSize]; + var random = new Random(13); + var data = new Dictionary(); - // fill set - raw.Slice(child0 * Keccak.Size, Keccak.Size).Fill(1); - raw.Slice(child1 * Keccak.Size, Keccak.Size).Fill(17); + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + data[i] = random.NextKeccak(); + data[i].Span.CopyTo(raw[(i * Keccak.Size)..]); + } + + var memo = new RlpMemo(raw); + + // Shrink the RLPMemo + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + memo = RlpMemo.Delete(memo, i, raw[..GetExpectedSize(NibbleSet.NibbleCount - i - 1)]); + data.Remove(i); + CompareMemoAndDict(memo, data); + } - var children = new NibbleSet((byte)child0, (byte)child1); + memo.Length.Should().Be(0); + + // Grow the RLPMemo + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + data[i] = random.NextKeccak(); + memo = RlpMemo.Insert(memo, i, data[i].Span, raw[..GetExpectedSize(i + 1)]); + CompareMemoAndDict(memo, data); + } - Run(raw, 0, children, EvenKey); + memo.Length.Should().Be(RlpMemo.MaxSize); } - [TestCase(0, NibbleSet.NibbleCount - 1)] - [TestCase(1, 11)] - public void Two_children_set_with_no_Keccak(int child0, int child1) + [Test] + public void Copy_data() { - Span raw = stackalloc byte[RlpMemo.Size]; + Span raw = stackalloc byte[RlpMemo.MaxSize]; + Span rawCopy = stackalloc byte[RlpMemo.MaxSize]; - var children = new NibbleSet((byte)child0, (byte)child1); + for (var i = 0; i < RlpMemo.MaxSize; i++) + { + raw[i] = (byte)(i & 0xFF); + } - // just empty map encoded - Run(raw, 0, children, OddKey); + var memo = new RlpMemo(raw); + memo = RlpMemo.Copy(memo.Raw, rawCopy); + memo.Raw.SequenceEqual(raw).Should().BeTrue(); } - private static void Run(Span memoRaw, int compressedSize, NibbleSet.Readonly children, bool oddKey) + [Test] + public void Keccak_to_rlp_children() { - var isOdd = oddKey == OddKey; + NibbleSet.Readonly children = new NibbleSet(1, 2); + Span workingMemory = new byte[RlpMemo.MaxSize]; + + // Create memo with random keccak for the corresponding children + var memo = new RlpMemo([]); + InsertRandomKeccak(ref memo, children, out _, workingMemory); + + // create E->B->L + // ->L + // leaves without any key and very small value cause to be inlined in branch + // encoded branch rlp is also < 32 bytes which causes it to be encoded as RLP in extension node + const string prefix = "ccccccccccccccccccccddddddddddddddddeeeeeeeeeeeeeeeeeeeeeeeeeeb"; + Keccak storageKey1 = + new Keccak(Convert.FromHexString(prefix + "1")); + Keccak storageKey2 = + new Keccak(Convert.FromHexString(prefix + "2")); - var key = Key.Merkle(isOdd ? NibblePath.Single(1, 0) : NibblePath.Empty); - var memo = new RlpMemo(memoRaw); + var commit = new Commit(); + commit.Set(Key.Account(Values.Key0), + new Account(0, 1).WriteTo(stackalloc byte[Paprika.Account.MaxByteCount])); + commit.Set(Key.StorageCell(NibblePath.FromKey(Values.Key0), storageKey1), new byte[] { 1, 2, 3 }); + commit.Set(Key.StorageCell(NibblePath.FromKey(Values.Key0), storageKey2), new byte[] { 10, 20, 30 }); - Span writeTo = stackalloc byte[compressedSize]; - var written = RlpMemo.Compress(key, memo.Raw, children, writeTo); - written.Should().Be(compressedSize); + using var merkle = new ComputeMerkleBehavior(); - if (!isOdd && children.SetCount == 2) + merkle.BeforeCommit(commit, CacheBudget.Options.None.Build()); + + // Update the branch with memo + commit.SetBranch(Key.Raw(NibblePath.FromKey(Values.Key0), DataType.Merkle, NibblePath.Parse(prefix)), children, + memo.Raw); + + merkle.RecalculateStorageTrie(commit, Values.Key0, CacheBudget.Options.None.Build()); + } + + [TestCase(1000)] + [TestCase(10_000)] + [TestCase(100_000)] + public void Large_random_operations(int numOperations) + { + Span workingSet = new byte[RlpMemo.MaxSize]; + var rand = new Random(13); + var memo = new RlpMemo([]); + NibbleSet children = new NibbleSet(); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) { - // cleanup - var decompressed = RlpMemo.Decompress(writeTo, children, stackalloc byte[RlpMemo.Size]); - decompressed.Raw.SequenceEqual(RlpMemo.Empty).Should().BeTrue(); + children[i] = true; } - else + + // Start with full RLPMemo. + InsertRandomKeccak(ref memo, children, out var data, workingSet); + + for (var i = 0; i < numOperations; i++) { - var decompressed = RlpMemo.Decompress(writeTo, children, stackalloc byte[RlpMemo.Size]); - decompressed.Raw.SequenceEqual(memoRaw).Should().BeTrue(); + var child = (byte)rand.Next(NibbleSet.NibbleCount); + var op = (RlpMemoOperation)rand.Next(Enum.GetValues().Length); + + switch (op) + { + case RlpMemoOperation.Set: + if (memo.Exists(child)) + { + data[child] = rand.NextKeccak(); + memo.Set(data[child].Span, child); + + memo.TryGetKeccak(child, out var k).Should().BeTrue(); + k.SequenceEqual(data[child].Span).Should().BeTrue(); + } + + break; + case RlpMemoOperation.Clear: + if (memo.Exists(child)) + { + data[child] = Keccak.Zero; + memo.Clear(child); + + memo.TryGetKeccak(child, out var k).Should().BeFalse(); + k.IsEmpty.Should().BeTrue(); + } + + break; + case RlpMemoOperation.Delete: + if (memo.Exists(child)) + { + children[child] = false; + data.Remove(child); + memo = RlpMemo.Delete(memo, child, workingSet); + + memo.TryGetKeccak(child, out var k).Should().BeFalse(); + k.IsEmpty.Should().BeTrue(); + } + + break; + case RlpMemoOperation.Insert: + if (!memo.Exists(child)) + { + children[child] = true; + data[child] = rand.NextKeccak(); + memo = RlpMemo.Insert(memo, child, data[child].Span, workingSet); + + memo.TryGetKeccak(child, out var k).Should().BeTrue(); + k.SequenceEqual(data[child].Span).Should().BeTrue(); + } + break; + } + + CompareMemoAndDict(memo, data); } } -} \ No newline at end of file + + private static void InsertRandomKeccak(ref RlpMemo memo, NibbleSet.Readonly children, out Dictionary data + , Span workingMemory) + { + data = new Dictionary(); + var random = new Random(13); + + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (children[i]) + { + data[i] = random.NextKeccak(); + memo = RlpMemo.Insert(memo, i, data[i].Span, workingMemory); + } + } + } + + private static void CompareMemoAndDict(RlpMemo memo, Dictionary data) + { + // All the elements in dictionary should be in the memo. + foreach (var child in data) + { + memo.Exists(child.Key).Should().BeTrue(); + + if (child.Value == Keccak.Zero) + { + memo.TryGetKeccak(child.Key, out var k).Should().BeFalse(); + k.IsEmpty.Should().BeTrue(); + } + else + { + memo.TryGetKeccak(child.Key, out var k).Should().BeTrue(); + k.SequenceEqual(child.Value.Span).Should().BeTrue(); + } + } + + // All the elements in the memo should be in the dictionary. + for (byte i = 0; i < NibbleSet.NibbleCount; i++) + { + if (memo.Exists(i)) + { + data.ContainsKey(i).Should().BeTrue(); + + if (memo.TryGetKeccak(i, out var k)) + { + k.SequenceEqual(data[i].Span).Should().BeTrue(); + } + else + { + k.IsEmpty.Should().BeTrue(); + Keccak.Zero.Span.SequenceEqual(data[i].Span).Should().BeTrue(); + } + } + } + + memo.Length.Should().Be(GetExpectedSize(data.Count)); + } + + private static int GetExpectedSize(int numElements) + { + var size = numElements * Keccak.Size; + + // Empty and full memo doesn't contain the index. + if (size != 0 && size != RlpMemo.MaxSize) + { + size += NibbleSet.MaxByteSize; + } + + return size; + } +} diff --git a/src/Paprika/Chain/Blockchain.cs b/src/Paprika/Chain/Blockchain.cs index 0e9788a2..87a7c858 100644 --- a/src/Paprika/Chain/Blockchain.cs +++ b/src/Paprika/Chain/Blockchain.cs @@ -1499,7 +1499,7 @@ protected override void CleanUp() public void ProcessProofNodes(Keccak accountKeccak, Span packedProofPaths, int proofCount) { Span workingSpan = stackalloc byte[NibblePath.MaxLengthValue * 2 + 1]; - Span rlpMemoization = stackalloc byte[RlpMemo.Size]; + Span rlpMemoization = stackalloc byte[RlpMemo.MaxSize]; Span targetKeySpan = stackalloc byte[NibblePath.FullKeccakByteLength * 2 + 1]; Span nodeWorkingSpan = stackalloc byte[Node.Extension.MaxByteLength]; @@ -1536,7 +1536,7 @@ public void ProcessProofNodes(Keccak accountKeccak, Span packedProofPaths, bool allChildrenPersisted = true; - RlpMemo memo = RlpMemo.Decompress(leftover, branch.Children, rlpMemoization); + var memo = RlpMemo.Copy(leftover, rlpMemoization); for (byte i = 0; i < NibbleSet.NibbleCount; i++) { @@ -2007,15 +2007,25 @@ public void CreateMerkleBranch(in Keccak account, in NibblePath storagePath, byt Key key = account == Keccak.Zero ? Key.Merkle(storagePath) : Key.Raw(NibblePath.FromKey(account), DataType.Merkle, storagePath); NibbleSet set = new NibbleSet(); - Span rlpMemoization = stackalloc byte[RlpMemo.Size]; + Span rlpMemoization = stackalloc byte[RlpMemo.MaxSize]; RlpMemo memo = new RlpMemo(rlpMemoization); for (int i = 0; i < childNibbles.Length; i++) { set[childNibbles[i]] = true; if (childHashes[i] != Keccak.Zero) - memo.Set(childHashes[i], childNibbles[i]); + { + if (memo.Exists(childNibbles[i])) + { + memo.Set(childHashes[i].Span, childNibbles[i]); + } + else + { + memo = RlpMemo.Insert(memo, childNibbles[i], childHashes[i].Span, rlpMemoization); + } + } } + _current.SetBranch(key, set, memo.Raw, persist ? EntryType.Persistent : EntryType.Proof); } diff --git a/src/Paprika/Merkle/ComputeMerkleBehavior.cs b/src/Paprika/Merkle/ComputeMerkleBehavior.cs index 82598a4e..b4f6fc7d 100644 --- a/src/Paprika/Merkle/ComputeMerkleBehavior.cs +++ b/src/Paprika/Merkle/ComputeMerkleBehavior.cs @@ -120,11 +120,11 @@ public Keccak GetHash(scoped in NibblePath path, IReadOnlyWorldState commit, boo using var merkleData = commit.Get(parentKey); if (!merkleData.Span.IsEmpty) { - var leftover = Node.ReadFrom(out var parenType, out var leaf, out _, out var branch, merkleData.Span); - if (parenType == Node.Type.Branch && !ignoreCache) + var leftover = Node.ReadFrom(out var parentType, out var leaf, out _, out var branch, merkleData.Span); + if (parentType == Node.Type.Branch && !ignoreCache) { - Span rlpMemoization = stackalloc byte[RlpMemo.Size]; - RlpMemo memo = RlpMemo.Decompress(leftover, branch.Children, rlpMemoization); + Span rlpMemoization = stackalloc byte[leftover.Length]; + var memo = RlpMemo.Copy(leftover, rlpMemoization); if (memo.TryGetKeccak(path[NibblePath.KeccakNibbleCount - 1], out var keccakSpan)) return new Keccak(keccakSpan); } @@ -158,11 +158,11 @@ public Keccak GetStorageHash(IReadOnlyWorldState commit, in Keccak account, Nibb using var merkleData = prefixed.Get(parentKey); if (!merkleData.Span.IsEmpty) { - var leftover = Node.ReadFrom(out var parenType, out var leaf, out _, out var branch, merkleData.Span); - if (parenType == Node.Type.Branch && !ignoreCache) + var leftover = Node.ReadFrom(out var parentType, out var leaf, out _, out var branch, merkleData.Span); + if (parentType == Node.Type.Branch && !ignoreCache) { - Span rlpMemoization = stackalloc byte[RlpMemo.Size]; - RlpMemo memo = RlpMemo.Decompress(leftover, branch.Children, rlpMemoization); + Span rlpMemoization = stackalloc byte[leftover.Length]; + var memo = RlpMemo.Copy(leftover, rlpMemoization); if (memo.TryGetKeccak(storagePath[NibblePath.KeccakNibbleCount - 1], out var keccakSpan)) return new Keccak(keccakSpan); } @@ -379,11 +379,10 @@ private BuildStorageTriesItem[] GetStorageWorkItems(ICommitWithStats commit, Cac public ReadOnlySpan InspectBeforeApply(in Key key, ReadOnlySpan data, Span workingSet) { - if (data.IsEmpty) - return data; - - if (key.Type != DataType.Merkle) + if (data.IsEmpty || key.Type != DataType.Merkle) + { return data; + } var node = Node.Header.Peek(data).NodeType; @@ -399,22 +398,17 @@ public ReadOnlySpan InspectBeforeApply(in Key key, ReadOnlySpan data return Node.Branch.GetOnlyBranchData(data); } - var memoizedRlp = Node.Branch.ReadFrom(data, out var branch); - if (memoizedRlp.Length == 0) + Node.Branch.ReadFrom(data, out var branch); + + // Optimization, omitting some of the branches to memoize. + // It omits only these with two children where the cost of the recompute is not big. + // To prevent an attack of spawning multiple levels of such branches, only even are skipped + if (branch.Children.SetCount == 2 && (key.Path.Length + key.StoragePath.Length) % 2 == 0) { - // no RLP of children memoized, return - return data; + return Node.Branch.GetOnlyBranchData(data); } - Debug.Assert(memoizedRlp.Length == RlpMemo.Size); - - // There are RLPs here, compress them - var dataLength = data.Length - RlpMemo.Size; - data[..dataLength].CopyTo(workingSet); - - var compressedLength = RlpMemo.Compress(key, memoizedRlp, branch.Children, workingSet[dataLength..]); - - return workingSet[..(dataLength + compressedLength)]; + return data; } public Keccak RootHash { get; private set; } @@ -590,16 +584,17 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope const int rlpSlice = 1024; var rlp = buffer.Span[..rlpSlice]; - var rlpMemoization = buffer.Span.Slice(rlpSlice, RlpMemo.Size); + var rlpMemoization = buffer.Span.Slice(rlpSlice, RlpMemo.MaxSize); var memoizedUpdated = false; + var memoizedUpdatedUsingBuffer = false; RlpMemo memo = default; if (memoize) { - var childRlpRequiresUpdate = isOwnedByThisCommit == false || previousRlp.Length != RlpMemo.Size; + var childRlpRequiresUpdate = isOwnedByThisCommit == false; memo = childRlpRequiresUpdate - ? RlpMemo.Decompress(previousRlp, branch.Children, rlpMemoization) + ? RlpMemo.Copy(previousRlp, rlpMemoization) : new RlpMemo(MakeRlpWritable(previousRlp)); } @@ -613,7 +608,7 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope if (!runInParallel) { - var childSpan = buffer.Span[(RlpMemo.Size + rlpSlice)..]; + var childSpan = buffer.Span[(RlpMemo.MaxSize + rlpSlice)..]; for (byte i = 0; i < NibbleSet.NibbleCount; i++) { @@ -648,20 +643,31 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope stream.Write(keccakOrRlp.Span); } - if (memoize) + // Memoize all the keccak values. + if (memoize && keccakOrRlp.DataType == KeccakOrRlp.Type.Keccak) { + if (memo.Exists(i)) + { + memo.Set(keccakOrRlp.Span, i); + } + else + { + memo = RlpMemo.Insert(memo, i, keccakOrRlp.Span, rlpMemoization); + memoizedUpdatedUsingBuffer = true; + } + memoizedUpdated = true; - memo.Set(keccakOrRlp, i); } } else { stream.EncodeEmptyArray(); - if (memoize) + if (memoize && memo.Exists(i)) { - memo.Clear(i); + memo = RlpMemo.Delete(memo, i, rlpMemoization); memoizedUpdated = true; + memoizedUpdatedUsingBuffer = true; } } } @@ -729,15 +735,24 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope stream.Encode(value); if (memoize) { + memoizedUpdated = true; + if (value.Length == Keccak.Size) { - memoizedUpdated = true; - memo.SetRaw(value, i); + if (memo.Exists(i)) + { + memo.Set(value, i); + } + else + { + memo = RlpMemo.Insert(memo, i, value, rlpMemoization); + memoizedUpdatedUsingBuffer = true; + } } - else + else if (memo.Exists(i)) { - memoizedUpdated = true; - memo.Clear(i); + memo = RlpMemo.Delete(memo, i, rlpMemoization); + memoizedUpdatedUsingBuffer = true; } } } @@ -756,10 +771,10 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope stream.Position = from; stream.StartSequence(actualLength); - if (memoize && !isOwnedByThisCommit && memoizedUpdated) + if (memoize && ((!isOwnedByThisCommit && memoizedUpdated) || memoizedUpdatedUsingBuffer)) { - // - ctx.Commit.SetBranch(key, branch.Children, rlpMemoization, EntryType.Persistent); + // Set the branch if the memo has been updated using the new buffer memory. + ctx.Commit.SetBranch(key, branch.Children, memo.Raw, EntryType.Persistent); } KeccakOrRlp.FromSpan(rlp.Slice(from, end - from), out keccakOrRlp); @@ -1040,27 +1055,41 @@ or DeleteStatus.ExtensionToLeaf static void UpdateBranchOnDelete(ICommit commit, in Node.Branch branch, NibbleSet.Readonly children, ReadOnlySpan leftover, ReadOnlySpanOwnerWithMetadata owner, byte nibble, in Key key) { - var childRlpRequiresUpdate = owner.IsOwnedBy(commit) == false || leftover.Length != RlpMemo.Size; + var childRlpRequiresUpdate = owner.IsOwnedBy(commit) == false; RlpMemo memo; byte[]? rlpWorkingSet = null; + var memoizedUpdatedUsingBuffer = false; if (childRlpRequiresUpdate) { // TODO: make it a context and pass through all the layers - rlpWorkingSet = ArrayPool.Shared.Rent(RlpMemo.Size); - memo = RlpMemo.Decompress(leftover, branch.Children, rlpWorkingSet.AsSpan()); + rlpWorkingSet = ArrayPool.Shared.Rent(leftover.Length); + memo = RlpMemo.Copy(leftover, rlpWorkingSet.AsSpan()); } else { memo = new RlpMemo(MakeRlpWritable(leftover)); } - memo.Clear(nibble); + if (memo.Exists(nibble)) + { + // If this child still exists, only clear the memo. Otherwise, delete it from the memo. + if (children[nibble]) + { + memo.Clear(nibble); + } + else + { + rlpWorkingSet ??= ArrayPool.Shared.Rent(leftover.Length - Keccak.Size); + memo = RlpMemo.Delete(memo, nibble, rlpWorkingSet); + memoizedUpdatedUsingBuffer = true; + } + } var shouldUpdate = !branch.Children.Equals(children); // There's the cached RLP - if (shouldUpdate || childRlpRequiresUpdate) + if (shouldUpdate || childRlpRequiresUpdate || memoizedUpdatedUsingBuffer) { commit.SetBranch(key, children, memo.Raw); } @@ -1288,20 +1317,31 @@ private static void MarkPathDirty(in NibblePath path, in Span rlpMemoWorki { var nibble = path[i]; - var childRlpRequiresUpdate = owner.IsOwnedBy(commit) == false || leftover.Length != RlpMemo.Size; + var childRlpRequiresUpdate = owner.IsOwnedBy(commit) == false; var memo = childRlpRequiresUpdate - ? RlpMemo.Decompress(leftover, branch.Children, rlpMemoWorkingSet) + ? RlpMemo.Copy(leftover, rlpMemoWorkingSet) : new RlpMemo(MakeRlpWritable(leftover)); - memo.Clear(nibble); + var memoizedUpdatedUsingBuffer = false; + + // If this nibble exists in the memo, clear it from the memo. Otherwise, insert an empty Keccak. + if (memo.Exists(nibble)) + { + memo.Clear(nibble); + } + else + { + memo = RlpMemo.Insert(memo, nibble, Keccak.Zero.Span, rlpMemoWorkingSet); + memoizedUpdatedUsingBuffer = true; + } createLeaf = !branch.Children[nibble]; var children = branch.Children.Set(nibble); var shouldUpdateBranch = createLeaf; - if (shouldUpdateBranch || childRlpRequiresUpdate) + if (shouldUpdateBranch || childRlpRequiresUpdate || memoizedUpdatedUsingBuffer) { - // Set the branch if either the children has hanged or the RLP requires the update + // Set the branch if either the children has changed or the RLP requires the update commit.SetBranch(key, children, memo.Raw); } @@ -1393,7 +1433,7 @@ public void Accept(IMerkleTrieVisitor visitor, NibblePath path, MerkleVisitorCon context.Level++; Span workingSpan = stackalloc byte[NibblePath.MaxLengthValue * 2 + 1]; - Span rlpMemoization = stackalloc byte[RlpMemo.Size]; + Span rlpMemoization = stackalloc byte[leftover.Length]; for (byte i = 0; i < NibbleSet.NibbleCount; i++) { if (branch.Children[i]) @@ -1403,7 +1443,7 @@ public void Accept(IMerkleTrieVisitor visitor, NibblePath path, MerkleVisitorCon if (childPath.Length == NibblePath.KeccakNibbleCount) { - RlpMemo memo = RlpMemo.Decompress(leftover, branch.Children, rlpMemoization); + var memo = RlpMemo.Copy(leftover, rlpMemoization); KeccakOrRlp childKeccakOrRlp = Keccak.Zero; if (memo.TryGetKeccak(i, out var keccakSpan)) { @@ -1655,23 +1695,7 @@ private static ReadOnlySpan Transform(in ReadOnlySpan data, in Span< // Branch should be always persistent, already copied and ready to work with. type = EntryType.Persistent; - - var leftoverLength = Node.Branch.ReadFrom(data, out var branch).Length; - if (leftoverLength == RlpMemo.Size) - { - // Rlp memo is decompressed, good to be stored as is. - return data; - } - - // RlpMemo not decompressed. - - // Write branch first - var leftover = branch.WriteToWithLeftover(workspace); - - // Decompress to the leftover - RlpMemo.Decompress(leftover, branch.Children, leftover); - - return workspace[..(workspace.Length - leftover.Length + RlpMemo.Size)]; + return data; } [SkipLocalsInit] diff --git a/src/Paprika/Merkle/NibbleSet.cs b/src/Paprika/Merkle/NibbleSet.cs index 86019147..4c0952a1 100644 --- a/src/Paprika/Merkle/NibbleSet.cs +++ b/src/Paprika/Merkle/NibbleSet.cs @@ -55,6 +55,13 @@ public bool this[byte nibble] public int SetCount => BitOperations.PopCount(_value); + public int SetCountBefore(byte nibble) + { + // Compute the number of set bits before `nibble` (including itself). + var leftChildren = (ushort)(_value & ((1U << (nibble + 1)) - 1)); + return BitOperations.PopCount(leftChildren); + } + public byte SmallestNibbleSet => (byte)BitOperations.TrailingZeroCount(_value); public byte BiggestNibbleSet => (byte)(31 - BitOperations.LeadingZeroCount((uint)_value)); @@ -102,10 +109,14 @@ public Readonly(ushort value) public static Readonly AllWithout(byte nibble) => new((ushort)(AllSetValue & ~(1 << nibble))); + public static Readonly None => new(0); + public bool AllSet => _value == AllSetValue; public int SetCount => new NibbleSet(_value).SetCount; + public int SetCountBefore(byte nibble) => new NibbleSet(_value).SetCountBefore(nibble); + public byte SmallestNibbleSet => new NibbleSet(_value).SmallestNibbleSet; public byte SmallestNibbleNotSet => new NibbleSet(_value).SmallestNibbleNotSet; diff --git a/src/Paprika/Merkle/RlpMemo.cs b/src/Paprika/Merkle/RlpMemo.cs index 96a1dc3c..190673d8 100644 --- a/src/Paprika/Merkle/RlpMemo.cs +++ b/src/Paprika/Merkle/RlpMemo.cs @@ -9,52 +9,45 @@ namespace Paprika.Merkle; public readonly ref struct RlpMemo { - public static readonly byte[] Empty = new byte[Size]; + public static readonly byte[] Empty = []; private readonly Span _buffer; - public const int Size = NibbleSet.NibbleCount * Keccak.Size; + public const int MaxSize = NibbleSet.NibbleCount * Keccak.Size; public RlpMemo(Span buffer) { - Debug.Assert(buffer.Length == Size); - _buffer = buffer; } public ReadOnlySpan Raw => _buffer; - public void SetRaw(ReadOnlySpan keccak, byte nibble) + public int Length => _buffer.Length; + + public void Set(ReadOnlySpan keccak, byte nibble) { - Debug.Assert(keccak.Length == Keccak.Size); - keccak.CopyTo(GetAtNibble(nibble)); + var span = GetAtNibble(nibble); + Debug.Assert(!span.IsEmpty, "Attempted to set a value on a non-existent index"); + Debug.Assert(keccak.Length == Keccak.Size && span.Length == Keccak.Size, "Attempted to set incorrect length"); + + keccak.CopyTo(span); } - public void Set(in KeccakOrRlp keccakOrRlp, byte nibble) + public void Clear(byte nibble) { var span = GetAtNibble(nibble); - if (keccakOrRlp.DataType == KeccakOrRlp.Type.Keccak) - { - keccakOrRlp.Span.CopyTo(span); - } - else + if (!span.IsEmpty) { - // on rlp, memoize none span.Clear(); } } - public void Clear(byte nibble) - { - GetAtNibble(nibble).Clear(); - } - public bool TryGetKeccak(byte nibble, out ReadOnlySpan keccak) { var span = GetAtNibble(nibble); - if (span.IndexOfAnyExcept((byte)0) >= 0) + if (!span.IsEmpty && span.IndexOfAnyExcept((byte)0) >= 0) { keccak = span; return true; @@ -64,139 +57,161 @@ public bool TryGetKeccak(byte nibble, out ReadOnlySpan keccak) return false; } - private Span GetAtNibble(byte nibble) => _buffer.Slice(nibble * Keccak.Size, Keccak.Size); - - public static RlpMemo Decompress(scoped in ReadOnlySpan leftover, NibbleSet.Readonly children, - scoped in Span workingSet) + public bool Exists(byte nibble) { - if (_decompressionForbidden) + if (_buffer.Length == 0) { - ThrowDecompressionForbidden(); + return false; } - var span = workingSet[..Size]; + var index = GetIndex(); + + return index[nibble]; + } - if (leftover.IsEmpty) + private Span GetAtNibble(byte nibble) + { + if (_buffer.Length == 0) { - // no RLP cached yet - span.Clear(); - return new RlpMemo(span); + return []; } - if (leftover.Length == Size) + var index = GetIndex(); + + // Check if the element exists + if (!index[nibble]) { - leftover.CopyTo(span); - return new RlpMemo(span); + return []; } - // It's neither empty nor full. It must be the compressed form, prepare setup first - span.Clear(); - var memo = new RlpMemo(span); + var nibbleIndex = index.SetCountBefore(nibble) - 1; + return _buffer.Slice(nibbleIndex * Keccak.Size, Keccak.Size); + } - // Extract empty bits if any - NibbleSet.Readonly empty; + private NibbleSet.Readonly GetIndex() + { + // Extract the index bits. + var indexLength = _buffer.Length % Keccak.Size; + NibbleSet.Readonly index; - // The empty bytes length is anything that is not aligned to the Keccak size - var emptyBytesLength = leftover.Length % Keccak.Size; - if (emptyBytesLength > 0) + if (indexLength != 0) { - var bits = leftover[^emptyBytesLength..]; - NibbleSet.Readonly.ReadFrom(bits, out empty); + Debug.Assert(indexLength == NibbleSet.MaxByteSize, "Unexpected index length"); + var bits = _buffer[^indexLength..]; + NibbleSet.Readonly.ReadFrom(bits, out index); } else { - empty = default; - } - - var at = 0; - for (byte i = 0; i < NibbleSet.NibbleCount; i++) - { - if (children[i] && empty[i] == false) - { - var keccak = leftover.Slice(at * Keccak.Size, Keccak.Size); - at++; + Debug.Assert(_buffer.Length is 0 or MaxSize, "Only empty or full RlpMemo can have no index"); - memo.SetRaw(keccak, i); - } + index = _buffer.IsEmpty ? NibbleSet.Readonly.None : NibbleSet.Readonly.All; } - return memo; + return index; + } - [DoesNotReturn] - [MethodImpl(MethodImplOptions.NoInlining)] - static void ThrowDecompressionForbidden() => throw new InvalidOperationException("Decompression is forbidden."); + public static RlpMemo Copy(ReadOnlySpan from, scoped in Span to) + { + var span = to[..from.Length]; + from.CopyTo(span); + return new RlpMemo(span); } - [SkipLocalsInit] - public static int Compress(in Key key, scoped in ReadOnlySpan memoizedRlp, NibbleSet.Readonly children, scoped in Span writeTo) + public static RlpMemo Insert(RlpMemo memo, byte nibble, ReadOnlySpan keccak, scoped in Span workingSet) { - // Optimization, omitting some of the branches to memoize. - // It omits only these with two children where the cost of the recompute is not big. - // To prevent an attack of spawning multiple levels of such branches, only even are skipped - if (children.SetCount == 2 && (key.Path.Length + key.StoragePath.Length) % 2 == 0) + var index = memo.GetIndex(); + + Debug.Assert(!index[nibble], "Attempted to insert a value into an already existing index"); + + // Update the index and then compute the destination size for copying. + index = index.Set(nibble); + var size = ComputeRlpMemoSize(index); + + Debug.Assert(size is >= Keccak.Size + NibbleSet.MaxByteSize and <= MaxSize, "Unexpected size during insert"); + Debug.Assert(workingSet.Length >= size, "Insufficient destination length for insertion"); + + var span = workingSet[..size]; + + // Compute the index of this nibble in the destination memo + var insertIndex = index.SetCountBefore(nibble) - 1; + var insertOffset = insertIndex * Keccak.Size; + + // Copy all the elements before the new element + if (insertOffset > 0) { - return 0; + memo._buffer[..insertOffset].CopyTo(span); } - var memo = new RlpMemo(ComputeMerkleBehavior.MakeRlpWritable(memoizedRlp)); - var at = 0; + // Copy all the elements after the new element (except the index) + if (memo.Length > insertOffset + NibbleSet.MaxByteSize) + { + memo._buffer[insertOffset..^NibbleSet.MaxByteSize].CopyTo(span[(insertOffset + Keccak.Size)..]); + } - var empty = new NibbleSet(); + keccak.CopyTo(span[insertOffset..]); - for (byte i = 0; i < NibbleSet.NibbleCount; i++) + // Insert the new index only if the destination memo is not full. + if (size != MaxSize) { - if (children[i]) - { - if (memo.TryGetKeccak(i, out var keccak)) - { - var dest = writeTo.Slice(at * Keccak.Size, Keccak.Size); - at++; - keccak.CopyTo(dest); - } - else - { - empty[i] = true; - } - } + index.WriteToWithLeftover(span[^NibbleSet.MaxByteSize..]); } - Debug.Assert(at != 16 || empty.SetCount == 0, "If at = 16, empty should be empty"); + return new RlpMemo(span); + } + + public static RlpMemo Delete(RlpMemo memo, byte nibble, scoped in Span workingSet) + { + var index = memo.GetIndex(); + + Debug.Assert(index[nibble], "Attempted to delete a non-existing index"); + + // Update the index and then compute the destination size for copying. + index = index.Remove(nibble); + var size = ComputeRlpMemoSize(index); + + Debug.Assert(size is >= 0 and < MaxSize, "Unexpected size during deletion"); + Debug.Assert(workingSet.Length >= size, "Insufficient destination length for deletion"); - if (empty.SetCount == children.SetCount) + if (size == 0) { - // None of children has their Keccak memoized. Instead of reporting it, return nothing written. - return 0; + // Return empty RlpMemo + return new RlpMemo(workingSet[..size]); } - if (empty.SetCount > 0) + var span = workingSet[..size]; + + // Compute the index of this nibble in the memo + var deleteIndex = index.SetCountBefore(nibble); + var deleteOffset = deleteIndex * Keccak.Size; + + // Copy all the elements before the deleted element + if (deleteOffset > 0) { - var dest = writeTo.Slice(at * Keccak.Size, NibbleSet.MaxByteSize); - new NibbleSet.Readonly(empty).WriteToWithLeftover(dest); - return at * Keccak.Size + NibbleSet.MaxByteSize; + memo._buffer[..deleteOffset].CopyTo(span); } - // Return only children that were written - return at * Keccak.Size; - } + // Copy all the elements after the deleted element + if (memo.Length > deleteOffset + Keccak.Size + NibbleSet.MaxByteSize) + { + memo._buffer[(deleteOffset + Keccak.Size)..].CopyTo(span[deleteOffset..]); + } - /// - /// Test only method to forbid decompression. - /// - /// - public static NoDecompressionScope NoDecompression() => new(); + // Since the destination memo is neither empty nor full here, it must always contain the index. + index.WriteToWithLeftover(span[^NibbleSet.MaxByteSize..]); - private static volatile bool _decompressionForbidden; + return new RlpMemo(span); + } - public readonly struct NoDecompressionScope : IDisposable + private static int ComputeRlpMemoSize(NibbleSet.Readonly index) { - public NoDecompressionScope() - { - _decompressionForbidden = true; - } + var size = index.SetCount * Keccak.Size; - public void Dispose() + // Add extra space for the index. Empty and full memo doesn't contain the index. + if (size != 0 && size != MaxSize) { - _decompressionForbidden = false; + size += NibbleSet.MaxByteSize; } + + return size; } }