Skip to content

Commit 688e162

Browse files
BobrisBobris
committed
Cursor FastIterate
1 parent 4497a52 commit 688e162

13 files changed

+211
-0
lines changed

‎BTDB/BTreeLib/BTreeImpl12.cs

+82
Original file line numberDiff line numberDiff line change
@@ -2585,4 +2585,86 @@ internal static void GatherUsedFiles(IntPtr nodePtr, CancellationToken cancellat
25852585
}
25862586
}
25872587
}
2588+
2589+
internal static void FastIterate(int deepness, IntPtr top, ref StructList<CursorItem> stack, ref Span<byte> buffer,
2590+
ref long keyIndex, CursorIterateCallback callback)
2591+
{
2592+
if (top == IntPtr.Zero)
2593+
return;
2594+
if (deepness == stack.Count)
2595+
{
2596+
stack.AddRef().Set(top, 0);
2597+
}
2598+
2599+
ref var header = ref NodeUtils12.Ptr2NodeHeader(top);
2600+
if (header.IsNodeLeaf)
2601+
{
2602+
var prefixSpan = NodeUtils12.GetPrefixSpan(top);
2603+
if (prefixSpan.Length > buffer.Length)
2604+
{
2605+
buffer = GC.AllocateUninitializedArray<byte>(NewSize(prefixSpan.Length, buffer.Length));
2606+
}
2607+
2608+
prefixSpan.CopyTo(buffer);
2609+
if (header.HasLongKeys)
2610+
{
2611+
var longKeys = NodeUtils12.GetLongKeyPtrs(top);
2612+
for (var i = (int)stack.Last._posInNode; i < longKeys.Length; i++, stack.Last._posInNode++)
2613+
{
2614+
var key = NodeUtils12.LongKeyPtrToSpan(longKeys[i]);
2615+
if (key.Length + prefixSpan.Length > buffer.Length)
2616+
{
2617+
buffer = GC.AllocateUninitializedArray<byte>(NewSize(key.Length + prefixSpan.Length,
2618+
buffer.Length));
2619+
prefixSpan.CopyTo(buffer);
2620+
}
2621+
2622+
key.CopyTo(buffer[prefixSpan.Length..]);
2623+
callback.Invoke(keyIndex, buffer[..(prefixSpan.Length + key.Length)]);
2624+
keyIndex++;
2625+
}
2626+
}
2627+
else
2628+
{
2629+
var keyOfs = NodeUtils12.GetKeySpans(top, out var keyData);
2630+
for (var i = (int)stack.Last._posInNode; i < keyOfs.Length - 1; i++, stack.Last._posInNode++)
2631+
{
2632+
var key = keyData.Slice(keyOfs[i], keyOfs[i + 1] - keyOfs[i]);
2633+
if (key.Length + prefixSpan.Length > buffer.Length)
2634+
{
2635+
buffer = GC.AllocateUninitializedArray<byte>(NewSize(key.Length + prefixSpan.Length,
2636+
buffer.Length));
2637+
prefixSpan.CopyTo(buffer);
2638+
}
2639+
2640+
key.CopyTo(buffer[prefixSpan.Length..]);
2641+
callback.Invoke(keyIndex, buffer[..(prefixSpan.Length + key.Length)]);
2642+
keyIndex++;
2643+
}
2644+
}
2645+
}
2646+
else
2647+
{
2648+
var children = NodeUtils12.GetBranchValuePtrs(top);
2649+
for (var i = (int)stack[deepness]._posInNode; i < children.Length; i++, stack[deepness]._posInNode++)
2650+
{
2651+
FastIterate(deepness + 1, children[i], ref stack, ref buffer, ref keyIndex, callback);
2652+
}
2653+
}
2654+
2655+
stack.Pop();
2656+
}
2657+
2658+
static int NewSize(int size, int existingSize)
2659+
{
2660+
if (existingSize * 2L > Array.MaxLength)
2661+
{
2662+
if (size > Array.MaxLength)
2663+
throw new ArgumentOutOfRangeException();
2664+
return Array.MaxLength;
2665+
}
2666+
2667+
size = Math.Max(size, existingSize * 2);
2668+
return size;
2669+
}
25882670
}

‎BTDB/BTreeLib/Cursor12.cs

+5
Original file line numberDiff line numberDiff line change
@@ -424,4 +424,9 @@ public void TestTreeCorrectness()
424424
{
425425
BTreeImpl12.TestTreeCorrectness(_rootNode.Root, ref _stack);
426426
}
427+
428+
public void FastIterate(ref Span<byte> buffer, ref long keyIndex, CursorIterateCallback callback)
429+
{
430+
BTreeImpl12.FastIterate(0, _rootNode.Root, ref _stack, ref buffer, ref keyIndex, callback);
431+
}
427432
}

‎BTDB/BTreeLib/ICursor.cs

+1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ byte[] GetKeyAsArray()
4545
}
4646

4747
void TestTreeCorrectness();
48+
void FastIterate(ref Span<byte> buffer, ref long keyIndex, CursorIterateCallback callback);
4849
}
4950

5051
public delegate void BuildTreeCallback(ref MemReader reader, ref ByteBuffer key, in Span<byte> value);

‎BTDB/KVDBLayer/BTreeMem/BTreeBranch.cs

+16
Original file line numberDiff line numberDiff line change
@@ -490,4 +490,20 @@ public void CalcBTreeStats(RefDictionary<(uint Depth, uint Children), uint> stat
490490
child.CalcBTreeStats(stats, depth + 1);
491491
}
492492
}
493+
494+
public void FastIterate(int deepness, ref StructList<NodeIdxPair> stack, ref long keyIndex,
495+
CursorIterateCallback callback)
496+
{
497+
if (deepness == stack.Count)
498+
{
499+
stack.Add(new() { Node = this, Idx = 0 });
500+
}
501+
502+
for (var i = stack[deepness].Idx; i < _children.Length; i++, stack[deepness].Idx++)
503+
{
504+
_children[i].FastIterate(deepness + 1, ref stack, ref keyIndex, callback);
505+
}
506+
507+
stack.Pop();
508+
}
493509
}

‎BTDB/KVDBLayer/BTreeMem/BTreeLeaf.cs

+17
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,23 @@ public void CalcBTreeStats(RefDictionary<(uint Depth, uint Children), uint> stat
288288
stats.GetOrAddValueRef((depth, (uint)_keyValues.Length))++;
289289
}
290290

291+
public void FastIterate(int deepness, ref StructList<NodeIdxPair> stack, ref long keyIndex,
292+
CursorIterateCallback callback)
293+
{
294+
if (deepness == stack.Count)
295+
{
296+
stack.Add(new() { Node = this, Idx = 0 });
297+
}
298+
299+
for (var i = stack.Last.Idx; i < _keyValues.Length; i++, stack.Last.Idx++)
300+
{
301+
callback.Invoke(keyIndex, _keyValues[i].Key);
302+
keyIndex++;
303+
}
304+
305+
stack.Pop();
306+
}
307+
291308
public ReadOnlyMemory<byte> GetKey(int idx)
292309
{
293310
return _keyValues[idx].Key;

‎BTDB/KVDBLayer/BTreeMem/BTreeLeafComp.cs

+17
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,23 @@ public void CalcBTreeStats(RefDictionary<(uint Depth, uint Children), uint> stat
430430
stats.GetOrAddValueRef((depth, (uint)_keyValues.Length))++;
431431
}
432432

433+
public void FastIterate(int deepness, ref StructList<NodeIdxPair> stack, ref long keyIndex,
434+
CursorIterateCallback callback)
435+
{
436+
if (deepness == stack.Count)
437+
{
438+
stack.Add(new() { Node = this, Idx = 0 });
439+
}
440+
441+
for (var i = stack.Last.Idx; i < _keyValues.Length; i++, stack.Last.Idx++)
442+
{
443+
callback.Invoke(keyIndex, _keyBytes.AsSpan(_keyValues[i].KeyOffset, _keyValues[i].KeyLength));
444+
keyIndex++;
445+
}
446+
447+
stack.Pop();
448+
}
449+
433450
static void RecalculateOffsets(Member[] keyValues)
434451
{
435452
ushort ofs = 0;

‎BTDB/KVDBLayer/BTreeMem/BTreeRoot.cs

+13
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,12 @@ public void CalcBTreeStats(RefDictionary<(uint Depth, uint Children), uint> stat
201201
_rootNode?.CalcBTreeStats(stats, depth);
202202
}
203203

204+
public void FastIterate(int deepness, ref StructList<NodeIdxPair> stack, ref long keyIndex,
205+
CursorIterateCallback callback)
206+
{
207+
throw new ArgumentException();
208+
}
209+
204210
public long TransactionId => _transactionId;
205211
public ulong CommitUlong { get; set; }
206212

@@ -292,6 +298,13 @@ public void BuildTree(long keyCount, Func<BTreeLeafMember> memberGenerator)
292298
_rootNode = BuildTreeNode(keyCount, memberGenerator);
293299
}
294300

301+
public void FastIterate(ref StructList<NodeIdxPair> stack, ref long keyIndex, ref Span<byte> buffer,
302+
CursorIterateCallback callback)
303+
{
304+
if (_rootNode == null) return;
305+
_rootNode.FastIterate(0, ref stack, ref keyIndex, callback);
306+
}
307+
295308
IBTreeNode BuildTreeNode(long keyCount, Func<BTreeLeafMember> memberGenerator)
296309
{
297310
var leafCount = (keyCount + BTreeLeafComp.MaxMembers - 1) / BTreeLeafComp.MaxMembers;

‎BTDB/KVDBLayer/BTreeMem/IBTreeNode.cs

+3
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,7 @@ interface IBTreeNode
2020
IBTreeNode EraseRange(long transactionId, long firstKeyIndex, long lastKeyIndex);
2121
IBTreeNode EraseOne(long transactionId, long keyIndex);
2222
void CalcBTreeStats(RefDictionary<(uint Depth, uint Children), uint> stats, uint depth);
23+
24+
void FastIterate(int deepness, ref StructList<NodeIdxPair> stack, ref long keyIndex,
25+
CursorIterateCallback callback);
2326
}

‎BTDB/KVDBLayer/BTreeMem/IBTreeRootNode.cs

+3
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,7 @@ interface IBTreeRootNode : IBTreeNode
1818
bool FindNextKey(ref StructList<NodeIdxPair> stack);
1919
bool FindPreviousKey(ref StructList<NodeIdxPair> stack);
2020
void BuildTree(long keyCount, Func<BTreeLeafMember> memberGenerator);
21+
22+
void FastIterate(ref StructList<NodeIdxPair> stack, ref long keyIndex, ref Span<byte> buffer,
23+
CursorIterateCallback callback);
2124
}

‎BTDB/KVDBLayer/Implementation/BTreeKeyValueDBCursor.cs

+20
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class BTreeKeyValueDBCursor : IKeyValueDBCursorInternal
1313
ICursor? _cursor;
1414
bool _modifiedFromLastFind;
1515
bool _removedCurrent;
16+
bool _modificationForbidden;
1617
long _keyIndex;
1718

1819
public static BTreeKeyValueDBCursor Create(BTreeKeyValueDBTransaction transaction)
@@ -555,8 +556,25 @@ public UpdateKeySuffixResult UpdateKeySuffix(in ReadOnlySpan<byte> key, uint pre
555556
return UpdateKeySuffixResult.Updated;
556557
}
557558

559+
public void FastIterate(ref Span<byte> buffer, CursorIterateCallback callback)
560+
{
561+
ObjectDisposedException.ThrowIf(_transaction == null, this);
562+
ObjectDisposedException.ThrowIf(_transaction.BTreeRoot == null, _transaction);
563+
_modificationForbidden = true;
564+
try
565+
{
566+
if (_keyIndex == -1) _keyIndex = _cursor!.CalcIndex();
567+
_cursor!.FastIterate(ref buffer, ref _keyIndex, callback);
568+
}
569+
finally
570+
{
571+
_modificationForbidden = false;
572+
}
573+
}
574+
558575
public void NotifyRemove(ulong startIndex, ulong endIndex)
559576
{
577+
if (_modificationForbidden) throw new BTDBException("DB modification during FastIterate is forbidden");
560578
if (_modifiedFromLastFind)
561579
{
562580
if (_keyIndex == -1) return;
@@ -586,6 +604,7 @@ public void NotifyRemove(ulong startIndex, ulong endIndex)
586604

587605
public void PreNotifyUpsert()
588606
{
607+
if (_modificationForbidden) throw new BTDBException("DB modification during FastIterate is forbidden");
589608
if (_keyIndex == -1)
590609
{
591610
if (!_cursor!.IsValid()) return;
@@ -617,6 +636,7 @@ public void NotifyInsert(ulong index)
617636

618637
public void NotifyWritableTransaction()
619638
{
639+
if (_modificationForbidden) throw new BTDBException("DB modification during FastIterate is forbidden");
620640
_cursor!.SetNewRoot(_transaction!.BTreeRoot!);
621641
}
622642
}

‎BTDB/KVDBLayer/Interface/IKeyValueDBCursor.cs

+10
Original file line numberDiff line numberDiff line change
@@ -155,4 +155,14 @@ public interface IKeyValueDBCursor : IDisposable
155155
/// Else it will update key and return Updated.
156156
/// </returns>
157157
UpdateKeySuffixResult UpdateKeySuffix(in ReadOnlySpan<byte> key, uint prefixLen);
158+
159+
/// <summary>
160+
/// From current position it will repeatedly call callback for each key in ascending order.
161+
/// It is forbidden to modify database in callback. It is also not allowed to call any Cursor method on this cursor.
162+
/// </summary>
163+
/// <param name="buffer">It will be used and enlarged to fit keys</param>
164+
/// <param name="callback">Iteration will stop if callback returns true</param>
165+
void FastIterate(ref Span<byte> buffer, CursorIterateCallback callback);
158166
}
167+
168+
public delegate bool CursorIterateCallback(long keyIndex, ReadOnlySpan<byte> key);

‎BTDB/KVDBLayer/MemoryImplementation/InMemoryKeyValueDBCursor.cs

+19
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ class InMemoryKeyValueDBCursor : IKeyValueDBCursorInternal
1313
bool _modifiedFromLastFind;
1414
bool _removedCurrent;
1515
long _keyIndex;
16+
bool _modificationForbiden;
1617

1718
public static InMemoryKeyValueDBCursor Create(InMemoryKeyValueDBTransaction transaction)
1819
{
@@ -508,8 +509,24 @@ public UpdateKeySuffixResult UpdateKeySuffix(in ReadOnlySpan<byte> key, uint pre
508509
return ctx.Result;
509510
}
510511

512+
public void FastIterate(ref Span<byte> buffer, CursorIterateCallback callback)
513+
{
514+
ObjectDisposedException.ThrowIf(_transaction == null, this);
515+
ObjectDisposedException.ThrowIf(_transaction._btreeRoot == null, _transaction);
516+
_modificationForbiden = true;
517+
try
518+
{
519+
_transaction._btreeRoot!.FastIterate(ref _stack, ref _keyIndex, ref buffer, callback);
520+
}
521+
finally
522+
{
523+
_modificationForbiden = false;
524+
}
525+
}
526+
511527
public void NotifyRemove(ulong startIndex, ulong endIndex)
512528
{
529+
if (_modificationForbiden) throw new BTDBException("DB cannot be modified during fast iteration");
513530
if (_modifiedFromLastFind)
514531
{
515532
if (_keyIndex == -1) return;
@@ -534,6 +551,7 @@ public void NotifyRemove(ulong startIndex, ulong endIndex)
534551

535552
public void PreNotifyUpsert()
536553
{
554+
if (_modificationForbiden) throw new BTDBException("DB cannot be modified during fast iteration");
537555
_stack.Clear();
538556
if (!_modifiedFromLastFind)
539557
{
@@ -560,6 +578,7 @@ public void NotifyInsert(ulong index)
560578

561579
public void NotifyWritableTransaction()
562580
{
581+
if (_modificationForbiden) throw new BTDBException("DB cannot be modified during fast iteration");
563582
_stack.Clear();
564583
}
565584
}

‎BTDBTest/ObjectDbTableRemoveOptimalizeTest.cs

+5
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,11 @@ public UpdateKeySuffixResult UpdateKeySuffix(in ReadOnlySpan<byte> key, uint pre
453453
return _keyValueDBCursorInternalImplementation.UpdateKeySuffix(in key, prefixLen);
454454
}
455455

456+
public void FastIterate(ref Span<byte> buffer, CursorIterateCallback callback)
457+
{
458+
_keyValueDBCursorInternalImplementation.FastIterate(ref buffer, callback);
459+
}
460+
456461
public void NotifyRemove(ulong startIndex, ulong endIndex)
457462
{
458463
_keyValueDBCursorInternalImplementation.NotifyRemove(startIndex, endIndex);

0 commit comments

Comments
 (0)