Skip to content

Commit 08b8054

Browse files
peter-hoferpeter-hofer
committed
[GR-70729] Split object arrays in ranges for marking.
PullRequest: graal/22361
2 parents c75b6af + 59eb933 commit 08b8054

File tree

6 files changed

+147
-36
lines changed

6 files changed

+147
-36
lines changed

‎sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/StackValue.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -72,7 +72,7 @@ private StackValue() {
7272
* <pre>
7373
* ComplexValue numberOnStack = StackValue.get(ComplexValue.class);
7474
* numberOnStack.realPart(3.0);
75-
* numberOnStack.imagineryPart(4.0);
75+
* numberOnStack.imaginaryPart(4.0);
7676
* double absoluteValue = absoluteValue(numberOnStack);
7777
* assert 5.0 == absoluteValue;
7878
* </pre>

‎sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/c/struct/CStruct.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2013, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -67,10 +67,10 @@
6767
* void realPart(double re);
6868
*
6969
* &#64;CField("im")
70-
* double imagineryPart();
70+
* double imaginaryPart();
7171
*
7272
* &#64;CField("im")
73-
* void imagineryPart(double im);
73+
* void imaginaryPart(double im);
7474
* }
7575
* </pre>
7676
*

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java‎

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge;
2626

27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
2728
import static com.oracle.svm.core.snippets.KnownIntrinsics.readCallerStackPointer;
2829

2930
import org.graalvm.nativeimage.IsolateThread;
@@ -52,12 +53,16 @@
5253
import com.oracle.svm.core.heap.Heap;
5354
import com.oracle.svm.core.heap.ObjectHeader;
5455
import com.oracle.svm.core.heap.ObjectVisitor;
56+
import com.oracle.svm.core.hub.DynamicHub;
57+
import com.oracle.svm.core.hub.HubType;
5558
import com.oracle.svm.core.log.Log;
5659
import com.oracle.svm.core.metaspace.Metaspace;
60+
import com.oracle.svm.core.snippets.KnownIntrinsics;
5761
import com.oracle.svm.core.thread.VMThreads;
5862
import com.oracle.svm.core.threadlocal.VMThreadLocalSupport;
5963
import com.oracle.svm.core.util.Timer;
6064

65+
import jdk.graal.compiler.nodes.java.ArrayLengthNode;
6166
import jdk.graal.compiler.word.Word;
6267

6368
/**
@@ -111,6 +116,7 @@ final class CompactingOldGeneration extends OldGeneration {
111116

112117
private final Space space = new Space("Old", "O", false, getAge());
113118
private final MarkStack markStack = new MarkStack();
119+
private final MarkStack arrayMarkStack = new MarkStack();
114120

115121
private final GreyObjectsWalker toGreyObjectsWalker = new GreyObjectsWalker();
116122
private final PlanningVisitor planningVisitor = new PlanningVisitor();
@@ -158,17 +164,56 @@ boolean scanGreyObjects(boolean incrementalGc) {
158164
}
159165
toGreyObjectsWalker.walkGreyObjects();
160166
} else {
161-
if (markStack.isEmpty()) {
167+
if (markStack.isEmpty() && arrayMarkStack.isEmpty()) {
162168
return false;
163169
}
164170
GreyToBlackObjectVisitor visitor = GCImpl.getGCImpl().getGreyToBlackObjectVisitor();
165171
do {
166-
visitor.visitObject(markStack.pop());
167-
} while (!markStack.isEmpty());
172+
while (!markStack.isEmpty()) {
173+
visitor.visitObject(markStack.popObject());
174+
}
175+
176+
// Process array ranges one at a time to avoid bloating the marking stack
177+
if (!arrayMarkStack.isEmpty()) {
178+
scanArrayRange(visitor);
179+
}
180+
} while (!markStack.isEmpty() || !arrayMarkStack.isEmpty());
168181
}
169182
return true;
170183
}
171184

185+
@AlwaysInline("GC performance")
186+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
187+
private void pushOntoMarkStack(Object obj) {
188+
DynamicHub objHub = KnownIntrinsics.readHub(obj);
189+
if (objHub.getHubType() == HubType.OBJECT_ARRAY) {
190+
if (ArrayLengthNode.arrayLength(obj) != 0) {
191+
arrayMarkStack.pushObject(obj);
192+
arrayMarkStack.pushInt(0);
193+
}
194+
} else {
195+
markStack.pushObject(obj);
196+
}
197+
}
198+
199+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
200+
private void scanArrayRange(GreyToBlackObjectVisitor visitor) {
201+
int index = arrayMarkStack.popInt();
202+
Object array = arrayMarkStack.popObject();
203+
204+
int length = ArrayLengthNode.arrayLength(array);
205+
final int stride = 2048;
206+
int endIndex = index + stride;
207+
if (endIndex < length) {
208+
arrayMarkStack.pushObject(array);
209+
arrayMarkStack.pushInt(endIndex);
210+
} else {
211+
endIndex = length;
212+
}
213+
214+
visitor.visitObjectArrayRange(array, index, endIndex - index);
215+
}
216+
172217
@AlwaysInline("GC performance")
173218
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
174219
@Override
@@ -196,7 +241,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead
196241
assert !ObjectHeaderImpl.hasIdentityHashFromAddressInline(oh.readHeaderFromObject(result));
197242
}
198243
ObjectHeaderImpl.setMarked(result);
199-
markStack.push(result);
244+
pushOntoMarkStack(result);
200245
return result;
201246
}
202247

@@ -212,7 +257,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal
212257
assert originalSpace == space;
213258
if (!ObjectHeaderImpl.isMarked(original)) {
214259
ObjectHeaderImpl.setMarked(original);
215-
markStack.push(original);
260+
pushOntoMarkStack(original);
216261
}
217262
return original;
218263
}
@@ -238,7 +283,7 @@ protected boolean promotePinnedObject(Object obj, HeapChunk.Header<?> originalCh
238283
((AlignedHeapChunk.AlignedHeader) originalChunk).setShouldSweepInsteadOfCompact(true);
239284
}
240285
ObjectHeaderImpl.setMarked(obj);
241-
markStack.push(obj);
286+
pushOntoMarkStack(obj);
242287
return true;
243288
}
244289

@@ -507,17 +552,20 @@ boolean verifySpaces() {
507552
@Override
508553
void checkSanityBeforeCollection() {
509554
assert markStack.isEmpty();
555+
assert arrayMarkStack.isEmpty();
510556
}
511557

512558
@Override
513559
void checkSanityAfterCollection() {
514560
assert markStack.isEmpty();
561+
assert arrayMarkStack.isEmpty();
515562
}
516563

517564
@Override
518565
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
519566
void tearDown() {
520567
markStack.tearDown();
568+
arrayMarkStack.tearDown();
521569
space.tearDown();
522570
}
523571
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge;
2626

27+
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28+
2729
import org.graalvm.nativeimage.Platform;
2830
import org.graalvm.nativeimage.Platforms;
2931

@@ -53,4 +55,9 @@ public void visitObject(Object o) {
5355
ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor);
5456
InteriorObjRefWalker.walkObjectInline(o, objRefVisitor);
5557
}
58+
59+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
60+
public void visitObjectArrayRange(Object o, int firstIndex, int count) {
61+
InteriorObjRefWalker.walkObjectArrayRangeInline(o, firstIndex, count, objRefVisitor);
62+
}
5663
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/MarkStack.java‎

Lines changed: 64 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,13 @@
2727
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
2828
import static jdk.vm.ci.code.CodeUtil.K;
2929

30-
import jdk.graal.compiler.word.Word;
3130
import org.graalvm.nativeimage.Platform;
3231
import org.graalvm.nativeimage.Platforms;
3332
import org.graalvm.nativeimage.c.struct.RawField;
3433
import org.graalvm.nativeimage.c.struct.RawStructure;
3534
import org.graalvm.nativeimage.c.struct.SizeOf;
3635
import org.graalvm.nativeimage.c.struct.UniqueLocationIdentity;
36+
import org.graalvm.word.Pointer;
3737
import org.graalvm.word.PointerBase;
3838
import org.graalvm.word.UnsignedWord;
3939

@@ -42,21 +42,31 @@
4242
import com.oracle.svm.core.config.ConfigurationValues;
4343
import com.oracle.svm.core.memory.NullableNativeMemory;
4444
import com.oracle.svm.core.nmt.NmtCategory;
45+
import com.oracle.svm.core.stack.StackOverflowCheck;
4546
import com.oracle.svm.core.util.VMError;
4647

4748
import jdk.graal.compiler.api.replacements.Fold;
4849
import jdk.graal.compiler.word.ObjectAccess;
50+
import jdk.graal.compiler.word.Word;
4951

5052
/**
5153
* LIFO stack for objects to visit during the mark phase. Without it, recursive calls could exhaust
52-
* the {@linkplain com.oracle.svm.core.stack.StackOverflowCheck yellow zone stack space} during GC.
54+
* the {@linkplain StackOverflowCheck yellow zone stack space} during GC. Callers can also push
55+
* other kinds of values, for example indexes for object arrays to keep track of the scanned range.
5356
*/
5457
public final class MarkStack {
5558
private static final int SEGMENT_SIZE = 64 * K - /* avoid potential malloc() overallocation */ 64;
5659

5760
@Fold
5861
static int entriesPerSegment() {
59-
return (SEGMENT_SIZE - SizeOf.get(Segment.class)) / ConfigurationValues.getObjectLayout().getReferenceSize();
62+
int headerSize = SizeOf.get(Segment.class);
63+
assert headerSize % entrySize() == 0 : "must be aligned";
64+
return (SEGMENT_SIZE - headerSize) / entrySize();
65+
}
66+
67+
@Fold
68+
static int entrySize() {
69+
return ConfigurationValues.getObjectLayout().getReferenceSize();
6070
}
6171

6272
private Segment top;
@@ -68,42 +78,68 @@ public MarkStack() {
6878

6979
@AlwaysInline("GC performance")
7080
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
71-
public void push(Object obj) {
81+
public void pushObject(Object obj) {
7282
assert obj != null;
83+
Pointer entry = pushSlot();
84+
ObjectAccess.writeObject(Word.nullPointer(), entry, obj);
85+
}
86+
87+
@AlwaysInline("GC performance")
88+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
89+
public void pushInt(int value) {
90+
Pointer entry = pushSlot();
91+
entry.writeInt(0, value);
92+
}
7393

94+
@AlwaysInline("GC performance")
95+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
96+
private Pointer pushSlot() {
7497
if (top.isNull() || cursor == entriesPerSegment()) {
7598
top = allocateSegment(top);
7699
cursor = 0;
77100
}
78101

79-
UnsignedWord offset = getOffsetAtIndex(cursor);
80-
ObjectAccess.writeObject(top, offset, obj);
102+
Pointer entry = getEntryAddress(cursor);
81103
cursor++;
104+
return entry;
82105
}
83106

84107
@AlwaysInline("GC performance")
85108
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
86-
public Object pop() {
109+
public Object popObject() {
87110
assert !isEmpty();
88111

89-
cursor--;
90-
UnsignedWord offset = getOffsetAtIndex(cursor);
91-
Object obj = ObjectAccess.readObject(top, offset);
92-
112+
Pointer entry = getEntryAddress(cursor - 1);
113+
Object obj = ObjectAccess.readObject(Word.nullPointer(), entry);
93114
assert obj != null;
94115

116+
popSlot();
117+
return obj;
118+
}
119+
120+
@AlwaysInline("GC performance")
121+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
122+
public int popInt() {
123+
assert !isEmpty();
124+
125+
Pointer entry = getEntryAddress(cursor - 1);
126+
int v = entry.readInt(0);
127+
128+
popSlot();
129+
return v;
130+
}
131+
132+
@AlwaysInline("GC performance")
133+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
134+
private void popSlot() {
135+
cursor--;
95136
if (cursor == 0) {
96137
if (top.getNext().isNonNull()) { // free eagerly, use cursor==0 only if completely empty
97-
Segment t = top;
98-
top = top.getNext();
99-
cursor = entriesPerSegment();
100-
NullableNativeMemory.free(t);
138+
freeTopSegment();
101139
} else {
102140
// keep a single segment
103141
}
104142
}
105-
106-
return obj;
107143
}
108144

109145
@AlwaysInline("GC performance")
@@ -134,11 +170,19 @@ private static Segment allocateSegment(Segment next) {
134170
return segment;
135171
}
136172

173+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
174+
private void freeTopSegment() {
175+
Segment t = top;
176+
top = top.getNext();
177+
cursor = entriesPerSegment();
178+
NullableNativeMemory.free(t);
179+
}
180+
137181
@AlwaysInline("GC performance")
138182
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
139-
private static UnsignedWord getOffsetAtIndex(int index) {
140-
int refSize = ConfigurationValues.getObjectLayout().getReferenceSize();
141-
return Word.unsigned(index).multiply(refSize).add(SizeOf.unsigned(Segment.class));
183+
private Pointer getEntryAddress(int index) {
184+
Pointer firstEntry = ((Pointer) top).add(SizeOf.unsigned(Segment.class));
185+
return firstEntry.add(Word.unsigned(index).multiply(entrySize()));
142186
}
143187

144188
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)

0 commit comments

Comments
 (0)