Merge branch '2.14' into 2.15

Author: cowtowncoder

release-notes/CREDITS-2.x

@@ -412,6 +412,9 @@ Jonas Konrad (yawkat@github)
   * Contributed fix for #3655: `Enum` values can not be read from single-element array even
     with `DeserializationFeature.UNWRAP_SINGLE_VALUE_ARRAYS`
    (2.14.1)
+  * Contributed fix for #3655: `ObjectMapper` default heap consumption increased significantly
+    from 2.13.x to 2.14.0
+   (2.14.1)
 
 Jirka Kremser (Jiri-Kremser@github)
   * Suggested #924: SequenceWriter.writeAll() could accept Iterable
@@ -1520,3 +1523,8 @@ Andrej Mitrovic (AndrejMitrovic@github)
   * Reported #3655: `Enum` values can not be read from single-element array even with
     `DeserializationFeature.UNWRAP_SINGLE_VALUE_ARRAYS`
   (2.14.1)
+
+Moritz Halbritter (mhalbritter@github)
+  * Reported #3665: `ObjectMapper` default heap consumption increased significantly
+    from 2.13.x to 2.14.0
+  (2.14.1)

release-notes/VERSION-2.x

@@ -10,6 +10,9 @@ Project: jackson-databind
   `DeserializationFeature.UNWRAP_SINGLE_VALUE_ARRAYS`
  (reported by Andrej M)
  (fix contributed by Jonas K)
+#3665: `ObjectMapper` default heap consumption increased significantly from 2.13.x to 2.14.0
+ (reported by Moritz H)
+ (fix contributed by Jonas K)
 
 2.14.0 (05-Nov-2022)
 

src/main/java/com/fasterxml/jackson/databind/util/internal/PrivateMaxEntriesMap.java

@@ -35,7 +35,9 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -132,14 +134,20 @@ public final class PrivateMaxEntriesMap<K, V> extends AbstractMap<K, V>
     /** The maximum capacity of the map. */
     static final long MAXIMUM_CAPACITY = Long.MAX_VALUE - Integer.MAX_VALUE;
 
-    /** The number of read buffers to use. */
-    static final int NUMBER_OF_READ_BUFFERS = ceilingNextPowerOfTwo(NCPU);
+    /**
+     * The number of read buffers to use.
+     * The max of 4 was introduced due to https://github.com/FasterXML/jackson-databind/issues/3665.
+     */
+    static final int NUMBER_OF_READ_BUFFERS = Math.min(4, ceilingNextPowerOfTwo(NCPU));
 
     /** Mask value for indexing into the read buffers. */
     static final int READ_BUFFERS_MASK = NUMBER_OF_READ_BUFFERS - 1;
 
-    /** The number of pending read operations before attempting to drain. */
-    static final int READ_BUFFER_THRESHOLD = 32;
+    /**
+     * The number of pending read operations before attempting to drain.
+     * The threshold of 4 was introduced due to https://github.com/FasterXML/jackson-databind/issues/3665.
+     */
+    static final int READ_BUFFER_THRESHOLD = 4;
 
     /** The maximum number of read operations to perform per amortized drain. */
     static final int READ_BUFFER_DRAIN_THRESHOLD = 2 * READ_BUFFER_THRESHOLD;
@@ -175,16 +183,20 @@ static int ceilingNextPowerOfTwo(int x) {
 
     final Lock evictionLock;
     final Queue<Runnable> writeBuffer;
-    final AtomicLong[] readBufferWriteCount;
-    final AtomicLong[] readBufferDrainAtWriteCount;
-    final AtomicReference<Node<K, V>>[][] readBuffers;
+    final AtomicLongArray readBufferWriteCount;
+    final AtomicLongArray readBufferDrainAtWriteCount;
+    final AtomicReferenceArray<Node<K, V>> readBuffers;
 
     final AtomicReference<DrainStatus> drainStatus;
 
     transient Set<K> keySet;
     transient Collection<V> values;
     transient Set<Entry<K, V>> entrySet;
 
+    private static int readBufferIndex(int bufferIndex, int entryIndex) {
+        return READ_BUFFER_SIZE * bufferIndex + entryIndex;
+    }
+
     /**
      * Creates an instance based on the builder's configuration.
      */
@@ -203,17 +215,9 @@ private PrivateMaxEntriesMap(Builder<K, V> builder) {
         drainStatus = new AtomicReference<DrainStatus>(IDLE);
 
         readBufferReadCount = new long[NUMBER_OF_READ_BUFFERS];
-        readBufferWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS];
-        readBufferDrainAtWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS];
-        readBuffers = new AtomicReference[NUMBER_OF_READ_BUFFERS][READ_BUFFER_SIZE];
-        for (int i = 0; i < NUMBER_OF_READ_BUFFERS; i++) {
-            readBufferWriteCount[i] = new AtomicLong();
-            readBufferDrainAtWriteCount[i] = new AtomicLong();
-            readBuffers[i] = new AtomicReference[READ_BUFFER_SIZE];
-            for (int j = 0; j < READ_BUFFER_SIZE; j++) {
-                readBuffers[i][j] = new AtomicReference<Node<K, V>>();
-            }
-        }
+        readBufferWriteCount = new AtomicLongArray(NUMBER_OF_READ_BUFFERS);
+        readBufferDrainAtWriteCount = new AtomicLongArray(NUMBER_OF_READ_BUFFERS);
+        readBuffers = new AtomicReferenceArray<>(NUMBER_OF_READ_BUFFERS * READ_BUFFER_SIZE);
     }
 
     /** Ensures that the object is not null. */
@@ -330,12 +334,11 @@ long recordRead(int bufferIndex, Node<K, V> node) {
         // The location in the buffer is chosen in a racy fashion as the increment
         // is not atomic with the insertion. This means that concurrent reads can
         // overlap and overwrite one another, resulting in a lossy buffer.
-        final AtomicLong counter = readBufferWriteCount[bufferIndex];
-        final long writeCount = counter.get();
-        counter.lazySet(writeCount + 1);
+        final long writeCount = readBufferWriteCount.get(bufferIndex);
+        readBufferWriteCount.lazySet(bufferIndex, writeCount + 1);
 
         final int index = (int) (writeCount & READ_BUFFER_INDEX_MASK);
-        readBuffers[bufferIndex][index].lazySet(node);
+        readBuffers.lazySet(readBufferIndex(bufferIndex, index), node);
 
         return writeCount;
     }
@@ -348,7 +351,7 @@ long recordRead(int bufferIndex, Node<K, V> node) {
      * @param writeCount the number of writes on the chosen read buffer
      */
     void drainOnReadIfNeeded(int bufferIndex, long writeCount) {
-        final long pending = (writeCount - readBufferDrainAtWriteCount[bufferIndex].get());
+        final long pending = (writeCount - readBufferDrainAtWriteCount.get(bufferIndex));
         final boolean delayable = (pending < READ_BUFFER_THRESHOLD);
         final DrainStatus status = drainStatus.get();
         if (status.shouldDrainBuffers(delayable)) {
@@ -403,20 +406,20 @@ void drainReadBuffers() {
     /** Drains the read buffer up to an amortized threshold. */
     //@GuardedBy("evictionLock")
     void drainReadBuffer(int bufferIndex) {
-        final long writeCount = readBufferWriteCount[bufferIndex].get();
+        final long writeCount = readBufferWriteCount.get(bufferIndex);
         for (int i = 0; i < READ_BUFFER_DRAIN_THRESHOLD; i++) {
             final int index = (int) (readBufferReadCount[bufferIndex] & READ_BUFFER_INDEX_MASK);
-            final AtomicReference<Node<K, V>> slot = readBuffers[bufferIndex][index];
-            final Node<K, V> node = slot.get();
+            final int arrayIndex = readBufferIndex(bufferIndex, index);
+            final Node<K, V> node = readBuffers.get(arrayIndex);
             if (node == null) {
                 break;
             }
 
-            slot.lazySet(null);
+            readBuffers.lazySet(arrayIndex, null);
             applyRead(node);
             readBufferReadCount[bufferIndex]++;
         }
-        readBufferDrainAtWriteCount[bufferIndex].lazySet(writeCount);
+        readBufferDrainAtWriteCount.lazySet(bufferIndex, writeCount);
     }
 
     /** Updates the node's location in the page replacement policy. */
@@ -579,10 +582,8 @@ public void clear() {
             }
 
             // Discard all pending reads
-            for (AtomicReference<Node<K, V>>[] buffer : readBuffers) {
-                for (AtomicReference<Node<K, V>> slot : buffer) {
-                    slot.lazySet(null);
-                }
+            for (int i = 0; i < readBuffers.length(); i++) {
+                readBuffers.lazySet(i, null);
             }
 
             // Apply all pending writes

src/test/java/com/fasterxml/jackson/databind/MapperFootprintTest.java

@@ -0,0 +1,30 @@
+package com.fasterxml.jackson.databind;
+
+import com.google.common.testing.GcFinalization;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.openjdk.jol.info.GraphLayout;
+
+public class MapperFootprintTest {
+    @Test
+    @Ignore
+    public void testMapperFootprint() throws InterruptedException {
+        // memory footprint limit for the ObjectMapper
+
+        // force gc (see javadoc of GraphLayout.subtract)
+        GcFinalization.awaitFullGc();
+        // do this calculation twice. If there's a GC in one case, and the subtract call doesn't work well because of
+        // this, we can fall back to the other.
+        GraphLayout mapperLayoutA = GraphLayout.parseInstance(new ObjectMapper())
+                .subtract(GraphLayout.parseInstance(new ObjectMapper()));
+        GraphLayout mapperLayoutB = GraphLayout.parseInstance(new ObjectMapper())
+                .subtract(GraphLayout.parseInstance(new ObjectMapper()));
+        GraphLayout mapperLayout = mapperLayoutA.totalSize() > mapperLayoutB.totalSize() ?
+                mapperLayoutB : mapperLayoutA;
+
+        Assert.assertTrue(
+                "ObjectMapper memory footprint exceeded limit. Footprint details: " + mapperLayout.toFootprint(),
+                mapperLayout.totalSize() < 10000);
+    }
+}