count tokens

more changes

don't increment token count for NOT_AVAILABLE

handle case when token is set to null

use _updateTokenToNull

null check not needed now

add test

Update TokenCountTest.java

Update StreamReadConstraints.java

add validation

try to reduce overhead for when maxTokenCount is not needed

Update StreamReadConstraints.java

Update TokenCountTest.java

Author: pjfanning

src/main/java/com/fasterxml/jackson/core/JsonParser.java

@@ -2546,6 +2546,18 @@ public <T extends TreeNode> T readValueAsTree() throws IOException {
         return (T) _codec().readTree(this);
     }
 
+    /**
+     * Get an approximate count of the number of tokens that have been read.
+     * This count is likely to be only updated if {@link StreamReadConstraints.Builder.maxTokenCount(long)}
+     * has been used to set a limit on the number of tokens that can be read.
+     *
+     * @return the number of tokens that have been read (-1 if the count is not available)
+     * @since 2.18
+     */
+    public long getTokenCount() {
+        return -1L;
+    }
+
     protected ObjectCodec _codec() {
         ObjectCodec c = getCodec();
         if (c == null) {

src/main/java/com/fasterxml/jackson/core/StreamReadConstraints.java

@@ -43,6 +43,12 @@ public class StreamReadConstraints
      */
     public static final long DEFAULT_MAX_DOC_LEN = -1L;
 
+    /**
+     * Default setting for maximum token count:
+     * see {@link Builder#maxTokenCount} for details.
+     */
+    public static final long DEFAULT_MAX_TOKEN_COUNT = -1L;
+
     /**
      * @since 2.16
      */
@@ -74,6 +80,7 @@ public class StreamReadConstraints
 
     protected final int _maxNestingDepth;
     protected final long _maxDocLen;
+    protected final long _maxTokenCount;
 
     protected final int _maxNumLen;
     protected final int _maxStringLen;
@@ -112,6 +119,7 @@ public static void overrideDefaultStreamReadConstraints(final StreamReadConstrai
 
     public static final class Builder {
         private long maxDocLen;
+        private long maxTokenCount;
         private int maxNestingDepth;
         private int maxNumLen;
         private int maxStringLen;
@@ -156,6 +164,31 @@ public Builder maxDocumentLength(long maxDocLen) {
             return this;
         }
 
+        /**
+         * Sets the maximum allowed token count (for positive values over 0) or
+         * indicate that any count is acceptable ({@code 0} or negative number).
+         *
+         * <p>
+         *   A token is a single unit of input, such as a number, a string, an object
+         *   start or end, or an array start or end.
+         * </p>
+         *
+         * @param maxTokenCount the maximum allowed token count if positive number above 0; otherwise
+         *   ({@code 0} or negative number) means "unlimited".
+         *
+         * @return this builder
+         *
+         * @since 2.18
+         */
+        public Builder maxTokenCount(long maxTokenCount) {
+            // Negative values and 0 mean "unlimited", mark with -1L
+            if (maxTokenCount <= 0L) {
+                maxTokenCount = -1L;
+            }
+            this.maxTokenCount = maxTokenCount;
+            return this;
+        }
+
         /**
          * Sets the maximum number length (in chars or bytes, depending on input context).
          * The default is 1000.
@@ -220,14 +253,15 @@ public Builder maxNameLength(final int maxNameLen) {
         }
 
         Builder() {
-            this(DEFAULT_MAX_DEPTH, DEFAULT_MAX_DOC_LEN,
+            this(DEFAULT_MAX_DEPTH, DEFAULT_MAX_DOC_LEN, DEFAULT_MAX_TOKEN_COUNT,
                     DEFAULT_MAX_NUM_LEN, DEFAULT_MAX_STRING_LEN, DEFAULT_MAX_NAME_LEN);
         }
 
-        Builder(final int maxNestingDepth, final long maxDocLen,
+        Builder(final int maxNestingDepth, final long maxDocLen, final long maxTokenCount,
                 final int maxNumLen, final int maxStringLen, final int maxNameLen) {
             this.maxNestingDepth = maxNestingDepth;
             this.maxDocLen = maxDocLen;
+            this.maxTokenCount = maxTokenCount;
             this.maxNumLen = maxNumLen;
             this.maxStringLen = maxStringLen;
             this.maxNameLen = maxNameLen;
@@ -236,14 +270,15 @@ public Builder maxNameLength(final int maxNameLen) {
         Builder(StreamReadConstraints src) {
             maxNestingDepth = src._maxNestingDepth;
             maxDocLen = src._maxDocLen;
+            maxTokenCount = src._maxTokenCount;
             maxNumLen = src._maxNumLen;
             maxStringLen = src._maxStringLen;
             maxNameLen = src._maxNameLen;
         }
 
         public StreamReadConstraints build() {
             return new StreamReadConstraints(maxNestingDepth, maxDocLen,
-                    maxNumLen, maxStringLen, maxNameLen);
+                    maxNumLen, maxStringLen, maxNameLen, maxTokenCount);
         }
     }
 
@@ -257,7 +292,7 @@ public StreamReadConstraints build() {
     protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen,
             final int maxNumLen, final int maxStringLen) {
         this(maxNestingDepth, maxDocLen,
-                maxNumLen, maxStringLen, DEFAULT_MAX_NAME_LEN);
+                maxNumLen, maxStringLen, DEFAULT_MAX_NAME_LEN, DEFAULT_MAX_TOKEN_COUNT);
     }
 
     /**
@@ -269,13 +304,30 @@ protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen,
      *
      * @since 2.16
      */
+    @Deprecated // since 2.18
+    protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen,
+                                    final int maxNumLen, final int maxStringLen, final int maxNameLen) {
+        this(maxNestingDepth, maxDocLen, maxNumLen, maxStringLen, maxNameLen, DEFAULT_MAX_TOKEN_COUNT);
+    }
+
+    /**
+     * @param maxNestingDepth Maximum input document nesting to allow
+     * @param maxDocLen Maximum input document length to allow
+     * @param maxNumLen Maximum number representation length to allow
+     * @param maxStringLen Maximum String value length to allow
+     * @param maxNameLen Maximum Object property name length to allow
+     * @param maxTokenCount Maximum number of tokens to allow
+     *
+     * @since 2.18
+     */
     protected StreamReadConstraints(final int maxNestingDepth, final long maxDocLen,
-            final int maxNumLen, final int maxStringLen, final int maxNameLen) {
+            final int maxNumLen, final int maxStringLen, final int maxNameLen, final long maxTokenCount) {
         _maxNestingDepth = maxNestingDepth;
         _maxDocLen = maxDocLen;
         _maxNumLen = maxNumLen;
         _maxStringLen = maxStringLen;
         _maxNameLen = maxNameLen;
+        _maxTokenCount = maxTokenCount;
     }
 
     public static Builder builder() {
@@ -337,6 +389,31 @@ public boolean hasMaxDocumentLength() {
         return _maxDocLen > 0L;
     }
 
+    /**
+     * Accessor for maximum token count.
+     * see {@link Builder#maxTokenCount(long)} for details.
+     *
+     * @return Maximum allowed token count
+     * @since 2.18
+     */
+    public long getMaxTokenCount() {
+        return _maxTokenCount;
+    }
+
+    /**
+     * Convenience method, basically same as:
+     *<pre>
+     *  getMaxTokenCount() &gt; 0L
+     *</pre>
+     *
+     * @return {@code True} if this constraints instance has a limit for maximum
+     *    token count to enforce; {@code false} otherwise.
+     * @since 2.18
+     */
+    public boolean hasMaxTokenCount() {
+        return _maxTokenCount > 0L;
+    }
+
     /**
      * Accessor for maximum length of numbers to decode.
      * see {@link Builder#maxNumberLength(int)} for details.
@@ -419,6 +496,31 @@ public void validateDocumentLength(long len) throws StreamConstraintsException
         }
     }
 
+    /**
+     * Convenience method that can be used to verify that the
+     * token count does not exceed the maximum specified by this
+     * constraints object (if any): if it does, a
+     * {@link StreamConstraintsException}
+     * is thrown.
+     *
+     * @param count Current token count for processed document content
+     *
+     * @throws StreamConstraintsException If length exceeds maximum
+     *
+     * @since 2.18
+     */
+    public void validateTokenCount(long count) throws StreamConstraintsException
+    {
+        // for performance reasons, it is assumed that users check hasMaxTokenCount()
+        // before calling this method - this method will not work properly if hasMaxTokenCount() is false
+        if (count > _maxTokenCount) {
+            throw _constructException(
+                "Token count (%d) exceeds the maximum allowed (%d, from %s)",
+                count, _maxTokenCount,
+                _constrainRef("getMaxTokenCount"));
+        }
+    }
+
     /*
     /**********************************************************************
     /* Convenience methods for validation, token lengths

src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java

@@ -7,6 +7,7 @@
 
 import com.fasterxml.jackson.core.*;
 import com.fasterxml.jackson.core.exc.InputCoercionException;
+import com.fasterxml.jackson.core.exc.StreamConstraintsException;
 import com.fasterxml.jackson.core.io.JsonEOFException;
 import com.fasterxml.jackson.core.io.NumberInput;
 import com.fasterxml.jackson.core.util.ByteArrayBuilder;
@@ -147,6 +148,13 @@ public abstract class ParserMinimalBase extends JsonParser
      */
     protected JsonToken _currToken;
 
+    /**
+     * Current count of tokens
+     *
+     * @since 2.18
+     */
+    protected long _tokenCount;
+
     /**
      * Last cleared token, if any: that is, value that was in
      * effect when {@link #clearCurrentToken} was called.
@@ -502,6 +510,11 @@ public String getValueAsString(String defaultValue) throws IOException {
         return getText();
     }
 
+    @Override
+    public long getTokenCount() {
+        return _tokenCount;
+    }
+
     /*
     /**********************************************************
     /* Base64 decoding
@@ -782,6 +795,22 @@ protected final void _wrapError(String msg, Throwable t) throws JsonParseExcepti
         throw _constructReadException(msg, t);
     }
 
+    protected final JsonToken _updateToken(final JsonToken token) throws StreamConstraintsException {
+        _currToken = token;
+        if (streamReadConstraints().hasMaxTokenCount() && token != JsonToken.NOT_AVAILABLE) {
+            streamReadConstraints().validateTokenCount(++_tokenCount);
+        }
+        return token;
+    }
+
+    protected final JsonToken _updateTokenToNull() {
+        return (_currToken = null);
+    }
+
+    protected final JsonToken _updateTokenToNA() {
+        return (_currToken = JsonToken.NOT_AVAILABLE);
+    }
+
     @Deprecated // since 2.11
     protected static byte[] _asciiBytes(String str) {
         byte[] b = new byte[str.length()];