From 03b355e87b3b5f0be1b60417a9df07626ca47324 Mon Sep 17 00:00:00 2001 From: Fawzi Essam Date: Tue, 6 May 2025 01:47:45 +0200 Subject: [PATCH 1/3] Allow exposing CBOR undefined value as JsonToken.VALUE_EMBEDDED_OBJECT Signed-off-by: Fawzi Essam --- .../dataformat/cbor/CBORConstants.java | 2 + .../jackson/dataformat/cbor/CBORParser.java | 49 ++++++++++-- .../cbor/parse/UndefinedValueTest.java | 75 ++++++++++++++++++- 3 files changed, 117 insertions(+), 9 deletions(-) diff --git a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java index 8fec6eae9..02085bf59 100644 --- a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java +++ b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java @@ -113,6 +113,8 @@ public final class CBORConstants public final static int INT_BREAK = 0xFF; + public final static int SIMPLE_VALUE_UNDEFINED = 0xF7; + /* /********************************************************** /* Basic UTF-8 decode/encode table diff --git a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java index 01ab6184d..1b9161c43 100644 --- a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java +++ b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java @@ -45,7 +45,23 @@ public enum Feature implements FormatFeature * * @since 2.20 */ - DECODE_USING_STANDARD_NEGATIVE_BIGINT_ENCODING(false) + DECODE_USING_STANDARD_NEGATIVE_BIGINT_ENCODING(false), + + /** + * Feature that determines how an ` undefined ` value (0xF7) is decoded. + * + *

+ * When enabled, the parser returns {@link JsonToken#VALUE_EMBEDDED_OBJECT} with a + * value of {@code null}, allowing the caller to distinguish `undefined` from actual + * {@link JsonToken#VALUE_NULL}. + *

+ * When disabled (default, for backwards compatibility), `undefined` value is + * reported as {@link JsonToken#VALUE_NULL}, maintaining legacy behavior from Jackson 2.9.6 to 2.19. + *

+ * + * @since 2.20 + */ + HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT(false) ; final boolean _defaultState; @@ -1915,6 +1931,20 @@ private final byte[] _getBinaryFromString(Base64Variant variant) throws IOExcept return _binaryValue; } + /** + * Checking whether the current token represents an `undefined` value (0xF7). + *

+ * This method allows distinguishing between real {@code null} and `undefined`, + * even if {@link CBORParser.Feature#HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT} is disabled + * and the token is reported as {@link JsonToken#VALUE_NULL}. + * + * @return {@code true} if current token is an `undefined`, {@code false} otherwise + * @since 2.20 + */ + public boolean isUndefined() { + return (_inputBuffer[_inputPtr - 1] & 0xFF) == SIMPLE_VALUE_UNDEFINED; + } + /* /********************************************************** /* Numeric accessors of public API @@ -3656,13 +3686,22 @@ private final static long _long(int i1, int i2) * Helper method to encapsulate details of handling of mysterious `undefined` value * that is allowed to be used as something encoder could not handle (as per spec), * whatever the heck that should be. - * Current definition for 2.9 is that we will be return {@link JsonToken#VALUE_NULL}, but - * for later versions it is likely that we will alternatively allow decoding as - * {@link JsonToken#VALUE_EMBEDDED_OBJECT} with "embedded value" of `null`. + * + *

+ * For backward compatibility with Jackson 2.9.6 to 2.19, this value is decoded + * as {@link JsonToken#VALUE_NULL} by default. + *

+ * + * since 2.20 If {@link CBORParser.Feature#HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT} is enabled, + * the value will instead be decoded as {@link JsonToken#VALUE_EMBEDDED_OBJECT} + * with an embedded value of {@code null}. * * @since 2.9.6 */ - protected JsonToken _decodeUndefinedValue() throws IOException { + protected JsonToken _decodeUndefinedValue() { + if (CBORParser.Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT.enabledIn(_formatFeatures)) { + return JsonToken.VALUE_EMBEDDED_OBJECT; + } return JsonToken.VALUE_NULL; } diff --git a/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java b/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java index 7ee4aaa60..57262fedb 100644 --- a/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java +++ b/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java @@ -4,12 +4,12 @@ import org.junit.jupiter.api.Test; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.dataformat.cbor.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; // for [dataformat-binary#93] public class UndefinedValueTest extends CBORTestBase @@ -21,8 +21,23 @@ public class UndefinedValueTest extends CBORTestBase @Test public void testUndefinedLiteralStreaming() throws Exception { - JsonParser p = cborParser(CBOR_F, new byte[] { BYTE_UNDEFINED }); + CBORParser p = cborParser(CBOR_F, new byte[] { BYTE_UNDEFINED }); assertEquals(JsonToken.VALUE_NULL, p.nextToken()); + assertTrue(p.isUndefined()); + assertNull(p.nextToken()); + p.close(); + } + + // @since 2.20 [jackson-dataformats-binary/137] + @Test + public void testUndefinedLiteralAsEmbeddedObject() throws Exception { + CBORFactory f = CBORFactory.builder() + .enable(CBORParser.Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT) + .build(); + CBORParser p = cborParser(f, new byte[] { BYTE_UNDEFINED }); + + assertEquals(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken()); + assertTrue(p.isUndefined()); assertNull(p.nextToken()); p.close(); } @@ -34,9 +49,30 @@ public void testUndefinedInArray() throws Exception out.write(CBORConstants.BYTE_ARRAY_INDEFINITE); out.write(BYTE_UNDEFINED); out.write(CBORConstants.BYTE_BREAK); - JsonParser p = cborParser(CBOR_F, out.toByteArray()); + CBORParser p = cborParser(CBOR_F, out.toByteArray()); assertEquals(JsonToken.START_ARRAY, p.nextToken()); assertEquals(JsonToken.VALUE_NULL, p.nextToken()); + assertTrue(p.isUndefined()); + assertEquals(JsonToken.END_ARRAY, p.nextToken()); + assertNull(p.nextToken()); + p.close(); + } + + // @since 2.20 [jackson-dataformats-binary/137] + @Test + public void testUndefinedInArrayAsEmbeddedObject() throws Exception { + CBORFactory f = CBORFactory.builder() + .enable(CBORParser.Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT) + .build(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(CBORConstants.BYTE_ARRAY_INDEFINITE); + out.write(BYTE_UNDEFINED); + out.write(CBORConstants.BYTE_BREAK); + CBORParser p = cborParser(f, out.toByteArray()); + assertEquals(JsonToken.START_ARRAY, p.nextToken()); + assertEquals(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken()); + assertTrue(p.isUndefined()); assertEquals(JsonToken.END_ARRAY, p.nextToken()); assertNull(p.nextToken()); p.close(); @@ -57,11 +93,42 @@ public void testUndefinedInObject() throws Exception // assume we use end marker for Object, so doc[doc.length-2] = BYTE_UNDEFINED; - JsonParser p = cborParser(CBOR_F, doc); + CBORParser p = cborParser(CBOR_F, doc); assertEquals(JsonToken.START_OBJECT, p.nextToken()); assertEquals(JsonToken.FIELD_NAME, p.nextToken()); assertEquals("bar", p.currentName()); assertEquals(JsonToken.VALUE_NULL, p.nextToken()); + assertTrue(p.isUndefined()); + assertEquals(JsonToken.END_OBJECT, p.nextToken()); + assertNull(p.nextToken()); + p.close(); + } + + // @since 2.20 [jackson-dataformats-binary/137] + @Test + public void testUndefinedInObjectAsEmbeddedObject() throws Exception { + CBORFactory f = CBORFactory.builder() + .enable(CBORParser.Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT) + .build(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + CBORGenerator g = cborGenerator(out); + g.writeStartObject(); + g.writeFieldName("bar"); + g.writeBoolean(true); + g.writeEndObject(); + g.close(); + + byte[] doc = out.toByteArray(); + // assume we use end marker for Object, so + doc[doc.length - 2] = BYTE_UNDEFINED; + + CBORParser p = cborParser(f, doc); + assertEquals(JsonToken.START_OBJECT, p.nextToken()); + assertEquals(JsonToken.FIELD_NAME, p.nextToken()); + assertEquals("bar", p.currentName()); + assertEquals(JsonToken.VALUE_EMBEDDED_OBJECT, p.nextToken()); + assertTrue(p.isUndefined()); assertEquals(JsonToken.END_OBJECT, p.nextToken()); assertNull(p.nextToken()); p.close(); From bdf400ecef1e9989d2b5125f79be7fd43082b4d4 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Mon, 5 May 2025 17:51:08 -0700 Subject: [PATCH 2/3] Add release notes, minor tweaks --- .../jackson/dataformat/cbor/CBORParser.java | 15 ++++++++++----- release-notes/CREDITS-2.x | 3 +++ release-notes/VERSION-2.x | 4 ++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java index 1b9161c43..1b2613ad4 100644 --- a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java +++ b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java @@ -1939,10 +1939,15 @@ private final byte[] _getBinaryFromString(Base64Variant variant) throws IOExcept * and the token is reported as {@link JsonToken#VALUE_NULL}. * * @return {@code true} if current token is an `undefined`, {@code false} otherwise + * * @since 2.20 */ public boolean isUndefined() { - return (_inputBuffer[_inputPtr - 1] & 0xFF) == SIMPLE_VALUE_UNDEFINED; + if ((_currToken == JsonToken.VALUE_NULL) || (_currToken == JsonToken.VALUE_NULL)) { + return (_inputBuffer != null) + && (_inputBuffer[_inputPtr - 1] & 0xFF) == SIMPLE_VALUE_UNDEFINED; + } + return false; } /* @@ -3686,9 +3691,8 @@ private final static long _long(int i1, int i2) * Helper method to encapsulate details of handling of mysterious `undefined` value * that is allowed to be used as something encoder could not handle (as per spec), * whatever the heck that should be. - * *

- * For backward compatibility with Jackson 2.9.6 to 2.19, this value is decoded + * For backward compatibility with Jackson 2.10 to 2.19, this value is decoded * as {@link JsonToken#VALUE_NULL} by default. *

* @@ -3696,10 +3700,11 @@ private final static long _long(int i1, int i2) * the value will instead be decoded as {@link JsonToken#VALUE_EMBEDDED_OBJECT} * with an embedded value of {@code null}. * - * @since 2.9.6 + * @since 2.10 */ protected JsonToken _decodeUndefinedValue() { - if (CBORParser.Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT.enabledIn(_formatFeatures)) { + if (Feature.HANDLE_UNDEFINED_AS_EMBEDDED_OBJECT.enabledIn(_formatFeatures)) { + _binaryValue = null; // should be clear but just in case return JsonToken.VALUE_EMBEDDED_OBJECT; } return JsonToken.VALUE_NULL; diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 008bf45ac..f407d82b0 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -391,6 +391,9 @@ Brian Gruber (@bgruber) (2.20.0) Fawzi Essam (@iifawzi) + * Contributed implementation of #137: (cbor) Allow exposing CBOR "undefined" value as + `JsonToken.VALUE_EMBEDDED_OBJECT`; with embedded value of `null` + (2.20.0) * Contributed fix for #431: (cbor) Negative `BigInteger` values not encoded/decoded correctly (2.20.0) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index e753cdc4c..f35bc6b8a 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -16,6 +16,10 @@ Active maintainers: 2.20.0 (not yet released) +#137: (cbor) Allow exposing CBOR "undefined" value as `JsonToken.VALUE_EMBEDDED_OBJECT`; + with embedded value of `null` + (implementation contributed by Fawzi E) + #431: (cbor) Negative `BigInteger` values not encoded/decoded correctly (reported by Brian G) (fix contributed by Fawzi E) From cae0ab748407bf9ad3b1d311ef66ed8819623ec4 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Mon, 5 May 2025 17:55:05 -0700 Subject: [PATCH 3/3] Fix minor bug I introduced :) --- .../fasterxml/jackson/dataformat/cbor/CBORConstants.java | 5 +++++ .../com/fasterxml/jackson/dataformat/cbor/CBORParser.java | 6 ++---- .../jackson/dataformat/cbor/parse/UndefinedValueTest.java | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java index 02085bf59..9c708a9c6 100644 --- a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java +++ b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORConstants.java @@ -113,6 +113,11 @@ public final class CBORConstants public final static int INT_BREAK = 0xFF; + /** + * Marker for "undefined" value in CBOR spec. + * + * @since 2.20 + */ public final static int SIMPLE_VALUE_UNDEFINED = 0xF7; /* diff --git a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java index 1b2613ad4..351dbd1a4 100644 --- a/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java +++ b/cbor/src/main/java/com/fasterxml/jackson/dataformat/cbor/CBORParser.java @@ -49,15 +49,13 @@ public enum Feature implements FormatFeature /** * Feature that determines how an ` undefined ` value (0xF7) is decoded. - * *

* When enabled, the parser returns {@link JsonToken#VALUE_EMBEDDED_OBJECT} with a * value of {@code null}, allowing the caller to distinguish `undefined` from actual * {@link JsonToken#VALUE_NULL}. *

* When disabled (default, for backwards compatibility), `undefined` value is - * reported as {@link JsonToken#VALUE_NULL}, maintaining legacy behavior from Jackson 2.9.6 to 2.19. - *

+ * reported as {@link JsonToken#VALUE_NULL}, maintaining legacy behavior from Jackson 2.10 to 2.19. * * @since 2.20 */ @@ -1943,7 +1941,7 @@ private final byte[] _getBinaryFromString(Base64Variant variant) throws IOExcept * @since 2.20 */ public boolean isUndefined() { - if ((_currToken == JsonToken.VALUE_NULL) || (_currToken == JsonToken.VALUE_NULL)) { + if ((_currToken == JsonToken.VALUE_NULL) || (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT)) { return (_inputBuffer != null) && (_inputBuffer[_inputPtr - 1] & 0xFF) == SIMPLE_VALUE_UNDEFINED; } diff --git a/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java b/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java index 57262fedb..4b1454188 100644 --- a/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java +++ b/cbor/src/test/java/com/fasterxml/jackson/dataformat/cbor/parse/UndefinedValueTest.java @@ -14,7 +14,7 @@ // for [dataformat-binary#93] public class UndefinedValueTest extends CBORTestBase { - private final static byte BYTE_UNDEFINED = (byte) 0xF7; + private final static byte BYTE_UNDEFINED = (byte) CBORConstants.SIMPLE_VALUE_UNDEFINED; private final CBORFactory CBOR_F = cborFactory();