Skip to content

Commit 297a66b

Browse files
iifawziiifawzi
authored
Skip RS CTRL-CHAR to support JSON Text Sequence (RFC7464) (#1414)
1 parent a96327e commit 297a66b

File tree

11 files changed

+181
-5
lines changed

11 files changed

+181
-5
lines changed

‎release-notes/CREDITS-2.x

+8
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,14 @@ Haruki (@stackunderflow111)
470470
when custom characterEscape is used
471471
(2.18.3)
472472
473+
Yanming Zhou (@quaff)
474+
* Requested #633: Allow skipping `RS` CTRL-CHAR to support JSON Text Sequences
475+
(2.19.0)
476+
477+
Fawzi Essam (@iifawzi)
478+
* Contributed #633: Allow skipping `RS` CTRL-CHAR to support JSON Text Sequences
479+
(2.19.0)
480+
473481
Eduard Gomoliako (@Gems)
474482
* Contributed #1356: Make `JsonGenerator::writeTypePrefix` method to not write a
475483
`WRAPPER_ARRAY` when `typeIdDef.id == null`

‎release-notes/VERSION-2.x

+3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ a pure JSON library.
1616

1717
2.19.0 (not yet released)
1818

19+
#633: Allow skipping `RS` CTRL-CHAR to support JSON Text Sequences
20+
(requested by Yanming Z)
21+
(contributed by Fawzi E)
1922
#1328: Optimize handling of `JsonPointer.head()`
2023
#1356: Make `JsonGenerator::writeTypePrefix` method to not write a
2124
`WRAPPER_ARRAY` when `typeIdDef.id == null`

‎src/main/java/com/fasterxml/jackson/core/JsonParser.java

+6
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,12 @@ public enum Feature {
193193
@Deprecated
194194
ALLOW_UNQUOTED_CONTROL_CHARS(false),
195195

196+
/**
197+
* @deprecated Use {@link com.fasterxml.jackson.core.json.JsonReadFeature#ALLOW_RS_CONTROL_CHAR} instead
198+
*/
199+
@Deprecated // but due to technical reasons we need this entry too
200+
ALLOW_RS_CONTROL_CHAR(false),
201+
196202
/**
197203
* Feature that can be enabled to accept quoting of all character
198204
* using backslash quoting mechanism: if not enabled, only characters

‎src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java

+5
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public abstract class ParserMinimalBase extends JsonParser
3131
protected final static int INT_LF = '\n';
3232
protected final static int INT_CR = '\r';
3333
protected final static int INT_SPACE = 0x0020;
34+
protected final static int INT_RS = 0x001E;
3435

3536
// Markup
3637
protected final static int INT_LBRACKET = '[';
@@ -769,6 +770,10 @@ protected void reportUnexpectedNumberChar(int ch, String comment) throws JsonPar
769770
protected void _throwInvalidSpace(int i) throws JsonParseException {
770771
char c = (char) i;
771772
String msg = "Illegal character ("+_getCharDesc(c)+"): only regular white space (\\r, \\n, \\t) is allowed between tokens";
773+
774+
if (i == INT_RS) {
775+
msg += " (consider enabling `JsonReadFeature.ALLOW_RS_CONTROL_CHAR` feature to allow use of Record Separators (\\u001E).";
776+
}
772777
throw _constructReadException(msg);
773778
}
774779

‎src/main/java/com/fasterxml/jackson/core/json/JsonParserBase.java

+14
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ public abstract class JsonParserBase
2626
protected final static int FEAT_MASK_NON_NUM_NUMBERS = Feature.ALLOW_NON_NUMERIC_NUMBERS.getMask();
2727
@SuppressWarnings("deprecation")
2828
protected final static int FEAT_MASK_ALLOW_MISSING = Feature.ALLOW_MISSING_VALUES.getMask();
29+
@SuppressWarnings("deprecation")
30+
protected final static int FEAT_MASK_ALLOW_CTRL_RS = Feature.ALLOW_RS_CONTROL_CHAR.getMask();
31+
2932
protected final static int FEAT_MASK_ALLOW_SINGLE_QUOTES = Feature.ALLOW_SINGLE_QUOTES.getMask();
3033
protected final static int FEAT_MASK_ALLOW_UNQUOTED_NAMES = Feature.ALLOW_UNQUOTED_FIELD_NAMES.getMask();
3134
protected final static int FEAT_MASK_ALLOW_JAVA_COMMENTS = Feature.ALLOW_COMMENTS.getMask();
@@ -130,4 +133,15 @@ public final JsonLocation getCurrentLocation() {
130133
public final JsonLocation getTokenLocation() {
131134
return currentTokenLocation();
132135
}
136+
137+
/*
138+
/**********************************************************************
139+
/* Other helper methods
140+
/**********************************************************************
141+
*/
142+
143+
// @since 2.19
144+
protected boolean _isAllowedCtrlCharRS(int i) {
145+
return (i == INT_RS) && (_features & FEAT_MASK_ALLOW_CTRL_RS) != 0;
146+
}
133147
}

‎src/main/java/com/fasterxml/jackson/core/json/JsonReadFeature.java

+13
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@ public enum JsonReadFeature
8080
@SuppressWarnings("deprecation")
8181
ALLOW_UNESCAPED_CONTROL_CHARS(false, JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS),
8282

83+
/**
84+
* Feature that determines whether parser will allow
85+
* Record Separator (RS) control character ({@code 0x1E})
86+
* as part of ignorable whitespace in JSON input, similar to the TAB character.
87+
* <p>
88+
* Since the official JSON specification permits only a limited set of control
89+
* characters as whitespace, this is a non-standard feature and is disabled by default.
90+
*
91+
* @since 2.19
92+
*/
93+
@SuppressWarnings("deprecation")
94+
ALLOW_RS_CONTROL_CHAR(false, JsonParser.Feature.ALLOW_RS_CONTROL_CHAR),
95+
8396
/**
8497
* Feature that can be enabled to accept quoting of all character
8598
* using backslash quoting mechanism: if not enabled, only characters

‎src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -2504,7 +2504,7 @@ private final int _skipWSOrEnd() throws IOException
25042504
_currInputRowStart = _inputPtr;
25052505
} else if (i == INT_CR) {
25062506
_skipCR();
2507-
} else if (i != INT_TAB) {
2507+
} else if (i != INT_TAB && !_isAllowedCtrlCharRS(i)) {
25082508
_throwInvalidSpace(i);
25092509
}
25102510
}
@@ -2524,7 +2524,7 @@ private final int _skipWSOrEnd() throws IOException
25242524
_currInputRowStart = _inputPtr;
25252525
} else if (i == INT_CR) {
25262526
_skipCR();
2527-
} else if (i != INT_TAB) {
2527+
} else if (i != INT_TAB && !_isAllowedCtrlCharRS(i)) {
25282528
_throwInvalidSpace(i);
25292529
}
25302530
}

‎src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -3080,7 +3080,7 @@ private final int _skipWSOrEnd() throws IOException
30803080
_currInputRowStart = _inputPtr;
30813081
} else if (i == INT_CR) {
30823082
_skipCR();
3083-
} else if (i != INT_TAB) {
3083+
} else if (i != INT_TAB && !_isAllowedCtrlCharRS(i)) {
30843084
_throwInvalidSpace(i);
30853085
}
30863086
}
@@ -3100,7 +3100,7 @@ private final int _skipWSOrEnd() throws IOException
31003100
_currInputRowStart = _inputPtr;
31013101
} else if (i == INT_CR) {
31023102
_skipCR();
3103-
} else if (i != INT_TAB) {
3103+
} else if (i != INT_TAB && !_isAllowedCtrlCharRS(i)) {
31043104
_throwInvalidSpace(i);
31053105
}
31063106
}

‎src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ private final int _skipWS(int ch) throws IOException
943943
} else if (ch == INT_CR) {
944944
++_currInputRowAlt;
945945
_currInputRowStart = _inputPtr;
946-
} else if (ch != INT_TAB) {
946+
} else if (ch != INT_TAB && !_isAllowedCtrlCharRS(ch)) {
947947
_throwInvalidSpace(ch);
948948
}
949949
}

‎src/test/java/com/fasterxml/jackson/core/read/NonStandardAllowRSTest.java

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package com.fasterxml.jackson.core.read;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.fail;
5+
6+
import java.io.StringReader;
7+
import java.nio.charset.StandardCharsets;
8+
9+
import org.junit.jupiter.api.Test;
10+
11+
import com.fasterxml.jackson.core.*;
12+
import com.fasterxml.jackson.core.exc.StreamReadException;
13+
import com.fasterxml.jackson.core.json.JsonReadFeature;
14+
import com.fasterxml.jackson.core.json.async.NonBlockingJsonParser;
15+
16+
// for [core#633]: optionally allow Record-Separator ctrl char
17+
class NonStandardAllowRSTest
18+
extends JUnit5TestBase
19+
{
20+
@Test
21+
void recordSeparatorEnabled() throws Exception {
22+
doRecordSeparationTest(true);
23+
}
24+
25+
@Test
26+
void recordSeparatorDisabled() throws Exception {
27+
doRecordSeparationTest(false);
28+
}
29+
30+
// Testing record separation for all parser implementations
31+
private void doRecordSeparationTest(boolean recordSeparation) throws Exception {
32+
String contents = "{\"key\":true}\u001E";
33+
JsonFactory factory = JsonFactory.builder()
34+
.configure(JsonReadFeature.ALLOW_RS_CONTROL_CHAR, recordSeparation)
35+
.build();
36+
try (JsonParser parser = factory.createParser(contents)) {
37+
verifyRecordSeparation(parser, recordSeparation);
38+
}
39+
try (JsonParser parser = factory.createParser(new StringReader(contents))) {
40+
verifyRecordSeparation(parser, recordSeparation);
41+
}
42+
try (JsonParser parser = factory.createParser(contents.getBytes(StandardCharsets.UTF_8))) {
43+
verifyRecordSeparation(parser, recordSeparation);
44+
}
45+
try (NonBlockingJsonParser parser = (NonBlockingJsonParser) factory.createNonBlockingByteArrayParser()) {
46+
byte[] data = contents.getBytes(StandardCharsets.UTF_8);
47+
parser.feedInput(data, 0, data.length);
48+
parser.endOfInput();
49+
verifyRecordSeparation(parser, recordSeparation);
50+
}
51+
}
52+
53+
private void verifyRecordSeparation(JsonParser parser, boolean recordSeparation) throws Exception {
54+
try {
55+
assertToken(JsonToken.START_OBJECT, parser.nextToken());
56+
String field1 = parser.nextFieldName();
57+
assertEquals("key", field1);
58+
assertToken(JsonToken.VALUE_TRUE, parser.nextToken());
59+
assertToken(JsonToken.END_OBJECT, parser.nextToken());
60+
parser.nextToken(); // RS token
61+
if (!recordSeparation) {
62+
fail("Should have thrown an exception");
63+
}
64+
} catch (StreamReadException e) {
65+
if (!recordSeparation) {
66+
verifyException(e, "Illegal character ((CTRL-CHAR");
67+
verifyException(e, "consider enabling `JsonReadFeature.ALLOW_RS_CONTROL_CHAR`");
68+
} else {
69+
throw e;
70+
}
71+
}
72+
}
73+
}

‎src/test/java/com/fasterxml/jackson/core/read/ParserFeaturesTest.java

+54
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.junit.jupiter.api.Test;
44

55
import com.fasterxml.jackson.core.*;
6+
import com.fasterxml.jackson.core.exc.StreamReadException;
67
import com.fasterxml.jackson.core.json.JsonReadFeature;
78

89
import static org.junit.jupiter.api.Assertions.*;
@@ -57,6 +58,20 @@ void tabsEnabled() throws Exception
5758
_testTabsEnabled(true);
5859
}
5960

61+
@Test
62+
void recordSeparatorDefault() throws Exception
63+
{
64+
_testRecordSeparatorDefault(false);
65+
_testRecordSeparatorDefault(true);
66+
}
67+
68+
@Test
69+
void recordSeparatorEnabled() throws Exception
70+
{
71+
_testRecordSeparatorEnabled(false);
72+
_testRecordSeparatorEnabled(true);
73+
}
74+
6075
/*
6176
/****************************************************************
6277
/* Secondary test methods
@@ -134,4 +149,43 @@ private void _testTabsEnabled(boolean useStream) throws Exception
134149
assertToken(JsonToken.END_OBJECT, p.nextToken());
135150
p.close();
136151
}
152+
153+
private void _testRecordSeparatorDefault(boolean useStream) throws Exception {
154+
JsonFactory f = new JsonFactory();
155+
String JSON = "[\"val:\"]\u001E";
156+
157+
try (JsonParser p = useStream ? createParserUsingStream(f, JSON, "UTF-8") : createParserUsingReader(f, JSON)) {
158+
assertToken(JsonToken.START_ARRAY, p.nextToken());
159+
try {
160+
p.nextToken(); // val
161+
p.nextToken(); // ]
162+
p.nextToken(); // RS token
163+
fail("Expected exception");
164+
} catch (StreamReadException e) {
165+
verifyException(e, "Illegal character ((CTRL-CHAR");
166+
verifyException(e, "consider enabling `JsonReadFeature.ALLOW_RS_CONTROL_CHAR`");
167+
}
168+
}
169+
}
170+
171+
private void _testRecordSeparatorEnabled(boolean useStream) throws Exception
172+
{
173+
JsonFactory f = JsonFactory.builder()
174+
.configure(JsonReadFeature.ALLOW_RS_CONTROL_CHAR, true)
175+
.build();
176+
177+
String FIELD = "key";
178+
String VALUE = "value";
179+
String JSON = "{ "+q(FIELD)+" : "+q(VALUE)+"}\u001E";
180+
JsonParser p = useStream ? createParserUsingStream(f, JSON, "UTF-8") : createParserUsingReader(f, JSON);
181+
182+
assertToken(JsonToken.START_OBJECT, p.nextToken());
183+
assertToken(JsonToken.FIELD_NAME, p.nextToken());
184+
assertEquals(FIELD, p.getText());
185+
assertToken(JsonToken.VALUE_STRING, p.nextToken());
186+
assertEquals(VALUE, p.getText());
187+
assertToken(JsonToken.END_OBJECT, p.nextToken());
188+
p.nextToken(); // RS token
189+
p.close();
190+
}
137191
}

0 commit comments

Comments
 (0)