Skip to content

Commit 211cf2c

Browse files
pjfanningpjfanning
authored
Honor maxErrorTokenLength in UTF8DataInputJsonParser [GHSA-7hhh-6rmp-j9qf] (#1698)
1 parent e7acd64 commit 211cf2c

7 files changed

Lines changed: 140 additions & 10 deletions

File tree

‎release-notes/VERSION-2.x‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ a pure JSON library.
1818

1919
#1649: Optimize `NumberInput.looksLikeValidNumber`
2020
(fix by @cowtowncoder, w/ Claude code)
21+
#1698: `UTF8DataInputJsonParser` does not honor `maxErrorTokenLength`
22+
when reporting an unrecognized token
23+
(fix by @pjfanning)
2124

2225
2.18.10 (15-Aug-2026)
2326

‎src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3018,14 +3018,15 @@ protected void _reportInvalidToken(String matchedPart, String msg) throws IOExce
30183018
* nothing fancy here.
30193019
*/
30203020
StringBuilder sb = new StringBuilder(matchedPart);
3021+
final int maxTokenLength = _ioContext.errorReportConfiguration().getMaxErrorTokenLength();
30213022
while ((_inputPtr < _inputEnd) || _loadMore()) {
30223023
char c = _inputBuffer[_inputPtr];
30233024
if (!Character.isJavaIdentifierPart(c)) {
30243025
break;
30253026
}
30263027
++_inputPtr;
30273028
sb.append(c);
3028-
if (sb.length() >= _ioContext.errorReportConfiguration().getMaxErrorTokenLength()) {
3029+
if (sb.length() >= maxTokenLength) {
30293030
sb.append("...");
30303031
break;
30313032
}

‎src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2132,8 +2132,14 @@ protected JsonToken _handleUnexpectedValue(int c)
21322132
return _handleInvalidNumberStart(_inputData.readUnsignedByte(), false, true);
21332133
}
21342134
// [core#77] Try to decode most likely token
2135-
if (Character.isJavaIdentifierStart(c)) {
2136-
_reportInvalidToken(c, ""+((char) c), _validJsonTokenList());
2135+
if (c > 0x7F) { // multi-byte UTF-8 char: decode first (consumes rest of its bytes)
2136+
c = _decodeCharForError(c);
2137+
if (Character.isJavaIdentifierStart(c)) {
2138+
_reportInvalidToken(_inputData.readUnsignedByte(), ""+((char) c), _validJsonTokenList());
2139+
}
2140+
} else if (Character.isJavaIdentifierStart(c)) {
2141+
// NOTE: 'c' is decoded (and appended) by _reportInvalidToken(); do not pre-append
2142+
_reportInvalidToken(c, "", _validJsonTokenList());
21372143
}
21382144
// but if it doesn't look like a token:
21392145
_reportUnexpectedChar(c, "expected a valid value "+_validJsonValueList());
@@ -2274,7 +2280,9 @@ private final void _checkMatchEnd(String matchStr, int i, int ch) throws IOExcep
22742280
// but actually only alphanums are problematic
22752281
char c = (char) _decodeCharForError(ch);
22762282
if (Character.isJavaIdentifierPart(c)) {
2277-
_reportInvalidToken(c, matchStr.substring(0, i));
2283+
// 'c' already decoded (all of its bytes consumed): include it as matched,
2284+
// continue from the following byte
2285+
_reportInvalidToken(_inputData.readUnsignedByte(), matchStr.substring(0, i) + c);
22782286
}
22792287
}
22802288

@@ -2765,6 +2773,7 @@ protected void _reportInvalidToken(int ch, String matchedPart, String msg)
27652773
throws IOException
27662774
{
27672775
StringBuilder sb = new StringBuilder(matchedPart);
2776+
final int maxTokenLength = _ioContext.errorReportConfiguration().getMaxErrorTokenLength();
27682777

27692778
/* Let's just try to find what appears to be the token, using
27702779
* regular Java identifier character rules. It's just a heuristic,
@@ -2776,6 +2785,10 @@ protected void _reportInvalidToken(int ch, String matchedPart, String msg)
27762785
break;
27772786
}
27782787
sb.append(c);
2788+
if (sb.length() >= maxTokenLength) {
2789+
sb.append("...");
2790+
break;
2791+
}
27792792
ch = _inputData.readUnsignedByte();
27802793
}
27812794
_reportError("Unrecognized token '"+sb.toString()+"': was expecting "+msg);

‎src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2745,6 +2745,9 @@ protected JsonToken _handleUnexpectedValue(int c) throws IOException
27452745
return _handleInvalidNumberStart(_inputBuffer[_inputPtr++] & 0xFF, false, true);
27462746
}
27472747
// [core#77] Try to decode most likely token
2748+
if (c > 0x7F) { // multi-byte UTF-8 char: decode first (consumes rest of its bytes)
2749+
c = _decodeCharForError(c);
2750+
}
27482751
if (Character.isJavaIdentifierStart(c)) {
27492752
_reportInvalidToken(""+((char) c), _validJsonTokenList());
27502753
}
@@ -2990,10 +2993,25 @@ private final void _matchToken2(String matchStr, int i) throws IOException
29902993

29912994
private final void _checkMatchEnd(String matchStr, int i, int ch) throws IOException {
29922995
// but actually only alphanums are problematic
2996+
if (ch < 0x80) { // single-byte char: can check without consuming it
2997+
if (Character.isJavaIdentifierPart((char) ch)) {
2998+
_reportInvalidToken(matchStr.substring(0, i));
2999+
}
3000+
return;
3001+
}
3002+
// Multi-byte char: must consume lead byte (decoding consumes the rest)
3003+
final int ptr = _inputPtr++;
3004+
final long processed = _currInputProcessed;
29933005
char c = (char) _decodeCharForError(ch);
29943006
if (Character.isJavaIdentifierPart(c)) {
2995-
_reportInvalidToken(matchStr.substring(0, i));
3007+
_reportInvalidToken(matchStr.substring(0, i) + c);
3008+
}
3009+
// Not part of token: rewind so regular handling reports it -- unless
3010+
// buffer was reloaded during decoding, in which case must report here
3011+
if (_currInputProcessed != processed) {
3012+
_reportUnexpectedChar(c, "expected white space, comma or end marker after token '"+matchStr+"'");
29963013
}
3014+
_inputPtr = ptr;
29973015
}
29983016

29993017
/*
@@ -3642,6 +3660,7 @@ protected void _reportInvalidToken(String matchedPart, String msg) throws IOExce
36423660
* nothing fancy here (nor fast).
36433661
*/
36443662
StringBuilder sb = new StringBuilder(matchedPart);
3663+
final int maxTokenLength = _ioContext.errorReportConfiguration().getMaxErrorTokenLength();
36453664
while ((_inputPtr < _inputEnd) || _loadMore()) {
36463665
int i = _inputBuffer[_inputPtr++];
36473666
char c = (char) _decodeCharForError(i);
@@ -3654,7 +3673,7 @@ protected void _reportInvalidToken(String matchedPart, String msg) throws IOExce
36543673
break;
36553674
}
36563675
sb.append(c);
3657-
if (sb.length() >= _ioContext.errorReportConfiguration().getMaxErrorTokenLength()) {
3676+
if (sb.length() >= maxTokenLength) {
36583677
sb.append("...");
36593678
break;
36603679
}

‎src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingUtf8JsonParserBase.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1235,6 +1235,7 @@ protected JsonToken _finishNonStdTokenWithEOF(int type, int matched) throws IOEx
12351235

12361236
protected JsonToken _finishErrorToken() throws IOException
12371237
{
1238+
final int maxTokenLength = _ioContext.errorReportConfiguration().getMaxErrorTokenLength();
12381239
while (_inputPtr < _inputEnd) {
12391240
int i = getNextSignedByteFromBuffer();
12401241

@@ -1246,7 +1247,7 @@ protected JsonToken _finishErrorToken() throws IOException
12461247
// 11-Jan-2016, tatu: note: we will fully consume the character,
12471248
// included or not, so if recovery was possible, it'd be off-by-one...
12481249
_textBuffer.append(ch);
1249-
if (_textBuffer.size() < _ioContext.errorReportConfiguration().getMaxErrorTokenLength()) {
1250+
if (_textBuffer.size() < maxTokenLength) {
12501251
continue;
12511252
}
12521253
}

‎src/test/java/com/fasterxml/jackson/core/ErrorReportConfigurationTest.java‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,57 @@ void expectedTokenLengthWithConfigurations()
216216
}
217217
}
218218

219+
// [core#1698]: every parser backend must honor `maxErrorTokenLength`;
220+
// `UTF8DataInputJsonParser` used to accumulate the whole token instead
221+
@Test
222+
void errorTokenLengthBoundedInAllModes()
223+
throws Exception
224+
{
225+
final int maxLen = 256;
226+
final JsonFactory f = streamFactoryBuilder()
227+
.errorReportConfiguration(ErrorReportConfiguration.builder()
228+
.maxErrorTokenLength(maxLen).build())
229+
.build();
230+
// Broken token far longer than the limit: must be truncated, not accumulated in full
231+
final String doc = _buildBrokenJsonOfLength(50 * maxLen);
232+
// limit, plus appended "..."
233+
final String expToken = _brokenToken(maxLen) + "...";
234+
235+
for (int mode : ALL_MODES) {
236+
try (JsonParser p = createParser(f, mode, doc)) {
237+
p.nextToken();
238+
p.nextToken();
239+
fail("Should not pass, mode: "+mode);
240+
} catch (JsonProcessingException e) {
241+
assertThat(_unrecognizedToken(e.getMessage()))
242+
.as("mode: %d", mode)
243+
.isEqualTo(expToken);
244+
}
245+
}
246+
}
247+
248+
// Short broken token (below limit) must be reported verbatim, without
249+
// duplicated leading char, in all modes
250+
@Test
251+
void shortErrorTokenReportedExactlyInAllModes()
252+
throws Exception
253+
{
254+
final JsonFactory f = newStreamFactory();
255+
final String doc = "{\"key\":abc!}";
256+
257+
for (int mode : ALL_MODES) {
258+
try (JsonParser p = createParser(f, mode, doc)) {
259+
p.nextToken();
260+
p.nextToken();
261+
fail("Should not pass, mode: "+mode);
262+
} catch (JsonProcessingException e) {
263+
assertThat(_unrecognizedToken(e.getMessage()))
264+
.as("mode: %d", mode)
265+
.isEqualTo("abc");
266+
}
267+
}
268+
}
269+
219270
@Test
220271
void nonPositiveErrorTokenConfig()
221272
{
@@ -318,13 +369,29 @@ private void _testWithMaxErrorTokenLength(int expectedSize, int tokenLen, JsonFa
318369
}
319370
}
320371

372+
// Extracts X from "Unrecognized token 'X': was expecting ..."
373+
private String _unrecognizedToken(String msg)
374+
{
375+
final String prefix = "Unrecognized token '";
376+
final int start = msg.indexOf(prefix);
377+
assertThat(start).as("message: %s", msg).isGreaterThanOrEqualTo(0);
378+
final int end = msg.indexOf("': was expecting", start);
379+
assertThat(end).as("message: %s", msg).isGreaterThan(start);
380+
return msg.substring(start + prefix.length(), end);
381+
}
382+
321383
private String _buildBrokenJsonOfLength(int len)
322384
{
323-
StringBuilder sb = new StringBuilder("{\"key\":");
385+
return "{\"key\":" + _brokenToken(len) + "!}";
386+
}
387+
388+
// Varied (not repeating single) chars so that duplicated/dropped chars are detectable
389+
private String _brokenToken(int len)
390+
{
391+
StringBuilder sb = new StringBuilder(len);
324392
for (int i = 0; i < len; i++) {
325-
sb.append("a");
393+
sb.append((char) ('a' + (i % 26)));
326394
}
327-
sb.append("!}");
328395
return sb.toString();
329396
}
330397
}

‎src/test/java/com/fasterxml/jackson/core/read/ParserErrorHandlingTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,21 @@ private void _testInvalidKeywords(int mode) throws Exception
6868
doTestInvalidKeyword1(mode, "treu");
6969
doTestInvalidKeyword1(mode, "trueenough");
7070
doTestInvalidKeyword1(mode, "C");
71+
72+
// Multi-byte (non-ASCII) identifier chars right after keyword
73+
doTestInvalidKeyword1(mode, "trueé");
74+
doTestInvalidKeyword1(mode, "nullé");
75+
doTestInvalidKeyword1(mode, "false中x");
76+
77+
// Multi-byte (non-ASCII) identifier char at start of token
78+
doTestInvalidKeyword1(mode, "éabc");
79+
doTestInvalidKeyword1(mode, "中x");
80+
81+
// Multi-byte char that is NOT part of token (NBSP): after keyword, or at value start
82+
_testNonTokenChar(mode, "[true\u00A0]");
83+
_testNonTokenChar(mode, "{\"a\":null\u00A0}");
84+
_testNonTokenChar(mode, "[\u00A0]");
85+
_testNonTokenChar(mode, "{\"a\":\u00A0}");
7186
}
7287

7388
private void doTestInvalidKeyword1(int mode, String value)
@@ -103,6 +118,17 @@ private void doTestInvalidKeyword1(int mode, String value)
103118
}
104119
}
105120

121+
// Must be reported as an error: not skipped, nor mis-decoded
122+
private void _testNonTokenChar(int mode, String doc) throws IOException
123+
{
124+
try (JsonParser p = createParser(JSON_F, mode, doc)) {
125+
while (p.nextToken() != null) { }
126+
fail("Expected an exception for invalid non-token char; doc: "+doc);
127+
} catch (JsonParseException jex) {
128+
verifyException(jex, "Unexpected character");
129+
}
130+
}
131+
106132
private void _testMangledNumbersInt(int mode) throws Exception
107133
{
108134
JsonParser p = createParser(JSON_F, mode, "123true");

0 commit comments

Comments
 (0)