From b645731a8e4410b0a942ff9cc48902433c1c2206 Mon Sep 17 00:00:00 2001 From: ydah Date: Fri, 2 Oct 2026 20:28:26 +0900 Subject: [PATCH] Preserve complete UTF-8 characters in parse error snippets --- ext/json/ext/parser/parser.c | 16 +++++++--------- test/json/json_parser_test.rb | 27 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 5cba2feb..1fd2ae69 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -631,17 +631,15 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta } if (len) { - buffer[0] = '\''; - MEMCPY(buffer + 1, ptr, char, len); - - while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte - len--; - } - - if (buffer[len] >= 0xC0) { // multibyte character start - len--; + if (len == PARSE_ERROR_FRAGMENT_LEN) { + // Only trim when the byte limit splits a multibyte character. + while (len && (unsigned char)ptr[len] >= 0x80 && (unsigned char)ptr[len] < 0xC0) { + len--; + } } + buffer[0] = '\''; + MEMCPY(buffer + 1, ptr, char, len); buffer[len + 1] = '\''; buffer[len + 2] = '\0'; ptr = (const char *)buffer; diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 025bf671..94b48688 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -842,7 +842,7 @@ def test_parse_error_snippet assert_equal "unexpected character: 'aああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abあああああああああ' at line 1 column 1", error.message + assert_equal "unexpected character: 'abああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") } assert_equal "unexpected character: 'abcあああああああああ' at line 1 column 1", error.message @@ -855,6 +855,31 @@ def test_parse_error_snippet error = assert_raise(JSON::ParserError) { JSON.parse("@") } assert_equal "unexpected character: '@' at line 1 column 1", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse('"a" 日本') } + assert_equal "unexpected token at end of stream '日本' at line 1 column 5", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse('[1, é') } + assert_equal "unexpected character: 'é' at line 1 column 5", error.message + end + + def test_parse_error_snippet_multibyte_boundaries + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + ['é', '日', '😀'].each do |char| + ['', ' ', "\n", "\t", "\r"].each do |suffix| + error = assert_raise(JSON::ParserError) { JSON.parse(char + suffix) } + assert_equal "unexpected character: '#{char}' at line 1 column 1", error.message + end + + (28..32).each do |padding| + prefix = 'a' * padding + fragment = prefix + char * ((32 - padding) / char.bytesize) + error = assert_raise(JSON::ParserError) { JSON.parse(prefix + char * 20) } + assert_equal "unexpected character: '#{fragment}' at line 1 column 1", error.message + assert_predicate error.message, :valid_encoding? + end + end end def test_parse_error_json_path