From 074aeccbeca0cb1a6d35462b3e07fb3d5afb2036 Mon Sep 17 00:00:00 2001 From: John Revill Date: Wed, 4 Feb 2015 16:07:48 +0000 Subject: [PATCH 1/4] Captured issue 48 in a unit test ("JSONValue.parse does not correctly decode UTF-8 bytes from an inputstream", https://code.google.com/p/json-smart/issues/detail?id=48) --- .../net/minidev/json/test/TestIssue48.java | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 json-smart/src/test/java/net/minidev/json/test/TestIssue48.java diff --git a/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java b/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java new file mode 100644 index 0000000..97cd14d --- /dev/null +++ b/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java @@ -0,0 +1,99 @@ +package net.minidev.json.test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; + +import junit.framework.TestCase; +import net.minidev.json.JSONObject; +import net.minidev.json.JSONValue; +import net.minidev.json.parser.ContentHandler; +import net.minidev.json.parser.ParseException; + +/** + * Catches issue 48: "JSONValue.parse does not correctly decode UTF-8 bytes from an inputstream" + * mentioned in https://code.google.com/p/json-smart/issues/detail?id=48 + */ +public class TestIssue48 extends TestCase { + /** + * Covers the case described in https://code.google.com/p/json-smart/issues/detail?id=48 + */ + public void testIssue48() throws UnsupportedEncodingException { + final String testJsonString = "{\"balance\":1000.21,\"num\":100,\"nickname\":null,\"is_vip\":true,\"Sinhalese\":\"සිංහල ජාතිය\",\"name\":\"foo\"}"; + + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); + + final JSONObject obj = (JSONObject) JSONValue.parse(bis); + + final String actual = (String) obj.get("Sinhalese"); + + assertEquals("සිංහල ජාතිය", actual); + } + + /** + * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-8 + */ + public void testIssue48_SAXParse_InputStream_UTF8() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; + + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); + + JSONValue.SAXParse(bis, new AssertingHandler(originalText)); + } + + /** + * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-16 + */ + public void testIssue48_SAXParse_InputStream_UTF16() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; + + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-16")); + + JSONValue.SAXParse(bis, new AssertingHandler(originalText)); + } + + /** + * Covers the same issue seen when using SAXParse with a Reader source. + * + * A Reader doesn't seem to be affected by the issue, probably because it reads + * a character at a time, rather than a byte at a time. + */ + public void testIssue48_SAXParse_Reader() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; + + final Reader source = new StringReader(testJsonString); + + JSONValue.SAXParse(source, new AssertingHandler(originalText)); + } + + /** + * A ContentHandler that checks any parsed primitive against a single expected + * value and fails the test if it's not the same + */ + private static class AssertingHandler implements ContentHandler { + final String expectedPrimitive; + + public AssertingHandler(final String expectedPrimitive) { + this.expectedPrimitive = expectedPrimitive; + } + + public void startJSON() throws ParseException, IOException { } + public void endJSON() throws ParseException, IOException { } + public boolean startObject() throws ParseException, IOException { return true; } + public boolean endObject() throws ParseException, IOException { return true; } + public boolean startObjectEntry(String key) throws ParseException, IOException { return true; } + public boolean endObjectEntry() throws ParseException, IOException { return true; } + public boolean startArray() throws ParseException, IOException { return true; } + public boolean endArray() throws ParseException, IOException { return true; } + + public boolean primitive(final Object value) throws ParseException, IOException { + assertEquals(this.expectedPrimitive, value); + return true; + } + } +} From faaa95f9929702cd12240bdb906e482fe54edad2 Mon Sep 17 00:00:00 2001 From: John Revill Date: Wed, 4 Feb 2015 16:09:12 +0000 Subject: [PATCH 2/4] Fix for issue 48 ("JSONValue.parse does not correctly decode UTF-8 bytes from an inputstream", https://code.google.com/p/json-smart/issues/detail?id=48). Instead of taking a byte at a time from the InputStream, this now wraps the InputStream in an InputStreamReader and uses the existing JSONParserReader to read one character at a time. A few extra parse() methods were added to take the source bytes' character set as a parameter. This allows for cases when the platform's default character set is not wanted. --- .../json/parser/JSONParserInputStream.java | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java index 0432655..998f465 100644 --- a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java +++ b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java @@ -15,19 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import static net.minidev.json.parser.ParseException.ERROR_UNEXPECTED_EOF; - -import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; /** * Parser for JSON text. Please note that JSONParser is NOT thread-safe. * * @author Uriel Chemouni */ -class JSONParserInputStream extends JSONParserStream { - private InputStream in; - +class JSONParserInputStream extends JSONParserReader { // len public JSONParserInputStream(int permissiveMode) { super(permissiveMode); @@ -40,50 +37,56 @@ public JSONParserInputStream(int permissiveMode) { public Object parse(InputStream in) throws ParseException { return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER); } + + /** + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * + * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader + */ + public Object parse(InputStream in, String charset) throws ParseException, UnsupportedEncodingException { + return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER, charset); + } /** * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory + * + * NB: This uses the default charset */ public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException { return parse(in, containerFactory, ContentHandlerDumy.HANDLER); } + + /** + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * + * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader + */ + public Object parse(InputStream in, ContainerFactory containerFactory, String charset) throws ParseException, UnsupportedEncodingException { + return parse(in, containerFactory, ContentHandlerDumy.HANDLER, charset); + } /** * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory + * + * NB: This uses the default charset */ public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler) throws ParseException { - // - this.in = in; - this.pos = -1; - return super.parse(containerFactory, handler); - } - - protected void read() throws IOException { - int i = in.read(); - c = (i == -1) ? (char) EOI : (char) i; - pos++; - // - } - - protected void readS() throws IOException { - sb.append(c); - int i = in.read(); - if (i == -1) { - c = EOI; - } else { - c = (char) i; - pos++; - } - } - - protected void readNoEnd() throws ParseException, IOException { - int i = in.read(); - if (i == -1) - throw new ParseException(pos - 1, ERROR_UNEXPECTED_EOF, "EOF"); - c = (char) i; - // + return super.parse(new InputStreamReader(in)); } + + /** + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * + * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader + */ + public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler, String charset) + throws ParseException, UnsupportedEncodingException { + return super.parse(new InputStreamReader(in, charset)); + } } From 033817dfdd1c1b42adeef2985d57930bea030aa6 Mon Sep 17 00:00:00 2001 From: John Revill Date: Wed, 4 Feb 2015 16:14:20 +0000 Subject: [PATCH 3/4] Correcting indentation --- .../json/parser/JSONParserInputStream.java | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java index 998f465..e422879 100644 --- a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java +++ b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java @@ -39,14 +39,14 @@ public Object parse(InputStream in) throws ParseException { } /** - * use to return Primitive Type, or String, Or JsonObject or JsonArray - * generated by a ContainerFactory - * + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader - */ - public Object parse(InputStream in, String charset) throws ParseException, UnsupportedEncodingException { - return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER, charset); - } + */ + public Object parse(InputStream in, String charset) throws ParseException, UnsupportedEncodingException { + return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER, charset); + } /** * use to return Primitive Type, or String, Or JsonObject or JsonArray @@ -58,15 +58,15 @@ public Object parse(InputStream in, ContainerFactory containerFactory) throws Pa return parse(in, containerFactory, ContentHandlerDumy.HANDLER); } - /** - * use to return Primitive Type, or String, Or JsonObject or JsonArray - * generated by a ContainerFactory - * + /** + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader - */ - public Object parse(InputStream in, ContainerFactory containerFactory, String charset) throws ParseException, UnsupportedEncodingException { - return parse(in, containerFactory, ContentHandlerDumy.HANDLER, charset); - } + */ + public Object parse(InputStream in, ContainerFactory containerFactory, String charset) throws ParseException, UnsupportedEncodingException { + return parse(in, containerFactory, ContentHandlerDumy.HANDLER, charset); + } /** * use to return Primitive Type, or String, Or JsonObject or JsonArray @@ -79,14 +79,13 @@ public Object parse(InputStream in, ContainerFactory containerFactory, ContentHa return super.parse(new InputStreamReader(in)); } - /** - * use to return Primitive Type, or String, Or JsonObject or JsonArray - * generated by a ContainerFactory - * + /** + * use to return Primitive Type, or String, Or JsonObject or JsonArray + * generated by a ContainerFactory + * * @throws UnsupportedEncodingException If the named charset is not supported by an InputStreamReader - */ - public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler, String charset) - throws ParseException, UnsupportedEncodingException { - return super.parse(new InputStreamReader(in, charset)); - } + */ + public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler, String charset) throws ParseException, UnsupportedEncodingException { + return super.parse(new InputStreamReader(in, charset)); + } } From 14b5fef5f02eb445ee56e59072073b3f69e86223 Mon Sep 17 00:00:00 2001 From: John Revill Date: Wed, 4 Feb 2015 16:18:42 +0000 Subject: [PATCH 4/4] Correcting indentation --- .../net/minidev/json/test/TestIssue48.java | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java b/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java index 97cd14d..4744e4d 100644 --- a/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java +++ b/json-smart/src/test/java/net/minidev/json/test/TestIssue48.java @@ -17,83 +17,83 @@ * mentioned in https://code.google.com/p/json-smart/issues/detail?id=48 */ public class TestIssue48 extends TestCase { - /** - * Covers the case described in https://code.google.com/p/json-smart/issues/detail?id=48 - */ - public void testIssue48() throws UnsupportedEncodingException { - final String testJsonString = "{\"balance\":1000.21,\"num\":100,\"nickname\":null,\"is_vip\":true,\"Sinhalese\":\"සිංහල ජාතිය\",\"name\":\"foo\"}"; + /** + * Covers the case described in https://code.google.com/p/json-smart/issues/detail?id=48 + */ + public void testIssue48() throws UnsupportedEncodingException { + final String testJsonString = "{\"balance\":1000.21,\"num\":100,\"nickname\":null,\"is_vip\":true,\"Sinhalese\":\"සිංහල ජාතිය\",\"name\":\"foo\"}"; - final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); - final JSONObject obj = (JSONObject) JSONValue.parse(bis); + final JSONObject obj = (JSONObject) JSONValue.parse(bis); - final String actual = (String) obj.get("Sinhalese"); + final String actual = (String) obj.get("Sinhalese"); - assertEquals("සිංහල ජාතිය", actual); - } + assertEquals("සිංහල ජාතිය", actual); + } - /** - * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-8 - */ - public void testIssue48_SAXParse_InputStream_UTF8() throws Exception { - final String originalText = "僧伽罗人"; - final String testJsonString = "[ \"" + originalText + "\" ]"; + /** + * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-8 + */ + public void testIssue48_SAXParse_InputStream_UTF8() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; - final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-8")); - JSONValue.SAXParse(bis, new AssertingHandler(originalText)); + JSONValue.SAXParse(bis, new AssertingHandler(originalText)); } - /** - * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-16 - */ - public void testIssue48_SAXParse_InputStream_UTF16() throws Exception { - final String originalText = "僧伽罗人"; - final String testJsonString = "[ \"" + originalText + "\" ]"; + /** + * Covers the same issue seen when using SAXParse with an InputStream source drawing on bytes of UTF-16 + */ + public void testIssue48_SAXParse_InputStream_UTF16() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; - final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-16")); + final ByteArrayInputStream bis = new ByteArrayInputStream(testJsonString.getBytes("UTF-16")); - JSONValue.SAXParse(bis, new AssertingHandler(originalText)); - } + JSONValue.SAXParse(bis, new AssertingHandler(originalText)); + } - /** - * Covers the same issue seen when using SAXParse with a Reader source. - * - * A Reader doesn't seem to be affected by the issue, probably because it reads - * a character at a time, rather than a byte at a time. - */ - public void testIssue48_SAXParse_Reader() throws Exception { - final String originalText = "僧伽罗人"; - final String testJsonString = "[ \"" + originalText + "\" ]"; + /** + * Covers the same issue seen when using SAXParse with a Reader source. + * + * A Reader doesn't seem to be affected by the issue, probably because it reads + * a character at a time, rather than a byte at a time. + */ + public void testIssue48_SAXParse_Reader() throws Exception { + final String originalText = "僧伽罗人"; + final String testJsonString = "[ \"" + originalText + "\" ]"; - final Reader source = new StringReader(testJsonString); + final Reader source = new StringReader(testJsonString); - JSONValue.SAXParse(source, new AssertingHandler(originalText)); - } + JSONValue.SAXParse(source, new AssertingHandler(originalText)); + } - /** - * A ContentHandler that checks any parsed primitive against a single expected - * value and fails the test if it's not the same - */ - private static class AssertingHandler implements ContentHandler { - final String expectedPrimitive; + /** + * A ContentHandler that checks any parsed primitive against a single expected + * value and fails the test if it's not the same + */ + private static class AssertingHandler implements ContentHandler { + final String expectedPrimitive; - public AssertingHandler(final String expectedPrimitive) { - this.expectedPrimitive = expectedPrimitive; - } + public AssertingHandler(final String expectedPrimitive) { + this.expectedPrimitive = expectedPrimitive; + } - public void startJSON() throws ParseException, IOException { } - public void endJSON() throws ParseException, IOException { } - public boolean startObject() throws ParseException, IOException { return true; } - public boolean endObject() throws ParseException, IOException { return true; } - public boolean startObjectEntry(String key) throws ParseException, IOException { return true; } - public boolean endObjectEntry() throws ParseException, IOException { return true; } - public boolean startArray() throws ParseException, IOException { return true; } - public boolean endArray() throws ParseException, IOException { return true; } + public void startJSON() throws ParseException, IOException { } + public void endJSON() throws ParseException, IOException { } + public boolean startObject() throws ParseException, IOException { return true; } + public boolean endObject() throws ParseException, IOException { return true; } + public boolean startObjectEntry(String key) throws ParseException, IOException { return true; } + public boolean endObjectEntry() throws ParseException, IOException { return true; } + public boolean startArray() throws ParseException, IOException { return true; } + public boolean endArray() throws ParseException, IOException { return true; } - public boolean primitive(final Object value) throws ParseException, IOException { - assertEquals(this.expectedPrimitive, value); - return true; - } - } + public boolean primitive(final Object value) throws ParseException, IOException { + assertEquals(this.expectedPrimitive, value); + return true; + } + } }