diff --git a/pom.xml b/pom.xml
index b3f817c0..4840503f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
com.jsoniter
- 0.9.23
+ 0.9.24-SNAPSHOT
jsoniter
json iterator
jsoniter (json-iterator) is fast and flexible JSON parser available in Java and Go
diff --git a/src/main/java/com/jsoniter/Codegen.java b/src/main/java/com/jsoniter/Codegen.java
index ad2013d5..7cf7318d 100644
--- a/src/main/java/com/jsoniter/Codegen.java
+++ b/src/main/java/com/jsoniter/Codegen.java
@@ -96,16 +96,12 @@ private static void addPlaceholderDecoderToSupportRecursiveStructure(final Strin
public Object decode(JsonIterator iter) throws IOException {
Decoder decoder = JsoniterSpi.getDecoder(cacheKey);
if (this == decoder) {
- for(int i = 0; i < 30; i++) {
+ for(int i = 0; (i < 30) && (this == decoder); i++) {
decoder = JsoniterSpi.getDecoder(cacheKey);
- if (this == decoder) {
- try {
- Thread.sleep(1000);
+ try {
+ Thread.sleep(1000);
} catch (InterruptedException e) {
throw new JsonException(e);
- }
- } else {
- break;
}
}
if (this == decoder) {
diff --git a/src/main/java/com/jsoniter/IterImpl.java b/src/main/java/com/jsoniter/IterImpl.java
index 1cd9515e..ad779fd8 100644
--- a/src/main/java/com/jsoniter/IterImpl.java
+++ b/src/main/java/com/jsoniter/IterImpl.java
@@ -59,7 +59,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
case '[': // If open symbol, increase level
level++;
break;
- case ']': // If close symbol, increase level
+ case ']': // If close symbol, decrease level
level--;
// If we have returned to the original level, we're done
@@ -85,7 +85,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
case '{': // If open symbol, increase level
level++;
break;
- case '}': // If close symbol, increase level
+ case '}': // If close symbol, decrease level
level--;
// If we have returned to the original level, we're done
diff --git a/src/main/java/com/jsoniter/IterImplForStreaming.java b/src/main/java/com/jsoniter/IterImplForStreaming.java
index a2802cc7..2cef3a16 100644
--- a/src/main/java/com/jsoniter/IterImplForStreaming.java
+++ b/src/main/java/com/jsoniter/IterImplForStreaming.java
@@ -71,7 +71,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
case '[': // If open symbol, increase level
level++;
break;
- case ']': // If close symbol, increase level
+ case ']': // If close symbol, decrease level
level--;
// If we have returned to the original level, we're done
@@ -101,7 +101,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
case '{': // If open symbol, increase level
level++;
break;
- case '}': // If close symbol, increase level
+ case '}': // If close symbol, decrease level
level--;
// If we have returned to the original level, we're done
@@ -147,7 +147,8 @@ final static void skipString(JsonIterator iter) throws IOException {
throw iter.reportError("skipString", "incomplete string");
}
if (escaped) {
- iter.head = 1; // skip the first char as last char is \
+ // TODO add unit test to prove/verify bug
+ iter.head += 1; // skip the first char as last char is \
}
} else {
iter.head = end;
@@ -274,19 +275,19 @@ public final static boolean loadMore(JsonIterator iter) throws IOException {
}
private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOException {
- int n;
- int offset;
- if (iter.skipStartedAt == 0 || iter.skipStartedAt < iter.tail / 2) {
- byte[] newBuf = new byte[iter.buf.length * 2];
- offset = iter.tail - iter.skipStartedAt;
- System.arraycopy(iter.buf, iter.skipStartedAt, newBuf, 0, offset);
- iter.buf = newBuf;
- n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
- } else {
- offset = iter.tail - iter.skipStartedAt;
- System.arraycopy(iter.buf, iter.skipStartedAt, iter.buf, 0, offset);
- n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
- }
+ int offset = iter.tail - iter.skipStartedAt;
+ byte[] srcBuffer = iter.buf;
+ // Check there is no unused buffer capacity
+ if ((getUnusedBufferByteCount(iter)) == 0) {
+ // If auto expand buffer enabled, then create larger buffer
+ if (iter.autoExpandBufferStep > 0) {
+ iter.buf = new byte[iter.buf.length + iter.autoExpandBufferStep];
+ } else {
+ throw iter.reportError("loadMore", String.format("buffer is full and autoexpansion is disabled. tail: [%s] skipStartedAt: [%s]", iter.tail, iter.skipStartedAt));
+ }
+ }
+ System.arraycopy(srcBuffer, iter.skipStartedAt, iter.buf, 0, offset);
+ int n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
iter.skipStartedAt = 0;
if (n < 1) {
if (n == -1) {
@@ -301,6 +302,11 @@ private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOExce
return true;
}
+ private static int getUnusedBufferByteCount(JsonIterator iter) {
+ // Get bytes from 0 to skipStart + from tail till end
+ return iter.buf.length - iter.tail + iter.skipStartedAt;
+ }
+
final static byte readByte(JsonIterator iter) throws IOException {
if (iter.head == iter.tail) {
if (!loadMore(iter)) {
@@ -643,8 +649,7 @@ static final int readInt(final JsonIterator iter, final byte c) throws IOExcepti
static void assertNotLeadingZero(JsonIterator iter) throws IOException {
try {
- byte nextByte = IterImpl.readByte(iter);
- iter.unreadByte();
+ byte nextByte = iter.buf[iter.head];
int ind2 = IterImplNumber.intDigits[nextByte];
if (ind2 == IterImplNumber.INVALID_CHAR_FOR_NUMBER) {
return;
diff --git a/src/main/java/com/jsoniter/JsonIterator.java b/src/main/java/com/jsoniter/JsonIterator.java
index 1f8d077e..c198540b 100644
--- a/src/main/java/com/jsoniter/JsonIterator.java
+++ b/src/main/java/com/jsoniter/JsonIterator.java
@@ -21,6 +21,9 @@ public class JsonIterator implements Closeable {
final static ValueType[] valueTypes = new ValueType[256];
InputStream in;
byte[] buf;
+ // Whenever buf is not large enough new one is created with size of
+ // buf.length + autoExpandBufferStep. Set to < 1 to disable auto expanding.
+ int autoExpandBufferStep;
int head;
int tail;
int skipStartedAt = -1; // skip should keep bytes starting at this pos
@@ -60,13 +63,22 @@ private JsonIterator(InputStream in, byte[] buf, int head, int tail) {
this.tail = tail;
}
+ private JsonIterator(InputStream in, byte[] buf, int autoExpandBufferStep) {
+ this(in, buf, 0, 0);
+ this.autoExpandBufferStep = autoExpandBufferStep;
+ }
+
public JsonIterator() {
this(null, new byte[0], 0, 0);
}
public static JsonIterator parse(InputStream in, int bufSize) {
+ return parse(in, bufSize, bufSize);
+ }
+
+ public static JsonIterator parse(InputStream in, int bufSize, int autoExpandBufferStep) {
enableStreamingSupport();
- return new JsonIterator(in, new byte[bufSize], 0, 0);
+ return new JsonIterator(in, new byte[bufSize], autoExpandBufferStep);
}
public static JsonIterator parse(byte[] buf) {
diff --git a/src/main/java/com/jsoniter/output/JsonStream.java b/src/main/java/com/jsoniter/output/JsonStream.java
index 88f77077..7886bc05 100644
--- a/src/main/java/com/jsoniter/output/JsonStream.java
+++ b/src/main/java/com/jsoniter/output/JsonStream.java
@@ -442,52 +442,59 @@ public static void serialize(TypeLiteral typeLiteral, Object obj, OutputStream o
}
public static void serialize(Type type, Object obj, OutputStream out) {
- JsonStream stream = JsonStreamPool.borrowJsonStream();
- try {
- try {
- stream.reset(out);
- stream.writeVal(type, obj);
- } finally {
- stream.close();
- }
- } catch (IOException e) {
- throw new JsonException(e);
- } finally {
- JsonStreamPool.returnJsonStream(stream);
- }
+ serialize(type, obj, out, false);
}
public static String serialize(Config config, Object obj) {
- JsoniterSpi.setCurrentConfig(config);
- try {
- return serialize(config.escapeUnicode(), obj.getClass(), obj);
- } finally {
- JsoniterSpi.clearCurrentConfig();
- }
+ return serialize(config, obj.getClass(), obj);
}
public static String serialize(Object obj) {
- return serialize(JsoniterSpi.getCurrentConfig().escapeUnicode(), obj.getClass(), obj);
+ return serialize(obj.getClass(), obj);
}
public static String serialize(Config config, TypeLiteral typeLiteral, Object obj) {
+ return serialize(config, typeLiteral.getType(), obj);
+ }
+
+ private static String serialize(Config config, Type type, Object obj) {
+ final Config configBackup = JsoniterSpi.getCurrentConfig();
+ // Set temporary config
JsoniterSpi.setCurrentConfig(config);
try {
- return serialize(config.escapeUnicode(), typeLiteral.getType(), obj);
+ return serialize(type, obj);
} finally {
- JsoniterSpi.clearCurrentConfig();
+ // Revert old config
+ JsoniterSpi.setCurrentConfig(configBackup);
}
}
public static String serialize(TypeLiteral typeLiteral, Object obj) {
- return serialize(JsoniterSpi.getCurrentConfig().escapeUnicode(), typeLiteral.getType(), obj);
+ return serialize(typeLiteral.getType(), obj);
}
public static String serialize(boolean escapeUnicode, Type type, Object obj) {
- JsonStream stream = JsonStreamPool.borrowJsonStream();
+ final Config currentConfig = JsoniterSpi.getCurrentConfig();
+ return serialize(currentConfig.copyBuilder().escapeUnicode(escapeUnicode).build(), type, obj);
+ }
+
+ private static String serialize(Type type, Object obj) {
+ return serialize(type, obj, null, true);
+ }
+
+ private static String serialize(Type type, Object obj, OutputStream out, boolean returnObjAsString) {
+ final JsonStream stream = JsonStreamPool.borrowJsonStream();
+ final boolean escapeUnicode = JsoniterSpi.getCurrentConfig().escapeUnicode();
try {
- stream.reset(null);
- stream.writeVal(type, obj);
+ try {
+ stream.reset(out);
+ stream.writeVal(type, obj);
+ } finally {
+ stream.close();
+ }
+ if (!returnObjAsString) {
+ return "";
+ }
if (escapeUnicode) {
return new String(stream.buf, 0, stream.count);
} else {
diff --git a/src/main/java/com/jsoniter/output/StreamImplString.java b/src/main/java/com/jsoniter/output/StreamImplString.java
index 7116f359..7c4a27a7 100644
--- a/src/main/java/com/jsoniter/output/StreamImplString.java
+++ b/src/main/java/com/jsoniter/output/StreamImplString.java
@@ -48,7 +48,7 @@ class StreamImplString {
static {
for (int i = 0; i < CAN_DIRECT_WRITE.length; i++) {
- if (i > 31 && i < 126 && i != '"' && i != '\\') {
+ if (i > 31 && i <= 126 && i != '"' && i != '\\') {
CAN_DIRECT_WRITE[i] = true;
}
}
@@ -132,7 +132,7 @@ private static void writeStringSlowPath(JsonStream stream, String val, int i, in
if (escapeUnicode) {
for (; i < valLen; i++) {
int c = val.charAt(i);
- if (c > 125) {
+ if (c > 127) {
writeAsSlashU(stream, c);
} else {
writeAsciiChar(stream, c);
@@ -147,7 +147,7 @@ private static void writeStringSlowPathWithoutEscapeUnicode(JsonStream stream, S
int _surrogate;
for (; i < valLen; i++) {
int c = val.charAt(i);
- if (c > 125) {
+ if (c > 127) {
if (c < 0x800) { // 2-byte
stream.write(
(byte) (0xc0 | (c >> 6)),
diff --git a/src/main/java/com/jsoniter/spi/JsoniterSpi.java b/src/main/java/com/jsoniter/spi/JsoniterSpi.java
index 7f505e1a..4b40e77e 100644
--- a/src/main/java/com/jsoniter/spi/JsoniterSpi.java
+++ b/src/main/java/com/jsoniter/spi/JsoniterSpi.java
@@ -43,6 +43,7 @@ public static void setCurrentConfig(Config val) {
currentConfig.set(val);
}
+ // TODO usage of this method leads to potentially unexpected side effects. All usage should be checked.
public static void clearCurrentConfig() {
currentConfig.set(defaultConfig);
}
diff --git a/src/test/java/com/jsoniter/IterImplForStreamingTest.java b/src/test/java/com/jsoniter/IterImplForStreamingTest.java
index c190d469..e0432d39 100644
--- a/src/test/java/com/jsoniter/IterImplForStreamingTest.java
+++ b/src/test/java/com/jsoniter/IterImplForStreamingTest.java
@@ -1,6 +1,13 @@
package com.jsoniter;
+import com.jsoniter.any.Any;
+import com.jsoniter.spi.JsonException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import junit.framework.TestCase;
+import org.junit.experimental.categories.Category;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
public class IterImplForStreamingTest extends TestCase {
@@ -11,4 +18,76 @@ public void testReadMaxDouble() throws Exception {
String number = new String(numberChars.chars, 0, numberChars.charsLength);
assertEquals(maxDouble, number);
}
-}
\ No newline at end of file
+
+ @Category(StreamingCategory.class)
+ public void testLoadMore() throws IOException {
+ final String originalContent = "1234567890";
+ final byte[] src = ("{\"a\":\"" + originalContent + "\"}").getBytes();
+
+ int initialBufferSize;
+ Any parsedString;
+ // Case #1: Data fits into initial buffer, autoresizing on
+ // Input must definitely fit into such large buffer
+ initialBufferSize = src.length * 2;
+ JsonIterator jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 512);
+ jsonIterator.readObject();
+ parsedString = jsonIterator.readAny();
+ assertEquals(originalContent, parsedString.toString());
+ // Check buffer was not expanded
+ assertEquals(initialBufferSize, jsonIterator.buf.length);
+
+ // Case #2: Data does not fit into initial buffer, autoresizing off
+ initialBufferSize = originalContent.length() / 2;
+ jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 0);
+ jsonIterator.readObject();
+ try {
+ jsonIterator.readAny();
+ fail("Expect to fail because buffer is too small.");
+ } catch (JsonException e) {
+ if (!e.getMessage().startsWith("loadMore")) {
+ throw e;
+ }
+ }
+ // Check buffer was not expanded
+ assertEquals(initialBufferSize, jsonIterator.buf.length);
+
+ // Case #3: Data does fit into initial buffer, autoresizing on
+ initialBufferSize = originalContent.length() / 2;
+ int autoExpandBufferStep = initialBufferSize * 3;
+ jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, autoExpandBufferStep);
+ jsonIterator.readObject();
+ parsedString = jsonIterator.readAny();
+ assertEquals(originalContent, parsedString.toString());
+ // Check buffer was expanded exactly once
+ assertEquals(initialBufferSize + autoExpandBufferStep, jsonIterator.buf.length);
+
+ // Case #4: Data does not fit (but largest string does) into initial buffer, autoresizing on
+ initialBufferSize = originalContent.length() + 2;
+ jsonIterator = JsonIterator.parse(new ByteArrayInputStream(src), initialBufferSize, 0);
+ jsonIterator.readObject();
+ parsedString = jsonIterator.readAny();
+ assertEquals(originalContent, parsedString.toString());
+ // Check buffer was expanded exactly once
+ assertEquals(initialBufferSize, jsonIterator.buf.length);
+ }
+
+ private static InputStream getSluggishInputStream(final byte[] src) {
+ return new InputStream() {
+ int position = 0;
+
+ @Override
+ public int read() throws IOException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (position < src.length) {
+ b[off] = src[position++];
+ return 1;
+ }
+ return -1;
+ }
+ };
+ }
+}
diff --git a/src/test/java/com/jsoniter/any/TestLong.java b/src/test/java/com/jsoniter/any/TestLong.java
index a5591cd9..247dbe8f 100644
--- a/src/test/java/com/jsoniter/any/TestLong.java
+++ b/src/test/java/com/jsoniter/any/TestLong.java
@@ -1,5 +1,6 @@
package com.jsoniter.any;
+import com.jsoniter.spi.JsonException;
import junit.framework.TestCase;
public class TestLong extends TestCase {
@@ -7,4 +8,21 @@ public void test_to_string_should_trim() {
Any any = Any.lazyLong(" 1000".getBytes(), 0, " 1000".length());
assertEquals("1000", any.toString());
}
+
+ public void test_should_fail_with_leading_zero() {
+ byte[] bytes = "01".getBytes();
+ Any any = Any.lazyLong(bytes, 0, bytes.length);
+ try {
+ any.toLong();
+ fail("This should fail.");
+ } catch (JsonException e) {
+
+ }
+ }
+
+ public void test_should_work_with_zero() {
+ byte[] bytes = "0".getBytes();
+ Any any = Any.lazyLong(bytes, 0, bytes.length);
+ assertEquals(0L, any.toLong());
+ }
}
diff --git a/src/test/java/com/jsoniter/output/TestString.java b/src/test/java/com/jsoniter/output/TestString.java
index 96dda062..186c770a 100644
--- a/src/test/java/com/jsoniter/output/TestString.java
+++ b/src/test/java/com/jsoniter/output/TestString.java
@@ -1,15 +1,40 @@
package com.jsoniter.output;
import com.jsoniter.spi.Config;
+import com.jsoniter.spi.Config.Builder;
+import com.jsoniter.spi.JsoniterSpi;
+import java.io.ByteArrayOutputStream;
import junit.framework.TestCase;
public class TestString extends TestCase {
+
+ public static final String UTF8_GREETING = "Привет čau 你好 ~";
+
public void test_unicode() {
String output = JsonStream.serialize(new Config.Builder().escapeUnicode(false).build(), "中文");
assertEquals("\"中文\"", output);
}
+ public void test_unicode_tilde() {
+ final String tilde = "~";
+ String output = JsonStream.serialize(new Config.Builder().escapeUnicode(false).build(), tilde);
+ assertEquals("\""+tilde+"\"", output);
+ }
+ public void test_escape_unicode() {
+ final Config config = new Builder().escapeUnicode(false).build();
+
+ assertEquals("\""+UTF8_GREETING+"\"", JsonStream.serialize(config, UTF8_GREETING));
+ assertEquals("\""+UTF8_GREETING+"\"", JsonStream.serialize(config.escapeUnicode(), UTF8_GREETING.getClass(), UTF8_GREETING));
+ }
public void test_escape_control_character() {
String output = JsonStream.serialize(new String(new byte[]{0}));
assertEquals("\"\\u0000\"", output);
}
+ public void test_serialize_into_output_stream() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ boolean escapeUnicode = JsoniterSpi.getCurrentConfig().escapeUnicode();
+ JsoniterSpi.setCurrentConfig(JsoniterSpi.getCurrentConfig().copyBuilder().escapeUnicode(false).build());
+ JsonStream.serialize(String.class, UTF8_GREETING, baos);
+ JsoniterSpi.setCurrentConfig(JsoniterSpi.getCurrentConfig().copyBuilder().escapeUnicode(escapeUnicode).build());
+ assertEquals("\"" + UTF8_GREETING + "\"", baos.toString());
+ }
}
diff --git a/src/test/java/com/jsoniter/suite/AllTestCases.java b/src/test/java/com/jsoniter/suite/AllTestCases.java
index 70c904c8..d3196ed4 100644
--- a/src/test/java/com/jsoniter/suite/AllTestCases.java
+++ b/src/test/java/com/jsoniter/suite/AllTestCases.java
@@ -53,6 +53,7 @@
TestGson.class,
com.jsoniter.output.TestGson.class,
TestStreamBuffer.class,
+ IterImplForStreamingTest.class,
TestCollection.class,
TestList.class,
TestAnnotationJsonObject.class,