Remove JsonStream. Use parse_many() instead.

jkeiser · jkeiser · commit 5aec2671eaf9 · 2020-03-26T09:25:07.000-07:00
diff --git a/Makefile b/Makefile
@@ -61,7 +61,7 @@ SRCHEADERS_WESTMERE=src/westmere/bitmanipulation.h src/westmere/bitmask.h src/we
 SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h
 SRCHEADERS=$(SRCHEADERS_SRC) $(SRCHEADERS_GENERIC) $(SRCHEADERS_ARM64) $(SRCHEADERS_HASWELL) $(SRCHEADERS_WESTMERE) $(SRCHEADERS_FALLBACK)
 
-INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/inline/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/parsedjson_iterator.h include/simdjson/inline/parsedjson_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/jsonstream.h include/simdjson/inline/jsonstream.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h
+INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/inline/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/parsedjson_iterator.h include/simdjson/inline/parsedjson_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h
 
 ifeq ($(SIMDJSON_TEST_AMALGAMATED_HEADERS),1)
 	HEADERS=singleheader/simdjson.h
diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt
@@ -10,13 +10,11 @@ set(SIMDJSON_INCLUDE
     ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/document_stream.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/document.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/error.h
-    ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/jsonstream.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/padded_string.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/parsedjson_iterator.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/internal/jsonformatutils.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/jsonioutil.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/jsonparser.h
-    ${SIMDJSON_INCLUDE_DIR}/simdjson/jsonstream.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/padded_string.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/parsedjson.h
     ${SIMDJSON_INCLUDE_DIR}/simdjson/parsedjson_iterator.h
diff --git a/include/simdjson.h b/include/simdjson.h
@@ -15,13 +15,11 @@
 #include "simdjson/jsonparser.h"
 #include "simdjson/parsedjson.h"
 #include "simdjson/parsedjson_iterator.h"
-#include "simdjson/jsonstream.h"
 
 // Inline functions
 #include "simdjson/inline/document.h"
 #include "simdjson/inline/document_stream.h"
 #include "simdjson/inline/error.h"
-#include "simdjson/inline/jsonstream.h"
 #include "simdjson/inline/padded_string.h"
 #include "simdjson/inline/parsedjson_iterator.h"
 
diff --git a/include/simdjson/document_stream.h b/include/simdjson/document_stream.h
@@ -6,8 +6,6 @@
 
 namespace simdjson {
 
-template <class string_container = padded_string> class JsonStream;
-
 /**
  * A forward-only stream of documents.
  *
@@ -137,7 +135,6 @@ class document::stream {
   std::thread stage_1_thread;
   document::parser parser_thread;
 #endif
-  template <class string_container> friend class JsonStream;
   friend class document::parser;
 }; // class document::stream
 
diff --git a/include/simdjson/inline/document_stream.h b/include/simdjson/inline/document_stream.h
@@ -1,7 +1,7 @@
 #ifndef SIMDJSON_INLINE_DOCUMENT_STREAM_H
 #define SIMDJSON_INLINE_DOCUMENT_STREAM_H
 
-#include "simdjson/jsonstream.h"
+#include "simdjson/document_stream.h"
 #include <algorithm>
 #include <limits>
 #include <stdexcept>
diff --git a/include/simdjson/inline/jsonstream.h b/include/simdjson/inline/jsonstream.h
diff --git a/include/simdjson/jsonstream.h b/include/simdjson/jsonstream.h
diff --git a/tests/basictests.cpp b/tests/basictests.cpp
@@ -440,134 +440,6 @@ namespace document_stream_tests {
     return true;
   }
 
-  // returns true if successful
-  bool JsonStream_utf8_test() {
-    std::cout << "Running " << __func__ << std::endl;
-    fflush(NULL);
-    const size_t n_records = 10000;
-    std::string data;
-    char buf[1024];
-    for (size_t i = 0; i < n_records; ++i) {
-      auto n = sprintf(buf,
-                      "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
-                      "\"été\": {\"id\": %zu, \"name\": \"éventail%zu\"}}",
-                      i, i, (i % 2) ? "⺃" : "⺕", i % 10, i % 10);
-      data += std::string(buf, n);
-    }
-    const size_t batch_size = 1000;
-    printf(".");
-    fflush(NULL);
-    simdjson::padded_string str(data);
-    simdjson::JsonStream<simdjson::padded_string> js{str, batch_size};
-    int parse_res = simdjson::SUCCESS_AND_HAS_MORE;
-    size_t count = 0;
-    simdjson::ParsedJson pj;
-    while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) {
-      parse_res = js.json_parse(pj);
-      if (parse_res != simdjson::SUCCESS && parse_res != simdjson::SUCCESS_AND_HAS_MORE) {
-        break;
-      }
-      simdjson::ParsedJson::Iterator iter(pj);
-      if(!iter.is_object()) {
-        printf("Root should be object\n");
-        return false;
-      }
-      if(!iter.down()) {
-        printf("Root should not be emtpy\n");
-        return false;
-      }
-      if(!iter.is_string()) {
-        printf("Object should start with string key\n");
-        return false;
-      }
-      if(strcmp(iter.get_string(),"id")!=0) {
-        printf("There should a single key, id.\n");
-        return false;
-      }
-      iter.move_to_value();
-      if(!iter.is_integer()) {
-        printf("Value of image should be integer\n");
-        return false;
-      }
-      int64_t keyid = iter.get_integer();
-      if(keyid != (int64_t)count) {
-        printf("key does not match %d, expected %d\n",(int) keyid, (int) count);
-        return false;
-      }
-      count++;
-    }
-    if(count != n_records) {
-      printf("Something is wrong in JsonStream_utf8_test at window size = %zu.\n", batch_size);
-      return false;
-    }
-    printf("ok\n");
-    return true;
-  }
-
-  // returns true if successful
-  bool JsonStream_test() {
-    std::cout << "Running " << __func__ << std::endl;
-    fflush(NULL);
-    const size_t n_records = 10000;
-    std::string data;
-    char buf[1024];
-    for (size_t i = 0; i < n_records; ++i) {
-      auto n = sprintf(buf,
-                      "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", "
-                      "\"ete\": {\"id\": %zu, \"name\": \"eventail%zu\"}}",
-                      i, i, (i % 2) ? "homme" : "femme", i % 10, i % 10);
-      data += std::string(buf, n);
-    }
-    const size_t batch_size = 1000;
-    printf(".");
-    fflush(NULL);
-    simdjson::padded_string str(data);
-    simdjson::JsonStream<simdjson::padded_string> js{str, batch_size};
-    int parse_res = simdjson::SUCCESS_AND_HAS_MORE;
-    size_t count = 0;
-    simdjson::ParsedJson pj;
-    while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) {
-      parse_res = js.json_parse(pj);
-      if (parse_res != simdjson::SUCCESS && parse_res != simdjson::SUCCESS_AND_HAS_MORE) {
-        break;
-      }
-      simdjson::ParsedJson::Iterator iter(pj);
-      if(!iter.is_object()) {
-        printf("Root should be object\n");
-        return false;
-      }
-      if(!iter.down()) {
-        printf("Root should not be emtpy\n");
-        return false;
-      }
-      if(!iter.is_string()) {
-        printf("Object should start with string key\n");
-        return false;
-      }
-      if(strcmp(iter.get_string(),"id")!=0) {
-        printf("There should a single key, id.\n");
-        return false;
-      }
-      iter.move_to_value();
-      if(!iter.is_integer()) {
-        printf("Value of image should be integer\n");
-        return false;
-      }
-      int64_t keyid = iter.get_integer();
-      if(keyid != (int64_t)count) {
-        printf("key does not match %d, expected %d\n",(int) keyid, (int) count);
-        return false;
-      }
-      count++;
-    }
-    if(count != n_records) {
-      printf("Something is wrong in JsonStream_test at window size = %zu.\n", batch_size);
-      return false;
-    }
-    printf("ok\n");
-    return true;
-  }
-
   // returns true if successful
   bool document_stream_test() {
     std::cout << "Running " << __func__ << std::endl;
@@ -666,9 +538,7 @@ namespace document_stream_tests {
 
   bool run() {
     return document_stream_test() &&
-           document_stream_utf8_test() &&
-           JsonStream_test() &&
-           JsonStream_utf8_test();
+           document_stream_utf8_test();
   }
 }
 
