This will guard the batch_size so it cannot be so low as to accidentally burn through your CPU. (simdjson#1319)

lemire · web-flow · commit 56098517470b · 2020-11-27T09:37:54.000-05:00
* This will guard the batch_size so it cannot be so low as to accidentally burn through your CPU.
diff --git a/include/simdjson/dom/document_stream-inl.h b/include/simdjson/dom/document_stream-inl.h
@@ -73,7 +73,7 @@ simdjson_really_inline document_stream::document_stream(
   : parser{&_parser},
     buf{_buf},
     len{_len},
-    batch_size{_batch_size},
+    batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
     error{SUCCESS}
 #ifdef SIMDJSON_THREADS_ENABLED
     , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
diff --git a/include/simdjson/dom/parser-inl.h b/include/simdjson/dom/parser-inl.h
@@ -83,6 +83,7 @@ inline simdjson_result<document_stream> parser::load_many(const std::string &pat
   size_t len;
   auto _error = read_file(path).get(len);
   if (_error) { return _error; }
+  if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
   return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size);
 }
 
@@ -112,6 +113,7 @@ simdjson_really_inline simdjson_result<element> parser::parse(const padded_strin
 }
 
 inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
+  if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
   return document_stream(*this, buf, len, batch_size);
 }
 inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {
diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h
@@ -21,6 +21,12 @@ class element;
 
 /** The default batch size for parser.parse_many() and parser.load_many() */
 static constexpr size_t DEFAULT_BATCH_SIZE = 1000000;
+/**
+ * Some adversary might try to set the batch size to 0 or 1, which might cause problems.
+ * We set a minimum of 32B since anything else is highly likely to be an error. In practice,
+ * most users will want a much larger batch size.
+ */
+static constexpr size_t MINIMAL_BATCH_SIZE = 32;
 
 /**
  * A persistent document parser.
@@ -249,7 +255,10 @@ class parser {
    * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
    *                   spot is cache-related: small enough to fit in cache, yet big enough to
    *                   parse as many documents as possible in one tight loop.
-   *                   Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet spot in our tests.
+   *                   Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet
+   *                   spot in our tests.
+   *                   If you set the batch_size to a value smaller than simdjson::dom::MINIMAL_BATCH_SIZE
+   *                   (currently 32B), it will be replaced by simdjson::dom::MINIMAL_BATCH_SIZE.
    * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
    *         - IO_ERROR if there was an error opening or reading the file.
    *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails.
diff --git a/tests/document_stream_tests.cpp b/tests/document_stream_tests.cpp
@@ -135,10 +135,16 @@ namespace document_stream_tests {
 
   bool small_window() {
     std::cout << "Running " << __func__ << std::endl;
-    auto json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded;
+    char input[2049];
+    input[0] = '[';
+    for(size_t i = 1; i < 1024; i++) {
+      input[2*i+1]= '1';
+      input[2*i+2]= i < 1023 ? ',' : ']';
+    }
+    auto json = simdjson::padded_string(input,2049);
     simdjson::dom::parser parser;
     size_t count = 0;
-    size_t window_size = 10; // deliberately too small
+    size_t window_size = 1024; // deliberately too small
     simdjson::dom::document_stream stream;
     ASSERT_SUCCESS( parser.parse_many(json, window_size).get(stream) );
     for (auto doc : stream) {
@@ -158,11 +164,17 @@ namespace document_stream_tests {
 #ifdef SIMDJSON_THREADS_ENABLED
   bool threaded_disabled() {
     std::cout << "Running " << __func__ << std::endl;
-    auto json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded;
+    char input[2049];
+    input[0] = '[';
+    for(size_t i = 1; i < 1024; i++) {
+      input[2*i+1]= '1';
+      input[2*i+2]= i < 1023 ? ',' : ']';
+    }
+    auto json = simdjson::padded_string(input,2049);
     simdjson::dom::parser parser;
     parser.threaded = false;
     size_t count = 0;
-    size_t window_size = 10; // deliberately too small
+    size_t window_size = 1024; // deliberately too small
     simdjson::dom::document_stream stream;
     ASSERT_SUCCESS( parser.parse_many(json, window_size).get(stream) );
     for (auto doc : stream) {

Original file line number	Diff line number	Diff line change
`@@ -83,6 +83,7 @@ inline simdjson_result<document_stream> parser::load_many(const std::string &pat`
`83`	`83`	`size_t len;`
`84`	`84`	`auto _error = read_file(path).get(len);`
`85`	`85`	`if (_error) { return _error; }`
	`86`	`+ if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }`
`86`	`87`	`return document_stream(this, (const uint8_t)loaded_bytes.get(), len, batch_size);`
`87`	`88`	`}`
`88`	`89`
`@@ -112,6 +113,7 @@ simdjson_really_inline simdjson_result<element> parser::parse(const padded_strin`
`112`	`113`	`}`
`113`	`114`
`114`	`115`	`inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {`
	`116`	`+ if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }`
`115`	`117`	`return document_stream(*this, buf, len, batch_size);`
`116`	`118`	`}`
`117`	`119`	`inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {`