Skip to content

Commit c3c3899

Browse files
jkeiserjkeiser
authored
Merge pull request simdjson#1179 from simdjson/jkeiser/stream-parse-scratch
Allow direct document iteration in ondemand
2 parents e774c50 + dbdbfd4 commit c3c3899

File tree

8 files changed

+169
-75
lines changed

8 files changed

+169
-75
lines changed

‎include/simdjson/generic/ondemand/array-inl.h

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,26 @@ simdjson_really_inline array array::started(json_iterator_ref &&iter) noexcept {
6464
return array(std::forward<json_iterator_ref>(iter));
6565
}
6666

67-
simdjson_really_inline array_iterator array::begin() & noexcept {
68-
return iter;
67+
//
68+
// For array_iterator
69+
//
70+
simdjson_really_inline json_iterator &array::get_iterator() noexcept {
71+
return *iter;
72+
}
73+
simdjson_really_inline json_iterator_ref array::borrow_iterator() noexcept {
74+
return iter.borrow();
75+
}
76+
simdjson_really_inline bool array::is_iteration_finished() const noexcept {
77+
return iter.is_alive();
78+
}
79+
simdjson_really_inline void array::iteration_finished() noexcept {
80+
iter.release();
81+
}
82+
83+
simdjson_really_inline array_iterator<array> array::begin() & noexcept {
84+
return *this;
6985
}
70-
simdjson_really_inline array_iterator array::end() & noexcept {
86+
simdjson_really_inline array_iterator<array> array::end() & noexcept {
7187
return {};
7288
}
7389

@@ -92,11 +108,11 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>
92108
{
93109
}
94110

95-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::begin() & noexcept {
111+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::begin() & noexcept {
96112
if (error()) { return error(); }
97113
return first.begin();
98114
}
99-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::end() & noexcept {
115+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::end() & noexcept {
100116
if (error()) { return error(); }
101117
return first.end();
102118
}

‎include/simdjson/generic/ondemand/array.h

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ class array {
3333
*
3434
* Part of the std::iterable interface.
3535
*/
36-
simdjson_really_inline array_iterator begin() & noexcept;
36+
simdjson_really_inline array_iterator<array> begin() & noexcept;
3737
/**
3838
* Sentinel representing the end of the array.
3939
*
4040
* Part of the std::iterable interface.
4141
*/
42-
simdjson_really_inline array_iterator end() & noexcept;
42+
simdjson_really_inline array_iterator<array> end() & noexcept;
4343

4444
protected:
4545
/**
@@ -69,6 +69,14 @@ class array {
6969
*/
7070
simdjson_really_inline array(json_iterator_ref &&iter) noexcept;
7171

72+
//
73+
// For array_iterator
74+
//
75+
simdjson_really_inline json_iterator &get_iterator() noexcept;
76+
simdjson_really_inline json_iterator_ref borrow_iterator() noexcept;
77+
simdjson_really_inline bool is_iteration_finished() const noexcept;
78+
simdjson_really_inline void iteration_finished() noexcept;
79+
7280
/**
7381
* Iterator marking current position.
7482
*
@@ -79,6 +87,7 @@ class array {
7987
friend class value;
8088
friend struct simdjson_result<value>;
8189
friend struct simdjson_result<array>;
90+
friend class array_iterator<array>;
8291
};
8392

8493
} // namespace ondemand
@@ -97,8 +106,8 @@ struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> : public SIMDJS
97106
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> &&a) noexcept = default;
98107
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
99108

100-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
101-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
109+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> begin() & noexcept;
110+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::array>> end() & noexcept;
102111
};
103112

104113
} // namespace simdjson

‎include/simdjson/generic/ondemand/array_iterator-inl.h

Lines changed: 41 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,27 @@ namespace simdjson {
22
namespace SIMDJSON_IMPLEMENTATION {
33
namespace ondemand {
44

5-
simdjson_really_inline array_iterator::array_iterator(json_iterator_ref &_iter) noexcept : iter{&_iter} {}
5+
template<typename T>
6+
simdjson_really_inline array_iterator<T>::array_iterator(T &_iter) noexcept : iter{&_iter} {}
67

7-
simdjson_really_inline simdjson_result<value> array_iterator::operator*() noexcept {
8-
if ((*iter)->error()) { iter->release(); return (*iter)->error(); }
9-
return value::start(iter->borrow());
8+
template<typename T>
9+
simdjson_really_inline simdjson_result<value> array_iterator<T>::operator*() noexcept {
10+
if (iter->get_iterator().error()) { iter->iteration_finished(); return iter->get_iterator().error(); }
11+
return value::start(iter->borrow_iterator());
1012
}
11-
simdjson_really_inline bool array_iterator::operator==(const array_iterator &other) noexcept {
13+
template<typename T>
14+
simdjson_really_inline bool array_iterator<T>::operator==(const array_iterator<T> &other) noexcept {
1215
return !(*this != other);
1316
}
14-
simdjson_really_inline bool array_iterator::operator!=(const array_iterator &) noexcept {
15-
return iter->is_alive();
17+
template<typename T>
18+
simdjson_really_inline bool array_iterator<T>::operator!=(const array_iterator<T> &) noexcept {
19+
return iter->is_iteration_finished();
1620
}
17-
simdjson_really_inline array_iterator &array_iterator::operator++() noexcept {
21+
template<typename T>
22+
simdjson_really_inline array_iterator<T> &array_iterator<T>::operator++() noexcept {
1823
bool has_value;
19-
error_code error = (*iter)->has_next_element().get(has_value); // If there's an error, has_next stays true.
20-
if (!(error || has_value)) { iter->release(); }
24+
error_code error = iter->get_iterator().has_next_element().get(has_value); // If there's an error, has_next stays true.
25+
if (!(error || has_value)) { iter->iteration_finished(); }
2126
return *this;
2227
}
2328

@@ -27,35 +32,38 @@ simdjson_really_inline array_iterator &array_iterator::operator++() noexcept {
2732

2833
namespace simdjson {
2934

30-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::simdjson_result(
31-
SIMDJSON_IMPLEMENTATION::ondemand::array_iterator &&value
35+
template<typename T>
36+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::simdjson_result(
37+
SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T> &&value
3238
) noexcept
33-
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>(value))
39+
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>(value))
3440
{
3541
}
36-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::simdjson_result(error_code error) noexcept
37-
: implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>({}, error)
42+
template<typename T>
43+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::simdjson_result(error_code error) noexcept
44+
: SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>({}, error)
3845
{
3946
}
4047

41-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator*() noexcept {
42-
if (error()) { second = SUCCESS; return error(); }
43-
return *first;
44-
}
45-
// Assumes it's being compared with the end. true if depth < iter->depth.
46-
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &other) noexcept {
47-
if (error()) { return true; }
48-
return first == other.first;
49-
}
50-
// Assumes it's being compared with the end. true if depth >= iter->depth.
51-
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &other) noexcept {
52-
if (error()) { return false; }
53-
return first != other.first;
54-
}
55-
// Checks for ']' and ','
56-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator>::operator++() noexcept {
57-
if (error()) { return *this; }
58-
++first;
48+
template<typename T>
49+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator*() noexcept {
50+
if (this->error()) { this->second = SUCCESS; return this->error(); }
51+
return *this->first;
52+
}
53+
template<typename T>
54+
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &other) noexcept {
55+
if (this->error()) { return true; }
56+
return this->first == other.first;
57+
}
58+
template<typename T>
59+
simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &other) noexcept {
60+
if (this->error()) { return false; }
61+
return this->first != other.first;
62+
}
63+
template<typename T>
64+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>>::operator++() noexcept {
65+
if (this->error()) { return *this; }
66+
++(this->first);
5967
return *this;
6068
}
6169

‎include/simdjson/generic/ondemand/array_iterator.h

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@ class document;
1616
* - * must be called exactly once per element.
1717
* - ++ must be called exactly once in between each * (*, ++, *, ++, * ...)
1818
*/
19+
template<typename T>
1920
class array_iterator {
2021
public:
2122
/** Create a new, invalid array iterator. */
2223
simdjson_really_inline array_iterator() noexcept = default;
23-
simdjson_really_inline array_iterator(const array_iterator &a) noexcept = default;
24-
simdjson_really_inline array_iterator &operator=(const array_iterator &a) noexcept = default;
24+
simdjson_really_inline array_iterator(const array_iterator<T> &a) noexcept = default;
25+
simdjson_really_inline array_iterator<T> &operator=(const array_iterator<T> &a) noexcept = default;
2526

2627
//
2728
// Iterator interface
@@ -40,30 +41,30 @@ class array_iterator {
4041
*
4142
* @return true if there are no more elements in the JSON array.
4243
*/
43-
simdjson_really_inline bool operator==(const array_iterator &) noexcept;
44+
simdjson_really_inline bool operator==(const array_iterator<T> &) noexcept;
4445
/**
4546
* Check if there are more elements in the JSON array.
4647
*
4748
* Part of the std::iterator interface.
4849
*
4950
* @return true if there are more elements in the JSON array.
5051
*/
51-
simdjson_really_inline bool operator!=(const array_iterator &) noexcept;
52+
simdjson_really_inline bool operator!=(const array_iterator<T> &) noexcept;
5253
/**
5354
* Move to the next element.
5455
*
5556
* Part of the std::iterator interface.
5657
*/
57-
simdjson_really_inline array_iterator &operator++() noexcept;
58+
simdjson_really_inline array_iterator<T> &operator++() noexcept;
5859

5960
private:
60-
json_iterator_ref *iter{};
61+
T *iter{};
6162

62-
simdjson_really_inline array_iterator(json_iterator_ref &iter) noexcept;
63+
simdjson_really_inline array_iterator(T &iter) noexcept;
6364

6465
friend class array;
6566
friend class value;
66-
friend struct simdjson_result<array_iterator>;
67+
friend struct simdjson_result<array_iterator<T>>;
6768
};
6869

6970
} // namespace ondemand
@@ -72,24 +73,24 @@ class array_iterator {
7273

7374
namespace simdjson {
7475

75-
template<>
76-
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> {
76+
template<typename T>
77+
struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> {
7778
public:
78-
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::array_iterator &&value) noexcept; ///< @private
79+
simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T> &&value) noexcept; ///< @private
7980
simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
8081

8182
simdjson_really_inline simdjson_result() noexcept = default;
82-
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &&a) noexcept = default;
83+
simdjson_really_inline simdjson_result(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &&a) noexcept = default;
8384
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
8485

8586
//
8687
// Iterator interface
8788
//
8889

8990
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION.
90-
simdjson_really_inline bool operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &) noexcept;
91-
simdjson_really_inline bool operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &) noexcept;
92-
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &operator++() noexcept;
91+
simdjson_really_inline bool operator==(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &) noexcept;
92+
simdjson_really_inline bool operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &) noexcept;
93+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<T>> &operator++() noexcept;
9394
};
9495

9596
} // namespace simdjson

‎include/simdjson/generic/ondemand/document-inl.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,22 @@ simdjson_really_inline simdjson_result<value> document::operator[](const char *k
9292
return get_object()[key];
9393
}
9494

95+
//
96+
// For array_iterator
97+
//
98+
simdjson_really_inline json_iterator &document::get_iterator() noexcept {
99+
return iter;
100+
}
101+
simdjson_really_inline json_iterator_ref document::borrow_iterator() noexcept {
102+
return iter.borrow();
103+
}
104+
simdjson_really_inline bool document::is_iteration_finished() const noexcept {
105+
return json;
106+
}
107+
simdjson_really_inline void document::iteration_finished() noexcept {
108+
json = nullptr;
109+
}
110+
95111
} // namespace ondemand
96112
} // namespace SIMDJSON_IMPLEMENTATION
97113
} // namespace simdjson

‎include/simdjson/generic/ondemand/document.h

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,19 @@ class document {
171171
simdjson_really_inline operator bool() noexcept(false);
172172
#endif
173173

174-
// We don't have an array_iterator that can point at an owned json_iterator
175-
// simdjson_really_inline simdjson_result<array::iterator> begin() & noexcept;
176-
// simdjson_really_inline simdjson_result<array::iterator> end() & noexcept;
174+
/**
175+
* Begin array iteration.
176+
*
177+
* Part of the std::iterable interface.
178+
*/
179+
simdjson_really_inline array_iterator<document> begin() & noexcept;
180+
/**
181+
* Sentinel representing the end of the array.
182+
*
183+
* Part of the std::iterable interface.
184+
*/
185+
simdjson_really_inline array_iterator<document> end() & noexcept;
186+
177187
/**
178188
* Look up a field by name on an object.
179189
*
@@ -209,11 +219,22 @@ class document {
209219
template<typename T>
210220
simdjson_result<T> consume_if_success(simdjson_result<T> &&result) noexcept;
211221

222+
simdjson_really_inline void assert_at_start() const noexcept;
223+
224+
//
225+
// For array_iterator
226+
//
227+
simdjson_really_inline json_iterator &get_iterator() noexcept;
228+
simdjson_really_inline json_iterator_ref borrow_iterator() noexcept;
229+
simdjson_really_inline bool is_iteration_finished() const noexcept;
230+
simdjson_really_inline void iteration_finished() noexcept;
231+
232+
//
233+
// Fields
234+
//
212235
json_iterator iter{}; ///< Current position in the document
213236
const uint8_t *json{}; ///< JSON for the value in the document (nullptr if value has been consumed)
214237

215-
simdjson_really_inline void assert_at_start() const noexcept;
216-
217238
friend struct simdjson_result<document>;
218239
friend class value;
219240
friend class ondemand::parser;
@@ -260,8 +281,8 @@ struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> : public SIM
260281
simdjson_really_inline operator bool() noexcept(false);
261282
#endif
262283

263-
// simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array::iterator> begin() & noexcept;
264-
// simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array::iterator> end() & noexcept;
284+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> begin() & noexcept;
285+
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator<SIMDJSON_IMPLEMENTATION::ondemand::document>> end() & noexcept;
265286
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
266287
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
267288
};

0 commit comments

Comments
 (0)