Skip to content

Commit 3a06453

Browse files
lemirelemire
authored
Merge pull request simdjson#994 from simdjson/issue976
Fix for issue 976 (something like 32-bit support)
2 parents 13e1794 + 444ec4a commit 3a06453

File tree

11 files changed

+154
-48
lines changed

11 files changed

+154
-48
lines changed

‎.drone.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,46 @@
11
kind: pipeline
2+
name: i386-gcc # we do not support 32-bit systems, but we run tests
3+
platform: { os: linux, arch: amd64 }
4+
steps:
5+
- name: Build and Test
6+
image: i386/ubuntu
7+
environment:
8+
CC: gcc
9+
CXX: g++
10+
BUILD_FLAGS: -- -j
11+
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
12+
CTEST_FLAGS: -j4 --output-on-failure -E checkperf
13+
commands:
14+
- apt-get update -qq
15+
- apt-get install -y g++ cmake gcc
16+
- mkdir build
17+
- cd build
18+
- cmake $CMAKE_FLAGS ..
19+
- cmake --build . $BUILD_FLAGS
20+
- ctest $CTEST_FLAGS
21+
---
22+
kind: pipeline
23+
name: i386-clang # we do not support 32-bit systems, but we run tests
24+
platform: { os: linux, arch: amd64 }
25+
steps:
26+
- name: Build and Test
27+
image: i386/ubuntu
28+
environment:
29+
CC: clang-6.0
30+
CXX: clang++-6.0
31+
BUILD_FLAGS: -- -j
32+
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
33+
CTEST_FLAGS: -j4 --output-on-failure -E checkperf
34+
commands:
35+
- apt-get update -qq
36+
- apt-get install -y clang++-6.0 cmake
37+
- mkdir build
38+
- cd build
39+
- cmake $CMAKE_FLAGS ..
40+
- cmake --build . $BUILD_FLAGS
41+
- ctest $CTEST_FLAGS
42+
---
43+
kind: pipeline
244
name: gcc9
345
platform: { os: linux, arch: amd64 }
446
steps:

‎include/simdjson/inline/tape_ref.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,14 @@ really_inline T tape_ref::next_tape_value() const noexcept {
8181
}
8282

8383
really_inline uint32_t internal::tape_ref::get_string_length() const noexcept {
84-
uint64_t string_buf_index = size_t(tape_value());
84+
size_t string_buf_index = size_t(tape_value());
8585
uint32_t len;
8686
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
8787
return len;
8888
}
8989

9090
really_inline const char * internal::tape_ref::get_c_str() const noexcept {
91-
uint64_t string_buf_index = size_t(tape_value());
91+
size_t string_buf_index = size_t(tape_value());
9292
return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
9393
}
9494

‎include/simdjson/portability.h

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <cstddef>
55
#include <cstdint>
66
#include <cstdlib>
7+
#include <cfloat>
78

89

910
#ifdef _MSC_VER
@@ -37,24 +38,28 @@
3738

3839
#if defined(__x86_64__) || defined(_M_AMD64)
3940
#define SIMDJSON_IS_X86_64 1
40-
#endif
41-
#if defined(__aarch64__) || defined(_M_ARM64)
41+
#elif defined(__aarch64__) || defined(_M_ARM64)
4242
#define SIMDJSON_IS_ARM64 1
43+
#else
44+
#define SIMDJSON_IS_32BITS 1
45+
46+
// We do not support 32-bit platforms, but it can be
47+
// handy to identify them.
48+
#if defined(_M_IX86) || defined(__i386__)
49+
#define SIMDJSON_IS_X86_32BITS 1
50+
#elif defined(__arm__) || defined(_M_ARM)
51+
#define SIMDJSON_IS_ARM_32BITS 1
4352
#endif
4453

45-
#if (!defined(SIMDJSON_IS_X86_64)) && (!defined(SIMDJSON_IS_ARM64))
46-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
47-
#pragma message("The simdjson library is designed\
48-
for 64-bit processors and it seems that you are not \
54+
#endif // defined(__x86_64__) || defined(_M_AMD64)
55+
56+
#ifdef SIMDJSON_IS_32BITS
57+
#pragma message("The simdjson library is designed \
58+
for 64-bit processors and it seems that you are not \
4959
compiling for a known 64-bit platform. All fast kernels \
5060
will be disabled and performance may be poor. Please \
5161
use a 64-bit target such as x64 or 64-bit ARM.")
52-
#else
53-
#error "The simdjson library is designed\
54-
for 64-bit processors. It seems that you are not \
55-
compiling for a known 64-bit platform."
56-
#endif
57-
#endif // (!defined(SIMDJSON_IS_X86_64)) && (!defined(SIMDJSON_IS_ARM64))
62+
#endif // SIMDJSON_IS_32BITS
5863
5964
// this is almost standard?
6065
#undef STRINGIFY_IMPLEMENTATION_
@@ -74,6 +79,15 @@ compiling for a known 64-bit platform."
7479
#define SIMDJSON_IMPLEMENTATION_WESTMERE 0
7580
#endif // SIMDJSON_IS_ARM64
7681
82+
// Our fast kernels require 64-bit systems.
83+
//
84+
// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions.
85+
// Furthermore, the number of SIMD registers is reduced.
86+
//
87+
// On 32-bit ARM, we would have smaller registers.
88+
//
89+
// The simdjson users should still have the fallback kernel. It is
90+
// slower, but it should run everywhere.
7791
#if SIMDJSON_IS_X86_64
7892
#ifndef SIMDJSON_IMPLEMENTATION_HASWELL
7993
#define SIMDJSON_IMPLEMENTATION_HASWELL 1
@@ -84,7 +98,7 @@ compiling for a known 64-bit platform."
8498
#define SIMDJSON_IMPLEMENTATION_ARM64 0
8599
#endif // SIMDJSON_IS_X86_64
86100
87-
// we are going to use runtime dispatch
101+
// We are going to use runtime dispatch.
88102
#ifdef SIMDJSON_IS_X86_64
89103
#ifdef __clang__
90104
// clang does not have GCC push pop

‎singleheader/amalgamate_demo.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* auto-generated on Tue 23 Jun 2020 20:51:12 EDT. Do not edit! */
1+
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
22

33
#include <iostream>
44
#include "simdjson.h"
@@ -43,3 +43,4 @@ int main(int argc, char *argv[]) {
4343
}
4444
return EXIT_SUCCESS;
4545
}
46+

‎singleheader/simdjson.cpp

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* auto-generated on Tue 23 Jun 2020 20:51:12 EDT. Do not edit! */
1+
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
22
/* begin file src/simdjson.cpp */
33
#include "simdjson.h"
44

@@ -934,9 +934,8 @@ struct value128 {
934934
uint64_t high;
935935
};
936936

937-
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) && \
938-
!defined(_M_X64) && !defined(_M_ARM64)// _umul128 for x86, arm
939-
// this is a slow emulation routine for 32-bit Windows
937+
#ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm
938+
// this is a slow emulation routine for 32-bit
940939
//
941940
static inline uint64_t __emulu(uint32_t x, uint32_t y) {
942941
return x * (uint64_t)y;
@@ -955,15 +954,15 @@ static inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) {
955954

956955
really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) {
957956
value128 answer;
958-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
957+
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
959958
#ifdef _M_ARM64
960959
// ARM64 has native support for 64-bit multiplications, no need to emultate
961960
answer.high = __umulh(value1, value2);
962961
answer.low = value1 * value2;
963962
#else
964963
answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64
965964
#endif // _M_ARM64
966-
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
965+
#else // defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
967966
__uint128_t r = ((__uint128_t)value1) * value2;
968967
answer.low = uint64_t(r);
969968
answer.high = uint64_t(r >> 64);
@@ -7826,7 +7825,7 @@ really_inline bool add_overflow(uint64_t value1, uint64_t value2,
78267825
#endif
78277826
}
78287827

7829-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
7828+
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
78307829
#pragma intrinsic(_umul128)
78317830
#endif
78327831
really_inline bool mul_overflow(uint64_t value1, uint64_t value2,
@@ -11114,7 +11113,7 @@ really_inline bool add_overflow(uint64_t value1, uint64_t value2,
1111411113
#endif
1111511114
}
1111611115

11117-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
11116+
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
1111811117
#pragma intrinsic(_umul128)
1111911118
#endif
1112011119
really_inline bool mul_overflow(uint64_t value1, uint64_t value2,

‎singleheader/simdjson.h

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* auto-generated on Tue 23 Jun 2020 20:51:12 EDT. Do not edit! */
1+
/* auto-generated on Fri Jun 26 15:35:58 UTC 2020. Do not edit! */
22
/* begin file include/simdjson.h */
33
#ifndef SIMDJSON_H
44
#define SIMDJSON_H
@@ -91,13 +91,14 @@
9191

9292
#if defined(__x86_64__) || defined(_M_AMD64)
9393
#define SIMDJSON_IS_X86_64 1
94-
#endif
95-
#if defined(__aarch64__) || defined(_M_ARM64)
94+
#elif defined(__aarch64__) || defined(_M_ARM64)
9695
#define SIMDJSON_IS_ARM64 1
96+
#else
97+
#define SIMDJSON_IS_32BITS 1
9798
#endif
9899

99-
#if (!defined(SIMDJSON_IS_X86_64)) && (!defined(SIMDJSON_IS_ARM64))
100-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
100+
#ifdef SIMDJSON_IS_32BITS
101+
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(__GNUC__)
101102
#pragma message("The simdjson library is designed\
102103
for 64-bit processors and it seems that you are not \
103104
compiling for a known 64-bit platform. All fast kernels \
@@ -108,7 +109,7 @@ use a 64-bit target such as x64 or 64-bit ARM.")
108109
for 64-bit processors. It seems that you are not \
109110
compiling for a known 64-bit platform."
110111
#endif
111-
#endif // (!defined(SIMDJSON_IS_X86_64)) && (!defined(SIMDJSON_IS_ARM64))
112+
#endif // SIMDJSON_IS_32BITS
112113
113114
// this is almost standard?
114115
#undef STRINGIFY_IMPLEMENTATION_
@@ -2632,7 +2633,6 @@ inline error_code dom_parser_implementation::allocate(size_t capacity, size_t ma
26322633
26332634
#endif // SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H
26342635
/* end file include/simdjson/internal/dom_parser_implementation.h */
2635-
#include <optional>
26362636
#include <string>
26372637
#include <atomic>
26382638
#include <vector>
@@ -4008,21 +4008,42 @@ class element {
40084008
*/
40094009
inline simdjson_result<object> get_object() const noexcept;
40104010
/**
4011-
* Cast this element to a string.
4011+
* Cast this element to a null-terminated C string.
4012+
*
4013+
* The string is guaranteed to be valid UTF-8.
40124014
*
4013-
* Equivalent to get<const char *>().
4015+
* The get_c_str() function is equivalent to get<const char *>().
4016+
*
4017+
* The length of the string is given by get_string_length(). Because JSON strings
4018+
* may contain null characters, it may be incorrect to use strlen to determine the
4019+
* string length.
4020+
*
4021+
* It is possible to get a single string_view instance which represents both the string
4022+
* content and its length: see get_string().
40144023
*
4015-
* @returns An pointer to a null-terminated string. This string is stored in the parser and will
4024+
* @returns A pointer to a null-terminated UTF-8 string. This string is stored in the parser and will
40164025
* be invalidated the next time it parses a document or when it is destroyed.
40174026
* Returns INCORRECT_TYPE if the JSON element is not a string.
40184027
*/
40194028
inline simdjson_result<const char *> get_c_str() const noexcept;
40204029
/**
4021-
* Cast this element to a string.
4030+
* Gives the length in bytes of the string.
4031+
*
4032+
* It is possible to get a single string_view instance which represents both the string
4033+
* content and its length: see get_string().
4034+
*
4035+
* @returns A string length in bytes.
4036+
* Returns INCORRECT_TYPE if the JSON element is not a string.
4037+
*/
4038+
inline simdjson_result<size_t> get_string_length() const noexcept;
4039+
/**
4040+
* Cast this element to a string.
4041+
*
4042+
* The string is guaranteed to be valid UTF-8.
40224043
*
40234044
* Equivalent to get<std::string_view>().
40244045
*
4025-
* @returns A string. The string is stored in the parser and will be invalidated the next time it
4046+
* @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next time it
40264047
* parses a document or when it is destroyed.
40274048
* Returns INCORRECT_TYPE if the JSON element is not a string.
40284049
*/
@@ -4199,7 +4220,9 @@ class element {
41994220
inline operator bool() const noexcept(false);
42004221

42014222
/**
4202-
* Read this element as a null-terminated string.
4223+
* Read this element as a null-terminated UTF-8 string.
4224+
*
4225+
* Be mindful that JSON allows strings to contain null characters.
42034226
*
42044227
* Does *not* convert other types to a string; requires that the JSON type of the element was
42054228
* an actual string.
@@ -4210,7 +4233,7 @@ class element {
42104233
inline explicit operator const char*() const noexcept(false);
42114234

42124235
/**
4213-
* Read this element as a null-terminated string.
4236+
* Read this element as a null-terminated UTF-8 string.
42144237
*
42154238
* Does *not* convert other types to a string; requires that the JSON type of the element was
42164239
* an actual string.
@@ -4410,6 +4433,7 @@ struct simdjson_result<dom::element> : public internal::simdjson_result_base<dom
44104433
really_inline simdjson_result<dom::array> get_array() const noexcept;
44114434
really_inline simdjson_result<dom::object> get_object() const noexcept;
44124435
really_inline simdjson_result<const char *> get_c_str() const noexcept;
4436+
really_inline simdjson_result<size_t> get_string_length() const noexcept;
44134437
really_inline simdjson_result<std::string_view> get_string() const noexcept;
44144438
really_inline simdjson_result<int64_t> get_int64() const noexcept;
44154439
really_inline simdjson_result<uint64_t> get_uint64() const noexcept;
@@ -5820,6 +5844,10 @@ really_inline simdjson_result<const char *> simdjson_result<dom::element>::get_c
58205844
if (error()) { return error(); }
58215845
return first.get_c_str();
58225846
}
5847+
really_inline simdjson_result<size_t> simdjson_result<dom::element>::get_string_length() const noexcept {
5848+
if (error()) { return error(); }
5849+
return first.get_string_length();
5850+
}
58235851
really_inline simdjson_result<std::string_view> simdjson_result<dom::element>::get_string() const noexcept {
58245852
if (error()) { return error(); }
58255853
return first.get_string();
@@ -5960,6 +5988,15 @@ inline simdjson_result<const char *> element::get_c_str() const noexcept {
59605988
return INCORRECT_TYPE;
59615989
}
59625990
}
5991+
inline simdjson_result<size_t> element::get_string_length() const noexcept {
5992+
switch (tape.tape_ref_type()) {
5993+
case internal::tape_type::STRING: {
5994+
return tape.get_string_length();
5995+
}
5996+
default:
5997+
return INCORRECT_TYPE;
5998+
}
5999+
}
59636000
inline simdjson_result<std::string_view> element::get_string() const noexcept {
59646001
switch (tape.tape_ref_type()) {
59656002
case internal::tape_type::STRING:
@@ -7610,14 +7647,14 @@ really_inline T tape_ref::next_tape_value() const noexcept {
76107647
}
76117648

76127649
really_inline uint32_t internal::tape_ref::get_string_length() const noexcept {
7613-
uint64_t string_buf_index = size_t(tape_value());
7650+
size_t string_buf_index = size_t(tape_value());
76147651
uint32_t len;
7615-
memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
7652+
memcpy(&len, &doc->string_buf[size_t(string_buf_index)], sizeof(len));
76167653
return len;
76177654
}
76187655

76197656
really_inline const char * internal::tape_ref::get_c_str() const noexcept {
7620-
uint64_t string_buf_index = size_t(tape_value());
7657+
size_t string_buf_index = size_t(tape_value());
76217658
return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
76227659
}
76237660

‎src/generic/stage2/numberparsing.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,15 @@ really_inline double compute_float_64(int64_t power, uint64_t i, bool negative,
1313
// It was described in
1414
// Clinger WD. How to read floating point numbers accurately.
1515
// ACM SIGPLAN Notices. 1990
16+
#ifndef FLT_EVAL_METHOD
17+
#error "FLT_EVAL_METHOD should be defined, please include cfloat."
18+
#endif
19+
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
20+
// We cannot be certain that x/y is rounded to nearest.
21+
if (0 <= power && power <= 22 && i <= 9007199254740991) {
22+
#else
1623
if (-22 <= power && power <= 22 && i <= 9007199254740991) {
24+
#endif
1725
// convert the integer into a double. This is lossless since
1826
// 0 <= i <= 2^53 - 1.
1927
double d = double(i);

‎src/haswell/bitmanipulation.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ really_inline bool add_overflow(uint64_t value1, uint64_t value2,
5858
#endif
5959
}
6060

61-
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
61+
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS)
6262
#pragma intrinsic(_umul128)
6363
#endif
6464
really_inline bool mul_overflow(uint64_t value1, uint64_t value2,

0 commit comments

Comments
 (0)