Skip to content

Commit a64d2f4

Browse files
jkeiserjkeiser
authored
Merge pull request simdjson#857 from simdjson/jkeiser/benchfeatures
Make benchfeatures work again
2 parents 25fe6d7 + 1e8a54a commit a64d2f4

File tree

9 files changed

+95
-93
lines changed

9 files changed

+95
-93
lines changed

‎.gitignore

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,12 @@ objs
8787

8888
# Build outputs
8989
/build*/
90+
/visual_studio/
9091

9192
# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md
9293
/corpus.zip
9394
/ossfuzz-out/
9495
/out/
9596

96-
# Don't check in generated API docs
97+
# Generated docs
9798
/doc/api
98-
99-
# Don't check in generated examples
100-
/jsonexamples/generated
101-
/visual_studio

‎benchmark/CMakeLists.txt

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
include_directories( . linux )
2-
link_libraries(simdjson simdjson-windows-headers)
3-
# add_executable(benchfeatures benchfeatures.cpp) # doesn't presently compile at all
2+
link_libraries(simdjson simdjson-flags simdjson-windows-headers test-data)
3+
add_executable(benchfeatures benchfeatures.cpp)
44
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
55
add_executable(perfdiff perfdiff.cpp)
66
add_executable(parse parse.cpp)
@@ -18,8 +18,6 @@ if (SIMDJSON_GOOGLE_BENCHMARKS)
1818
link_libraries(benchmark::benchmark)
1919
add_executable(bench_parse_call bench_parse_call.cpp)
2020
add_executable(bench_dom_api bench_dom_api.cpp)
21-
target_link_libraries(bench_dom_api test-data)
22-
target_link_libraries(bench_parse_call test-data)
2321
endif()
2422

2523
if (SIMDJSON_COMPETITION)
@@ -37,16 +35,3 @@ if (SIMDJSON_COMPETITION)
3735
endif()
3836

3937
include(checkperf.cmake)
40-
41-
# IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
42-
# add_test(NAME checkperf
43-
# COMMAND ${CMAKE_COMMAND} -E env
44-
# CHECKPERF_REPOSITORY=${SIMDJSON_GITHUB_REPOSITORY}
45-
# CHECKPERF_BRANCH=master
46-
# CHECKPERF_DIR=${CMAKE_CURRENT_BINARY_DIR}/simdjson-master
47-
# CHECKPERF_CMAKECACHE=${SIMDJSON_USER_CMAKECACHE}
48-
# bash ${CMAKE_CURRENT_SOURCE_DIR}/checkperf.sh ${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json)
49-
# set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
50-
# set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
51-
# set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
52-
# ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")

‎benchmark/benchfeatures.cpp

Lines changed: 50 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ void exit_usage(string message) {
7575
}
7676

7777
struct option_struct {
78-
architecture arch = architecture::UNSUPPORTED;
7978
bool stage1_only = false;
8079

8180
int32_t iterations = 400;
@@ -98,10 +97,7 @@ struct option_struct {
9897
verbose = true;
9998
break;
10099
case 'a':
101-
arch = parse_architecture(optarg);
102-
if (arch == architecture::UNSUPPORTED) {
103-
exit_usage(string("Unsupported option value -a ") + optarg + ": expected -a HASWELL, WESTMERE or ARM64");
104-
}
100+
simdjson::active_implementation = simdjson::available_implementations[optarg];
105101
break;
106102
case 's':
107103
if (!strcmp(optarg, "stage1")) {
@@ -113,15 +109,9 @@ struct option_struct {
113109
}
114110
break;
115111
default:
116-
exit_error("Unexpected argument " + c);
112+
exit_error(string("Unexpected argument ") + std::string(1,static_cast<char>(c)));
117113
}
118114
}
119-
120-
// If architecture is not specified, pick the best supported architecture by default
121-
if (arch == architecture::UNSUPPORTED) {
122-
arch = find_best_supported_architecture();
123-
}
124-
dom::parser::use_implementation(arch);
125115
}
126116

127117
template<typename F>
@@ -150,20 +140,20 @@ struct feature_benchmarker {
150140
benchmarker struct23;
151141
benchmarker struct23_miss;
152142

153-
feature_benchmarker(const simdjson::implementation &parser, event_collector& collector) :
154-
utf8 ("jsonexamples/generated/utf-8.json", parser, collector),
155-
utf8_miss ("jsonexamples/generated/utf-8-miss.json", parser, collector),
156-
escape ("jsonexamples/generated/escape.json", parser, collector),
157-
escape_miss ("jsonexamples/generated/escape-miss.json", parser, collector),
158-
empty ("jsonexamples/generated/0-structurals.json", parser, collector),
159-
empty_miss ("jsonexamples/generated/0-structurals-miss.json", parser, collector),
160-
struct7 ("jsonexamples/generated/7-structurals.json", parser, collector),
161-
struct7_miss ("jsonexamples/generated/7-structurals-miss.json", parser, collector),
162-
struct7_full ("jsonexamples/generated/7-structurals-full.json", parser, collector),
163-
struct15 ("jsonexamples/generated/15-structurals.json", parser, collector),
164-
struct15_miss("jsonexamples/generated/15-structurals-miss.json", parser, collector),
165-
struct23 ("jsonexamples/generated/23-structurals.json", parser, collector),
166-
struct23_miss("jsonexamples/generated/23-structurals-miss.json", parser, collector)
143+
feature_benchmarker(event_collector& collector) :
144+
utf8 (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8.json", collector),
145+
utf8_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8-miss.json", collector),
146+
escape (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape.json", collector),
147+
escape_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape-miss.json", collector),
148+
empty (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals.json", collector),
149+
empty_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals-miss.json", collector),
150+
struct7 (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals.json", collector),
151+
struct7_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-miss.json", collector),
152+
struct7_full (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-full.json", collector),
153+
struct15 (SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals.json", collector),
154+
struct15_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals-miss.json", collector),
155+
struct23 (SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals.json", collector),
156+
struct23_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals-miss.json", collector)
167157
{
168158

169159
}
@@ -185,7 +175,7 @@ struct feature_benchmarker {
185175
}
186176

187177
double cost_per_block(BenchmarkStage stage, const benchmarker& feature, size_t feature_blocks, const benchmarker& base) const {
188-
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / feature_blocks;
178+
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / double(feature_blocks);
189179
}
190180

191181
// Whether we're recording cache miss and branch miss events
@@ -195,7 +185,7 @@ struct feature_benchmarker {
195185

196186
// Base cost of any block (including empty ones)
197187
double base_cost(BenchmarkStage stage) const {
198-
return (empty[stage].best.elapsed_ns() / empty.stats->blocks);
188+
return (empty[stage].best.elapsed_ns() / double(empty.stats->blocks));
199189
}
200190

201191
// Extra cost of a 1-7 structural block over an empty block
@@ -209,7 +199,7 @@ struct feature_benchmarker {
209199
// Rate of 1-7-structural misses per 8-structural flip
210200
double struct1_7_miss_rate(BenchmarkStage stage) const {
211201
if (!has_events()) { return 1; }
212-
return double(struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses()) / struct7_miss.stats->blocks_with_1_structural_flipped;
202+
return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
213203
}
214204

215205
// Extra cost of an 8-15 structural block over a 1-7 structural block
@@ -223,7 +213,7 @@ struct feature_benchmarker {
223213
// Rate of 8-15-structural misses per 8-structural flip
224214
double struct8_15_miss_rate(BenchmarkStage stage) const {
225215
if (!has_events()) { return 1; }
226-
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / struct15_miss.stats->blocks_with_8_structurals_flipped;
216+
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
227217
}
228218

229219
// Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -237,7 +227,7 @@ struct feature_benchmarker {
237227
// Rate of 16-structural misses per 16-structural flip
238228
double struct16_miss_rate(BenchmarkStage stage) const {
239229
if (!has_events()) { return 1; }
240-
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / struct23_miss.stats->blocks_with_16_structurals_flipped;
230+
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
241231
}
242232

243233
// Extra cost of having UTF-8 in a block
@@ -251,7 +241,7 @@ struct feature_benchmarker {
251241
// Rate of UTF-8 misses per UTF-8 flip
252242
double utf8_miss_rate(BenchmarkStage stage) const {
253243
if (!has_events()) { return 1; }
254-
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / utf8_miss.stats->blocks_with_utf8_flipped;
244+
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
255245
}
256246

257247
// Extra cost of having escapes in a block
@@ -265,39 +255,39 @@ struct feature_benchmarker {
265255
// Rate of escape misses per escape flip
266256
double escape_miss_rate(BenchmarkStage stage) const {
267257
if (!has_events()) { return 1; }
268-
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / escape_miss.stats->blocks_with_escapes_flipped;
258+
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
269259
}
270260

271261
double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
272262
// Expected base ns/block (empty)
273263
json_stats& stats = *file.stats;
274-
double expected = base_cost(stage) * stats.blocks;
275-
expected += struct1_7_cost(stage) * stats.blocks_with_1_structural;
276-
expected += utf8_cost(stage) * stats.blocks_with_utf8;
277-
expected += escape_cost(stage) * stats.blocks_with_escapes;
278-
expected += struct8_15_cost(stage) * stats.blocks_with_8_structurals;
279-
expected += struct16_cost(stage) * stats.blocks_with_16_structurals;
280-
return expected / stats.blocks;
264+
double expected = base_cost(stage) * double(stats.blocks);
265+
expected += struct1_7_cost(stage) * double(stats.blocks_with_1_structural);
266+
expected += utf8_cost(stage) * double(stats.blocks_with_utf8);
267+
expected += escape_cost(stage) * double(stats.blocks_with_escapes);
268+
expected += struct8_15_cost(stage) * double(stats.blocks_with_8_structurals);
269+
expected += struct16_cost(stage) * double(stats.blocks_with_16_structurals);
270+
return expected / double(stats.blocks);
281271
}
282272

283273
double calc_expected_miss_cost(BenchmarkStage stage, const benchmarker& file) const {
284274
// Expected base ns/block (empty)
285275
json_stats& stats = *file.stats;
286-
double expected = struct1_7_miss_cost(stage) * stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
287-
expected += utf8_miss_cost(stage) * stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
288-
expected += escape_miss_cost(stage) * stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
289-
expected += struct8_15_miss_cost(stage) * stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
290-
expected += struct16_miss_cost(stage) * stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
291-
return expected / stats.blocks;
276+
double expected = struct1_7_miss_cost(stage) * double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
277+
expected += utf8_miss_cost(stage) * double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
278+
expected += escape_miss_cost(stage) * double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
279+
expected += struct8_15_miss_cost(stage) * double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
280+
expected += struct16_miss_cost(stage) * double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
281+
return expected / double(stats.blocks);
292282
}
293283

294284
double calc_expected_misses(BenchmarkStage stage, const benchmarker& file) const {
295285
json_stats& stats = *file.stats;
296-
double expected = stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
297-
expected += stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
298-
expected += stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
299-
expected += stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
300-
expected += stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
286+
double expected = double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
287+
expected += double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
288+
expected += double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
289+
expected += double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
290+
expected += double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
301291
return expected;
302292
}
303293

@@ -364,10 +354,10 @@ struct feature_benchmarker {
364354
};
365355

366356
void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
367-
double actual = results[stage].best.elapsed_ns() / results.stats->blocks;
357+
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
368358
double calc = features.calc_expected(stage, results);
369-
uint64_t actual_misses = results[stage].best.branch_misses();
370-
uint64_t calc_misses = uint64_t(features.calc_expected_misses(stage, results));
359+
double actual_misses = results[stage].best.branch_misses();
360+
double calc_misses = features.calc_expected_misses(stage, results);
371361
double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
372362
printf(" | %-8s ", benchmark_stage_name(stage));
373363
printf("| %-15s ", filename);
@@ -376,10 +366,10 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
376366
printf("| %8.3g ", calc);
377367
printf("| %8.3g ", actual);
378368
printf("| %+8.3g ", actual - calc);
379-
printf("| %13lu ", calc_misses);
369+
printf("| %13llu ", (long long unsigned)(calc_misses));
380370
if (features.has_events()) {
381-
printf("| %13lu ", actual_misses);
382-
printf("| %+13ld ", int64_t(actual_misses - calc_misses));
371+
printf("| %13llu ", (long long unsigned)(actual_misses));
372+
printf("| %+13lld ", (long long int)(actual_misses - calc_misses));
383373
double miss_adjustment = calc_miss_cost * (double(int64_t(actual_misses - calc_misses)) / calc_misses);
384374
printf("| %8.3g ", calc_miss_cost + miss_adjustment);
385375
printf("| %+8.3g ", actual - (calc + miss_adjustment));
@@ -401,9 +391,9 @@ int main(int argc, char *argv[]) {
401391

402392
// Set up benchmarkers by reading all files
403393
feature_benchmarker features(collector);
404-
benchmarker gsoc_2018("jsonexamples/gsoc-2018.json", collector);
405-
benchmarker twitter("jsonexamples/twitter.json", collector);
406-
benchmarker random("jsonexamples/random.json", collector);
394+
benchmarker gsoc_2018(SIMDJSON_BENCHMARK_DATA_DIR "gsoc-2018.json", collector);
395+
benchmarker twitter(SIMDJSON_BENCHMARK_DATA_DIR "twitter.json", collector);
396+
benchmarker random(SIMDJSON_BENCHMARK_DATA_DIR "random.json", collector);
407397

408398
// Run the benchmarks
409399
progress_bar progress(options.iterations, 100);

‎jsonexamples/CMakeLists.txt

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
1-
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
2-
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
3-
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
1+
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
2+
set(EXAMPLE_JSON ${CMAKE_CURRENT_BINARY_DIR}/twitter.json PARENT_SCOPE)
3+
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_BINARY_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
4+
5+
# Copy static files to the build dir so they live alongside the generated ones
6+
file(GLOB_RECURSE example_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.json *.ndjson)
7+
foreach(example_file ${example_files})
8+
configure_file(${example_file} ${example_file} COPYONLY)
9+
endforeach(example_file)
10+
11+
add_subdirectory(generated)
12+
413
add_library(jsonexamples-data INTERFACE)
5-
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
14+
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/")
15+
add_dependencies(jsonexamples-data ${example_files})
16+
add_dependencies(jsonexamples-data generated-data)

‎jsonexamples/generated/CMakeLists.txt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
set(generated_files
2+
utf-8.json escape.json
3+
0-structurals.json 7-structurals.json 15-structurals.json 23-structurals.json
4+
)
5+
find_package(Ruby QUIET)
6+
if (RUBY_EXECUTABLE)
7+
add_custom_command(
8+
OUTPUT ${generated_files}
9+
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/miss-templates/*.txt
10+
COMMAND ${RUBY_EXECUTABLE} genfeaturejson.rb ${CMAKE_CURRENT_BINARY_DIR}
11+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
12+
)
13+
add_custom_target(generated-data DEPENDS ${generated_files})
14+
else (RUBY_EXECUTABLE)
15+
# TODO make this work without Ruby and make it part of the normal build
16+
add_custom_target(generated-data)
17+
endif (RUBY_EXECUTABLE)

0 commit comments

Comments
 (0)