Mark branches as likely/unlikely

jkeiser · jkeiser · commit 060b622e6197 · 2019-09-23T18:31:22.000-07:00
diff --git a/src/generic/stage1_find_marks.h b/src/generic/stage1_find_marks.h
@@ -169,7 +169,7 @@ really_inline void find_structural_bits_64(
 }
 
 int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) {
-  if (len > pj.byte_capacity) {
+  if (unlikely(len > pj.byte_capacity)) {
     std::cerr << "Your ParsedJson object only supports documents up to "
               << pj.byte_capacity << " bytes but you are trying to process "
               << len << " bytes" << std::endl;
@@ -207,7 +207,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p
   /* If we have a final chunk of less than 64 bytes, pad it to 64 with
    * spaces  before processing it (otherwise, we risk invalidating the UTF-8
    * checks). */
-  if (idx < len) {
+  if (likely(idx < len)) {
     uint8_t tmp_buf[64];
     memset(tmp_buf, 0x20, 64);
     memcpy(tmp_buf, buf + idx, len - idx);
@@ -217,22 +217,21 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p
     idx += 64;
   }
 
-  /* finally, flatten out the remaining structurals from the last iteration
-   */
+  /* finally, flatten out the remaining structurals from the last iteration */
   flatten_bits(base_ptr, idx, structurals);
 
   simdjson::ErrorValues error = detect_errors_on_eof(unescaped_chars_error, prev_in_string);
-  if (error != simdjson::SUCCESS) {
+  if (unlikely(error != simdjson::SUCCESS)) {
     return error;
   }
 
   pj.n_structural_indexes = base_ptr - pj.structural_indexes;
   /* a valid JSON file cannot have zero structural indexes - we should have
    * found something */
-  if (pj.n_structural_indexes == 0u) {
+  if (unlikely(pj.n_structural_indexes == 0u)) {
     return simdjson::EMPTY;
   }
-  if (pj.structural_indexes[pj.n_structural_indexes - 1] > len) {
+  if (unlikely(pj.structural_indexes[pj.n_structural_indexes - 1] > len)) {
     return simdjson::UNEXPECTED_ERROR;
   }
   if (len != pj.structural_indexes[pj.n_structural_indexes - 1]) {
diff --git a/src/generic/stage1_find_marks_flatten.h b/src/generic/stage1_find_marks_flatten.h
@@ -43,7 +43,7 @@ really_inline void flatten_bits(uint32_t *&base_ptr, uint32_t idx, uint64_t bits
 
   // Do the next 8 all together (we hope in most cases it won't happen at all
   // and the branch is easily predicted).
-  if (cnt > 8) {
+  if (unlikely(cnt > 8)) {
     for (int i=8; i<16; i++) {
       base_ptr[i] = idx + trailing_zeroes(bits);
       bits = bits & (bits - 1);
@@ -53,14 +53,15 @@ really_inline void flatten_bits(uint32_t *&base_ptr, uint32_t idx, uint64_t bits
   // Most files don't have 16+ structurals per block, so we take several basically guaranteed
   // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
   // or the start of a value ("abc" true 123) every 4 characters.
-  if (cnt > 16) {
+  if (unlikely(cnt > 16)) {
     uint32_t i = 16;
     do {
       base_ptr[i] = idx + trailing_zeroes(bits);
       bits = bits & (bits - 1);
       i++;
     } while (i < cnt);
   }
+
   base_ptr += cnt;
 }
 #endif // SIMDJSON_NAIVE_FLATTEN
diff --git a/src/haswell/simdutf8check.h b/src/haswell/simdutf8check.h
@@ -218,7 +218,7 @@ struct utf8_checker<Architecture::HASWELL> {
     __m256i any_bits_on = in.reduce([&](auto a, auto b) {
       return _mm256_or_si256(a, b);
     });
-    if ((_mm256_testz_si256(any_bits_on, high_bit)) == 1) {
+    if (likely(_mm256_testz_si256(any_bits_on, high_bit) == 1)) {
       // it is ascii, we just check continuation
       this->has_error = _mm256_or_si256(
           _mm256_cmpgt_epi8(this->previous.carried_continuations,
diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h
@@ -107,7 +107,7 @@ really_inline void flatten_bits(uint32_t *&base_ptr, uint32_t idx, uint64_t bits
 
   // Do the next 8 all together (we hope in most cases it won't happen at all
   // and the branch is easily predicted).
-  if (cnt > 8) {
+  if (unlikely(cnt > 8)) {
     for (int i=8; i<16; i++) {
       base_ptr[i] = idx + trailing_zeroes(bits);
       bits = _blsr_u64(bits);
@@ -117,14 +117,15 @@ really_inline void flatten_bits(uint32_t *&base_ptr, uint32_t idx, uint64_t bits
   // Most files don't have 16+ structurals per block, so we take several basically guaranteed
   // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
   // or the start of a value ("abc" true 123) every four characters.
-  if (cnt > 16) {
+  if (unlikely(cnt > 16)) {
     uint32_t i = 16;
     do {
       base_ptr[i] = idx + trailing_zeroes(bits);
       bits = _blsr_u64(bits);
       i++;
     } while (i < cnt);
   }
+
   base_ptr += cnt;
 }
 
