py/formatfloat: Back off float comparisons by 2 eps to avoid rounding errors.

dpwe · dpwe · commit af117625b744 · 2022-07-15T12:39:57.000-04:00
diff --git a/py/formatfloat.c b/py/formatfloat.c
@@ -96,8 +96,22 @@ static inline int fp_isless1(float x) {
 #define fp_iszero(x) (x == 0)
 #define fp_isless1(x) (x < 1.0)
 
+union floatbits {
+    double f;
+    uint64_t u;
+};
+
 #endif // MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT/DOUBLE
 
+static inline int fp_ge_eps(FPTYPE x, FPTYPE y) {
+    union floatbits fb_y = {y};
+    // Back off 2 eps.
+    // This is valid for almost all values, but in practice
+    // it's only used when y = 1eX for X>=0.
+    fb_y.u -= 2;
+    return (x >= fb_y.f);
+}
+
 static const FPTYPE g_pos_pow[] = {
     #if FPDECEXP > 32
     MICROPY_FLOAT_CONST(1e256), MICROPY_FLOAT_CONST(1e128), MICROPY_FLOAT_CONST(1e64),
@@ -254,14 +268,14 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch
         // mantissa.
         FPTYPE u_base = FPCONST(1.0);
         for (e = 0, e1 = FPDECEXP; e1; e1 >>= 1, pos_pow++) {
-            // Use "volatile" to force the result to be stored in a float64.
-            // Without this, nanbox build would sometimes fail to report
-            // "equality" leading to weird results.
-            #if defined(__i386__)
-            volatile
-            #endif
             FPTYPE next_u = u_base * *pos_pow;
-            if (f >= next_u) {
+            // fp_ge_eps performs "f >= (next_u - 2eps)" so that if, for
+            // numerical reasons, f is very close to a power of ten but
+            // not strictly equal, we still treat it as that power of 10.
+            // The comparison was failing for maybe 10% of 1eX values, but
+            // although rounding fixed many of them, there were still some
+            // rendering as 9.99999998e(X-1).
+            if (fp_ge_eps(f, next_u)) {
                 u_base = next_u;
                 e += e1;
             }
@@ -359,9 +373,11 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch
                 }
             }
         }
-        // Figure how many u_bases we can fit into f.
         for (d = 0; d < 9; ++d) {
-            if (f < u_base) {
+            // This is essentially "if (f < u_base)", but with 2eps margin
+            // so that if f is just a tiny bit smaller, we treat it as
+            // equal (and accept the additional digit value).
+            if (!fp_ge_eps(f, u_base)) {
                 break;
             }
             f -= u_base;
diff --git a/tests/float/float_format_ints.py b/tests/float/float_format_ints.py
@@ -12,6 +12,9 @@
             print(title, "with format", f_fmt, "gives", f_fmt.format(f))
             print(title, "with format", g_fmt, "gives", g_fmt.format(f))
 
+# Check that all the powers of 10 (that fit in float32) format right.
+for i in range(32):
+    print("{:.12g}".format(float("1e" + str(i))))
 
 # 16777215 is 2^24 - 1, the largest integer that can be completely held
 # in a float32.
