diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
index 3d72bc9bd5c0..6f8dfe315cbd 100644
--- a/lib/matplotlib/font_manager.py
+++ b/lib/matplotlib/font_manager.py
@@ -1356,17 +1356,20 @@ def _rebuild():
     if _fmcache:
         try:
             fontManager = json_load(_fmcache)
+        except FileNotFoundError:
+            _log.debug("No font cache found %s", _fmcache)
+        except json.JSONDecodeError:
+            _log.warning("Font cache parsing failed %s", _fmcache)
+        else:
             if (not hasattr(fontManager, '_version') or
                 fontManager._version != FontManager.__version__):
-                _rebuild()
+                _log.debug("Font cache needs rebuild (version mismatch)")
+                fontManager = None
             else:
                 fontManager.default_size = None
                 _log.debug("Using fontManager instance from %s", _fmcache)
-        except TimeoutError:
-            raise
-        except Exception:
-            _rebuild()
-    else:
+
+    if fontManager is None:
         _rebuild()
 
     def findfont(prop, **kw):