diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
index 9bc5ee9ee712..b6882826a17e 100644
--- a/lib/matplotlib/font_manager.py
+++ b/lib/matplotlib/font_manager.py
@@ -952,7 +952,45 @@ def pickle_load(filename):
         data = pickle.load(fh)
     return data
 
-class FontManager:
+
+class TempCache(object):
+    """
+    A class to store temporary caches that are (a) not saved to disk
+    and (b) invalidated whenever certain font-related
+    rcParams---namely the family lookup lists---are changed or the
+    font cache is reloaded.  This avoids the expensive linear search
+    through all fonts every time a font is looked up.
+    """
+    # A list of rcparam names that, when changed, invalidated this
+    # cache.
+    invalidating_rcparams = (
+        'font.serif', 'font.sans-serif', 'font.cursive', 'font.fantasy',
+        'font.monospace')
+
+    def __init__(self):
+        self._lookup_cache = {}
+        self._last_rcParams = self.make_rcparams_key()
+
+    def make_rcparams_key(self):
+        return [id(fontManager)] + [
+            rcParams[param] for param in self.invalidating_rcparams]
+
+    def get(self, prop):
+        key = self.make_rcparams_key()
+        if key != self._last_rcParams:
+            self._lookup_cache = {}
+            self._last_rcParams = key
+        return self._lookup_cache.get(prop)
+
+    def set(self, prop, value):
+        key = self.make_rcparams_key()
+        if key != self._last_rcParams:
+            self._lookup_cache = {}
+            self._last_rcParams = key
+        self._lookup_cache[prop] = value
+
+
+class FontManager(object):
     """
     On import, the :class:`FontManager` singleton instance creates a
     list of TrueType fonts based on the font properties: name, style,
@@ -1015,9 +1053,6 @@ def __init__(self, size=None, weight='normal'):
         else:
             self.defaultFont['afm'] = None
 
-        self.ttf_lookup_cache = {}
-        self.afm_lookup_cache = {}
-
     def get_default_weight(self):
         """
         Return the default font weight.
@@ -1200,15 +1235,13 @@ def findfont(self, prop, fontext='ttf', directory=None,
             return fname
 
         if fontext == 'afm':
-            font_cache = self.afm_lookup_cache
             fontlist = self.afmlist
         else:
-            font_cache = self.ttf_lookup_cache
             fontlist = self.ttflist
 
         if directory is None:
-            cached = font_cache.get(hash(prop))
-            if cached:
+            cached = _lookup_cache[fontext].get(prop)
+            if cached is not None:
                 return cached
 
         best_score = 1e64
@@ -1266,7 +1299,7 @@ def findfont(self, prop, fontext='ttf', directory=None,
                 raise ValueError("No valid font could be found")
 
         if directory is None:
-            font_cache[hash(prop)] = result
+            _lookup_cache[fontext].set(prop, result)
         return result
 
 
@@ -1348,6 +1381,11 @@ def findfont(prop, fontext='ttf'):
 
     fontManager = None
 
+    _lookup_cache = {
+        'ttf': TempCache(),
+        'afm': TempCache()
+    }
+
     def _rebuild():
         global fontManager
         fontManager = FontManager()
diff --git a/lib/matplotlib/tests/test_font_manager.py b/lib/matplotlib/tests/test_font_manager.py
index c55be8846759..df2675219023 100644
--- a/lib/matplotlib/tests/test_font_manager.py
+++ b/lib/matplotlib/tests/test_font_manager.py
@@ -6,7 +6,7 @@
 
 import os
 
-from matplotlib.font_manager import findfont, FontProperties, _rebuild
+from matplotlib.font_manager import findfont, FontProperties
 from matplotlib import rc_context
 
 
@@ -14,10 +14,6 @@ def test_font_priority():
     with rc_context(rc={
             'font.sans-serif':
             ['cmmi10', 'Bitstream Vera Sans']}):
-        # force the font manager to rebuild it self
-        _rebuild()
         font = findfont(
             FontProperties(family=["sans-serif"]))
     assert_equal(os.path.basename(font), 'cmmi10.ttf')
-    # force it again
-    _rebuild()