diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
index 6f8dfe315cbd..1015800cbece 100644
--- a/lib/matplotlib/font_manager.py
+++ b/lib/matplotlib/font_manager.py
@@ -947,7 +947,7 @@ class FontManager(object):
     # Increment this version number whenever the font cache data
     # format or behavior has changed and requires a existing font
     # cache files to be rebuilt.
-    __version__ = 300
+    __version__ = 310
 
     def __init__(self, size=None, weight='normal'):
         self._version = self.__version__
@@ -975,19 +975,13 @@ def __init__(self, size=None, weight='normal'):
         self.defaultFamily = {
             'ttf': 'DejaVu Sans',
             'afm': 'Helvetica'}
-        self.defaultFont = {}
 
         ttffiles = findSystemFonts(paths) + findSystemFonts()
-        self.defaultFont['ttf'] = next(
-            (fname for fname in ttffiles
-             if fname.lower().endswith("dejavusans.ttf")),
-            ttffiles[0])
         self.ttflist = createFontList(ttffiles)
 
         afmfiles = (findSystemFonts(paths, fontext='afm')
                     + findSystemFonts(fontext='afm'))
         self.afmlist = createFontList(afmfiles, fontext='afm')
-        self.defaultFont['afm'] = afmfiles[0] if afmfiles else None
 
     @cbook.deprecated("3.0")
     @property
@@ -999,6 +993,13 @@ def ttffiles(self):
     def afmfiles(self):
         return [font.fname for font in self.afmlist]
 
+    @property
+    def defaultFont(self):
+        # Lazily evaluated (findfont then caches the result) to avoid including
+        # the venv path in the json serialization.
+        return {ext: self.findfont(family, fontext=ext)
+                for ext, family in self.defaultFamily.items()}
+
     def get_default_weight(self):
         """
         Return the default font weight.