diff --git a/lib/matplotlib/backend_bases.py b/lib/matplotlib/backend_bases.py
index 0e1e5d41beab..cfd98db38f4d 100644
--- a/lib/matplotlib/backend_bases.py
+++ b/lib/matplotlib/backend_bases.py
@@ -2802,15 +2802,6 @@ def __init__(self, canvas):
         self.mode = ''  # a mode string for the status bar
         self.set_history_buttons()
 
-        @partial(canvas.mpl_connect, 'draw_event')
-        def update_stack(event):
-            nav_info = self._nav_stack()
-            if (nav_info is None  # True initial navigation info.
-                    # An axes has been added or removed, so update the
-                    # navigation info too.
-                    or set(nav_info) != set(self.canvas.figure.axes)):
-                self.push_current()
-
     def set_message(self, s):
         """Display a message on toolbar or in status bar."""
 
@@ -2955,6 +2946,10 @@ def press_pan(self, event):
             self._button_pressed = None
             return
 
+        if self._nav_stack() is None:
+            # set the home button to this view
+            self.push_current()
+
         x, y = event.x, event.y
         self._xypress = []
         for i, a in enumerate(self.canvas.figure.get_axes()):
@@ -2990,6 +2985,10 @@ def press_zoom(self, event):
             self._button_pressed = None
             return
 
+        if self._nav_stack() is None:
+            # set the home button to this view
+            self.push_current()
+
         x, y = event.x, event.y
         self._xypress = []
         for i, a in enumerate(self.canvas.figure.get_axes()):