diff --git a/lib/matplotlib/backend_bases.py b/lib/matplotlib/backend_bases.py
index 5af60bddf9c9..70eff2cf566d 100644
--- a/lib/matplotlib/backend_bases.py
+++ b/lib/matplotlib/backend_bases.py
@@ -2976,11 +2976,7 @@ def _update_cursor(self, event):
         """
         Update the cursor after a mouse move event or a tool (de)activation.
         """
-        if not event.inaxes or not self.mode:
-            if self._lastCursor != cursors.POINTER:
-                self.set_cursor(cursors.POINTER)
-                self._lastCursor = cursors.POINTER
-        else:
+        if self.mode and event.inaxes and event.inaxes.get_navigate():
             if (self.mode == _Mode.ZOOM
                     and self._lastCursor != cursors.SELECT_REGION):
                 self.set_cursor(cursors.SELECT_REGION)
@@ -2989,6 +2985,9 @@ def _update_cursor(self, event):
                   and self._lastCursor != cursors.MOVE):
                 self.set_cursor(cursors.MOVE)
                 self._lastCursor = cursors.MOVE
+        elif self._lastCursor != cursors.POINTER:
+            self.set_cursor(cursors.POINTER)
+            self._lastCursor = cursors.POINTER
 
     @contextmanager
     def _wait_cursor_for_draw_cm(self):
diff --git a/lib/matplotlib/backend_tools.py b/lib/matplotlib/backend_tools.py
index 4b594442f899..459e71e4f526 100644
--- a/lib/matplotlib/backend_tools.py
+++ b/lib/matplotlib/backend_tools.py
@@ -223,12 +223,11 @@ class SetCursorBase(ToolBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._id_drag = None
-        self._cursor = None
+        self._current_tool = None
         self._default_cursor = cursors.POINTER
         self._last_cursor = self._default_cursor
         self.toolmanager.toolmanager_connect('tool_added_event',
                                              self._add_tool_cbk)
-
         # process current tools
         for tool in self.toolmanager.tools.values():
             self._add_tool(tool)
@@ -243,10 +242,9 @@ def set_figure(self, figure):
 
     def _tool_trigger_cbk(self, event):
         if event.tool.toggled:
-            self._cursor = event.tool.cursor
+            self._current_tool = event.tool
         else:
-            self._cursor = None
-
+            self._current_tool = None
         self._set_cursor_cbk(event.canvasevent)
 
     def _add_tool(self, tool):
@@ -264,16 +262,14 @@ def _add_tool_cbk(self, event):
     def _set_cursor_cbk(self, event):
         if not event:
             return
-
-        if not getattr(event, 'inaxes', False) or not self._cursor:
-            if self._last_cursor != self._default_cursor:
-                self.set_cursor(self._default_cursor)
-                self._last_cursor = self._default_cursor
-        elif self._cursor:
-            cursor = self._cursor
-            if cursor and self._last_cursor != cursor:
-                self.set_cursor(cursor)
-                self._last_cursor = cursor
+        if (self._current_tool and getattr(event, "inaxes", None)
+                and event.inaxes.get_navigate()):
+            if self._last_cursor != self._current_tool.cursor:
+                self.set_cursor(self._current_tool.cursor)
+                self._last_cursor = self._current_tool.cursor
+        elif self._last_cursor != self._default_cursor:
+            self.set_cursor(self._default_cursor)
+            self._last_cursor = self._default_cursor
 
     def set_cursor(self, cursor):
         """