diff --git a/lib/matplotlib/offsetbox.py b/lib/matplotlib/offsetbox.py
index 32c5bafcde1d..194b950a8a30 100644
--- a/lib/matplotlib/offsetbox.py
+++ b/lib/matplotlib/offsetbox.py
@@ -1486,11 +1486,13 @@ def on_motion(self, evt):
                 self.canvas.draw()
 
     def on_pick(self, evt):
-        if self._check_still_parented() and evt.artist == self.ref_artist:
-            self.mouse_x = evt.mouseevent.x
-            self.mouse_y = evt.mouseevent.y
-            self.got_artist = True
-            if self._use_blit:
+        if self._check_still_parented():
+            if evt.artist == self.ref_artist:
+                self.mouse_x = evt.mouseevent.x
+                self.mouse_y = evt.mouseevent.y
+                self.save_offset()
+                self.got_artist = True
+            if self.got_artist and self._use_blit:
                 self.ref_artist.set_animated(True)
                 self.canvas.draw()
                 self.background = \
@@ -1498,13 +1500,15 @@ def on_pick(self, evt):
                 self.ref_artist.draw(
                     self.ref_artist.figure._get_renderer())
                 self.canvas.blit()
-            self.save_offset()
 
     def on_release(self, event):
         if self._check_still_parented() and self.got_artist:
             self.finalize_offset()
             self.got_artist = False
             if self._use_blit:
+                self.canvas.restore_region(self.background)
+                self.ref_artist.draw(self.ref_artist.figure._get_renderer())
+                self.canvas.blit()
                 self.ref_artist.set_animated(False)
 
     def _check_still_parented(self):