diff --git a/lib/matplotlib/offsetbox.py b/lib/matplotlib/offsetbox.py
index 77768e738351..f890634c993f 100644
--- a/lib/matplotlib/offsetbox.py
+++ b/lib/matplotlib/offsetbox.py
@@ -1500,16 +1500,23 @@ def __init__(self, ref_artist, use_blit=False):
             ref_artist.set_picker(True)
         self.got_artist = False
         self._use_blit = use_blit and self.canvas.supports_blit
-        self.cids = [
-            self.canvas.callbacks._connect_picklable(
-                'pick_event', self.on_pick),
-            self.canvas.callbacks._connect_picklable(
-                'button_release_event', self.on_release),
+        callbacks = ref_artist.figure._canvas_callbacks
+        self._disconnectors = [
+            functools.partial(
+                callbacks.disconnect, callbacks._connect_picklable(name, func))
+            for name, func in [
+                ("pick_event", self.on_pick),
+                ("button_release_event", self.on_release),
+                ("motion_notify_event", self.on_motion),
+            ]
         ]
 
     # A property, not an attribute, to maintain picklability.
     canvas = property(lambda self: self.ref_artist.figure.canvas)
 
+    cids = property(lambda self: [
+        disconnect.args[0] for disconnect in self._disconnectors[:2]])
+
     def on_motion(self, evt):
         if self._check_still_parented() and self.got_artist:
             dx = evt.x - self.mouse_x
@@ -1536,16 +1543,12 @@ def on_pick(self, evt):
                 self.ref_artist.draw(
                     self.ref_artist.figure._get_renderer())
                 self.canvas.blit()
-            self._c1 = self.canvas.callbacks._connect_picklable(
-                "motion_notify_event", self.on_motion)
             self.save_offset()
 
     def on_release(self, event):
         if self._check_still_parented() and self.got_artist:
             self.finalize_offset()
             self.got_artist = False
-            self.canvas.mpl_disconnect(self._c1)
-
             if self._use_blit:
                 self.ref_artist.set_animated(False)
 
@@ -1558,14 +1561,8 @@ def _check_still_parented(self):
 
     def disconnect(self):
         """Disconnect the callbacks."""
-        for cid in self.cids:
-            self.canvas.mpl_disconnect(cid)
-        try:
-            c1 = self._c1
-        except AttributeError:
-            pass
-        else:
-            self.canvas.mpl_disconnect(c1)
+        for disconnector in self._disconnectors:
+            disconnector()
 
     def save_offset(self):
         pass
diff --git a/lib/matplotlib/tests/test_offsetbox.py b/lib/matplotlib/tests/test_offsetbox.py
index cd57122272df..2d35a602f1fe 100644
--- a/lib/matplotlib/tests/test_offsetbox.py
+++ b/lib/matplotlib/tests/test_offsetbox.py
@@ -450,3 +450,11 @@ def test_paddedbox():
     pb = PaddedBox(ta, pad=15, draw_frame=True)
     ab = AnchoredOffsetbox('lower right', child=pb)
     ax.add_artist(ab)
+
+
+def test_remove_draggable():
+    fig, ax = plt.subplots()
+    an = ax.annotate("foo", (.5, .5))
+    an.draggable(True)
+    an.remove()
+    MouseEvent("button_release_event", fig.canvas, 1, 1)._process()