diff --git a/lib/matplotlib/offsetbox.py b/lib/matplotlib/offsetbox.py
index 56b10c785ff9..7ac89450c813 100644
--- a/lib/matplotlib/offsetbox.py
+++ b/lib/matplotlib/offsetbox.py
@@ -1486,7 +1486,7 @@ def finalize_offset(self):
     def __init__(self, ref_artist, use_blit=False):
         self.ref_artist = ref_artist
         if not ref_artist.pickable():
-            ref_artist.set_picker(True)
+            ref_artist.set_picker(self._picker)
         self.got_artist = False
         self._use_blit = use_blit and self.canvas.supports_blit
         callbacks = self.canvas.callbacks
@@ -1500,6 +1500,11 @@ def __init__(self, ref_artist, use_blit=False):
             ]
         ]
 
+    @staticmethod
+    def _picker(artist, mouseevent):
+        # A custom picker to prevent dragging on mouse scroll events
+        return (artist.contains(mouseevent) and mouseevent.name != "scroll_event"), {}
+
     # A property, not an attribute, to maintain picklability.
     canvas = property(lambda self: self.ref_artist.get_figure(root=True).canvas)
     cids = property(lambda self: [