from pylab import *
from matplotlib.backend_bases import MouseEvent
rcdefaults()

fig, axs = plt.subplots(2)
for i, ax in enumerate(axs):
    ax.set(xlim=(0, 1), ylim=(0, 1))
    ann = ax.annotate(
        "hello", xy=(.4, .4), xytext=(.6, .6), arrowprops={"arrowstyle": "->"})
    ax.figure.canvas.draw()
    xs = np.linspace(*ax.get_xlim(), 30)
    ys = np.linspace(*ax.get_ylim(), 30)
    for x in xs:
        for y in ys:
            event = MouseEvent(
                "button_press_event",
                ax.figure.canvas,
                *ax.transData.transform_point((x, y)))
            art = [ann, ann.arrow_patch][i]
            ax.plot(x, y, ".", c="g" if art.contains(event)[0] else "r")
plt.show()