diff --git a/lib/matplotlib/_pylab_helpers.py b/lib/matplotlib/_pylab_helpers.py
index 907e69957ca2..26ba7a171244 100644
--- a/lib/matplotlib/_pylab_helpers.py
+++ b/lib/matplotlib/_pylab_helpers.py
@@ -71,9 +71,10 @@ def destroy(cls, num):
     @classmethod
     def destroy_fig(cls, fig):
         """Destroy figure *fig*."""
-        canvas = getattr(fig, "canvas", None)
-        manager = getattr(canvas, "manager", None)
-        cls.destroy(manager)
+        num = next((manager.num for manager in cls.figs.values()
+                    if manager.canvas.figure == fig), None)
+        if num is not None:
+            cls.destroy(num)
 
     @classmethod
     def destroy_all(cls):
diff --git a/lib/matplotlib/tests/test_backend_bases.py b/lib/matplotlib/tests/test_backend_bases.py
index d8d7f252af1c..7da6ed026b13 100644
--- a/lib/matplotlib/tests/test_backend_bases.py
+++ b/lib/matplotlib/tests/test_backend_bases.py
@@ -60,6 +60,15 @@ def test_get_default_filename(tmpdir):
     assert filename == 'image.png'
 
 
+def test_canvas_change():
+    fig = plt.figure()
+    # Replaces fig.canvas
+    canvas = FigureCanvasBase(fig)
+    # Should still work.
+    plt.close(fig)
+    assert not plt.fignum_exists(fig.number)
+
+
 @pytest.mark.backend('pdf')
 def test_non_gui_warning(monkeypatch):
     plt.subplots()
diff --git a/lib/matplotlib/tests/test_backends_interactive.py b/lib/matplotlib/tests/test_backends_interactive.py
index a14530c2cd75..f701f30df1fc 100644
--- a/lib/matplotlib/tests/test_backends_interactive.py
+++ b/lib/matplotlib/tests/test_backends_interactive.py
@@ -146,8 +146,13 @@ def check_alt_backend(alt_backend):
 @pytest.mark.parametrize("toolbar", ["toolbar2", "toolmanager"])
 @pytest.mark.flaky(reruns=3)
 def test_interactive_backend(backend, toolbar):
-    if backend == "macosx" and toolbar == "toolmanager":
-        pytest.skip("toolmanager is not implemented for macosx.")
+    if backend == "macosx":
+        if toolbar == "toolmanager":
+            pytest.skip("toolmanager is not implemented for macosx.")
+        if toolbar == "toolbar2" and os.environ.get('TRAVIS'):
+            # See https://github.com/matplotlib/matplotlib/issues/18213
+            pytest.skip("toolbar2 for macosx is buggy on Travis.")
+
     proc = subprocess.run(
         [sys.executable, "-c", _test_script,
          json.dumps({"toolbar": toolbar})],