diff --git a/lib/matplotlib/colorbar.py b/lib/matplotlib/colorbar.py
index d826649af167..a5439bf8ea75 100644
--- a/lib/matplotlib/colorbar.py
+++ b/lib/matplotlib/colorbar.py
@@ -992,6 +992,12 @@ def remove(self):
         If the colorbar was created with ``use_gridspec=True`` the previous
         gridspec is restored.
         """
+        if hasattr(self.ax, '_colorbar_info'):
+            parents = self.ax._colorbar_info['parents']
+            for a in parents:
+                if self.ax in a._colorbars:
+                    a._colorbars.remove(self.ax)
+
         self.ax.remove()
 
         self.mappable.callbacks.disconnect(self.mappable.colorbar_cid)
diff --git a/lib/matplotlib/tests/test_colorbar.py b/lib/matplotlib/tests/test_colorbar.py
index 00a6f52698e2..6b691713e7e6 100644
--- a/lib/matplotlib/tests/test_colorbar.py
+++ b/lib/matplotlib/tests/test_colorbar.py
@@ -221,7 +221,7 @@ def test_colorbar_single_scatter():
                          ids=['no gridspec', 'with gridspec'])
 def test_remove_from_figure(use_gridspec):
     """
-    Test `remove_from_figure` with the specified ``use_gridspec`` setting
+    Test `remove` with the specified ``use_gridspec`` setting
     """
     fig, ax = plt.subplots()
     sc = ax.scatter([1, 2], [3, 4], cmap="spring")
@@ -235,6 +235,23 @@ def test_remove_from_figure(use_gridspec):
     assert (pre_position.get_points() == post_position.get_points()).all()
 
 
+def test_remove_from_figure_cl():
+    """
+    Test `remove` with constrained_layout
+    """
+    fig, ax = plt.subplots(constrained_layout=True)
+    sc = ax.scatter([1, 2], [3, 4], cmap="spring")
+    sc.set_array(np.array([5, 6]))
+    fig.draw_without_rendering()
+    pre_position = ax.get_position()
+    cb = fig.colorbar(sc)
+    cb.remove()
+    fig.draw_without_rendering()
+    post_position = ax.get_position()
+    np.testing.assert_allclose(pre_position.get_points(),
+                               post_position.get_points())
+
+
 def test_colorbarbase():
     # smoke test from #3805
     ax = plt.gca()