diff --git a/lib/matplotlib/_constrained_layout.py b/lib/matplotlib/_constrained_layout.py
index 3561fbfef2b0..c90bacd590ba 100644
--- a/lib/matplotlib/_constrained_layout.py
+++ b/lib/matplotlib/_constrained_layout.py
@@ -361,15 +361,19 @@ def do_constrained_layout(fig, renderer, h_pad, w_pad,
                                         'bottom')
 
                     ###########
-                    # Now we make the widths and heights similar.
+                    # Now we make the widths and heights of position boxes
+                    # similar. (i.e the spine locations)
                     # This allows vertically stacked subplots to have
-                    # different sizes if they occupy different ammounts
+                    # different sizes if they occupy different amounts
                     # of the gridspec:  i.e.
                     # gs = gridspec.GridSpec(3,1)
                     # ax1 = gs[0,:]
                     # ax2 = gs[1:,:]
                     # then drows0 = 1, and drowsC = 2, and ax2
                     # should be at least twice as large as ax1.
+                    # But it can be more than twice as large because
+                    # it needs less room for the labeling.
+                    #
                     # For height, this only needs to be done if the
                     # subplots share a column.  For width if they
                     # share a row.
@@ -387,31 +391,41 @@ def do_constrained_layout(fig, renderer, h_pad, w_pad,
                     dcolsC = (colnumCmax - colnumCmin + 1)
                     dcols0 = (colnum0max - colnum0min + 1)
 
-                    if drowsC > drows0:
+                    if height0 > heightC:
                         if in_same_column(ss0, ssc):
                             ax._poslayoutbox.constrain_height_min(
-                                axc._poslayoutbox.height * drows0 * height0
-                                / drowsC / heightC)
-                    elif drowsC < drows0:
-                        if in_same_column(ss0, ssc):
+                                axc._poslayoutbox.height * height0 / heightC)
+                            # these constraints stop the smaller axes from
+                            # being allowed to go to zero height...
                             axc._poslayoutbox.constrain_height_min(
-                                ax._poslayoutbox.height * drowsC * heightC
-                                / drows0 / drowsC)
+                                ax._poslayoutbox.height * heightC /
+                                (height0*1.8))
                     else:
+                        if in_same_column(ss0, ssc):
+                            axc._poslayoutbox.constrain_height_min(
+                                ax._poslayoutbox.height * heightC / height0)
+                            ax._poslayoutbox.constrain_height_min(
+                                ax._poslayoutbox.height * height0 /
+                                (heightC*1.8))
+                    if drows0 == drowsC:
                         ax._poslayoutbox.constrain_height(
                                 axc._poslayoutbox.height * height0 / heightC)
                     # widths...
-                    if dcolsC > dcols0:
+                    if width0 > widthC:
                         if in_same_row(ss0, ssc):
                             ax._poslayoutbox.constrain_width_min(
-                                    axc._poslayoutbox.width * dcols0 * width0
-                                    / dcolsC / widthC)
-                    elif dcolsC < dcols0:
-                        if in_same_row(ss0, ssc):
+                                    axc._poslayoutbox.width * width0 / widthC)
                             axc._poslayoutbox.constrain_width_min(
-                                    ax._poslayoutbox.width * dcolsC * widthC
-                                    / dcols0 / width0)
+                                    ax._poslayoutbox.width * widthC /
+                                    (width0*1.8))
                     else:
+                        if in_same_row(ss0, ssc):
+                            axc._poslayoutbox.constrain_width_min(
+                                    ax._poslayoutbox.width * widthC / width0)
+                            ax._poslayoutbox.constrain_width_min(
+                                    axc._poslayoutbox.width * width0 /
+                                    (widthC*1.8))
+                    if dcols0 == dcolsC:
                         ax._poslayoutbox.constrain_width(
                                 axc._poslayoutbox.width * width0 / widthC)
 
diff --git a/lib/matplotlib/tests/baseline_images/test_constrainedlayout/constrained_layout17.png b/lib/matplotlib/tests/baseline_images/test_constrainedlayout/constrained_layout17.png
new file mode 100644
index 000000000000..41ceab8f38d2
Binary files /dev/null and b/lib/matplotlib/tests/baseline_images/test_constrainedlayout/constrained_layout17.png differ
diff --git a/lib/matplotlib/tests/test_constrainedlayout.py b/lib/matplotlib/tests/test_constrainedlayout.py
index 5333ba8c85b5..de3bee3bba8a 100644
--- a/lib/matplotlib/tests/test_constrainedlayout.py
+++ b/lib/matplotlib/tests/test_constrainedlayout.py
@@ -345,3 +345,21 @@ def test_constrained_layout16():
     fig, ax = plt.subplots(constrained_layout=True)
     example_plot(ax, fontsize=12)
     ax2 = fig.add_axes([0.2, 0.2, 0.4, 0.4])
+
+
+@image_comparison(baseline_images=['constrained_layout17'],
+        extensions=['png'])
+def test_constrained_layout17():
+    'Test uneven gridspecs'
+    fig = plt.figure(constrained_layout=True)
+    gs = gridspec.GridSpec(3, 3, figure=fig)
+
+    ax1 = fig.add_subplot(gs[0, 0])
+    ax2 = fig.add_subplot(gs[0, 1:])
+    ax3 = fig.add_subplot(gs[1:, 0:2])
+    ax4 = fig.add_subplot(gs[1:, -1])
+
+    example_plot(ax1)
+    example_plot(ax2)
+    example_plot(ax3)
+    example_plot(ax4)