diff --git a/lib/matplotlib/tests/test_image.py b/lib/matplotlib/tests/test_image.py
index ae1379a4e796..d8a5bf54614e 100644
--- a/lib/matplotlib/tests/test_image.py
+++ b/lib/matplotlib/tests/test_image.py
@@ -481,6 +481,40 @@ def test_jpeg_alpha():
     assert image.getpixel((0, 0)) == (254, 0, 0)
 
 
+@cleanup
+def test_minimized_rasterized():
+    # This ensures that the rasterized content in the colorbars is
+    # only as thick as the colorbar, and doesn't extend to other parts
+    # of the image.  See #5814.  While the original bug exists only
+    # in Postscript, the best way to detect it is to generate SVG
+    # and then parse the output to make sure the two colorbar images
+    # are the same size.
+    from xml.etree import ElementTree
+
+    np.random.seed(0)
+    data = np.random.rand(10, 10)
+
+    fig, ax = plt.subplots(1, 2)
+    p1 = ax[0].pcolormesh(data)
+    p2 = ax[1].pcolormesh(data)
+
+    plt.colorbar(p1, ax=ax[0])
+    plt.colorbar(p2, ax=ax[1])
+
+    buff = io.BytesIO()
+    plt.savefig(buff, format='svg')
+
+    buff = io.BytesIO(buff.getvalue())
+    tree = ElementTree.parse(buff)
+    width = None
+    for image in tree.iter('image'):
+        if width is None:
+            width = image['width']
+        else:
+            if image['width'] != width:
+                assert False
+
+
 if __name__=='__main__':
     import nose
     nose.runmodule(argv=['-s','--with-doctest'], exit=False)
diff --git a/src/_backend_agg.cpp b/src/_backend_agg.cpp
index 9e9ae5ce9f3f..e4721565f52f 100644
--- a/src/_backend_agg.cpp
+++ b/src/_backend_agg.cpp
@@ -210,10 +210,10 @@ agg::rect_i RendererAgg::get_content_extents()
         }
     }
 
-    r.x1 = std::max(0, r.x1 - 1);
-    r.y1 = std::max(0, r.y1 - 1);
-    r.x2 = std::max(r.x2 + 1, (int)width);
-    r.y2 = std::max(r.y2 + 1, (int)height);
+    r.x1 = std::max(0, r.x1);
+    r.y1 = std::max(0, r.y1);
+    r.x2 = std::min(r.x2 + 1, (int)width);
+    r.y2 = std::min(r.y2 + 1, (int)height);
 
     return r;
 }