diff --git a/lib/matplotlib/axes/_base.py b/lib/matplotlib/axes/_base.py
index 8a105f4da67d..8e348fea4675 100644
--- a/lib/matplotlib/axes/_base.py
+++ b/lib/matplotlib/axes/_base.py
@@ -2545,9 +2545,7 @@ def _process_unit_info(self, datasets=None, kwargs=None, *, convert=True):
             except KeyError:
                 raise ValueError(f"Invalid axis name: {axis_name!r}") from None
             # Update from data if axis is already set but no unit is set yet.
-            if (axis is not None and
-                    data is not None and
-                    not axis._have_units_and_converter()):
+            if axis is not None and data is not None and not axis.have_units():
                 axis.update_units(data)
         for axis_name, axis in axis_map.items():
             # Return if no axis is set.
diff --git a/lib/matplotlib/axis.py b/lib/matplotlib/axis.py
index c0e706e2b3a1..97cf5745c656 100644
--- a/lib/matplotlib/axis.py
+++ b/lib/matplotlib/axis.py
@@ -1728,17 +1728,8 @@ def _update_axisinfo(self):
         self.set_default_intervals()
 
     def have_units(self):
-        """
-        Return `True` if units or a converter have been set.
-        """
         return self.converter is not None or self.units is not None
 
-    def _have_units_and_converter(self):
-        """
-        Return `True` if units and a converter have been set.
-        """
-        return self.converter is not None and self.units is not None
-
     def convert_units(self, x):
         # If x is natively supported by Matplotlib, doesn't need converting
         if munits._is_natively_supported(x):
diff --git a/lib/matplotlib/testing/jpl_units/UnitDbl.py b/lib/matplotlib/testing/jpl_units/UnitDbl.py
index 5b0d773f7731..5226c06ad54b 100644
--- a/lib/matplotlib/testing/jpl_units/UnitDbl.py
+++ b/lib/matplotlib/testing/jpl_units/UnitDbl.py
@@ -91,8 +91,6 @@ def __bool__(self):
 
     def _cmp(self, op, rhs):
         """Check that *self* and *rhs* share units; compare them using *op*."""
-        if rhs is None:
-            return NotImplemented
         self.checkSameUnits(rhs, "compare")
         return op(self._value, rhs._value)
 
diff --git a/lib/matplotlib/tests/baseline_images/test_patches/units_rectangle.png b/lib/matplotlib/tests/baseline_images/test_patches/units_rectangle.png
index f60e962f342c..651e1e2ef95a 100644
Binary files a/lib/matplotlib/tests/baseline_images/test_patches/units_rectangle.png and b/lib/matplotlib/tests/baseline_images/test_patches/units_rectangle.png differ
diff --git a/lib/matplotlib/tests/test_dates.py b/lib/matplotlib/tests/test_dates.py
index 7abebcde7e56..cc9a83a5c5b6 100644
--- a/lib/matplotlib/tests/test_dates.py
+++ b/lib/matplotlib/tests/test_dates.py
@@ -636,6 +636,27 @@ def test_concise_formatter_show_offset(t_delta, expected):
     assert formatter.get_offset() == expected
 
 
+def test_concise_converter_stays():
+    # This test demonstrates problems introduced by gh-23417 (reverted in gh-25278)
+    # In particular, downstream libraries like Pandas had their designated converters
+    # overridden by actions like setting xlim (or plotting additional points using
+    # stdlib/numpy dates and string date representation, which otherwise work fine with
+    # their date converters)
+    # While this is a bit of a toy example that would be unusual to see it demonstrates
+    # the same ideas (namely having a valid converter already applied that is desired)
+    # without introducing additional subclasses.
+    # See also discussion at gh-25219 for how Pandas was affected
+    x = [datetime.datetime(2000, 1, 1), datetime.datetime(2020, 2, 20)]
+    y = [0, 1]
+    fig, ax = plt.subplots()
+    ax.plot(x, y)
+    # Bypass Switchable date converter
+    ax.xaxis.converter = conv = mdates.ConciseDateConverter()
+    assert ax.xaxis.units is None
+    ax.set_xlim(*x)
+    assert ax.xaxis.converter == conv
+
+
 def test_offset_changes():
     fig, ax = plt.subplots()
 
diff --git a/lib/matplotlib/tests/test_units.py b/lib/matplotlib/tests/test_units.py
index 85a63ecc2f10..d3b8c5a71643 100644
--- a/lib/matplotlib/tests/test_units.py
+++ b/lib/matplotlib/tests/test_units.py
@@ -283,19 +283,3 @@ def test_plot_kernel():
     # just a smoketest that fail
     kernel = Kernel([1, 2, 3, 4, 5])
     plt.plot(kernel)
-
-
-@pytest.mark.parametrize('plot_meth_name', ['scatter', 'plot'])
-def test_unit_axis_label(plot_meth_name):
-    # Check that the correct Axis labels are set on plots with units
-    import matplotlib.testing.jpl_units as units
-    units.register()
-
-    fig, ax = plt.subplots()
-    ax.xaxis.set_units('m')
-    ax.yaxis.set_units('sec')
-    plot_method = getattr(ax, plot_meth_name)
-    plot_method(np.arange(3) * units.m, np.arange(3) * units.sec)
-    assert ax.get_xlabel() == 'm'
-    assert ax.get_ylabel() == 'sec'
-    plt.close('all')