Fix unit info setting axis label

Author: dstansby

lib/matplotlib/axes/_base.py

@@ -2580,7 +2580,9 @@ def _process_unit_info(self, datasets=None, kwargs=None, *, convert=True):
             except KeyError:
                 raise ValueError(f"Invalid axis name: {axis_name!r}") from None
             # Update from data if axis is already set but no unit is set yet.
-            if axis is not None and data is not None and not axis.have_units():
+            if (axis is not None and
+                    data is not None and not
+                    axis.have_units_and_converter()):
                 axis.update_units(data)
         for axis_name, axis in axis_map.items():
             # Return if no axis is set.

lib/matplotlib/axis.py

@@ -1652,8 +1652,17 @@ def _update_axisinfo(self):
         self.set_default_intervals()
 
     def have_units(self):
+        """
+        Return `True` if units or a converter have been set.
+        """
         return self.converter is not None or self.units is not None
 
+    def have_units_and_converter(self):
+        """
+        Return `True` if units and a converter have been set.
+        """
+        return self.converter is not None and self.units is not None
+
     def convert_units(self, x):
         # If x is natively supported by Matplotlib, doesn't need converting
         if munits._is_natively_supported(x):