FIX: correctly manage rasterization state via zoreder

If an artist was rasterized both by setting state on the artist and
be being below the z-order threshold than we would try to stop rasterization
one too many times.

This constructions lets us re-use the `allow_rasterization` logic.

closes matplotlib#24235

Author: tacaswell

lib/matplotlib/axes/_base.py

@@ -3127,23 +3127,21 @@ def draw(self, renderer):
 
         if (rasterization_zorder is not None and
                 artists and artists[0].zorder < rasterization_zorder):
-            renderer.start_rasterizing()
             artists_rasterized = [a for a in artists
                                   if a.zorder < rasterization_zorder]
             artists = [a for a in artists
                        if a.zorder >= rasterization_zorder]
         else:
             artists_rasterized = []
 
-        # the patch draws the background rectangle -- the frame below
-        # will draw the edges
         if self.axison and self._frameon:
-            self.patch.draw(renderer)
+            if artists_rasterized:
+                artists_rasterized = [self.patch] + artists_rasterized
+            else:
+                artists = [self.patch] + artists
 
         if artists_rasterized:
-            for a in artists_rasterized:
-                a.draw(renderer)
-            renderer.stop_rasterizing()
+            _draw_rasterized(self.figure, artists_rasterized, renderer)
 
         mimage._draw_list_compositing_images(
             renderer, self, artists, self.figure.suppressComposite)
@@ -4636,3 +4634,23 @@ def _label_outer_yaxis(self, *, check_patch):
             self.yaxis.set_tick_params(which="both", labelright=False)
             if self.yaxis.offsetText.get_position()[0] == 1:
                 self.yaxis.offsetText.set_visible(False)
+
+
+def _draw_rasterized(figure, artists, renderer):
+    class _MinimalArtist:
+        def get_rasterized(self):
+            return True
+
+        def get_agg_filter(self):
+            return None
+
+        def __init__(self, figure, artists):
+            self.figure = figure
+            self.artists = artists
+
+        @martist.allow_rasterization
+        def draw(self, renderer):
+            for a in self.artists:
+                a.draw(renderer)
+
+    return _MinimalArtist(figure, artists).draw(renderer)

lib/matplotlib/tests/test_axes.py

@@ -8449,3 +8449,11 @@ def get_next_color():
         c = 'red\n'
         mpl.axes.Axes._parse_scatter_color_args(
             c, None, kwargs={}, xsize=2, get_next_color_func=get_next_color)
+
+
+def test_zorder_and_explicit_rasterization():
+    fig, ax = plt.subplots()
+    ax.set_rasterization_zorder(5)
+    ln, = ax.plot(range(5), rasterized=True, zorder=1)
+    b = io.BytesIO()
+    fig.savefig(b, format='pdf')