Hack around VDPAU race conditions when compositing.

Once in a while we fail to redirect windows.

Author: dancingmirrors

video/out/vo_nouveau.c

@@ -977,6 +977,26 @@ static void destroy_vdpau_objects(struct vo *vo)
 
     free_video_specific(vo);
 
+    // Wait for all queued surfaces to become idle before destroying them.
+    if (vc->flip_queue != VDP_INVALID_HANDLE) {
+        VdpTime dummy;
+        for (int i = 0; i < vc->num_output_surfaces; i++) {
+            if (vc->output_surfaces[i] != VDP_INVALID_HANDLE) {
+                vdp_st = vdp->presentation_queue_block_until_surface_idle(
+                    vc->flip_queue, vc->output_surfaces[i], &dummy);
+                CHECK_VDP_WARNING(vo, "Error waiting for surface idle");
+            }
+        }
+        if (vc->rotation_surface != VDP_INVALID_HANDLE) {
+            vdp_st = vdp->presentation_queue_block_until_surface_idle(
+                vc->flip_queue, vc->rotation_surface, &dummy);
+            CHECK_VDP_WARNING(vo, "Error waiting for rotation surface idle");
+        }
+
+        // Unfortunately VDPAU seems to be inherently racy.
+        mp_sleep_ns(20000);
+    }
+
     if (vc->flip_queue != VDP_INVALID_HANDLE) {
         vdp_st = vdp->presentation_queue_destroy(vc->flip_queue);
         CHECK_VDP_WARNING(vo, "Error when calling vdp_presentation_queue_destroy");

video/out/x11_common.c

@@ -897,8 +897,15 @@ void vo_x11_uninit(struct vo *vo)
 
     set_screensaver(x11, true);
 
-    if (x11->window != None && x11->window != x11->rootwin)
+    if (x11->window != None && x11->window != x11->rootwin) {
+        // Reset compositor bypass hint before destroying the window to avoid
+        // leaving compositors in a bad state.
+        long hint = 0; // 0 = enable compositor
+        XChangeProperty(x11->display, x11->window, XA(x11, _NET_WM_BYPASS_COMPOSITOR),
+                        XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&hint, 1);
+        XSync(x11->display, False);
         XDestroyWindow(x11->display, x11->window);
+    }
     if (x11->xic)
         XDestroyIC(x11->xic);
     if (x11->colormap != None)