drm_common: Be smarter when deciding on which CRTC and Encoder to use

Inspired by kmscube, first try to pick the Encoder and CRTC already
associated with the selected Connector, if any. Otherwise try to find
the first matching encoder & CRTC like before.

The previous behavior had problems when using atomic
modesetting (crtc_setup_atomic) when we picked an Encoder & CRTC that
was currently being used by the fbcon together with another Encoder.
drmModeSetCrtc was able to "steal" the CRTC in this case, but using
atomic modesetting we do not seem to get this behavior automatically.

This should also improve behavior somewhat when run on a multi screen
setup with regards to deinit and VT switching (still sometimes you end
up with a blank screen where you previously had a cloned display of
your fbcon)

Author: Anton Kindestam

video/out/drm_common.c

@@ -173,6 +173,27 @@ static bool setup_connector(struct kms *kms, const drmModeRes *res,
 
 static bool setup_crtc(struct kms *kms, const drmModeRes *res)
 {
+    // First try to find currently connected encoder and its current CRTC
+    for (unsigned int i = 0; i < res->count_encoders; i++) {
+        drmModeEncoder *encoder = drmModeGetEncoder(kms->fd, res->encoders[i]);
+        if (!encoder) {
+            MP_WARN(kms, "Cannot retrieve encoder %u:%u: %s\n",
+                    i, res->encoders[i], mp_strerror(errno));
+            continue;
+        }
+
+        if (encoder->encoder_id == kms->connector->encoder_id && encoder->crtc_id != 0) {
+            MP_VERBOSE(kms, "Connector %u currently connected to encoder %u\n",
+                       kms->connector->connector_id, kms->connector->encoder_id);
+            kms->encoder = encoder;
+            kms->crtc_id = encoder->crtc_id;
+            goto success;
+        }
+
+        drmModeFreeEncoder(encoder);
+    }
+
+    // Otherwise pick first legal encoder and CRTC combo for the connector
     for (unsigned int i = 0; i < kms->connector->count_encoders; ++i) {
         drmModeEncoder *encoder
             = drmModeGetEncoder(kms->fd, kms->connector->encoders[i]);
@@ -190,7 +211,7 @@ static bool setup_crtc(struct kms *kms, const drmModeRes *res)
 
             kms->encoder = encoder;
             kms->crtc_id = res->crtcs[j];
-            return true;
+            goto success;
         }
 
         drmModeFreeEncoder(encoder);
@@ -199,6 +220,11 @@ static bool setup_crtc(struct kms *kms, const drmModeRes *res)
     MP_ERR(kms, "Connector %u has no suitable CRTC\n",
            kms->connector->connector_id);
     return false;
+
+  success:
+    MP_VERBOSE(kms, "Selected Encoder %u with CRTC %u\n",
+               kms->encoder->encoder_id, kms->crtc_id);
+    return true;
 }
 
 static bool setup_mode(struct kms *kms, int mode_id)