libvncclient: tls_openssl: add expected X.509 fingerprint handling

...for self-signed certs.

Author: bk138

src/libvncclient/tls_openssl.c

@@ -34,6 +34,8 @@
 #endif
 
 static rfbBool rfbTLSInitialized = FALSE;
+static int rfbTLSExpectedFingerprintIndex = -1;
+static int rfbTLSClientIndex = -1;
 
 // Locking callbacks are only initialized if we have mutex support.
 #if defined(LIBVNCSERVER_HAVE_LIBPTHREAD) || defined(LIBVNCSERVER_HAVE_WIN32THREADS)
@@ -154,6 +156,9 @@ InitializeTLS(void)
   SSLeay_add_ssl_algorithms();
   RAND_load_file("/dev/urandom", 1024);
 
+  rfbTLSExpectedFingerprintIndex = SSL_get_ex_new_index(0, "rfbTLSExpectedFingerprintIndex", NULL, NULL, NULL);
+  rfbTLSClientIndex = SSL_get_ex_new_index(0, "rfbTLSClientIndex", NULL, NULL, NULL);
+
   rfbClientLog("OpenSSL version %s initialized.\n", SSLeay_version(SSLEAY_VERSION));
   rfbTLSInitialized = TRUE;
   return TRUE;
@@ -221,6 +226,104 @@ load_crls_from_file(char *file, SSL_CTX *ssl_ctx)
     return FALSE;
 }
 
+/** Convert ASN1_TIME to time_t */
+static time_t asn1time_to_time_t(const ASN1_TIME *t) {
+    struct tm tm;
+    memset(&tm, 0, sizeof(tm));
+
+    // If t is NULL, ASN1_TIME_to_tm() converts the current time, which is not what we want.
+    if (!t || !ASN1_TIME_to_tm(t, &tm)) {
+        return (time_t)-1;
+    }
+
+    return mktime(&tm);  // time_t in localtime; for UTC, use timegm if available
+}
+
+static int cert_fingerprint_mismatch_callback(rfbClient *client, X509 *cert) {
+    if(!cert) {
+        rfbClientErr("No cert given in fingerprint mismatch handling\n");
+        return 0;
+    }
+
+    if (!client || !client->GetX509CertFingerprintMismatchDecision) {
+        rfbClientErr("No client callback given in fingerprint mismatch handling\n");
+        return 0;
+    }
+
+    /* Subject */
+    BIO *bio = BIO_new(BIO_s_mem());
+    if (!bio) {
+        return 0;
+    }
+    X509_NAME_print_ex(bio, X509_get_subject_name(cert), 0, XN_FLAG_RFC2253);
+    char *data;
+    long data_len = BIO_get_mem_data(bio, &data); // owned by OpenSSL
+    if (data_len <= 0) {
+        BIO_free(bio);
+        return 0;
+    }
+    // Allocate a null-terminated copy, bio does not have that
+    char *subject = malloc(data_len + 1);
+    if (subject) {
+        memcpy(subject, data, data_len);
+        subject[data_len] = '\0';
+    }
+    BIO_free(bio);
+
+    /* Validity */
+    time_t not_before = asn1time_to_time_t(X509_get0_notBefore(cert));
+    time_t not_after = asn1time_to_time_t(X509_get0_notAfter(cert));
+
+    /* SHA-256 fingerprint */
+    unsigned char fingerprint[32];
+    if (! X509_digest(cert, EVP_sha256(), fingerprint, NULL)) {
+        free(subject);
+        return 0;
+    }
+
+    /* Call the callback (just reads values) */
+    rfbBool decision = client->GetX509CertFingerprintMismatchDecision(client, subject, not_before, not_after, fingerprint, 32);
+
+    /* Free all allocated memory here */
+    free(subject);
+
+    return decision ? 1 : 0;
+}
+
+static int cert_verify_callback(int preverify_ok, X509_STORE_CTX *ctx) {
+    if (preverify_ok) {
+        rfbClientLog("Server cert trusted\n");
+        return 1; // Accept
+    }
+
+    /*
+      If preverify failed, compare fingerprints
+    */
+    unsigned char remote_fingerprint[32];
+    if (! X509_digest(X509_STORE_CTX_get_current_cert(ctx), EVP_sha256(), remote_fingerprint, NULL)) {
+        return 0;
+    }
+
+    SSL *ssl = X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx());
+    const unsigned char *expected_fingerprint = SSL_get_ex_data(ssl, rfbTLSExpectedFingerprintIndex);
+
+    int verify;
+    if (expected_fingerprint && memcmp(remote_fingerprint, expected_fingerprint, 32) == 0) {
+        // accept
+        verify = 1;
+    } else {
+        // ask user
+        verify = cert_fingerprint_mismatch_callback(SSL_get_ex_data(ssl, rfbTLSClientIndex), X509_STORE_CTX_get_current_cert(ctx));
+    }
+
+    if(verify) {
+        X509_STORE_CTX_set_error(ctx, X509_V_OK);
+    }
+
+    return verify;
+}
+
+
 static SSL *
 open_ssl_connection (rfbClient *client, int sockfd, rfbBool anonTLS, rfbCredential *cred)
 {
@@ -286,7 +389,7 @@ open_ssl_connection (rfbClient *client, int sockfd, rfbBool anonTLS, rfbCredenti
       }
     }
 
-    SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+    SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, cert_verify_callback);
 
     if (verify_crls == rfbX509CrlVerifyClient) 
       X509_VERIFY_PARAM_set_flags(param, X509_V_FLAG_CRL_CHECK);
@@ -325,7 +428,12 @@ open_ssl_connection (rfbClient *client, int sockfd, rfbBool anonTLS, rfbCredenti
   }
 
   SSL_set_fd (ssl, sockfd);
-  SSL_CTX_set_app_data (ssl_ctx, client);
+  SSL_set_ex_data(ssl, rfbTLSClientIndex, client);
+
+  if (!anonTLS && cred->x509Credential.x509ExpectedFingerprint) {
+      // store expected fingerprint
+      SSL_set_ex_data(ssl, rfbTLSExpectedFingerprintIndex, cred->x509Credential.x509ExpectedFingerprint);
+  }
 
   do
   {
@@ -526,6 +634,7 @@ FreeX509Credential(rfbCredential *cred)
   free(cred->x509Credential.x509CACrlFile);
   free(cred->x509Credential.x509ClientCertFile);
   free(cred->x509Credential.x509ClientKeyFile);
+  free(cred->x509Credential.x509ExpectedFingerprint);
   free(cred);
 }