diff --git a/src/libp11-int.h b/src/libp11-int.h
index c3d7d899..fd84f97b 100644
--- a/src/libp11-int.h
+++ b/src/libp11-int.h
@@ -355,9 +355,15 @@ extern int pkcs11_private_decrypt(
 /* Retrieve PKCS11_KEY from an RSA key */
 extern PKCS11_OBJECT_private *pkcs11_get_ex_data_rsa(const RSA *rsa);
 
+/* Set PKCS11_KEY for an RSA key */
+void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key);
+
 /* Retrieve PKCS11_KEY from an EC_KEY */
 extern PKCS11_OBJECT_private *pkcs11_get_ex_data_ec(const EC_KEY *ec);
 
+/* Set PKCS11_KEY for an EC_KEY */
+extern void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key);
+
 /* Free the global RSA_METHOD */
 extern void pkcs11_rsa_method_free(void);
 
diff --git a/src/p11_ec.c b/src/p11_ec.c
index e8027f0d..88a4cbdf 100644
--- a/src/p11_ec.c
+++ b/src/p11_ec.c
@@ -351,7 +351,7 @@ PKCS11_OBJECT_private *pkcs11_get_ex_data_ec(const EC_KEY *ec)
 #endif
 }
 
-static void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key)
+void pkcs11_set_ex_data_ec(EC_KEY *ec, PKCS11_OBJECT_private *key)
 {
 #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
 	EC_KEY_set_ex_data(ec, ec_ex_index, key);
diff --git a/src/p11_key.c b/src/p11_key.c
index dbef9207..d1afd7a0 100644
--- a/src/p11_key.c
+++ b/src/p11_key.c
@@ -1,6 +1,6 @@
 /* libp11, a simple layer on to of PKCS#11 API
  * Copyright (C) 2005 Olaf Kirch <okir@lst.de>
- * Copyright (C) 2016-2018 Michał Trojnara <Michal.Trojnara@stunnel.org>
+ * Copyright (C) 2016-2024 Michał Trojnara <Michal.Trojnara@stunnel.org>
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
@@ -481,7 +481,10 @@ EVP_PKEY *pkcs11_get_key(PKCS11_OBJECT_private *key0, CK_OBJECT_CLASS object_cla
 			EVP_PKEY_free(ret);
 			goto err;
 		}
-		pkcs11_object_ref(key);
+		if (key->object_class == CKO_PRIVATE_KEY)
+			pkcs11_object_ref(key);
+		else /* Public key -> detach PKCS11_OBJECT */
+			pkcs11_set_ex_data_rsa(rsa, NULL);
 		break;
 	case EVP_PKEY_EC:
 #if OPENSSL_VERSION_NUMBER < 0x30000000L || defined(LIBRESSL_VERSION_NUMBER)
@@ -498,8 +501,13 @@ EVP_PKEY *pkcs11_get_key(PKCS11_OBJECT_private *key0, CK_OBJECT_CLASS object_cla
 			EVP_PKEY_free(ret);
 			goto err;
 		}
-		pkcs11_object_ref(key);
+		if (key->object_class == CKO_PRIVATE_KEY)
+			pkcs11_object_ref(key);
+		else /* Public key -> detach PKCS11_OBJECT */
+			pkcs11_set_ex_data_ec(ec_key, NULL);
 #else
+		/* pkcs11_ec_copy() method is only set for private keys,
+		 * so public keys do not have a PKCS11_OBJECT reference */
 		ret = EVP_PKEY_dup(key->evp_key);
 #endif
 		break;
diff --git a/src/p11_rsa.c b/src/p11_rsa.c
index 0558e674..359e9de9 100644
--- a/src/p11_rsa.c
+++ b/src/p11_rsa.c
@@ -280,7 +280,7 @@ PKCS11_OBJECT_private *pkcs11_get_ex_data_rsa(const RSA *rsa)
 	return RSA_get_ex_data(rsa, rsa_ex_index);
 }
 
-static void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key)
+void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key)
 {
 	RSA_set_ex_data(rsa, rsa_ex_index, key);
 }