Fix PKey.check for some broken keys (#897)

* fix PKey.check for some broken keys

RSA_check_key is documented to return 1 for valid keys.
It (currently) returns 0 or -1 for invalid ones.
The previous code accepted invalid keys if RSA_check_key returns -1!

* add test

Author: Mrmaxmeier

src/OpenSSL/crypto.py

@@ -345,7 +345,7 @@ def check(self):
         rsa = _lib.EVP_PKEY_get1_RSA(self._pkey)
         rsa = _ffi.gc(rsa, _lib.RSA_free)
         result = _lib.RSA_check_key(rsa)
-        if result:
+        if result == 1:
             return True
         _raise_current_error()
 

tests/test_crypto.py

@@ -561,6 +561,12 @@ def normalize_privatekey_pem(pem):
 Td8GMrwKz0557OxxtKN6uVVy4ACFMqEw0zN/KJI1vxc9
 -----END CERTIFICATE-----"""
 
+rsa_p_not_prime_pem = """
+-----BEGIN RSA PRIVATE KEY-----
+MBsCAQACAS0CAQcCAQACAQ8CAQMCAQACAQACAQA=
+-----END RSA PRIVATE KEY-----
+"""
+
 
 @pytest.fixture
 def x509_data():
@@ -966,6 +972,14 @@ def test_check_public_key(self):
         with pytest.raises(TypeError):
             pub.check()
 
+    def test_check_pr_897(self):
+        """
+        `PKey.check` raises `OpenSSL.crypto.Error` if provided with broken key
+        """
+        pkey = load_privatekey(FILETYPE_PEM, rsa_p_not_prime_pem)
+        with pytest.raises(Error):
+            pkey.check()
+
 
 def x509_name(**attrs):
     """