rsa: consistent handling of zero-padded ciphertext

Author: tomato42

tlslite/utils/openssl_rsakey.py

@@ -30,6 +30,8 @@ def password_callback(v, prompt1='Enter private key passphrase:',
 
 
 if m2cryptoLoaded:
+    import M2Crypto
+
     class OpenSSL_RSAKey(RSAKey):
         def __init__(self, n=0, e=0, key_type="rsa"):
             self.rsa = None
@@ -69,8 +71,9 @@ def _rawPrivateKeyOp(self, message):
             return ciphertext
 
         def _raw_private_key_op_bytes(self, message):
-            return bytearray(m2.rsa_private_encrypt(self.rsa, bytes(message),
-                                                    m2.no_padding))
+            return self._call_m2crypto(
+                m2.rsa_private_encrypt, message,
+                "Bad parameters to private key operation")
 
         def _rawPublicKeyOp(self, ciphertext):
             data = numberToByteArray(ciphertext, numBytes(self.n))
@@ -79,9 +82,16 @@ def _rawPublicKeyOp(self, ciphertext):
             message = bytesToNumber(bytearray(string))
             return message
 
+        def _call_m2crypto(self, method, param, err_msg):
+            try:
+                return bytearray(method(self.rsa, bytes(param), m2.no_padding))
+            except M2Crypto.RSA.RSAError:
+                raise ValueError(err_msg)
+
         def _raw_public_key_op_bytes(self, ciphertext):
-            return bytearray(m2.rsa_public_decrypt(self.rsa, bytes(ciphertext),
-                                                   m2.no_padding))
+            return self._call_m2crypto(
+                m2.rsa_public_decrypt, ciphertext,
+                "Bad parameters to public key operation")
 
         def acceptsPassword(self): return True
 

unit_tests/test_tlslite_utils_rsakey.py

@@ -2051,6 +2051,85 @@ def test_negative_11_byte_long_wrong_type_byte(self):
         self.assertNotEqual(msg, b'lorem ipsum')
         self.assertEqual(msg, plaintext)
 
+    def test_negative_11_bytes_long_with_null_padded_ciphertext(self):
+        # an invalid ciphertext, with a zero byte in first byte of
+        # ciphertext, decrypts to a random 11 byte long synthethic
+        # plaintext
+        ciphertext = a2b_hex(remove_whitespace("""
+0096136621faf36d5290b16bd26295de27f895d1faa51c800dafce73d001d60796cd4e2ac3f
+a2162131d859cd9da5a0c8a42281d9a63e5f353971b72e36b5722e4ac444d77f892a5443deb
+3dca49fa732fe855727196e23c26eeac55eeced8267a209ebc0f92f4656d64a6c13f7f7ce54
+4ebeb0f668fe3a6c0f189e4bcd5ea12b73cf63e0c8350ee130dd62f01e5c97a1e13f52fde96
+a9a1bc9936ce734fdd61f27b18216f1d6de87f49cf4f2ea821fb8efd1f92cdad529baf7e31a
+ff9bff4074f2cad2b4243dd15a711adcf7de900851fbd6bcb53dac399d7c880531d06f25f70
+02e1aaf1722765865d2c2b902c7736acd27bc6cbd3e38b560e2eecf7d4b576
+"""))
+        self.assertEqual(len(ciphertext), numBytes(self.pub_key.n))
+
+        # sanity check that the decrypted ciphertext is invalid
+        dec = self.priv_key._raw_private_key_op_bytes(ciphertext)
+        self.assertEqual(dec[0:3], b'\x00\x02\x00')
+        self.assertNotEqual(dec[-12:-11], b'\x00')
+
+        plaintext = a2b_hex(remove_whitespace("ba27b1842e7c21c0e7ef6a"))
+
+        msg = self.priv_key.decrypt(ciphertext)
+
+        self.assertEqual(msg, plaintext)
+
+    def test_negative_11_bytes_long_with_null_truncated_ciphertext(self):
+        # same as test_negative_11_bytes_long_with_null_padded_ciphertext
+        # but with the zero bytes at the beginning removed
+        ciphertext = a2b_hex(remove_whitespace("""
+96136621faf36d5290b16bd26295de27f895d1faa51c800dafce73d001d60796cd4e2ac3f
+a2162131d859cd9da5a0c8a42281d9a63e5f353971b72e36b5722e4ac444d77f892a5443deb
+3dca49fa732fe855727196e23c26eeac55eeced8267a209ebc0f92f4656d64a6c13f7f7ce54
+4ebeb0f668fe3a6c0f189e4bcd5ea12b73cf63e0c8350ee130dd62f01e5c97a1e13f52fde96
+a9a1bc9936ce734fdd61f27b18216f1d6de87f49cf4f2ea821fb8efd1f92cdad529baf7e31a
+ff9bff4074f2cad2b4243dd15a711adcf7de900851fbd6bcb53dac399d7c880531d06f25f70
+02e1aaf1722765865d2c2b902c7736acd27bc6cbd3e38b560e2eecf7d4b576
+"""))
+        self.assertEqual(len(ciphertext), numBytes(self.pub_key.n)-1)
+
+        # sanity check that the decrypted ciphertext is invalid
+        dec = self.priv_key._raw_private_key_op_bytes(b"\x00" + ciphertext)
+        self.assertEqual(dec[0:3], b'\x00\x02\x00')
+        self.assertNotEqual(dec[-12:-11], b'\x00')
+
+        plaintext = a2b_hex(remove_whitespace("ba27b1842e7c21c0e7ef6a"))
+
+        msg = self.priv_key.decrypt(ciphertext)
+
+        # tlslite-ng considers a too short ciphertext a publicly known error
+        # so it returns an error (None)
+        self.assertEqual(msg, None)
+
+    def test_negative_11_byte_long_with_double_null_padded_ciphertext(self):
+        # an invalid ciphertext, with two zero byte in first bytes of
+        # ciphertext, that decrypts to a random 11 byte long synthethic
+        # plaintext
+        ciphertext = a2b_hex(remove_whitespace("""
+0000587cccc6b264bdfe0dc2149a988047fa921801f3502ea64624c510c6033d2f427e3f136
+c26e88ea9f6519e86a542cec96aad1e5e9013c3cc203b6de15a69183050813af5c9ad797031
+36d4b92f50ce171eefc6aa7988ecf02f319ffc5eafd6ee7a137f8fce64b255bb1b8dd19cfe7
+67d64fdb468b9b2e9e7a0c24dae03239c8c714d3f40b7ee9c4e59ac15b17e4d328f1100756b
+ce17133e8e7493b54e5006c3cbcdacd134130c5132a1edebdbd01a0c41452d16ed7a0788003
+c34730d0808e7e14c797a21f2b45a8aa1644357fd5e988f99b017d9df37563a354c788dc0e2
+f9466045622fa3f3e17db63414d27761f57392623a2bef6467501c63e8d645"""))
+        self.assertEqual(len(ciphertext), numBytes(self.pub_key.n))
+
+        # sanity check that the decrypted ciphertext is invalid
+        dec = self.priv_key._raw_private_key_op_bytes(ciphertext)
+        self.assertEqual(dec[0:3], b'\x00\x02\x00')
+        self.assertNotEqual(dec[-12:-11], b'\x00')
+
+        plaintext = a2b_hex(remove_whitespace("d5cf555b1d6151029a429a"))
+
+        msg = self.priv_key.decrypt(ciphertext)
+
+        self.assertEqual(msg, plaintext)
+
+
     def test_negative_11_byte_long_null_type_byte(self):
         # an otherwise correct plaintext, but with wrong second byte
         # (0x00 instead of 0x02), and a 0x02 on third position, generates a