Fix OpenSSL string/bytes python3 issues

OpenSSL works a bit differently in python3 which requires delicate
handling.

Author: SpamapS

src/saml2/cert.py

@@ -4,6 +4,7 @@
 import datetime
 import dateutil.parser
 import pytz
+import six
 from OpenSSL import crypto
 from os.path import join
 from os import remove
@@ -154,10 +155,13 @@ def create_certificate(self, cert_info, request=False, valid_from=0,
                 tmp_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
             tmp_key = None
             if cipher_passphrase is not None:
+                passphrase = cipher_passphrase["passphrase"]
+                if isinstance(cipher_passphrase["passphrase"],
+                              six.string_types):
+                    passphrase = passphrase.encode('utf-8')
                 tmp_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, k,
                                                  cipher_passphrase["cipher"],
-                                                 cipher_passphrase[
-                                                     "passphrase"])
+                                                 passphrase)
             else:
                 tmp_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, k)
             if write_to_file:
@@ -190,7 +194,7 @@ def write_str_to_file(self, file, str_data):
         f.close()
 
     def read_str_from_file(self, file, type="pem"):
-        f = open(file)
+        f = open(file, 'rt')
         str_data = f.read()
         f.close()
 
@@ -257,7 +261,10 @@ def create_cert_signed_certificate(self, sign_cert_str, sign_key_str,
         cert.set_pubkey(req_cert.get_pubkey())
         cert.sign(ca_key, hash_alg)
 
-        return crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+        cert_dump = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+        if isinstance(cert_dump, six.string_types):
+            return cert_dump
+        return cert_dump.decode('utf-8')
 
     def verify_chain(self, cert_chain_str_list, cert_str):
         """
@@ -327,6 +334,8 @@ def verify(self, signing_cert_str, cert_str):
                                "signed certificate.")
 
             cert_algorithm = cert.get_signature_algorithm()
+            if six.PY3:
+                cert_algorithm = cert_algorithm.decode('ascii')
 
             cert_asn1 = crypto.dump_certificate(crypto.FILETYPE_ASN1, cert)
 
@@ -342,7 +351,9 @@ def verify(self, signing_cert_str, cert_str):
 
             signature_payload = cert_signature_decoded.payload
 
-            if signature_payload[0] != '\x00':
+            sig_pay0 = signature_payload[0]
+            if ((isinstance(sig_pay0, int) and sig_pay0 != 0) or
+                (isinstance(sig_pay0, str) and sig_pay0 != '\x00')):
                 return (False,
                        "The certificate should not contain any unused bits.")
 
@@ -355,4 +366,4 @@ def verify(self, signing_cert_str, cert_str):
             except crypto.Error as e:
                 return False, "Certificate is incorrectly signed."
         except Exception as e:
-            return False, "Certificate is not valid for an unknown reason."
+            return False, "Certificate is not valid for an unknown reason. %s" % str(e)

tests/test_81_certificates.py

@@ -174,7 +174,7 @@ def test_validate_passphrase(self):
                                                            request=True)
         cert_str = osw.create_cert_signed_certificate(ca_cert_str, ca_key_str,
                                                       req_cert_str,
-                                                      passphrase="qwerty")
+                                                      passphrase=b"qwerty")
 
         valid = False
         try: