Clean up error handling around unsupported rsa functions (#227)

Author: DmitriyMusatkin

source/unix/openssl_rsa.c

@@ -71,7 +71,7 @@ static int s_rsa_encrypt(
 
     EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key_pair_impl->key, NULL);
     if (ctx == NULL) {
-        return aws_raise_error(AWS_ERROR_CAL_CRYPTO_OPERATION_FAILED);
+        return aws_raise_error(AWS_ERROR_CAL_UNSUPPORTED_ALGORITHM);
     }
 
     if (aws_reinterpret_lc_evp_error_as_crt(EVP_PKEY_encrypt_init(ctx), "EVP_PKEY_encrypt_init", AWS_LS_CAL_RSA)) {
@@ -129,7 +129,7 @@ static int s_rsa_decrypt(
 
     EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key_pair_impl->key, NULL);
     if (ctx == NULL) {
-        return aws_raise_error(AWS_ERROR_CAL_CRYPTO_OPERATION_FAILED);
+        return aws_raise_error(AWS_ERROR_CAL_UNSUPPORTED_ALGORITHM);
     }
 
     if (aws_reinterpret_lc_evp_error_as_crt(EVP_PKEY_decrypt_init(ctx), "EVP_PKEY_decrypt_init", AWS_LS_CAL_RSA)) {
@@ -191,7 +191,14 @@ static int s_set_signature_ctx_from_algo(EVP_PKEY_CTX *ctx, enum aws_rsa_signatu
         }
         if (aws_reinterpret_lc_evp_error_as_crt(
                 EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha1()), "EVP_PKEY_CTX_set_signature_md", AWS_LS_CAL_RSA)) {
-            return AWS_OP_ERR;
+            /*
+             * This can fail with invalid digest on platforms that disabled sha1 for fips (ex. openssl 3.5+).
+             * Unfortunately, error code for invalid digest is wildly inconsistent between versions, making it
+             * impossible to write a backwards compatible error handling. In practice however the only way this should
+             * fail is when algo is not supported, so lets just hardcode the error. Still call the helper to get
+             * consistent logging for error.
+             */
+            return aws_raise_error(AWS_ERROR_CAL_UNSUPPORTED_ALGORITHM);
         }
     } else if (algorithm == AWS_CAL_RSA_SIGNATURE_PSS_SHA256) {
         if (aws_reinterpret_lc_evp_error_as_crt(
@@ -232,7 +239,7 @@ static int s_rsa_sign(
 
     EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key_pair_impl->key, NULL);
     if (ctx == NULL) {
-        return aws_raise_error(AWS_ERROR_CAL_CRYPTO_OPERATION_FAILED);
+        return aws_raise_error(AWS_ERROR_CAL_UNSUPPORTED_ALGORITHM);
     }
 
     if (aws_reinterpret_lc_evp_error_as_crt(EVP_PKEY_sign_init(ctx), "EVP_PKEY_sign_init", AWS_LS_CAL_RSA)) {
@@ -290,7 +297,7 @@ static int s_rsa_verify(
 
     EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key_pair_impl->key, NULL);
     if (ctx == NULL) {
-        return aws_raise_error(AWS_ERROR_CAL_CRYPTO_OPERATION_FAILED);
+        return aws_raise_error(AWS_ERROR_CAL_UNSUPPORTED_ALGORITHM);
     }
 
     if (aws_reinterpret_lc_evp_error_as_crt(EVP_PKEY_verify_init(ctx), "EVP_PKEY_verify_init", AWS_LS_CAL_RSA)) {

tests/rsa_test.c

@@ -12,6 +12,25 @@
 
 #include "test_case_helper.h"
 
+#if defined(AWS_OS_LINUX)
+#    include <aws/cal/private/opensslcrypto_common.h>
+#endif
+
+/*
+ * Helper to figure out whether rsa signing with sha1 is supported with currently crypto lib.
+ */
+static bool s_crypto_supports_sha1_signing(void) {
+    bool is_supported = true;
+#if defined(AWS_OS_LINUX)
+#    if defined(OPENSSL_IS_OPENSSL)
+#        if OPENSSL_VERSION_NUMBER >= 0x30500000L
+    is_supported = false;
+#        endif
+#    endif
+#endif
+    return is_supported;
+}
+
 /*
  * TODO: Need better test vectors. NIST ones are a pain to use.
  * For now using manually generated vectors and relying on round tripping.
@@ -335,6 +354,10 @@ AWS_TEST_CASE(rsa_verify_signing_pkcs1_sha256, s_rsa_verify_signing_pkcs1_sha256
 
 static int s_rsa_verify_signing_pkcs1_sha1(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;
+    if (!s_crypto_supports_sha1_signing()) {
+        return AWS_OP_SKIP;
+    }
+
     struct aws_byte_cursor message = aws_byte_cursor_from_c_str(TEST_ENCRYPTION_STRING);
 
     aws_cal_library_init(allocator);
@@ -626,6 +649,10 @@ AWS_TEST_CASE(rsa_signing_roundtrip_pkcs1_sha256_from_user, s_rsa_signing_roundt
 static int s_rsa_signing_roundtrip_pkcs1_sha1_from_user(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;
 
+    if (!s_crypto_supports_sha1_signing()) {
+        return AWS_OP_SKIP;
+    }
+
     aws_cal_library_init(allocator);
 
     ASSERT_SUCCESS(s_rsa_signing_roundtrip_from_user(
@@ -893,6 +920,10 @@ AWS_TEST_CASE(rsa_signing_mismatch_pkcs1_sha256, s_rsa_signing_mismatch_pkcs1_sh
 
 static int s_rsa_signing_mismatch_pkcs1_sha1(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;
+    if (!s_crypto_supports_sha1_signing()) {
+        return AWS_OP_SKIP;
+    }
+
     struct aws_byte_cursor message = aws_byte_cursor_from_c_str(TEST_ENCRYPTION_STRING);
 
     aws_cal_library_init(allocator);