pkey: stop retrying after non-retryable error from OSSL_DECODER

Continue processing only when OSSL_DECODER_from_bio() returns the error
code ERR_R_UNSUPPORTED. Otherwise, raise an exception without retrying
decoding the input in another format.

This fixes another case where OpenSSL::PKey.read prompts for a
passphrase multiple times when the input contains multiple
passphrase-protected PEM blocks and the first one cannot be decoded.

I am not entirely sure if the error code ERR_R_UNSUPPORTED is considered
part of the public interface of OpenSSL, but this seems to be the only
option available and is the approach used internally by the
PEM_read_bio_*() functions.

Fixes #927

Author: rhenium

ext/openssl/ossl_pkey.c

@@ -83,13 +83,15 @@ ossl_pkey_wrap(EVP_PKEY *pkey)
 # include <openssl/decoder.h>
 
 static EVP_PKEY *
-ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass)
+ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass,
+               int *retryable)
 {
     void *ppass = (void *)pass;
     OSSL_DECODER_CTX *dctx;
     EVP_PKEY *pkey = NULL;
     int pos = 0, pos2;
 
+    *retryable = 0;
     dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL,
                                          selection, NULL, NULL);
     if (!dctx)
@@ -100,25 +102,29 @@ ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass)
         goto out;
     while (1) {
         if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-            goto out;
-        if (BIO_eof(bio))
             break;
+        if (ERR_GET_REASON(ERR_peek_error()) != ERR_R_UNSUPPORTED)
+            break;
+        if (BIO_eof(bio) == 1) {
+            *retryable = 1;
+            break;
+        }
         pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
+        if (pos2 < 0 || pos2 <= pos) {
+            *retryable = 1;
             break;
+        }
         ossl_clear_error();
         pos = pos2;
     }
   out:
-    OSSL_BIO_reset(bio);
     OSSL_DECODER_CTX_free(dctx);
     return pkey;
 }
 
 EVP_PKEY *
 ossl_pkey_read_generic(BIO *bio, VALUE pass)
 {
-    EVP_PKEY *pkey = NULL;
     /* First check DER, then check PEM. */
     const char *input_types[] = {"DER", "PEM"};
     int input_type_num = (int)(sizeof(input_types) / sizeof(char *));
@@ -167,18 +173,22 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
         EVP_PKEY_PUBLIC_KEY
     };
     int selection_num = (int)(sizeof(selections) / sizeof(int));
-    int i, j;
 
-    for (i = 0; i < input_type_num; i++) {
-        for (j = 0; j < selection_num; j++) {
-            pkey = ossl_pkey_read(bio, input_types[i], selections[j], pass);
-            if (pkey) {
-                goto out;
+    for (int i = 0; i < input_type_num; i++) {
+        for (int j = 0; j < selection_num; j++) {
+            if (i || j) {
+                ossl_clear_error();
+                BIO_reset(bio);
             }
+
+            int retryable;
+            EVP_PKEY *pkey = ossl_pkey_read(bio, input_types[i], selections[j],
+                                            pass, &retryable);
+            if (pkey || !retryable)
+                return pkey;
         }
     }
-  out:
-    return pkey;
+    return NULL;
 }
 #else
 EVP_PKEY *

test/openssl/test_pkey.rb

@@ -124,6 +124,17 @@ def test_s_read_passphrase
       }
     }
     assert_equal(1, called)
+
+    # Incorrect passphrase returned by the block. The input contains two PEM
+    # blocks.
+    called = 0
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      OpenSSL::PKey.read(encrypted_pem + encrypted_pem) {
+        called += 1
+        "incorrect_passphrase"
+      }
+    }
+    assert_equal(1, called)
   end
 
   def test_s_read_passphrase_tty