pkey: pass pem_password_cb to OSSL_DECODER only when it is needed

Specify OSSL_DECODER_CTX_set_pem_password_cb() only when we expect a
passphrase-protected private key.

OSSL_DECODER appears to try to decrypt every PEM block in the input even
when the PEM header does not match the requested selection. This can
cause repeated prompts for a passphrase in a single OpenSSL::PKey.read
call.

Author: rhenium

ext/openssl/ossl_pkey.c

@@ -94,7 +94,8 @@ ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass)
                                          selection, NULL, NULL);
     if (!dctx)
         goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb,
+    if (selection == EVP_PKEY_KEYPAIR &&
+        OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb,
                                              ppass) != 1)
         goto out;
     while (1) {

test/openssl/test_pkey.rb

@@ -85,6 +85,86 @@ def test_s_read_der_then_pem
     assert_not_predicate(pkey, :private?)
   end
 
+  def test_s_read_passphrase
+    orig = Fixtures.pkey("rsa-1")
+    encrypted_pem = orig.private_to_pem("AES-256-CBC", "correct_passphrase")
+    assert_match(/\A-----BEGIN ENCRYPTED PRIVATE KEY-----/, encrypted_pem)
+
+    # Correct passphrase passed as the second argument
+    pkey1 = OpenSSL::PKey.read(encrypted_pem, "correct_passphrase")
+    assert_equal(orig.private_to_der, pkey1.private_to_der)
+
+    # Correct passphrase returned by the block. The block gets false
+    called = 0
+    flag = nil
+    pkey2 = OpenSSL::PKey.read(encrypted_pem) { |f|
+      called += 1
+      flag = f
+      "correct_passphrase"
+    }
+    assert_equal(orig.private_to_der, pkey2.private_to_der)
+    assert_equal(1, called)
+    assert_false(flag)
+
+    # Incorrect passphrase passed. The block is not called
+    called = 0
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      OpenSSL::PKey.read(encrypted_pem, "incorrect_passphrase") {
+        called += 1
+      }
+    }
+    assert_equal(0, called)
+
+    # Incorrect passphrase returned by the block. The block is called only once
+    called = 0
+    assert_raise(OpenSSL::PKey::PKeyError) {
+      OpenSSL::PKey.read(encrypted_pem) {
+        called += 1
+        "incorrect_passphrase"
+      }
+    }
+    assert_equal(1, called)
+  end
+
+  def test_s_read_passphrase_tty
+    omit "https://github.com/aws/aws-lc/pull/2555" if aws_lc?
+
+    orig = Fixtures.pkey("rsa-1")
+    encrypted_pem = orig.private_to_pem("AES-256-CBC", "correct_passphrase")
+
+    # Correct passphrase passed to OpenSSL's prompt
+    script = <<~"end;"
+      require "openssl"
+      Process.setsid
+      OpenSSL::PKey.read(#{encrypted_pem.dump})
+      puts "ok"
+    end;
+    assert_in_out_err([*$:.map { |l| "-I#{l}" }, "-e#{script}"],
+                      "correct_passphrase\n") { |stdout, stderr|
+      assert_equal(["Enter PEM pass phrase:"], stderr)
+      assert_equal(["ok"], stdout)
+    }
+
+    # Incorrect passphrase passed to OpenSSL's prompt
+    script = <<~"end;"
+      require "openssl"
+      Process.setsid
+      begin
+        OpenSSL::PKey.read(#{encrypted_pem.dump})
+      rescue OpenSSL::PKey::PKeyError
+        puts "ok"
+      else
+        puts "expected OpenSSL::PKey::PKeyError"
+      end
+    end;
+    stdin = "incorrect_passphrase\n" * 5
+    assert_in_out_err([*$:.map { |l| "-I#{l}" }, "-e#{script}"],
+                      stdin) { |stdout, stderr|
+      assert_equal(1, stderr.count("Enter PEM pass phrase:"))
+      assert_equal(["ok"], stdout)
+    }
+  end if ENV["OSSL_TEST_ALL"] == "1" && Process.respond_to?(:setsid)
+
   def test_hmac_sign_verify
     pkey = OpenSSL::PKey.generate_key("HMAC", { "key" => "abcd" })