Add digest algo param to private encrypt and public decrypt

Specifically, it is added to openssl_private_encrypt() and
openssl_public_decrypt() functions. The purpose is to specify digest
algorithm for OEAP padding. It currently defaults to SHA1 which is not
preferred for modern setup and causes problems in compatibility with
web crypto.

Author: bukka

ext/openssl/openssl.c

@@ -3725,6 +3725,29 @@ PHP_FUNCTION(openssl_cms_decrypt)
 
 /* }}} */
 
+/* Helper to set RSA padding and digest for OAEP */
+static int php_openssl_set_rsa_padding_and_digest(EVP_PKEY_CTX *ctx, zend_long padding, const char *digest_algo)
+{
+	if (EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0) {
+		return 0;
+	}
+
+	if (digest_algo != NULL) {
+		const EVP_MD *md = php_openssl_get_evp_md_by_name(digest_algo);
+		if (md == NULL) {
+			php_error_docref(NULL, E_WARNING, "Unknown digest algorithm: %s", digest_algo);
+			return 0;
+		}
+
+		if (padding == RSA_PKCS1_OAEP_PADDING) {
+			if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md) <= 0) {
+				return 0;
+			}
+		}
+	}
+
+	return 1;
+}
 
 /* {{{ Encrypts data with private key */
 PHP_FUNCTION(openssl_private_encrypt)
@@ -3780,10 +3803,11 @@ PHP_FUNCTION(openssl_private_decrypt)
 {
 	zval *key, *crypted;
 	zend_long padding = RSA_PKCS1_PADDING;
-	char * data;
-	size_t data_len;
+	char *data;
+	char *digest_algo = NULL;
+	size_t data_len, digest_algo_len = 0;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|l", &data, &data_len, &crypted, &key, &padding) == FAILURE) {
+	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|ls!", &data, &data_len, &crypted, &key, &padding, &digest_algo, &digest_algo_len) == FAILURE) {
 		RETURN_THROWS();
 	}
 
@@ -3798,7 +3822,7 @@ PHP_FUNCTION(openssl_private_decrypt)
 	size_t out_len = 0;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
 	if (!ctx || EVP_PKEY_decrypt_init(ctx) <= 0 ||
-			EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0 ||
+			!php_openssl_set_rsa_padding_and_digest(ctx, padding, digest_algo) ||
 			EVP_PKEY_decrypt(ctx, NULL, &out_len, (unsigned char *) data, data_len) <= 0) {
 		php_openssl_store_errors();
 		RETVAL_FALSE;
@@ -3879,10 +3903,11 @@ PHP_FUNCTION(openssl_public_decrypt)
 {
 	zval *key, *crypted;
 	zend_long padding = RSA_PKCS1_PADDING;
-	char * data;
-	size_t data_len;
+	char *data;
+	char *digest_algo = NULL;
+	size_t data_len, digest_algo_len = 0;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|l", &data, &data_len, &crypted, &key, &padding) == FAILURE) {
+	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|ls!", &data, &data_len, &crypted, &key, &padding, &digest_algo, &digest_algo_len) == FAILURE) {
 		RETURN_THROWS();
 	}
 
@@ -3897,7 +3922,7 @@ PHP_FUNCTION(openssl_public_decrypt)
 	size_t out_len = 0;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
 	if (!ctx || EVP_PKEY_verify_recover_init(ctx) <= 0 ||
-			EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0 ||
+			!php_openssl_set_rsa_padding_and_digest(ctx, padding, digest_algo) ||
 			EVP_PKEY_verify_recover(ctx, NULL, &out_len, (unsigned char *) data, data_len) <= 0) {
 		php_openssl_store_errors();
 		RETVAL_FALSE;

ext/openssl/openssl.stub.php

@@ -575,13 +575,13 @@ function openssl_private_encrypt(#[\SensitiveParameter] string $data, &$encrypte
  * @param string $decrypted_data
  * @param OpenSSLAsymmetricKey|OpenSSLCertificate|array|string $private_key
  */
-function openssl_private_decrypt(string $data, #[\SensitiveParameter] &$decrypted_data, #[\SensitiveParameter] $private_key, int $padding = OPENSSL_PKCS1_PADDING): bool {}
+function openssl_private_decrypt(string $data, #[\SensitiveParameter] &$decrypted_data, #[\SensitiveParameter] $private_key, int $padding = OPENSSL_PKCS1_PADDING, ?string $digest_algo = null): bool {}
 
 /**
  * @param string $encrypted_data
  * @param OpenSSLAsymmetricKey|OpenSSLCertificate|array|string $public_key
  */
-function openssl_public_encrypt(#[\SensitiveParameter] string $data, &$encrypted_data, $public_key, int $padding = OPENSSL_PKCS1_PADDING): bool {}
+function openssl_public_encrypt(#[\SensitiveParameter] string $data, &$encrypted_data, $public_key, int $padding = OPENSSL_PKCS1_PADDING, ?string $digest_algo = null): bool {}
 
 /**
  * @param string $decrypted_data

ext/openssl/openssl_arginfo.h

@@ -0,0 +1,41 @@
+--TEST--
+openssl_private_decrypt() with digest algorithm tests
+--EXTENSIONS--
+openssl
+--FILE--
+<?php
+$data = "Testing openssl_private_decrypt() with digest algorithms";
+$privkey = "file://" . __DIR__ . "/private_rsa_1024.key";
+$pubkey = "file://" . __DIR__ . "/public.key";
+
+openssl_public_encrypt($data, $encrypted_sha256, $pubkey, OPENSSL_PKCS1_OAEP_PADDING, "sha256");
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_sha256, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "sha256"));
+var_dump($output_sha256);
+
+openssl_public_encrypt($data, $encrypted_default, $pubkey, OPENSSL_PKCS1_OAEP_PADDING);
+var_dump(openssl_private_decrypt($encrypted_default, $output_default, $privkey, OPENSSL_PKCS1_OAEP_PADDING));
+var_dump($output_default);
+
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_mismatch, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "sha512"));
+var_dump($output_mismatch);
+
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_invalid, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "invalid_hash"));
+var_dump($output_invalid);
+
+openssl_public_encrypt($data, $encrypted_pkcs1, $pubkey, OPENSSL_PKCS1_PADDING);
+var_dump(openssl_private_decrypt($encrypted_pkcs1, $output_pkcs1, $privkey, OPENSSL_PKCS1_PADDING, "sha256"));
+var_dump($output_pkcs1);
+?>
+--EXPECTF--
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"
+bool(false)
+NULL
+
+Warning: openssl_private_decrypt(): Unknown digest algorithm: invalid_hash in %s on line %d
+bool(false)
+NULL
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"