Add digest algo param to public encrypt and private decrypt

Specifically, it is added to openssl_public_encrypt() and
openssl_private_decrypt() functions. The purpose is to specify digest
algorithm for OEAP padding. It currently defaults to SHA1 which is not
preferred for modern setup and causes problems in compatibility with
web crypto.

Closes phpGH-19223

Author: bukka

ext/openssl/openssl.c

@@ -3725,6 +3725,29 @@ PHP_FUNCTION(openssl_cms_decrypt)
 
 /* }}} */
 
+/* Helper to set RSA padding and digest for OAEP */
+static int php_openssl_set_rsa_padding_and_digest(EVP_PKEY_CTX *ctx, zend_long padding, const char *digest_algo)
+{
+	if (EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0) {
+		return 0;
+	}
+
+	if (digest_algo != NULL) {
+		const EVP_MD *md = php_openssl_get_evp_md_by_name(digest_algo);
+		if (md == NULL) {
+			php_error_docref(NULL, E_WARNING, "Unknown digest algorithm: %s", digest_algo);
+			return 0;
+		}
+
+		if (padding == RSA_PKCS1_OAEP_PADDING) {
+			if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md) <= 0) {
+				return 0;
+			}
+		}
+	}
+
+	return 1;
+}
 
 /* {{{ Encrypts data with private key */
 PHP_FUNCTION(openssl_private_encrypt)
@@ -3780,10 +3803,11 @@ PHP_FUNCTION(openssl_private_decrypt)
 {
 	zval *key, *crypted;
 	zend_long padding = RSA_PKCS1_PADDING;
-	char * data;
-	size_t data_len;
+	char *data;
+	char *digest_algo = NULL;
+	size_t data_len, digest_algo_len = 0;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|l", &data, &data_len, &crypted, &key, &padding) == FAILURE) {
+	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|ls!", &data, &data_len, &crypted, &key, &padding, &digest_algo, &digest_algo_len) == FAILURE) {
 		RETURN_THROWS();
 	}
 
@@ -3798,7 +3822,7 @@ PHP_FUNCTION(openssl_private_decrypt)
 	size_t out_len = 0;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
 	if (!ctx || EVP_PKEY_decrypt_init(ctx) <= 0 ||
-			EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0 ||
+			!php_openssl_set_rsa_padding_and_digest(ctx, padding, digest_algo) ||
 			EVP_PKEY_decrypt(ctx, NULL, &out_len, (unsigned char *) data, data_len) <= 0) {
 		php_openssl_store_errors();
 		RETVAL_FALSE;
@@ -3879,10 +3903,11 @@ PHP_FUNCTION(openssl_public_decrypt)
 {
 	zval *key, *crypted;
 	zend_long padding = RSA_PKCS1_PADDING;
-	char * data;
-	size_t data_len;
+	char *data;
+	char *digest_algo = NULL;
+	size_t data_len, digest_algo_len = 0;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|l", &data, &data_len, &crypted, &key, &padding) == FAILURE) {
+	if (zend_parse_parameters(ZEND_NUM_ARGS(), "szz|ls!", &data, &data_len, &crypted, &key, &padding, &digest_algo, &digest_algo_len) == FAILURE) {
 		RETURN_THROWS();
 	}
 
@@ -3897,7 +3922,7 @@ PHP_FUNCTION(openssl_public_decrypt)
 	size_t out_len = 0;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
 	if (!ctx || EVP_PKEY_verify_recover_init(ctx) <= 0 ||
-			EVP_PKEY_CTX_set_rsa_padding(ctx, padding) <= 0 ||
+			!php_openssl_set_rsa_padding_and_digest(ctx, padding, digest_algo) ||
 			EVP_PKEY_verify_recover(ctx, NULL, &out_len, (unsigned char *) data, data_len) <= 0) {
 		php_openssl_store_errors();
 		RETVAL_FALSE;

ext/openssl/openssl.stub.php

@@ -575,13 +575,13 @@ function openssl_private_encrypt(#[\SensitiveParameter] string $data, &$encrypte
  * @param string $decrypted_data
  * @param OpenSSLAsymmetricKey|OpenSSLCertificate|array|string $private_key
  */
-function openssl_private_decrypt(string $data, #[\SensitiveParameter] &$decrypted_data, #[\SensitiveParameter] $private_key, int $padding = OPENSSL_PKCS1_PADDING): bool {}
+function openssl_private_decrypt(string $data, #[\SensitiveParameter] &$decrypted_data, #[\SensitiveParameter] $private_key, int $padding = OPENSSL_PKCS1_PADDING, ?string $digest_algo = null): bool {}
 
 /**
  * @param string $encrypted_data
  * @param OpenSSLAsymmetricKey|OpenSSLCertificate|array|string $public_key
  */
-function openssl_public_encrypt(#[\SensitiveParameter] string $data, &$encrypted_data, $public_key, int $padding = OPENSSL_PKCS1_PADDING): bool {}
+function openssl_public_encrypt(#[\SensitiveParameter] string $data, &$encrypted_data, $public_key, int $padding = OPENSSL_PKCS1_PADDING, ?string $digest_algo = null): bool {}
 
 /**
  * @param string $decrypted_data

ext/openssl/openssl_arginfo.h

@@ -0,0 +1,41 @@
+--TEST--
+openssl_private_decrypt() with digest algorithm tests
+--EXTENSIONS--
+openssl
+--FILE--
+<?php
+$data = "Testing openssl_private_decrypt() with digest algorithms";
+$privkey = "file://" . __DIR__ . "/private_rsa_1024.key";
+$pubkey = "file://" . __DIR__ . "/public.key";
+
+openssl_public_encrypt($data, $encrypted_sha256, $pubkey, OPENSSL_PKCS1_OAEP_PADDING, "sha256");
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_sha256, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "sha256"));
+var_dump($output_sha256);
+
+openssl_public_encrypt($data, $encrypted_default, $pubkey, OPENSSL_PKCS1_OAEP_PADDING);
+var_dump(openssl_private_decrypt($encrypted_default, $output_default, $privkey, OPENSSL_PKCS1_OAEP_PADDING));
+var_dump($output_default);
+
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_mismatch, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "sha512"));
+var_dump($output_mismatch);
+
+var_dump(openssl_private_decrypt($encrypted_sha256, $output_invalid, $privkey, OPENSSL_PKCS1_OAEP_PADDING, "invalid_hash"));
+var_dump($output_invalid);
+
+openssl_public_encrypt($data, $encrypted_pkcs1, $pubkey, OPENSSL_PKCS1_PADDING);
+var_dump(openssl_private_decrypt($encrypted_pkcs1, $output_pkcs1, $privkey, OPENSSL_PKCS1_PADDING, "sha256"));
+var_dump($output_pkcs1);
+?>
+--EXPECTF--
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"
+bool(false)
+NULL
+
+Warning: openssl_private_decrypt(): Unknown digest algorithm: invalid_hash in %s on line %d
+bool(false)
+NULL
+bool(true)
+string(56) "Testing openssl_private_decrypt() with digest algorithms"