Fix GH-11952: better locale strings canonicalization for IntlDateFormatter and NumberFormatter

Author: alexandre-daubois

NEWS

@@ -10,6 +10,10 @@ PHP                                                                        NEWS
   . Fixed bug GH-19544 (GC treats ZEND_WEAKREF_TAG_MAP references as WeakMap
     references). (Arnaud, timwolla)
 
+- Intl:
+  . Fixed bug GH-11952 (Fix locale strings canonicalization for IntlDateFormatter
+    and NumberFormatter). (alexandre-daubois)
+
 - OpenSSL:
   . Fixed bug GH-19245 (Success error message on TLS stream accept failure).
     (Jakub Zelenka)

ext/intl/dateformat/dateformat_create.cpp

@@ -22,6 +22,7 @@
 extern "C" {
 #include <unicode/ustring.h>
 #include <unicode/udat.h>
+#include <unicode/uloc.h>
 
 #include "php_intl.h"
 #include "dateformat_create.h"
@@ -110,7 +111,12 @@ static zend_result datefmt_ctor(INTERNAL_FUNCTION_PARAMETERS, zend_error_handlin
 	if (locale_len == 0) {
 		locale_str = (char *) intl_locale_get_default();
 	}
-	locale = Locale::createFromName(locale_str);
+
+	char* canonicalized_locale = canonicalize_locale_string(locale_str);
+	const char* final_locale = canonicalized_locale ? canonicalized_locale : locale_str;
+	const char* stored_locale = canonicalized_locale ? canonicalized_locale : locale_str;
+
+	locale = Locale::createFromName(final_locale);
 	/* get*Name accessors being set does not preclude being bogus */
 	if (locale.isBogus() || ((locale_len == 1 && locale_str[0] != 'C') || (locale_len > 1 && strlen(locale.getISO3Language()) == 0))) {
 		goto error;
@@ -148,7 +154,7 @@ static zend_result datefmt_ctor(INTERNAL_FUNCTION_PARAMETERS, zend_error_handlin
 	}
 
 	DATE_FORMAT_OBJECT(dfo) = udat_open((UDateFormatStyle)time_type,
-			(UDateFormatStyle)date_type, locale_str, NULL, 0, svalue,
+			(UDateFormatStyle)date_type, final_locale, NULL, 0, svalue,
 			slength, &INTL_DATA_ERROR_CODE(dfo));
 
 	if (pattern_str && pattern_str_len > 0) {
@@ -181,9 +187,13 @@ static zend_result datefmt_ctor(INTERNAL_FUNCTION_PARAMETERS, zend_error_handlin
 	dfo->date_type			= date_type;
 	dfo->time_type			= time_type;
 	dfo->calendar			= calendar_type;
-	dfo->requested_locale	= estrdup(locale_str);
+	/* Store the canonicalized locale, or fallback to original if canonicalization failed */
+	dfo->requested_locale	= estrdup(stored_locale);
 
 error:
+	if (canonicalized_locale) {
+		efree(canonicalized_locale);
+	}
 	if (svalue) {
 		efree(svalue);
 	}

ext/intl/formatter/formatter_main.c

@@ -17,6 +17,7 @@
 #endif
 
 #include <unicode/ustring.h>
+#include <unicode/uloc.h>
 
 #include "php_intl.h"
 #include "formatter_class.h"
@@ -63,12 +64,18 @@ static int numfmt_ctor(INTERNAL_FUNCTION_PARAMETERS, zend_error_handling *error_
 		locale = intl_locale_get_default();
 	}
 
-	/* Create an ICU number formatter. */
-	FORMATTER_OBJECT(nfo) = unum_open(style, spattern, spattern_len, locale, NULL, &INTL_DATA_ERROR_CODE(nfo));
+	char* canonicalized_locale = canonicalize_locale_string(locale);
+	const char* final_locale = canonicalized_locale ? canonicalized_locale : locale;
 
-	if(spattern) {
+	FORMATTER_OBJECT(nfo) = unum_open(style, spattern, spattern_len, final_locale, NULL, &INTL_DATA_ERROR_CODE(nfo));
+
+	if (spattern) {
 		efree(spattern);
 	}
+	
+	if (canonicalized_locale) {
+		efree(canonicalized_locale);
+	}
 
 	INTL_CTOR_CHECK_STATUS(nfo, "numfmt_create: number formatter creation failed");
 	return SUCCESS;

ext/intl/php_intl.c

@@ -94,6 +94,20 @@ const char *intl_locale_get_default( void )
 	return INTL_G(default_locale);
 }
 
+static char* canonicalize_locale_string(const char* locale) {
+	char canonicalized[ULOC_FULLNAME_CAPACITY];
+	UErrorCode status = U_ZERO_ERROR;
+	int32_t canonicalized_len;
+
+	canonicalized_len = uloc_canonicalize(locale, canonicalized, sizeof(canonicalized), &status);
+
+	if (U_FAILURE(status) || canonicalized_len <= 0) {
+		return NULL;
+	}
+
+	return estrdup(canonicalized);
+}
+
 /* {{{ INI Settings */
 PHP_INI_BEGIN()
 	STD_PHP_INI_ENTRY(LOCALE_INI_NAME, NULL, PHP_INI_ALL, OnUpdateStringUnempty, default_locale, zend_intl_globals, intl_globals)

ext/intl/php_intl.h

@@ -68,6 +68,7 @@ PHP_RSHUTDOWN_FUNCTION(intl);
 PHP_MINFO_FUNCTION(intl);
 
 const char *intl_locale_get_default( void );
+static char* canonicalize_locale_string(const char* locale);
 
 #define PHP_INTL_VERSION PHP_VERSION
 

ext/intl/tests/gh11942_datefmt_locale_canonicalization.phpt

@@ -0,0 +1,37 @@
+--TEST--
+Fix GH-11942: IntlDateFormatter should canonicalize locale strings
+--EXTENSIONS--
+intl
+--FILE--
+<?php
+
+$test_cases = [
+    ['pt', 'pt'],
+    ['pt-PT', 'pt_PT'],
+    ['pt_PT.utf8', 'pt_PT'],
+    ['fr_CA@euro', 'fr_CA'],
+];
+
+echo "Testing IntlDateFormatter locale canonicalization:\n";
+foreach ($test_cases as $test_case) {
+    [$input, $expected] = $test_case;
+    
+    $formatter = new IntlDateFormatter($input, IntlDateFormatter::SHORT, IntlDateFormatter::NONE, 'UTC');
+    $actual = $formatter->getLocale();
+
+    $status = ($actual === $expected) ? 'PASS' : 'FAIL';
+    echo "Input: $input -> Expected: $expected -> Actual: $actual -> $status\n";
+}
+
+$dateFormatter = new IntlDateFormatter('pt_PT.utf8', IntlDateFormatter::SHORT, IntlDateFormatter::NONE, 'UTC');
+$dateResult = $dateFormatter->format(1691585260);
+echo "\nDateFormatter with pt_PT.utf8: " . $dateResult . "\n";
+?>
+--EXPECT--
+Testing IntlDateFormatter locale canonicalization:
+Input: pt -> Expected: pt -> Actual: pt -> PASS
+Input: pt-PT -> Expected: pt_PT -> Actual: pt_PT -> PASS
+Input: pt_PT.utf8 -> Expected: pt_PT -> Actual: pt_PT -> PASS
+Input: fr_CA@euro -> Expected: fr_CA -> Actual: fr_CA -> PASS
+
+DateFormatter with pt_PT.utf8: 09/08/23

ext/intl/tests/gh11942_numfmt_locale_canonicalization.phpt

@@ -0,0 +1,37 @@
+--TEST--
+Fix GH-11942: NumberFormatter should canonicalize locale strings
+--EXTENSIONS--
+intl
+--FILE--
+<?php
+
+$test_cases = [
+    ['pt', 'pt'],
+    ['pt-PT', 'pt_PT'],
+    ['pt_PT.utf8', 'pt_PT'],
+    ['fr_CA@euro', 'fr_CA'],
+];
+
+echo "Testing NumberFormatter locale canonicalization:\n";
+foreach ($test_cases as $test_case) {
+    [$input, $expected] = $test_case;
+    
+    $formatter = new NumberFormatter($input, NumberFormatter::DECIMAL);
+    $actual = $formatter->getLocale();
+
+    $status = ($actual === $expected) ? 'PASS' : 'FAIL';
+    echo "Input: $input -> Expected: $expected -> Actual: $actual -> $status\n";
+}
+
+$numFormatter = new NumberFormatter('pt_PT.utf8', NumberFormatter::DECIMAL);
+$numResult = $numFormatter->format(1234.56);
+echo "\nNumberFormatter with pt_PT.utf8: " . $numResult . "\n";
+?>
+--EXPECT--
+Testing NumberFormatter locale canonicalization:
+Input: pt -> Expected: pt -> Actual: pt -> PASS
+Input: pt-PT -> Expected: pt_PT -> Actual: pt_PT -> PASS
+Input: pt_PT.utf8 -> Expected: pt_PT -> Actual: pt_PT -> PASS
+Input: fr_CA@euro -> Expected: fr_CA -> Actual: fr_CA -> PASS
+
+NumberFormatter with pt_PT.utf8: 1 234,56