Charset: Improve UTF-8 scrubbing ability via new UTF-8 scanning pipeline.

This is the fourth in a series of patches to modernize and standardize UTF-8 handling.

`wp_check_invalid_utf8()` has long been dependent on the runtime configuration of the system running it. This has led to hard-to-diagnose issues with text containing invalid UTF-8. The function has also had an apparent defect since its inception: when requesting to strip invalid bytes it returns an empty string.

This patch updates the function to remove all dependency on the system running it. It defers to the `mbstring` extension if that’s available, falling back to the new UTF-8 scanning pipeline.

To support this work, `wp_scrub_utf8()` is created with a proper fallback so that the remaining logic inside of `wp_check_invalid_utf8()` can be minimized. The defect in this function has been fixed, but instead of stripping the invalid bytes it will replace them with the Unicode replacement character for stronger security guarantees.

Developed in #9498
Discussed in https://core.trac.wordpress.org/ticket/63837

Follow-up to: [60768].
Props askapache, chriscct7, Cyrille37, desrosj, dmsnell, helen, jonsurrell, kitchin, miqrogroove, pbearne, shailu25.
Fixes #63837, #29717.
See #63863.


git-svn-id: https://develop.svn.wordpress.org/trunk@60793 602fd350-edb4-49c9-b593-d223f7449a82

Author: dmsnell

src/wp-includes/compat-utf8.php

@@ -227,11 +227,11 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
 /**
  * Fallback mechanism for safely validating UTF-8 bytes.
  *
- * @see wp_is_valid_utf8()
- *
  * @since 6.9.0
  * @access private
  *
+ * @see wp_is_valid_utf6()
+ *
  * @param string $bytes String which might contain text encoded as UTF-8.
  * @return bool Whether the provided bytes can decode as valid UTF-8.
  */
@@ -248,3 +248,46 @@ function _wp_is_valid_utf8_fallback( string $bytes ): bool {
 
 	return $bytes_length === $next_byte_at && 0 === $invalid_length;
 }
+
+/**
+ * Fallback mechanism for replacing invalid spans of UTF-8 bytes.
+ *
+ * Example:
+ *
+ *     'Pi�a' === _wp_scrub_utf8_fallback( "Pi\xF1a" ); // “ñ” is 0xF1 in Windows-1252.
+ *
+ * @since 6.9.0
+ * @access private
+ *
+ * @see wp_scrub_utf8()
+ *
+ * @param string $bytes UTF-8 encoded string which might contain spans of invalid bytes.
+ * @return string Input string with spans of invalid bytes swapped with the replacement character.
+ */
+function _wp_scrub_utf8_fallback( string $bytes ): string {
+	$bytes_length   = strlen( $bytes );
+	$next_byte_at   = 0;
+	$was_at         = 0;
+	$invalid_length = 0;
+	$scrubbed       = '';
+
+	while ( $next_byte_at <= $bytes_length ) {
+		_wp_scan_utf8( $bytes, $next_byte_at, $invalid_length );
+
+		if ( $next_byte_at >= $bytes_length ) {
+			if ( 0 === $was_at ) {
+				return $bytes;
+			}
+
+			return $scrubbed . substr( $bytes, $was_at, $next_byte_at - $was_at - $invalid_length );
+		}
+
+		$scrubbed .= substr( $bytes, $was_at, $next_byte_at - $was_at );
+		$scrubbed .= "\u{FFFD}";
+
+		$next_byte_at += $invalid_length;
+		$was_at        = $next_byte_at;
+	}
+
+	return $scrubbed;
+}

src/wp-includes/formatting.php

@@ -917,58 +917,6 @@ function seems_utf8( $str ) {
 	return true;
 }
 
-/**
- * Determines if a given byte string represents a valid UTF-8 encoding.
- *
- * Note that it’s unlikely for non-UTF-8 data to validate as UTF-8, but
- * it is still possible. Many texts are simultaneously valid UTF-8,
- * valid US-ASCII, and valid ISO-8859-1 (`latin1`).
- *
- * Example:
- *
- *     true === wp_is_valid_utf8( '' );
- *     true === wp_is_valid_utf8( 'just a test' );
- *     true === wp_is_valid_utf8( "\xE2\x9C\x8F" );    // Pencil, U+270F.
- *     true === wp_is_valid_utf8( "\u{270F}" );        // Pencil, U+270F.
- *     true === wp_is_valid_utf8( '✏' );              // Pencil, U+270F.
- *
- *     false === wp_is_valid_utf8( "just \xC0 test" ); // Invalid bytes.
- *     false === wp_is_valid_utf8( "\xE2\x9C" );       // Invalid/incomplete sequences.
- *     false === wp_is_valid_utf8( "\xC1\xBF" );       // Overlong sequences.
- *     false === wp_is_valid_utf8( "\xED\xB0\x80" );   // Surrogate halves.
- *     false === wp_is_valid_utf8( "B\xFCch" );        // ISO-8859-1 high-bytes.
- *                                                     // E.g. The “ü” in ISO-8859-1 is a single byte 0xFC,
- *                                                     // but in UTF-8 is the two-byte sequence 0xC3 0xBC.
- *
- * A “valid” string consists of “well-formed UTF-8 code unit sequence[s],” meaning
- * that the bytes conform to the UTF-8 encoding scheme, all characters use the minimal
- * byte sequence required by UTF-8, and that no sequence encodes a UTF-16 surrogate
- * code point or any character above the representable range.
- *
- * @see https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G32860
- *
- * @see _wp_is_valid_utf8_fallback
- *
- * @since 6.9.0
- *
- * @param string $bytes String which might contain text encoded as UTF-8.
- * @return bool Whether the provided bytes can decode as valid UTF-8.
- */
-function wp_is_valid_utf8( string $bytes ): bool {
-	/*
-	 * Since PHP 8.3.0 the UTF-8 validity is cached internally
-	 * on string objects, making this a direct property lookup.
-	 *
-	 * This is to be preferred exclusively once PHP 8.3.0 is
-	 * the minimum supported version, because even when the
-	 * status isn’t cached, it uses highly-optimized code to
-	 * validate the byte stream.
-	 */
-	return function_exists( 'mb_check_encoding' )
-		? mb_check_encoding( $bytes, 'UTF-8' )
-		: _wp_is_valid_utf8_fallback( $bytes );
-}
-
 /**
  * Converts a number of special characters into their HTML entities.
  *
@@ -1141,10 +1089,39 @@ function wp_specialchars_decode( $text, $quote_style = ENT_NOQUOTES ) {
 /**
  * Checks for invalid UTF8 in a string.
  *
+ * Note! This function only performs its work if the `blog_charset` is set
+ * to UTF-8. For all other values it returns the input text unchanged.
+ *
+ * Note! Unless requested, this returns an empty string if the input contains
+ * any sequences of invalid UTF-8. To replace invalid byte sequences, pass
+ * `true` as the optional `$strip` parameter.
+ *
+ * Consider using {@see wp_scrub_utf8()} instead which does not depend on
+ * the value of `blog_charset`.
+ *
+ * Example:
+ *
+ *     // The `blog_charset` is `latin1`, so this returns the input unchanged.
+ *     $every_possible_input === wp_check_invalid_utf8( $every_possible_input );
+ *
+ *     // Valid strings come through unchanged.
+ *     'test' === wp_check_invalid_utf8( 'test' );
+ *
+ *     $invalid = "the byte \xC0 is never allowed in a UTF-8 string.";
+ *
+ *     // Invalid strings are rejected outright.
+ *     '' === wp_check_invalid_utf8( $invalid );
+ *
+ *     // “Stripping” invalid sequences produces the replacement character instead.
+ *     "the byte \u{FFFD} is never allowed in a UTF-8 string." === wp_check_invalid_utf8( $invalid, true );
+ *     'the byte � is never allowed in a UTF-8 string.' === wp_check_invalid_utf8( $invalid, true );
+ *
  * @since 2.8.0
+ * @since 6.9.0 Stripping replaces invalid byte sequences with the Unicode replacement character U+FFFD (�).
  *
- * @param string $text   The text which is to be checked.
- * @param bool   $strip  Optional. Whether to attempt to strip out invalid UTF8. Default false.
+ * @param string $text   String which is expected to be encoded as UTF-8 unless `blog_charset` is another encoding.
+ * @param bool   $strip  Optional. Whether to replace invalid sequences of bytes with the Unicode replacement
+ *                       character (U+FFFD `�`). Default `false` returns an empty string for invalid UTF-8 inputs.
  * @return string The checked text.
  */
 function wp_check_invalid_utf8( $text, $strip = false ) {
@@ -1159,32 +1136,14 @@ function wp_check_invalid_utf8( $text, $strip = false ) {
 	if ( ! isset( $is_utf8 ) ) {
 		$is_utf8 = is_utf8_charset();
 	}
-	if ( ! $is_utf8 ) {
-		return $text;
-	}
-
-	// Check for support for utf8 in the installed PCRE library once and store the result in a static.
-	static $utf8_pcre = null;
-	if ( ! isset( $utf8_pcre ) ) {
-		// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
-		$utf8_pcre = @preg_match( '/^./u', 'a' );
-	}
-	// We can't demand utf8 in the PCRE installation, so just return the string in those cases.
-	if ( ! $utf8_pcre ) {
-		return $text;
-	}
 
-	// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- preg_match fails when it encounters invalid UTF8 in $text.
-	if ( 1 === @preg_match( '/^./us', $text ) ) {
+	if ( ! $is_utf8 || wp_is_valid_utf8( $text ) ) {
 		return $text;
 	}
 
-	// Attempt to strip the bad chars if requested (not recommended).
-	if ( $strip && function_exists( 'iconv' ) ) {
-		return iconv( 'utf-8', 'utf-8', $text );
-	}
-
-	return '';
+	return $strip
+		? wp_scrub_utf8( $text )
+		: '';
 }
 
 /**

src/wp-includes/utf8.php

@@ -0,0 +1,135 @@
+<?php
+
+if ( extension_loaded( 'mbstring' ) ) :
+	/**
+	 * Determines if a given byte string represents a valid UTF-8 encoding.
+	 *
+	 * Note that it’s unlikely for non-UTF-8 data to validate as UTF-8, but
+	 * it is still possible. Many texts are simultaneously valid UTF-8,
+	 * valid US-ASCII, and valid ISO-8859-1 (`latin1`).
+	 *
+	 * Example:
+	 *
+	 *     true === wp_is_valid_utf8( '' );
+	 *     true === wp_is_valid_utf8( 'just a test' );
+	 *     true === wp_is_valid_utf8( "\xE2\x9C\x8F" );    // Pencil, U+270F.
+	 *     true === wp_is_valid_utf8( "\u{270F}" );        // Pencil, U+270F.
+	 *     true === wp_is_valid_utf8( '✏' );              // Pencil, U+270F.
+	 *
+	 *     false === wp_is_valid_utf8( "just \xC0 test" ); // Invalid bytes.
+	 *     false === wp_is_valid_utf8( "\xE2\x9C" );       // Invalid/incomplete sequences.
+	 *     false === wp_is_valid_utf8( "\xC1\xBF" );       // Overlong sequences.
+	 *     false === wp_is_valid_utf8( "\xED\xB0\x80" );   // Surrogate halves.
+	 *     false === wp_is_valid_utf8( "B\xFCch" );        // ISO-8859-1 high-bytes.
+	 *                                                     // E.g. The “ü” in ISO-8859-1 is a single byte 0xFC,
+	 *                                                     // but in UTF-8 is the two-byte sequence 0xC3 0xBC.
+	 *
+	 *  A “valid” string consists of “well-formed UTF-8 code unit sequence[s],” meaning
+	 *  that the bytes conform to the UTF-8 encoding scheme, all characters use the minimal
+	 *  byte sequence required by UTF-8, and that no sequence encodes a UTF-16 surrogate
+	 *  code point or any character above the representable range.
+	 *
+	 * @see https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G32860
+	 *
+	 * @since 6.9.0
+	 *
+	 * @param string $bytes String which might contain text encoded as UTF-8.
+	 * @return bool Whether the provided bytes can decode as valid UTF-8.
+	 */
+	function wp_is_valid_utf8( string $bytes ): bool {
+		return mb_check_encoding( $bytes, 'UTF-8' );
+	}
+else :
+	/**
+	 * Fallback function for validating UTF-8.
+	 *
+	 * @ignore
+	 * @private
+	 *
+	 * @since 6.9.0
+	 */
+	function wp_is_valid_utf8( string $string ): bool {
+		return _wp_is_valid_utf8_fallback( $string );
+	}
+endif;
+
+if (
+	extension_loaded( 'mbstring' ) &&
+	// Maximal subpart substitution introduced by php/php-src@04e59c916f12b322ac55f22314e31bd0176d01cb.
+	version_compare( PHP_VERSION, '8.1.6', '>=' )
+) :
+	/**
+	 * Replaces ill-formed UTF-8 byte sequences with the Unicode Replacement Character.
+	 *
+	 * Knowing what to do in the presence of text encoding issues can be complicated.
+	 * This function replaces invalid spans of bytes to neutralize any corruption that
+	 * may be there and prevent it from causing further problems downstream.
+	 *
+	 * However, it’s not always ideal to replace those bytes. In some settings it may
+	 * be best to leave the invalid bytes in the string so that downstream code can handle
+	 * them in a specific way. Replacing the bytes too early, like escaping for HTML too
+	 * early, can introduce other forms of corruption and data loss.
+	 *
+	 * When in doubt, use this function to replace spans of invalid bytes.
+	 *
+	 * Replacement follows the “maximal subpart” algorithm for secure and interoperable
+	 * strings. This can lead to sequences of multiple replacement characters in a row.
+	 *
+	 * Example:
+	 *
+	 *     // Valid strings come through unchanged.
+	 *     'test' === wp_scrub_utf8( 'test' );
+	 *
+	 *     // Invalid sequences of bytes are replaced.
+	 *     $invalid = "the byte \xC0 is never allowed in a UTF-8 string.";
+	 *     "the byte \u{FFFD} is never allowed in a UTF-8 string." === wp_scrub_utf8( $invalid, true );
+	 *     'the byte � is never allowed in a UTF-8 string.' === wp_scrub_utf8( $invalid, true );
+	 *
+	 *     // Maximal subparts are replaced individually.
+	 *     '.�.' === wp_scrub_utf8( ".\xC0." );              // C0 is never valid.
+	 *     '.�.' === wp_scrub_utf8( ".\xE2\x8C." );          // Missing A3 at end.
+	 *     '.��.' === wp_scrub_utf8( ".\xE2\x8C\xE2\x8C." ); // Maximal subparts replaced separately.
+	 *     '.��.' === wp_scrub_utf8( ".\xC1\xBF." );         // Overlong sequence.
+	 *     '.���.' === wp_scrub_utf8( ".\xED\xA0\x80." );    // Surrogate half.
+	 *
+	 * Note! The Unicode Replacement Character is itself a Unicode character (U+FFFD).
+	 * Once a span of invalid bytes has been replaced by one, it will not be possible
+	 * to know whether the replacement character was originally intended to be there
+	 * or if it is the result of scrubbing bytes. It is ideal to leave replacement for
+	 * display only, but some contexts (e.g. generating XML or passing data into a
+	 * large language model) require valid input strings.
+	 *
+	 * @since 6.9.0
+	 *
+	 * @see https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-5/#G40630
+	 *
+	 * @param string $text String which is assumed to be UTF-8 but may contain invalid sequences of bytes.
+	 * @return string Input text with invalid sequences of bytes replaced with the Unicode replacement character.
+	 */
+	function wp_scrub_utf8( $text ) {
+		/*
+		 * While it looks like setting the substitute character could fail,
+		 * the internal PHP code will never fail when provided a valid
+		 * code point as a number. In this case, there’s no need to check
+		 * its return value to see if it succeeded.
+		 */
+		$prev_replacement_character = mb_substitute_character();
+		mb_substitute_character( 0xFFFD );
+		$scrubbed = mb_scrub( $text, 'UTF-8' );
+		mb_substitute_character( $prev_replacement_character );
+
+		return $scrubbed;
+	}
+else :
+	/**
+	 * Fallback function for scrubbing UTF-8.
+	 *
+	 * @ignore
+	 * @private
+	 *
+	 * @since 6.9.0
+	 */
+	function wp_scrub_utf8( $text ) {
+		return _wp_scrub_utf8_fallback( $text );
+	}
+endif;

src/wp-settings.php

@@ -111,6 +111,7 @@
 // Load early WordPress files.
 require ABSPATH . WPINC . '/class-wp-list-util.php';
 require ABSPATH . WPINC . '/class-wp-token-map.php';
+require ABSPATH . WPINC . '/utf8.php';
 require ABSPATH . WPINC . '/formatting.php';
 require ABSPATH . WPINC . '/meta.php';
 require ABSPATH . WPINC . '/functions.php';