Skip to content

Commit 4975f20

Browse files
dmsnelldmsnell
committed
Charset: Track detection of non-characters when scanning UTF-8.
Noncharacters are code points that are permantently reserved in the Unicode Standard for internal use. They are not recommended for use in open interchange of Unicode text data. However, they are valid code points and will not cause a string to return as invalid. Still, HTML and XML both impose semantic rules on their use and it may be important for code to know whether they are present in a string. This patch introduces a new function, `wp_has_noncharacters()`, which answers this question. This is accomplished through an inline check with the fallback UTF-8 scanner. There are 66 noncharacters, making it difficult to find them properly with common string search functionality. While the inline check adds overhead to the scanning process, the rare occurrance of noncharacters should lead to minimal actual overhead due to strong branch prediction. See https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-23/#G12612
1 parent 843b569 commit 4975f20

File tree

3 files changed

+159
-11
lines changed

3 files changed

+159
-11
lines changed

‎src/wp-includes/compat-utf8.php‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,19 +35,21 @@
3535
* @since 6.9.0
3636
* @access private
3737
*
38-
* @param string $bytes UTF-8 encoded string which might include invalid spans of bytes.
39-
* @param int $at Where to start scanning.
40-
* @param int $invalid_length Will be set to how many bytes are to be ignored after `$at`.
41-
* @param int|null $max_bytes Stop scanning after this many bytes have been seen.
42-
* @param int|null $max_code_points Stop scanning after this many code points have been seen.
38+
* @param string $bytes UTF-8 encoded string which might include invalid spans of bytes.
39+
* @param int $at Where to start scanning.
40+
* @param int $invalid_length Will be set to how many bytes are to be ignored after `$at`.
41+
* @param int|null $max_bytes Stop scanning after this many bytes have been seen.
42+
* @param int|null $max_code_points Stop scanning after this many code points have been seen.
43+
* @param bool $has_noncharacters Set to indicate if scanned string contained noncharacters.
4344
* @return int How many code points were successfully scanned.
4445
*/
45-
function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max_bytes = null, ?int $max_code_points = null ): int {
46-
$byte_length = strlen( $bytes );
47-
$end = min( $byte_length, $at + ( $max_bytes ?? PHP_INT_MAX ) );
48-
$invalid_length = 0;
49-
$count = 0;
50-
$max_count = $max_code_points ?? PHP_INT_MAX;
46+
function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max_bytes = null, ?int $max_code_points = null, ?bool &$has_noncharacters = null ): int {
47+
$byte_length = strlen( $bytes );
48+
$end = min( $byte_length, $at + ( $max_bytes ?? PHP_INT_MAX ) );
49+
$invalid_length = 0;
50+
$count = 0;
51+
$max_count = $max_code_points ?? PHP_INT_MAX;
52+
$has_noncharacters = false;
5153

5254
for ( $i = $at; $i < $end && $count <= $max_count; $i++ ) {
5355
/*
@@ -145,6 +147,15 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
145147
) {
146148
++$count;
147149
$i += 2;
150+
151+
// Covers the range U+FDD0–U+FDEF, U+FFFE, U+FFFF.
152+
if ( 0xEF === $b1 ) {
153+
$has_noncharacters |= (
154+
( 0xB7 === $b2 && $b3 >= 0x90 && $b3 <= 0xAF ) ||
155+
( 0xBF === $b2 && ( 0xBE === $b3 || 0xBF === $b3 ) )
156+
);
157+
}
158+
148159
continue;
149160
}
150161

@@ -162,6 +173,14 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
162173
) {
163174
++$count;
164175
$i += 3;
176+
177+
// Covers U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, …, U+10FFFE, U+10FFFF.
178+
$has_noncharacters |= (
179+
( 0x0F === ( $b2 & 0x0F ) ) &&
180+
0xBF === $b3 &&
181+
( 0xBE === $b4 || 0xBF === $b4 )
182+
);
183+
165184
continue;
166185
}
167186

‎src/wp-includes/utf8.php‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,36 @@ function wp_scrub_utf8( $text ) {
133133
return _wp_scrub_utf8_fallback( $text );
134134
}
135135
endif;
136+
137+
/**
138+
* Returns whether the given string contains Unicode noncharacters.
139+
*
140+
* XML recommends against using noncharacters and HTML forbids their
141+
* use in attribute names. Unicode recommends that they not be used
142+
* in open exchange of data.
143+
*
144+
* Noncharacters are code points within the following ranges:
145+
* - U+FDD0–U+FDEF
146+
* - U+FFFE–U+FFFF
147+
* - U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, …, U+10FFFE, U+10FFFF
148+
*
149+
* @see https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-23/#G12612
150+
* @see https://www.w3.org/TR/xml/#charsets
151+
* @see https://html.spec.whatwg.org/#attributes-2
152+
*
153+
* @param string $text Are there noncharacters in this string?
154+
* @return bool Whether noncharacters were found in the string.
155+
*/
156+
function wp_has_noncharacters( string $text ): bool {
157+
$at = 0;
158+
$invalid_length = 0;
159+
$has_noncharacters = false;
160+
$end = strlen( $text );
161+
162+
while ( $at < $end && ! $has_noncharacters ) {
163+
_wp_scan_utf8( $text, $at, $invalid_length, null, null, $has_noncharacters );
164+
$at += $invalid_length;
165+
}
166+
167+
return $has_noncharacters;
168+
}

‎tests/phpunit/tests/unicode/wpHasNoncharacters.php‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
<?php
2+
/**
3+
* Unit tests covering WordPress’ UTF-8 handling: noncharacter detection.
4+
*
5+
* @package WordPress
6+
* @group unicode
7+
*/
8+
9+
class Tests_WpHasNoncharacters extends WP_UnitTestCase {
10+
/**
11+
* Ensures that a noncharacter inside a string will be properly detected.
12+
*
13+
* @dataProvider data_noncharacters
14+
*
15+
* @param string $noncharacter
16+
*/
17+
public function test_detects_non_characters( string $noncharacter ) {
18+
$this->assertTrue(
19+
wp_has_noncharacters( $noncharacter ),
20+
'Failed to detect entire string as noncharacter.'
21+
);
22+
23+
$this->assertTrue(
24+
wp_has_noncharacters( "{$noncharacter} and more." ),
25+
'Failed to detect noncharacter prefix.'
26+
);
27+
28+
$this->assertTrue(
29+
wp_has_noncharacters( "Some text and then a {$noncharacter} and more." ),
30+
'Failed to detect medial noncharacter.'
31+
);
32+
33+
$this->assertTrue(
34+
wp_has_noncharacters( "Some text and a {$noncharacter}." ),
35+
'Failed to detect noncharacter suffix.'
36+
);
37+
}
38+
39+
public function test_avoids_false_positives() {
40+
// Get all the noncharacters in one long string, each surrounded on both sides by null bytes.
41+
$noncharacters = array_column( array_values( iterator_to_array( self::data_noncharacters() ) ), 0 );
42+
$noncharacters = implode( "\x00", array_map( fn ( $c ) => "\x00{$c}", $noncharacters ) ) . "\x00";
43+
44+
$this->assertFalse(
45+
wp_has_noncharacters( "\x00" ),
46+
'Falsely detected noncharacter in U+0000'
47+
);
48+
49+
for ( $code_point = 1; $code_point <= 0x10FFFF; $code_point++ ) {
50+
// Surrogate halves are invalid UTF-8.
51+
if ( $code_point >= 0xD800 && $code_point <= 0xDFFF ) {
52+
continue;
53+
}
54+
55+
$char = mb_chr( $code_point );
56+
$hex_char = strtoupper( str_pad( dechex( $code_point ), 4, '0', STR_PAD_LEFT ) );
57+
58+
if ( str_contains( $noncharacters, $char ) ) {
59+
$this->assertTrue(
60+
wp_has_noncharacters( $char ),
61+
"Failed to detect noncharacter as test verification for U+{$hex_char}"
62+
);
63+
} else {
64+
$this->assertFalse(
65+
wp_has_noncharacters( $char ),
66+
"Falsely detected noncharacter in U+{$hex_char}."
67+
);
68+
}
69+
}
70+
}
71+
72+
/**
73+
* Data provider
74+
*
75+
* @return array[]
76+
*/
77+
public static function data_noncharacters() {
78+
for ( $code_point = 0xFDD0; $code_point <= 0xFDEF; $code_point++ ) {
79+
$hex_char = strtoupper( str_pad( dechex( $code_point ), 4, '0', STR_PAD_LEFT ) );
80+
yield "U+{$hex_char}" => array( mb_chr( $code_point ) );
81+
}
82+
83+
yield 'U+FFFE' => array( "\u{FFFE}" );
84+
yield 'U+FFFF' => array( "\u{FFFF}" );
85+
86+
for ( $plane = 0x10000; $plane <= 0x10FFFF; $plane += 0x10000 ) {
87+
$code_point = $plane + 0xFFFE;
88+
$hex_char = strtoupper( str_pad( dechex( $code_point ), 4, '0', STR_PAD_LEFT ) );
89+
yield "U+{$hex_char}" => array( mb_chr( $code_point ) );
90+
91+
$code_point = $plane + 0xFFFF;
92+
$hex_char = strtoupper( str_pad( dechex( $code_point ), 4, '0', STR_PAD_LEFT ) );
93+
yield "U+{$hex_char}" => array( mb_chr( $code_point ) );
94+
}
95+
}
96+
}

0 commit comments

Comments
 (0)