MIME: Introduce MIME type parser

src/wp-settings.php

@@ -256,6 +256,7 @@
 require ABSPATH . WPINC . '/class-wp-oembed-controller.php';
 require ABSPATH . WPINC . '/media.php';
 require ABSPATH . WPINC . '/http.php';
+require ABSPATH . WPINC . '/class-wp-mime-sniffer.php';
 require ABSPATH . WPINC . '/html-api/html5-named-character-references.php';
 require ABSPATH . WPINC . '/html-api/class-wp-html-attribute-token.php';
 require ABSPATH . WPINC . '/html-api/class-wp-html-span.php';

tests/phpunit/data/mime/fetch-wpt-mimesniff-data.php

@@ -0,0 +1,106 @@
+<?php
+
+/**
+ * Downloads the web-platform-test test suite for MIME sniffing.
+ *
+ * Update the SHA as appropriate to keep updated with the upstream
+ * project. It may happen that changes to the project move files
+ * around, rename, or delete files. When that happens, update the
+ * manifest accordingly. This script is locked to a specific SHA
+ * to ensure that when backwards-incompatible changes occur, that
+ * WordPress tests aren’t broken because of it.
+ */
+
+$opts     = getopt( 'q', array( 'color::', 'force' ) );
+$force_it = isset( $opts['force'] );
+if ( isset( $opts['q'] ) || 'never' === ( $opts['color'] ?? 'auto' ) ) {
+	$use_color = false;
+} elseif ( 'always' === ( $opts['color'] ?? 'auto' ) ) {
+	$use_color = true;
+} else {
+	$use_color = posix_isatty( STDOUT );
+}
+$r = $use_color ? "\e[31m" : '';
+$g = $use_color ? "\e[32m" : '';
+$y = $use_color ? "\e[33m" : '';
+$b = $use_color ? "\e[34m" : '';
+$v = $use_color ? "\e[35m" : '';
+$c = $use_color ? "\e[36m" : '';
+$_ = $use_color ? "\e[90m" : '';
+$z = $use_color ? "\e[m" : '';
+
+$base_url = 'https://raw.githubusercontent.com';
+$repo     = 'web-platform-tests/wpt';
+$sha      = 'f1edffe3511493e684402fdfed42433e79374d55';
+
+// Contains a map of { "FilePath:ETag": { "/path/to/file.json": "LastEtagValue" | null } }
+$manifest_path = __DIR__ . '/manifest.json';
+$manifest      = json_decode( file_get_contents( $manifest_path ), true );
+
+foreach ( $manifest['FilePath:ETag'] as $file_path => $etag ) {
+	$url = "{$base_url}/{$repo}/{$sha}/{$file_path}";
+	$ch  = curl_init();
+
+	echo "{$_}Fetching '{$c}{$url}{$_}'...{$z}\n";
+	$use_etag = ! $force_it && is_string( $etag );
+
+	curl_setopt_array( $ch, [
+		CURLOPT_URL            => $url,
+		CURLOPT_FOLLOWLOCATION => true,
+		CURLOPT_MAXREDIRS      => 3,
+		CURLOPT_RETURNTRANSFER => true,
+		CURLOPT_USERAGENT      => 'PHP/WordPress Tests/updating-test-data',
+		CURLOPT_TIMEOUT        => 30,
+		CURLOPT_CONNECTTIMEOUT => 10,
+		CURLOPT_HTTPHEADER     => $use_etag ? array( "If-None-Match: \"{$etag}\"" ) : array(),
+		CURLOPT_HEADER         => true,
+	] );
+
+	$response = curl_exec( $ch );
+	if ( $response === false ) {
+		$error = curl_error( $ch );
+		echo "{$_}Failed to download '{$g}{$file_path}{$_}'. Skipping.{$z}\n";
+		continue;
+	}
+
+	$header_size = curl_getinfo( $ch, CURLINFO_HEADER_SIZE );
+	$headers     = substr( $response, 0, $header_size );
+	$test_data   = substr( $response, $header_size );
+
+	$status_code = curl_getinfo( $ch, CURLINFO_HTTP_CODE );
+
+	if ( $status_code === 304 ) {
+		echo "{$_}Test file '{$g}{$file_path}{$_}' unchanged since last request. Skipping.{$z}\n";
+		continue;
+	}
+
+	if ( $status_code !== 200 ) {
+		echo "{$_}Unhandled HTTP {$y}{$status_code}{$_} response for '{$g}{$file_path}{$_}'. Skipping.{$z}\n";
+		continue;
+	}
+
+	if ( str_ends_with( $file_path, ".json" ) ) {
+		$test_decode = json_decode( $test_data, true );
+		if ( null === $test_decode || JSON_ERROR_NONE !== json_last_error() ) {
+			$error = json_last_error_msg();
+			echo "{$_}Unable to parse JSON for '{$g}{$file_path}{$_}'. Skipping.{$z}\n";
+			echo "{$r}  {$error}{$z}\n";
+			continue;
+		}
+	}
+
+	$new_etag   = null;
+	if ( 1 === preg_match( '~(^|\r\n)ETag:\s+"(?P<ETAG_VALUE>[^"]+)"\r\n~i', $headers, $etag_match ) ) {
+		$new_etag = $etag_match['ETAG_VALUE'];
+	}
+
+	file_put_contents( __DIR__ . '/wpt-tests/' . basename( $file_path ), $test_data );
+	if ( isset( $new_etag ) ) {
+		$manifest[ 'FilePath:ETag' ][ $file_path ] = $new_etag;
+		echo "{$_}Updated '{$g}{$file_path}{$_}' setting ETag to '{$v}{$new_etag}{$_}'.{$z}\n";
+	} else {
+		echo "{$_}Updated '{$g}{$file_path}{$_}' but {$r}received no ETag.{$z}\n";
+	}
+}
+
+file_put_contents( $manifest_path, json_encode( $manifest, JSON_PRETTY_PRINT ) );

tests/phpunit/data/mime/manifest.json

@@ -0,0 +1,10 @@
+{
+    "FilePath:ETag": {
+        "LICENSE.md": "df56994c53fd4d23b794258400a8e3519473cae98a4c590148028c25525238b3",
+        "mimesniff\/mime-types\/README.md": "688cfc802263c15caa21e810d91fe9d960b80cea8f4f505139b55cdd19a25676",
+        "mimesniff\/mime-types\/resources\/generated-mime-types.json": "66586a88f18e7b6e1c8bcf8b6348470181977b41f1ec85618251582aac4ae4a3",
+        "mimesniff\/mime-types\/resources\/mime-groups.json": "d3769412d4ed166ac262cd0b403588362aaa874b79328a92f27749374600a2c5",
+        "mimesniff\/mime-types\/resources\/mime-types-minimized.json": "4bdd2793b286fda8548c6e96c435fd037daf3029a5d5048fdbb183ff6f666ad4",
+        "mimesniff\/mime-types\/resources\/mime-types.json": "68e83b8c060a17aa88b307415c4e5b17e00442a6cf99d99ad42b2f1ea23323a6"
+    }
+}

tests/phpunit/data/mime/wpt-tests/LICENSE.md

@@ -0,0 +1,11 @@
+# The 3-Clause BSD License
+
+Copyright © web-platform-tests contributors
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tests/phpunit/data/mime/wpt-tests/README.md

@@ -0,0 +1,47 @@
+## MIME types
+
+`resources/mime-types.json` and `resources/generated-mime-types.json` contain MIME type tests. The tests are encoded as a JSON array. String values in the array serve as documentation. All other values are objects with the following fields:
+
+* `input`: The string to be parsed.
+* `output`: Null if parsing resulted in failure and the MIME type record serialized as string otherwise.
+* `navigable`: True if the MIME type can be used for a document to be loaded in a browsing context (i.e., does not result in a download) and omitted otherwise.
+* `encoding`: The encoding that can be extracted from the MIME type or null if no encoding can be extracted, and omitted otherwise.
+
+Note: the object description implies that there tests without `navigable` or `encoding` set.
+
+A wrapper for these JSON MIME type tests needs to take care that not all `input` values can be tested in all entrypoints. Some entrypoints only accept bytes and some have further restrictions. A function such as the one below can be used to differentiate:
+
+```js
+function isByteCompatible(str) {
+  // see https://fetch.spec.whatwg.org/#concept-header-value-normalize
+  if(/^[\u0009\u0020\u000A\u000D]+|[\u0009\u0020\u000A\u000D]+$/.test(str)) {
+    return "header-value-incompatible";
+  }
+
+  for(let i = 0; i < str.length; i++) {
+    const charCode = str.charCodeAt(i);
+    // See https://fetch.spec.whatwg.org/#concept-header-value
+    if(charCode > 0xFF) {
+      return "incompatible";
+    } else if(charCode === 0x00 || charCode === 0x0A || charCode === 0x0D) {
+      return "header-value-incompatible";
+    }
+  }
+  return "compatible";
+}
+```
+
+`resources/generated-mime-types.json` is generated by running `resources/generated-mime-types.py`. Modify the latter to correct the former.
+
+These tests are used by resources in this directory to test various aspects of MIME types.
+
+## MIME type groups
+
+`resources/mime-groups.json` contains MIME type group-membership tests. The tests are encoded as a JSON array. String values in the array serve as documentation. All other values are objects with the following fields:
+
+* `input`: The MIME type to test.
+* `groups`: An array of zero or more groups to which the MIME type belongs.
+
+In order to pass the tests an implementation must treat each MIME type as belonging to the exact set of groups listed, with no additions or omissions.
+
+Note: As MIME type groups are used only while determining the computed MIME type of a resource and are not exposed in any API, no browser-based test harness is available.