Fix Default Encoding for application/json Content-Type in HTTP Responses (#1422)

Author: snoopdoggy322

pkgs/http/CHANGELOG.md

@@ -1,4 +1,9 @@
-## 1.3.0
+## 1.4.0-wip
+
+* Fixed default encoding for application/json without a charset
+  to use utf8 instead of latin1, ensuring proper JSON decoding.
+
+## 1.3.0-wip
 
 * Fixed unintended HTML tags in doc comments.
 * Switched `BrowserClient` to use Fetch API instead of `XMLHttpRequest`.

pkgs/http/lib/src/multipart_file.dart

@@ -73,7 +73,7 @@ class MultipartFile {
   factory MultipartFile.fromString(String field, String value,
       {String? filename, MediaType? contentType}) {
     contentType ??= MediaType('text', 'plain');
-    var encoding = encodingForCharset(contentType.parameters['charset'], utf8);
+    var encoding = encodingForContentTypeHeader(contentType, utf8);
     contentType = contentType.change(parameters: {'charset': encoding.name});
 
     return MultipartFile.fromBytes(field, encoding.encode(value),

pkgs/http/lib/src/response.dart

@@ -21,9 +21,11 @@ class Response extends BaseResponse {
   ///
   /// This is converted from [bodyBytes] using the `charset` parameter of the
   /// `Content-Type` header field, if available. If it's unavailable or if the
-  /// encoding name is unknown, [latin1] is used by default, as per
-  /// [RFC 2616][].
+  /// encoding name is unknown:
+  /// - [utf8] is used when the content-type is 'application/json' (see [RFC 3629][]).
+  /// - [latin1] is used in all other cases (see [RFC 2616][])
   ///
+  /// [RFC 3629]: https://www.rfc-editor.org/rfc/rfc3629.
   /// [RFC 2616]: http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
   String get body => _encodingForHeaders(headers).decode(bodyBytes);
 
@@ -66,10 +68,13 @@ class Response extends BaseResponse {
 
 /// Returns the encoding to use for a response with the given headers.
 ///
-/// Defaults to [latin1] if the headers don't specify a charset or if that
-/// charset is unknown.
+/// If the `Content-Type` header specifies a charset, it will use that charset.
+/// If no charset is provided or the charset is unknown:
+/// - Defaults to [utf8] if the `Content-Type` is `application/json`
+///   (since JSON is defined to use UTF-8 by default).
+/// - Otherwise, defaults to [latin1] for compatibility.
 Encoding _encodingForHeaders(Map<String, String> headers) =>
-    encodingForCharset(_contentTypeForHeaders(headers).parameters['charset']);
+    encodingForContentTypeHeader(_contentTypeForHeaders(headers));
 
 /// Returns the [MediaType] object for the given headers' content-type.
 ///

pkgs/http/lib/src/utils.dart

@@ -6,6 +6,8 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:typed_data';
 
+import 'package:http_parser/http_parser.dart';
+
 import 'byte_stream.dart';
 
 /// Converts a [Map] from parameter names to values to a URL query string.
@@ -18,13 +20,27 @@ String mapToQuery(Map<String, String> map, {required Encoding encoding}) =>
             '=${Uri.encodeQueryComponent(e.value, encoding: encoding)}')
         .join('&');
 
-/// Returns the [Encoding] that corresponds to [charset].
+/// Determines the appropriate [Encoding] based on the given [contentTypeHeader]
 ///
-/// Returns [fallback] if [charset] is null or if no [Encoding] was found that
-/// corresponds to [charset].
-Encoding encodingForCharset(String? charset, [Encoding fallback = latin1]) {
-  if (charset == null) return fallback;
-  return Encoding.getByName(charset) ?? fallback;
+/// - If the `Content-Type` is `application/json` and no charset is specified,
+///   it defaults to [utf8].
+/// - If a charset is specified in the parameters,
+///   it attempts to find a matching [Encoding].
+/// - If no charset is specified or the charset is unknown,
+///   it falls back to the provided [fallback], which defaults to [latin1].
+Encoding encodingForContentTypeHeader(MediaType contentTypeHeader,
+    [Encoding fallback = latin1]) {
+  final charset = contentTypeHeader.parameters['charset'];
+
+  // Default to utf8 for application/json when charset is unspecified.
+  if (contentTypeHeader.type == 'application' &&
+      contentTypeHeader.subtype == 'json' &&
+      charset == null) {
+    return utf8;
+  }
+
+  // Attempt to find the encoding or fall back to the default.
+  return charset != null ? Encoding.getByName(charset) ?? fallback : fallback;
 }
 
 /// Returns the [Encoding] that corresponds to [charset].

pkgs/http/pubspec.yaml

@@ -1,5 +1,5 @@
 name: http
-version: 1.3.0
+version: 1.4.0-wip
 description: A composable, multi-platform, Future-based API for HTTP requests.
 repository: https://github.com/dart-lang/http/tree/master/pkgs/http
 

pkgs/http/test/response_test.dart

@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 
 import 'package:http/http.dart' as http;
 import 'package:test/test.dart';
@@ -45,6 +46,49 @@ void main() {
           headers: {'content-type': 'text/plain; charset=iso-8859-1'});
       expect(response.body, equals('föøbãr'));
     });
+
+    test(
+        'decoding with empty charset if content type is application/json, Russian text',
+        () {
+      final utf8Bytes = utf8.encode('{"foo":"Привет, мир!"}');
+      var response = http.Response.bytes(utf8Bytes, 200,
+          headers: {'content-type': 'application/json'});
+      expect(response.body, equals('{"foo":"Привет, мир!"}'));
+    });
+
+    test(
+        'test decoding with empty charset if content type is application/json, Chinese text',
+        () {
+      final chineseUtf8Bytes = utf8.encode('{"foo":"你好，世界！"}');
+      var responseChinese = http.Response.bytes(chineseUtf8Bytes, 200,
+          headers: {'content-type': 'application/json'});
+
+      expect(responseChinese.body, equals('{"foo":"你好，世界！"}'));
+    });
+
+    test(
+        'test decoding with empty charset if content type is application/json, Korean text',
+        () {
+      final koreanUtf8Bytes = utf8.encode('{"foo":"안녕하세요, 세계!"}');
+      var responseKorean = http.Response.bytes(koreanUtf8Bytes, 200,
+          headers: {'content-type': 'application/json'});
+      expect(responseKorean.body, equals('{"foo":"안녕하세요, 세계!"}'));
+    });
+
+    test('respects the inferred encoding for application/json content-type',
+        () {
+      final latin1Bytes = latin1.encode('{"foo":"Olá, mundo!"}');
+      var response = http.Response.bytes(latin1Bytes, 200,
+          headers: {'content-type': 'application/json; charset=iso-8859-1'});
+      expect(response.body, equals('{"foo":"Olá, mundo!"}'));
+    });
+
+    test('latin1 decode on empty charset if content type is text/plain', () {
+      final latin1Bytes = latin1.encode('Hello, wold!');
+      var response = http.Response.bytes(latin1Bytes, 200,
+          headers: {'content-type': 'text/plain'});
+      expect(response.body, equals('Hello, wold!'));
+    });
   });
 
   group('.fromStream()', () {