SNOW-787480: fix json load encoding error (#1528)

Author: sfc-gh-aling

DESCRIPTION.md

@@ -10,6 +10,7 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 
 - v3.0.4(TBD)
   - Fixed a bug in which `cursor.execute()` could modify the argument statement_params dictionary object when executing a multistatement query.
+  - Added the json_result_force_utf8_decoding connection parameter to force decoding JSON content in utf-8 when the result format is JSON.
   - Fixed a bug in which we cannot call `SnowflakeCursor.nextset` before fetching the result of the first query if the cursor runs an async multistatement query.
   - Bumped vendored library urllib3 to 1.26.15
   - Bumped vendored library requests to 2.29.0

src/snowflake/connector/connection.py

@@ -204,6 +204,10 @@ def DefaultConverterClass() -> type:
         True,
         bool,
     ),  # Whether to log imported packages in telemetry
+    "json_result_force_utf8_decoding": (
+        False,
+        bool,
+    ),  # Whether to force the JSON content to be decoded in utf-8, it is only effective when result format is JSON
 }
 
 APPLICATION_RE = re.compile(r"[\w\d_]+")
@@ -265,6 +269,9 @@ class SnowflakeConnection:
         enable_connection_diag: when true, clients will generate a connectivity diagnostic report.
         connection_diag_log_path: path to location to create diag report with enable_connection_diag.
         connection_diag_whitelist_path: path to a whitelist.json file to test with enable_connection_diag.
+        json_result_force_utf8_decoding: When true, json result will be decoded in utf-8,
+          when false, the encoding of the content is auto-detected. Default value is false.
+          This parameter is only effective when the result format is JSON.
     """
 
     OCSP_ENV_LOCK = Lock()

src/snowflake/connector/result_batch.py

@@ -132,6 +132,7 @@ def remote_chunk_info(c: dict[str, Any]) -> RemoteChunkInfo:
                     schema,
                     column_converters,
                     cursor._use_dict_result,
+                    json_result_force_utf8_decoding=cursor._connection._json_result_force_utf8_decoding,
                 )
                 for c in chunks
             ]
@@ -384,6 +385,8 @@ def __init__(
         schema: Sequence[ResultMetadata],
         column_converters: Sequence[tuple[str, SnowflakeConverterType]],
         use_dict_result: bool,
+        *,
+        json_result_force_utf8_decoding: bool = False,
     ) -> None:
         super().__init__(
             rowcount,
@@ -392,6 +395,7 @@ def __init__(
             schema,
             use_dict_result,
         )
+        self._json_result_force_utf8_decoding = json_result_force_utf8_decoding
         self.column_converters = column_converters
 
     @classmethod
@@ -420,10 +424,21 @@ def _load(self, response: Response) -> list:
 
         Returns:
             Whatever ``json.loads`` return, but in a list.
-            Unfortunately there's not type hint for this.
+            Unfortunately there's no type hint for this.
             For context: https://github.com/python/typing/issues/182
         """
-        read_data = response.text
+        # if users specify how to decode the data, we decode the bytes using the specified encoding
+        if self._json_result_force_utf8_decoding:
+            try:
+                read_data = str(response.content, "utf-8", errors="strict")
+            except Exception as exc:
+                err_msg = f"failed to decode json result content due to error {exc!r}"
+                logger.error(err_msg)
+                raise Error(msg=err_msg)
+        else:
+            # note: SNOW-787480 response.apparent_encoding is unreliable, chardet.detect can be wrong which is used by
+            # response.text to decode content, check issue: https://github.com/chardet/chardet/issues/148
+            read_data = response.text
         return json.loads("".join(["[", read_data, "]"]))
 
     def _parse(

test/integ/test_cursor.py

@@ -51,6 +51,7 @@ class ResultMetadata(NamedTuple):
     ER_FAILED_TO_REWRITE_MULTI_ROW_INSERT,
     ER_NOT_POSITIVE_SIZE,
 )
+from snowflake.connector.errors import Error
 from snowflake.connector.sqlstate import SQLSTATE_FEATURE_NOT_SUPPORTED
 from snowflake.connector.telemetry import TelemetryField
 
@@ -1618,3 +1619,42 @@ def test_multi_statement_failure(conn_cnx):
             CLIENT_VERSION,
             (type(None), str),
         )
+
+
+@pytest.mark.skipolddriver
+def test_decoding_utf8_for_json_result(conn_cnx):
+    # SNOW-787480, if not explicitly setting utf-8 decoding, the data will be
+    # detected decoding as windows-1250 by chardet.detect
+    with conn_cnx(
+        session_parameters={"python_connector_query_result_format": "JSON"}
+    ) as con, con.cursor() as cur:
+        sql = """select '"",' || '"",' || '"",' || '"",' || '"",' || 'Ofigràfic' || '"",' from TABLE(GENERATOR(ROWCOUNT => 5000)) v;"""
+        ret = cur.execute(sql).fetchall()
+        assert len(ret) == 5000
+        # This test case is tricky, for most of the test cases, the decoding is incorrect and can could be different
+        # on different platforms, however, due to randomness, in rare cases the decoding is indeed utf-8,
+        # the backend behavior is flaky
+        assert ret[0] in (
+            ('"","","","","",OfigrĂ\xa0fic"",',),  # AWS Cloud
+            ('"","","","","",OfigrÃ\xa0fic"",',),  # GCP Mac and Linux Cloud
+            ('"","","","","",Ofigr\xc3\\xa0fic"",',),  # GCP Windows Cloud
+            (
+                '"","","","","",Ofigràfic"",',
+            ),  # regression environment gets the correct decoding
+        )
+
+    with conn_cnx(
+        session_parameters={"python_connector_query_result_format": "JSON"},
+        json_result_force_utf8_decoding=True,
+    ) as con, con.cursor() as cur:
+        ret = cur.execute(sql).fetchall()
+        assert len(ret) == 5000
+        assert ret[0] == ('"","","","","",Ofigràfic"",',)
+
+    result_batch = JSONResultBatch(
+        None, None, None, None, None, False, json_result_force_utf8_decoding=True
+    )
+    mock_resp = mock.Mock()
+    mock_resp.content = "À".encode("latin1")
+    with pytest.raises(Error):
+        result_batch._load(mock_resp)