pybricksdev: use explicit text encoding for text files

Ensure that text files are opened with UTF-8 encoding by default to
avoid issues with different default encodings. This also requires
updating some dependencies that also had issues with text encoding.

PYTHONWARNDEFAULTENCODING is set for tests to try to catch any future
issues with text encoding. And PYTHONDEVMODE is set for good measure.

Author: dlech

.github/workflows/test.yml

@@ -38,6 +38,9 @@ jobs:
       - run: poetry install --only=main --only=test
       - run: poetry run pytest
         if: matrix.python-version != '3.10'
+        env:
+          PYTHONDEVMODE: 1
+          PYTHONWARNDEFAULTENCODING: 1
       - run: poetry run coverage run
         if: matrix.python-version == '3.10'
       - run: poetry run coverage xml

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Fixed calling `PybricksHub.write()` methods.
+- Fixed using default text encoding when opening text files.
 
 ### Changed
 - Downloading programs without starting them can now be done by

poetry.lock

@@ -68,7 +68,9 @@ def _get_script_path(file: TextIO) -> ContextManager[PathLike]:
         @contextlib.contextmanager
         def temp_context():
             try:
-                with NamedTemporaryFile(suffix=".py", delete=False) as temp:
+                with NamedTemporaryFile(
+                    suffix=".py", delete=False, encoding="utf-8"
+                ) as temp:
                     temp.write(file.buffer.read())
 
                 yield temp.name
@@ -98,7 +100,7 @@ def add_parser(self, subparsers: argparse._SubParsersAction):
             "file",
             metavar="<file>",
             help="path to a MicroPython script or `-` for stdin",
-            type=argparse.FileType(),
+            type=argparse.FileType(encoding="utf-8"),
         )
         parser.add_argument(
             "--abi",
@@ -135,7 +137,7 @@ def add_parser(self, subparsers: argparse._SubParsersAction):
             "file",
             metavar="<file>",
             help="path to a MicroPython script or `-` for stdin",
-            type=argparse.FileType(),
+            type=argparse.FileType(encoding="utf-8"),
         )
         parser.add_argument(
             "-n",

pybricksdev/cli/__init__.py

@@ -108,7 +108,7 @@ async def download_and_run(client: BleakClient, script: str, abi: int) -> None:
         script: The script to be compiled and run.
         abi: The MPY ABI version.
     """
-    with NamedTemporaryFile("w", suffix=".py") as temp:
+    with NamedTemporaryFile("w", suffix=".py", encoding="utf-8") as temp:
         temp.write(script)
 
         # file has to be closed so mpy-cross can open it

pybricksdev/cli/flash.py

@@ -55,7 +55,7 @@ async def compile_file(
     """
 
     # Get version info
-    with open(os.path.join(proj_dir, proj_path), "r") as f:
+    with open(os.path.join(proj_dir, proj_path), "r", encoding="utf-8") as f:
         loop = asyncio.get_running_loop()
         script = f.read()
 
@@ -188,7 +188,7 @@ def save_script(py_string):
     py_path = os.path.join(BUILD_DIR, TMP_PY_SCRIPT)
 
     # Write Python command to a file.
-    with open(py_path, "w") as f:
+    with open(py_path, "w", encoding="utf-8") as f:
         f.write(py_string)
         f.write("\n")
 

pybricksdev/compile.py

@@ -194,7 +194,7 @@ def _line_handler(self, line: bytes) -> None:
                 os.makedirs(dir_path)
 
             logger.info("Saving log to {0}.".format(full_path))
-            self.log_file = open(full_path, "w")
+            self.log_file = open(full_path, "w", encoding="utf-8")
             return
 
         # The line tells us to close a log file, so do it.

pybricksdev/connections/pybricks.py

@@ -28,16 +28,16 @@ include = [
 pybricksdev = 'pybricksdev.cli:main'
 
 [tool.poetry.dependencies]
-argcomplete = ">=1.11.1"
+argcomplete = ">=3.6.2"
 bleak = ">=0.22.0"
-mpy-cross-v5 = ">=1.0.0"
+mpy-cross-v5 = ">=1.1.0"
 python = ">=3.10,<3.14"
 tqdm = ">=4.62.3"
 pyusb = ">=1.0.2"
 semver = ">=2.13.0"
 appdirs = ">=1.4.4"
 prompt-toolkit = ">=3.0.18"
-mpy-cross-v6 = ">=1.0.0"
+mpy-cross-v6 = ">=1.1.0"
 packaging = ">=22"
 typing-extensions = ">=4.3.0"
 reactivex = ">=4.0.4"
@@ -74,6 +74,11 @@ known_third_party = ["usb"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "strict"
+filterwarnings = [
+    "error",
+    # Known issue with reactivex package
+    "ignore::DeprecationWarning:reactivex.internal",
+]
 
 [tool.coverage.run]
 branch = true

pyproject.toml

@@ -37,7 +37,9 @@ async def test_download_modern_protocol(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -62,7 +64,9 @@ async def test_download_legacy_firmware(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -86,7 +90,9 @@ async def test_download_unsupported_capabilities(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -114,7 +120,9 @@ async def test_download_compile_error(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test'  # Missing closing parenthesis")
             temp_path = temp.name
@@ -157,7 +165,9 @@ async def test_run_modern_protocol(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name

tests/connections/test_pybricks.py

@@ -71,7 +71,9 @@ async def test_download_ble(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -80,7 +82,7 @@ async def test_download_ble(self):
             # Create args
             args = argparse.Namespace(
                 conntype="ble",
-                file=open(temp_path, "r"),
+                file=open(temp_path, "r", encoding="utf-8"),
                 name="MyHub",
                 start=False,
                 wait=False,
@@ -118,7 +120,9 @@ async def test_download_usb(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -127,7 +131,7 @@ async def test_download_usb(self):
             # Create args
             args = argparse.Namespace(
                 conntype="usb",
-                file=open(temp_path, "r"),
+                file=open(temp_path, "r", encoding="utf-8"),
                 name=None,
                 start=False,
                 wait=False,
@@ -208,7 +212,9 @@ async def test_download_connection_error(self):
         with contextlib.ExitStack() as stack:
             # Create and manage temporary file
             temp = stack.enter_context(
-                tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False)
+                tempfile.NamedTemporaryFile(
+                    suffix=".py", mode="w+", delete=False, encoding="utf-8"
+                )
             )
             temp.write("print('test')")
             temp_path = temp.name
@@ -217,7 +223,7 @@ async def test_download_connection_error(self):
             # Create args
             args = argparse.Namespace(
                 conntype="ble",
-                file=open(temp_path, "r"),
+                file=open(temp_path, "r", encoding="utf-8"),
                 name="MyHub",
                 start=False,
                 wait=False,