fix: ensure sys.std* has utf-8 encoding (#771)

This change:
* fixes the encoding issue
* setup the CI to run on windows and macos to ensure it works.
* replace os.linesep by "\n"

Fixes MRGFY-5769

Change-Id: I4b9928f563539fc89cb6f917b21457eea2d581e2

Author: sileht

.github/workflows/ci.yaml

@@ -10,16 +10,34 @@ on:
 jobs:
   test:
     timeout-minutes: 5
-    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        os: [ubuntu-24.04, windows-2025, macos-15]
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/[email protected]
       - uses: actions/[email protected]
         with:
           python-version: ">=3.10"
-      - run: |
+
+      - name: Setup
+        shell: bash
+        run: |
           # nosemgrep: generic.ci.security.use-frozen-lockfile.use-frozen-lockfile-pip
           pip install -r requirements-poetry.txt
-          poetry install --sync
-          poetry run poe linters
-          poetry run poe test
-          poetry build
+          poetry sync
+
+      - name: Linters
+        if: matrix.os == 'ubuntu-24.04'
+        shell: bash
+        run: poetry run poe linters
+
+      - name: Tests
+        shell: bash
+        env:
+          PYTHONUTF8: "${{ matrix.os == 'windows-2025' && 1 || 0 }}"
+        run: poetry run poe test
+
+      - name: Build
+        shell: bash
+        run: poetry build

.mergify.yml

@@ -1,7 +1,9 @@
 extends: .github
 shared:
   check_runs: &CheckRuns
-    - check-success=test
+    - check-success=test (ubuntu-24.04)
+    - check-success=test (windows-2022)
+    - check-success=test (mac-15)
 
 queue_rules:
   - name: dependencies

mergify_cli/ci/cli.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
 import click
@@ -307,13 +306,13 @@ async def _process_junit_files(  # noqa: PLR0913
         )
 
     if quarantine_final_failure_message is None:
-        click.echo(f"{os.linesep}🎉 Verdict")
+        click.echo("\n🎉 Verdict")
         click.echo(
             f"• Status: ✅ OK — all {nb_failing_spans} failures are quarantined (ignored for CI status)",
         )
         quarantine_exit_error_code = 0
     else:
-        click.echo(f"{os.linesep}❌ Verdict")
+        click.echo("\n❌ Verdict")
         click.echo(
             f"• Status: 🔴 FAIL — {quarantine_final_failure_message}",
         )

mergify_cli/ci/quarantine.py

@@ -1,5 +1,4 @@
 import dataclasses
-import os
 import typing
 
 import click
@@ -38,7 +37,7 @@ async def check_and_update_failing_spans(
     Returns the number of failing tests that are not quarantined.
     """
 
-    click.echo(f"{os.linesep}🛡️ Quarantine")
+    click.echo("\n🛡️ Quarantine")
 
     failing_spans = [
         span

mergify_cli/cli.py

@@ -15,6 +15,10 @@
 
 from __future__ import annotations
 
+import os
+import subprocess
+import sys
+
 import click
 import click.decorators
 import click_default_group
@@ -43,5 +47,22 @@ def cli(
 cli.add_command(ci_cli_mod.ci)
 
 
+def enforce_utf8_mode() -> None:
+    if sys.flags.utf8_mode:
+        return
+
+    argv = [sys.executable, "-X", "utf8"]
+    argv.extend(subprocess._args_from_interpreter_flags())  # type: ignore[attr-defined]  # noqa: SLF001
+    argv.extend(sys.argv)
+
+    os.execv(argv[0], argv)  # noqa: S606
+
+
 def main() -> None:
+    # NOTE:
+    #   It's unlikely this day that a platform does not support unicode.
+    #   But on windows, the default encoding may not be an unicode one.
+    #   Let's try our best by forcing utf-8 and if it's impossible, just returns escaped character
+    if os.name == "nt":
+        enforce_utf8_mode()
     cli()

mergify_cli/tests/ci/test_upload.py

@@ -120,6 +120,8 @@ async def test_junit_upload_http_error_console(
     )
     captured = capsys.readouterr()
     assert (
-        '• ❌ Error uploading spans: Failed to export batch code: 422, reason: {"detail":\n"Not enabled on this repository"}'
+        "• ❌ Error uploading spans: Failed to export batch code: 422, reason:"
         in captured.out
     )
+    # Open Telemetry does not render JSON the same way between Windows and Linux
+    assert "Not enabled on this repository" in captured.out

mergify_cli/tests/test_cli.py

@@ -0,0 +1,60 @@
+import os
+import pathlib
+import subprocess
+import sys
+import textwrap
+
+
+def test_reexec_enables_utf8_and_prints_emoji(
+    tmp_path: pathlib.Path,
+) -> None:
+    """
+    Run in a child process so os.execv can safely replace it.
+    We patch mymodule.cli inside the child (no mocks), then call main().
+    We assert:
+      - sys.flags.utf8_mode == 1 at print time
+      - the emoji is present in stdout
+    """
+    script = tmp_path / "runner.py"
+    script.write_text(
+        textwrap.dedent(
+            """
+            import sys
+            import os
+
+            from mergify_cli import cli
+
+            # Make cli() print a probe + the emoji
+            def _cli():
+                print(f"utf8_mode={int(sys.flags.utf8_mode)}")
+                print("✅")
+
+            cli.cli = _cli
+            cli.main()
+            """,
+        ),
+        encoding="utf-8",
+    )
+
+    # Force utf8_mode OFF initially so enforce_utf8_mode triggers on Windows.
+    env = os.environ.copy()
+    env["PYTHONUTF8"] = "0"  # ensure not already in UTF-8 mode
+
+    proc = subprocess.run(
+        [sys.executable, str(script)],
+        check=False,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    assert proc.returncode == 0, proc.stderr
+    stdout = proc.stdout
+    if os.name == "nt":
+        # After re-exec with -X utf8, utf8_mode should be 1 at print time
+        assert "utf8_mode=1" in stdout
+    else:
+        # No reexec on linux, no need the utf8 mode
+        assert "utf8_mode=0" in stdout
+
+    assert "✅" in stdout

poe.toml

@@ -17,6 +17,17 @@ sequence = [
 ]
 
 [tool.poe.tasks.setup]
+env.CI.default = ""
 help = "Sync poetry virtualenv"
-executor = { type  = "simple" }
-cmd = "./tools/poetry-install.sh"
+control.expr = "${CI}.strip().lower()"
+
+    [[tool.poe.tasks.setup.switch]]
+    case = ["true", "1", "y", "yes"]
+    env = { POETRY_VIRTUALENVS_OPTIONS_NO_PIP = "true", POETRY_VIRTUALENVS_OPTIONS_NO_SETUPTOOLS = "true" }
+    executor = { type  = "simple" }
+    cmd = "poetry sync --no-cache"
+
+    [[tool.poe.tasks.setup.switch]]
+    case = ["", "false", "0", "n", "no"]
+    executor = { type  = "simple" }
+    cmd = "poetry sync"