Fix stdio encoding issue: Enforce explicit UTF-8 for correct Unicode handling (modelcontextprotocol#73)

* fix: Enforce UTF-8 encoding for stdio transport

This change replaces the default system encoding with an explicit UTF8Encoding (without BOM)
for both client and server transports. This ensures proper handling of Unicode characters,
including Chinese characters and emoji.

- Use UTF8Encoding explicitly for StreamReader and StreamWriter.
- Add tests for Chinese characters ("上下文伺服器") and emoji (🔍🚀👍) to confirm the fix.

Fixes modelcontextprotocol#35.

* Remove relaxed JSON encoder

* Address PR feedback

---------

Co-authored-by: Stephen Toub <[email protected]>

Author: willibrandon

src/ModelContextProtocol/Logging/Log.cs

@@ -114,7 +114,7 @@ internal static partial class Log
     internal static partial void TransportNotConnected(this ILogger logger, string endpointName);
 
     [LoggerMessage(Level = LogLevel.Information, Message = "Transport sending message for {endpointName} with ID {messageId}, JSON {json}")]
-    internal static partial void TransportSendingMessage(this ILogger logger, string endpointName, string messageId, string json);
+    internal static partial void TransportSendingMessage(this ILogger logger, string endpointName, string messageId, string? json = null);
 
     [LoggerMessage(Level = LogLevel.Information, Message = "Transport message sent for {endpointName} with ID {messageId}")]
     internal static partial void TransportSentMessage(this ILogger logger, string endpointName, string messageId);
@@ -347,4 +347,35 @@ public static partial void SSETransportPostNotAccepted(
         string endpointName,
         string messageId,
         string responseContent);
+
+    /// <summary>
+    /// Logs the byte representation of a message in UTF-8 encoding.
+    /// </summary>
+    /// <param name="logger">The logger to use.</param>
+    /// <param name="endpointName">The name of the endpoint.</param>
+    /// <param name="byteRepresentation">The byte representation as a hex string.</param>
+    [LoggerMessage(EventId = 39000, Level = LogLevel.Trace, Message = "Transport {EndpointName}: Message bytes (UTF-8): {ByteRepresentation}")]
+    private static partial void TransportMessageBytes(this ILogger logger, string endpointName, string byteRepresentation);
+
+    /// <summary>
+    /// Logs the byte representation of a message for diagnostic purposes.
+    /// This is useful for diagnosing encoding issues with non-ASCII characters.
+    /// </summary>
+    /// <param name="logger">The logger to use.</param>
+    /// <param name="endpointName">The name of the endpoint.</param>
+    /// <param name="message">The message to log bytes for.</param>
+    internal static void TransportMessageBytesUtf8(this ILogger logger, string endpointName, string message)
+    {
+        if (logger.IsEnabled(LogLevel.Trace))
+        {
+            var bytes = System.Text.Encoding.UTF8.GetBytes(message);
+            var byteRepresentation =
+#if NET
+                Convert.ToHexString(bytes);
+#else
+                BitConverter.ToString(bytes).Replace("-", " ");
+#endif
+            logger.TransportMessageBytes(endpointName, byteRepresentation);
+        }
+    }
 }

src/ModelContextProtocol/Protocol/Transport/StdioClientTransport.cs

@@ -1,12 +1,13 @@
-using System.Diagnostics;
-using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using ModelContextProtocol.Configuration;
 using ModelContextProtocol.Logging;
 using ModelContextProtocol.Protocol.Messages;
 using ModelContextProtocol.Utils;
 using ModelContextProtocol.Utils.Json;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
+using System.Diagnostics;
+using System.Text;
+using System.Text.Json;
 
 namespace ModelContextProtocol.Protocol.Transport;
 
@@ -59,6 +60,8 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default)
 
             _shutdownCts = new CancellationTokenSource();
 
+            UTF8Encoding noBomUTF8 = new(encoderShouldEmitUTF8Identifier: false);
+
             var startInfo = new ProcessStartInfo
             {
                 FileName = _options.Command,
@@ -68,6 +71,11 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default)
                 UseShellExecute = false,
                 CreateNoWindow = true,
                 WorkingDirectory = _options.WorkingDirectory ?? Environment.CurrentDirectory,
+                StandardOutputEncoding = noBomUTF8,
+                StandardErrorEncoding = noBomUTF8,
+#if NET
+                StandardInputEncoding = noBomUTF8,
+#endif
             };
 
             if (!string.IsNullOrWhiteSpace(_options.Arguments))
@@ -92,13 +100,35 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default)
             // Set up error logging
             _process.ErrorDataReceived += (sender, args) => _logger.TransportError(EndpointName, args.Data ?? "(no data)");
 
-            if (!_process.Start())
+            // We need both stdin and stdout to use a no-BOM UTF-8 encoding. On .NET Core,
+            // we can use ProcessStartInfo.StandardOutputEncoding/StandardInputEncoding, but
+            // StandardInputEncoding doesn't exist on .NET Framework; instead, it always picks
+            // up the encoding from Console.InputEncoding. As such, when not targeting .NET Core,
+            // we temporarily change Console.InputEncoding to no-BOM UTF-8 around the Process.Start
+            // call, to ensure it picks up the correct encoding.
+#if NET
+            _processStarted = _process.Start();
+#else
+            Encoding originalInputEncoding = Console.InputEncoding;
+            try
+            {
+                Console.InputEncoding = noBomUTF8;
+                _processStarted = _process.Start();
+            }
+            finally
+            {
+                Console.InputEncoding = originalInputEncoding;
+            }
+#endif
+
+            if (!_processStarted)
             {
                 _logger.TransportProcessStartFailed(EndpointName);
                 throw new McpTransportException("Failed to start MCP server process");
             }
+
             _logger.TransportProcessStarted(EndpointName, _process.Id);
-            _processStarted = true;
+            
             _process.BeginErrorReadLine();
 
             // Start reading messages in the background
@@ -134,9 +164,10 @@ public override async Task SendMessageAsync(IJsonRpcMessage message, Cancellatio
         {
             var json = JsonSerializer.Serialize(message, _jsonOptions.GetTypeInfo<IJsonRpcMessage>());
             _logger.TransportSendingMessage(EndpointName, id, json);
+            _logger.TransportMessageBytesUtf8(EndpointName, json);
 
-            // Write the message followed by a newline
-            await _process!.StandardInput.WriteLineAsync(json.AsMemory(), cancellationToken).ConfigureAwait(false);
+            // Write the message followed by a newline using our UTF-8 writer
+            await _process!.StandardInput.WriteLineAsync(json).ConfigureAwait(false);
             await _process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false);
 
             _logger.TransportSentMessage(EndpointName, id);
@@ -161,12 +192,10 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
         {
             _logger.TransportEnteringReadMessagesLoop(EndpointName);
 
-            using var reader = _process!.StandardOutput;
-
-            while (!cancellationToken.IsCancellationRequested && !_process.HasExited)
+            while (!cancellationToken.IsCancellationRequested && !_process!.HasExited)
             {
                 _logger.TransportWaitingForMessage(EndpointName);
-                var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
+                var line = await _process.StandardOutput.ReadLineAsync(cancellationToken).ConfigureAwait(false);
                 if (line == null)
                 {
                     _logger.TransportEndOfStream(EndpointName);
@@ -179,6 +208,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
                 }
 
                 _logger.TransportReceivedMessage(EndpointName, line);
+                _logger.TransportMessageBytesUtf8(EndpointName, line);
 
                 await ProcessMessageAsync(line, cancellationToken).ConfigureAwait(false);
             }
@@ -230,28 +260,27 @@ private async Task ProcessMessageAsync(string line, CancellationToken cancellati
     private async Task CleanupAsync(CancellationToken cancellationToken)
     {
         _logger.TransportCleaningUp(EndpointName);
-        if (_process != null && _processStarted && !_process.HasExited)
+
+        if (_process is Process process && _processStarted && !process.HasExited)
         {
             try
             {
-                // Try to close stdin to signal the process to exit
-                _logger.TransportClosingStdin(EndpointName);
-                _process.StandardInput.Close();
-
                 // Wait for the process to exit
                 _logger.TransportWaitingForShutdown(EndpointName);
 
                 // Kill the while process tree because the process may spawn child processes
                 // and Node.js does not kill its children when it exits properly
-                _process.KillTree(_options.ShutdownTimeout);
+                process.KillTree(_options.ShutdownTimeout);
             }
             catch (Exception ex)
             {
                 _logger.TransportShutdownFailed(EndpointName, ex);
             }
-
-            _process.Dispose();
-            _process = null;
+            finally
+            {
+                process.Dispose();
+                _process = null;
+            }
         }
 
         if (_shutdownCts is { } shutdownCts)
@@ -261,29 +290,30 @@ private async Task CleanupAsync(CancellationToken cancellationToken)
             _shutdownCts = null;
         }
 
-        if (_readTask != null)
+        if (_readTask is Task readTask)
         {
             try
             {
                 _logger.TransportWaitingForReadTask(EndpointName);
-                await _readTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
+                await readTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
             }
             catch (TimeoutException)
             {
                 _logger.TransportCleanupReadTaskTimeout(EndpointName);
-                // Continue with cleanup
             }
             catch (OperationCanceledException)
             {
                 _logger.TransportCleanupReadTaskCancelled(EndpointName);
-                // Ignore cancellation
             }
             catch (Exception ex)
             {
                 _logger.TransportCleanupReadTaskFailed(EndpointName, ex);
             }
-            _readTask = null;
-            _logger.TransportReadTaskCleanedUp(EndpointName);
+            finally
+            {
+                _logger.TransportReadTaskCleanedUp(EndpointName);
+                _readTask = null;
+            }
         }
 
         SetConnected(false);

src/ModelContextProtocol/Protocol/Transport/StdioServerTransport.cs

@@ -1,12 +1,13 @@
-using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
 using ModelContextProtocol.Logging;
 using ModelContextProtocol.Protocol.Messages;
 using ModelContextProtocol.Server;
 using ModelContextProtocol.Utils;
 using ModelContextProtocol.Utils.Json;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.Extensions.Options;
+using System.Text;
+using System.Text.Json;
 
 namespace ModelContextProtocol.Protocol.Transport;
 
@@ -15,12 +16,14 @@ namespace ModelContextProtocol.Protocol.Transport;
 /// </summary>
 public sealed class StdioServerTransport : TransportBase, IServerTransport
 {
+    private static readonly byte[] s_newlineBytes = "\n"u8.ToArray();
+
     private readonly string _serverName;
     private readonly ILogger _logger;
 
     private readonly JsonSerializerOptions _jsonOptions = McpJsonUtilities.DefaultOptions;
-    private readonly TextReader _stdin = Console.In;
-    private readonly TextWriter _stdout = Console.Out;
+    private readonly TextReader _stdInReader;
+    private readonly Stream _stdOutStream;
 
     private Task? _readTask;
     private CancellationTokenSource? _shutdownCts;
@@ -83,16 +86,50 @@ public StdioServerTransport(string serverName, ILoggerFactory? loggerFactory = n
 
         _serverName = serverName;
         _logger = (ILogger?)loggerFactory?.CreateLogger<StdioClientTransport>() ?? NullLogger.Instance;
+        
+        // Get raw console streams and wrap them with UTF-8 encoding
+        _stdInReader = new StreamReader(Console.OpenStandardInput(), Encoding.UTF8);
+        _stdOutStream = new BufferedStream(Console.OpenStandardOutput());
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="StdioServerTransport"/> class with explicit input/output streams.
+    /// </summary>
+    /// <param name="serverName">The name of the server.</param>
+    /// <param name="stdinStream">The input TextReader to use.</param>
+    /// <param name="stdoutStream">The output TextWriter to use.</param>
+    /// <param name="loggerFactory">Optional logger factory used for logging employed by the transport.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="serverName"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// <para>
+    /// This constructor is useful for testing scenarios where you want to redirect input/output.
+    /// </para>
+    /// </remarks>
+    public StdioServerTransport(string serverName, Stream stdinStream, Stream stdoutStream, ILoggerFactory? loggerFactory = null)
+        : base(loggerFactory)
+    {
+        Throw.IfNull(serverName);
+        Throw.IfNull(stdinStream);
+        Throw.IfNull(stdoutStream);
+        
+        _serverName = serverName;
+        _logger = (ILogger?)loggerFactory?.CreateLogger<StdioClientTransport>() ?? NullLogger.Instance;
+        
+        _stdInReader = new StreamReader(stdinStream, Encoding.UTF8);
+        _stdOutStream = stdoutStream;
     }
 
     /// <inheritdoc/>
     public Task StartListeningAsync(CancellationToken cancellationToken = default)
     {
+        _logger.LogDebug("Starting StdioServerTransport listener for {EndpointName}", EndpointName);
+        
         _shutdownCts = new CancellationTokenSource();
 
         _readTask = Task.Run(async () => await ReadMessagesAsync(_shutdownCts.Token).ConfigureAwait(false), CancellationToken.None);
 
         SetConnected(true);
+        _logger.LogDebug("StdioServerTransport now connected for {EndpointName}", EndpointName);
 
         return Task.CompletedTask;
     }
@@ -114,11 +151,11 @@ public override async Task SendMessageAsync(IJsonRpcMessage message, Cancellatio
 
         try
         {
-            var json = JsonSerializer.Serialize(message, _jsonOptions.GetTypeInfo<IJsonRpcMessage>());
-            _logger.TransportSendingMessage(EndpointName, id, json);
+            _logger.TransportSendingMessage(EndpointName, id);
 
-            await _stdout.WriteLineAsync(json.AsMemory(), cancellationToken).ConfigureAwait(false);
-            await _stdout.FlushAsync(cancellationToken).ConfigureAwait(false);
+            await JsonSerializer.SerializeAsync(_stdOutStream, message, _jsonOptions.GetTypeInfo<IJsonRpcMessage>(), cancellationToken).ConfigureAwait(false);
+            await _stdOutStream.WriteAsync(s_newlineBytes, cancellationToken).ConfigureAwait(false);
+            await _stdOutStream.FlushAsync(cancellationToken).ConfigureAwait(false);;
 
             _logger.TransportSentMessage(EndpointName, id);
         }
@@ -146,7 +183,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
             {
                 _logger.TransportWaitingForMessage(EndpointName);
 
-                var reader = _stdin;
+                var reader = _stdInReader;
                 var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
                 if (line == null)
                 {
@@ -160,6 +197,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
                 }
 
                 _logger.TransportReceivedMessage(EndpointName, line);
+                _logger.TransportMessageBytesUtf8(EndpointName, line);
 
                 try
                 {
@@ -207,19 +245,20 @@ private async Task CleanupAsync(CancellationToken cancellationToken)
     {
         _logger.TransportCleaningUp(EndpointName);
 
-        if (_shutdownCts != null)
+        if (_shutdownCts is { } shutdownCts)
         {
-            await _shutdownCts.CancelAsync().ConfigureAwait(false);
-            _shutdownCts.Dispose();
+            await shutdownCts.CancelAsync().ConfigureAwait(false);
+            shutdownCts.Dispose();
+
             _shutdownCts = null;
         }
 
-        if (_readTask != null)
+        if (_readTask is { } readTask)
         {
             try
             {
                 _logger.TransportWaitingForReadTask(EndpointName);
-                await _readTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
+                await readTask.WaitAsync(TimeSpan.FromSeconds(5), cancellationToken).ConfigureAwait(false);
             }
             catch (TimeoutException)
             {
@@ -235,10 +274,16 @@ private async Task CleanupAsync(CancellationToken cancellationToken)
             {
                 _logger.TransportCleanupReadTaskFailed(EndpointName, ex);
             }
-            _readTask = null;
-            _logger.TransportReadTaskCleanedUp(EndpointName);
+            finally
+            {
+                _logger.TransportReadTaskCleanedUp(EndpointName);
+                _readTask = null;
+            }
         }
 
+        _stdInReader?.Dispose();
+        _stdOutStream?.Dispose();
+
         SetConnected(false);
         _logger.TransportCleanedUp(EndpointName);
     }