fix: Enforce UTF-8 encoding for stdio transport

This change replaces the default system encoding with an explicit UTF8Encoding (without BOM)
for both client and server transports. This ensures proper handling of Unicode characters,
including Chinese characters and emoji.

- Use UTF8Encoding explicitly for StreamReader and StreamWriter.
- Add tests for Chinese characters ("上下文伺服器") and emoji (🔍🚀👍) to confirm the fix.

Fixes #35.

Author: willibrandon

src/ModelContextProtocol/Logging/LoggerExtensions.cs

@@ -0,0 +1,35 @@
+using Microsoft.Extensions.Logging;
+
+namespace ModelContextProtocol.Logging;
+
+/// <summary>
+/// Extensions methods for ILogger instances used in MCP protocol.
+/// </summary>
+public static partial class LoggerExtensions
+{
+    /// <summary>
+    /// Logs the byte representation of a message in UTF-8 encoding.
+    /// </summary>
+    /// <param name="logger">The logger to use.</param>
+    /// <param name="endpointName">The name of the endpoint.</param>
+    /// <param name="byteRepresentation">The byte representation as a hex string.</param>
+    [LoggerMessage(EventId = 39000, Level = LogLevel.Trace, Message = "Transport {EndpointName}: Message bytes (UTF-8): {ByteRepresentation}")]
+    public static partial void TransportMessageBytes(this ILogger logger, string endpointName, string byteRepresentation);
+
+    /// <summary>
+    /// Logs the byte representation of a message for diagnostic purposes.
+    /// This is useful for diagnosing encoding issues with non-ASCII characters.
+    /// </summary>
+    /// <param name="logger">The logger to use.</param>
+    /// <param name="endpointName">The name of the endpoint.</param>
+    /// <param name="message">The message to log bytes for.</param>
+    public static void TransportMessageBytesUtf8(this ILogger logger, string endpointName, string message)
+    {
+        if (logger.IsEnabled(LogLevel.Trace))
+        {
+            var bytes = System.Text.Encoding.UTF8.GetBytes(message);
+            var byteRepresentation = string.Join(" ", bytes.Select(b => $"{b:X2}"));
+            logger.TransportMessageBytes(endpointName, byteRepresentation);
+        }
+    }
+}

src/ModelContextProtocol/Protocol/Transport/StdioClientTransport.cs

@@ -1,12 +1,13 @@
-using System.Diagnostics;
-using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using ModelContextProtocol.Configuration;
 using ModelContextProtocol.Logging;
 using ModelContextProtocol.Protocol.Messages;
 using ModelContextProtocol.Utils;
 using ModelContextProtocol.Utils.Json;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
+using System.Diagnostics;
+using System.Text;
+using System.Text.Json;
 
 namespace ModelContextProtocol.Protocol.Transport;
 
@@ -20,6 +21,8 @@ public sealed class StdioClientTransport : TransportBase, IClientTransport
     private readonly ILogger _logger;
     private readonly JsonSerializerOptions _jsonOptions;
     private Process? _process;
+    private StreamWriter? _stdInWriter;
+    private StreamReader? _stdOutReader;
     private Task? _readTask;
     private CancellationTokenSource? _shutdownCts;
     private bool _processStarted;
@@ -99,6 +102,13 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default)
             }
             _logger.TransportProcessStarted(EndpointName, _process.Id);
             _processStarted = true;
+            
+            // Create streams with explicit UTF-8 encoding to ensure proper Unicode character handling
+            // This is especially important for non-ASCII characters like Chinese text and emoji
+            var utf8Encoding = new UTF8Encoding(false); // No BOM
+            _stdInWriter = new StreamWriter(_process.StandardInput.BaseStream, utf8Encoding) { AutoFlush = true };
+            _stdOutReader = new StreamReader(_process.StandardOutput.BaseStream, utf8Encoding);
+            
             _process.BeginErrorReadLine();
 
             // Start reading messages in the background
@@ -118,7 +128,7 @@ public async Task ConnectAsync(CancellationToken cancellationToken = default)
     /// <inheritdoc/>
     public override async Task SendMessageAsync(IJsonRpcMessage message, CancellationToken cancellationToken = default)
     {
-        if (!IsConnected || _process?.HasExited == true)
+        if (!IsConnected || _process?.HasExited == true || _stdInWriter == null)
         {
             _logger.TransportNotConnected(EndpointName);
             throw new McpTransportException("Transport is not connected");
@@ -134,10 +144,11 @@ public override async Task SendMessageAsync(IJsonRpcMessage message, Cancellatio
         {
             var json = JsonSerializer.Serialize(message, _jsonOptions.GetTypeInfo<IJsonRpcMessage>());
             _logger.TransportSendingMessage(EndpointName, id, json);
+            _logger.TransportMessageBytesUtf8(EndpointName, json);
 
-            // Write the message followed by a newline
-            await _process!.StandardInput.WriteLineAsync(json.AsMemory(), cancellationToken).ConfigureAwait(false);
-            await _process.StandardInput.FlushAsync(cancellationToken).ConfigureAwait(false);
+            // Write the message followed by a newline using our UTF-8 writer
+            await _stdInWriter.WriteLineAsync(json).ConfigureAwait(false);
+            await _stdInWriter.FlushAsync(cancellationToken).ConfigureAwait(false);
 
             _logger.TransportSentMessage(EndpointName, id);
         }
@@ -161,12 +172,10 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
         {
             _logger.TransportEnteringReadMessagesLoop(EndpointName);
 
-            using var reader = _process!.StandardOutput;
-
-            while (!cancellationToken.IsCancellationRequested && !_process.HasExited)
+            while (!cancellationToken.IsCancellationRequested && !_process!.HasExited && _stdOutReader != null)
             {
                 _logger.TransportWaitingForMessage(EndpointName);
-                var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
+                var line = await _stdOutReader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
                 if (line == null)
                 {
                     _logger.TransportEndOfStream(EndpointName);
@@ -179,6 +188,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
                 }
 
                 _logger.TransportReceivedMessage(EndpointName, line);
+                _logger.TransportMessageBytesUtf8(EndpointName, line);
 
                 await ProcessMessageAsync(line, cancellationToken).ConfigureAwait(false);
             }
@@ -230,14 +240,28 @@ private async Task ProcessMessageAsync(string line, CancellationToken cancellati
     private async Task CleanupAsync(CancellationToken cancellationToken)
     {
         _logger.TransportCleaningUp(EndpointName);
-        if (_process != null && _processStarted && !_process.HasExited)
+        
+        if (_stdInWriter != null)
         {
             try
             {
-                // Try to close stdin to signal the process to exit
                 _logger.TransportClosingStdin(EndpointName);
-                _process.StandardInput.Close();
+                _stdInWriter.Close();
+            }
+            catch (Exception ex)
+            {
+                _logger.TransportShutdownFailed(EndpointName, ex);
+            }
 
+            _stdInWriter = null;
+        }
+        
+        _stdOutReader = null;
+        
+        if (_process != null && _processStarted && !_process.HasExited)
+        {
+            try
+            {
                 // Wait for the process to exit
                 _logger.TransportWaitingForShutdown(EndpointName);
 

src/ModelContextProtocol/Protocol/Transport/StdioServerTransport.cs

@@ -1,12 +1,13 @@
-using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
 using ModelContextProtocol.Logging;
 using ModelContextProtocol.Protocol.Messages;
 using ModelContextProtocol.Server;
 using ModelContextProtocol.Utils;
 using ModelContextProtocol.Utils.Json;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.Extensions.Options;
+using System.Text;
+using System.Text.Json;
 
 namespace ModelContextProtocol.Protocol.Transport;
 
@@ -19,8 +20,8 @@ public sealed class StdioServerTransport : TransportBase, IServerTransport
     private readonly ILogger _logger;
 
     private readonly JsonSerializerOptions _jsonOptions = McpJsonUtilities.DefaultOptions;
-    private readonly TextReader _stdin = Console.In;
-    private readonly TextWriter _stdout = Console.Out;
+    private readonly TextReader _stdInReader;
+    private readonly TextWriter _stdOutWriter;
 
     private Task? _readTask;
     private CancellationTokenSource? _shutdownCts;
@@ -83,16 +84,57 @@ public StdioServerTransport(string serverName, ILoggerFactory? loggerFactory = n
 
         _serverName = serverName;
         _logger = (ILogger?)loggerFactory?.CreateLogger<StdioClientTransport>() ?? NullLogger.Instance;
+        
+        // Create console streams with explicit UTF-8 encoding to ensure proper Unicode character handling
+        // This is especially important for non-ASCII characters like Chinese text and emoji
+        var utf8Encoding = new UTF8Encoding(false); // No BOM
+        
+        // Get raw console streams and wrap them with UTF-8 encoding
+        Stream inputStream = Console.OpenStandardInput();
+        Stream outputStream = Console.OpenStandardOutput();
+        
+        _stdInReader = new StreamReader(inputStream, utf8Encoding);
+        _stdOutWriter = new StreamWriter(outputStream, utf8Encoding) { AutoFlush = true };
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="StdioServerTransport"/> class with explicit input/output streams.
+    /// </summary>
+    /// <param name="serverName">The name of the server.</param>
+    /// <param name="input">The input TextReader to use.</param>
+    /// <param name="output">The output TextWriter to use.</param>
+    /// <param name="loggerFactory">Optional logger factory used for logging employed by the transport.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="serverName"/> is <see langword="null"/>.</exception>
+    /// <remarks>
+    /// <para>
+    /// This constructor is useful for testing scenarios where you want to redirect input/output.
+    /// </para>
+    /// </remarks>
+    public StdioServerTransport(string serverName, TextReader input, TextWriter output, ILoggerFactory? loggerFactory = null)
+        : base(loggerFactory)
+    {
+        Throw.IfNull(serverName);
+        Throw.IfNull(input);
+        Throw.IfNull(output);
+        
+        _serverName = serverName;
+        _logger = (ILogger?)loggerFactory?.CreateLogger<StdioClientTransport>() ?? NullLogger.Instance;
+        
+        _stdInReader = input;
+        _stdOutWriter = output;
     }
 
     /// <inheritdoc/>
     public Task StartListeningAsync(CancellationToken cancellationToken = default)
     {
+        _logger.LogDebug("Starting StdioServerTransport listener for {EndpointName}", EndpointName);
+        
         _shutdownCts = new CancellationTokenSource();
 
         _readTask = Task.Run(async () => await ReadMessagesAsync(_shutdownCts.Token).ConfigureAwait(false), CancellationToken.None);
 
         SetConnected(true);
+        _logger.LogDebug("StdioServerTransport now connected for {EndpointName}", EndpointName);
 
         return Task.CompletedTask;
     }
@@ -116,9 +158,10 @@ public override async Task SendMessageAsync(IJsonRpcMessage message, Cancellatio
         {
             var json = JsonSerializer.Serialize(message, _jsonOptions.GetTypeInfo<IJsonRpcMessage>());
             _logger.TransportSendingMessage(EndpointName, id, json);
+            _logger.TransportMessageBytesUtf8(EndpointName, json);
 
-            await _stdout.WriteLineAsync(json.AsMemory(), cancellationToken).ConfigureAwait(false);
-            await _stdout.FlushAsync(cancellationToken).ConfigureAwait(false);
+            await _stdOutWriter.WriteLineAsync(json).ConfigureAwait(false);
+            await _stdOutWriter.FlushAsync(cancellationToken).ConfigureAwait(false);
 
             _logger.TransportSentMessage(EndpointName, id);
         }
@@ -146,7 +189,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
             {
                 _logger.TransportWaitingForMessage(EndpointName);
 
-                var reader = _stdin;
+                var reader = _stdInReader;
                 var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
                 if (line == null)
                 {
@@ -160,6 +203,7 @@ private async Task ReadMessagesAsync(CancellationToken cancellationToken)
                 }
 
                 _logger.TransportReceivedMessage(EndpointName, line);
+                _logger.TransportMessageBytesUtf8(EndpointName, line);
 
                 try
                 {