Merge pull request connamara#946 from gbirchmeier/rebase-VAllens-getbytes

[Rebase connamara#940] Refactor the GetBytes method of the CharEncoding class to use ArrayPool<byte> to improve memory allocation performance.

Author: gbirchmeier

QuickFIXn/CharEncoding.cs

@@ -1,30 +1,64 @@
-namespace QuickFix;
+using System;
+using System.Buffers;
+
+namespace QuickFix;
 
 /// <summary>
 /// Static class that manages the selected character encoding.
 /// This impl is kind of a hack -- we're using a static class
 /// so we don't have to alter a bunch of other classes to pass
 /// a CharEncoding reference around.
 /// </summary>
-public static class CharEncoding {
+public static class CharEncoding
+{
     public const string DefaultCharEncoding = "iso-8859-1";
 
     public static System.Text.Encoding SelectedEncoding { get; private set; }
 
-    static CharEncoding() {
+    static CharEncoding()
+    {
         System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
         SelectedEncoding = System.Text.Encoding.GetEncoding(DefaultCharEncoding);
     }
 
-    public static void ResetToDefaultEncoding() {
-        SetEncoding(DefaultCharEncoding);
-    }
+    public static void ResetToDefaultEncoding() => SetEncoding(DefaultCharEncoding);
+
+    public static void SetEncoding(string encoding) => SelectedEncoding = System.Text.Encoding.GetEncoding(encoding);
+
+    /// <summary>
+    /// Convert a string to a byte array using the current globally-set encoding.
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    public static byte[] GetBytes(string data) => SelectedEncoding.GetBytes(data);
 
-    public static void SetEncoding(string encoding) {
-        SelectedEncoding = System.Text.Encoding.GetEncoding(encoding);
+    /// <summary>
+    /// Convert a string to a byte array using the current globally-set encoding.
+    /// This function uses a shared byte-pool for efficiency.
+    /// The return value needs to be disposed; `using` is recommended (see existing usages).
+    /// This function is still internal; we're not sure if we want to make it part of the public interface.
+    /// </summary>
+    /// <param name="data"></param>
+    /// <param name="bytes"></param>
+    /// <returns></returns>
+    internal static ValueDisposable GetBytes(ReadOnlySpan<char> data, out ReadOnlySpan<byte> bytes)
+    {
+        System.Text.Encoding encoding = SelectedEncoding;
+        int byteCount = encoding.GetByteCount(data);
+        byte[] buffer = ArrayPool<byte>.Shared.Rent(byteCount);
+
+        encoding.GetBytes(data, new Span<byte>(buffer, 0, byteCount));
+        bytes = new ReadOnlySpan<byte>(buffer, 0, byteCount);
+
+        return new ValueDisposable(buffer);
     }
+}
 
-    public static byte[] GetBytes(string data) {
-        return SelectedEncoding.GetBytes(data);
+internal readonly ref struct ValueDisposable(byte[]? bytes)
+{
+    public void Dispose()
+    {
+        if (bytes == null) return;
+        if (bytes.Length > 0) ArrayPool<byte>.Shared.Return(bytes);
     }
 }

QuickFIXn/Fields/FieldBase.cs

@@ -99,11 +99,13 @@ public int GetTotal()
             MakeStringFields();
 
         int sum = 0;
-        byte[] array = CharEncoding.GetBytes(_stringField);
+
+        using ValueDisposable _ = CharEncoding.GetBytes(_stringField, out ReadOnlySpan<byte> array);
         foreach (byte b in array)
         {
             sum += b;
         }
+
         return sum + 1; // +1 for SOH
     }
 

QuickFIXn/Parser.cs

@@ -30,10 +30,10 @@ public Parser(Encoding encoding)
         _checkSumBytes = encoding.GetBytes('\u0001' + "10=");
     }
 
-    public void AddToStream(byte[] data, int bytesAdded)
-        => AddToStream(data.AsSpan(0, bytesAdded));
+    [Obsolete("Use AddToStream(ReadOnlySpan<byte>) instead.  This will be removed in v1.15")]
+    public void AddToStream(byte[] data, int bytesAdded) => AddToStream(new ReadOnlySpan<byte>(data, 0, bytesAdded));
 
-    public void AddToStream(Span<byte> data)
+    public void AddToStream(ReadOnlySpan<byte> data)
     {
         // We attempt to copy the new bytes into the existing buffer.
         if (data.TryCopyTo(_buffer.AsSpan(_bufferStartIndex + _usedBufferLength)))

QuickFIXn/SocketInitiatorThread.cs

@@ -90,7 +90,7 @@ public bool Read()
         {
             int bytesRead = ReadSome(_readBuffer, 1000);
             if (bytesRead > 0)
-                _parser.AddToStream(_readBuffer, bytesRead);
+                _parser.AddToStream(new ReadOnlySpan<byte>(_readBuffer, 0, bytesRead));
             else
                 Session.Next();
 
@@ -189,8 +189,9 @@ public bool Send(string data)
             throw new ApplicationException("Initiator is not connected (uninitialized stream)");
         }
 
-        byte[] rawData = CharEncoding.GetBytes(data);
-        _stream.Write(rawData, 0, rawData.Length);
+        using ValueDisposable _ = CharEncoding.GetBytes(data, out ReadOnlySpan<byte> rawData);
+        _stream.Write(rawData);
+
         return true;
     }
 

QuickFIXn/SocketReader.cs

@@ -46,7 +46,7 @@ public void Read()
         {
             int bytesRead = ReadSome(_readBuffer, 1000);
             if (bytesRead > 0)
-                _parser.AddToStream(_readBuffer, bytesRead);
+                _parser.AddToStream(new ReadOnlySpan<byte>(_readBuffer, 0, bytesRead));
             else
                 _qfSession?.Next();
 
@@ -271,8 +271,9 @@ private void LogEvent(string s)
 
     public int Send(string data)
     {
-        byte[] rawData = CharEncoding.GetBytes(data);
-        _stream.Write(rawData, 0, rawData.Length);
+        using ValueDisposable _ = CharEncoding.GetBytes(data, out ReadOnlySpan<byte> rawData);
+        _stream.Write(rawData);
+
         return rawData.Length;
     }
 

QuickFIXn/Store/FileStore.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Text;
 using QuickFix.Util;
@@ -10,16 +11,11 @@ namespace QuickFix.Store;
 /// </summary>
 public class FileStore : IMessageStore
 {
-    private class MsgDef
+    private readonly struct MsgDef(long index, int size)
     {
-        public long Index { get; }
-        public int Size { get; }
+        public long Index { get; } = index;
 
-        public MsgDef(long index, int size)
-        {
-            Index = index;
-            Size = size;
-        }
+        public int Size { get; } = size;
     }
 
     private readonly string _seqNumsFileName;
@@ -198,13 +194,19 @@ public void Get(SeqNumType startSeqNum, SeqNumType endSeqNum, List<string> messa
     {
         for (SeqNumType i = startSeqNum; i <= endSeqNum; i++)
         {
-            if (_offsets.ContainsKey(i))
+            if (_offsets.TryGetValue(i, out MsgDef msgDef))
             {
-                _msgFile.Seek(_offsets[i].Index, System.IO.SeekOrigin.Begin);
-                byte[] msgBytes = new byte[_offsets[i].Size];
-                _msgFile.Read(msgBytes, 0, msgBytes.Length);
-
-                messages.Add(CharEncoding.SelectedEncoding.GetString(msgBytes));
+                _msgFile.Seek(msgDef.Index, System.IO.SeekOrigin.Begin);
+                byte[] msgBytes = ArrayPool<byte>.Shared.Rent(msgDef.Size);
+                try
+                {
+                    _msgFile.ReadExactly(new Span<byte>(msgBytes, 0, msgDef.Size));
+                    messages.Add(CharEncoding.SelectedEncoding.GetString(new ReadOnlySpan<byte>(msgBytes, 0, msgDef.Size)));
+                }
+                finally
+                {
+                    ArrayPool<byte>.Shared.Return(msgBytes);
+                }
             }
         }
 
@@ -221,20 +223,19 @@ public bool Set(SeqNumType msgSeqNum, string msg)
         _msgFile.Seek(0, System.IO.SeekOrigin.End);
 
         long offset = _msgFile.Position;
-        byte[] msgBytes = CharEncoding.GetBytes(msg);
-        int size = msgBytes.Length;
+
+        using ValueDisposable _ = CharEncoding.GetBytes(msg.AsSpan(), out ReadOnlySpan<byte> msgBytes);
 
         StringBuilder b = new StringBuilder();
-        b.Append(msgSeqNum).Append(',').Append(offset).Append(',').Append(size);
+        b.Append(msgSeqNum).Append(',').Append(offset).Append(',').Append(msgBytes.Length);
         _headerFile.WriteLine(b.ToString());
         _headerFile.Flush();
 
-        _offsets[msgSeqNum] = new MsgDef(offset, size);
+        _offsets[msgSeqNum] = new MsgDef(offset, msgBytes.Length);
 
-        _msgFile.Write(msgBytes, 0, size);
+        _msgFile.Write(msgBytes);
         _msgFile.Flush();
 
-
         return true;
     }
 
@@ -299,7 +300,7 @@ public void Dispose()
         Dispose(true);
         GC.SuppressFinalize(this);
     }
-    private bool _disposed = false;
+    private bool _disposed;
     protected virtual void Dispose(bool disposing)
     {
         if (_disposed) return;

RELEASE_NOTES.md

@@ -9,18 +9,38 @@ What's New
 ----------
 
 **CAUTION:**  
-* **1.13.0 has moved to .NET 8 (as Microsoft is ending .NET 6 support on Nov 12, 2024)
+* **Starting with 1.14, the QuickFIX message **nuget** packages have been renamed!**
+    **Please remove the old package and import the new package!**
+    (See issue #627 for more information.)
+
+    The new names are as follows (note the deleted period):
+    * ~~QuickFIX.FIX4.0.{ver}~~ becomes **QuickFIX.FIX40.{ver}**
+    * ~~QuickFIX.FIX4.1.{ver}~~ becomes **QuickFIX.FIX41.{ver}**
+    * ~~QuickFIX.FIX4.2.{ver}~~ becomes **QuickFIX.FIX42.{ver}**
+    * ~~QuickFIX.FIX4.3.{ver}~~ becomes **QuickFIX.FIX43.{ver}**
+    * ~~QuickFIX.FIX4.4.{ver}~~ becomes **QuickFIX.FIX44.{ver}**
+    * ~~QuickFIX.FIX5.0.{ver}~~ becomes **QuickFIX.FIX50.{ver}**
+    * ~~QuickFIX.FIX5.0SP1.{ver}~~ becomes **QuickFIX.FIX50SP1.{ver}**
+    * ~~QuickFIX.FIX5.0SP2.{ver}~~ becomes **QuickFIX.FIX50SP2.{ver}**
+    * ~~QuickFIX.FIXT1.1.{ver}~~ becomes **QuickFIX.FIXT11.{ver}**
+  
+* **1.13.0 has moved to .NET 8 (as Microsoft is ending .NET 6 support on Nov 12, 2024)**
 * **There are breaking changes between 1.12 and 1.13!  Please review the 1.13.0 notes below.**
 * **There are breaking changes between 1.11 and 1.12!  Please review the 1.12.0 notes below.**
 * **There are breaking changes between 1.10 and 1.11!  Please review the 1.11.0 notes below.**
 
 ### next release
 
+**Breaking changes**
+* #627 - rename message packages to get rid of superfluous period (gbirchmeier)
+    * e.g. QuickFIX.FIX4.4 is now QuickFIX.FIX44, etc.
+
 **Non-breaking changes**
 * #939 - minor checkTooHigh/checkTooLow refactor in Session.cs (gbirchmeier)
 * #941 - clarify ResendRequest-related log message, add UT coverage for Session (gbirchmeier)
 * #895 - fix: When SSLCACertificate is empty an error is logged and it fails to start (dckorben)
 * #942 - fix #942: field 369 (LastMsgSeqNumProcessed) wrong in ResendRequest message (gbirchmeier)
+* #940 - Create an alternate CharEncoding.GetBytes impl which uses ArrayPool to improve memory performance (VAllens)
 
 ### v1.13.0
 

UnitTests/CharEncodingTests.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Text;
+using NUnit.Framework;
+using QuickFix;
+
+namespace UnitTests;
+
+[TestFixture]
+public class CharEncodingTests
+{
+    [TearDown]
+    public void TearDown()
+    {
+        CharEncoding.ResetToDefaultEncoding();
+    }
+
+    [Test]
+    public void GetBytes_Simple()
+    {
+        CharEncoding.SetEncoding("iso-8859-1");
+        byte[] isoBytes = CharEncoding.GetBytes("ïèâî");
+        Assert.That(isoBytes, Is.EqualTo(new byte[] {0xEF, 0xE8, 0xE2, 0xEE}));
+
+        CharEncoding.SetEncoding("windows-1251");
+        byte[] altBytes = CharEncoding.GetBytes("пиво");
+        Assert.That(altBytes, Is.EqualTo(new byte[] {0xEF, 0xE8, 0xE2, 0xEE}));
+    }
+
+    [Test]
+    public void GetBytes_Pooled()
+    {
+        CharEncoding.SetEncoding("iso-8859-1");
+        using ValueDisposable _ = CharEncoding.GetBytes("ïèâî", out ReadOnlySpan<byte> isoByteSpan);
+        Assert.That(isoByteSpan.ToArray(), Is.EqualTo(new byte[] {0xEF, 0xE8, 0xE2, 0xEE}));
+
+        CharEncoding.SetEncoding("windows-1251");
+        using ValueDisposable _2 = CharEncoding.GetBytes("пиво", out ReadOnlySpan<byte> altByteSpan);
+        Assert.That(altByteSpan.ToArray(), Is.EqualTo(new byte[] {0xEF, 0xE8, 0xE2, 0xEE}));
+    }
+}