Implement RedisValue.Length for all underlying storage kinds (#2370)

* fix #2368

- implement Length() for other encodings (using format layout)
- unify format code
- switch to C# 11 for u8 strings (needed a few "scoped" modifiers adding)
- tests for format and Length

* tweak langver

* cleanup double format

* use 7.0.101 SDK (102 not yet on ubuntu?)

* tweak SDK in CI.yml; add CI.yml to sln

* We need Redis 6 runtime for tests, so let's grab both

* Add release notes

---------

Co-authored-by: Nick Craver <[email protected]>

Author: mgravell

.github/workflows/CI.yml

@@ -24,6 +24,7 @@ jobs:
       with:
         dotnet-version: | 
           6.0.x
+          7.0.x
     - name: .NET Build
       run: dotnet build Build.csproj -c Release /p:CI=true
     - name: Start Redis Services (docker-compose)
@@ -56,6 +57,7 @@ jobs:
     #   with:
     #     dotnet-version: | 
     #       6.0.x
+    #       7.0.x
     - name: .NET Build
       run: dotnet build Build.csproj -c Release /p:CI=true
     - name: Start Redis Services (v3.0.503)

Directory.Build.props

@@ -15,7 +15,7 @@
     <PackageProjectUrl>https://stackexchange.github.io/StackExchange.Redis/</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
 
-    <LangVersion>10.0</LangVersion>
+    <LangVersion>11</LangVersion>
     <RepositoryType>git</RepositoryType>
     <RepositoryUrl>https://github.com/StackExchange/StackExchange.Redis/</RepositoryUrl>
 

StackExchange.Redis.sln

@@ -9,6 +9,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		build.cmd = build.cmd
 		Build.csproj = Build.csproj
 		build.ps1 = build.ps1
+		.github\workflows\CI.yml = .github\workflows\CI.yml
 		Directory.Build.props = Directory.Build.props
 		Directory.Build.targets = Directory.Build.targets
 		Directory.Packages.props = Directory.Packages.props

appveyor.yml

@@ -6,7 +6,7 @@ init:
 
 install:
 - cmd: >-
-    choco install dotnet-sdk --version 6.0.101
+    choco install dotnet-sdk --version 7.0.102
 
     cd tests\RedisConfigs\3.0.503
 

docs/ReleaseNotes.md

@@ -10,6 +10,7 @@ Current package versions:
 
 - Fix [#2350](https://github.com/StackExchange/StackExchange.Redis/issues/2350): Properly parse lua script paramters in all cultures ([#2351 by NickCraver](https://github.com/StackExchange/StackExchange.Redis/pull/2351))
 - Fix [#2362](https://github.com/StackExchange/StackExchange.Redis/issues/2362): Set `RedisConnectionException.FailureType` to `AuthenticationFailure` on all authentication scenarios for better handling ([#2367 by NickCraver](https://github.com/StackExchange/StackExchange.Redis/pull/2367))
+- Fix [#2368](https://github.com/StackExchange/StackExchange.Redis/issues/2368): Support `RedisValue.Length()` for all storage types ([#2370 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2370))
 
 ## 2.6.90
 

src/StackExchange.Redis/BufferReader.cs

@@ -44,7 +44,7 @@ private bool FetchNextSegment()
             return true;
         }
 
-        public BufferReader(ReadOnlySequence<byte> buffer)
+        public BufferReader(scoped in ReadOnlySequence<byte> buffer)
         {
             _buffer = buffer;
             _lastSnapshotPosition = buffer.Start;

src/StackExchange.Redis/Format.cs

@@ -174,7 +174,7 @@ internal static bool TryParseInt64(ReadOnlySpan<byte> s, out long value) =>
 
         internal static bool CouldBeInteger(string s)
         {
-            if (string.IsNullOrEmpty(s) || s.Length > PhysicalConnection.MaxInt64TextLen) return false;
+            if (string.IsNullOrEmpty(s) || s.Length > Format.MaxInt64TextLen) return false;
             bool isSigned = s[0] == '-';
             for (int i = isSigned ? 1 : 0; i < s.Length; i++)
             {
@@ -185,7 +185,7 @@ internal static bool CouldBeInteger(string s)
         }
         internal static bool CouldBeInteger(ReadOnlySpan<byte> s)
         {
-            if (s.IsEmpty | s.Length > PhysicalConnection.MaxInt64TextLen) return false;
+            if (s.IsEmpty | s.Length > Format.MaxInt64TextLen) return false;
             bool isSigned = s[0] == '-';
             for (int i = isSigned ? 1 : 0; i < s.Length; i++)
             {
@@ -355,5 +355,77 @@ internal static unsafe string GetString(ReadOnlySpan<byte> span)
                 return Encoding.UTF8.GetString(ptr, span.Length);
             }
         }
+
+        [DoesNotReturn]
+        private static void ThrowFormatFailed() => throw new InvalidOperationException("TryFormat failed");
+
+        internal const int
+            MaxInt32TextLen = 11, // -2,147,483,648 (not including the commas)
+            MaxInt64TextLen = 20; // -9,223,372,036,854,775,808 (not including the commas)
+
+        internal static int MeasureDouble(double value)
+        {
+            if (double.IsInfinity(value)) return 4; // +inf / -inf
+            var s = value.ToString("G17", NumberFormatInfo.InvariantInfo); // this looks inefficient, but is how Utf8Formatter works too, just: more direct
+            return s.Length;
+        }
+
+        internal static int FormatDouble(double value, Span<byte> destination)
+        {
+            if (double.IsInfinity(value))
+            {
+                if (double.IsPositiveInfinity(value))
+                {
+                    if (!"+inf"u8.TryCopyTo(destination)) ThrowFormatFailed();
+                }
+                else
+                {
+                    if (!"-inf"u8.TryCopyTo(destination)) ThrowFormatFailed();
+                }
+                return 4;
+            }
+            var s = value.ToString("G17", NumberFormatInfo.InvariantInfo); // this looks inefficient, but is how Utf8Formatter works too, just: more direct
+            if (s.Length > destination.Length) ThrowFormatFailed();
+
+            var chars = s.AsSpan();
+            for (int i = 0; i < chars.Length; i++)
+            {
+                destination[i] = (byte)chars[i];
+            }
+            return chars.Length;
+        }
+
+        internal static int MeasureInt64(long value)
+        {
+            Span<byte> valueSpan = stackalloc byte[MaxInt64TextLen];
+            return FormatInt64(value, valueSpan);
+        }
+
+        internal static int FormatInt64(long value, Span<byte> destination)
+        {
+            if (!Utf8Formatter.TryFormat(value, destination, out var len))
+                ThrowFormatFailed();
+            return len;
+        }
+
+        internal static int MeasureUInt64(ulong value)
+        {
+            Span<byte> valueSpan = stackalloc byte[MaxInt64TextLen];
+            return FormatUInt64(value, valueSpan);
+        }
+
+        internal static int FormatUInt64(ulong value, Span<byte> destination)
+        {
+            if (!Utf8Formatter.TryFormat(value, destination, out var len))
+                ThrowFormatFailed();
+            return len;
+        }
+
+        internal static int FormatInt32(int value, Span<byte> destination)
+        {
+            if (!Utf8Formatter.TryFormat(value, destination, out var len))
+                ThrowFormatFailed();
+            return len;
+        }
     }
 }

src/StackExchange.Redis/PhysicalConnection.cs

@@ -1,6 +1,7 @@
-using System;
+using Pipelines.Sockets.Unofficial;
+using Pipelines.Sockets.Unofficial.Arenas;
+using System;
 using System.Buffers;
-using System.Buffers.Text;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -15,8 +16,6 @@
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using Pipelines.Sockets.Unofficial;
-using Pipelines.Sockets.Unofficial.Arenas;
 
 namespace StackExchange.Redis
 {
@@ -449,7 +448,7 @@ void add(string lk, string sk, string? v)
                             add("Outstanding-Responses", "outstanding", GetSentAwaitingResponseCount().ToString());
                             add("Last-Read", "last-read", (unchecked(now - lastRead) / 1000) + "s ago");
                             add("Last-Write", "last-write", (unchecked(now - lastWrite) / 1000) + "s ago");
-                            if(unansweredWriteTime != 0) add("Unanswered-Write", "unanswered-write", (unchecked(now - unansweredWriteTime) / 1000) + "s ago");
+                            if (unansweredWriteTime != 0) add("Unanswered-Write", "unanswered-write", (unchecked(now - unansweredWriteTime) / 1000) + "s ago");
                             add("Keep-Alive", "keep-alive", bridge.ServerEndPoint?.WriteEverySeconds + "s");
                             add("Previous-Physical-State", "state", oldState.ToString());
                             add("Manager", "mgr", bridge.Multiplexer.SocketManager?.GetState());
@@ -777,8 +776,7 @@ internal static void WriteBulkString(in RedisValue value, PipeWriter output)
 
         internal void WriteHeader(RedisCommand command, int arguments, CommandBytes commandBytes = default)
         {
-            var bridge = BridgeCouldBeNull;
-            if (bridge == null) throw new ObjectDisposedException(ToString());
+            var bridge = BridgeCouldBeNull ?? throw new ObjectDisposedException(ToString());
 
             if (command == RedisCommand.UNKNOWN)
             {
@@ -801,7 +799,7 @@ internal void WriteHeader(RedisCommand command, int arguments, CommandBytes comm
             // *{argCount}\r\n      = 3 + MaxInt32TextLen
             // ${cmd-len}\r\n       = 3 + MaxInt32TextLen
             // {cmd}\r\n            = 2 + commandBytes.Length
-            var span = _ioPipe!.Output.GetSpan(commandBytes.Length + 8 + MaxInt32TextLen + MaxInt32TextLen);
+            var span = _ioPipe!.Output.GetSpan(commandBytes.Length + 8 + Format.MaxInt32TextLen + Format.MaxInt32TextLen);
             span[0] = (byte)'*';
 
             int offset = WriteRaw(span, arguments + 1, offset: 1);
@@ -817,16 +815,12 @@ internal void WriteHeader(RedisCommand command, int arguments, CommandBytes comm
         internal static void WriteMultiBulkHeader(PipeWriter output, long count)
         {
             // *{count}\r\n         = 3 + MaxInt32TextLen
-            var span = output.GetSpan(3 + MaxInt32TextLen);
+            var span = output.GetSpan(3 + Format.MaxInt32TextLen);
             span[0] = (byte)'*';
             int offset = WriteRaw(span, count, offset: 1);
             output.Advance(offset);
         }
 
-        internal const int
-            MaxInt32TextLen = 11, // -2,147,483,648 (not including the commas)
-            MaxInt64TextLen = 20; // -9,223,372,036,854,775,808 (not including the commas)
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int WriteCrlf(Span<byte> span, int offset)
         {
@@ -906,25 +900,16 @@ internal static int WriteRaw(Span<byte> span, long value, bool withLengthPrefix
             {
                 // we're going to write it, but *to the wrong place*
                 var availableChunk = span.Slice(offset);
-                if (!Utf8Formatter.TryFormat(value, availableChunk, out int formattedLength))
-                {
-                    throw new InvalidOperationException("TryFormat failed");
-                }
+                var formattedLength = Format.FormatInt64(value, availableChunk);
                 if (withLengthPrefix)
                 {
                     // now we know how large the prefix is: write the prefix, then write the value
-                    if (!Utf8Formatter.TryFormat(formattedLength, availableChunk, out int prefixLength))
-                    {
-                        throw new InvalidOperationException("TryFormat failed");
-                    }
+                    var prefixLength = Format.FormatInt32(formattedLength, availableChunk);
                     offset += prefixLength;
                     offset = WriteCrlf(span, offset);
 
                     availableChunk = span.Slice(offset);
-                    if (!Utf8Formatter.TryFormat(value, availableChunk, out int finalLength))
-                    {
-                        throw new InvalidOperationException("TryFormat failed");
-                    }
+                    var finalLength = Format.FormatInt64(value, availableChunk);
                     offset += finalLength;
                     Debug.Assert(finalLength == formattedLength);
                 }
@@ -1035,15 +1020,15 @@ private static void WriteUnifiedSpan(PipeWriter writer, ReadOnlySpan<byte> value
             }
             else if (value.Length <= MaxQuickSpanSize)
             {
-                var span = writer.GetSpan(5 + MaxInt32TextLen + value.Length);
+                var span = writer.GetSpan(5 + Format.MaxInt32TextLen + value.Length);
                 span[0] = (byte)'$';
                 int bytes = AppendToSpan(span, value, 1);
                 writer.Advance(bytes);
             }
             else
             {
                 // too big to guarantee can do in a single span
-                var span = writer.GetSpan(3 + MaxInt32TextLen);
+                var span = writer.GetSpan(3 + Format.MaxInt32TextLen);
                 span[0] = (byte)'$';
                 int bytes = WriteRaw(span, value.Length, offset: 1);
                 writer.Advance(bytes);
@@ -1136,7 +1121,7 @@ internal static void WriteUnifiedPrefixedString(PipeWriter writer, byte[]? prefi
                 }
                 else
                 {
-                    var span = writer.GetSpan(3 + MaxInt32TextLen);
+                    var span = writer.GetSpan(3 + Format.MaxInt32TextLen);
                     span[0] = (byte)'$';
                     int bytes = WriteRaw(span, totalLength, offset: 1);
                     writer.Advance(bytes);
@@ -1228,7 +1213,7 @@ private static void WriteUnifiedPrefixedBlob(PipeWriter writer, byte[]? prefix,
             }
             else
             {
-                var span = writer.GetSpan(3 + MaxInt32TextLen); // note even with 2 max-len, we're still in same text range
+                var span = writer.GetSpan(3 + Format.MaxInt32TextLen); // note even with 2 max-len, we're still in same text range
                 span[0] = (byte)'$';
                 int bytes = WriteRaw(span, prefix.LongLength + value.LongLength, offset: 1);
                 writer.Advance(bytes);
@@ -1249,7 +1234,7 @@ private static void WriteUnifiedInt64(PipeWriter writer, long value)
 
             // ${asc-len}\r\n           = 3 + MaxInt32TextLen
             // {asc}\r\n                = MaxInt64TextLen + 2
-            var span = writer.GetSpan(5 + MaxInt32TextLen + MaxInt64TextLen);
+            var span = writer.GetSpan(5 + Format.MaxInt32TextLen + Format.MaxInt64TextLen);
 
             span[0] = (byte)'$';
             var bytes = WriteRaw(span, value, withLengthPrefix: true, offset: 1);
@@ -1263,11 +1248,10 @@ private static void WriteUnifiedUInt64(PipeWriter writer, ulong value)
 
             // ${asc-len}\r\n           = 3 + MaxInt32TextLen
             // {asc}\r\n                = MaxInt64TextLen + 2
-            var span = writer.GetSpan(5 + MaxInt32TextLen + MaxInt64TextLen);
+            var span = writer.GetSpan(5 + Format.MaxInt32TextLen + Format.MaxInt64TextLen);
 
-            Span<byte> valueSpan = stackalloc byte[MaxInt64TextLen];
-            if (!Utf8Formatter.TryFormat(value, valueSpan, out var len))
-                throw new InvalidOperationException("TryFormat failed");
+            Span<byte> valueSpan = stackalloc byte[Format.MaxInt64TextLen];
+            var len = Format.FormatUInt64(value, valueSpan);
             span[0] = (byte)'$';
             int offset = WriteRaw(span, len, withLengthPrefix: false, offset: 1);
             valueSpan.Slice(0, len).CopyTo(span.Slice(offset));
@@ -1280,7 +1264,7 @@ internal static void WriteInteger(PipeWriter writer, long value)
             //note: client should never write integer; only server does this
 
             // :{asc}\r\n                = MaxInt64TextLen + 3
-            var span = writer.GetSpan(3 + MaxInt64TextLen);
+            var span = writer.GetSpan(3 + Format.MaxInt64TextLen);
 
             span[0] = (byte)':';
             var bytes = WriteRaw(span, value, withLengthPrefix: false, offset: 1);

src/StackExchange.Redis/RawResult.cs

@@ -77,7 +77,7 @@ internal ref struct Tokenizer
             public Tokenizer GetEnumerator() => this;
             private BufferReader _value;
 
-            public Tokenizer(in ReadOnlySequence<byte> value)
+            public Tokenizer(scoped in ReadOnlySequence<byte> value)
             {
                 _value = new BufferReader(value);
                 Current = default;
@@ -384,7 +384,7 @@ internal bool TryGetDouble(out double val)
 
         internal bool TryGetInt64(out long value)
         {
-            if (IsNull || Payload.IsEmpty || Payload.Length > PhysicalConnection.MaxInt64TextLen)
+            if (IsNull || Payload.IsEmpty || Payload.Length > Format.MaxInt64TextLen)
             {
                 value = 0;
                 return false;

src/StackExchange.Redis/RedisValue.cs

@@ -333,6 +333,9 @@ internal StorageType Type
             StorageType.Null => 0,
             StorageType.Raw => _memory.Length,
             StorageType.String => Encoding.UTF8.GetByteCount((string)_objectOrSentinel!),
+            StorageType.Int64 => Format.MeasureInt64(OverlappedValueInt64),
+            StorageType.UInt64 => Format.MeasureUInt64(OverlappedValueUInt64),
+            StorageType.Double => Format.MeasureDouble(OverlappedValueDouble),
             _ => throw new InvalidOperationException("Unable to compute length of type: " + Type),
         };
 
@@ -824,16 +827,15 @@ private static string ToHex(ReadOnlySpan<byte> src)
 
                     return value._memory.ToArray();
                 case StorageType.Int64:
-                    Span<byte> span = stackalloc byte[PhysicalConnection.MaxInt64TextLen + 2];
+                    Span<byte> span = stackalloc byte[Format.MaxInt64TextLen + 2];
                     int len = PhysicalConnection.WriteRaw(span, value.OverlappedValueInt64, false, 0);
                     arr = new byte[len - 2]; // don't need the CRLF
                     span.Slice(0, arr.Length).CopyTo(arr);
                     return arr;
                 case StorageType.UInt64:
                     // we know it is a huge value - just jump straight to Utf8Formatter
-                    span = stackalloc byte[PhysicalConnection.MaxInt64TextLen];
-                    if (!Utf8Formatter.TryFormat(value.OverlappedValueUInt64, span, out len))
-                        throw new InvalidOperationException("TryFormat failed");
+                    span = stackalloc byte[Format.MaxInt64TextLen];
+                    len = Format.FormatUInt64(value.OverlappedValueUInt64, span);
                     arr = new byte[len];
                     span.Slice(0, len).CopyTo(arr);
                     return arr;
@@ -1123,11 +1125,11 @@ private ReadOnlyMemory<byte> AsMemory(out byte[]? leased)
                     s = Format.ToString(OverlappedValueDouble);
                     goto HaveString;
                 case StorageType.Int64:
-                    leased = ArrayPool<byte>.Shared.Rent(PhysicalConnection.MaxInt64TextLen + 2); // reused code has CRLF terminator
+                    leased = ArrayPool<byte>.Shared.Rent(Format.MaxInt64TextLen + 2); // reused code has CRLF terminator
                     len = PhysicalConnection.WriteRaw(leased, OverlappedValueInt64) - 2; // drop the CRLF
                     return new ReadOnlyMemory<byte>(leased, 0, len);
                 case StorageType.UInt64:
-                    leased = ArrayPool<byte>.Shared.Rent(PhysicalConnection.MaxInt64TextLen); // reused code has CRLF terminator
+                    leased = ArrayPool<byte>.Shared.Rent(Format.MaxInt64TextLen); // reused code has CRLF terminator
                     // value is huge, jump direct to Utf8Formatter
                     if (!Utf8Formatter.TryFormat(OverlappedValueUInt64, leased, out len))
                         throw new InvalidOperationException("TryFormat failed");