fix: prevent EVM memory size overflow crash for extreme memory requests (#9887)

fix: prevent EVM memory size overflow crash for large memory requests

The memory validation in CheckMemoryAccessViolation was checking against
long.MaxValue, but .NET arrays are limited to int.MaxValue. When memory
requests exceeded int.MaxValue but were less than long.MaxValue, the
word-aligned size calculation would overflow when cast to int, causing
ArrayPool.Rent() to allocate an incorrectly sized array and subsequent
Array.Copy operations to crash.

This fix changes the limit from long.MaxValue to (int.MaxValue - WordSize + 1)
to ensure that after word alignment, the resulting size still fits in int.

Root cause:
- Memory size validation checked: totalSize > long.MaxValue
- But .NET arrays require: (int)Size to be valid
- Word alignment adds up to 31 bytes: Size = totalSize + (32 - totalSize % 32)
- When totalSize > int.MaxValue - 31, (int)Size overflows

Example crash scenario:
1. Contract requests 4GB (0xffffffff) memory via DELEGATECALL
2. Validation passes (4GB < long.MaxValue)
3. Word-aligned Size = 0x100000000 (exceeds int.MaxValue)
4. (int)Size = 0 (overflow)
5. ArrayPool.Rent(0) returns tiny array
6. Array.Copy crashes with "Destination array was not long enough"

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <[email protected]>

Author: bshastry

src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs

@@ -54,7 +54,8 @@ public void Div32Ceiling(int input, int expectedResult)
     [TestCase(100 * MaxCodeSize, MaxCodeSize)]
     [TestCase(1000 * MaxCodeSize, MaxCodeSize)]
     [TestCase(0, 1024 * 1024)]
-    [TestCase(0, Int32.MaxValue)]
+    // Note: Int32.MaxValue was removed as a test case because after word alignment
+    // it exceeds the maximum allowed memory size and correctly returns out-of-gas.
     public void MemoryCost(int destination, int memoryAllocation)
     {
         EvmPooledMemory memory = new();
@@ -114,6 +115,50 @@ public void CalculateMemoryCost_TotalSizeExceedsLongMax_ShouldReturnOutOfGas()
         Assert.That(result, Is.EqualTo(0L));
     }
 
+    [Test]
+    public void CalculateMemoryCost_TotalSizeExceedsIntMaxAfterWordAlignment_ShouldReturnOutOfGas()
+    {
+        // Test that memory requests that would overflow int.MaxValue after word alignment
+        // are properly rejected. This prevents crashes in .NET array operations.
+        // The limit is int.MaxValue - WordSize + 1 to ensure word-aligned size fits in int.
+        EvmPooledMemory memory = new();
+
+        // Request exactly at the limit should succeed
+        UInt256 maxAllowedSize = (UInt256)(int.MaxValue - EvmPooledMemory.WordSize + 1);
+        long result = memory.CalculateMemoryCost(0, in maxAllowedSize, out bool outOfGas);
+        Assert.That(outOfGas, Is.EqualTo(false), "Size at limit should be allowed");
+
+        // Request one byte over the limit should fail
+        UInt256 overLimitSize = maxAllowedSize + 1;
+        result = memory.CalculateMemoryCost(0, in overLimitSize, out outOfGas);
+        Assert.That(outOfGas, Is.EqualTo(true), "Size over limit should return out of gas");
+        Assert.That(result, Is.EqualTo(0L));
+    }
+
+    [Test]
+    public void CalculateMemoryCost_4GBMemoryRequest_ShouldReturnOutOfGas()
+    {
+        // Regression test: 4GB memory request (0xffffffff) should return out-of-gas
+        // instead of causing integer overflow crash in array operations.
+        EvmPooledMemory memory = new();
+        UInt256 size4GB = 0xffffffffUL;
+        long result = memory.CalculateMemoryCost(0, in size4GB, out bool outOfGas);
+        Assert.That(outOfGas, Is.EqualTo(true), "4GB memory request should return out of gas");
+        Assert.That(result, Is.EqualTo(0L));
+    }
+
+    [Test]
+    public void CalculateMemoryCost_LargeOffsetPlusLength_ShouldReturnOutOfGas()
+    {
+        // Test that location + length exceeding int.MaxValue - WordSize + 1 returns out-of-gas
+        EvmPooledMemory memory = new();
+        UInt256 location = (UInt256)(int.MaxValue / 2);
+        UInt256 length = (UInt256)(int.MaxValue / 2 + 100); // Sum exceeds limit
+        long result = memory.CalculateMemoryCost(in location, in length, out bool outOfGas);
+        Assert.That(outOfGas, Is.EqualTo(true), "Location + length exceeding limit should return out of gas");
+        Assert.That(result, Is.EqualTo(0L));
+    }
+
     [Test]
     public void Save_LocationExceedsULong_ShouldReturnOutOfGas()
     {

src/Nethermind/Nethermind.Evm/EvmPooledMemory.cs

@@ -84,15 +84,20 @@ private static void CheckMemoryAccessViolation(in UInt256 location, in UInt256 l
 
     private static void CheckMemoryAccessViolation(in UInt256 location, ulong length, out ulong newLength, out bool outOfGas)
     {
-        if (location.IsLargerThanULong() || length > long.MaxValue)
+        // Check for overflow and ensure the word-aligned size fits in int.
+        // Word alignment can add up to 31 bytes, so we use (int.MaxValue - WordSize + 1) as the limit.
+        // This ensures that after word alignment, the size still fits in int for .NET array operations.
+        const ulong MaxMemorySize = int.MaxValue - WordSize + 1;
+
+        if (location.IsLargerThanULong() || length > MaxMemorySize)
         {
             outOfGas = true;
             newLength = 0;
             return;
         }
 
         ulong totalSize = location.u0 + length;
-        if (totalSize < location.u0 || totalSize > long.MaxValue)
+        if (totalSize < location.u0 || totalSize > MaxMemorySize)
         {
             outOfGas = true;
             newLength = 0;