Feature/paprika backend

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "src/bench_precompiles"]
 	path = src/bench_precompiles
 	url = https://github.com/shamatar/bench_precompiles.git
+[submodule "src/Nethermind/Paprika"]
+	path = src/Nethermind/Paprika
+	url = [email protected]:NethermindEth/Paprika.git

Directory.Packages.props

@@ -24,7 +24,9 @@
     <PackageVersion Include="Google.Protobuf.Tools" Version="3.33.1" />
     <PackageVersion Include="Grpc" Version="2.46.6" />
     <PackageVersion Include="Grpc.Tools" Version="2.76.0" />
+    <PackageVersion Include="HdrHistogram" Version="2.5.0" />
     <PackageVersion Include="HexMate" Version="0.0.3" />
+    <PackageVersion Include="JetBrains.dotMemoryUnit" Version="3.2.20220510" />
     <PackageVersion Include="MathNet.Numerics.FSharp" Version="5.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="10.0.0" />
@@ -84,6 +86,7 @@
     <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="10.0.0" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="22.0.16" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="22.0.16" />
+    <PackageVersion Include="Verify.NUnit" Version="31.7.1" />
     <PackageVersion Include="Websocket.Client" Version="5.3.0" />
     <PackageVersion Include="ZstdSharp.Port" Version="0.8.6" />
   </ItemGroup>

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -36,6 +36,9 @@
 using Nethermind.Merge.Plugin;
 using Nethermind.JsonRpc;
 using System.Reflection;
+using Paprika;
+using Paprika.Store;
+using PaprikaPlugin;
 
 namespace Ethereum.Test.Base;
 
@@ -133,6 +136,12 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             .AddSingleton(specProvider)
             .AddSingleton(_logManager)
             .AddSingleton(rewardCalculator)
+            .AddModule(new PaprikaPluginModule(new PaprikaConfig()
+            {
+            }))
+            .AddSingleton<IDb>(PagedDb.NativeMemoryDb(
+                10.MB(),
+                16))
             .AddSingleton<IDifficultyCalculator>(DifficultyCalculator);
 
         if (isEngineTest)

src/Nethermind/Ethereum.Test.Base/Ethereum.Test.Base.csproj

@@ -19,5 +19,6 @@
     <ProjectReference Include="..\Nethermind.Core.Test\Nethermind.Core.Test.csproj" />
     <ProjectReference Include="..\Nethermind.Core\Nethermind.Core.csproj" />
     <ProjectReference Include="..\Nethermind.Evm\Nethermind.Evm.csproj" />
+    <ProjectReference Include="..\Paprika\src\PaprikaPlugin\PaprikaPlugin.csproj" />
   </ItemGroup>
 </Project>

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -25,6 +25,9 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using Paprika;
+using Paprika.Store;
+using PaprikaPlugin;
 
 namespace Ethereum.Test.Base
 {
@@ -82,6 +85,12 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
                 .AddSingleton<IBlockhashProvider>(new TestBlockhashProvider())
                 .AddSingleton(specProvider)
                 .AddSingleton(_logManager)
+                .AddModule(new PaprikaPluginModule(new PaprikaConfig()
+                {
+                }))
+                .AddSingleton<IDb>(PagedDb.NativeMemoryDb(
+                    10.MB(),
+                    16))
                 .Build();
 
             IMainProcessingContext mainBlockProcessingContext = container.Resolve<IMainProcessingContext>();

src/Nethermind/EthereumTests.slnx

@@ -67,4 +67,6 @@
   <Project Path="Nethermind.EofParse.Runner/Nethermind.EofParse.Runner.csproj" />
   <Project Path="Nethermind.State.Test.Runner.Test/Nethermind.State.Test.Runner.Test.csproj" />
   <Project Path="Nethermind.Test.Runner/Nethermind.Test.Runner.csproj" />
+  <Project Path="Paprika/src/Paprika/Paprika.csproj" />
+  <Project Path="Paprika/src/PaprikaPlugin/PaprikaPlugin.csproj" />
 </Solution>

src/Nethermind/Nethermind.Blockchain.Test.Runner/PerfTest.cs

@@ -28,6 +28,7 @@ public IEnumerable<EthereumTestResult> RunTests()
             bool isNewLine = true;
             foreach (GeneralStateTest test in tests)
             {
+                Console.WriteLine($"RUNNING {test.Name}");
                 if (test.LoadFailure is not null)
                 {
                     continue;

src/Nethermind/Nethermind.Blockchain.Test.Runner/Program.cs

@@ -34,6 +34,7 @@ private static async Task RunManualTestingLoop()
                 string[] input = Console.ReadLine().Split();
                 string command = input[0];
                 string testWildcard = input.Length <= 1 ? null : input[1];
+                Console.Error.WriteLine($"Command {command}");
 
                 Stopwatch stopwatch = new Stopwatch();
                 if (command == "p")
@@ -53,6 +54,7 @@ private static async Task RunManualTestingLoop()
                 }
                 else
                 {
+                    Console.Error.WriteLine("Break?");
                     break;
                 }
 
@@ -76,6 +78,7 @@ private static async Task RunManualTestingLoop()
 
         private static void WrapAndRunDirectoryStateTests(IStateTestRunner stateTest)
         {
+            Console.Error.WriteLine($"Running {stateTest}");
             var result = stateTest.RunTests().ToList();
             var failedTestsInCategory = result.Where(r => !r.Pass).Select(t => t.Name + " " + t.LoadFailure).ToArray();
             AllFailingTests.AddRange(failedTestsInCategory);
@@ -169,7 +172,7 @@ private static void RunAllStateTests(string testWildcard, Func<ITestSourceLoader
             WrapAndRunDirectoryStateTests(testRunnerBuilder(new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "vmLogTests", testWildcard)));
             WrapAndRunDirectoryStateTests(testRunnerBuilder(new TestsSourceLoader(new LoadGeneralStateTestsStrategy(), "vmTests", testWildcard)));
 
-            /* 
+            /*
             await Run(testRunnerBuilder(new DirectoryTestsSource("bcEIP158ToByzantium", testWildcard));
             await Run(testRunnerBuilder(new DirectoryTestsSource("bcFrontierToHomestead", testWildcard));
             await Run(testRunnerBuilder(new DirectoryTestsSource("bcHomesteadToDao", testWildcard));

src/Nethermind/Nethermind.Core/Account.cs

@@ -127,6 +127,11 @@ public Account WithChangedCodeHash(Hash256 newCodeHash)
         }
 
         public AccountStruct ToStruct() => new(Nonce, Balance, StorageRoot, CodeHash);
+
+        public override string ToString()
+        {
+            return $"N:{Nonce},B:{Balance},S:{StorageRoot},C:{CodeHash}";
+        }
     }
 
     public readonly struct AccountStruct : IEquatable<AccountStruct>

src/Nethermind/Nethermind.Evm/State/IWorldStateScopeProvider.cs

@@ -79,7 +79,7 @@ public interface IStorageTree
     {
         Hash256 RootHash { get; }
 
-        byte[] Get(in UInt256 index);
+        byte[]? Get(in UInt256 index);
 
         void HintGet(in UInt256 index, byte[]? value);
 
@@ -88,7 +88,7 @@ public interface IStorageTree
         /// </summary>
         /// <param name="hash"></param>
         /// <returns></returns>
-        byte[] Get(in ValueHash256 hash);
+        byte[]? Get(in ValueHash256 hash);
     }
 
     public interface IWorldStateWriteBatch : IDisposable

src/Nethermind/Nethermind.Paprika/IPaprikaConfig.cs

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Config;
+
+namespace Nethermind.Paprika;
+
+public interface IPaprikaConfig: IConfig
+{
+    [ConfigItem(Description = "Enable paprika", DefaultValue = "false", HiddenFromDocs = true)]
+    public bool Enabled { get; set; }
+
+    [ConfigItem(Description = "Paprika db size in GB", DefaultValue = "128", HiddenFromDocs = true)]
+    public long SizeGb { get; set; }
+
+    [ConfigItem(Description = "Paprika history depth", DefaultValue = "128", HiddenFromDocs = true)]
+    public byte HistoryDepth { get; set; }
+
+    [ConfigItem(Description = "Paprika flush", DefaultValue = "false", HiddenFromDocs = true)]
+    public bool FlushToDisk { get; set; }
+
+    [ConfigItem(Description = "Paprika flush", DefaultValue = "2", HiddenFromDocs = true)]
+    int FinalizationQueueLimit { get; set; }
+
+    [ConfigItem(Description = "Automatic finalaization", DefaultValue = "2", HiddenFromDocs = true)]
+    int AutomaticallyFinalizeAfter { get; set; }
+
+    [ConfigItem(Description = "Import from triestore", DefaultValue = "false", HiddenFromDocs = true)]
+    bool ImportFromTrieStore { get; set; }
+}

src/Nethermind/Nethermind.Paprika/Importer/ImporterStep.cs

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Autofac.Features.AttributeFilters;
+using Nethermind.Api.Steps;
+using Nethermind.Blockchain;
+using Nethermind.Config;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Db;
+using Nethermind.Init.Steps;
+using Nethermind.Logging;
+using Nethermind.Monitoring;
+using Nethermind.Trie;
+using IDb = Paprika.IDb;
+
+namespace Nethermind.Paprika.Importer;
+
+[RunnerStepDependencies(
+    dependencies: [typeof(InitializeBlockTree)],
+    dependents: [typeof(InitializeBlockchain)]
+)]
+public class ImporterStep(
+    IBlockTree blockTree,
+    global::Paprika.Chain.Blockchain paprikaBlockchain,
+    [KeyFilter(DbNames.State)] INodeStorage stateNodeStorage,
+    IProcessExitSource exitSource,
+    IDisposableStack disposableStack,
+    IDb db,
+    ILogManager logManager): IStep
+{
+    ILogger _logger = logManager.GetClassLogger<ImporterStep>();
+
+    public async Task Execute(CancellationToken cancellationToken)
+    {
+        Console.Error.WriteLine("execute start");
+        new NethermindKestrelMetricServer(9999).Start();
+        disposableStack.Push(Prometheus.MeterAdapter.StartListening());
+
+        Hash256 rootHash;
+        using (var readOnly = paprikaBlockchain.StartReadOnlyLatestFromDb())
+        {
+            rootHash = readOnly.Hash.ToNethHash();
+        }
+
+        if (rootHash != Hash256.Zero)
+        {
+            _logger.Warn($"Import skipped as has existing root hash. {rootHash}");
+            return;
+        }
+
+        BlockHeader? head = blockTree.Head?.Header;
+        if (head is null)
+        {
+            _logger.Warn($"Import skipped as head is null. {rootHash}");
+            return;
+        }
+
+        _logger.Info($"Starting paprika import from head {head.ToString(BlockHeader.Format.Short)}. StateRoot: {head.StateRoot}");
+
+        // LMDB
+        // db.ForceSync();
+        var visitor = new PaprikaCopyingVisitor(paprikaBlockchain, 50000, false, logManager);
+
+        var trieStore = new RawTrieStore(stateNodeStorage);
+        PatriciaTree trie = new PatriciaTree(trieStore, logManager);
+        trie.RootHash = head.StateRoot!;
+
+        var visit = Task.Run(() =>
+        {
+            trie.Accept(visitor, trie.RootHash, new VisitingOptions
+            {
+                MaxDegreeOfParallelism = 8,
+                //FullScanMemoryBudget = 1L * 1024 * 1024 * 1024
+            });
+
+            visitor.Finish();
+        });
+
+        var copy = visitor.Copy();
+        await Task.WhenAll(visit, copy);
+
+        var finalRootHash = await copy;
+        var rootHashNeth = finalRootHash.ToNethHash();
+
+        if (rootHashNeth != head.StateRoot)
+        {
+            throw new Exception($"Import failed. Root hash mismatched. Expected {head.StateRoot}, got {rootHashNeth}");
+        }
+
+        db.Flush();
+
+        _logger.Info("Imported statedb to paprika. Exiting....");
+        exitSource.Exit(0);
+    }
+}