Merge remote-tracking branch 'origin/compatible' into HEAD

Author: georgeee

buildkite/src/Command/Rosetta/Connectivity.dhall

@@ -26,6 +26,8 @@ let Profiles = ../../Constants/Profiles.dhall
 
 let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
+let B/If = B.definitions/commandStep/properties/if/Type
+
 let Spec =
       { Type =
           { dockerType : Dockers.Type
@@ -35,6 +37,7 @@ let Spec =
           , timeout : Natural
           , profile : Profiles.Type
           , scope : List PipelineScope.Type
+          , if : B/If
           }
       , default =
           { dockerType = Dockers.Type.Bullseye
@@ -44,6 +47,8 @@ let Spec =
           , timeout = 1000
           , profile = Profiles.Type.Devnet
           , scope = PipelineScope.Full
+          , if =
+              "build.pull_request.base_branch != \"develop\" || build.branch == \"develop\""
           }
       }
 
@@ -68,6 +73,7 @@ let command
                 "rosetta-${Network.lowerName spec.network}-connectivity-test"
             , target = Size.XLarge
             , soft_fail = Some spec.softFail
+            , if = Some spec.if
             , depends_on =
                 Dockers.dependsOn
                   Dockers.DepsSpec::{

flake.lock

@@ -14,6 +14,7 @@
   inputs.utils.url = "github:gytis-ivaskevicius/flake-utils-plus";
   inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11-small";
   inputs.nixpkgs-old.url = "github:nixos/nixpkgs/nixos-23.05-small";
+  inputs.nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
 
   inputs.mix-to-nix.url = "github:serokell/mix-to-nix";
   inputs.nix-npm-buildPackage.url = "github:serokell/nix-npm-buildpackage";
@@ -38,7 +39,8 @@
   # - flake.nix (and flake.lock after running
   #   `nix flake update opam-repository`).
   # - scripts/update_opam_switch.sh
-  inputs.opam-repository.url = "github:ocaml/opam-repository/08d8c16c16dc6b23a5278b06dff0ac6c7a217356";
+  inputs.opam-repository.url =
+    "github:ocaml/opam-repository/08d8c16c16dc6b23a5278b06dff0ac6c7a217356";
   inputs.opam-repository.flake = false;
 
   inputs.nixpkgs-mozilla.url = "github:mozilla/nixpkgs-mozilla";
@@ -60,7 +62,7 @@
 
   outputs = inputs@{ self, nixpkgs, utils, mix-to-nix, nix-npm-buildPackage
     , opam-nix, opam-repository, nixpkgs-mozilla, flake-buildkite-pipeline
-    , nix-utils, flockenzeit, nixpkgs-old, ... }:
+    , nix-utils, flockenzeit, nixpkgs-old, nixpkgs-unstable, ... }:
     let
       inherit (nixpkgs) lib;
 
@@ -103,9 +105,7 @@
         # Skip tests on nodejs dep due to known issue with nixpkgs 24.11 https://github.com/NixOS/nixpkgs/issues/402079
         # this can be removed after upgrading
         skipNodeTests = final: prev: {
-          nodejs = prev.nodejs.overrideAttrs(old: {
-            doCheck = false;
-          });
+          nodejs = prev.nodejs.overrideAttrs (old: { doCheck = false; });
         };
       };
       nixosModules.mina = import ./nix/modules/mina.nix inputs;
@@ -273,12 +273,47 @@
         };
     } // utils.lib.eachDefaultSystem (system:
       let
-        rocksdbOverlay = pkgs: prev:
-          if prev.stdenv.isx86_64 then {
-            rocksdb-mina = pkgs.rocksdb511;
-          } else {
-            rocksdb-mina = pkgs.rocksdb;
-          };
+        # Helper function to map dependencies to current nixpkgs equivalents
+        mapDepsToCurrentPkgs = pkgs: deps:
+          map (dep:
+            if pkgs ? ${dep.pname or dep.name or ""} then
+              pkgs.${dep.pname or dep.name or ""}
+            else
+              dep) deps;
+
+        # Helper function to disable compression libraries in cmake flags
+        disableCompressionLibs = flags:
+          builtins.filter (flag: flag != [ ]) (map (flag:
+            if builtins.isString flag
+            && builtins.match ".*WITH_(BZ2|LZ4|SNAPPY|ZLIB|ZSTD)=1.*" flag
+            != null then
+              builtins.replaceStrings [ "=1" ] [ "=0" ] flag
+            else
+              flag) flags);
+
+        rocksdbOverlay = pkgs: prev: {
+          rocksdb-mina = let
+            # Get the full derivation from unstable but build with current nixpkgs
+            unstableRocksdb =
+              (nixpkgs-unstable.legacyPackages.${system}.rocksdb.override {
+                enableShared = false;
+                enableLiburing = false;
+                bzip2 = null;
+                lz4 = null;
+                snappy = null;
+                zlib = null;
+                zstd = null;
+              });
+          in pkgs.stdenv.mkDerivation (unstableRocksdb.drvAttrs // {
+            cmakeFlags =
+              disableCompressionLibs unstableRocksdb.drvAttrs.cmakeFlags;
+            # Override the build environment to use current nixpkgs toolchain
+            nativeBuildInputs = mapDepsToCurrentPkgs pkgs
+              (unstableRocksdb.nativeBuildInputs or [ ]);
+            buildInputs =
+              mapDepsToCurrentPkgs pkgs (unstableRocksdb.buildInputs or [ ]);
+          });
+        };
         go119Overlay = (_: _: {
           inherit (nixpkgs-old.legacyPackages.${system})
             go_1_19 buildGo119Module;
@@ -326,7 +361,7 @@
           (pkgs.python3.withPackages
             (python-pkgs: [ python-pkgs.click python-pkgs.requests ]))
           jq
-          rocksdb.tools
+          rocksdb-mina.tools
         ];
       in {
 

flake.nix

@@ -21,7 +21,7 @@ pkgs.mkShell {
     wasm-pack
     lmdb
     rosetta-cli
-    rocksdb.tools
+    rocksdb-mina.tools
   ];
   OPAMSWITCH = "mina";
   MINA_ROCKSDB = "${pkgs.rocksdb-mina}/lib/librocksdb.a";

nix/impure-shell.nix

@@ -2,41 +2,4 @@
 final: prev: {
   sodium-static =
     final.libsodium.overrideAttrs (o: { dontDisableStatic = true; });
-
-  rocksdb = (prev.rocksdb.override {
-    snappy = null;
-    lz4 = null;
-    zstd = null;
-    bzip2 = null;
-  }).overrideAttrs (_: {
-    cmakeFlags = [
-      "-DPORTABLE=1"
-      "-DWITH_JEMALLOC=0"
-      "-DWITH_JNI=0"
-      "-DWITH_BENCHMARK_TOOLS=0"
-      "-DWITH_TESTS=1"
-      "-DWITH_TOOLS=0"
-      "-DWITH_BZ2=0"
-      "-DWITH_LZ4=0"
-      "-DWITH_SNAPPY=0"
-      "-DWITH_ZLIB=0"
-      "-DWITH_ZSTD=0"
-      "-DWITH_GFLAGS=0"
-      "-DUSE_RTTI=1"
-    ];
-  });
-
-  rocksdb511 = let
-    impl = (import (fetchTarball {
-      url = "https://github.com/NixOS/nixpkgs/archive/nixos-19.03-small.tar.gz";
-      sha256 = "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb";
-    }) { system = "x86_64-linux"; }).rocksdb.override {
-      snappy = null;
-      lz4 = null;
-      bzip2 = null;
-    };
-  in if impl.version == "5.11.3" then
-    impl
-  else
-    throw "Expected rocksdb version 5.11.3, but got ${impl.version}";
 }

nix/misc.nix

@@ -65,19 +65,14 @@ let
         '';
       };
 
-      rocksdb_stubs = super.rocksdb_stubs.overrideAttrs (oa: {
-        MINA_ROCKSDB = let
-          mainPath = "${pkgs.rocksdb-mina}/lib/librocksdb.a";
-          staticPath =
-            "${pkgs.rocksdb-mina.static or pkgs.rocksdb-mina}/lib/librocksdb.a";
-        in if builtins.pathExists mainPath then
-          mainPath
-        else if builtins.pathExists staticPath then
-          staticPath
-        else
-          throw
-          "Could not find librocksdb.a in either ${mainPath} or ${staticPath}";
-      });
+      rocksdb_stubs =
+        # TODO uncomment after updating rocksdb_stubs to 10.5.1
+        # assert (super.rocksdb_stubs.version == pkgs.rocksdb-mina.version)
+        #   || builtins.throw
+        #   "rocksdb_stubs version (${super.rocksdb_stubs.version}) does not match rocksdb-mina version (${pkgs.rocksdb-mina.version})";
+        super.rocksdb_stubs.overrideAttrs {
+          MINA_ROCKSDB = "${pkgs.rocksdb-mina}/lib/librocksdb.a";
+        };
 
       # This is needed because
       # - lld package is not wrapped to pick up the correct linker flags

nix/ocaml.nix

@@ -4,6 +4,6 @@
 # Find the root of the Mina repo
 ROOT=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )"/.. &> /dev/null && pwd )
 # Update the submodules
-pushd "$ROOT" && git submodule sync && git submodule update --init --recursive && popd
+pushd "$ROOT" && git submodule sync && git submodule update --init --recursive --depth 1 && popd
 # Add the flake registry entry
 nix registry add mina "git+file://$ROOT?submodules=1"

nix/pin.sh

@@ -59,7 +59,7 @@ readonly DEFAULT_PAYMENT_TX_INTERVAL="2"
 readonly DEFAULT_ZKAPP_TX_INTERVAL="1"
 
 # Default memory threshold for PostgreSQL processes
-readonly DEFAULT_POSTGRES_MEMORY_THRESHOLD="2500"  # MB
+readonly DEFAULT_POSTGRES_MEMORY_THRESHOLD="3000"  # MB
 
 # Default memory threshold for Rosetta API processes
 readonly DEFAULT_ROSETTA_MEMORY_THRESHOLD="300"   # MB

scripts/tests/rosetta-load.sh

@@ -1704,11 +1704,11 @@ let snark_hashes =
       fun () -> if json then Core.printf "[]\n%!"]
 
 let internal_commands logger ~itn_features =
+  let signature_kind = Mina_signature_kind.t_DEPRECATED in
   [ ( Snark_worker.Intf.command_name
     , Snark_worker.command ~proof_level:Genesis_constants.Compiled.proof_level
         ~constraint_constants:Genesis_constants.Compiled.constraint_constants
-        ~commit_id:Mina_version.commit_id
-        ~signature_kind:Mina_signature_kind.t_DEPRECATED )
+        ~commit_id:Mina_version.commit_id ~signature_kind )
   ; ("snark-hashes", snark_hashes)
   ; ( "run-prover"
     , Command.async
@@ -1727,8 +1727,7 @@ let internal_commands logger ~itn_features =
                  let%bind prover =
                    Prover.create ~commit_id:Mina_version.commit_id ~logger
                      ~proof_level ~constraint_constants
-                     ~pids:(Pid.Table.create ()) ~conf_dir
-                     ~signature_kind:Mina_signature_kind.t_DEPRECATED ()
+                     ~pids:(Pid.Table.create ()) ~conf_dir ~signature_kind ()
                  in
                  Prover.prove_from_input_sexp prover sexp >>| ignore
              | `Eof ->
@@ -1755,7 +1754,6 @@ let internal_commands logger ~itn_features =
                 Reader.read_sexp reader )
           with
           | `Ok sexp -> (
-              let signature_kind = Mina_signature_kind.t_DEPRECATED in
               let%bind worker_state =
                 Snark_worker.Inputs.Worker_state.create ~proof_level
                   ~constraint_constants ~signature_kind ()
@@ -1988,7 +1986,7 @@ let internal_commands logger ~itn_features =
             Prover.create ~commit_id:Mina_version.commit_id ~logger ~pids
               ~conf_dir ~proof_level
               ~constraint_constants:precomputed_values.constraint_constants
-              ~signature_kind:Mina_signature_kind.t_DEPRECATED ()
+              ~signature_kind ()
           in
           match%bind
             Prover.create_genesis_block prover

src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml

@@ -184,7 +184,7 @@ let command =
      in
      let proof_level = Genesis_constants.Proof_level.Full in
      let proof_cache_db = Proof_cache_tag.create_identity_db () in
-     let signature_kind = Mina_signature_kind.t_DEPRECATED in
+     let signature_kind = Mina_signature_kind.Testnet in
      if witness_only then
        witness ~proof_cache_db ~genesis_constants ~constraint_constants
          ~proof_level ~max_num_updates ~logger ~signature_kind ?min_num_updates