test: add eigenda v1 to v2 migration kurtosis test (#44)

* test(kt-devnet): add v1->v2 migration test

Using the proxy v1.7.0-rc docker image
updated the test harness to add a client for the new proxy admin API which allows changing dispersal backend (v1 -> v2)
added test to test this migration

* test(kt-devnet): fix test harness

proxyClient was not being initialized properly, causing a nil pointer error.

* ci(kt-devnet): add v2 files to test matrix

* docs(kt-devnet): fix comment on TestEigenDAV2Migration_Memstore test

* test(kt-devnet): make GetDispersalBackend use passed ctx

* test(kt-devnet): add finalization assert to migration test

We check that op-node is still finalizing after the migration, just like we do in the failover test.

* test(kt-devnet): also test migration against holesky

* test(kt-devnet): fix wrong context bug

was passing old context in places, causing context timed out error

* chore(kt-devnet): add max-blob-length flag to make proxy load faster

proxy takes ~1min to load and sometimes longer. add the --eigenda.v2.max-blob-length flag set to 1MiB to make loading faster (loads fewer SRS points at startup)

* chore(kt-devnet): add justfile command to switch proxy backend

allows switching from v1 to v2 when doing manual testing locally

* test(kt-devnet): add log line to show discarded commitments

in failover test

* test(kt-devnet): skip migration test on holesky to make CI faster

Copy paste of code comment explaining reasoning:
	// The issue is that because of the holocene strict ordering rules,
	// after migrating from v1 to v2, the v2 certs, even though they finalize very quickly
	// and are returned to op-batcher, op-batcher cannot send them to the batch-inbox until
	// the v1 certs have been sent. So we need to wait at least 10-15 mins for the v1 certs to be returned
	// from proxy to op-batcher and submitted to batch-inbox.
	// starting kurtosis devnet takes ~10 mins and batcher logs tests take 15 mins,
	// so we are already at 25 mins. Trying to keep CI time under 30 mins,
	// so we skip this test for now. We could turn on if we had a nice inabox eigenda that would allow
	// for much faster v1 finalization.

* test(kt-devnet): reenable v2migration test on holesky

Decided to live with increased CI time in order to be safe.

* test(kt-devnet): fix migration test by increasing waitForBlock timer

* chore(kt-devnet): bump eigenda.yaml proxy to v1.7.0

* test(kt-devnet): add extra minute to WaitForBlock to prevent timeout

* test(kt-devnet): increase go test timeout to 30/40mins

* test(kt-devnet): bump v1 stage to 40 l1 blocks in migration test

* test(kt-devnet): add extra minute to WaitForBlock to prevent timeout

* test(kt-devnet): add -v flags to get more info in ci

* test(kt-devnet): change max-channel-duration 150->10 for holesky tests

* ci(kt-devnet): remove holesky-sequential-large-blobs v1 test

Kept having flukes. It was taking super long. Test getting throttled. Was just a ci nightmare, so opted to remove.

Author: samlaf

.github/workflows/kurtosis-devnet.yml

@@ -52,8 +52,12 @@ jobs:
         # This list should be kept up-to-date with the files in eigenda-template-values
         valuesFile:
           [
-            "holesky-sequential-large-blobs.json",
+            # v1 is too slow for sequential large blobs.
+            # Could send a single massive blob every few minutes, but then migration test
+            # takes too long waiting for the few certs to land onchain, so opted to just not run it.
             "holesky-concurrent-small-blobs.json",
+            "holesky-v2-sequential-large-blobs.json",
+            "holesky-v2-concurrent-small-blobs.json",
           ]
     steps:
       - uses: actions/checkout@v4
@@ -68,6 +72,7 @@ jobs:
             "eigenda-proxy": {
               "secrets": {
                 "eigenda.signer-private-key-hex": "${{ secrets.EIGENDA_V1_HOLESKY_TESTNET_SIGNER_KEY }}",
+                "eigenda.v2.signer-payment-key-hex": "${{ secrets.EIGENDA_V2_HOLESKY_TESTNET_SIGNER_KEY }}",
                 "eigenda.eth_rpc": "https://ethereum-holesky-rpc.publicnode.com"
               }
             }

kurtosis-devnet/eigenda-template-values/holesky-concurrent-small-blobs.json

@@ -1,13 +1,15 @@
 {
   "eigenda-proxy": {
     "useMemstore": false,
+    "dispersalBackend": "v1",
     "secrets": {
       "eigenda.signer-private-key-hex": "",
+      "eigenda.v2.signer-payment-key-hex": "",
       "eigenda.eth_rpc": ""
     }
   },
   "batcher": {
-    "max-channel-duration": 150,
+    "max-channel-duration": 10,
     "altda.max-concurrent-da-requests": 100,
     "target-num-frames": 10,
     "max-l1-tx-size-bytes": 1000,

kurtosis-devnet/eigenda-template-values/holesky-v2-concurrent-small-blobs.json

@@ -0,0 +1,18 @@
+{
+  "eigenda-proxy": {
+    "useMemstore": false,
+    "dispersalBackend": "v2",
+    "secrets": {
+      "eigenda.signer-private-key-hex": "",
+      "eigenda.v2.signer-payment-key-hex": "",
+      "eigenda.eth_rpc": ""
+    }
+  },
+  "batcher": {
+    "max-channel-duration": 10,
+    "altda.max-concurrent-da-requests": 100,
+    "target-num-frames": 10,
+    "max-l1-tx-size-bytes": 1000,
+    "batch-type": 1
+  }
+}

…lues/holesky-sequential-large-blobs.json …s/holesky-v2-sequential-large-blobs.jsonkurtosis-devnet/eigenda-template-values/holesky-sequential-large-blobs.json renamed to kurtosis-devnet/eigenda-template-values/holesky-v2-sequential-large-blobs.json kurtosis-devnet/eigenda-template-values/holesky-sequential-large-blobs.json renamed to kurtosis-devnet/eigenda-template-values/holesky-v2-sequential-large-blobs.json

@@ -1,13 +1,15 @@
 {
   "eigenda-proxy": {
     "useMemstore": false,
+    "dispersalBackend": "v2",
     "secrets": {
       "eigenda.signer-private-key-hex": "",
+      "eigenda.v2.signer-payment-key-hex": "",
       "eigenda.eth_rpc": ""
     }
   },
   "batcher": {
-    "max-channel-duration": 150,
+    "max-channel-duration": 10,
     "altda.max-concurrent-da-requests": 1,
     "target-num-frames": 1000,
     "max-l1-tx-size-bytes": 1000,

kurtosis-devnet/eigenda.yaml

@@ -8,11 +8,13 @@
 # TODO: I hate yaml templating... should really move to something more reasonable. CUE perhaps?
 # use proxy by default, otherwise connect with eigenda holesky backend
 {{- $useProxyMemstore := dig "eigenda-proxy" "useMemstore" true $context }}
-{{- $eigendaBackendSignerKey := dig "eigenda-proxy" "secrets" "eigenda.signer-private-key-hex" "" $context }}
+{{- $proxyDispersalBackend := dig "eigenda-proxy" "dispersalBackend" "v1" $context }}
+{{- $eigendaV1SignerKey := dig "eigenda-proxy" "secrets" "eigenda.signer-private-key-hex" "" $context }}
+{{- $eigendaV2SignerKey := dig "eigenda-proxy" "secrets" "eigenda.v2.signer-payment-key-hex" "" $context }}
 {{- $eigendaBackendEthRpc := dig "eigenda-proxy" "secrets" "eigenda.eth_rpc" "" $context }}
 {{- if not $useProxyMemstore}}
   # make sure the secrets are populated when using eigenda-holesky backend
-  {{- if or (eq $eigendaBackendSignerKey "") (eq $eigendaBackendEthRpc "") }}
+  {{- if or (eq $eigendaV1SignerKey "") (eq $eigendaV2SignerKey "") (eq $eigendaBackendEthRpc "") }}
     {{- fail "Missing required eigenda-proxy eigenda-backend secrets." }}
   {{- end }}
 {{- end }}
@@ -90,19 +92,29 @@ optimism_package:
         cannon_prestates_url: "http://fileserver/proofs/op-program/cannon"
         extra_params: []
       da_server_params:
-        image: ghcr.io/layr-labs/eigenda-proxy:v1.6.5
+        image: ghcr.io/layr-labs/eigenda-proxy:v1.7.0
         cmd:
           - --addr=0.0.0.0
           - --port=3100
+          - --storage.backends-to-enable=V1,V2
+          - --storage.dispersal-backend={{ $proxyDispersalBackend }} #v1 or v2
+          - --api-enabled=admin # to enable changing dispersal-backend to v2 without restarting
+          - --eigenda.v2.max-blob-length=1MiB # makes startup faster by only loading 1MiB/2MiB of G1/G2 SRS points.
         {{- if $useProxyMemstore }}
           - --memstore.enabled
           - --memstore.expiration=30m
         {{- else }} # connect to eigenda holesky backend
+          # V1 flags
           - --eigenda.disperser-rpc=disperser-holesky.eigenda.xyz:443
           - --eigenda.svc-manager-addr=0xD4A7E1Bd8015057293f0D0A557088c286942e84b
           # The two params below are loaded from the eigenda-secrets.json file
-          - --eigenda.signer-private-key-hex={{ $eigendaBackendSignerKey }}
+          - --eigenda.signer-private-key-hex={{ $eigendaV1SignerKey }}
           - --eigenda.eth-rpc={{ $eigendaBackendEthRpc }}
+          # V2 flags
+          - --eigenda.v2.disperser-rpc=disperser-holesky.eigenda.xyz:443
+          - --eigenda.v2.cert-verifier-addr=0xFe52fE1940858DCb6e12153E2104aD0fDFbE1162
+          - --eigenda.v2.signer-payment-key-hex={{ $eigendaV2SignerKey }}
+          - --eigenda.v2.eth-rpc={{ $eigendaBackendEthRpc }}
         {{- end }}
       additional_services:
         - da_server

kurtosis-devnet/justfile

@@ -121,13 +121,13 @@ eigenda-devnet-clean ENCLAVE_NAME="eigenda-devnet":
 # meaning with a config file in eigenda-template-values/memstore-* .
 [group('eigenda')]
 eigenda-devnet-test-memstore *ARGS="":
-    go test ./tests/eigenda/... -run "_Memstore$" -timeout 20m {{ARGS}}
+    go test ./tests/eigenda/... -run "_Memstore$" -v -timeout 30m {{ARGS}}
 # Runs all tests that end with the `_Holesky` suffix, assuming the devnet was started with proxy using holesky backend.
 # To start the devnet with holesky backend, the files in eigenda-template-values/holesky-* need have secrets injected.
 # Take a look at how CI does it in .github/workflows/kurtosis-devnet.yml .
 [group('eigenda')]
 eigenda-devnet-test-holesky *ARGS="":
-    go test ./tests/eigenda/... -run "_Holesky$" -timeout 20m {{ARGS}}
+    go test ./tests/eigenda/... -run "_Holesky$" -v -timeout 40m {{ARGS}}
 [group('eigenda')]
 eigenda-devnet-add-tx-fuzzer ENCLAVE_NAME="eigenda-devnet" *ARGS="":
   kurtosis service add {{ENCLAVE_NAME}} tx-fuzz ethpandaops/tx-fuzz:master -- \
@@ -174,6 +174,14 @@ eigenda-devnet-configs ENCLAVE_NAME="eigenda-devnet":
   echo "PROXY MEMSTORE CONFIG:"
   PROXY_ENDPOINT=$(kurtosis port print {{ENCLAVE_NAME}} da-server-op-kurtosis http)
   curl $PROXY_ENDPOINT/memstore/config | jq
+  curl $PROXY_ENDPOINT/admin/eigenda-dispersal-backend | jq
+[group('eigenda')]
+eigenda-devnet-change-proxy-backend backend="V1" ENCLAVE_NAME="eigenda-devnet":
+  #!/usr/bin/env bash
+  PROXY_ENDPOINT=$(kurtosis port print {{ENCLAVE_NAME}} da-server-op-kurtosis http)
+  curl -X PUT $PROXY_ENDPOINT/admin/eigenda-dispersal-backend \
+    -H "Content-Type: application/json" \
+    -d '{"eigenDADispersalBackend": "{{backend}}"}'
 # We unfortunately have to restart the batcher in this ugly way right now just to change even a single flag.
 # This is b/c op's kurtosis setup right now is not idempotent so if we change a param in a values file
 # and rerun `just eigenda-devnet-start`, the entire devnet gets respun up which takes a long time.

kurtosis-devnet/tests/eigenda/batcher_logs_test.go

@@ -15,14 +15,16 @@ func TestBatcherFromLogs_Holesky(t *testing.T) {
 		t.Logf("TestBatcherFromLogs_Holesky needs a timeout of at least 15 minutes to run.")
 		t.FailNow()
 	}
+	harness := NewHarness(t)
 	// Batching time on Holesky can be up to 10 minutes, so we need long time to see a tx getting confirmed.
-	testBatcherFromLogs(t, 15*time.Minute)
+	testBatcherFromLogs(t, harness, 15*time.Minute)
 }
 
 func TestBatcherFromLogs_Memstore(t *testing.T) {
+	harness := NewHarness(t)
 	// 2 minutes is arbitrary here but should be long enough to observe interesting behavior using memstore.
 	// Also need to run the failover test which takes quite a while and can't run in parallel with these tests (or can it..?)
-	testBatcherFromLogs(t, 2*time.Minute)
+	testBatcherFromLogs(t, harness, 2*time.Minute)
 }
 
 // These tests are log driven. The batcher doesn't expose an API to query its state outside of logs and metrics,
@@ -32,13 +34,11 @@ func TestBatcherFromLogs_Memstore(t *testing.T) {
 // A better approach might be to generate txs from inside the golang test instead of relying on the external tx-fuzzer.
 // We could then increase traffic until the point where DA gets throttled, then change batcher parameters to increase blob size, etc.
 // Updating the batcher params is currently hard to do however; see comments above the eigenda-devnet-restart-batcher command in the justfile.
-func testBatcherFromLogs(t *testing.T, testTimeout time.Duration) {
+func testBatcherFromLogs(t *testing.T, harness *Harness, testTimeout time.Duration) {
 	// We stream logs for testTimeout, and run all the below tests in parallel (they read the same log outputs)
 	ctxWithTestTimeout, cancel := context.WithTimeout(context.Background(), testTimeout)
 	t.Cleanup(cancel)
 
-	harness := NewHarness(t)
-
 	// Make sure that no channel is ever timing out (fails to be sent to L1 in timely manner).
 	// Make sure the testsTimer is longer than max-channel-duration in the batcher config (found in the eigenda-template-values/ files).
 	// Currently max-channel-duration is set to 10 L1 blocks, meaning 10*6 seconds = 60 seconds.

kurtosis-devnet/tests/eigenda/failover_test.go

@@ -41,7 +41,7 @@ func TestFailoverToEthDACalldata_Memstore(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
 		t.Logf("Cleanup; Failing back to eigenda... resetting enclave proxy to start posting to eigenda again")
-		err := harness.Clients.ProxyMemconfigClient.Failback(ctx)
+		err := harness.Clients.ProxyClients.Failback(ctx)
 		if err != nil {
 			t.Logf("Error failing back... you might need to reset proxy to normal mode manually: %v", err)
 		}
@@ -61,13 +61,18 @@ func TestFailoverToEthDACalldata_Memstore(t *testing.T) {
 	require.GreaterOrEqual(t, harness.TestStartL1BlockNum, l1BlocksQueriedForBatcherTxs, "Test started too early in the chain")
 	fromBlock := harness.TestStartL1BlockNum - l1BlocksQueriedForBatcherTxs
 
+	eigenDABackend, err := harness.Clients.ProxyClients.GetDispersalBackend(ctxWithDeadline)
+	require.NoError(t, err)
+	DALayerEigenDA, err := certVersionToDALayer(eigenDABackend)
+	require.NoError(t, err)
+
 	// 1. Check that the original commitments are EigenDA
 	t.Logf("[Stage1: EigenDA] Checking that the initial commitments are EigenDA")
 	requireBatcherTxsToBeFromLayer(t, fromBlock, fromBlock+l1BlocksQueriedForBatcherTxs, DALayerEigenDA, harness.Endpoints.GethL1Endpoint, harness.BatchInboxAddr)
 
 	// 2. Failover and check that the commitments are now EthDACalldata
 	t.Logf("[Stage2: EthDA-Calldata] Failing over... changing proxy's config to return 503 errors")
-	err := harness.Clients.ProxyMemconfigClient.Failover(ctxWithDeadline)
+	err = harness.Clients.ProxyClients.Failover(ctxWithDeadline)
 	require.NoError(t, err)
 
 	afterFailoverFromBlockNum, err := harness.Clients.GethL1Client.BlockNumber(ctxWithDeadline)
@@ -90,7 +95,7 @@ func TestFailoverToEthDACalldata_Memstore(t *testing.T) {
 
 	// 3. Failback and check that the commitments are EigenDA again
 	t.Logf("[Stage3: EigenDA] Failing back... changing proxy's config to start processing PUT requests normally again")
-	err = harness.Clients.ProxyMemconfigClient.Failback(ctxWithDeadline)
+	err = harness.Clients.ProxyClients.Failback(ctxWithDeadline)
 	require.NoError(t, err)
 
 	afterFailbackFromBlockNum, err := harness.Clients.GethL1Client.BlockNumber(ctxWithDeadline)
@@ -115,6 +120,7 @@ func requireBatcherTxsToBeFromLayer(t *testing.T, fromBlockNum, toBlockNum uint6
 	wrongCommitmentsToDiscard := 0
 	for _, batcherTx := range batcherTxs {
 		if batcherTx.daLayer != expectedLayer {
+			t.Logf("Discarding batcher tx @ block %d with wrong commitment type %s (expected %s)", batcherTx.block, batcherTx.daLayer, expectedLayer)
 			wrongCommitmentsToDiscard++
 		}
 		// as soon as we see a commitment from expectedLayer, we stop discarding.
@@ -138,15 +144,28 @@ func requireBatcherTxsToBeFromLayer(t *testing.T, fromBlockNum, toBlockNum uint6
 // Note that 4844 txs are completely different and don't use normal txs with a prefix in the calldata,
 // see https://github.com/ethereum-optimism/optimism/blob/develop/op-node/rollup/derive/blob_data_source.go#L134-L137
 const ethDACalldataCommitmentPrefix = "0x00"
-const eigenDACommitmentPrefix = "0x010100"
+const eigenDAV1CommitmentPrefix = "0x01010000"
+const eigenDAV2CommitmentPrefix = "0x01010001"
 
 type DALayer string
 
 const (
 	DALayerEthCalldata DALayer = "ethda-calldata"
-	DALayerEigenDA     DALayer = "eigenda"
+	DALayerEigenDAV1   DALayer = "eigendaV1"
+	DALayerEigenDAV2   DALayer = "eigendaV2"
 )
 
+func certVersionToDALayer(certVersion EigenDACertVersion) (DALayer, error) {
+	switch certVersion {
+	case EigenDACertVersionV1:
+		return DALayerEigenDAV1, nil
+	case EigenDACertVersionV2:
+		return DALayerEigenDAV2, nil
+	default:
+		return "", fmt.Errorf("unknown EigenDA cert version: %s", certVersion)
+	}
+}
+
 type BatcherTx struct {
 	commitment string
 	daLayer    DALayer // commitment starts with respective prefix
@@ -241,19 +260,20 @@ func fetchBatcherTxs(gethL1Endpoint string, batchInbox string, fromBlockNum, toB
 	for _, block := range graphQLResp.Data.Blocks {
 		for _, tx := range block.Transactions {
 			if strings.EqualFold(tx.To.Address, batchInbox) {
-				var daLayer DALayer
-				if strings.HasPrefix(tx.InputData, eigenDACommitmentPrefix) {
-					daLayer = DALayerEigenDA
+				batcherTx := BatcherTx{
+					commitment: tx.InputData,
+					block:      uint64(tx.Block.Number),
+				}
+				if strings.HasPrefix(tx.InputData, eigenDAV1CommitmentPrefix) {
+					batcherTx.daLayer = DALayerEigenDAV1
+				} else if strings.HasPrefix(tx.InputData, eigenDAV2CommitmentPrefix) {
+					batcherTx.daLayer = DALayerEigenDAV2
 				} else if strings.HasPrefix(tx.InputData, ethDACalldataCommitmentPrefix) {
-					daLayer = DALayerEthCalldata
+					batcherTx.daLayer = DALayerEthCalldata
 				} else {
 					return nil, fmt.Errorf("unknown commitment prefix: %s", tx.InputData)
 				}
-				batcherTxs = append(batcherTxs, BatcherTx{
-					commitment: tx.InputData,
-					daLayer:    daLayer,
-					block:      uint64(tx.Block.Number),
-				})
+				batcherTxs = append(batcherTxs, batcherTx)
 			}
 		}
 	}