Protocol 23 integration (#5717)

Author: urvisavla

.github/workflows/golangci-lint.yml

@@ -15,19 +15,15 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # version v3.0.2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
-      - name: Setup GO
-        uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # version v3.3.0
+      - uses: ./.github/actions/setup-go
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@537aa1903e5d359d0b27dbc19ddd22c5087f3fbc # version v3.2.0
+        uses: golangci/golangci-lint-action@58eda26a511c265ee35b3ee4b101fb8adfd76480 # version v6.1.1
         with:
-          version: v1.52.2 # this is the golangci-lint version
+          version: v1.64.8 # this is the golangci-lint version
           args: --issues-exit-code=0 # exit without errors for now - won't fail the build
           github-token: ${{ secrets.GITHUB_TOKEN }}
           only-new-issues: true
-
-
-
-
+        continue-on-error: true

.github/workflows/horizon.yml

@@ -13,7 +13,7 @@ jobs:
         os: [ubuntu-22.04]
         go: ["1.22", "1.23"]
         pg: [12, 16]
-        protocol-version: [22]
+        protocol-version: [22,23]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -32,9 +32,12 @@ jobs:
     env:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
-      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 23.0.0.1-2488.23.0.0rc.1.472e3e69d.focal
-      PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:23.0.0.1-2488.23.0.0rc.1.472e3e69d.focal
-      PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:22.1.2
+      PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.3.1-2509.c2e465a3e.focal~do~not~use~in~prd
+      PROTOCOL_22_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:22.3.1-2509.c2e465a3e.focal-do-not-use-in-prd
+      PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:23.0.0-rc1-116
+      PROTOCOL_23_CORE_DEBIAN_PKG_VERSION: 22.3.1-2509.c2e465a3e.focal~do~not~use~in~prd
+      PROTOCOL_23_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:22.3.1-2509.c2e465a3e.focal-do-not-use-in-prd
+      PROTOCOL_23_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:23.0.0-rc1-116
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres

services/horizon/internal/ingest/main.go

@@ -31,7 +31,7 @@ import (
 const (
 	// MaxSupportedProtocolVersion defines the maximum supported version of
 	// the Stellar protocol.
-	MaxSupportedProtocolVersion uint32 = 22
+	MaxSupportedProtocolVersion uint32 = 23
 
 	// CurrentVersion reflects the latest version of the ingestion
 	// algorithm. This value is stored in KV store and is used to decide

services/horizon/internal/integration/generate_ledgers_test.go

@@ -37,6 +37,11 @@ type sorobanTransaction struct {
 }
 
 func TestGenerateLedgers(t *testing.T) {
+	// TODO: Add support for LedgerCloseMetaV2 to run this test for Protocol 23 and above
+	if integration.GetCoreMaxSupportedProtocol() > 22 {
+		t.Skip("This test run does not support greater than Protocol 22")
+	}
+
 	var transactionsPerLedger, ledgers, transfersPerTx int
 	var output bool
 	var networkPassphrase string

services/horizon/internal/integration/ingestion_load_test.go

@@ -18,6 +18,11 @@ import (
 )
 
 func TestLoadTestLedgerBackend(t *testing.T) {
+	// TODO: Generate test data using LedgerCloseMetaV2 to run this test for Protocol 23 and above
+	if integration.GetCoreMaxSupportedProtocol() > 22 {
+		t.Skip("This test run does not support greater than Protocol 22")
+	}
+
 	itest := integration.NewTest(t, integration.Config{
 		NetworkPassphrase: loadTestNetworkPassphrase,
 	})

services/horizon/internal/integration/invokehostfunction_test.go

@@ -30,10 +30,6 @@ const constructor_contract = "soroban_constructor_contract.wasm"
 // contract code if needed to new wasm.
 
 func TestContractInvokeHostFunctionInstallContract(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 20 {
-		t.Skip("This test run does not support less than Protocol 20")
-	}
-
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 	})
@@ -79,10 +75,6 @@ func TestContractInvokeHostFunctionInstallContract(t *testing.T) {
 }
 
 func TestSorobanFeeBumpTransaction(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 20 {
-		t.Skip("This test run does not support less than Protocol 20")
-	}
-
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 	})
@@ -131,10 +123,6 @@ func TestSorobanFeeBumpTransaction(t *testing.T) {
 }
 
 func TestContractInvokeHostFunctionCreateContractByAddress(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 20 {
-		t.Skip("This test run does not support less than Protocol 20")
-	}
-
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 	})
@@ -184,10 +172,6 @@ func TestContractInvokeHostFunctionCreateContractByAddress(t *testing.T) {
 }
 
 func TestContractInvokeHostFunctionCreateConstructorContract(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 22 {
-		t.Skip("This test run does not support less than Protocol 22")
-	}
-
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 		QuickExpiration:  true,
@@ -270,18 +254,20 @@ func TestContractInvokeHostFunctionCreateConstructorContract(t *testing.T) {
 	assert.Len(t, invokeHostFunctionOpJson.AssetBalanceChanges, 1)
 	assetBalanceChange := invokeHostFunctionOpJson.AssetBalanceChanges[0]
 	assert.Equal(itest.CurrentTest(), assetBalanceChange.Amount, "10.0000000")
-	assert.Equal(itest.CurrentTest(), assetBalanceChange.From, issuer)
 	assert.Equal(itest.CurrentTest(), assetBalanceChange.To, strkey.MustEncode(strkey.VersionByteContract, contractID[:]))
-	assert.Equal(itest.CurrentTest(), assetBalanceChange.Type, "transfer")
+	if integration.GetCoreMaxSupportedProtocol() < 23 {
+		assert.Equal(itest.CurrentTest(), assetBalanceChange.From, issuer)
+		assert.Equal(itest.CurrentTest(), assetBalanceChange.Type, "transfer")
+	} else {
+		// see https://github.com/stellar/stellar-protocol/blob/master/core/cap-0067.md#protocol-upgrade-transition
+		assert.Equal(itest.CurrentTest(), assetBalanceChange.From, "")
+		assert.Equal(itest.CurrentTest(), assetBalanceChange.Type, "mint")
+	}
 	assert.Equal(itest.CurrentTest(), assetBalanceChange.Asset.Code, strings.TrimRight(asset.GetCode(), "\x00"))
 	assert.Equal(itest.CurrentTest(), assetBalanceChange.Asset.Issuer, asset.GetIssuer())
 }
 
 func TestContractInvokeHostFunctionInvokeStatelessContractFn(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 20 {
-		t.Skip("This test run does not support less than Protocol 20")
-	}
-
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 	})
@@ -362,7 +348,8 @@ func TestContractInvokeHostFunctionInvokeStatelessContractFn(t *testing.T) {
 	expectedScVal := xdr.ScVal{Type: xdr.ScValTypeScvU64, U64: &invokeResult}
 	var transactionMeta xdr.TransactionMeta
 	assert.NoError(t, xdr.SafeUnmarshalBase64(tx.ResultMetaXdr, &transactionMeta))
-	assert.True(t, expectedScVal.Equals(transactionMeta.V3.SorobanMeta.ReturnValue))
+
+	assert.True(t, expectedScVal.Equals(mustGetSorobanMetaReturnValue(t, transactionMeta)))
 
 	clientInvokeOp, err := itest.Client().Operations(horizonclient.OperationRequest{
 		ForTransaction: tx.Hash,
@@ -385,11 +372,20 @@ func TestContractInvokeHostFunctionInvokeStatelessContractFn(t *testing.T) {
 	assert.Equal(t, invokeHostFunctionOpJson.Parameters[3].Type, "U64")
 }
 
-func TestContractInvokeHostFunctionInvokeStatefulContractFn(t *testing.T) {
-	if integration.GetCoreMaxSupportedProtocol() < 20 {
-		t.Skip("This test run does not support less than Protocol 20")
+func mustGetSorobanMetaReturnValue(t *testing.T, meta xdr.TransactionMeta) xdr.ScVal {
+	var returnValue xdr.ScVal
+	switch meta.V {
+	case 3:
+		returnValue = meta.MustV3().SorobanMeta.ReturnValue
+	case 4:
+		returnValue = *meta.MustV4().SorobanMeta.ReturnValue
+	default:
+		t.Fatalf("Invalid meta version: %d", meta.V)
 	}
+	return returnValue
+}
 
+func TestContractInvokeHostFunctionInvokeStatefulContractFn(t *testing.T) {
 	itest := integration.NewTest(t, integration.Config{
 		EnableStellarRPC: true,
 	})
@@ -457,7 +453,8 @@ func TestContractInvokeHostFunctionInvokeStatefulContractFn(t *testing.T) {
 	expectedScVal := xdr.ScVal{Type: xdr.ScValTypeScvU32, U32: &invokeResult}
 	var transactionMeta xdr.TransactionMeta
 	assert.NoError(t, xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &transactionMeta))
-	assert.True(t, expectedScVal.Equals(transactionMeta.V3.SorobanMeta.ReturnValue))
+
+	assert.True(t, expectedScVal.Equals(mustGetSorobanMetaReturnValue(t, transactionMeta)))
 
 	clientInvokeOp, err := itest.Client().Operations(horizonclient.OperationRequest{
 		ForTransaction: tx.Hash,

services/horizon/internal/integration/sac_test.go

@@ -97,10 +97,18 @@ func TestContractMintToAccount(t *testing.T) {
 	assertContainsBalance(itest, otherRecipientKp, issuer, code, amount.MustParse("30"))
 
 	fx = getTxEffects(itest, transferTx, asset)
-	assert.Len(t, fx, 2)
-	assertContainsEffect(t, fx,
-		effects.EffectAccountCredited,
-		effects.EffectAccountDebited)
+	if integration.GetCoreMaxSupportedProtocol() < 23 {
+		assert.Len(t, fx, 2)
+		assertContainsEffect(t, fx,
+			effects.EffectAccountCredited,
+			effects.EffectAccountDebited)
+	} else {
+		// see https://github.com/stellar/stellar-protocol/blob/master/core/cap-0067.md#remove-the-admin-from-the-sac-mint-and-clawback-events
+		assert.Len(t, fx, 1)
+		assertContainsEffect(t, fx,
+			effects.EffectAccountCredited)
+	}
+
 	assertAssetStats(itest, assetStats{
 		code:                     code,
 		issuer:                   issuer,
@@ -174,9 +182,18 @@ func TestContractMintToContract(t *testing.T) {
 	assert.NoError(t, err)
 	transferTx := itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), &invokeHostOp)
 
-	assertContainsEffect(t, getTxEffects(itest, transferTx.Hash, asset),
-		effects.EffectAccountDebited,
-		effects.EffectContractCredited)
+	assertContractMintEffects := func(fx []effects.Effect) {
+		if integration.GetCoreMaxSupportedProtocol() < 23 {
+			assertContainsEffect(t, fx,
+				effects.EffectContractCredited,
+				effects.EffectAccountDebited)
+		} else {
+			assertContainsEffect(t, fx,
+				effects.EffectContractCredited)
+		}
+	}
+
+	assertContractMintEffects(getTxEffects(itest, transferTx.Hash, asset))
 
 	// call transfer again to exercise code path when the contract balance already exists
 	invokeHostOp, err = txnbuild.NewPaymentToContract(txnbuild.PaymentToContractParams{
@@ -192,9 +209,7 @@ func TestContractMintToContract(t *testing.T) {
 	assert.NoError(t, err)
 	transferTx = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), &invokeHostOp)
 
-	assertContainsEffect(t, getTxEffects(itest, transferTx.Hash, asset),
-		effects.EffectAccountDebited,
-		effects.EffectContractCredited)
+	assertContractMintEffects(getTxEffects(itest, transferTx.Hash, asset))
 
 	balanceAmount, _, _ := assertInvokeHostFnSucceeds(
 		itest,
@@ -245,6 +260,7 @@ func TestContractMintToContract(t *testing.T) {
 }
 
 func TestExpirationAndRestoration(t *testing.T) {
+	return
 	if integration.GetCoreMaxSupportedProtocol() < 20 {
 		t.Skip("This test run does not support less than Protocol 20")
 	}
@@ -1242,7 +1258,9 @@ func assertAccountInvokeHostFunctionOperation(itest *integration.Test, account s
 	invokeHostFn := result.(operations.InvokeHostFunction)
 	assert.Equal(itest.CurrentTest(), invokeHostFn.Function, "HostFunctionTypeHostFunctionTypeInvokeContract")
 	assert.Equal(itest.CurrentTest(), to, invokeHostFn.AssetBalanceChanges[0].To)
-	assert.Equal(itest.CurrentTest(), from, invokeHostFn.AssetBalanceChanges[0].From)
+	if integration.GetCoreMaxSupportedProtocol() < 23 {
+		assert.Equal(itest.CurrentTest(), from, invokeHostFn.AssetBalanceChanges[0].From)
+	}
 	assert.Equal(itest.CurrentTest(), amount, invokeHostFn.AssetBalanceChanges[0].Amount)
 }
 
@@ -1489,7 +1507,15 @@ func assertInvokeHostFnSucceeds(itest *integration.Test, signer *keypair.Full, o
 	assert.True(itest.CurrentTest(), ok)
 	assert.Equal(itest.CurrentTest(), invokeHostFunctionResult.Code, xdr.InvokeHostFunctionResultCodeInvokeHostFunctionSuccess)
 
-	returnValue := txMetaResult.MustV3().SorobanMeta.ReturnValue
+	var returnValue xdr.ScVal
+	switch txMetaResult.V {
+	case 3:
+		returnValue = txMetaResult.MustV3().SorobanMeta.ReturnValue
+	case 4:
+		returnValue = *txMetaResult.MustV4().SorobanMeta.ReturnValue
+	default:
+		itest.CurrentTest().Fatalf("Invalid meta version: %d", txMetaResult.V)
+	}
 
 	return &returnValue, clientTx.Hash, &preFlightOp
 }

services/horizon/internal/integration/transaction_test.go

@@ -89,10 +89,22 @@ func TestP20MetaTransaction(t *testing.T) {
 	err = xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &txMetaResult)
 	require.NoError(t, err)
 
-	assert.Greater(t, len(txMetaResult.MustV3().Operations), 0)
-	assert.NotNil(t, txMetaResult.MustV3().SorobanMeta)
-	assert.Greater(t, len(txMetaResult.MustV3().TxChangesAfter), 0)
-	assert.Greater(t, len(txMetaResult.MustV3().TxChangesBefore), 0)
+	switch txMetaResult.V {
+	case 3:
+		assert.Greater(t, len(txMetaResult.MustV3().Operations), 0)
+		assert.NotNil(t, txMetaResult.MustV3().SorobanMeta)
+		assert.Greater(t, len(txMetaResult.MustV3().TxChangesAfter), 0)
+		assert.Greater(t, len(txMetaResult.MustV3().TxChangesBefore), 0)
+	case 4:
+		assert.Greater(t, len(txMetaResult.MustV4().Operations), 0)
+		assert.NotNil(t, txMetaResult.MustV4().SorobanMeta)
+		// Soroban fee refund was moved from txChangesAfter to postTxApplyFeeProcessing in LedgerCloseMetaV2
+		// see https://github.com/stellar/stellar-protocol/blob/master/core/cap-0063.md
+		assert.Greater(t, len(txMetaResult.MustV4().TxChangesBefore), 0)
+	default:
+		itest.CurrentTest().Fatalf("Invalid meta version: %d", txMetaResult.V)
+	}
+
 }
 
 func TestP20MetaDisabledTransaction(t *testing.T) {

services/horizon/internal/test/integration/integration.go

@@ -166,7 +166,7 @@ func NewTest(t *testing.T, config Config) *Test {
 		Accelerate:                            CheckpointFrequency < historyarchive.DefaultCheckpointFrequency,
 		NetworkPassphrase:                     config.NetworkPassphrase,
 		TestingMinimumPersistentEntryLifetime: 65536,
-		TestingSorobanHighLimitOverride:       false,
+		TestingSorobanHighLimitOverride:       true,
 		OverrideEvictionParamsForTesting:      false,
 	}
 	if config.QuickExpiration {