diff --git a/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg new file mode 100644 index 0000000000..41f584845a --- /dev/null +++ b/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg @@ -0,0 +1,12 @@ +BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000 + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" + diff --git a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg index 9040d6dc57..a0d80ce7ed 100644 --- a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg @@ -1,6 +1,7 @@ # Generated file, do not edit BUCKETLIST_DB_INDEX_CUTOFF = 20 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15 +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DATABASE = "sqlite3://stellar.db" DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = -1 diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core-old-version.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core-old-version.cfg new file mode 100644 index 0000000000..12fe5e36a5 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core-old-version.cfg @@ -0,0 +1,21 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg index 12fe5e36a5..c18ff3498a 100644 --- a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg @@ -1,5 +1,6 @@ # Generated file, do not edit BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DATABASE = "sqlite3://stellar.db" DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = -1 diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg index ec37e504fc..3e9772c91c 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg @@ -1,5 +1,6 @@ # Generated file, do not edit BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DATABASE = "sqlite3://stellar.db" DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = 0 diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg index fa25d69b98..14882a8ebd 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg index 4a971a19d2..b4c428495c 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg index 30159c150a..16dae50af7 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg index eab3b67bf8..38d84bd0a0 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg index 9c5fd26769..dcaa6aaf13 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg index b8aeff746c..098c268301 100644 --- a/ingest/ledgerbackend/testdata/expected-online-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 diff --git a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg index 8ecc0fe958..94812437d8 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg index 3960c1b546..8f2602b340 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg index 6b3830e6ff..7fb5a9c77e 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg index 93bb61b8be..3b04e0478e 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 diff --git a/ingest/ledgerbackend/testdata/expected-query-params.cfg b/ingest/ledgerbackend/testdata/expected-query-params.cfg index 86657a46a7..8a76da1a82 100644 --- a/ingest/ledgerbackend/testdata/expected-query-params.cfg +++ b/ingest/ledgerbackend/testdata/expected-query-params.cfg @@ -1,5 +1,6 @@ # Generated file, do not edit BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +BUCKETLIST_DB_MEMORY_FOR_CACHING = 0 DATABASE = "sqlite3://stellar.db" DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = -1 diff --git a/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg new file mode 100644 index 0000000000..96eb32cbe7 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg @@ -0,0 +1,22 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index b2d4d239d4..19f468df70 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" "github.com/stellar/go/support/errors" @@ -109,6 +110,7 @@ type captiveCoreTomlValues struct { HTTPQueryPort *uint `toml:"HTTP_QUERY_PORT,omitempty"` QueryThreadPoolSize *uint `toml:"QUERY_THREAD_POOL_SIZE,omitempty"` QuerySnapshotLedgers *uint `toml:"QUERY_SNAPSHOT_LEDGERS,omitempty"` + BucketListDBMemoryForCaching *uint `toml:"BUCKETLIST_DB_MEMORY_FOR_CACHING,omitempty"` } // QuorumSetIsConfigured returns true if there is a quorum set defined in the configuration. @@ -357,6 +359,8 @@ type CaptiveCoreTomlParams struct { EnforceSorobanTransactionMetaExtV1 bool // Fast HTTP Query Server parameters HTTPQueryServerParams *HTTPQueryServerParams + // CoreBuildVersionFn is a function that returns the build version of the stellar-core binary. + CoreBuildVersionFn CoreBuildVersionFunc } // NewCaptiveCoreTomlFromFile constructs a new CaptiveCoreToml instance by merging configuration @@ -454,6 +458,61 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) { return offline, nil } +// coreVersion helper struct identify a core version and provides the +// utilities to compare the version ( i.e. minor + major pair ) to a predefined +// version. +type coreVersion struct { + major int + minor int +} + +// greaterThanOrEqual compares the core version to a version specific. If unable +// to make the decision, the result is always "false", leaning toward the +// common denominator. +func (c coreVersion) greaterThanOrEqual(other coreVersion) bool { + if c.major == 0 && c.minor == 0 { + return false + } + return (c.major == other.major && c.minor >= other.minor) || (c.major > other.major) +} + +func (c *CaptiveCoreToml) checkCoreVersion(params CaptiveCoreTomlParams) coreVersion { + if params.CoreBinaryPath == "" { + return coreVersion{} + } + + getCoreVersion := params.CoreBuildVersionFn + if getCoreVersion == nil { + getCoreVersion = CoreBuildVersion + } + + versionRaw, err := getCoreVersion(params.CoreBinaryPath) + if err != nil { + return coreVersion{} + } + + var version [2]int + + re := regexp.MustCompile(`\D*(\d*)\.(\d*).*`) + versionStr := re.FindStringSubmatch(versionRaw) + if err == nil && len(versionStr) == 3 { + for i := 1; i < len(versionStr); i++ { + val, err := strconv.Atoi((versionStr[i])) + if err != nil { + break + } + version[i-1] = val + } + } + + return coreVersion{ + major: version[0], + minor: version[1], + } +} + +var minVersionForBucketlistCaching = coreVersion{major: 22, minor: 2} + func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { if params.UseDB && !c.tree.Has("DATABASE") { c.Database = "sqlite3://stellar.db" @@ -521,6 +580,17 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { c.QueryThreadPoolSize = &poolSize } + + if !c.tree.Has("BUCKETLIST_DB_MEMORY_FOR_CACHING") && + c.checkCoreVersion(params).greaterThanOrEqual(minVersionForBucketlistCaching) { + // set BUCKETLIST_DB_MEMORY_FOR_CACHING to 0 to disable allocation of + // memory for caching entries in BucketListDB. + // If we do not set BUCKETLIST_DB_MEMORY_FOR_CACHING, core will apply + // the default value which may result in additional 2-3 GB of memory usage + // in captive core + var disable uint = 0 + c.BucketListDBMemoryForCaching = &disable + } } func enforceOption(opt **bool) { diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index 9bee190d09..51ccf6f56e 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -243,6 +243,7 @@ func TestGenerateConfig(t *testing.T) { useDB bool enforceSorobanDiagnosticEvents bool enforceEmitMetaV1 bool + coreVersion string }{ { name: "offline config with no appendix", @@ -253,6 +254,7 @@ func TestGenerateConfig(t *testing.T) { peerPort: newUint(12345), logPath: nil, useDB: true, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config with no peer port", @@ -262,6 +264,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: nil, logPath: newString("/var/stellar-core/test.log"), + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "online config with appendix", @@ -271,6 +274,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: newUint(12345), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "online config with unsupported field in appendix", @@ -280,6 +284,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: newUint(12345), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "online config with no peer port", @@ -289,6 +294,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: nil, logPath: newString("/var/stellar-core/test.log"), + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "online config with no http port", @@ -298,6 +304,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: nil, peerPort: newUint(12345), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config with appendix", @@ -307,6 +314,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: newUint(12345), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config with extra fields in appendix", @@ -316,6 +324,7 @@ func TestGenerateConfig(t *testing.T) { httpPort: newUint(6789), peerPort: newUint(12345), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config with enforce diagnostic events and metav1", @@ -324,6 +333,7 @@ func TestGenerateConfig(t *testing.T) { logPath: nil, enforceSorobanDiagnosticEvents: true, enforceEmitMetaV1: true, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config disabling enforced diagnostic events and metav1", @@ -333,6 +343,7 @@ func TestGenerateConfig(t *testing.T) { logPath: nil, enforceSorobanDiagnosticEvents: true, enforceEmitMetaV1: true, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "online config with enforce diagnostic events and meta v1", @@ -344,6 +355,7 @@ func TestGenerateConfig(t *testing.T) { logPath: nil, enforceSorobanDiagnosticEvents: true, enforceEmitMetaV1: true, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "offline config with minimum persistent entry in appendix", @@ -351,6 +363,7 @@ func TestGenerateConfig(t *testing.T) { appendPath: filepath.Join("testdata", "appendix-with-minimum-persistent-entry.cfg"), expectedPath: filepath.Join("testdata", "expected-online-with-appendix-minimum-persistent-entry.cfg"), logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "default BucketlistDB config", @@ -359,6 +372,16 @@ func TestGenerateConfig(t *testing.T) { expectedPath: filepath.Join("testdata", "expected-default-bucketlistdb-core.cfg"), useDB: true, logPath: nil, + coreVersion: "v23.0.0-127-jb50f3f919", + }, + { + name: "default BucketlistDB config with older version", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "sample-appendix.cfg"), + expectedPath: filepath.Join("testdata", "expected-default-bucketlistdb-core-old-version.cfg"), + useDB: true, + logPath: nil, + coreVersion: "v22.1.0-123-hb50f3f219", }, { name: "BucketlistDB config in appendix", @@ -367,6 +390,7 @@ func TestGenerateConfig(t *testing.T) { expectedPath: filepath.Join("testdata", "expected-bucketlistdb-core.cfg"), useDB: true, logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, { name: "Query parameters in appendix", @@ -375,6 +399,16 @@ func TestGenerateConfig(t *testing.T) { expectedPath: filepath.Join("testdata", "expected-query-params.cfg"), useDB: true, logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", + }, + { + name: "BUCKETLIST_DB_MEMORY_FOR_CACHING in appendix", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "appendix-with-memory-for-bucketlist-caching.cfg"), + expectedPath: filepath.Join("testdata", "expected-with-memory-for-bucketlist-caching.cfg"), + useDB: true, + logPath: nil, + coreVersion: "v22.2.0-124-ga50f3f919", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -390,6 +424,10 @@ func TestGenerateConfig(t *testing.T) { UseDB: testCase.useDB, EnforceSorobanDiagnosticEvents: testCase.enforceSorobanDiagnosticEvents, EnforceSorobanTransactionMetaExtV1: testCase.enforceEmitMetaV1, + CoreBinaryPath: "stellar-core", + CoreBuildVersionFn: func(string) (string, error) { + return testCase.coreVersion, nil + }, } if testCase.appendPath != "" { captiveCoreToml, err = NewCaptiveCoreTomlFromFile(testCase.appendPath, params) @@ -419,6 +457,10 @@ func TestGenerateCoreConfigInMemory(t *testing.T) { HistoryArchiveURLs: []string{"http://localhost:1170"}, Strict: false, UseDB: false, + CoreBinaryPath: "stellar-core", + CoreBuildVersionFn: func(string) (string, error) { + return "v21.9.0-124-ga50f3f919", nil + }, } captiveCoreToml, err = NewCaptiveCoreTomlFromFile(appendPath, params) assert.NoError(t, err)