ingest/ledgerbackend: Add core configuration support for BUCKETLIST_DB_MEMORY_FOR_CACHING (#5632)

tamirms · web-flow · commit 10ea13dded37 · 2025-03-26T13:16:31.000Z
diff --git a/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg
@@ -0,0 +1,12 @@
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"
+
diff --git a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg
@@ -1,6 +1,7 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_CUTOFF = 20
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core-old-version.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core-old-version.cfg
@@ -0,0 +1,21 @@
+# Generated file, do not edit
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+DATABASE = "sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"
diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
diff --git a/ingest/ledgerbackend/testdata/expected-query-params.cfg b/ingest/ledgerbackend/testdata/expected-query-params.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg
@@ -0,0 +1,22 @@
+# Generated file, do not edit
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000
+DATABASE = "sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/stellar/go/support/errors"
@@ -109,6 +110,7 @@ type captiveCoreTomlValues struct {
 	HTTPQueryPort                         *uint                `toml:"HTTP_QUERY_PORT,omitempty"`
 	QueryThreadPoolSize                   *uint                `toml:"QUERY_THREAD_POOL_SIZE,omitempty"`
 	QuerySnapshotLedgers                  *uint                `toml:"QUERY_SNAPSHOT_LEDGERS,omitempty"`
+	BucketListDBMemoryForCaching          *uint                `toml:"BUCKETLIST_DB_MEMORY_FOR_CACHING,omitempty"`
 }
 
 // QuorumSetIsConfigured returns true if there is a quorum set defined in the configuration.
@@ -357,6 +359,8 @@ type CaptiveCoreTomlParams struct {
 	EnforceSorobanTransactionMetaExtV1 bool
 	// Fast HTTP Query Server parameters
 	HTTPQueryServerParams *HTTPQueryServerParams
+	// CoreBuildVersionFn is a function that returns the build version of the stellar-core binary.
+	CoreBuildVersionFn CoreBuildVersionFunc
 }
 
 // NewCaptiveCoreTomlFromFile constructs a new CaptiveCoreToml instance by merging configuration
@@ -454,6 +458,61 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) {
 	return offline, nil
 }
 
+// coreVersion helper struct identify a core version and provides the
+// utilities to compare the version ( i.e. minor + major pair ) to a predefined
+// version.
+type coreVersion struct {
+	major int
+	minor int
+}
+
+// greaterThanOrEqual compares the core version to a version specific. If unable
+// to make the decision, the result is always "false", leaning toward the
+// common denominator.
+func (c coreVersion) greaterThanOrEqual(other coreVersion) bool {
+	if c.major == 0 && c.minor == 0 {
+		return false
+	}
+	return (c.major == other.major && c.minor >= other.minor) || (c.major > other.major)
+}
+
+func (c *CaptiveCoreToml) checkCoreVersion(params CaptiveCoreTomlParams) coreVersion {
+	if params.CoreBinaryPath == "" {
+		return coreVersion{}
+	}
+
+	getCoreVersion := params.CoreBuildVersionFn
+	if getCoreVersion == nil {
+		getCoreVersion = CoreBuildVersion
+	}
+
+	versionRaw, err := getCoreVersion(params.CoreBinaryPath)
+	if err != nil {
+		return coreVersion{}
+	}
+
+	var version [2]int
+
+	re := regexp.MustCompile(`\D*(\d*)\.(\d*).*`)
+	versionStr := re.FindStringSubmatch(versionRaw)
+	if err == nil && len(versionStr) == 3 {
+		for i := 1; i < len(versionStr); i++ {
+			val, err := strconv.Atoi((versionStr[i]))
+			if err != nil {
+				break
+			}
+			version[i-1] = val
+		}
+	}
+
+	return coreVersion{
+		major: version[0],
+		minor: version[1],
+	}
+}
+
+var minVersionForBucketlistCaching = coreVersion{major: 22, minor: 2}
+
 func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 	if params.UseDB && !c.tree.Has("DATABASE") {
 		c.Database = "sqlite3://stellar.db"
@@ -521,6 +580,17 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 		c.QueryThreadPoolSize = &poolSize
 
 	}
+
+	if !c.tree.Has("BUCKETLIST_DB_MEMORY_FOR_CACHING") &&
+		c.checkCoreVersion(params).greaterThanOrEqual(minVersionForBucketlistCaching) {
+		// set BUCKETLIST_DB_MEMORY_FOR_CACHING to 0 to disable allocation of
+		// memory for caching entries in BucketListDB.
+		// If we do not set BUCKETLIST_DB_MEMORY_FOR_CACHING, core will apply
+		// the default value which may result in additional 2-3 GB of memory usage
+		// in captive core
+		var disable uint = 0
+		c.BucketListDBMemoryForCaching = &disable
+	}
 }
 
 func enforceOption(opt **bool) {
diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`# Generated file, do not edit`
	`2`	`+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0`
`2`	`3`	`DEPRECATED_SQL_LEDGER_STATE = true`
`3`	`4`	`EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true`
`4`	`5`	`ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true`