Add core configuration support for BUCKETLIST_DB_MEMORY_FOR_CACHING

tamirms · tamirms · commit fc761d30aeda · 2025-03-18T18:47:16.000Z
diff --git a/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/appendix-with-memory-for-bucketlist-caching.cfg
@@ -0,0 +1,12 @@
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"
+
diff --git a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg
@@ -1,6 +1,7 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_CUTOFF = 20
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg b/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg
@@ -1,4 +1,5 @@
 # Generated file, do not edit
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
diff --git a/ingest/ledgerbackend/testdata/expected-query-params.cfg b/ingest/ledgerbackend/testdata/expected-query-params.cfg
@@ -1,5 +1,6 @@
 # Generated file, do not edit
 BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0
 DATABASE = "sqlite3://stellar.db"
 DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = -1
diff --git a/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg b/ingest/ledgerbackend/testdata/expected-with-memory-for-bucketlist-caching.cfg
@@ -0,0 +1,22 @@
+# Generated file, do not edit
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+BUCKETLIST_DB_MEMORY_FOR_CACHING = 3000
+DATABASE = "sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"
diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go
@@ -109,6 +109,7 @@ type captiveCoreTomlValues struct {
 	HTTPQueryPort                         *uint                `toml:"HTTP_QUERY_PORT,omitempty"`
 	QueryThreadPoolSize                   *uint                `toml:"QUERY_THREAD_POOL_SIZE,omitempty"`
 	QuerySnapshotLedgers                  *uint                `toml:"QUERY_SNAPSHOT_LEDGERS,omitempty"`
+	BucketListDBMemoryForCaching          *uint                `toml:"BUCKETLIST_DB_MEMORY_FOR_CACHING,omitempty"`
 }
 
 // QuorumSetIsConfigured returns true if there is a quorum set defined in the configuration.
@@ -521,6 +522,16 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 		c.QueryThreadPoolSize = &poolSize
 
 	}
+
+	if !c.tree.Has("BUCKETLIST_DB_MEMORY_FOR_CACHING") {
+		// set BUCKETLIST_DB_MEMORY_FOR_CACHING to 0 to disable allocation of
+		// memory for caching entries in BucketListDB.
+		// If we do not set BUCKETLIST_DB_MEMORY_FOR_CACHING, core will apply
+		// the default value which may result in additional 2-3 GB of memory usage
+		// in captive core
+		var disable uint = 0
+		c.BucketListDBMemoryForCaching = &disable
+	}
 }
 
 func enforceOption(opt **bool) {
diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go
@@ -376,6 +376,14 @@ func TestGenerateConfig(t *testing.T) {
 			useDB:        true,
 			logPath:      nil,
 		},
+		{
+			name:         "BUCKETLIST_DB_MEMORY_FOR_CACHING in appendix",
+			mode:         stellarCoreRunnerModeOnline,
+			appendPath:   filepath.Join("testdata", "appendix-with-memory-for-bucketlist-caching.cfg"),
+			expectedPath: filepath.Join("testdata", "expected-with-memory-for-bucketlist-caching.cfg"),
+			useDB:        true,
+			logPath:      nil,
+		},
 	} {
 		t.Run(testCase.name, func(t *testing.T) {
 			var err error

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`# Generated file, do not edit`
	`2`	`+BUCKETLIST_DB_MEMORY_FOR_CACHING = 0`
`2`	`3`	`DEPRECATED_SQL_LEDGER_STATE = true`
`3`	`4`	`FAILURE_SAFETY = -1`
`4`	`5`	`HTTP_PORT = 11626`