add optional postgresql support for stellar core

Author: leighmcculloch

README.md

@@ -205,6 +205,22 @@ curl "http://localhost:11626/ll?level=DEBUG&partition=SCP"
 
 Available partitions: `Fs`, `SCP`, `Bucket`, `Database`, `History`, `Process`, `Ledger`, `Overlay`, `Herder`, `Tx`, `LoadGen`, `Work`, `Invariant`, `Perf`
 
+### Core Database Options
+
+By default, Stellar Core uses SQLite for its database. To use PostgreSQL instead, set the `CORE_USE_POSTGRES` environment variable:
+
+```shell
+docker run -e CORE_USE_POSTGRES=true -p "8000:8000" stellar/quickstart --local
+```
+
+When `CORE_USE_POSTGRES=true`:
+- A `core` PostgreSQL database is created alongside the `horizon` database
+- Stellar Core connects to PostgreSQL instead of SQLite
+- PostgreSQL is started before Stellar Core
+
+> [!WARNING]
+> PostgreSQL support for Stellar Core in Quickstart is deprecated and will likely be removed in a future release. It is highly recommended not to use this feature if you are not already using PostgreSQL. If you are currently using PostgreSQL for Stellar Core in Quickstart, please comment on [this issue](https://github.com/stellar/quickstart/issues/875) with information about your use case for which PostgreSQL is necessary.
+
 ### Stellar Lab
 
 Stellar Lab is an interactive toolkit for exploring and interacting with the Stellar network. It allows developers to build, sign, simulate, and submit transactions, and to make requests to both the Friendbot, RPC, and Horizon APIs. Lab is also built-in to Quickstart.
@@ -495,7 +511,7 @@ The point of this project is to make running stellar's software within your own
 
 This image manages a postgres database `horizon` for horizon's data. The username to use when connecting with your postgresql client or library is `stellar`. The password to use is dependent upon the mode your container is running in: Persistent mode uses a password supplied by you and ephemeral mode generates a password and prints it to the console upon container startup.
 
-Note: In the past, stellar-core also used PostgreSQL with a `core` database. Stellar-core now uses SQLite, so PostgreSQL is only started when Horizon is enabled.
+Note: By default, stellar-core uses SQLite. To use PostgreSQL for stellar-core, set `CORE_USE_POSTGRES=true` (see [Core Database Options](#core-database-options)). When using PostgreSQL, a `core` database is also created.
 
 ## Example launch commands
 

common/core/bin/start

@@ -5,6 +5,14 @@ if [ -f /opt/stellar/core/etc/env ]; then
   source /opt/stellar/core/etc/env
 fi
 
+# Wait for postgres if using postgres for core database
+if [ "$CORE_USE_POSTGRES" = "true" ]; then
+  while ! psql -U stellar -c 'select 1' core &> /dev/null ; do
+    echo "Waiting for postgres to be available..."
+    sleep 1
+  done
+fi
+
 echo "starting core..."
 set -e
 exec /usr/bin/stellar-core --conf "/opt/stellar/core/etc/stellar-core.cfg" run ${CORE_LOG_LEVEL:+--ll "$CORE_LOG_LEVEL"}

futurenet/core/etc/stellar-core.cfg

@@ -4,7 +4,7 @@ LOG_FILE_PATH="/var/log/stellar-core/stellar-core-{datetime:%Y-%m-%d_%H-%M-%S}.l
 MANUAL_CLOSE=__MANUAL_CLOSE__
 
 NETWORK_PASSPHRASE="__NETWORK__"
-DATABASE="sqlite3:///opt/stellar/core/stellar.db"
+DATABASE="__DATABASE__"
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 CATCHUP_RECENT=100

local/core/etc/stellar-core.cfg

@@ -11,7 +11,7 @@ NETWORK_PASSPHRASE="__NETWORK__"
 NODE_SEED="SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2 self"
 NODE_IS_VALIDATOR=true
 
-DATABASE="sqlite3:///opt/stellar/core/stellar.db"
+DATABASE="__DATABASE__"
 
 FAILURE_SAFETY=0
 UNSAFE_QUORUM=true

pubnet/core/etc/stellar-core.cfg

@@ -3,7 +3,7 @@ PUBLIC_HTTP_PORT=true
 LOG_FILE_PATH="/var/log/stellar-core/stellar-core-{datetime:%Y-%m-%d_%H-%M-%S}.log"
 MANUAL_CLOSE=__MANUAL_CLOSE__
 
-DATABASE="sqlite3:///opt/stellar/core/stellar.db"
+DATABASE="__DATABASE__"
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 CATCHUP_RECENT=100
 

start

@@ -43,6 +43,7 @@ export PROTOCOL_VERSION_DEFAULT="$(< /image.json jq -r '.config.protocol_version
 : "${ENABLE_RPC_ADMIN_ENDPOINT:=$ENABLE_SOROBAN_RPC_ADMIN_ENDPOINT}"
 : "${ENABLE_CORE_MANUAL_CLOSE:=false}"
 : "${CORE_LOG_LEVEL:=}"
+: "${CORE_USE_POSTGRES:=}"
 : "${LIMITS:=testnet}"
 
 QUICKSTART_INITIALIZED=false
@@ -409,7 +410,8 @@ function copy_pgpass() {
 }
 
 function init_db() {
-  if [ "$ENABLE_HORIZON" != "true" ]; then
+  # Need postgres if horizon is enabled, or if core is enabled and using postgres
+  if [ "$ENABLE_HORIZON" != "true" ] && { [ "$ENABLE_CORE" != "true" ] || [ "$CORE_USE_POSTGRES" != "true" ]; }; then
     return 0
   fi
   if [ -f $PGHOME/.quickstart-initialized ]; then
@@ -441,10 +443,16 @@ function init_db() {
   run_silent "init-postgres" sudo -u postgres $PGBIN/initdb -D $PGDATA
 
   start_postgres
-  run_silent "create-horizon-db" sudo -u postgres createdb horizon
+  if [ "$ENABLE_HORIZON" = "true" ]; then
+    run_silent "create-horizon-db" sudo -u postgres createdb horizon
+  fi
+  if [ "$ENABLE_CORE" = "true" ] && [ "$CORE_USE_POSTGRES" = "true" ]; then
+    run_silent "create-core-db" sudo -u postgres createdb core
+  fi
   run_silent "stellar-postgres-user" sudo -u postgres psql <<-SQL
     CREATE USER $PGUSER WITH PASSWORD '$PGPASS';
-    GRANT ALL PRIVILEGES ON DATABASE horizon to $PGUSER;
+    $([ "$ENABLE_HORIZON" = "true" ] && echo "GRANT ALL PRIVILEGES ON DATABASE horizon to $PGUSER;")
+    $([ "$ENABLE_CORE" = "true" ] && [ "$CORE_USE_POSTGRES" = "true" ] && echo "GRANT ALL PRIVILEGES ON DATABASE core to $PGUSER;")
 SQL
 
   touch .quickstart-initialized
@@ -471,6 +479,7 @@ function init_stellar_core() {
   fi
   cat > $COREHOME/etc/env <<EOF
 CORE_LOG_LEVEL="$core_log_level"
+CORE_USE_POSTGRES="$CORE_USE_POSTGRES"
 EOF
 
   if [ -f $COREHOME/.quickstart-initialized ]; then
@@ -487,6 +496,15 @@ EOF
 
   run_silent "finalize-core-config-manual-close" perl -pi -e "s/__MANUAL_CLOSE__/$ENABLE_CORE_MANUAL_CLOSE/g" etc/stellar-core.cfg
 
+  # Set database based on CORE_USE_POSTGRES environment variable
+  if [ "$CORE_USE_POSTGRES" = "true" ]; then
+    local DATABASE_URL="postgresql://dbname=core host=localhost user=stellar password=$PGPASS"
+    start_postgres
+  else
+    local DATABASE_URL="sqlite3:///opt/stellar/core/stellar.db"
+  fi
+  run_silent "finalize-core-config-database" perl -pi -e "s|__DATABASE__|$DATABASE_URL|g" etc/stellar-core.cfg
+
   run_silent "init-core-db" sudo -u stellar stellar-core new-db --conf etc/stellar-core.cfg
 
   if [ "$NETWORK" = "local" ]; then
@@ -700,6 +718,9 @@ function start_optional_services() {
   done
 
   if [ "$ENABLE_CORE" == "true" ]; then
+    if [ "$CORE_USE_POSTGRES" == "true" ]; then
+      supervisorctl start postgresql
+    fi
     supervisorctl start stellar-core
   fi
 

testnet/core/etc/stellar-core.cfg

@@ -4,7 +4,7 @@ LOG_FILE_PATH="/var/log/stellar-core/stellar-core-{datetime:%Y-%m-%d_%H-%M-%S}.l
 MANUAL_CLOSE=__MANUAL_CLOSE__
 
 NETWORK_PASSPHRASE="__NETWORK__"
-DATABASE="sqlite3:///opt/stellar/core/stellar.db"
+DATABASE="__DATABASE__"
 CATCHUP_RECENT=100
 
 UNSAFE_QUORUM=true