Skip to content

feat(scripts): add LSM support as UTXO_BACKEND option #94

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mkoura merged 2 commits into from
Dec 12, 2025
Merged

feat(scripts): add LSM support as UTXO_BACKEND option #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1b99296
feat(scripts): add LSM support as UTXO_BACKEND option
mkoura Dec 12, 2025
615bf79
feat(scripts): support empty and mixed UTXO backends
mkoura Dec 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 40 additions & 9 deletions src/cardonnay_scripts/scripts/common/common-start-fast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ initialize_globals() {
fi

case "${UTXO_BACKEND:=""}" in
"" | mem | disk)
"" | mem | disk | disklmdb | empty)
;;
*)
echo "Unknown \`UTXO_BACKEND\`: '$UTXO_BACKEND', line $LINENO in ${BASH_SOURCE[0]}" >&2
Expand All @@ -65,6 +65,21 @@ initialize_globals() {
esac
readonly UTXO_BACKEND

local backend
UTXO_BACKENDS=()
for backend in ${MIXED_UTXO_BACKENDS:=""}; do
case "$backend" in
mem | disk | disklmdb | empty)
UTXO_BACKENDS+=("$backend")
;;
*)
echo "Unknown \`MIXED_UTXO_BACKENDS\` entry: '$backend', line $LINENO in ${BASH_SOURCE[0]}" >&2
exit 1
;;
esac
done
readonly UTXO_BACKENDS

SECURITY_PARAM="$(jq '.securityParam' < "${SCRIPT_DIR}/genesis.spec.json")"
readonly SECURITY_PARAM
NETWORK_MAGIC="$(jq '.networkMagic' < "${SCRIPT_DIR}/genesis.spec.json")"
Expand Down Expand Up @@ -187,22 +202,39 @@ get_genesis_data() {

edit_node_configs() {
local live_tables_base="${STATE_CLUSTER_NAME}/lmdb"
local conf fname node_name pool_num
local conf fname node_name pool_num utxo_backend

for conf in "$SCRIPT_DIR"/config-*.json; do
[ -e "$conf" ] || { echo "No config files found in ${SCRIPT_DIR}, line $LINENO in ${BASH_SOURCE[0]}" >&2; exit 1; }
fname="${conf##*/}"
node_name="${fname##config-}"
node_name="${node_name%.json}"

if [ "$fname" = "config-bft1.json" ]; then
pool_num=""
else
pool_num="${fname##*-pool}"
pool_num="${pool_num%.json}"
fi

utxo_backend="${UTXO_BACKEND}"
# Rotate through the mixed backends for block producing nodes, if set.
if [ -n "$pool_num" ] && [ "${#UTXO_BACKENDS[@]}" -gt 0 ]; then
index=$(( (pool_num - 1) % ${#UTXO_BACKENDS[@]} ))
utxo_backend="${UTXO_BACKENDS[$index]}"
fi
if [ "$utxo_backend" = "empty" ]; then
utxo_backend=""
fi

jq \
--arg byron_hash "$BYRON_GENESIS_HASH" \
--arg shelley_hash "$SHELLEY_GENESIS_HASH" \
--arg alonzo_hash "$ALONZO_GENESIS_HASH" \
--arg conway_hash "$CONWAY_GENESIS_HASH" \
--arg dijkstra_hash "$DIJKSTRA_GENESIS_HASH" \
--argjson prot_ver "$PROTOCOL_VERSION" \
--arg backend "$UTXO_BACKEND" \
--arg backend "$utxo_backend" \
--arg live_tables_path "${live_tables_base}-${node_name}" \
'.ByronGenesisHash = $byron_hash
| .ShelleyGenesisHash = $shelley_hash
Expand All @@ -223,12 +255,13 @@ edit_node_configs() {
| ."LastKnownBlockVersion-Major" = $prot_ver
| if $backend == "mem" then
(.LedgerDB.Backend = "V2InMemory"
| .LedgerDB.NumOfDiskSnapshots = 2
| .LedgerDB.QueryBatchSize = 100000
| .LedgerDB.SnapshotInterval = 216)
elif $backend == "disk" then
.LedgerDB.Backend = "V2LSM"
elif $backend == "disklmdb" then
(.LedgerDB.Backend = "V1LMDB"
| .LedgerDB.LiveTablesPath = $live_tables_path)
| .LedgerDB.LiveTablesPath = $live_tables_path
| .LedgerDB.SnapshotInterval = 300)
elif has("LedgerDB") then
.LedgerDB |= del(.Backend)
else
Expand All @@ -240,13 +273,11 @@ edit_node_configs() {
if [ -z "${ENABLE_LEGACY:-}" ]; then

if [ -n "${MIXED_P2P:-}" ]; then
if [ "$fname" = "config-bft1.json" ]; then
if [ -z "$pool_num" ]; then
cp -f "$SCRIPT_DIR"/topology-bft1.json "$STATE_CLUSTER"
continue
fi

pool_num="${fname##*-pool}"
pool_num="${pool_num%.json}"
if [ "$((pool_num % 2))" != 0 ]; then
cp -f "${SCRIPT_DIR}/topology-pool${pool_num}.json" "$STATE_CLUSTER"
continue
Expand Down
50 changes: 41 additions & 9 deletions src/cardonnay_scripts/scripts/common/common-start-slow
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ initialize_globals() {
fi

case "${UTXO_BACKEND:=""}" in
"" | mem | disk)
"" | mem | disk | disklmdb | empty)
;;
*)
echo "Unknown \`UTXO_BACKEND\`: '$UTXO_BACKEND', line $LINENO in ${BASH_SOURCE[0]}" >&2
Expand All @@ -71,6 +71,21 @@ initialize_globals() {
esac
readonly UTXO_BACKEND

local backend
UTXO_BACKENDS=()
for backend in ${MIXED_UTXO_BACKENDS:=""}; do
case "$backend" in
mem | disk | disklmdb | empty)
UTXO_BACKENDS+=("$backend")
;;
*)
echo "Unknown \`MIXED_UTXO_BACKENDS\` entry: '$backend', line $LINENO in ${BASH_SOURCE[0]}" >&2
exit 1
;;
esac
done
readonly UTXO_BACKENDS

SECURITY_PARAM="$(jq '.securityParam' < "${SCRIPT_DIR}/genesis.spec.json")"
readonly SECURITY_PARAM
NETWORK_MAGIC="$(jq '.networkMagic' < "${SCRIPT_DIR}/genesis.spec.json")"
Expand Down Expand Up @@ -189,21 +204,39 @@ get_genesis_data() {

edit_node_configs() {
local live_tables_base="${STATE_CLUSTER_NAME}/lmdb"
local conf fname node_name pool_num
local conf fname node_name pool_num utxo_backend

for conf in "$SCRIPT_DIR"/config-*.json; do
[ -e "$conf" ] || { echo "No config files found in ${SCRIPT_DIR}, line $LINENO in ${BASH_SOURCE[0]}" >&2; exit 1; }
fname="${conf##*/}"
node_name="${fname##config-}"
node_name="${node_name%.json}"

if [ "$fname" = "config-bft1.json" ]; then
pool_num=""
else
pool_num="${fname##*-pool}"
pool_num="${pool_num%.json}"
fi

utxo_backend="${UTXO_BACKEND}"
# Rotate through the mixed backends for block producing nodes, if set.
if [ -n "$pool_num" ] && [ "${#UTXO_BACKENDS[@]}" -gt 0 ]; then
index=$(( (pool_num - 1) % ${#UTXO_BACKENDS[@]} ))
utxo_backend="${UTXO_BACKENDS[$index]}"
fi
if [ "$utxo_backend" = "empty" ]; then
utxo_backend=""
fi


jq \
--arg byron_hash "$BYRON_GENESIS_HASH" \
--arg shelley_hash "$SHELLEY_GENESIS_HASH" \
--arg alonzo_hash "$ALONZO_GENESIS_HASH" \
--arg conway_hash "$CONWAY_GENESIS_HASH" \
--arg dijkstra_hash "$DIJKSTRA_GENESIS_HASH" \
--arg backend "$UTXO_BACKEND" \
--arg backend "$utxo_backend" \
--arg live_tables_path "${live_tables_base}-${node_name}" \
'.ByronGenesisHash = $byron_hash
| .ShelleyGenesisHash = $shelley_hash
Expand All @@ -219,12 +252,13 @@ edit_node_configs() {
end
| if $backend == "mem" then
(.LedgerDB.Backend = "V2InMemory"
| .LedgerDB.NumOfDiskSnapshots = 2
| .LedgerDB.QueryBatchSize = 100000
| .LedgerDB.SnapshotInterval = 216)
elif $backend == "disk" then
.LedgerDB.Backend = "V2LSM"
elif $backend == "disklmdb" then
(.LedgerDB.Backend = "V1LMDB"
| .LedgerDB.LiveTablesPath = $live_tables_path)
| .LedgerDB.LiveTablesPath = $live_tables_path
| .LedgerDB.SnapshotInterval = 300)
elif has("LedgerDB") then
.LedgerDB |= del(.Backend)
else
Expand All @@ -235,13 +269,11 @@ edit_node_configs() {

if [ -z "${ENABLE_LEGACY:-}" ]; then
if [ -n "${MIXED_P2P:-}" ]; then
if [ "$fname" = "config-bft1.json" ]; then
if [ -z "$pool_num" ]; then
cp -f "$SCRIPT_DIR"/topology-bft1.json "$STATE_CLUSTER"
continue
fi

pool_num="${fname##*-pool}"
pool_num="${pool_num%.json}"
if [ "$((pool_num % 2))" != 0 ]; then
cp -f "${SCRIPT_DIR}/topology-pool${pool_num}.json" "$STATE_CLUSTER"
continue
Expand Down
2 changes: 1 addition & 1 deletion src/cardonnay_scripts/scripts/conway_fast/testnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"SMASH": "if set, will start and configure smash",
"ENABLE_LEGACY": "if set, local cluster will use legacy networking",
"MIXED_P2P": "if set, local cluster will use P2P for some nodes and legacy topology for others",
"UTXO_BACKEND": "'mem' or 'disk', default is 'mem' (or legacy) if unset",
"UTXO_BACKEND": "'mem', 'disk', 'disklmdb' or `empty`, default is `empty` (mem without configuration) if unset",
"NO_CC": "if set, will not create committee",
"DRY_RUN": "if set, will not start the cluster",
"PROTOCOL_VERSION": "if set, will use the specified protocol version (e.g., 11 for latest Conway, etc.)"
Expand Down
2 changes: 1 addition & 1 deletion src/cardonnay_scripts/scripts/conway_slow/testnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"SMASH": "if set, will start and configure smash",
"ENABLE_LEGACY": "if set, local cluster will use legacy networking",
"MIXED_P2P": "if set, local cluster will use P2P for some nodes and legacy topology for others",
"UTXO_BACKEND": "'mem' or 'disk', default is 'mem' (or legacy) if unset",
"UTXO_BACKEND": "'mem', 'disk', 'disklmdb' or `empty`, default is `empty` (mem without configuration) if unset",
"NO_CC": "if set, will not create committee",
"DRY_RUN": "if set, will not start the cluster",
"PROTOCOL_VERSION": "if set, will use the specified protocol version (e.g., 11 for latest Conway, etc.)"
Expand Down
2 changes: 1 addition & 1 deletion src/cardonnay_scripts/scripts/mainnet_fast/testnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"SMASH": "if set, will start and configure smash",
"ENABLE_LEGACY": "if set, local cluster will use legacy networking",
"MIXED_P2P": "if set, local cluster will use P2P for some nodes and legacy topology for others",
"UTXO_BACKEND": "'mem' or 'disk', default is 'mem' (or legacy) if unset",
"UTXO_BACKEND": "'mem', 'disk', 'disklmdb' or `empty`, default is `empty` (mem without configuration) if unset",
"NO_CC": "if set, will not create committee",
"DRY_RUN": "if set, will not start the cluster",
"PROTOCOL_VERSION": "if set, will use the specified protocol version (e.g., 11 for latest Conway, etc.)"
Expand Down