Fix Docker Compose validator deployment environment persistence (#4586)

## Problem
Docker Compose validators lose configuration on restart because
environment variables aren't persisted. The deploy-validator.sh script
sets variables but doesn't save them to a .env file, causing validators
to fail after docker-compose restart.

## Solution
1. **Modified deploy-validator.sh** to create a .env file with all
configuration
2. **Created fix-validator-env.sh** migration script for existing
validators
3. **Fixed log output contamination** in PUBLIC_KEY generation
4. **Added backup scripts** for validator keys trapped in Docker volumes

## Changes
- `scripts/deploy-validator.sh`: Now creates .env file for Docker
Compose persistence and redirects logs to stderr to prevent PUBLIC_KEY
contamination
- `scripts/fix-validator-env.sh`: Migration script to generate .env from
legacy .deployment-info files (handles malformed PUBLIC_KEY with ANSI
codes)
- `scripts/backup-validator-keys.sh`: Comprehensive backup tool for
validator keys
- `scripts/extract-wallet-from-volume.sh`: Quick extraction tool for
immediate key recovery

## Testing
- Tested migration script with real .deployment-info files containing
malformed PUBLIC_KEY
- Verified .env file generation preserves all required variables
- Confirmed Docker Compose restart maintains configuration with .env
file

## Impact
Critical fix for validator operators - prevents configuration loss on
restart

Author: eldios

docker/.env.production.template

@@ -0,0 +1,28 @@
+# Production deployment environment variables
+# This file should be created by deploy-validator.sh and renamed to .env
+# These variables are critical for the validator to function properly after restart
+
+# Domain configuration (set by deploy script)
+DOMAIN=${DOMAIN}
+ACME_EMAIL=${ACME_EMAIL}
+
+# Genesis configuration (critical - validator won't work without this)
+GENESIS_URL=${GENESIS_URL}
+GENESIS_BUCKET=${GENESIS_BUCKET}
+GENESIS_PATH_PREFIX=${GENESIS_PATH_PREFIX}
+
+# Validator configuration
+VALIDATOR_KEY=${VALIDATOR_KEY}
+VALIDATOR_NAME=${VALIDATOR_NAME}
+
+# Network configuration
+FAUCET_URL=${FAUCET_URL}
+FAUCET_PORT=${FAUCET_PORT:-8080}
+
+# Storage configuration
+LINERA_STORAGE_SERVICE_PORT=${LINERA_STORAGE_SERVICE_PORT:-1235}
+
+# Docker image
+LINERA_IMAGE=${LINERA_IMAGE}
+
+# Add any other deployment-specific variables here

scripts/backup-validator-keys.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# Script to backup validator keys and wallet from Docker volumes
+# CRITICAL: Run this regularly to protect against data loss
+
+set -euo pipefail
+
+BACKUP_DIR="validator-backup-$(date +%Y%m%d-%H%M%S)"
+DOCKER_COMPOSE_DIR="docker"
+
+echo "🔑 Linera Validator Key Backup Tool"
+echo "===================================="
+echo ""
+
+# Create backup directory
+mkdir -p "${BACKUP_DIR}"
+cd "${BACKUP_DIR}"
+
+echo "📁 Created backup directory: ${BACKUP_DIR}"
+echo ""
+
+# Function to backup from container
+backup_container_files() {
+	local container="$1"
+	local paths="$2"
+	local description="$3"
+
+	echo "→ Checking ${description} in container: ${container}"
+
+	# Check if container exists and is running
+	if ! docker ps --format '{{.Names}}' | grep -q "^${container}$"; then
+		echo "  ⚠️  Container ${container} not running, skipping..."
+		return
+	fi
+
+	# Create container backup directory
+	mkdir -p "${container}"
+
+	# Try to backup each path
+	for path in ${paths}; do
+		# Check if path exists in container
+		if docker exec "${container}" test -e "${path}" 2>/dev/null; then
+			echo "  ✓ Found ${path}"
+			docker cp "${container}:${path}" "${container}/" 2>/dev/null || true
+		fi
+	done
+}
+
+# Backup from proxy container
+backup_container_files "proxy" \
+	"/linera/*.json /root/.config/linera /data /linera-storage" \
+	"proxy wallet and storage"
+
+# Backup from shard containers (there may be multiple)
+for shard in $(docker ps --format '{{.Names}}' | grep -E '^(docker-)?shard'); do
+	backup_container_files "${shard}" \
+		"/linera/*.json /root/.config/linera /data /linera-storage" \
+		"shard wallet and storage"
+done
+
+# Backup ScyllaDB data volume info (for reference)
+echo ""
+echo "→ Getting volume information..."
+docker volume inspect linera-scylla-data 2>/dev/null >scylla-volume-info.json ||
+	echo "  ⚠️  ScyllaDB volume not found"
+
+# Backup docker-compose environment
+echo ""
+echo "→ Backing up deployment configuration..."
+if [ -f "../${DOCKER_COMPOSE_DIR}/.env" ]; then
+	cp "../${DOCKER_COMPOSE_DIR}/.env" ./env-backup
+	echo "  ✓ Backed up .env file"
+elif [ -f "../${DOCKER_COMPOSE_DIR}/.deployment-info" ]; then
+	cp "../${DOCKER_COMPOSE_DIR}/.deployment-info" ./deployment-info-backup
+	echo "  ✓ Backed up .deployment-info file"
+fi
+
+# Get container configuration
+echo ""
+echo "→ Saving container configuration..."
+docker compose -f "../${DOCKER_COMPOSE_DIR}/docker-compose.yml" config >docker-compose-config.yml 2>/dev/null || true
+
+# Create restore instructions
+cat >RESTORE_INSTRUCTIONS.md <<'EOF'
+# Validator Key Restoration Instructions
+
+## ⚠️ CRITICAL FILES
+
+Look for these files in your backup:
+- `wallet.json` - Your validator wallet
+- `keystore.json` - Your validator keystore  
+- Any `.json` files containing keys
+- `.config/linera/` directory
+
+## To Restore
+
+1. Stop your validator:
+   ```bash
+   cd docker && docker compose down
+   ```
+
+2. Copy wallet files back to container volumes:
+   ```bash
+   # After starting containers
+   docker cp wallet.json proxy:/linera/
+   docker cp keystore.json proxy:/linera/
+   ```
+
+3. Restart services:
+   ```bash
+   docker compose up -d
+   ```
+
+## Emergency Recovery
+
+If volumes are corrupted, you may need to:
+1. Delete the old volumes: `docker volume rm linera-scylla-data`
+2. Recreate from this backup
+3. Re-sync with the network
+
+## Backup Contents
+
+This backup was created: $(date)
+Host: $(hostname)
+EOF
+
+# Create tarball of entire backup
+cd ..
+tar -czf "${BACKUP_DIR}.tar.gz" "${BACKUP_DIR}"
+
+echo ""
+echo "✅ Backup completed successfully!"
+echo ""
+echo "📦 Backup saved to:"
+echo "   Directory: $(pwd)/${BACKUP_DIR}/"
+echo "   Archive:   $(pwd)/${BACKUP_DIR}.tar.gz"
+echo ""
+echo "⚠️  IMPORTANT: Copy ${BACKUP_DIR}.tar.gz to a safe location OFF this server!"
+echo ""
+echo "Suggested backup locations:"
+echo "  - Your local machine: scp $(hostname):$(pwd)/${BACKUP_DIR}.tar.gz ~/"
+echo "  - Cloud storage: gsutil cp ${BACKUP_DIR}.tar.gz gs://your-backup-bucket/"
+echo "  - Another server: rsync -av ${BACKUP_DIR}.tar.gz user@backup-server:~/"

scripts/deploy-validator.sh

@@ -501,10 +501,11 @@ generate_validator_keys() {
 	local image="$1"
 	local config_file="validator-config.toml"
 
-	log INFO "Generating validator keys..."
+	# Log to stderr so it doesn't get captured in the output
+	log INFO "Generating validator keys..." >&2
 
 	if [[ "${DRY_RUN:-0}" == "1" ]]; then
-		log INFO "[DRY RUN] Would generate validator keys using image: ${image}"
+		log INFO "[DRY RUN] Would generate validator keys using image: ${image}" >&2
 		echo "DRY_RUN_PUBLIC_KEY"
 		return 0
 	fi
@@ -517,7 +518,7 @@ generate_validator_keys() {
 		/linera-server generate --validators "${config_file}")
 
 	if [ -z "${public_key}" ]; then
-		log ERROR "Failed to generate validator keys"
+		log ERROR "Failed to generate validator keys" >&2
 		return 1
 	fi
 
@@ -1014,7 +1015,7 @@ main() {
 	# Generate validator keys
 	local public_key
 	if ! public_key=$(generate_validator_keys "${LINERA_IMAGE}"); then
-		log ERROR "Failed to generate validator keys"
+		log ERROR "Failed to generate validator keys" >&2
 		exit 1
 	fi
 
@@ -1056,26 +1057,48 @@ main() {
 	log INFO "  Restart services:"
 	log INFO "    cd ${DOCKER_COMPOSE_DIR} && docker compose restart"
 
-	# Save deployment info
-	local deployment_info="${REPO_ROOT}/${DOCKER_COMPOSE_DIR}/.deployment-info"
-	cat >"${deployment_info}" <<EOF
-# Deployment Information
+	# Create .env file for Docker Compose (this is the source of truth)
+	local env_file="${REPO_ROOT}/${DOCKER_COMPOSE_DIR}/.env"
+	cat >"${env_file}" <<EOF
+# Validator Deployment Configuration
 # Generated: $(date -Iseconds)
-HOST=${host}
-EMAIL=${ACME_EMAIL}
-PUBLIC_KEY=${public_key}
-BRANCH=${branch_name}
-COMMIT=${git_commit}
-IMAGE=${LINERA_IMAGE}
-CUSTOM_TAG=${custom_tag:-N/A}
+# This file is the source of truth for Docker Compose configuration
+# It persists all settings across container restarts
+
+# Deployment metadata
+DEPLOYMENT_HOST=${host}
+DEPLOYMENT_EMAIL=${ACME_EMAIL}
+DEPLOYMENT_PUBLIC_KEY=${public_key}
+DEPLOYMENT_BRANCH=${branch_name}
+DEPLOYMENT_COMMIT=${git_commit}
+DEPLOYMENT_CUSTOM_TAG=${custom_tag:-N/A}
+DEPLOYMENT_DATE=$(date -Iseconds)
+
+# Domain and SSL configuration (used by docker-compose.yml)
+DOMAIN=${host}
+ACME_EMAIL=${ACME_EMAIL}
+
+# Genesis configuration (critical for validator operation)
+GENESIS_URL=${genesis_url}
 GENESIS_BUCKET=${genesis_bucket}
 GENESIS_PATH_PREFIX=${genesis_path_prefix}
-GENESIS_URL=${genesis_url}
-SHARDS=${num_shards}
-XFS_PATH=${xfs_path:-N/A}
-CACHE_SIZE=${cache_size}
+
+# Validator configuration
+VALIDATOR_PUBLIC_KEY=${public_key}
+
+# Docker image
+LINERA_IMAGE=${LINERA_IMAGE}
+
+# ScyllaDB configuration
+NUM_SHARDS=${num_shards}
+${xfs_path:+XFS_PATH=${xfs_path}}
+${xfs_path:+CACHE_SIZE=${cache_size}}
+
+# Network configuration
+FAUCET_PORT=8080
+LINERA_STORAGE_SERVICE_PORT=1235
 EOF
-	log DEBUG "Deployment info saved to: ${deployment_info}"
+	log INFO "Environment variables saved to ${env_file} for persistence across restarts"
 }
 
 # Run main function with all arguments

scripts/extract-wallet-from-volume.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Quick script to extract wallet/keys from Docker volumes
+# For operators who need their keys NOW
+
+set -euo pipefail
+
+echo "🔑 Extracting Validator Keys from Docker Volumes"
+echo "================================================"
+echo ""
+
+# Method 1: Direct copy from running containers
+echo "Method 1: Checking running containers..."
+for container in proxy shard; do
+	if docker exec ${container} ls /linera/*.json 2>/dev/null; then
+		echo "✓ Found keys in ${container}, extracting..."
+		docker cp ${container}:/linera/. ./validator-keys-${container}/ 2>/dev/null || true
+	fi
+done
+
+# Method 2: Mount volume to temporary container
+echo ""
+echo "Method 2: Checking Docker volumes..."
+for vol in $(docker volume ls -q | grep -E 'linera|scylla'); do
+	echo "→ Inspecting volume: ${vol}"
+	# Create temp container to access volume
+	docker run --rm -v ${vol}:/backup -v $(pwd):/output alpine sh -c "
+        if find /backup -name '*.json' -o -name 'wallet*' -o -name 'keystore*' 2>/dev/null; then
+            echo '  ✓ Found potential key files'
+            cp -r /backup/* /output/volume-${vol}/ 2>/dev/null || true
+        fi
+    " 2>/dev/null || true
+done
+
+# Method 3: Direct volume path access (requires root)
+echo ""
+echo "Method 3: Direct volume access (may require sudo)..."
+DOCKER_ROOT="/var/lib/docker/volumes"
+if [ -d "${DOCKER_ROOT}" ]; then
+	for vol_path in ${DOCKER_ROOT}/*/; do
+		vol_name=$(basename ${vol_path})
+		if [[ ${vol_name} == *"linera"* ]] || [[ ${vol_name} == *"scylla"* ]]; then
+			echo "→ Checking ${vol_name}"
+			sudo find ${vol_path} -name "*.json" -o -name "wallet*" -o -name "keystore*" 2>/dev/null | while read f; do
+				echo "  ✓ Found: ${f}"
+				sudo cp -p "${f}" "./direct-${vol_name}-$(basename ${f})" 2>/dev/null || true
+			done
+		fi
+	done
+fi
+
+echo ""
+echo "🔍 Searching complete. Check current directory for extracted files:"
+ls -la *.json wallet* keystore* 2>/dev/null || echo "No key files found in standard locations"
+echo ""
+echo "⚠️  If no files were found, try:"
+echo "  1. docker exec proxy cat /linera/wallet.json > wallet.json"
+echo "  2. docker exec proxy cat /linera/keystore.json > keystore.json"
+echo "  3. Check if keys are in environment: docker exec proxy env | grep -i key"