diff --git a/.gitignore b/.gitignore index eefeae3b..09f77ed5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ mkosi.tools.manifest .claudesync/ .claudeignore tmp/ +logs/ !/**/.gitkeep .vscode/ + +.venv/ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f5185663 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +# Makefile for flashbots-images +# Build VM images using mkosi +# +# NOTE: Current implementation uses venv with pip for dependency management. +# TODO: Revisit tooling choice (pip vs uv vs nix) in future PR. + +.DEFAULT_GOAL := help + +VERSION := $(shell git describe --tags --always --dirty="-dev" 2>/dev/null || echo "dev") +SHELL := /bin/bash +.SHELLFLAGS := -eu -o pipefail -c + +# mkosi version +MKOSI_COMMIT := a425313c5811d2ed840630dbfc45c6bc296bfd48 + +# Virtual environment paths +VENV := .venv +VENV_BIN := $(VENV)/bin +VENV_MARKER := $(VENV)/.installed-$(MKOSI_COMMIT) +MKOSI := $(VENV_BIN)/mkosi + +# Build logs +LOGS_DIR := logs +TIMESTAMP := $(shell date +%Y%m%d-%H%M%S) + +##@ Help + +# Awk script from https://github.com/paradigmxyz/reth/blob/main/Makefile +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.PHONY: v +v: ## Show the version + @echo "Version: $(VERSION)" + +##@ Build + +.PHONY: build +build: $(VENV_MARKER) ## Build VM image + @mkdir -p $(LOGS_DIR) + $(MKOSI) --force -I buildernet.conf 2>&1 | tee $(LOGS_DIR)/build-$(TIMESTAMP).log + +.PHONY: build-playground +build-playground: $(VENV_MARKER) ## Build VM image for playground + @mkdir -p $(LOGS_DIR) + $(MKOSI) -I buildernet.conf --profile="devtools,playground" 2>&1 | tee $(LOGS_DIR)/build-playground-$(TIMESTAMP).log + +##@ Setup + +# Create venv only if it doesn't exist +$(VENV_BIN)/activate: + @echo "Creating Python virtual environment at $(VENV)..." + python3 -m venv $(VENV) + +# Install/update dependencies when mkosi version changed +$(VENV_MARKER): $(VENV_BIN)/activate + @echo "Installing mkosi (commit: $(MKOSI_COMMIT))..." + @rm -f $(VENV)/.installed-* + $(VENV_BIN)/pip install -q --upgrade pip + $(VENV_BIN)/pip install -q git+https://github.com/systemd/mkosi.git@$(MKOSI_COMMIT) + @touch $@ + @echo "Installed: $$($(MKOSI) --version)" + +.PHONY: setup +setup: $(VENV_MARKER) ## Setup build environment (venv + mkosi) + @echo "Environment ready. mkosi: $$($(MKOSI) --version)" + +##@ Utilities + +.PHONY: clean +clean: ## Remove build artifacts (keeps venv) + git clean -fdX mkosi.output mkosi.builddir + rm -rf $(LOGS_DIR) + +.PHONY: clean-all +clean-all: clean ## Remove all artifacts including venv + rm -rf $(VENV) diff --git a/mkosi.images/buildernet/mkosi.conf.d/playground.conf b/mkosi.images/buildernet/mkosi.conf.d/playground.conf new file mode 100644 index 00000000..bea6e591 --- /dev/null +++ b/mkosi.images/buildernet/mkosi.conf.d/playground.conf @@ -0,0 +1,5 @@ +[Match] +Profiles=playground + +[Include] +Include=../../mkosi.profiles/playground \ No newline at end of file diff --git a/mkosi.profiles/playground/mkosi.conf b/mkosi.profiles/playground/mkosi.conf new file mode 100644 index 00000000..69325c91 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.conf @@ -0,0 +1,11 @@ +[Content] +# Use with devtools profile for full debugging capabilities: +# mkosi --profile=playground,devtools + +Autologin=true +Packages=login + +# BuilderHub URL override for playground +# 10.0.2.2 is the QEMU user networking default gateway that maps to host +# 8888 is the port of the builder-hub-proxy service +Environment=BUILDERNET_BUILDERHUB_URL=http://10.0.2.2:8888 diff --git a/mkosi.profiles/playground/mkosi.extra/etc/systemd/system-preset/08-playground.preset b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system-preset/08-playground.preset new file mode 100644 index 00000000..421f14d2 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system-preset/08-playground.preset @@ -0,0 +1,24 @@ +# Playground profile - disables services requiring external dependencies +# +# Services can be enabled at runtime with: +# systemctl enable --now .service + +# Fetches genesis.json and testnet config from playground host +enable playground-config-fetch.service + +# Downloads Reth state snapshot from S3 +disable reth-sync.service + +# Ships logs to CloudWatch, metrics to Prometheus +disable vector.service + +# ACME services +# - Issues TLS certs via Let's Encrypt +disable acme-le.service +# - Renews Let's Encrypt TLS certs +disable acme-le-renewal.service +# - Metrics exporter +disable acme-le-textfile-collector.service + +# Rebalances ETH across builder wallets +disable rbuilder-rebalancer.service diff --git a/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/persistent-setup.service.d/playground-override.conf b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/persistent-setup.service.d/playground-override.conf new file mode 100644 index 00000000..c539be48 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/persistent-setup.service.d/playground-override.conf @@ -0,0 +1,5 @@ +[Service] +Environment= +ExecStart= +ExecStart=/usr/bin/persistent-setup --device-lun 10 --key-file /etc/disk-encryption-key +TimeoutStartSec=120 diff --git a/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/playground-config-fetch.service b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/playground-config-fetch.service new file mode 100644 index 00000000..aae1e5e0 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/playground-config-fetch.service @@ -0,0 +1,15 @@ +[Unit] +Description=Fetch configuration files from playground +Wants=network-online.target render-config.service +After=network.target network-online.target render-config.service persistent-setup.service +Before=reth.service lighthouse.service +ConditionPathIsDirectory=!/var/lib/persistent/playground + +[Service] +Type=oneshot +User=root +StateDirectory=persistent/playground +ExecStart=/usr/local/bin/playground-config-fetch + +[Install] +WantedBy=multi-user.target diff --git a/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/rbuilder-bidding-downloader.service.d/playground-override.conf b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/rbuilder-bidding-downloader.service.d/playground-override.conf new file mode 100644 index 00000000..198eeb38 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/etc/systemd/system/rbuilder-bidding-downloader.service.d/playground-override.conf @@ -0,0 +1,18 @@ +[Service] +# Clear EnvironmentFile +EnvironmentFile= + +# Override environment with playground-specific values +Environment=RBUILDER_BIDDING_SHA256SUM=a2c3e02c40ce79e8e649fbea6cd1d4cda89139109e2d39cfb8646e1aa4a36c89 \ + RBUILDER_BIDDING_TAG=v1.3.3 + +# Clear original ExecStart commands from base service +ExecStart= + +# Download from public GitHub release (rbuilder repo) +ExecStart=/bin/bash -c '[[ -f /usr/bin/rbuilder-bidding ]] || /usr/bin/fetch-gh-release-artifact fetch \ + --repo rbuilder --tag "$RBUILDER_BIDDING_TAG" \ + --artifact-name tbv-bidding-service --output-file /usr/bin/rbuilder-bidding' +ExecStart=/bin/bash -c '[[ -f /usr/bin/rbuilder-bidding ]] || \ + echo "$RBUILDER_BIDDING_SHA256SUM /usr/bin/rbuilder-bidding" | sha256sum --check' +ExecStart=/bin/chmod +x /usr/bin/rbuilder-bidding \ No newline at end of file diff --git a/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/operator-api/config.toml.mustache b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/operator-api/config.toml.mustache new file mode 100644 index 00000000..6a4184c2 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/operator-api/config.toml.mustache @@ -0,0 +1,35 @@ +[general] +listen_addr = "0.0.0.0:3535" +pipe_file = "/run/operator-api/journal.fifo" +pprof = false +log_json = true +log_debug = false + +# HTTP Basic Auth: the salt, and where to persistently store the hashed secret +basic_auth_secret_path = "/var/lib/persistent/operator-api/basic-auth-hash" +basic_auth_secret_salt = "bb36ef0b6643" + +# HTTP server timeouts +# http_read_timeout_ms = 2500 +# http_write_timeout_ms = 2500 + +# TLS configuration +tls_enabled = true +tls_create_if_missing = true +tls_cert_path = "/var/lib/persistent/operator-api/cert.pem" +tls_key_path = "/var/lib/persistent/operator-api/key.pem" + +# Make sure to update buildernet/mkosi.extra/etc/sudoers.d/operator-api +# when adding a new action +[actions] +reboot = "sudo systemctl --no-pager --no-ask-password reboot" +rbuilder_restart = "sudo systemctl --no-pager --no-ask-password restart rbuilder-operator" +rbuilder_stop = "sudo systemctl --no-pager --no-ask-password stop rbuilder-operator" +fetch_config = "sudo systemctl --no-pager --no-ask-password start fetch-config" +rbuilder_bidding_restart = "sudo systemctl --no-pager --no-ask-password restart rbuilder-bidding" +ssh_stop = "sudo systemctl --no-pager --no-ask-password stop ssh.service ssh.socket" +ssh_start = "sudo systemctl --no-pager --no-ask-password start ssh.service ssh.socket" +haproxy_restart = "sudo systemctl --no-pager --no-ask-password restart haproxy" + +[file_uploads] +rbuilder_blocklist = "/var/lib/persistent/rbuilder-operator/rbuilder.blocklist.json" diff --git a/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/rbuilder-operator/config.toml.mustache_unsafe b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/rbuilder-operator/config.toml.mustache_unsafe new file mode 100644 index 00000000..0eb10979 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/rbuilder-operator/config.toml.mustache_unsafe @@ -0,0 +1,82 @@ +# ============================================================================= +# rbuilder-operator playground config +# ============================================================================= +# Service dependencies and their status: +# +# LOCAL SERVICES (within VM): +# reth - ENABLED, EL node via IPC (/run/reth/reth.ipc) +# lighthouse - ENABLED, CL node via HTTP (127.0.0.1:3500) +# rbuilder-bidding - ENABLED, bidding service via IPC (/run/rbuilder-bidding/rpc_bidding_server.sock) +# +# EXTERNAL SERVICES (disabled for playground): +# orderflow-archive - DISABLED (confirmed), blocks_processor_url omitted, only used if key_registration_url set +# builderhub - DISABLED (confirmed), key_registration_url omitted, skips key registration entirely +# clickhouse - DISABLED (confirmed), [built_blocks_clickhouse_config] omitted, returns None +# tbv_push_redis - DISABLED (confirmed), omitted, only used if key_registration_url set +# relay endpoints - DISABLED, [[relays]] empty, no block submission +# relay bid scrapers - BLOCKER, [[relay_bid_scrapers]] empty, rbuilder-operator requires non-empty +# +# ============================================================================= + +adjust_finalized_blocks = true +bidding_service_ipc_path = "/run/rbuilder-bidding/rpc_bidding_server.sock" +# blocklist - disabled +# blocklist = "{{rbuilder.blocklist}}" +# blocks_processor_url - disabled, no block archiving +# blocks_processor_url = "https://orderflow-archive.flashbots.net/api" +chain = "/var/lib/persistent/playground/genesis.json" +cl_node_url = ["http://127.0.0.1:3500"] +# coinbase_secret_key - omitted, rbuilder generates random key with warning +coinbase_secret_key = "{{rbuilder.coinbase_secret_key}}" +el_node_ipc_path = "/run/reth/reth.ipc" +error_storage_path = "/run/rbuilder-operator/rbuilder_errors.sqlite" +evm_caching_enable = {{rbuilder.evm_caching_enable}} +extra_data = "{{rbuilder.extra_data}}" +faster_finalize = true +full_telemetry_server_ip = "127.0.0.1" +full_telemetry_server_port = 6060 +ignore_blobs = {{rbuilder.ignore_blobs}} +ignore_cancellable_orders = false +jsonrpc_server_ip = "0.0.0.0" +jsonrpc_server_port = 8645 +# key_registration_url - disabled, no key registration with builderhub? +# key_registration_url = "http://127.0.0.1:7937" +live_builders = {{{rbuilder.live_builders}}} +log_color = false +log_json = true +log_level = "info,rbuilder=debug" +max_order_execution_duration_warning_us = {{rbuilder.max_order_execution_duration_warning_us}} +# optimistic_v3 - disabled for playground, no optimistic relay submissions +# optimistic_v3_server_ip = "0.0.0.0" +# optimistic_v3_server_port = 6000 +# optimistic_v3_public_url = "{{rbuilder.optimistic_v3_public_url}}" +# optimistic_v3_relay_pubkeys = {{{rbuilder.optimistic_v3_relay_pubkeys}}} +# BLS secret key for signing relay submissions +relay_secret_key = "{{rbuilder.relay_secret_key}}" +require_non_empty_blocklist = {{rbuilder.require_non_empty_blocklist}} +reth_db_path = "/var/lib/persistent/reth/db" +reth_static_files_path = "/var/lib/persistent/reth/static_files" +root_hash_sparse_trie_version = "v2" +root_hash_threads = {{rbuilder.root_hash_threads}} +root_hash_use_sparse_trie = true +simulation_threads = 8 +system_recipient_allowlist = {{{rbuilder.system_recipient_allowlist}}} +time_to_keep_mempool_txs_secs = {{rbuilder.time_to_keep_mempool_txs_secs}} +watchdog_timeout_sec = {{rbuilder.watchdog_timeout_sec}} + +# [built_blocks_clickhouse_config] - disabled, no metrics/analytics to ClickHouse +# host = "{{rbuilder.clickhouse_host}}" +# database = "{{rbuilder.clickhouse_database}}" +# username = "{{rbuilder.clickhouse_username}}" +# password = "{{rbuilder.clickhouse_password}}" +# disk_database_path = "/persistent/rbuilder/failed_clickhouse.db" +# disk_max_size_mb = 2000 +# memory_max_size_mb = 1024 +# builder_name = "{{instance_name}}" + +{{{rbuilder.builders}}} + +{{{rbuilder.relays}}} + +# TODO: explore what this is (missing service in playground?) +{{{rbuilder.relay_bid_scrapers}}} diff --git a/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/lighthouse.service.d/playground-override.conf.mustache b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/lighthouse.service.d/playground-override.conf.mustache new file mode 100644 index 00000000..62a46752 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/lighthouse.service.d/playground-override.conf.mustache @@ -0,0 +1,26 @@ +[Unit] +Wants=playground-config-fetch.service +After=playground-config-fetch.service + +[Service] +# Clear the default ExecStart and use playground testnet config (no checkpoint sync) +ExecStart= +ExecStart=/usr/bin/lighthouse bn \ + --testnet-dir /var/lib/persistent/playground/testnet \ + --execution-endpoint http://localhost:8551 \ + --execution-jwt /run/eth-jwt/jwt \ + --suggested-fee-recipient 0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990 \ + --always-prepare-payload \ + --prepare-payload-lookahead 8000 \ + --http \ + --http-port 3500 \ + --http-address 127.0.0.1 \ + --port 9000 \ + --metrics \ + --metrics-address 127.0.0.1 \ + --metrics-port 5054 \ + --datadir %S/persistent/lighthouse \ + --enable-private-discovery \ + --disable-peer-scoring \ + --disable-packet-filter \ + --libp2p-addresses "{{{playground.cl_libp2p_addr}}}" \ No newline at end of file diff --git a/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/reth.service.d/playground-override.conf.mustache b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/reth.service.d/playground-override.conf.mustache new file mode 100644 index 00000000..f5af52a7 --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/etc/systemd/system/reth.service.d/playground-override.conf.mustache @@ -0,0 +1,29 @@ +[Unit] +Wants=playground-config-fetch.service +After=playground-config-fetch.service + +[Service] +# Clear the default ExecStart and use playground genesis +ExecStart= +ExecStart=/usr/bin/reth node \ + --full \ + --chain /var/lib/persistent/playground/genesis.json \ + --datadir %S/persistent/reth \ + --authrpc.addr 127.0.0.1 \ + --authrpc.jwtsecret /run/eth-jwt/jwt \ + --authrpc.port 8551 \ + --http \ + --http.addr 127.0.0.1 \ + --http.port 8545 \ + --http.api "eth,net,web3,trace,rpc,debug,txpool" \ + --ws \ + --ws.addr 127.0.0.1 \ + --ws.port 8546 \ + --ws.api "eth,net,trace,web3,rpc,debug,txpool" \ + --log.stdout.format json \ + --log.file.max-files 0 \ + --metrics "127.0.0.1:9001" \ + --engine.persistence-threshold=0 \ + --engine.memory-block-buffer-target=0 \ + --ipcpath=%t/reth/reth.ipc \ + --trusted-peers "{{{playground.el_bootnode}}}" \ No newline at end of file diff --git a/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/usr/local/bin/playground-config-fetch.mustache_755 b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/usr/local/bin/playground-config-fetch.mustache_755 new file mode 100644 index 00000000..292db79b --- /dev/null +++ b/mkosi.profiles/playground/mkosi.extra/usr/lib/mustache-templates/usr/local/bin/playground-config-fetch.mustache_755 @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Fetches devnet configuration files from playground HTTP service. +# Base URL is injected via mustache from builderhub config. +# + +set -eu -o pipefail + +source /usr/bin/helper-functions.sh + +# Base URL from BuilderHub config (injected via mustache) +BASE_URL="{{{playground.artifacts_url}}}" + +# Target directory +PLAYGROUND_DIR="/var/lib/persistent/playground" + +# File mappings: source_path -> target_path +# Source paths are relative to BASE_URL +declare -A FILES=( + ["genesis.json"]="${PLAYGROUND_DIR}/genesis.json" + ["testnet/config.yaml"]="${PLAYGROUND_DIR}/testnet/config.yaml" + ["testnet/genesis.ssz"]="${PLAYGROUND_DIR}/testnet/genesis.ssz" + ["testnet/genesis_validators_root.txt"]="${PLAYGROUND_DIR}/testnet/genesis_validators_root.txt" + ["testnet/deploy_block.txt"]="${PLAYGROUND_DIR}/testnet/deploy_block.txt" + ["testnet/deposit_contract_block.txt"]="${PLAYGROUND_DIR}/testnet/deposit_contract_block.txt" + ["testnet/boot_enr.yaml"]="${PLAYGROUND_DIR}/testnet/boot_enr.yaml" +) + +log "playground-config-fetch: Starting download from ${BASE_URL}" + +# Create directories +mkdir -p "${PLAYGROUND_DIR}/testnet" + +# Download each file +for src in "${!FILES[@]}"; do + target="${FILES[$src]}" + url="${BASE_URL}/${src}" + + log "playground-config-fetch: Downloading ${url} -> ${target}" + + if ! curl -fsSL --retry 5 --retry-delay 2 -o "${target}" "${url}"; then + log "playground-config-fetch: ERROR: Failed to download ${url}" + exit 1 + fi +done + +log "playground-config-fetch: Successfully downloaded all config files" \ No newline at end of file