diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index c6701b4e..82a8f514 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -27,6 +27,7 @@ flashboxes/ │ └── debloat*.sh # System cleanup scripts ├── bob-common/ # TEE Searcher common image ├── bob-l1/ # L1 TEE Searcher sandbox image +├── bob-l2/ # L2 TEE Searcher sandbox image ├── buildernet/ # BuilderNet ├── tdx-dummy/ # TDX test environment ├── kernel/ # Kernel configuration diff --git a/Makefile b/Makefile index 5154bc6e..f896ce86 100644 --- a/Makefile +++ b/Makefile @@ -39,11 +39,11 @@ setup: ## Install dependencies (Linux only) # Build module build: check-perms setup ## Build the specified module - $(WRAPPER) mkosi --force -I $(IMAGE).conf + $(WRAPPER) mkosi --force --include=$(IMAGE).conf # Build module with devtools profile build-dev: check-perms setup ## Build module with development tools - $(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf + $(WRAPPER) mkosi --force --profile=devtools --include=$(IMAGE).conf ##@ Utilities diff --git a/bob-common/mkosi.build b/bob-common/mkosi.build index 100ac3e8..29f690bb 100755 --- a/bob-common/mkosi.build +++ b/bob-common/mkosi.build @@ -35,7 +35,7 @@ make_git_package \ # Build tdx-init make_git_package \ "tdx-init" \ - "v0.1.1" \ + "ilya/disk-resize" \ "https://github.com/flashbots/tdx-init" \ 'go build -trimpath -ldflags "-s -w -buildid=" -o ./build/tdx-init' \ "build/tdx-init:/usr/bin/tdx-init" diff --git a/bob-common/mkosi.extra/usr/bin/init-container.sh b/bob-common/mkosi.extra/usr/bin/init-container.sh index 5375ace7..38472761 100755 --- a/bob-common/mkosi.extra/usr/bin/init-container.sh +++ b/bob-common/mkosi.extra/usr/bin/init-container.sh @@ -5,7 +5,6 @@ NAME=searcher-container # PORT FORWARDS SEARCHER_SSH_PORT=10022 -EL_P2P_PORT=30303 SEARCHER_INPUT_CHANNEL=27017 # Run extra commands which are customized per image, @@ -21,8 +20,6 @@ su -s /bin/sh searcher -c "cd ~ && podman run -d \ --name $NAME --replace \ --init \ -p ${SEARCHER_SSH_PORT}:22 \ - -p ${EL_P2P_PORT}:${EL_P2P_PORT} \ - -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \ -p ${SEARCHER_INPUT_CHANNEL}:${SEARCHER_INPUT_CHANNEL}/udp \ -v /persistent/searcher:/persistent:rw \ -v /etc/searcher/ssh_hostkey:/etc/searcher/ssh_hostkey:rw \ diff --git a/bob-common/mkosi.extra/usr/bin/init-firewall.sh b/bob-common/mkosi.extra/usr/bin/init-firewall.sh index 9f9516f3..8701b55e 100755 --- a/bob-common/mkosi.extra/usr/bin/init-firewall.sh +++ b/bob-common/mkosi.extra/usr/bin/init-firewall.sh @@ -124,6 +124,18 @@ accept_dst_ip_port() { -m comment --comment "$comment" } +accept_src_ip_dst_port() { + chain="$1" + protocol="$2" + ip="$3" + port="$4" + comment="$5" + + iptables -A "$chain" -p "$protocol" -s "$ip" --dport "$port" \ + -m conntrack --ctstate NEW -j ACCEPT \ + -m comment --comment "$comment" +} + drop_dst_ip() { chain="$1" ip="$2" diff --git a/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init b/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init index 254b741e..803f5ce2 100644 --- a/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init +++ b/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init @@ -2,9 +2,12 @@ # See also: bob-common/mkosi.extra/usr/bin/init-container.sh ENGINE_API_PORT=8551 +EL_P2P_PORT=30303 BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\ -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \ + -p ${EL_P2P_PORT}:${EL_P2P_PORT} \ + -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \ -v /persistent/lighthouse_logs:/var/log/lighthouse:ro \ -v /tmp/jwt.hex:/secrets/jwt.hex:ro \ " diff --git a/bob-l2.conf b/bob-l2.conf new file mode 100644 index 00000000..81344371 --- /dev/null +++ b/bob-l2.conf @@ -0,0 +1,13 @@ +[Include] +Include=base/mkosi.conf +Include=bob-common/mkosi.conf +Include=bob-l2/mkosi.conf + +[Config] +Profiles=gcp + +[Distribution] +Mirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/ + +[Build] +ToolsTreeMirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/ diff --git a/bob-l2/kernel.config b/bob-l2/kernel.config new file mode 100644 index 00000000..44f58d2c --- /dev/null +++ b/bob-l2/kernel.config @@ -0,0 +1,45 @@ +CONFIG_NET_VENDOR_GOOGLE=y +CONFIG_GVE=y + +# Enable iptables interface for CONFIG_NF_TABLES +# Same config as in bob-l1 +CONFIG_IPV6=n +CONFIG_NETFILTER_NETLINK=y +CONFIG_NETFILTER_NETLINK_LOG=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_NF_CT_PROTO_UDPLITE=y +CONFIG_NF_CT_NETLINK=y +CONFIG_NF_NAT_NEEDED=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_BRIDGE=y +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NETFILTER_XTABLES_COMPAT=y +CONFIG_NFT_CT=y +CONFIG_NFT_COUNTER=y +CONFIG_NFT_LOG=y +CONFIG_NFT_LIMIT=y +CONFIG_NFT_MASQ=y +CONFIG_NFT_REJECT=y +CONFIG_NFT_REJECT_INET=y +CONFIG_NFT_COMPAT=y +CONFIG_NFT_NAT=y +CONFIG_NFT_REDIR=y +CONFIG_NFT_OBJREF=y +CONFIG_NETFILTER_XT_TARGET_LOG=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP_NF_TARGET_NETMAP=y +CONFIG_IP_NF_TARGET_REDIRECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_NET_SCHED=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_USER_API_RNG=y +CONFIG_CRYPTO_USER_API_AEAD=y diff --git a/bob-l2/mkosi.conf b/bob-l2/mkosi.conf new file mode 100644 index 00000000..e58d17b8 --- /dev/null +++ b/bob-l2/mkosi.conf @@ -0,0 +1,10 @@ +[Build] +Environment=KERNEL_CONFIG_SNIPPETS=kernel/snippets/ubuntu.config,bob-l2/kernel.config +WithNetwork=true + +[Content] +ExtraTrees=bob-l2/mkosi.extra +PostInstallationScripts=bob-l2/mkosi.postinst + +Packages=chrony + dmidecode diff --git a/bob-l2/mkosi.extra/etc/bob/firewall-config b/bob-l2/mkosi.extra/etc/bob/firewall-config new file mode 100644 index 00000000..ad166406 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/bob/firewall-config @@ -0,0 +1,85 @@ +# This script is sourced from firewall script and contains image-specific rules +# See also: bob-common/mkosi.extra/usr/bin/init-firewall.sh + +# Image-specific ports +SSH_CONTROL_PORT=22 +SSH_DATA_PORT=10022 +SSH_REGISTER_PORT=8080 +CVM_REVERSE_PROXY_PORT=8745 +SEARCHER_INPUT_PORT=27017 + +# Well-known ports +DNS_PORT=53 +HTTP_PORT=80 +HTTPS_PORT=443 +NTP_PORT=123 +OP_NODE_P2P_PORT=9222 +OP_GETH_P2P_PORT=40404 +ENGINE_API_PORT=8651 + +########################################################################### +# (1) ALWAYS_IN: Inbound rules that are always applied +########################################################################### + +accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port" +accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_PORT "Searcher input channel" + +# We drive op-geth in the searcher container from external op-node +# We assume here that static peers in config are only syn nodes +accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$CONFIG_EL_PEERS_IPS" $ENGINE_API_PORT "Engine API" + +# CVM reverse-proxy serves server attestation +# Also forwards request to ssh pubkey server on localhost:5001, +# which serves searcher-container openssh server pubkey +accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy" + +########################################################################### +# (2) ALWAYS_OUT: Outbound rules that are always applied +########################################################################### + +# Note: this is accessible only from host, searcher netns has DROP on those +# See also init-container.sh +accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP" + +accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp "$CONFIG_SIMULATOR_IP" $HTTP_PORT "bundle" + +# TODO: this is temporary for tests +iptables -A $CHAIN_ALWAYS_OUT -j ACCEPT + +########################################################################### +# (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode +########################################################################### + +accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_DATA_PORT "SSH data plane" +accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_REGISTER_PORT "SSH register service" + +accept_dst_port $CHAIN_MAINTENANCE_IN tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)" +accept_dst_port $CHAIN_MAINTENANCE_IN udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)" + +########################################################################### +# (4) MAINTENANCE_OUT: Outbound rules for Maintenance Mode +########################################################################### + +# Block tx endpoint during maintenance +drop_dst_ip $CHAIN_MAINTENANCE_OUT "$CONFIG_SIMULATOR_IP" "tx stream (DROP before accept-all rules)" + +accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)" +accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)" + +accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTP_PORT "HTTP" +accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTPS_PORT "HTTPS" + +accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)" +accept_dst_port $CHAIN_MAINTENANCE_OUT udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)" + +########################################################################### +# (5) PRODUCTION_IN: Inbound rules for Production Mode +########################################################################### + +# None at the moment + +########################################################################### +# (6) PRODUCTION_OUT: Outbound rules for Production Mode +########################################################################### + +accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp "$CONFIG_SIMULATOR_IP" $HTTP_PORT "tx stream" diff --git a/bob-l2/mkosi.extra/etc/bob/searcher-container-after-init b/bob-l2/mkosi.extra/etc/bob/searcher-container-after-init new file mode 100644 index 00000000..ba7cb042 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/bob/searcher-container-after-init @@ -0,0 +1,7 @@ +# This script is sourced from init-container.sh and contains image-specific stuff +# See also: bob-common/mkosi.extra/usr/bin/init-container.sh + +exec_in_container " + cat <> /etc/hosts +$CONFIG_SIMULATOR_IP simulator.internal +EOF" diff --git a/bob-l2/mkosi.extra/etc/bob/searcher-container-before-init b/bob-l2/mkosi.extra/etc/bob/searcher-container-before-init new file mode 100644 index 00000000..3ac916e1 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/bob/searcher-container-before-init @@ -0,0 +1,12 @@ +# This script is sourced from init-container.sh and contains image-specific stuff +# See also: bob-common/mkosi.extra/usr/bin/init-container.sh + +ENGINE_API_PORT=8651 +EL_P2P_PORT=40404 + +BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\ + -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \ + -p ${EL_P2P_PORT}:${EL_P2P_PORT} \ + -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \ + -v /etc/bob/config.env:/etc/config.env:ro \ +" diff --git a/bob-l2/mkosi.extra/etc/searcher-container-init-extra b/bob-l2/mkosi.extra/etc/searcher-container-init-extra new file mode 100644 index 00000000..25df8920 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/searcher-container-init-extra @@ -0,0 +1,7 @@ +# This script is sourced from init-container.sh and contains image-specific stuff +# See also: bob-common/mkosi.extra/usr/bin/init-container.sh + +echo "Injecting static hosts into searcher container..." +exec_in_container ' + cat <> /etc/hosts +EOF' diff --git a/bob-l2/mkosi.extra/etc/systemd/system/fetch-config.service b/bob-l2/mkosi.extra/etc/systemd/system/fetch-config.service new file mode 100644 index 00000000..258aeda0 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/systemd/system/fetch-config.service @@ -0,0 +1,14 @@ +[Unit] +Description=Fetch some configuration variables from Vault +After=network.target network-setup.service +Requires=network-setup.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/fetch-config.sh +RemainAfterExit=yes +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=minimal.target diff --git a/bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-config.conf b/bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-config.conf new file mode 100644 index 00000000..81df0d13 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-config.conf @@ -0,0 +1,6 @@ +[Unit] +After=fetch-config.service +Requires=fetch-config.service + +[Service] +EnvironmentFile=/etc/bob/config.env diff --git a/bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-config.conf b/bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-config.conf new file mode 100644 index 00000000..81df0d13 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-config.conf @@ -0,0 +1,6 @@ +[Unit] +After=fetch-config.service +Requires=fetch-config.service + +[Service] +EnvironmentFile=/etc/bob/config.env diff --git a/bob-l2/mkosi.extra/etc/tdx-init/disk-glob b/bob-l2/mkosi.extra/etc/tdx-init/disk-glob new file mode 100644 index 00000000..9a125c62 --- /dev/null +++ b/bob-l2/mkosi.extra/etc/tdx-init/disk-glob @@ -0,0 +1,2 @@ +/dev/disk/by-path/*nvme-2 +/dev/disk/by-path/*:10 diff --git a/bob-l2/mkosi.extra/usr/bin/fetch-config.sh b/bob-l2/mkosi.extra/usr/bin/fetch-config.sh new file mode 100755 index 00000000..e03f8c8e --- /dev/null +++ b/bob-l2/mkosi.extra/usr/bin/fetch-config.sh @@ -0,0 +1,95 @@ +#!/bin/sh +set -eu -o pipefail + +# This script fetches couple of pre-defined keys from Vault +# and writes them to /etc/bob/config.env as: +# CONFIG_{KEY}='{VALUE}' + +CONFIG_PATH=/etc/bob/config.env + +if dmidecode -s system-manufacturer 2>/dev/null | grep -q "QEMU"; then + echo "Running in local QEMU, using hardcoded metadata values" + + cat <> "$CONFIG_PATH" +CONFIG_NETWORK_ID='12345' +CONFIG_NETWORK_NAME='local-testnet' +CONFIG_JWT_SECRET='1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef' +CONFIG_EL_STATIC_PEERS='enode://abc123@192.168.1.10:30303' +CONFIG_EL_PEERS_IPS='192.168.1.10' +CONFIG_SIMULATOR_RPC_URL='http://192.168.1.100:8545' +CONFIG_SIMULATOR_WS_URL='ws://192.168.1.100:8546' +CONFIG_SIMULATOR_IP='192.168.1.100' +EOF + + # Ideally, this logic should be somewhere else, but it's fine for now + chattr -i /etc/resolv.conf || true + echo "nameserver 1.1.1.1" > /etc/resolv.conf + + exit 0 +fi + +fetch_metadata_value() { + curl -s \ + --header "Metadata-Flavor: Google" \ + "http://metadata/computeMetadata/v1/instance/attributes/$1" +} + +instance_name=$(fetch_metadata_value "name") +vault_addr=$(fetch_metadata_value "vault_addr") +vault_auth_mount=$(fetch_metadata_value "vault_auth_mount_gcp") +vault_kv_path=$(fetch_metadata_value "vault_kv_path") +vault_kv_common_suffix=$(fetch_metadata_value "vault_kv_common_suffix") + +gcp_token=$(curl \ + --header "Metadata-Flavor: Google" \ + --data-urlencode "audience=http://vault/$instance_name" \ + --data-urlencode "format=full" \ + "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity") + +vault_token=$(curl \ + --data "$(printf '{"role":"%s","jwt":"%s"}' "$instance_name" "$gcp_token")" \ + "${vault_addr}/v1/${vault_auth_mount}/login" | \ + jq -r .auth.client_token) + +common_data=$(curl \ + --header "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/${vault_kv_path}/node/${vault_kv_common_suffix}" | + jq -c .data.data) +secret_data=$(curl \ + --header "X-Vault-Token: ${vault_token}" \ + "${vault_addr}/v1/${vault_kv_path}/node/${instance_name}" | + jq -c .data.data) + +# merge objects +data=$(echo "$common_data $secret_data" | jq -s 'add') + +get_data_value() { + echo "$data" | jq -rc --arg key "$1" '.[$key]' +} + +get_ips_from_uris() { + # eh, good enough for our usecase + echo "$1" | grep -oE '[0-9]{1,3}(\.[0-9]{1,3}){3}' +} + +network_id=$(get_data_value network_id) +network_name=$(get_data_value network_name) +jwt_secret=$(get_data_value jwt_secret) + +el_static_peers=$(get_data_value el_static_peers | jq -r 'join(",")') +el_peers_ips=$(get_ips_from_uris "$el_static_peers" | tr '\n' ',' | sed 's/,$//') + +simulator_rpc_url=$(get_data_value simulator_rpc_url) +simulator_ws_url=$(get_data_value simulator_ws_url) +simulator_ip=$(get_ips_from_uris "$simulator_rpc_url") + +cat <> "$CONFIG_PATH" +CONFIG_NETWORK_ID='${network_id}' +CONFIG_NETWORK_NAME='${network_name}' +CONFIG_JWT_SECRET='${jwt_secret}' +CONFIG_EL_STATIC_PEERS='${el_static_peers}' +CONFIG_EL_PEERS_IPS='${el_peers_ips}' +CONFIG_SIMULATOR_RPC_URL='${simulator_rpc_url}' +CONFIG_SIMULATOR_WS_URL='${simulator_ws_url}' +CONFIG_SIMULATOR_IP='${simulator_ip}' +EOF diff --git a/bob-l2/mkosi.postinst b/bob-l2/mkosi.postinst new file mode 100755 index 00000000..006a55f8 --- /dev/null +++ b/bob-l2/mkosi.postinst @@ -0,0 +1,16 @@ +#!/bin/bash +set -euxo pipefail + +# Install chrony config +mkdir -p "$BUILDROOT/etc/chrony/" +install -m 644 services/chrony.conf "$BUILDROOT/etc/chrony/" + +# Enable services +mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants" + +for service in \ + chrony.service +do + mkosi-chroot systemctl enable "$service" + ln -sf "/etc/systemd/system/$service" "$BUILDROOT/etc/systemd/system/minimal.target.wants/" +done diff --git a/mkosi.profiles/gcp/mkosi.postoutput b/mkosi.profiles/gcp/mkosi.postoutput index 0294d531..bf1e54e1 100755 --- a/mkosi.profiles/gcp/mkosi.postoutput +++ b/mkosi.profiles/gcp/mkosi.postoutput @@ -51,3 +51,11 @@ touch -d "2024-01-01 00:00:00 UTC" "$TMP/disk.raw" 2>/dev/null || true tar --format=oldgnu -Sczf "$TAR" -C "$TMP" disk.raw rm -rf "$TMP" + +IMAGE_NAME="bob-l2-poc-ilya-tmp-$(date +%s)" +GS_PATH="gs://flashbots-artifacts-us-east5/${IMAGE_NAME}.tar.gz" + +echo export CLOUDSDK_CORE_PROJECT=flashbots-artifacts +echo gcloud storage cp ./build/tdx-debian.tar.gz "${GS_PATH}" +echo gcloud beta compute images create --guest-os-features TDX_CAPABLE,UEFI_COMPATIBLE --source-uri "${GS_PATH}" "${IMAGE_NAME}" +echo "This will create GCP image: projects/flashbots-artifacts/global/images/${IMAGE_NAME}"