feat: add txpool-viz

Author: ndeto

README.md

@@ -758,6 +758,7 @@ additional_services:
   - forky
   - apache
   - tracoor
+  - txpool_viz
 
 # Configuration place for blockscout explorer - https://github.com/blockscout/blockscout
 blockscout_params:
@@ -1058,6 +1059,20 @@ spamoor_params:
   # A list of optional params that will be passed to the spamoor command for modifying its behaviour
   extra_args: []
 
+# Configuration for txpool_viz. A mempool visualizer.
+txpool_viz_params:
+  # The image to use for txpool_viz
+  image: punkhazardlabs/txpool-viz:latest
+  # Polling configuration for txpool_viz
+  polling:
+    interval: 0.1s # How often to poll for new mempool data (in seconds)
+    timeout: 5s # Timeout for polling requests (in seconds)
+  # Filters to apply to the mempool data
+  filters:
+    min_gas_price: 1gwei # Minimum gas price filter for transactions
+  focil_enabled: "true" # Enable or disable FOCIL (set to "true" to enable)
+  log_level: "info" # Logging level for txpool_viz (e.g., "info", "debug", "warn", "error")
+
 # Ethereum genesis generator params
 ethereum_genesis_generator_params:
   # The image to use for ethereum genesis generator
@@ -1070,7 +1085,7 @@ port_publisher:
   # Set to "auto" to automatically detect public IP from ident.me
   # Defaults to KURTOSIS_IP_ADDR_PLACEHOLDER (uses per-service settings)
   nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
-  
+
   # Execution Layer public port exposed to your local machine
   # Disabled by default
   # Public port start defaults to 32000
@@ -1083,7 +1098,7 @@ port_publisher:
     # Set to "auto" to automatically detect public IP from ident.me
     # Defaults to KURTOSIS_IP_ADDR_PLACEHOLDER (container IP)
     nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
-  
+
   # Consensus Layer public port exposed to your local machine
   # Disabled by default
   # Public port start defaults to 33000
@@ -1096,7 +1111,7 @@ port_publisher:
     # Set to "auto" to automatically detect public IP from ident.me
     # Defaults to KURTOSIS_IP_ADDR_PLACEHOLDER (container IP)
     nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
-  
+
   # Validator client public port exposed to your local machine
   # Disabled by default
   # Public port start defaults to 34000
@@ -1109,7 +1124,7 @@ port_publisher:
     # Set to "auto" to automatically detect public IP from ident.me
     # Defaults to KURTOSIS_IP_ADDR_PLACEHOLDER (container IP)
     nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
-  
+
   # remote signer public port exposed to your local machine
   # Disabled by default
   # Public port start defaults to 35000
@@ -1122,7 +1137,7 @@ port_publisher:
     # Set to "auto" to automatically detect public IP from ident.me
     # Defaults to KURTOSIS_IP_ADDR_PLACEHOLDER (container IP)
     nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER
-  
+
   # Additional services public port exposed to your local machine
   # Disabled by default
   # Public port start defaults to 36000

main.star

@@ -56,6 +56,7 @@ get_prefunded_accounts = import_module(
     "./src/prefunded_accounts/get_prefunded_accounts.star"
 )
 spamoor = import_module("./src/spamoor/spamoor.star")
+txpool_viz = import_module("./src/txpool_viz/txpool_viz.star")
 
 GRAFANA_USER = "admin"
 GRAFANA_PASSWORD = "admin"
@@ -467,6 +468,18 @@ def run(plan, args={}):
                 global_node_selectors,
             )
             plan.print("Successfully launched tx-fuzz")
+        elif additional_service == "txpool_viz":
+            plan.print("Launching txpool-viz")
+            txpool_viz_config_template = read_file(
+                static_files.TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH
+            )
+            txpool_viz.launch_txpool_viz(
+                plan,
+                txpool_viz_config_template,
+                all_participants,
+                args_with_right_defaults.txpool_viz_params,
+                global_node_selectors
+            )
         elif additional_service == "forkmon":
             plan.print("Launching el forkmon")
             forkmon_config_template = read_file(

src/package_io/constants.star

@@ -93,6 +93,7 @@ MEV_RS_MEV_TYPE = "mev-rs"
 COMMIT_BOOST_MEV_TYPE = "commit-boost"
 DEFAULT_DORA_IMAGE = "ethpandaops/dora:latest"
 DEFAULT_SPAMOOR_IMAGE = "ethpandaops/spamoor:latest"
+DEFAULT_TXPOOL_VIZ_IMAGE = "punkhazardlabs/txpool-viz:latest"
 DEFAULT_ASSERTOOR_IMAGE = "ethpandaops/assertoor:latest"
 DEFAULT_SNOOPER_IMAGE = "ethpandaops/rpc-snooper:latest"
 DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE = (

src/package_io/input_parser.star

@@ -74,6 +74,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = (
     "dora_params",
     "docker_cache_params",
     "assertoor_params",
+    "txpool_viz_params",
     "prometheus_params",
     "grafana_params",
     "tx_fuzz_params",
@@ -179,6 +180,8 @@ def input_parser(plan, input_args):
             for sub_attr in input_args["spamoor_params"]:
                 sub_value = input_args["spamoor_params"][sub_attr]
                 result["spamoor_params"][sub_attr] = sub_value
+        elif attr == "txpool_viz_params":
+            result["txpool_viz_params"] = get_txpool_viz_params(input_args)
         elif attr == "ethereum_genesis_generator_params":
             for sub_attr in input_args["ethereum_genesis_generator_params"]:
                 sub_value = input_args["ethereum_genesis_generator_params"][sub_attr]
@@ -572,6 +575,19 @@ def input_parser(plan, input_args):
             spammers=result["spamoor_params"]["spammers"],
             extra_args=result["spamoor_params"]["extra_args"],
         ),
+        txpool_viz_params=struct(
+            image=result["txpool_viz_params"]["image"],
+            min_cpu=result["txpool_viz_params"]["min_cpu"],
+            max_cpu=result["txpool_viz_params"]["max_cpu"],
+            min_mem=result["txpool_viz_params"]["min_mem"],
+            max_mem=result["txpool_viz_params"]["max_mem"],
+            extra_args=result["txpool_viz_params"]["extra_args"],
+            polling=result["txpool_viz_params"]["polling"],
+            filters=result["txpool_viz_params"]["filters"],
+            focil_enabled=result["txpool_viz_params"]["focil_enabled"],
+            log_level=result["txpool_viz_params"]["log_level"],
+            env=result["txpool_viz_params"]["env"],
+        ),
         additional_services=result["additional_services"],
         wait_for_finalization=result["wait_for_finalization"],
         global_log_level=result["global_log_level"],
@@ -1698,6 +1714,7 @@ def docker_cache_image_override(plan, result):
         "prometheus_params.image",
         "grafana_params.image",
         "spamoor_params.image",
+        "txpool_viz_params.image",
         "ethereum_genesis_generator_params.image",
     ]
 
@@ -1792,7 +1809,6 @@ def get_default_ethereum_genesis_generator_params():
         "image": constants.DEFAULT_ETHEREUM_GENESIS_GENERATOR_IMAGE,
     }
 
-
 def get_devnet_image_tag(network_name, original_image):
     if "devnet" not in network_name:
         return original_image
@@ -1825,7 +1841,6 @@ def get_devnet_image_tag(network_name, original_image):
             image_name = original_image
         return "ethpandaops/{0}:{1}".format(image_name, network_name)
 
-
 def get_devnet_modified_images(network_name, default_images):
     if "devnet" not in network_name:
         return default_images
@@ -1835,3 +1850,39 @@ def get_devnet_modified_images(network_name, default_images):
         modified_images[client_type] = get_devnet_image_tag(network_name, image)
 
     return modified_images
+
+def get_txpool_viz_params(input_args):
+    image = input_args.get("txpool_viz_params", {}).get("image", constants.DEFAULT_TXPOOL_VIZ_IMAGE)
+    min_cpu = input_args.get("txpool_viz_params", {}).get("min_cpu", False)
+    max_cpu = input_args.get("txpool_viz_params", {}).get("max_cpu", False)
+    min_mem = input_args.get("txpool_viz_params", {}).get("min_mem", False)
+    max_mem = input_args.get("txpool_viz_params", {}).get("max_mem", False)
+    extra_args = input_args.get("txpool_viz_params", {}).get("extra_args", [])
+    polling_args = input_args.get("txpool_viz_params", {}).get("polling", {})
+    filters_args = input_args.get("txpool_viz_params", {}).get("filters", {})
+    focil_enabled = input_args.get("txpool_viz_params", {}).get("focil_enabled", "false")
+    log_level = input_args.get("txpool_viz_params", {}).get("log_level", "info")
+    env = input_args.get("txpool_viz_params", {}).get("env", {})
+
+    polling_config = {
+        "interval": polling_args.get("interval", "0.5s"),
+        "timeout": polling_args.get("timeout", "3s"),
+    }
+
+    filters_config = {
+        "min_gas_price": filters_args.get("min_gas_price", "1gwei"),
+    }
+
+    return {
+        "image": image,
+        "min_cpu": min_cpu,
+        "max_cpu": max_cpu,
+        "min_mem": min_mem,
+        "max_mem": max_mem,
+        "extra_args": extra_args,
+        "polling": polling_config,
+        "filters": filters_config,
+        "focil_enabled": focil_enabled,
+        "log_level": log_level,
+        "env": env,
+    }

src/package_io/sanity_check.star

@@ -299,6 +299,19 @@ SUBCATEGORY_PARAMS = {
         "mev",
         "other",
     ],
+    "txpool_viz_params": [
+        "image",
+        "min_cpu",
+        "max_cpu",
+        "min_mem",
+        "max_mem",
+        "extra_args",
+        "polling",
+        "filters",
+        "focil_enabled",
+        "log_level",
+        "env"
+    ]
 }
 
 ADDITIONAL_SERVICES_PARAMS = [
@@ -321,6 +334,7 @@ ADDITIONAL_SERVICES_PARAMS = [
     "nginx",
     "tracoor",
     "spamoor",
+    "txpool_viz"
 ]
 
 ADDITIONAL_CATEGORY_PARAMS = {

src/static_files/static_files.star

@@ -123,3 +123,10 @@ FLASHBOTS_RBUILDER_CONFIG_FILEPATH = (
 COMMIT_BOOST_CONFIG_FILEPATH = (
     STATIC_FILES_DIRPATH + "/mev/commit-boost/cb-config.toml.tmpl"
 )
+
+# txpool-viz config
+TXPOOL_VIZ_CONFIG_DIRPATH = "/txpool-viz-config"
+TXPOOL_VIZ_CONFIG_TEMPLATE_FILEPATH = (
+    STATIC_FILES_DIRPATH + TXPOOL_VIZ_CONFIG_DIRPATH + "/config.yaml.tmpl"
+)
+

src/txpool_viz/txpool_viz.star

@@ -0,0 +1,115 @@
+redis_module = import_module("github.com/kurtosis-tech/redis-package/main.star")
+postgres_module = import_module("github.com/kurtosis-tech/postgres-package/main.star")
+shared_utils = import_module("../shared_utils/shared_utils.star")
+
+HTTP_PORT_NUMBER= 8080
+TXPOOL_VIZ_SERVICE_NAME="txpool-viz"
+
+TXPOOL_VIZ_CONFIG_FILENAME = "config.yaml"
+TXPOOL_VIZ_CONFIG_PATH="/cfg/"
+
+# The min/max CPU/memory that txpool-viz can use.
+MIN_CPU = 100
+MAX_CPU = 1000
+MIN_MEM = 128
+MAX_MEM = 1024
+
+def launch_txpool_viz(
+    plan,
+    config_template,
+    network_participants,
+    txpool_viz_params,
+    global_node_selectors
+):
+    endpoints_list = []
+    for participant in network_participants:
+        el_metrics_info = participant.el_context.el_metrics_info
+        endpoints_list.append({
+            "Name": el_metrics_info[0]["name"] if el_metrics_info else "unknown",
+            "RPCUrl": participant.el_context.rpc_http_url,
+            "Socket": participant.el_context.ws_url
+        })
+
+    beacon_endpoints = []
+    for participant in network_participants:
+        beacon_endpoints.append({
+            "Name": participant.cl_context.beacon_service_name,
+            "BeaconUrl": participant.cl_context.beacon_http_url,
+        })
+
+    # config data & template
+    template_data = txpool_viz_config_template_data(txpool_viz_params, endpoints_list, beacon_endpoints)
+
+    template_and_data = shared_utils.new_template_and_data(
+        config_template, template_data
+    )
+
+    file_config = {}
+    file_config[TXPOOL_VIZ_CONFIG_FILENAME] = template_and_data
+
+    config_files_artifact_name = plan.render_templates(
+        config=file_config,
+        name="txpool-viz-config",
+    )
+
+    postgres = postgres_module.run(
+        plan,
+        service_name="txpool-viz-postgres",
+    )
+
+    # attaching redis server
+    redis = redis_module.run(
+        plan,
+        service_name="txpool-viz-redis",
+    )
+
+    redis_url = "redis://" + redis.hostname + ":" + str(redis.port_number) + "/0"
+
+    environment_variables = dict(txpool_viz_params.env)
+    environment_variables.update({
+        "POSTGRES_URL": postgres.url,
+        "REDIS_URL": redis_url,
+        "PORT": str(HTTP_PORT_NUMBER),
+        "ENV": "prod",
+    })
+
+    service_config = get_service_config(environment_variables, txpool_viz_params, global_node_selectors, config_files_artifact_name)
+
+    txpoolviz = plan.add_service(TXPOOL_VIZ_SERVICE_NAME, config=service_config)
+
+def get_service_config(environment_variables, txpool_viz_params, node_selectors, files_artifact):
+    return ServiceConfig(
+        image=txpool_viz_params.image,
+        ports= {
+            shared_utils.HTTP_APPLICATION_PROTOCOL:  PortSpec(
+                number=HTTP_PORT_NUMBER,
+                application_protocol=shared_utils.HTTP_APPLICATION_PROTOCOL,
+                transport_protocol=shared_utils.TCP_PROTOCOL,
+            ),
+        },
+        env_vars=environment_variables,
+        min_cpu=txpool_viz_params.min_cpu or MIN_CPU,
+        max_cpu=txpool_viz_params.max_cpu or MAX_CPU,
+        min_memory=txpool_viz_params.min_mem or MIN_MEM,
+        max_memory=txpool_viz_params.max_mem or MAX_MEM,
+        files = {
+           TXPOOL_VIZ_CONFIG_PATH : files_artifact
+        },
+        node_selectors=node_selectors,
+      )
+
+def txpool_viz_config_template_data(config, endpoints_list, beacon_endpoints):
+    cfg = dict(config.extra_args)
+    cfg.update({
+        "Endpoints": endpoints_list,
+        "Polling": config.polling,
+        "Filters": config.filters,
+        "LogLevel": config.log_level,
+    })
+
+    if config.focil_enabled == "true":
+        cfg["FocilEnabled"] = config.focil_enabled
+        cfg["BeaconEndpoints"] = beacon_endpoints
+
+    return cfg
+

static_files/txpool-viz-config/config.yaml.tmpl

@@ -0,0 +1,26 @@
+endpoints:
+{{- range .Endpoints }}
+  - name: {{ .Name }}
+    rpc_url: "{{ .RPCUrl }}"
+    socket: "{{ .Socket }}"
+{{- end }}
+
+{{- if eq .FocilEnabled "true" }}
+focil_enabled: true
+beacon_urls:
+{{- range .BeaconEndpoints }}
+  - name: {{ .Name }}
+    beacon_url: "{{ .BeaconUrl }}"
+{{- end }}
+{{- end }}
+
+polling:
+  interval: {{ .Polling.interval }}
+  timeout: {{ .Polling.timeout }}
+
+filters:
+  min_gas_price: {{ .Filters.MinGasPrice }}
+
+log_level: "{{ .LogLevel }}"
+
+extra_args: {{ .ExtraArgs }}