feat: add support for pull through cache (ethereum#833)

Author: barnabasbusa

.github/tests/mix-with-tools-minimal.yaml

@@ -31,3 +31,6 @@ additional_services:
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true
+docker_cache_params:
+  enabled: true
+  url: "docker.ethquokkaops.io"

README.md

@@ -637,19 +637,24 @@ additional_services:
 # Configuration place for dora the explorer - https://github.com/ethpandaops/dora
 dora_params:
   # Dora docker image to use
-  # Leave blank to use the default image according to your network params
-  image: ""
-
+  # Defaults to the latest image
+  image: "ethpandaops/dora:latest"
   # A list of optional extra env_vars the dora container should spin up with
   env: {}
 
 # Configuration place for transaction spammer - https://github.com/MariusVanDerWijden/tx-fuzz
 tx_spammer_params:
+  # TX Spammer docker image to use
+  # Defaults to the latest master image
+  image: "ethpandaops/tx-fuzz:master"
   # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour
   tx_spammer_extra_args: []
 
 # Configuration place for goomy the blob spammer - https://github.com/ethpandaops/goomy-blob
 goomy_blob_params:
+  # Goomy Blob docker image to use
+  # Defaults to the latest
+  image: "ethpandaops/goomy-blob:latest"
   # A list of optional params that will be passed to the blob-spammer comamnd for modifying its behaviour
   goomy_blob_args: []
 
@@ -664,6 +669,9 @@ prometheus_params:
   max_cpu: 1000
   min_mem: 128
   max_mem: 2048
+  # Prometheus docker image to use
+  # Defaults to the latest image
+  image: "prom/prometheus:latest"
 
 # Configuration place for grafana
 grafana_params:
@@ -676,12 +684,15 @@ grafana_params:
   max_cpu: 1000
   min_mem: 128
   max_mem: 2048
+  # Grafana docker image to use
+  # Defaults to the latest image
+  image: "grafana/grafana:latest"
 
 # Configuration place for the assertoor testing tool - https://github.com/ethpandaops/assertoor
 assertoor_params:
   # Assertoor docker image to use
-  # Leave blank to use the default image according to your network params
-  image: ""
+  # Defaults to the latest image
+  image: "ethpandaops/assertoor:latest"
 
   # Check chain stability
   # This check monitors the chain and succeeds if:
@@ -771,6 +782,20 @@ disable_peer_scoring: false
 # Defaults to false
 persistent: false
 
+# Docker cache url enables all docker images to be pulled through a custom docker registry
+# Disabled by default
+# Defaults to empty cache url
+# Images pulled from dockerhub will be prefixed with "/dh/" by default (docker.io)
+# Images pulled from github registry will be prefixed with "/gh/" by default (ghcr.io)
+# Images pulled from google registory will be prefixed with "/gcr/" by default (gcr.io)
+# If you want to use a local image in combination with the cache, do not put "/" in your local image name
+docker_cache_params:
+  enabled: false
+  url: ""
+  dockerhub_prefix: "/dh/"
+  github_prefix: "/gh/"
+  google_prefix: "/gcr/"
+
 # Supports three valeus
 # Default: "null" - no mev boost, mev builder, mev flood or relays are spun up
 # "mock" - mock-builder & mev-boost are spun up

main.star

@@ -92,6 +92,7 @@ def run(plan, args={}):
     global_node_selectors = args_with_right_defaults.global_node_selectors
     keymanager_enabled = args_with_right_defaults.keymanager_enabled
     apache_port = args_with_right_defaults.apache_port
+    docker_cache_params = args_with_right_defaults.docker_cache_params
 
     prefunded_accounts = genesis_constants.PRE_FUNDED_ACCOUNTS
     if (
@@ -158,9 +159,8 @@ def run(plan, args={}):
         network_id,
     ) = participant_network.launch_participant_network(
         plan,
-        args_with_right_defaults.participants,
+        args_with_right_defaults,
         network_params,
-        args_with_right_defaults.global_log_level,
         jwt_file,
         keymanager_file,
         persistent,
@@ -169,10 +169,6 @@ def run(plan, args={}):
         global_node_selectors,
         keymanager_enabled,
         parallel_keystore_generation,
-        args_with_right_defaults.checkpoint_sync_enabled,
-        args_with_right_defaults.checkpoint_sync_url,
-        args_with_right_defaults.port_publisher,
-        args_with_right_defaults.mev_type,
     )
 
     plan.print(
@@ -459,6 +455,7 @@ def run(plan, args={}):
                 network_params.seconds_per_slot,
                 network_params.genesis_delay,
                 global_node_selectors,
+                args_with_right_defaults.tx_spammer_params,
             )
             plan.print("Successfully launched blob spammer")
         elif additional_service == "goomy_blob":
@@ -488,6 +485,7 @@ def run(plan, args={}):
                 global_node_selectors,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched execution layer forkmon")
         elif additional_service == "beacon_metrics_gazer":
@@ -500,6 +498,7 @@ def run(plan, args={}):
                     global_node_selectors,
                     args_with_right_defaults.port_publisher,
                     index,
+                    args_with_right_defaults.docker_cache_params,
                 )
             )
             launch_prometheus_grafana = True
@@ -516,6 +515,7 @@ def run(plan, args={}):
                 global_node_selectors,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched blockscout")
         elif additional_service == "dora":
@@ -550,6 +550,7 @@ def run(plan, args={}):
                 global_node_selectors,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched dugtrio")
         elif additional_service == "blutgang":
@@ -566,6 +567,7 @@ def run(plan, args={}):
                 global_node_selectors,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched blutgang")
         elif additional_service == "blobscan":
@@ -580,6 +582,7 @@ def run(plan, args={}):
                 global_node_selectors,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched blobscan")
         elif additional_service == "forky":
@@ -598,6 +601,7 @@ def run(plan, args={}):
                 final_genesis_timestamp,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched forky")
         elif additional_service == "tracoor":
@@ -616,6 +620,7 @@ def run(plan, args={}):
                 final_genesis_timestamp,
                 args_with_right_defaults.port_publisher,
                 index,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched tracoor")
         elif additional_service == "apache":
@@ -627,6 +632,7 @@ def run(plan, args={}):
                 all_participants,
                 args_with_right_defaults.participants,
                 global_node_selectors,
+                args_with_right_defaults.docker_cache_params,
             )
             plan.print("Successfully launched apache")
         elif additional_service == "full_beaconchain_explorer":
@@ -673,6 +679,7 @@ def run(plan, args={}):
                 fuzz_target,
                 args_with_right_defaults.custom_flood_params,
                 global_node_selectors,
+                args_with_right_defaults.docker_cache_params,
             )
         else:
             fail("Invalid additional service %s" % (additional_service))

src/apache/apache_launcher.star

@@ -11,6 +11,7 @@ APACHE_ENR_LIST_FILENAME = "bootstrap_nodes.txt"
 
 APACHE_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/usr/local/apache2/htdocs/"
 
+IMAGE_NAME = "library/httpd:latest"
 # The min/max CPU/memory that assertoor can use
 MIN_CPU = 100
 MAX_CPU = 300
@@ -33,6 +34,7 @@ def launch_apache(
     participant_contexts,
     participant_configs,
     global_node_selectors,
+    docker_cache_params,
 ):
     config_files_artifact_name = plan.upload_files(
         src=static_files.APACHE_CONFIG_FILEPATH, name="apache-config"
@@ -93,6 +95,7 @@ def launch_apache(
         public_ports,
         bootstrap_info_files_artifact_name,
         global_node_selectors,
+        docker_cache_params,
     )
 
     plan.add_service(SERVICE_NAME, config)
@@ -104,6 +107,7 @@ def get_config(
     public_ports,
     bootstrap_info_files_artifact_name,
     node_selectors,
+    docker_cache_params,
 ):
     files = {
         constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data,
@@ -145,7 +149,10 @@ def get_config(
     cmd_str = " ".join(cmd)
 
     return ServiceConfig(
-        image="httpd:latest",
+        image=shared_utils.docker_cache_image_calc(
+            docker_cache_params,
+            IMAGE_NAME,
+        ),
         ports=USED_PORTS,
         cmd=[cmd_str],
         public_ports=public_ports,

src/assertoor/assertoor_launcher.star

@@ -119,12 +119,10 @@ def get_config(
         ASSERTOOR_CONFIG_FILENAME,
     )
 
-    if assertoor_params.image != "":
-        IMAGE_NAME = assertoor_params.image
-    elif network_params.electra_fork_epoch < constants.ELECTRA_FORK_EPOCH:
+    IMAGE_NAME = assertoor_params.image
+
+    if network_params.electra_fork_epoch < constants.ELECTRA_FORK_EPOCH:
         IMAGE_NAME = "ethpandaops/assertoor:electra-support"
-    else:
-        IMAGE_NAME = "ethpandaops/assertoor:latest"
 
     return ServiceConfig(
         image=IMAGE_NAME,

src/beacon_metrics_gazer/beacon_metrics_gazer_launcher.star

@@ -37,12 +37,14 @@ def launch_beacon_metrics_gazer(
     global_node_selectors,
     port_publisher,
     additional_service_index,
+    docker_cache_params,
 ):
     config = get_config(
         cl_contexts[0].beacon_http_url,
         global_node_selectors,
         port_publisher,
         additional_service_index,
+        docker_cache_params,
     )
 
     beacon_metrics_gazer_service = plan.add_service(SERVICE_NAME, config)
@@ -64,6 +66,7 @@ def get_config(
     node_selectors,
     port_publisher,
     additional_service_index,
+    docker_cache_params,
 ):
     config_file_path = shared_utils.path_join(
         BEACON_METRICS_GAZER_CONFIG_MOUNT_DIRPATH_ON_SERVICE,
@@ -78,7 +81,10 @@ def get_config(
     )
 
     return ServiceConfig(
-        image=IMAGE_NAME,
+        image=shared_utils.docker_cache_image_calc(
+            docker_cache_params,
+            IMAGE_NAME,
+        ),
         ports=USED_PORTS,
         public_ports=public_ports,
         files={

src/blob_spammer/blob_spammer.star

@@ -1,4 +1,3 @@
-IMAGE_NAME = "ethpandaops/tx-fuzz:master"
 SERVICE_NAME = "blob-spammer"
 
 ENTRYPOINT_ARGS = ["/bin/sh", "-c"]
@@ -19,6 +18,7 @@ def launch_blob_spammer(
     seconds_per_slot,
     genesis_delay,
     global_node_selectors,
+    tx_spammer_params,
 ):
     config = get_config(
         prefunded_addresses,
@@ -28,6 +28,7 @@ def launch_blob_spammer(
         seconds_per_slot,
         genesis_delay,
         global_node_selectors,
+        tx_spammer_params.image,
     )
     plan.add_service(SERVICE_NAME, config)
 
@@ -40,10 +41,11 @@ def get_config(
     seconds_per_slot,
     genesis_delay,
     node_selectors,
+    image,
 ):
     dencunTime = (deneb_fork_epoch * 32 * seconds_per_slot) + genesis_delay
     return ServiceConfig(
-        image=IMAGE_NAME,
+        image=image,
         entrypoint=ENTRYPOINT_ARGS,
         cmd=[
             " && ".join(