Use variables for execution explorer deployment

mainnet-shadow-fork-7/inventory/group_vars/all.yaml

@@ -114,6 +114,7 @@ local_forkmondata_host_dir: "{{inventory_dir}}/../forkmondata"
 local_forkmondata_host_archive: "{{inventory_dir}}/../forkmondata.tar.gz"
 local_custom_config_data_host_dir: "{{inventory_dir}}/../custom_config_data"
 local_custom_config_data_host_archive: "{{inventory_dir}}/../custom_config_data.tar.gz"
+local_eth1_genesis_filepath: "{{local_custom_config_data_host_dir}}/genesis.json"
 # private dirs
 # Validator assignments
 local_validator_host_dir: "{{inventory_dir}}/../validator_prep/{{ansible_hostname}}"

mainnet-shadow-fork-7/inventory/group_vars/explorer.yml

@@ -0,0 +1,64 @@
+explorer_docker_network_name: explorernet
+
+beacon_explorer_container_name: explorer
+execution_explorer_container_name: blockscout
+
+explorer_memory: 9000m
+beacon_explorer_image_name: "parithoshj/beacon-explorer:print-chain-id"
+execution_explorer_image_name: "skylenet/blockscout:v4.0.0-beta"
+
+explorer_first_block: 1
+explorer_eth1_source_hostname: "merge-devnet-0-ethx-explorer-lighthouse-nethermind-1"
+explorer_eth1_source_ip_address: "{{hostvars[explorer_eth1_source_hostname].ansible_host}}"
+explorer_eth2_source_ip_address: "{{explorer_eth1_source_ip_address}}"
+
+explorer_postgres_container_name: postgres
+explorer_postgres_image_name: "postgres:12.0"
+explorer_postgres_memory: 9000m
+explorer_postgres_port: "5432"
+explorer_postgres_ip: "172.1.1.100"
+explorer_postgres_user: postgres
+
+# Set in secrets.yaml
+# explorer_postgres_password: "{{ lookup('pipe', 'keybase fs read /keybase/team/ethereum.devops/eth2/merge-devnets/merge-devnet-3/explorer-postgres') }}"
+
+explorer_postgres_host_dir: "{{home_dir}}/explorer_postgres_data"
+explorer_postgres_volumes:
+  - "{{explorer_postgres_host_dir}}:/postgresql/data"
+explorer_postgres_start_args: "-i -h {{explorer_postgres_ip}}"
+
+explorer_postgres_init_volumes:
+  - "{{explorer_host_dir}}:/src"
+
+explorer_postgres_init_args: >
+  psql
+  -f /src/tables.sql
+  -d db
+  -h {{explorer_postgres_ip}}
+  -p {{explorer_postgres_port}}
+  -U {{explorer_postgres_user}}
+explorer_host_dir: "{{home_dir}}/explorer_data"
+execution_explorer_genesis_host_path: "{{explorer_host_dir}}/genesis.json"
+execution_explorer_genesis_cont_path: /data/genesis.json
+execution_explorer_volumes:
+  - "{{explorer_host_dir}}:/data"
+
+beacon_explorer_config_host_path: "{{explorer_host_dir}}/eth2_config.yaml"
+beacon_explorer_config_cont_path: /data/eth2_config.yaml
+beacon_explorer_volumes:
+  - "{{explorer_host_dir}}:/data"
+
+beacon_explorer_start_args: "./explorer --config=/data/config.yaml"
+
+execution_explorer_start_args: 'sh -c "echo $DATABASE_URL &&
+  mix do ecto.create, ecto.migrate &&
+  exec mix phx.server" '
+
+explorer_stats_container_name: explorerstats
+
+explorer_stats_volumes:
+  - "{{explorer_host_dir}}:/data"
+
+explorer_stats_start_args: "./statistics --config=/data/config.yaml"
+
+explorer_beacon_extra_args: "--slots-per-archive-point=64"

playbooks/setup_beacon_and_execution_explorer_full.yml

@@ -1,10 +1,12 @@
 ---
 #- import_playbook: tasks/setup_machine.yml
 - import_playbook: tasks/upload_beacon_explorer_data.yml
-# - import_playbook: tasks/upload_execution_explorer_data.yml
 - import_playbook: tasks/start_explorer_net.yml
 - import_playbook: tasks/start_explorer_postgres.yml
 - import_playbook: tasks/init_explorer_sql_tables.yml
 - import_playbook: tasks/start_beacon_explorer.yml
 - import_playbook: tasks/start_explorer_statistics.yml
-# - import_playbook: tasks/start_execution_explorer.yml
+
+# Execution
+- import_playbook: tasks/upload_execution_explorer_data.yml
+- import_playbook: tasks/start_execution_explorer.yml

playbooks/tasks/start_explorer_postgres.yml

@@ -18,16 +18,16 @@
         network_mode: bridge
         networks:
           - name: "{{explorer_docker_network_name}}"
-            ipv4_address: "172.1.1.100"
+            ipv4_address: "{{explorer_postgres_ip}}"
         expose:
-          - "5432"
+          - "{{explorer_postgres_port}}"
         restart: "{{ explorer_postgres_require_restart | default(false) }}"
         memory: "{{explorer_postgres_memory}}"
         command: "{{ explorer_postgres_start_args }}"
         volumes: "{{explorer_postgres_volumes}}"
         env:
           POSTGRES_PASSWORD: "{{ explorer_postgres_password }}"
           POSTGRES_USER: postgres
-          POSTGRES_PORT: "5432"
+          POSTGRES_PORT: "{{explorer_postgres_port}}"
           POSTGRES_DB: db
           PGDATA: "/postgresql/data"

playbooks/tasks/templates/blockscout_data/blockscout.env.j2

@@ -8,17 +8,17 @@ DISABLE_READ_API=true
 DISABLE_WRITE_API=true
 ENABLE_TXS_STATS=true
 SHOW_TXS_CHART=true
-DATABASE_URL=postgresql://postgres:{{ explorer_postgres_password }}@172.1.1.100:5432/explorer?ssl=false
+DATABASE_URL=postgresql://{{explorer_postgres_user}}:{{explorer_postgres_password}}@{{explorer_postgres_ip}}:{{explorer_postgres_port}}/explorer?ssl=false
 SECRET_KEY_BASE=VTIB3uHDNbvrY0+60ZWgUoUBKDn9ppLR8MI4CpRz4/qLyEFs54ktJfaNT6Z221No
 ETHEREUM_JSONRPC_VARIANT=geth
-ETHEREUM_JSONRPC_HTTP_URL=http://{{ explorer_ip_address }}:8545
-ETHEREUM_JSONRPC_WS_URL=ws://{{ explorer_ip_address }}:8546
+ETHEREUM_JSONRPC_HTTP_URL=http://{{explorer_eth1_source_ip_address}}:{{eth1_rpc_port}}
+ETHEREUM_JSONRPC_WS_URL=ws://{{explorer_eth1_source_ip_address}}:{{eth1_ws_rpc_port}}
 NETWORK=Ethereum
 SUBNETWORK=Testnet
 BLOCK_TRANSFORMER=base
 POOL_SIZE=15
-CHAIN_SPEC_PATH=/custom_config_data/genesis.json
+CHAIN_SPEC_PATH={{execution_explorer_genesis_cont_path}}
 {% if explorer_first_block is defined %}
 FIRST_BLOCK={{explorer_first_block}}
 TRACE_FIRST_BLOCK={{explorer_first_block}}
-{% endif %}
+{% endif %}

playbooks/tasks/upload_execution_explorer_data.yml

@@ -9,5 +9,11 @@
     - name: Move the execution explorer config j2 template to remote
       template:
         src: blockscout_data/blockscout.env.j2
-        dest:  "{{explorer_host_dir}}/blockscout.env"
-        mode: '755'
+        dest: "{{explorer_host_dir}}/blockscout.env"
+        mode: "755"
+    # Note: explicitly copying the file instead of relying on uploading the entire custom_config_data tar
+    # ensures that the playbook errors if paths are not correct.
+    - name: Copy genesis JSON file
+      copy:
+        src: "{{local_eth1_genesis_filepath}}"
+        dest: "{{execution_explorer_genesis_host_path}}"