Skip to content

Commit 542eea6

Browse files
jordanschalmjordanschalm
authored
Merge pull request #7744 from onflow/sre/jp/add-playbook-for-voting
Add playbook for root block voting
2 parents cdce991 + 0112194 commit 542eea6

File tree

3 files changed

+513
-0
lines changed

3 files changed

+513
-0
lines changed

‎cmd/bootstrap/transit/README.md‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,62 @@ Running `transit push-transit-key` will perform the following actions:
8383
- `transit-key.priv.<id>`
8484
1. Upload the node's public files to the server
8585
- `transit-key.pub.<id>`
86+
87+
## Collection nodes
88+
89+
The transit script has three commands applicable to collection nodes:
90+
91+
```shell
92+
$ transit pull-clustering -t ${server-token} -d ${bootstrap-dir}
93+
$ transit generate-cluster-block-vote -t ${server-token} -d ${bootstrap-dir}
94+
$ transit push-cluster-block-vote -t ${server-token} -d ${bootstrap-dir} -v ${vote-file}
95+
```
96+
97+
### Pull Clustering
98+
99+
Running `transit pull-clustering` will:
100+
101+
1. Fetch the collection cluster assignment for the upcoming spork and write it to `<bootstrap-dir>/public-root-information/root-clustering.json`
102+
103+
### Sign Cluster Root Block
104+
105+
After the root clustering has been fetched, running `transit generate-cluster-block-vote` will:
106+
107+
1. Create a signature over the root block of the collection node's assigned cluster, using the node's private staking key.
108+
2. Store the resulting vote to the file `<bootstrap-dir>/private-root-information/private-node-info_<node_id>/root-cluster-block-vote.json`
109+
110+
### Upload Vote
111+
112+
Once a vote has been generated, running `transit push-cluster-block-vote` will upload the vote file to the server.
113+
114+
## Root Block Signing Automation
115+
116+
The root block voting is a critical and time-sensitive step of a network upgrade (spork).
117+
During this process, Collection Node operators must download the cluster assignment, generate votes from each of their Collection Nodes, and upload those votes to the server (one vote per node).
118+
Then Consensus Node operators must download the root block, generate a vote for it using each of their Consensus Nodes, and upload those votes to the server (one vote per node).
119+
120+
To ensure this step is completed reliably and without delays, operators running multiple Collection Nodes or Consensus Nodes are strongly encouraged to automate the root block voting process.
121+
The provided `vote_on_cluster_block.yml` and `vote_on_root_block.yml` Ansible playbooks serve as an example for building such automation. They automate:
122+
123+
- Pulling the root information
124+
- For Collection Nodes: the collection cluster assignment
125+
- For Consensus Nodes: the root block and random beacon key
126+
- Generating the vote
127+
- Pushing the vote to the server
128+
129+
for Collection and Consensus nodes respectively.
130+
131+
Refer to this playbook as a reference for how to use the transit script in an automated environment.
132+
133+
Example usage:
134+
135+
```shell
136+
ansible-playbook -i inventories/mainnet/mainnet26.yml/ vote_on_root_block.yml \
137+
-e "boot_tools_tar=https://storage.googleapis.com/flow-genesis-bootstrap/boot-tools.tar" \
138+
-e "bootstrap_directory=/var/flow/bootstrap"
139+
-e "genesis_bucket=flow-genesis-bootstrap" \
140+
-e "network_version_token=mainnet-26" \
141+
-e "output_directory=/var/flow/bootstrap" \
142+
-e force_repull_transit=true \
143+
```
144+

‎cmd/bootstrap/transit/vote_on_cluster_block.yml‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
---
2+
- name: Prepare and sign collection cluster root block
3+
hosts: collection
4+
become: true
5+
gather_facts: false
6+
7+
vars:
8+
9+
# Required inputs (no defaults)
10+
# http(s) URL or absolute local path to boot-tools.tar (on controller)
11+
boot_tools_tar: "{{ boot_tools_tar }}"
12+
bootstrap_directory: "{{ bootstrap_directory }}"
13+
genesis_bucket: "{{ genesis_bucket }}"
14+
network_version_token: "{{ network_version_token }}"
15+
output_directory: "{{ output_directory }}"
16+
17+
# Derived
18+
transit_path: "{{ output_directory }}/transit"
19+
_is_url: "{{ (boot_tools_tar is string) and (boot_tools_tar is match('^https?://')) }}"
20+
21+
# Flag to force re-pull/install of transit
22+
force_repull_transit: false
23+
24+
pre_tasks:
25+
- name: Validate required inputs are provided
26+
assert:
27+
that:
28+
- boot_tools_tar is defined and (boot_tools_tar | string | length) > 0
29+
- output_directory is defined and (output_directory | string | length) > 0
30+
- network_version_token is defined and (network_version_token | string | length) > 0
31+
- genesis_bucket is defined and (genesis_bucket | string | length) > 0
32+
fail_msg: >-
33+
Missing one or more required inputs: boot_tools_tar, output_directory,
34+
network_version_token, genesis_bucket.
35+
tags: [validate]
36+
37+
- name: Validate output_directory is an absolute path
38+
assert:
39+
that:
40+
- output_directory is match('^/')
41+
fail_msg: "output_directory must be an absolute path (e.g., /opt/flow/transit-bin)."
42+
tags: [validate]
43+
44+
- name: Validate bootstrap_directory is an absolute path
45+
assert:
46+
that:
47+
- bootstrap_directory is match('^/')
48+
fail_msg: "bootstrap_directory must be an absolute path (e.g., /opt/flow/transit-bin)."
49+
tags: [validate]
50+
51+
- name: Validate boot_tools_tar looks like a URL or an absolute file path
52+
vars:
53+
_looks_like_path: "{{ (not _is_url) and (boot_tools_tar is match('^/')) }}"
54+
assert:
55+
that:
56+
- _is_url or _looks_like_path
57+
fail_msg: "boot_tools_tar must be an http(s) URL or an absolute local path on the controller."
58+
tags: [validate]
59+
60+
tasks:
61+
- name: Check if output directory exists
62+
stat:
63+
path: "{{ output_directory }}"
64+
register: dir_stat
65+
66+
- name: Create output directory (skip if exists)
67+
file:
68+
path: "{{ output_directory }}"
69+
state: directory
70+
mode: "0755"
71+
when: not dir_stat.stat.exists
72+
73+
- name: Check if transit already exists in output directory
74+
stat:
75+
path: "{{ transit_path }}"
76+
register: transit_stat
77+
when: not force_repull_transit
78+
79+
- name: Decide if we should fetch/reinstall transit
80+
set_fact:
81+
should_fetch_transit: "{{ true if force_repull_transit else (not transit_stat.stat.exists) }}"
82+
83+
- block:
84+
- name: Create temporary working directory
85+
tempfile:
86+
state: directory
87+
suffix: boottools
88+
register: tmpdir
89+
90+
- name: Download boot-tools.tar to temp directory (URL)
91+
get_url:
92+
url: "{{ boot_tools_tar }}"
93+
dest: "{{ tmpdir.path }}/boot-tools.tar"
94+
mode: "0644"
95+
force: true
96+
timeout: 120
97+
when: _is_url
98+
99+
- name: Copy boot-tools.tar to temp directory (local controller path)
100+
copy:
101+
src: "{{ boot_tools_tar }}"
102+
dest: "{{ tmpdir.path }}/boot-tools.tar"
103+
mode: "0644"
104+
when: not _is_url
105+
106+
- name: Extract boot-tools.tar into temp directory
107+
unarchive:
108+
src: "{{ tmpdir.path }}/boot-tools.tar"
109+
dest: "{{ tmpdir.path }}"
110+
remote_src: true
111+
112+
- name: Find the transit binary inside the extracted boot tools
113+
find:
114+
paths: "{{ tmpdir.path }}"
115+
patterns: "transit"
116+
file_type: file
117+
recurse: true
118+
register: transit_found
119+
120+
- name: Ensure transit binary was found in boot-tools.tar
121+
assert:
122+
that:
123+
- (transit_found.files | length) > 0
124+
fail_msg: "Could not locate 'transit' inside boot-tools.tar."
125+
126+
- name: Install transit into output directory
127+
copy:
128+
src: "{{ (transit_found.files | first).path }}"
129+
dest: "{{ transit_path }}"
130+
mode: "0755"
131+
remote_src: true
132+
133+
- name: Cleanup temporary working directory
134+
file:
135+
path: "{{ tmpdir.path }}"
136+
state: absent
137+
when: should_fetch_transit
138+
139+
- name: Ensure transit binary is executable
140+
file:
141+
path: "{{ transit_path }}"
142+
mode: "0755"
143+
state: file
144+
145+
# --- Transit commands ---
146+
- name: Pull root clustering assignment command
147+
command:
148+
argv:
149+
- "{{ transit_path }}"
150+
- pull-clustering
151+
- -t
152+
- "{{ network_version_token }}"
153+
- -b
154+
- "{{ bootstrap_directory }}"
155+
- -o
156+
- "{{ output_directory }}"
157+
- -g
158+
- "{{ genesis_bucket }}"
159+
register: pull_clustering_result
160+
ignore_errors: false
161+
162+
- name: Show output of pull clustering command
163+
debug:
164+
msg:
165+
- "stdout: {{ pull_clustering_result.stdout | default('') }}"
166+
- "stderr: {{ pull_clustering_result.stderr | default('') }}"
167+
168+
- name: Generate cluster block vote
169+
command:
170+
argv:
171+
- "{{ transit_path }}"
172+
- generate-cluster-block-vote
173+
- -b
174+
- "{{ bootstrap_directory }}"
175+
- -o
176+
- "{{ output_directory }}"
177+
register: gen_vote_result
178+
ignore_errors: false
179+
180+
- name: Show output of generate cluster block vote command
181+
debug:
182+
msg:
183+
- "stdout: {{ gen_vote_result.stdout | default('') }}"
184+
- "stderr: {{ gen_vote_result.stderr | default('') }}"
185+
186+
- name: Push cluster block vote
187+
command:
188+
argv:
189+
- "{{ transit_path }}"
190+
- push-cluster-block-vote
191+
- -t
192+
- "{{ network_version_token }}"
193+
- -b
194+
- "{{ bootstrap_directory }}"
195+
- -d
196+
- "{{ output_directory }}"
197+
- -g
198+
- "{{ genesis_bucket }}"
199+
register: push_vote_result
200+
ignore_errors: false
201+
202+
- name: Show output of push root block vote
203+
debug:
204+
msg:
205+
- "stdout: {{ push_vote_result.stdout | default('') }}"
206+
- "stderr: {{ push_vote_result.stderr | default('') }}"
207+
208+
- name: Get node id from file
209+
command: cat {{ boostrap_directory }}/public-root-information/node-id
210+
register: nodeid_result
211+
212+
- name: Strip whitespace from node id
213+
set_fact:
214+
node_id: "{{ nodeid_result.stdout | trim }}"
215+
216+
- name: Check if cluster block vote file exists in GCS
217+
command: >
218+
gsutil ls gs://{{ genesis_bucket }}/{{ network_version_token }}/root-cluster-block-vote.{{ node_id }}.json
219+
register: gsutil_result
220+
ignore_errors: true
221+
222+
- name: Assert cluster block vote file exists in bucket
223+
assert:
224+
that:
225+
- gsutil_result.rc == 0
226+
fail_msg: "Cluster block vote file not found in bucket"
227+

0 commit comments

Comments
 (0)