Skip to content

Commit 7545c06

Browse files
theekrystalleetheekrystallee
authored
update script for node table automation (#363)
Signed-off-by: krystal <56278409+theekrystallee@users.noreply.github.com>
1 parent 61024e7 commit 7545c06

File tree

2 files changed

+172
-131
lines changed

2 files changed

+172
-131
lines changed

‎.github/workflows/update-node-tables.yml‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ on:
55
branches: [main]
66
workflow_dispatch:
77
schedule:
8-
- cron: "0 6 * * *" # daily at 06:00 UTC
8+
# Run daily at 06:00 UTC
9+
- cron: "0 6 * * *"
910

1011
permissions:
1112
contents: write
@@ -23,10 +24,10 @@ jobs:
2324
- name: Checkout
2425
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744
2526

26-
- name: Install dependencies (jq)
27+
- name: Install dependencies (jq, curl)
2728
run: |
2829
sudo apt-get update
29-
sudo apt-get install -y jq
30+
sudo apt-get install -y jq curl
3031
3132
- name: Make script executable
3233
run: chmod +x ./update_node_tables.sh
@@ -41,7 +42,7 @@ jobs:
4142
git status --porcelain
4243
if git diff --quiet; then
4344
echo "changes=false" >> "$GITHUB_OUTPUT"
44-
echo "no changes detected."
45+
echo "No changes detected."
4546
else
4647
echo "changes=true" >> "$GITHUB_OUTPUT"
4748
echo "=== diffstat ==="
@@ -54,10 +55,25 @@ jobs:
5455
if: ${{ steps.git-check.outputs.changes == 'true' }}
5556
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
5657
with:
57-
commit-message: "chore: update node tables"
58+
commit-message: "chore: update node tables from Mirror Node API"
5859
sign-commits: true
5960
branch: update-node-tables
6061
title: "chore: update node tables"
61-
body: "automated update of node tables."
62+
body: |
63+
## Automated Node Table Update
64+
65+
This PR updates the mainnet consensus node tables with the latest data from the [Mirror Node API](https://mainnet.mirrornode.hedera.com/api/v1/network/nodes).
66+
67+
### Updated fields:
68+
- Service endpoints (IP addresses and ports)
69+
- Public keys
70+
- Certificate hashes (SHA384 thumbprints)
71+
72+
### Data source:
73+
- Mirror Node API: `https://mainnet.mirrornode.hedera.com/api/v1/network/nodes`
74+
- Same data displayed on [HashScan](https://hashscan.io/mainnet/nodes/table)
75+
76+
---
77+
*This is an automated PR generated by the node tables update workflow.*
6278
labels: automated-pr
6379
delete-branch: true

‎update_node_tables.sh‎

Lines changed: 150 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -2,45 +2,55 @@
22
set -euo pipefail
33

44
# ──────────────────────────────────────────────────────────────────────────────
5-
# config
5+
# Hedera Node Tables Update Script
6+
#
7+
# This script fetches the latest node data from the Hedera Mirror Node API
8+
# and updates the mainnet-nodes.mdx documentation file with:
9+
# - Service endpoints
10+
# - Public keys
11+
# - Certificate hashes
12+
#
13+
# Data source: https://mainnet.mirrornode.hedera.com/api/v1/network/nodes
14+
# (Same data displayed on https://hashscan.io/mainnet/nodes/table)
615
# ──────────────────────────────────────────────────────────────────────────────
7-
BASE_URL="https://mainnet.mirrornode.hedera.com/api/v1/network/nodes?limit=100&order=asc"
8-
DOC_FILE="networks/mainnet/mainnet-nodes/README.md"
916

10-
TABLE_A_START="<!-- TABLE A START -->"
11-
TABLE_A_END="<!-- TABLE A END -->"
12-
TABLE_B_START="<!-- TABLE B START -->"
13-
TABLE_B_END="<!-- TABLE B END -->"
17+
# ──────────────────────────────────────────────────────────────────────────────
18+
# Configuration
19+
# ──────────────────────────────────────────────────────────────────────────────
20+
BASE_URL="https://mainnet.mirrornode.hedera.com/api/v1/network/nodes?limit=100&order=asc"
21+
DOC_FILE="hedera/networks/mainnet/mainnet-nodes.mdx"
1422

1523
# ──────────────────────────────────────────────────────────────────────────────
16-
# temp files and cleanup
24+
# Temp files and cleanup
1725
# ──────────────────────────────────────────────────────────────────────────────
1826
acc_file="$(mktemp)"
1927
page_file="$(mktemp)"
20-
trap 'rm -f "$acc_file" "$page_file" "${DOC_FILE}.tmp" "${DOC_FILE}.bak" 2>/dev/null || true' EXIT
28+
tableA_file="$(mktemp)"
29+
tableB_file="$(mktemp)"
30+
trap 'rm -f "$acc_file" "$page_file" "$tableA_file" "$tableB_file" 2>/dev/null || true' EXIT
2131

22-
# initialize accumulator
32+
# Initialize accumulator
2333
printf '%s\n' '{"nodes":[]}' > "$acc_file"
2434

2535
# ──────────────────────────────────────────────────────────────────────────────
26-
# fetch all pages (no huge argv to jq)
36+
# Fetch all pages from Mirror Node API
2737
# ──────────────────────────────────────────────────────────────────────────────
28-
echo "ℹ️ fetching all nodes via pagination..."
38+
echo "ℹ️ Fetching all nodes via pagination from Mirror Node API..."
2939
url="$BASE_URL"
3040
while [ -n "${url:-}" ]; do
3141
echo "→ GET $url"
3242
if ! curl -sS --fail --max-time 30 "$url" -o "$page_file"; then
33-
echo "❌ fetch failed: $url"
43+
echo "❌ Fetch failed: $url"
3444
exit 1
3545
fi
3646

37-
# append nodes from this page to the accumulator using files (avoids argv blowups)
47+
# Append nodes from this page to the accumulator
3848
jq -s '{
3949
nodes: (.[0].nodes + (.[1].nodes // []))
4050
}' "$acc_file" "$page_file" > "${acc_file}.new"
4151
mv "${acc_file}.new" "$acc_file"
4252

43-
# follow pagination; mirror returns a relative path for next
53+
# Follow pagination
4454
next_rel="$(jq -r '.links.next // empty' < "$page_file")"
4555
if [ -n "$next_rel" ]; then
4656
case "$next_rel" in
@@ -53,7 +63,9 @@ while [ -n "${url:-}" ]; do
5363
fi
5464
done
5565

56-
# normalize, de-dupe, and project fields (stdin, not argv)
66+
# ──────────────────────────────────────────────────────────────────────────────
67+
# Process and normalize node data
68+
# ──────────────────────────────────────────────────────────────────────────────
5769
nodes_json="$(
5870
jq '
5971
.nodes
@@ -72,126 +84,139 @@ nodes_json="$(
7284
)"
7385

7486
node_count="$(jq 'length' <<<"$nodes_json")"
75-
echo "ℹ️ found $node_count nodes"
76-
[ "$node_count" -eq 0 ] && { echo "⚠️ no nodes found; aborting"; exit 1; }
77-
78-
# ──────────────────────────────────────────────────────────────────────────────
79-
# build table a (markdown)
80-
# ──────────────────────────────────────────────────────────────────────────────
81-
echo "ℹ️ building table a (markdown)..."
82-
tableA_header="| Node | Node ID | Node Account ID | Endpoints | Node Certificate Thumbprint |
83-
|------|---------|-----------------|-----------|--------------------------------|
84-
"
85-
86-
tableA_rows="$(
87-
jq -r '
88-
map({
89-
node: (
90-
if (.description // "") == "" then "N/A"
91-
else (
92-
# capture either "Hosted by X" or "Hosted for X" when present
93-
try ((.description | capture("Hosted (by|for) (?<node>[^|]+)")).node)
94-
catch .description
95-
) end
96-
),
97-
id: .node_id,
98-
acct: .node_account_id,
99-
endpoints: (
100-
(
101-
[.service_endpoints[]? |
102-
if (.ip_address_v4 // "") != "" and (.port // null) != null
103-
then "\(.ip_address_v4):\(.port)"
104-
elif (.ip_address_v6 // "") != "" and (.port // null) != null
105-
then "[\(.ip_address_v6)]:\(.port)"
106-
else empty
107-
end
108-
]
109-
) | join(",<br>")
110-
),
111-
thumb: .node_cert_hash
112-
})
113-
| sort_by(.id)
114-
| .[]
115-
| "| \(.node) | \(.id) | **\(.acct // "N/A")** | \(.endpoints // "N/A") | \(.thumb // "N/A") |"
87+
echo "ℹ️ Found $node_count nodes"
88+
[ "$node_count" -eq 0 ] && { echo "⚠️ No nodes found; aborting"; exit 1; }
89+
90+
# ──────────────────────────────────────────────────────────────────────────────
91+
# Build Table A: Node Address Book (HTML format for MDX - ALL ON ONE LINE)
92+
# Columns: Node | Node ID | Node Account ID | Endpoints | Node Certificate Thumbprint
93+
#
94+
# IMPORTANT: Mintlify/MDX requires the entire table to be on a SINGLE LINE
95+
# with NO newlines between <tr> elements. Newlines break the rendering.
96+
# ──────────────────────────────────────────────────────────────────────────────
97+
echo "ℹ️ Building Table A (Node Address Book)..."
98+
99+
{
100+
# Start table - everything must be on one line
101+
printf '<table><thead><tr><th>Node</th><th>Node ID</th><th>Node Account ID</th><th>Endpoints</th><th>Node Certificate Thumbprint</th></tr></thead><tbody>'
102+
103+
# Generate all rows WITHOUT newlines between them
104+
jq -rj '
105+
.[] |
106+
# Extract node name from description
107+
(
108+
if (.description // "") == "" then "N/A"
109+
else (
110+
(.description | split(" | ")[0] |
111+
if startswith("Hosted by ") then .[10:]
112+
elif startswith("Hosted for ") then .[11:]
113+
else .
114+
end
115+
) // .description
116+
) end
117+
) as $node_name |
118+
119+
# Format endpoints with <br/> separators
120+
(
121+
[.service_endpoints[]? |
122+
if (.ip_address_v4 // "") != "" and (.port // null) != null
123+
then "\(.ip_address_v4):\(.port)"
124+
elif (.ip_address_v6 // "") != "" and (.port // null) != null
125+
then "[\(.ip_address_v6)]:\(.port)"
126+
else empty
127+
end
128+
] | join(",<br/>")
129+
) as $endpoints |
130+
131+
"<tr><td>\($node_name)</td><td>\(.node_id)</td><td><strong>\(.node_account_id // "N/A")</strong></td><td>\($endpoints // "N/A")</td><td>\(.node_cert_hash // "N/A")</td></tr>"
116132
' <<<"$nodes_json"
117-
)"
118-
tableA_content="${tableA_header}${tableA_rows}"
119-
echo "ℹ️ table a built."
133+
134+
printf '</tbody></table>'
135+
} > "$tableA_file"
136+
137+
echo "ℹ️ Table A built."
120138

121139
# ──────────────────────────────────────────────────────────────────────────────
122-
# build table b (markdown)
140+
# Build Table B: Node Public Keys (HTML format for MDX - ALL ON ONE LINE)
141+
# Columns: Node Account ID | Public Key
123142
# ──────────────────────────────────────────────────────────────────────────────
124-
echo "ℹ️ building table b (markdown)..."
125-
tableB_header="| Node Account ID | Public Key |
126-
|-----------------|-----------|
127-
"
143+
echo "ℹ️ Building Table B (Node Public Keys)..."
128144

129-
tableB_rows="$(
130-
jq -r '
131-
map(select(.public_key != null))
132-
| sort_by(.node_id)
133-
| .[]
134-
| "| **\(.node_account_id // "N/A")** | \(.public_key // "N/A") |"
145+
{
146+
printf '<table><thead><tr><th>Node Account ID</th><th>Public Key</th></tr></thead><tbody>'
147+
148+
# Generate all rows WITHOUT newlines between them
149+
jq -rj '
150+
map(select(.public_key != null and .public_key != "")) |
151+
sort_by(.node_id) |
152+
.[] |
153+
"<tr><td><strong>\(.node_account_id // "N/A")</strong></td><td>\(.public_key)</td></tr>"
135154
' <<<"$nodes_json"
136-
)"
137-
tableB_content="${tableB_header}${tableB_rows}"
138-
echo "ℹ️ table b built."
139-
140-
# ──────────────────────────────────────────────────────────────────────────────
141-
# injection helper
142-
# ──────────────────────────────────────────────────────────────────────────────
143-
inject_table() {
144-
local file_to_update="$1" start_marker="$2" end_marker="$3" content_to_inject="$4"
145-
local tmp_content
146-
tmp_content="$(mktemp)"
147-
printf "%s\n" "$content_to_inject" > "$tmp_content"
148-
149-
cp "$file_to_update" "${file_to_update}.bak" || true
150-
echo "ℹ️ created backup: ${file_to_update}.bak"
151-
152-
awk -v start="$start_marker" -v end="$end_marker" -v tf="$tmp_content" '
153-
{
154-
if ($0 == start) {
155-
print
156-
print ""
157-
while ((getline line < tf) > 0) { print line }
158-
print ""
159-
inblock=1; next
160-
}
161-
if ($0 == end) { inblock=0; print; next }
162-
if (!inblock) { print }
163-
}' "$file_to_update" > "${file_to_update}.tmp" && mv "${file_to_update}.tmp" "$file_to_update"
164-
165-
rm -f "$tmp_content"
166-
}
167-
168-
# ensure doc file exists with markers
155+
156+
printf '</tbody></table>'
157+
} > "$tableB_file"
158+
159+
echo "ℹ️ Table B built."
160+
161+
# ──────────────────────────────────────────────────────────────────────────────
162+
# Verify doc file exists
163+
# ──────────────────────────────────────────────────────────────────────────────
169164
if [ ! -f "$DOC_FILE" ]; then
170-
echo "ℹ️ $DOC_FILE not found. creating a placeholder with markers."
171-
mkdir -p "$(dirname "$DOC_FILE")"
172-
printf "%s\n%s\n\n%s\n%s\n" "$TABLE_A_START" "$TABLE_A_END" "$TABLE_B_START" "$TABLE_B_END" > "$DOC_FILE"
165+
echo "❌ Error: $DOC_FILE not found."
166+
echo " Please ensure you're running this script from the repository root."
167+
exit 1
173168
fi
174169

175170
# ──────────────────────────────────────────────────────────────────────────────
176-
# inject tables
171+
# Update the MDX file using Python
172+
# Strategy: Read table content from files to avoid shell escaping issues
177173
# ──────────────────────────────────────────────────────────────────────────────
178-
echo "ℹ️ injecting table a into $DOC_FILE..."
179-
inject_table "$DOC_FILE" "$TABLE_A_START" "$TABLE_A_END" "$tableA_content"
180-
echo "ℹ️ injecting table b into $DOC_FILE..."
181-
inject_table "$DOC_FILE" "$TABLE_B_START" "$TABLE_B_END" "$tableB_content"
174+
echo "ℹ️ Updating $DOC_FILE..."
175+
176+
python3 - "$DOC_FILE" "$tableA_file" "$tableB_file" << 'PYTHON_SCRIPT'
177+
import re
178+
import sys
179+
180+
doc_file = sys.argv[1]
181+
tableA_file = sys.argv[2]
182+
tableB_file = sys.argv[3]
183+
184+
# Read the table content from files
185+
with open(tableA_file, "r") as f:
186+
tableA = f.read()
187+
188+
with open(tableB_file, "r") as f:
189+
tableB = f.read()
190+
191+
# Read the original file
192+
with open(doc_file, "r") as f:
193+
content = f.read()
194+
195+
# Pattern to match the first table (Node Address Book)
196+
# This table has headers: Node | Node ID | Node Account ID | Endpoints | Node Certificate Thumbprint
197+
pattern_tableA = r'<table><thead><tr><th>Node</th><th>Node ID</th><th>Node Account ID</th><th>Endpoints</th><th>Node Certificate Thumbprint</th></tr></thead><tbody>.*?</tbody></table>'
198+
199+
# Pattern to match the second table (Public Keys)
200+
# This table has headers: Node Account ID | Public Key
201+
pattern_tableB = r'<table><thead><tr><th>Node Account ID</th><th>Public Key</th></tr></thead><tbody>.*?</tbody></table>'
202+
203+
# Replace the tables
204+
new_content = re.sub(pattern_tableA, tableA, content, count=1, flags=re.DOTALL)
205+
new_content = re.sub(pattern_tableB, tableB, new_content, count=1, flags=re.DOTALL)
206+
207+
# Write the updated file
208+
with open(doc_file, "w") as f:
209+
f.write(new_content)
210+
211+
print("✅ Tables replaced successfully")
212+
PYTHON_SCRIPT
182213

183214
# ──────────────────────────────────────────────────────────────────────────────
184-
# diagnostics
215+
# Summary
185216
# ──────────────────────────────────────────────────────────────────────────────
186-
echo "ℹ️ change summary:"
187-
git status --porcelain || true
188-
if git diff --quiet; then
189-
echo "no changes detected."
190-
else
191-
echo "=== diffstat ==="
192-
git --no-pager diff --stat
193-
echo "=== first 200 lines of diff ==="
194-
git --no-pager diff | head -n 200
195-
fi
196-
197-
echo "✅ updated $DOC_FILE with latest nodes"
217+
echo ""
218+
echo "✅ Updated $DOC_FILE with latest node data from Mirror Node API"
219+
echo " - $node_count nodes processed"
220+
echo " - Service endpoints updated"
221+
echo " - Public keys updated"
222+
echo " - Certificate hashes updated"

0 commit comments

Comments
 (0)