Skip to content

Commit 46f4826

Browse files
zahzah
committed
Backwards compatible handling of the web3-url parameter in TOML
1 parent 3a35809 commit 46f4826

File tree

9 files changed

+226
-35
lines changed

9 files changed

+226
-35
lines changed

‎AllTests-mainnet.md‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,19 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
161161
+ Testing uints inputs - valid OK
162162
```
163163
OK: 10/12 Fail: 0/12 Skip: 2/12
164+
## EL Configuration
165+
```diff
166+
+ Empty config file OK
167+
+ Invalid URls OK
168+
+ New style config files OK
169+
+ Old style config files OK
170+
+ URL parsing OK
171+
```
172+
OK: 5/5 Fail: 0/5 Skip: 0/5
164173
## Eth1 monitor
165174
```diff
166175
+ Deposits chain OK
167-
+ Rewrite HTTPS Infura URLs OK
176+
+ Rewrite URLs OK
168177
+ Roundtrip engine RPC V1 and bellatrix ExecutionPayload representations OK
169178
+ Roundtrip engine RPC V2 and capella ExecutionPayload representations OK
170179
+ Roundtrip engine RPC V3 and deneb ExecutionPayload representations OK
@@ -626,4 +635,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
626635
OK: 9/9 Fail: 0/9 Skip: 0/9
627636

628637
---TOTAL---
629-
OK: 347/352 Fail: 0/352 Skip: 5/352
638+
OK: 352/357 Fail: 0/357 Skip: 5/357

‎beacon_chain/eth1/el_conf.nim‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
import
22
std/[options, strutils, uri],
33
stew/results, chronicles, confutils,
4+
confutils/toml/defs as confTomlDefs,
5+
confutils/toml/std/net as confTomlNet,
6+
confutils/toml/std/uri as confTomlUri,
47
json_serialization, # for logging
58
toml_serialization, toml_serialization/lexer,
69
../spec/engine_authentication
710

11+
export
12+
toml_serialization, confTomlDefs, confTomlNet, confTomlUri
13+
814
type
915
EngineApiRole* = enum
1016
DepositSyncing = "sync-deposits"
@@ -20,8 +26,8 @@ type
2026

2127
EngineApiUrlConfigValue* = object
2228
url*: string # TODO: Use the URI type here
23-
jwtSecret*: Option[string]
24-
jwtSecretFile*: Option[InputFile]
29+
jwtSecret* {.serializedFieldName: "jwt-secret".}: Option[string]
30+
jwtSecretFile* {.serializedFieldName: "jwt-secret-file".}: Option[InputFile]
2531
roles*: Option[EngineApiRoles]
2632

2733
const
@@ -97,9 +103,9 @@ proc parseCmdArg*(T: type EngineApiUrlConfigValue, input: string): T
97103
if uri.anchor != "":
98104
for key, value in decodeQuery(uri.anchor):
99105
case key
100-
of "jwtSecret":
106+
of "jwtSecret", "jwt-secret":
101107
jwtSecret = some value
102-
of "jwtSecretFile":
108+
of "jwtSecretFile", "jwt-secret-file":
103109
jwtSecretFile = some InputFile.parseCmdArg(value)
104110
of "roles":
105111
var uriRoles: EngineApiRoles = {}
@@ -126,6 +132,17 @@ proc parseCmdArg*(T: type EngineApiUrlConfigValue, input: string): T
126132
jwtSecretFile: jwtSecretFile,
127133
roles: roles)
128134

135+
proc readValue*(reader: var TomlReader, value: var EngineApiUrlConfigValue)
136+
{.raises: [Defect, SerializationError, IOError].} =
137+
if reader.lex.readable and reader.lex.peekChar in ['\'', '"']:
138+
# If the input is a string, we'll reuse the command-line parsing logic
139+
value = try: parseCmdArg(EngineApiUrlConfigValue, reader.readValue(string))
140+
except ValueError as err:
141+
reader.lex.raiseUnexpectedValue("Valid Engine API URL expected: " & err.msg)
142+
else:
143+
# Else, we'll use the standard object-serializer in TOML
144+
toml_serialization.readValue(reader, value)
145+
129146
proc fixupWeb3Urls*(web3Url: var string) =
130147
var normalizedUrl = toLowerAscii(web3Url)
131148
if not (normalizedUrl.startsWith("https://") or

‎docs/the_nimbus_book/src/eth1.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,21 @@ You can increase the resilience of your setup and eliminate any downtime during
104104
```
105105

106106
!!! tip
107-
You can use a different secret for each connection by specifying `jwtSecret` or `jwtSecretFile` as a query parameter in the anchor section of the URL (e.g. `http://127.0.0.1:8551/#jwtSecret=0x12345...` or `http://127.0.0.1:8551/#jwtSecretFile=/tmp/jwtsecret`).
107+
You can use a different secret for each connection by specifying `jwt-secret` or `jwt-secret-file` as a query parameter in the anchor section of the URL (e.g. `http://127.0.0.1:8551/#jwt-secret=0x12345...` or `http://127.0.0.1:8551/#jwt-secret-file=/tmp/jwtsecret`). If you use a [TOML config file](./options.html#configuration-files), you can also use the following more natural syntax:
108+
109+
```toml
110+
data-dir = "my-data-dir"
111+
rest = true
112+
...
113+
114+
[[el]]
115+
url = "http://127.0.0.1:8551"
116+
jwt-secret-file="/path/to/jwt/file"
117+
118+
[[el]]
119+
url = "http://192.168.1.2:8551"
120+
jwt-secret = ""
121+
```
108122

109123
As long as only one of execution clients remains operational and fully synced, Nimbus will keep performing all validator duties.
110124

‎ncli/deposit_downloader.nim‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,18 @@ import
1010

1111
type
1212
CliFlags = object
13-
network {.
13+
network* {.
1414
defaultValue: "mainnet"
1515
name: "network".}: string
16-
elUrls {.
16+
elUrls* {.
1717
name: "el".}: seq[EngineApiUrlConfigValue]
18-
jwtSecret {.
18+
jwtSecret* {.
1919
name: "jwt-secret".}: Option[InputFile]
20-
outDepositsFile {.
20+
outDepositsFile* {.
2121
name: "out-deposits-file".}: Option[OutFile]
22+
configFile* {.
23+
desc: "Loads the configuration from a TOML file"
24+
name: "config-file" .}: Option[InputFile]
2225

2326
proc main(flags: CliFlags) {.async.} =
2427
let
@@ -70,4 +73,8 @@ proc main(flags: CliFlags) {.async.} =
7073

7174
info "All deposits downloaded"
7275

73-
waitFor main(load CliFlags)
76+
waitFor main(
77+
load(CliFlags,
78+
secondarySources = proc (config: CliFlags, sources: auto) =
79+
if config.configFile.isSome:
80+
sources.addConfigFile(Toml, config.configFile.get)))

‎tests/all_tests.nim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import # Unit test
2424
./test_discovery,
2525
./test_engine_authentication,
2626
./test_eth1_monitor,
27+
./test_el_conf,
2728
./test_eth2_ssz_serialization,
2829
./test_exit_pool,
2930
./test_forks,

‎tests/test_el_conf.nim‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
# beacon_chain
2+
# Copyright (c) 2021-2023 Status Research & Development GmbH
3+
# Licensed and distributed under either of
4+
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
5+
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
6+
# at your option. This file may not be copied, modified, or distributed except according to those terms.
7+
8+
{.used.}
9+
10+
import
11+
unittest2, confutils,
12+
stew/byteutils,
13+
../beacon_chain/eth1/el_conf,
14+
../beacon_chain/spec/engine_authentication
15+
16+
type
17+
ExampleConfigFile = object
18+
dataDir* {.name: "data-dir".}: string
19+
el* {.name: "el".}: seq[EngineApiUrlConfigValue]
20+
21+
proc loadExampleConfig(content: string, cmdLine = newSeq[string]()): ExampleConfigFile =
22+
ExampleConfigFile.load(
23+
cmdLine = cmdLine,
24+
secondarySources = proc (config: ExampleConfigFile, sources: auto) =
25+
sources.addConfigFileContent(Toml, content))
26+
27+
const
28+
validJwtToken = parseJwtTokenValue(
29+
"aa95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098cc").get
30+
31+
suite "EL Configuration":
32+
test "URL parsing":
33+
let url1 = EngineApiUrlConfigValue.parseCmdArg("localhost:8484")
34+
check:
35+
url1.url == "localhost:8484"
36+
url1.roles.isNone
37+
url1.jwtSecret.isNone
38+
url1.jwtSecretFile.isNone
39+
40+
let
41+
url1Final1 = url1.toFinalUrl(some validJwtToken)
42+
url1Final2 = url1.toFinalUrl(none seq[byte])
43+
44+
check:
45+
url1Final1.isOk
46+
url1Final1.get.url == "ws://localhost:8484"
47+
url1Final1.get.jwtSecret.get == validJwtToken
48+
url1Final1.get.roles == defaultEngineApiRoles
49+
50+
url1Final2.isOk
51+
url1Final2.get.url == "ws://localhost:8484"
52+
url1Final2.get.jwtSecret.isNone
53+
url1Final2.get.roles == defaultEngineApiRoles
54+
55+
let url2 = EngineApiUrlConfigValue.parseCmdArg(
56+
"https://eth-node.io:2020#jwt-secret-file=jwt.hex")
57+
check:
58+
url2.url == "https://eth-node.io:2020"
59+
url2.roles.isNone
60+
url2.jwtSecret.isNone
61+
url2.jwtSecretFile.get.string == "jwt.hex"
62+
63+
let url3 = EngineApiUrlConfigValue.parseCmdArg(
64+
"http://localhost/#roles=sync-deposits&jwt-secret=ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba")
65+
check:
66+
url3.url == "http://localhost/"
67+
url3.roles == some({DepositSyncing})
68+
url3.jwtSecret == some("ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba")
69+
url3.jwtSecretFile.isNone
70+
71+
let url3Final = url3.toFinalUrl(some validJwtToken)
72+
check:
73+
url3Final.isOk
74+
url3Final.get.jwtSecret.get.toHex == "ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
75+
url3Final.get.roles == {DepositSyncing}
76+
77+
let url4 = EngineApiUrlConfigValue.parseCmdArg(
78+
"localhost#roles=sync-deposits,validate-blocks&jwt-secret=ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba23")
79+
check:
80+
url4.url == "localhost"
81+
url4.roles == some({DepositSyncing, BlockValidation})
82+
url4.jwtSecret == some("ee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba23")
83+
url4.jwtSecretFile.isNone
84+
85+
let url4Final = url4.toFinalUrl(some validJwtToken)
86+
check:
87+
not url4Final.isOk # the JWT secret is invalid
88+
89+
let url5 = EngineApiUrlConfigValue.parseCmdArg(
90+
"http://127.0.0.1:9090/#roles=sync-deposits,validate-blocks,produce-blocks,sync-deposits")
91+
check:
92+
url5.url == "http://127.0.0.1:9090/"
93+
url5.roles == some({DepositSyncing, BlockValidation, BlockProduction})
94+
url5.jwtSecret.isNone
95+
url5.jwtSecretFile.isNone
96+
97+
test "Invalid URls":
98+
template testInvalidUrl(url: string) =
99+
expect ValueError:
100+
echo "This URL should be invalid: ", EngineApiUrlConfigValue.parseCmdArg(url)
101+
102+
testInvalidUrl "http://127.0.0.1:9090/#roles="
103+
testInvalidUrl "http://127.0.0.1:9090/#roles=sy"
104+
testInvalidUrl "http://127.0.0.1:9090/#roles=sync-deposits,"
105+
testInvalidUrl "http://127.0.0.1:9090/#roles=sync-deposits;validate-blocks"
106+
testInvalidUrl "http://127.0.0.1:9090/#roles=validate-blocks,sync-deps"
107+
108+
test "Old style config files":
109+
let cfg = loadExampleConfig """
110+
data-dir = "/foo"
111+
el = ["http://localhost:8585", "eth-data.io#roles=sync-deposits", "wss://eth-nodes.io/21312432"]
112+
"""
113+
114+
check:
115+
cfg.dataDir == "/foo"
116+
cfg.el.len == 3
117+
cfg.el[0].url == "http://localhost:8585"
118+
cfg.el[1].url == "eth-data.io"
119+
cfg.el[1].roles == some({DepositSyncing})
120+
cfg.el[2].url == "wss://eth-nodes.io/21312432"
121+
122+
test "New style config files":
123+
let cfg = loadExampleConfig """
124+
data-dir = "my-data-dir"
125+
126+
[[el]]
127+
url = "http://localhost:8585"
128+
jwt-secret-file = "jwt.hex"
129+
130+
[[el]]
131+
url = "eth-data.io"
132+
roles = ["sync-deposits", "produce-blocks"]
133+
134+
[[el]]
135+
url = "wss://eth-nodes.io/21312432"
136+
jwt-secret = "0xee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
137+
"""
138+
139+
check:
140+
cfg.dataDir == "my-data-dir"
141+
142+
cfg.el.len == 3
143+
cfg.el[0].url == "http://localhost:8585"
144+
cfg.el[0].roles.isNone
145+
cfg.el[0].jwtSecret.isNone
146+
cfg.el[0].jwtSecretFile.get.string == "jwt.hex"
147+
148+
cfg.el[1].url == "eth-data.io"
149+
cfg.el[1].roles == some({DepositSyncing, BlockProduction})
150+
cfg.el[1].jwtSecret.isNone
151+
cfg.el[1].jwtSecretFile.isNone
152+
153+
cfg.el[2].url == "wss://eth-nodes.io/21312432"
154+
cfg.el[2].roles.isNone
155+
cfg.el[2].jwtSecret.get == "0xee95565a2cc95553d4bf2185f58658939ba3074ce5695cbabfab4a1eaf7098ba"
156+
cfg.el[2].jwtSecretFile.isNone
157+
158+
test "Empty config file":
159+
let cfg = loadExampleConfig("", cmdLine = @["--data-dir=foo"])
160+
161+
check:
162+
cfg.dataDir == "foo"
163+
cfg.el.len == 0

‎tests/test_eth1_monitor.nim‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,39 +23,19 @@ from ../beacon_chain/spec/presets import
2323
MAX_BYTES_PER_TRANSACTION, MAX_EXTRA_DATA_BYTES, MAX_TRANSACTIONS_PER_PAYLOAD
2424

2525
suite "Eth1 monitor":
26-
test "Rewrite HTTPS Infura URLs":
26+
test "Rewrite URLs":
2727
var
28-
mainnetWssUrl = "wss://mainnet.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
29-
mainnetHttpUrl = "http://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
30-
mainnetHttpsUrl = "https://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
31-
goerliWssUrl = "wss://goerli.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
32-
goerliHttpUrl = "http://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
33-
goerliHttpsUrl = "https://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
3428
gethHttpUrl = "http://localhost:8545"
3529
gethHttpsUrl = "https://localhost:8545"
3630
gethWsUrl = "ws://localhost:8545"
3731
unspecifiedProtocolUrl = "localhost:8545"
3832

39-
fixupWeb3Urls mainnetWssUrl
40-
fixupWeb3Urls mainnetHttpUrl
41-
fixupWeb3Urls mainnetHttpsUrl
42-
fixupWeb3Urls goerliWssUrl
43-
fixupWeb3Urls goerliHttpUrl
44-
fixupWeb3Urls goerliHttpsUrl
4533
fixupWeb3Urls gethHttpUrl
4634
fixupWeb3Urls gethHttpsUrl
4735
fixupWeb3Urls gethWsUrl
4836
fixupWeb3Urls unspecifiedProtocolUrl
4937

5038
check:
51-
mainnetWssUrl == "wss://mainnet.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
52-
mainnetHttpUrl == "http://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
53-
mainnetHttpsUrl == "https://mainnet.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
54-
55-
goerliWssUrl == "wss://goerli.infura.io/ws/v3/6224f3c792cc443fafb64e70a98f871e"
56-
goerliHttpUrl == "http://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
57-
goerliHttpsUrl == "https://goerli.infura.io/v3/6224f3c792cc443fafb64e70a98f871e"
58-
5939
gethHttpUrl == "http://localhost:8545"
6040
gethHttpsUrl == "https://localhost:8545"
6141
unspecifiedProtocolUrl == "ws://localhost:8545"

‎vendor/nim-confutils‎

‎vendor/nim-toml-serialization‎

0 commit comments

Comments
 (0)