Skip to content

Commit eabfb34

Browse files
authored
feat: dump config (#741)
* feat: dump config * do not dump sensitive data * fix default logfile location * fix tmp
1 parent c917adc commit eabfb34

File tree

9 files changed

+283
-88
lines changed

9 files changed

+283
-88
lines changed

images/utils/launcher/__init__.py

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
import logging
22
import shlex
33
import traceback
4-
import os
4+
import os.path
55

6-
from .config import Config, ConfigLoader
7-
from .shell import Shell
8-
from .node import NodeManager, NodeNotFound
9-
from .utils import ParallelExecutionError, ArgumentError
6+
from launcher.config import Config, ConfigLoader
7+
from launcher.shell import Shell
8+
from launcher.node import NodeManager, NodeNotFound
9+
from launcher.utils import ParallelExecutionError, ArgumentError
1010

11-
from .check_wallets import Action as CheckWalletsAction
12-
from .close_other_utils import Action as CloseOtherUtilsAction
13-
from .auto_unlock import Action as AutoUnlockAction
14-
from .warm_up import Action as WarmUpAction
15-
from .errors import FatalError, ConfigError, ConfigErrorScope
11+
from launcher.check_wallets import Action as CheckWalletsAction
12+
from launcher.close_other_utils import Action as CloseOtherUtilsAction
13+
from launcher.auto_unlock import Action as AutoUnlockAction
14+
from launcher.warm_up import Action as WarmUpAction
15+
from launcher.errors import FatalError, ConfigError, ConfigErrorScope
1616

1717

1818
HELP = """\
@@ -96,9 +96,14 @@
9696

9797

9898
def init_logging():
99-
fmt = "%(asctime)s.%(msecs)03d %(levelname)s %(process)d --- [%(threadName)s] %(name)s: %(message)s"
99+
fmt = "%(asctime)s.%(msecs)03d %(levelname)5s %(process)d --- [%(threadName)-15s] %(name)-30s: %(message)s"
100100
datefmt = "%Y-%m-%d %H:%M:%S"
101-
logging.basicConfig(format=fmt, datefmt=datefmt, level=logging.ERROR, filename="/var/log/launcher.log")
101+
if os.path.exists("/mnt/hostfs/tmp"):
102+
logfile = "/mnt/hostfs/tmp/xud-docker.log"
103+
else:
104+
logfile = "xud-docker.log"
105+
106+
logging.basicConfig(format=fmt, datefmt=datefmt, level=logging.INFO, filename=logfile, filemode="w")
102107

103108
level_config = {
104109
"launcher": logging.DEBUG,
@@ -235,16 +240,15 @@ def pre_start(self):
235240
self.close_other_utils()
236241

237242
def start(self):
238-
up_env = True
239-
try:
240-
up_env = self.node_manager.update()
241-
except ParallelExecutionError:
242-
pass
243+
self.logger.info("Start %s", self.config.network)
244+
245+
up_env = self.node_manager.update()
243246

244247
if up_env:
245248
self.node_manager.up()
246249
self.pre_start()
247250

251+
self.logger.info("Start shell")
248252
self.shell.start(f"{self.config.network} > ", self.handle_command)
249253

250254

@@ -267,7 +271,6 @@ def launch(self):
267271
config = None
268272
try:
269273
config = Config(ConfigLoader())
270-
assert config.network_dir is not None
271274
shell.set_network_dir(config.network_dir) # will create shell history file in network_dir
272275
env = XudEnv(config, shell)
273276
env.start()
@@ -288,6 +291,8 @@ def launch(self):
288291
print("{}. For more details, see {}".format(e, config.logfile))
289292
else:
290293
traceback.print_exc()
294+
except ParallelExecutionError:
295+
pass
291296
except Exception: # exclude system exceptions like SystemExit
292297
self.logger.exception("Unexpected exception during launching")
293298
traceback.print_exc()

images/utils/launcher/check_wallets.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def lnd_restart(chain):
9797
c: Container = client.containers.get(name)
9898
cmd = f"lncli -n {network} -c {chain} getinfo"
9999
exit_code, output = c.exec_run(cmd)
100-
self.logger.debug("[Execute] %s: exit_code=%s, output=%s", cmd, exit_code, output)
100+
self.logger.debug("[Execute] %s: exit_code=%s\n%s", cmd, exit_code, output.decode())
101101

102102
if exit_code == 0:
103103
self.logger.debug("Skip restarting %s", name)
@@ -119,7 +119,7 @@ def lnd_restart(chain):
119119
self.logger.debug("Sleep 15 seconds. For God's sake may %s work normally!!!", short_name)
120120
time.sleep(15)
121121

122-
with ThreadPoolExecutor(max_workers=3) as executor:
122+
with ThreadPoolExecutor(max_workers=3, thread_name_prefix="RestartLnd") as executor:
123123
f1 = executor.submit(lnd_restart, "bitcoin")
124124
f2 = executor.submit(lnd_restart, "litecoin")
125125

@@ -223,7 +223,7 @@ def lnd_ready(self, chain: LndChain) -> bool:
223223
cmd = f"lncli -n {network} -c {chain} getinfo"
224224
try:
225225
exit_code, output = lnd.exec_run(cmd)
226-
self.logger.debug("[Execute] %s: exit_code=%s, output=%s", cmd, exit_code, output)
226+
self.logger.debug("[Execute] %s: exit_code=%s\n%s", cmd, exit_code, output.decode())
227227
except:
228228
self.logger.exception("Failed to exec \"%s\" in container %s", cmd, name)
229229
return False
@@ -261,7 +261,7 @@ def ensure_layer2_ready(self) -> None:
261261
# xud is locked, run 'xucli unlock', 'xucli create', or 'xucli restore' then try again
262262
while True:
263263
exit_code, output = xud.exec_run(cmd)
264-
self.logger.debug("[Execute] %s: exit_code=%s, output=%s", cmd, exit_code, output)
264+
self.logger.debug("[Execute] %s: exit_code=%s\n%s", cmd, exit_code, output.decode())
265265
if exit_code == 0:
266266
xud_ok = True
267267
break
@@ -287,7 +287,7 @@ def ensure_layer2_ready(self) -> None:
287287
print("Syncing light clients:")
288288
self._print_lnd_cfheaders(erase_last_line=False)
289289

290-
with ThreadPoolExecutor(max_workers=2) as executor:
290+
with ThreadPoolExecutor(max_workers=2, thread_name_prefix="LndReady") as executor:
291291
f1 = executor.submit(self.ensure_lnd_ready, "bitcoin")
292292
f2 = executor.submit(self.ensure_lnd_ready, "litecoin")
293293

images/utils/launcher/config/config.py

Lines changed: 128 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
from __future__ import annotations
12
import argparse
23
import logging
34
from logging.handlers import TimedRotatingFileHandler
45
import os
6+
from typing import Optional, List
57

68
import toml
79

@@ -12,6 +14,16 @@
1214

1315

1416
class Config:
17+
branch: str
18+
network: str
19+
home_dir: str
20+
network_dir: str
21+
disable_update: bool
22+
external_ip: Optional[str]
23+
backup_dir: Optional[str]
24+
restore_dir: Optional[str]
25+
eth_providers: List[str]
26+
1527
def __init__(self, loader: ConfigLoader):
1628
self.logger = logging.getLogger("launcher.Config")
1729

@@ -23,44 +35,49 @@ def __init__(self, loader: ConfigLoader):
2335
self.network = os.environ["NETWORK"]
2436

2537
self.home_dir = self.loader.ensure_home_dir(os.environ["HOST_HOME"])
26-
self.network_dir = None
38+
self.network_dir = os.path.join(self.home_dir, self.network)
2739
self.backup_dir = None
2840
self.restore_dir = None
2941

3042
self.eth_providers = general_config[self.network]["eth_providers"]
3143

3244
self.nodes = nodes_config[self.network]
3345

34-
self.args = None
46+
self._parse_command_line_arguments()
47+
self._parse_general_config_file()
48+
self.network_dir = self.loader.ensure_network_dir(self.network_dir)
49+
50+
self._parse_network_config_file()
51+
52+
for node in self.nodes.values():
53+
for v in node["volumes"]:
54+
v["host"] = self.expand_vars(v["host"])
3555

36-
self.parse()
56+
self.dump()
3757

38-
def parse(self):
58+
def _parse_command_line_arguments(self) -> None:
59+
self.args = None
3960
try:
4061
self.parse_command_line_arguments()
4162
except Exception as e:
4263
raise ConfigError(ConfigErrorScope.COMMAND_LINE_ARGS) from e
4364

44-
self.network_dir = "{}/{}".format(self.home_dir, self.network)
45-
65+
def _parse_general_config_file(self) -> None:
66+
filename = "xud-docker.conf"
67+
conf_file = os.path.join(self.home_dir, filename)
4668
try:
4769
self.parse_general_config()
4870
except Exception as e:
49-
conf_file = "{}/{}.conf".format(self.home_dir, "xud-docker")
5071
raise ConfigError(ConfigErrorScope.GENERAL_CONF, conf_file=conf_file) from e
5172

52-
self.network_dir = self.loader.ensure_network_dir(self.network_dir)
53-
73+
def _parse_network_config_file(self) -> None:
74+
filename = "{}.conf".format(self.network)
75+
conf_file = os.path.join(self.network_dir, filename)
5476
try:
5577
self.parse_network_config()
5678
except Exception as e:
57-
conf_file = "{}/{}.conf".format(self.network_dir, self.network)
5879
raise ConfigError(ConfigErrorScope.NETWORK_CONF, conf_file=conf_file) from e
5980

60-
for node in self.nodes.values():
61-
for v in node["volumes"]:
62-
v["host"] = self.expand_vars(v["host"])
63-
6481
def parse_command_line_arguments(self):
6582
parser = ArgumentParser(argument_default=argparse.SUPPRESS, prog="xud.sh", usage="bash xud.sh [OPTIONS]")
6683
parser.add_argument(
@@ -915,8 +932,100 @@ def expand_vars(self, value):
915932
return value
916933

917934
@property
918-
def logfile(self):
919-
if self.network_dir:
920-
network = self.network
921-
return f"{self.network_dir}/logs/{network}.log"
922-
return None
935+
def logs_dir(self) -> str:
936+
return os.path.join(self.network_dir, "logs")
937+
938+
@property
939+
def logfile(self) -> str:
940+
filename = f"{self.network}.log"
941+
return os.path.join(self.logs_dir, filename)
942+
943+
@property
944+
def dumpfile(self) -> str:
945+
filename = f"config.sh"
946+
return os.path.join(self.logs_dir, filename)
947+
948+
def dump(self) -> None:
949+
"""Dump xud-docker configurations as bash key-value file in logs_dir"""
950+
prefix = "XUD_DOCKER"
951+
952+
with open("/mnt/hostfs" + self.dumpfile, "w") as f:
953+
def dump_attr(attr: str) -> None:
954+
key = f"{prefix}_{attr.upper()}"
955+
value = getattr(self, attr)
956+
if not value:
957+
value = ""
958+
if isinstance(value, bool):
959+
value = str(value).lower()
960+
print("{}=\"{}\"".format(key, value), file=f)
961+
dump_attr("branch")
962+
dump_attr("disable_update")
963+
dump_attr("external_ip")
964+
dump_attr("network")
965+
dump_attr("home_dir")
966+
dump_attr("network_dir")
967+
dump_attr("backup_dir")
968+
dump_attr("restore_dir")
969+
970+
# dump nodes config
971+
def dump_node_attr(node: str, attr: str) -> None:
972+
node_config = self.nodes[node]
973+
node_prefix = f"{prefix}_SERVICE_{node.upper()}"
974+
if attr == "volumes":
975+
for volume in node_config["volumes"]:
976+
key = f"{node_prefix}_VOLUME"
977+
value = "{}:{}".format(volume["host"], volume["container"])
978+
print("{}=\"{}\"".format(key, value), file=f)
979+
elif attr == "ports":
980+
for port in node_config["ports"]:
981+
key = f"{node_prefix}_PORT"
982+
value = str(port)
983+
print("{}=\"{}\"".format(key, value), file=f)
984+
else:
985+
key = f"{node_prefix}_{attr.upper()}"
986+
value = ""
987+
try:
988+
value = node_config[attr]
989+
except KeyError:
990+
if node == "arby":
991+
value = node_config.get(attr.replace("_", "-"), "")
992+
if not value:
993+
value = ""
994+
if isinstance(value, bool):
995+
value = str(value).lower()
996+
print("{}=\"{}\"".format(key, value), file=f)
997+
998+
for node in self.nodes.keys():
999+
dump_node_attr(node, "image")
1000+
dump_node_attr(node, "volumes")
1001+
dump_node_attr(node, "ports")
1002+
dump_node_attr(node, "mode")
1003+
dump_node_attr(node, "disabled")
1004+
dump_node_attr(node, "preserve_config")
1005+
dump_node_attr(node, "use_local_image")
1006+
1007+
if node in ["bitcoind", "litecoind"]:
1008+
dump_node_attr(node, "external_rpc_host")
1009+
dump_node_attr(node, "external_rpc_port")
1010+
#dump_node_attr(node, "external_rpc_user")
1011+
#dump_node_attr(node, "external_rpc_password")
1012+
dump_node_attr(node, "external_zmqpubrawblock")
1013+
dump_node_attr(node, "external_zmqpubrawtx")
1014+
elif node == "geth":
1015+
dump_node_attr(node, "external_rpc_host")
1016+
dump_node_attr(node, "external_rpc_port")
1017+
#dump_node_attr(node, "infura_project_id")
1018+
#dump_node_attr(node, "infura_project_secret")
1019+
dump_node_attr(node, "cache")
1020+
elif node == "arby":
1021+
dump_node_attr(node, "test_centralized_baseasset_balance")
1022+
dump_node_attr(node, "test_centralized_quoteasset_balance")
1023+
dump_node_attr(node, "opendex_base_asset")
1024+
dump_node_attr(node, "opendex_quote_asset")
1025+
dump_node_attr(node, "cex_base_asset")
1026+
dump_node_attr(node, "cex_quote_asset")
1027+
dump_node_attr(node, "live_cex")
1028+
dump_node_attr(node, "cex")
1029+
#dump_node_attr(node, "cex_api_key")
1030+
#dump_node_attr(node, "cex_api_secret")
1031+
dump_node_attr(node, "margin")

images/utils/launcher/config/template.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,21 @@ def __eq__(self, other):
5454
return False
5555
return True
5656

57+
def __str__(self):
58+
if self.host:
59+
host = "{}:{}".format(self.host, self.host_port)
60+
else:
61+
host = "{}".format(self.host_port)
62+
63+
container = "{}".format(self.port)
64+
65+
if self.protocol == "tcp":
66+
return "{}:{}".format(host, container)
67+
elif self.protocol == "udp":
68+
return "{}:{}/udp".format(host, container)
69+
elif self.protocol == "sctp":
70+
return "{}:{}/sctp".format(host, container)
71+
5772

5873
nodes_config = {
5974
"simnet": {

0 commit comments

Comments
 (0)