backup & logs

Author: reliveyy

images/utils/launcher/config/config.py

@@ -1133,3 +1133,6 @@ def dump_node_attr(node: str, attr: str) -> None:
                     # dump_node_attr(node, "cex_api_key")
                     # dump_node_attr(node, "cex_api_secret")
                     dump_node_attr(node, "margin")
+
+    def expand_path(self, value) -> str:
+        pass

images/utils/launcher/node/__init__.py

@@ -2,7 +2,6 @@
 
 import functools
 import logging
-import os
 import sys
 import threading
 from abc import ABC, abstractmethod
@@ -14,7 +13,7 @@
 from docker.errors import NotFound
 from docker.types import IPAMConfig, IPAMPool
 
-from launcher.utils import get_hostfs_file, ArgumentParser, yes_or_no
+from launcher.utils import ArgumentParser, yes_or_no
 from .DockerTemplate import DockerTemplate
 from .arby import Arby
 from .base import Node, ContainerNotFound, NoWaiting
@@ -52,19 +51,25 @@ def execute(self, *args) -> None:
 class LogsCommand(Command):
     def create_parser(self) -> ArgumentParser:
         parser = ArgumentParser(prog="logs", description="fetch the logs of a container")
-        parser.add_argument("--tail", "-t", metavar='N', type=int,
-                            help="number of lines to show from the end of the logs", default=100)
-        parser.add_argument("--since", "-s")
+        parser.add_argument("--tail", metavar='N',
+                            help="number of lines to show from the end of the logs (default \"100\")",
+                            default="100")
+        parser.add_argument("--since",
+                            help="show logs since timestamp (e.g. 2013-01-02T13:23:37) or relative (e.g. 42m for 42 minutes)")
+        parser.add_argument("--until",
+                            help="show logs before a timestamp (e.g. 2013-01-02T13:23:37) or relative (e.g. 42m for 42 minutes)")
+        parser.add_argument("--follow", "-f", action="store_true",
+                            help="follow log output")
+        parser.add_argument("--timestamps", "-t", action="store_true",
+                            help="show timestamps")
         parser.add_argument("service")
         return parser
 
     def execute(self, *args) -> None:
         args = self.parser.parse_args(args)
         name = args.service
-        tail = args.tail
-        since = args.since
         service = self.get_service(name)
-        for line in service.logs(tail=tail, since=since):
+        for line in service.logs(tail=args.tail, since=args.since, until=args.until, follow=args.follow, timestamps=args.timestamps):
             print(line)
 
 
@@ -354,12 +359,12 @@ def status(self):
         print(f"{border_style}┌─%s─┬─%s─┐{RESET}" % ("─" * col1_width, "─" * col2_width))
         print(
             f"{border_style}│{RESET} {title_style}%s{RESET} {border_style}│{RESET} {title_style}%s{RESET} {border_style}│{RESET}" % (
-            col1_fmt % col1_title, col2_fmt % col2_title))
+                col1_fmt % col1_title, col2_fmt % col2_title))
         for name in names:
             print(f"{border_style}├─%s─┼─%s─┤{RESET}" % ("─" * col1_width, "─" * col2_width))
             print(
                 f"{border_style}│{RESET} {service_style}%s{RESET} {border_style}│{RESET} {border_style}%s{RESET} {border_style}│{RESET}" % (
-                col1_fmt % name, col2_fmt % ""))
+                    col1_fmt % name, col2_fmt % ""))
         print(f"{border_style}└─%s─┴─%s─┘{RESET}" % ("─" * col1_width, "─" * col2_width))
 
         lock = threading.Lock()
@@ -372,10 +377,10 @@ def update_line(name, text, fetching=False):
             x = col1_width + 2
             if fetching:
                 print(f"\033[%dA\033[%dC{border_style}%s{RESET}\033[%dD\033[%dB" % (
-                y, x + 3, col2_fmt % text[:col2_width], x + col2_width + 3, y), end="")
+                    y, x + 3, col2_fmt % text[:col2_width], x + col2_width + 3, y), end="")
             else:
                 print("\033[%dA\033[%dC%s\033[%dD\033[%dB" % (
-                y, x + 3, col2_fmt % text[:col2_width], x + col2_width + 3, y), end="")
+                    y, x + 3, col2_fmt % text[:col2_width], x + col2_width + 3, y), end="")
             sys.stdout.flush()
 
         result = {name: None for name in names}

images/utils/launcher/node/base.py

@@ -296,12 +296,29 @@ def _get_container_created_timestamp(self):
         t = datetime.datetime.strptime(parts[0], "%Y-%m-%dT%H:%M:%S")
         return t
 
-    def logs(self, tail="all"):
-        if self._container is None:
-            return None
-        t = self._get_container_created_timestamp()
-        result = self._container.logs(since=t, tail=tail)
-        return itertools.chain(result.decode().splitlines())
+    def logs(self, tail: str = None, since: str = None, until: str = None, follow: bool = False, timestamps: bool = False):
+        assert since is None, "String since is not supported yet"
+        assert until is None, "String until is not supported yet"
+        try:
+            tail = int(tail)
+        except:
+            pass
+
+        kwargs = {
+            "tail": tail,
+            "follow": follow,
+            "timestamps": timestamps,
+        }
+
+        if follow:
+            kwargs["stream"] = True
+
+        result = self.container.logs(**kwargs)
+        if isinstance(result, bytes):
+            return itertools.chain(result.decode().splitlines())
+        else:
+            for line in result:
+                yield line.decode().rstrip()
 
     def _compare_image(self, container: Container) -> bool:
         attrs = container.attrs

images/utils/launcher/node/image.py

@@ -228,10 +228,11 @@ def _get_update_status(self) -> str:
         - LOCAL_MISSING: The cloud image exists but no local image.
         - LOCAL_ONLY: The image only exists locally.
         - UNAVAILABLE: The image is not found locally or remotely.
+        - USE_LOCAL
         """
         if self.node.node_config["use_local_image"]:
             self.cloud_metadata = None
-            return "LOCAL_NEWER"
+            return "USE_LOCAL"
 
         local = self.local_metadata
 
@@ -271,6 +272,8 @@ def get_update_action(self) -> str:
             action = "PULL"
         elif status == "UP_TO_DATE":
             action = "NONE"
+        elif status == "USE_LOCAL":
+            action = "NONE"
         else:
             raise Exception("Unexpected status " + status)
 

images/utils/launcher/node/lnd.py

@@ -127,6 +127,7 @@ def get_current_height(self):
         try:
             c = self.get_container()
             since = datetime.now() - timedelta(hours=1)
+            # TODO use base logs
             lines = c.logs(since=since).decode().splitlines()
             p = re.compile(r".*New block: height=(\d+),.*")
             for line in reversed(lines):

images/utils/launcher/node/xud.py

@@ -1,14 +1,16 @@
+from __future__ import annotations
+
 import functools
 import json
 import logging
 import os
 import re
 from concurrent.futures import wait
-from threading import Event, Thread
-from typing import List
+from threading import Event
+from typing import List, Optional
 
 from launcher.table import ServiceTable
-from launcher.utils import yes_or_no, get_percentage
+from launcher.utils import yes_or_no, normalize_path
 from .base import Node, CliBackend, CliError
 from .lnd import CFHeaderState
 
@@ -219,9 +221,79 @@ def _setup_wallets(self) -> None:
                 self._restore_wallets()
                 break
 
+    @property
+    def backup_dir(self) -> Optional[str]:
+        value_file = os.path.join(self.data_dir, ".backup-dir-value")
+        if os.path.exists(value_file):
+            with open(value_file) as f:
+                value = f.read().strip()
+                value = value.replace("/mnt/hostfs", "")
+                return value
+        return None
+
+    def update_backup_dir(self, value: str) -> None:
+        cmd = "/update-backup-dir.sh '/mnt/hostfs{}'".format(value)
+        exit_code, output = self.exec(cmd)
+        print(output)
+        if exit_code != 0:
+            raise Exception("Failed to update backup location")
+
+    def _check_backup_dir(self, value: str) -> None:
+        value = "/mnt/hostfs" + value
+
+        if not os.path.exists(value):
+            raise Exception("not existed")
+
+        if not os.path.isdir(value):
+            raise Exception("not a directory")
+
+        if not os.access(value, os.R_OK):
+            raise Exception("not readable")
+
+        if not os.access(value, os.W_OK):
+            raise Exception("not writable")
+
     def _setup_backup(self) -> None:
         logger.info("Setup backup")
 
+        current_backup_dir = self.backup_dir
+
+        if current_backup_dir:
+            backup_dir = self.config.backup_dir
+
+            if backup_dir:
+                if current_backup_dir != backup_dir:
+                    self.update_backup_dir(backup_dir)
+        else:
+            backup_dir = self.config.backup_dir
+
+            if not backup_dir:
+                print()
+                print("Please enter a path to a destination where to store a backup of your environment. "
+                      "It includes everything, but NOT your wallet balance which is secured by your XUD SEED. "
+                      "The path should be an external drive, like a USB or network drive, which is permanently "
+                      "available on your device since backups are written constantly.")
+                print()
+
+                while True:
+                    reply = input("Enter path to backup location: ")
+                    reply = reply.strip()
+                    path = normalize_path(reply)
+                    print("Checking backup location... ", end="", flush=True)
+                    try:
+                        self._check_backup_dir(path)
+                        print("OK.")
+                    except Exception as e:
+                        print("Failed (%s)." % e)
+                        continue
+
+                    self.update_backup_dir(path)
+                    break
+
+            else:
+                if current_backup_dir != backup_dir:
+                    self.update_backup_dir(backup_dir)
+
     def has_wallets(self) -> bool:
         nodekey = os.path.join(self.data_dir, "nodekey.dat")
         return os.path.exists(nodekey)

setup.sh

@@ -394,6 +394,7 @@ docker run --rm -it \
 --name "$(get_utils_name)" \
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v "$NETWORK_DIR:/root/$NETWORK" \
+-v /:/mnt/hostfs \
 -e LOG_TIMESTAMP="$LOG_TIMESTAMP" \
 -e NETWORK="$NETWORK" \
 -e NETWORK_DIR="$NETWORK_DIR" \