From 6fd91b43bdcedec848652bad7bcf6a6a65daefba Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:15:33 +0200 Subject: [PATCH 01/11] Add support for telegram notifications --- snapraid-runner.py | 92 +++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 26 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index e8aadce..e212a43 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -16,7 +16,7 @@ # Global variables config = None -email_log = None +notification_log = None def tee_log(infile, out_lines, log_level): @@ -24,6 +24,7 @@ def tee_log(infile, out_lines, log_level): Create a thread that saves all the output on infile to out_lines and logs every line with log_level """ + def tee_thread(): for line in iter(infile.readline, ""): line = line.strip() @@ -33,6 +34,7 @@ def tee_thread(): logging.log(log_level, line.strip()) out_lines.append(line) infile.close() + t = threading.Thread(target=tee_thread) t.daemon = True t.start() @@ -83,26 +85,15 @@ def send_email(success): # use quoted-printable instead of the default base64 charset.add_charset("utf-8", charset.SHORTEST, charset.QP) - if success: - body = "SnapRAID job completed successfully:\n\n\n" - else: - body = "Error during SnapRAID job:\n\n\n" + body = get_success_message(success) - log = email_log.getvalue() maxsize = config['email'].get('maxsize', 500) * 1024 - if maxsize and len(log) > maxsize: - cut_lines = log.count("\n", maxsize // 2, -maxsize // 2) - log = ( - "NOTE: Log was too big for email and was shortened\n\n" + - log[:maxsize // 2] + - "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( - cut_lines) + - log[-maxsize // 2:]) + log = get_log(maxsize) body += log msg = MIMEText(body, "plain", "utf-8") msg["Subject"] = config["email"]["subject"] + \ - (" SUCCESS" if success else " ERROR") + (" SUCCESS" if success else " ERROR") msg["From"] = config["email"]["from"] msg["To"] = config["email"]["to"] smtp = {"host": config["smtp"]["host"]} @@ -123,12 +114,61 @@ def send_email(success): server.quit() +def send_telegram(success): + from email import charset + import requests + + if len(config["telegram"]["token"]) == 0: + logging.error("Failed to send telegram because token is not set.") + return + + if len(config["telegram"]["chat_id"]) == 0: + logging.error("Failed to send telegram because chat_id is not set.") + return + + # use quoted-printable instead of the default base64 + charset.add_charset("utf-8", charset.SHORTEST, charset.QP) + body = get_success_message(success) + + maxsize = 4096 + log = get_log(maxsize) + body += log + + url = 'https://api.telegram.org/bot{}/sendMessage'.format(config["telegram"]["token"]) + data = {"chat_id": config["telegram"]["chat_id"], "text": body} + requests.post(url, data) + + +def get_log(maxsize): + log = notification_log.getvalue() + if maxsize and len(log) > maxsize: + cut_lines = log.count("\n", maxsize // 2, -maxsize // 2) + log = ( + "NOTE: Log was too big and was shortened\n\n" + + log[:maxsize // 2] + + "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( + cut_lines) + + log[-maxsize // 2:]) + return log + + +def get_success_message(success): + if success: + return "SnapRAID job completed successfully:\n\n\n" + return "Error during SnapRAID job:\n\n\n" + + def finish(is_success): if ("error", "success")[is_success] in config["email"]["sendon"]: try: send_email(is_success) except Exception: logging.exception("Failed to send email") + if ("error", "success")[is_success] in config["telegram"]["sendon"]: + try: + send_telegram(is_success) + except Exception: + logging.exception("Failed to send telegram") if is_success: logging.info("Run finished successfully") else: @@ -140,7 +180,7 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "email", "smtp", "scrub"] + sections = ["snapraid", "logging", "email", "smtp", "telegram", "scrub"] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): @@ -188,15 +228,15 @@ def setup_logger(): file_logger.setFormatter(log_format) root_logger.addHandler(file_logger) - if config["email"]["sendon"]: - global email_log - email_log = StringIO() - email_logger = logging.StreamHandler(email_log) - email_logger.setFormatter(log_format) - if config["email"]["short"]: - # Don't send programm stdout in email - email_logger.setLevel(logging.INFO) - root_logger.addHandler(email_logger) + if config["notifications"]["on"]: + global notification_log + notification_log = StringIO() + notification_logger = logging.StreamHandler(notification_log) + notification_logger.setFormatter(log_format) + if config["notifications"]["short"]: + # Don't send program stdout in notification + notification_logger.setLevel(logging.INFO) + root_logger.addHandler(notification_logger) def main(): @@ -244,7 +284,7 @@ def run(): if not os.path.isfile(config["snapraid"]["executable"]): logging.error("The configured snapraid executable \"{}\" does not " "exist or is not a file".format( - config["snapraid"]["executable"])) + config["snapraid"]["executable"])) finish(False) if not os.path.isfile(config["snapraid"]["config"]): logging.error("Snapraid config does not exist at " + From 0eebd4b1d5131c3d2b5bc5454873e0d56342dde4 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:16:39 +0200 Subject: [PATCH 02/11] Format with Black --- snapraid-runner.py | 132 +++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index e212a43..d0799c1 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -57,12 +57,13 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]): # Snapraid always outputs utf-8 on windows. On linux, utf-8 # also seems a sensible assumption. encoding="utf-8", - errors="replace" + errors="replace", ) out = [] threads = [ tee_log(p.stdout, out, logging.OUTPUT), - tee_log(p.stderr, [], logging.OUTERR)] + tee_log(p.stderr, [], logging.OUTERR), + ] for t in threads: t.join() ret = p.wait() @@ -87,13 +88,12 @@ def send_email(success): charset.add_charset("utf-8", charset.SHORTEST, charset.QP) body = get_success_message(success) - maxsize = config['email'].get('maxsize', 500) * 1024 + maxsize = config["email"].get("maxsize", 500) * 1024 log = get_log(maxsize) body += log msg = MIMEText(body, "plain", "utf-8") - msg["Subject"] = config["email"]["subject"] + \ - (" SUCCESS" if success else " ERROR") + msg["Subject"] = config["email"]["subject"] + (" SUCCESS" if success else " ERROR") msg["From"] = config["email"]["from"] msg["To"] = config["email"]["to"] smtp = {"host": config["smtp"]["host"]} @@ -107,10 +107,7 @@ def send_email(success): server.starttls() if config["smtp"]["user"]: server.login(config["smtp"]["user"], config["smtp"]["password"]) - server.sendmail( - config["email"]["from"], - [config["email"]["to"]], - msg.as_string()) + server.sendmail(config["email"]["from"], [config["email"]["to"]], msg.as_string()) server.quit() @@ -134,7 +131,9 @@ def send_telegram(success): log = get_log(maxsize) body += log - url = 'https://api.telegram.org/bot{}/sendMessage'.format(config["telegram"]["token"]) + url = "https://api.telegram.org/bot{}/sendMessage".format( + config["telegram"]["token"] + ) data = {"chat_id": config["telegram"]["chat_id"], "text": body} requests.post(url, data) @@ -144,11 +143,13 @@ def get_log(maxsize): if maxsize and len(log) > maxsize: cut_lines = log.count("\n", maxsize // 2, -maxsize // 2) log = ( - "NOTE: Log was too big and was shortened\n\n" + - log[:maxsize // 2] + - "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( - cut_lines) + - log[-maxsize // 2:]) + "NOTE: Log was too big and was shortened\n\n" + + log[: maxsize // 2] + + "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( + cut_lines + ) + + log[-maxsize // 2 :] + ) return log @@ -187,8 +188,11 @@ def load_config(args): config[section][k] = v.strip() int_options = [ - ("snapraid", "deletethreshold"), ("logging", "maxsize"), - ("scrub", "percentage"), ("scrub", "older-than"), ("email", "maxsize"), + ("snapraid", "deletethreshold"), + ("logging", "maxsize"), + ("scrub", "percentage"), + ("scrub", "older-than"), + ("email", "maxsize"), ] for section, option in int_options: try: @@ -196,19 +200,18 @@ def load_config(args): except ValueError: config[section][option] = 0 - config["smtp"]["ssl"] = (config["smtp"]["ssl"].lower() == "true") - config["smtp"]["tls"] = (config["smtp"]["tls"].lower() == "true") - config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") - config["email"]["short"] = (config["email"]["short"].lower() == "true") - config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") + config["smtp"]["ssl"] = config["smtp"]["ssl"].lower() == "true" + config["smtp"]["tls"] = config["smtp"]["tls"].lower() == "true" + config["scrub"]["enabled"] = config["scrub"]["enabled"].lower() == "true" + config["email"]["short"] = config["email"]["short"].lower() == "true" + config["snapraid"]["touch"] = config["snapraid"]["touch"].lower() == "true" if args.scrub is not None: config["scrub"]["enabled"] = args.scrub def setup_logger(): - log_format = logging.Formatter( - "%(asctime)s [%(levelname)-6.6s] %(message)s") + log_format = logging.Formatter("%(asctime)s [%(levelname)-6.6s] %(message)s") root_logger = logging.getLogger() logging.OUTPUT = 15 logging.addLevelName(logging.OUTPUT, "OUTPUT") @@ -222,9 +225,8 @@ def setup_logger(): if config["logging"]["file"]: max_log_size = min(config["logging"]["maxsize"], 0) * 1024 file_logger = logging.handlers.RotatingFileHandler( - config["logging"]["file"], - maxBytes=max_log_size, - backupCount=9) + config["logging"]["file"], maxBytes=max_log_size, backupCount=9 + ) file_logger.setFormatter(log_format) root_logger.addHandler(file_logger) @@ -241,13 +243,20 @@ def setup_logger(): def main(): parser = argparse.ArgumentParser() - parser.add_argument("-c", "--conf", - default="snapraid-runner.conf", - metavar="CONFIG", - help="Configuration file (default: %(default)s)") - parser.add_argument("--no-scrub", action='store_false', - dest='scrub', default=None, - help="Do not scrub (overrides config)") + parser.add_argument( + "-c", + "--conf", + default="snapraid-runner.conf", + metavar="CONFIG", + help="Configuration file (default: %(default)s)", + ) + parser.add_argument( + "--no-scrub", + action="store_false", + dest="scrub", + default=None, + help="Do not scrub (overrides config)", + ) args = parser.parse_args() if not os.path.exists(args.conf): @@ -282,13 +291,15 @@ def run(): logging.info("=" * 60) if not os.path.isfile(config["snapraid"]["executable"]): - logging.error("The configured snapraid executable \"{}\" does not " - "exist or is not a file".format( - config["snapraid"]["executable"])) + logging.error( + 'The configured snapraid executable "{}" does not ' + "exist or is not a file".format(config["snapraid"]["executable"]) + ) finish(False) if not os.path.isfile(config["snapraid"]["config"]): - logging.error("Snapraid config does not exist at " + - config["snapraid"]["config"]) + logging.error( + "Snapraid config does not exist at " + config["snapraid"]["config"] + ) finish(False) if config["snapraid"]["touch"]: @@ -301,20 +312,34 @@ def run(): logging.info("*" * 60) diff_results = Counter(line.split(" ")[0] for line in diff_out) - diff_results = dict((x, diff_results[x]) for x in - ["add", "remove", "move", "update"]) - logging.info(("Diff results: {add} added, {remove} removed, " + - "{move} moved, {update} modified").format(**diff_results)) + diff_results = dict( + (x, diff_results[x]) for x in ["add", "remove", "move", "update"] + ) + logging.info( + ( + "Diff results: {add} added, {remove} removed, " + + "{move} moved, {update} modified" + ).format(**diff_results) + ) - if (config["snapraid"]["deletethreshold"] >= 0 and - diff_results["remove"] > config["snapraid"]["deletethreshold"]): + if ( + config["snapraid"]["deletethreshold"] >= 0 + and diff_results["remove"] > config["snapraid"]["deletethreshold"] + ): logging.error( "Deleted files exceed delete threshold of {}, aborting".format( - config["snapraid"]["deletethreshold"])) + config["snapraid"]["deletethreshold"] + ) + ) finish(False) - if (diff_results["remove"] + diff_results["add"] + diff_results["move"] + - diff_results["update"] == 0): + if ( + diff_results["remove"] + + diff_results["add"] + + diff_results["move"] + + diff_results["update"] + == 0 + ): logging.info("No changes detected, no sync required") else: logging.info("Running sync...") @@ -328,10 +353,13 @@ def run(): if config["scrub"]["enabled"]: logging.info("Running scrub...") try: - snapraid_command("scrub", { - "percentage": config["scrub"]["percentage"], - "older-than": config["scrub"]["older-than"], - }) + snapraid_command( + "scrub", + { + "percentage": config["scrub"]["percentage"], + "older-than": config["scrub"]["older-than"], + }, + ) except subprocess.CalledProcessError as e: logging.error(e) finish(False) From cfb345a0d233110c84633a5ed4d552df26e21a01 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:18:55 +0200 Subject: [PATCH 03/11] Update example conf --- snapraid-runner.conf.example | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index ed64810..4f8c23e 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -14,11 +14,15 @@ file = snapraid.log ; maximum logfile size in KiB, leave empty for infinite maxsize = 5000 +[notifications] +; set to true to activate +on = true +; set to false to get full programm output via email +short = true + [email] ; when to send an email, comma-separated list of [success, error] sendon = success,error -; set to false to get full programm output via email -short = true subject = [SnapRAID] Status Report: from = to = @@ -35,6 +39,14 @@ tls = false user = password = +[telegram] +; when to send a message, comma-separated list of [success, error] +sendon = success,error +; Telegram bot token +token = +; chat_id for telegram bot +chat_id = + [scrub] ; set to true to run scrub after sync enabled = false From c6f7e3bb155af5469fef4e9d215bcbc8cd886a18 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:42:12 +0200 Subject: [PATCH 04/11] Add notifications to sections --- snapraid-runner.conf.example | 2 +- snapraid-runner.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index 4f8c23e..6a64429 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -17,7 +17,7 @@ maxsize = 5000 [notifications] ; set to true to activate on = true -; set to false to get full programm output via email +; set to false to get full program output via email short = true [email] diff --git a/snapraid-runner.py b/snapraid-runner.py index d0799c1..55eac32 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -203,7 +203,8 @@ def load_config(args): config["smtp"]["ssl"] = config["smtp"]["ssl"].lower() == "true" config["smtp"]["tls"] = config["smtp"]["tls"].lower() == "true" config["scrub"]["enabled"] = config["scrub"]["enabled"].lower() == "true" - config["email"]["short"] = config["email"]["short"].lower() == "true" + config["notifications"]["on"] = config["notifications"]["on"].lower() == "true" + config["notifications"]["short"] = config["notifications"]["short"].lower() == "true" config["snapraid"]["touch"] = config["snapraid"]["touch"].lower() == "true" if args.scrub is not None: From 5aa05cd9a49c5d8447e2519f194a163ae0896d53 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:44:11 +0200 Subject: [PATCH 05/11] Fix missing key --- snapraid-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 55eac32..51ac7f4 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -181,7 +181,7 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "email", "smtp", "telegram", "scrub"] + sections = ["snapraid", "logging", "notifications", "email", "smtp", "telegram", "scrub"] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): From 182b466e4adc4e33d5b4bac7115e0ecb7534332b Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:57:02 +0200 Subject: [PATCH 06/11] Format with Black --- snapraid-runner.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 51ac7f4..9c5ac8c 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -181,7 +181,15 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "notifications", "email", "smtp", "telegram", "scrub"] + sections = [ + "snapraid", + "logging", + "notifications", + "email", + "smtp", + "telegram", + "scrub", + ] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): @@ -204,7 +212,9 @@ def load_config(args): config["smtp"]["tls"] = config["smtp"]["tls"].lower() == "true" config["scrub"]["enabled"] = config["scrub"]["enabled"].lower() == "true" config["notifications"]["on"] = config["notifications"]["on"].lower() == "true" - config["notifications"]["short"] = config["notifications"]["short"].lower() == "true" + config["notifications"]["short"] = ( + config["notifications"]["short"].lower() == "true" + ) config["snapraid"]["touch"] = config["snapraid"]["touch"].lower() == "true" if args.scrub is not None: From 864d735aa40264dcee823ae74e7a40285f93a772 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 20:57:15 +0200 Subject: [PATCH 07/11] Update example conf --- snapraid-runner.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index 6a64429..8a15d9b 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -15,7 +15,7 @@ file = snapraid.log maxsize = 5000 [notifications] -; set to true to activate +; if you want notifications to be activated on = true ; set to false to get full program output via email short = true From 3f64ac8aed90f748e601d225785be1fe88193e5d Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 21:01:59 +0200 Subject: [PATCH 08/11] Update example conf --- snapraid-runner.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index 8a15d9b..7da54b5 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -17,7 +17,7 @@ maxsize = 5000 [notifications] ; if you want notifications to be activated on = true -; set to false to get full program output via email +; set to false to get full program output via notifications short = true [email] From 54d1acc86b07b8bcd060986a045a8b8546df730f Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 21:05:20 +0200 Subject: [PATCH 09/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fe4130e..673386f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ It supports Windows, Linux and macOS and requires at least python3.7. * Runs `diff` before `sync` to see how many files were deleted and aborts if that number exceeds a set threshold. * Can create a size-limited rotated logfile. -* Can send notification emails after each run or only for failures. +* Can send notification via email or telegram after each run or only for failures. * Can run `scrub` after `sync` ## Changelog From f627efab4e8024e4354f04c17909433086d4a5f5 Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 21:11:11 +0200 Subject: [PATCH 10/11] Simplify expression --- snapraid-runner.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 9c5ac8c..92fdb82 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -333,10 +333,7 @@ def run(): ).format(**diff_results) ) - if ( - config["snapraid"]["deletethreshold"] >= 0 - and diff_results["remove"] > config["snapraid"]["deletethreshold"] - ): + if 0 <= config["snapraid"]["deletethreshold"] < diff_results["remove"]: logging.error( "Deleted files exceed delete threshold of {}, aborting".format( config["snapraid"]["deletethreshold"] From 2b7ed2252a6f3484252692b8877fe19b7ea2b4bd Mon Sep 17 00:00:00 2001 From: Tommy Larsson <45052383+larssont@users.noreply.github.com> Date: Wed, 24 Jun 2020 21:17:31 +0200 Subject: [PATCH 11/11] Fix typo --- snapraid-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 92fdb82..1682072 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -67,7 +67,7 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]): for t in threads: t.join() ret = p.wait() - # sleep for a while to make pervent output mixup + # sleep for a while to make prevent output mixup time.sleep(0.3) if ret == 0 or ret in allow_statuscodes: return out